blob: cda841adc777d1f3e288ac3f88e36f9826624d24 [file] [log] [blame]
Jose Marinho75509b42019-04-09 09:34:59 +01001/*
2 * Copyright 2019 The Hafnium Authors.
3 *
Andrew Walbrane959ec12020-06-17 15:01:09 +01004 * Use of this source code is governed by a BSD-style
5 * license that can be found in the LICENSE file or at
6 * https://opensource.org/licenses/BSD-3-Clause.
Jose Marinho75509b42019-04-09 09:34:59 +01007 */
8
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01009#include "hf/ffa_memory.h"
Andrew Walbran475c1452020-02-07 13:22:22 +000010
J-Alves7b9cc432024-04-04 10:57:17 +010011#include "hf/arch/memcpy_trapped.h"
Federico Recanati4fd065d2021-12-13 20:06:23 +010012#include "hf/arch/mm.h"
Olivier Deprez112d2b52020-09-30 07:39:23 +020013#include "hf/arch/other_world.h"
Olivier Deprez55a189e2021-06-09 15:45:27 +020014#include "hf/arch/plat/ffa.h"
Andrew Walbran290b0c92020-02-03 16:37:14 +000015
J-Alves5952d942022-12-22 16:03:00 +000016#include "hf/addr.h"
Jose Marinho75509b42019-04-09 09:34:59 +010017#include "hf/api.h"
Daniel Boulbya2f8c662021-11-26 17:52:53 +000018#include "hf/assert.h"
Jose Marinho09b1db82019-08-08 09:16:59 +010019#include "hf/check.h"
Jose Marinho75509b42019-04-09 09:34:59 +010020#include "hf/dlog.h"
J-Alves3456e032023-07-20 12:20:05 +010021#include "hf/ffa.h"
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010022#include "hf/ffa_internal.h"
J-Alves66652252022-07-06 09:49:51 +010023#include "hf/ffa_memory_internal.h"
J-Alves3456e032023-07-20 12:20:05 +010024#include "hf/ffa_partition_manifest.h"
J-Alves5952d942022-12-22 16:03:00 +000025#include "hf/mm.h"
Andrew Walbran475c1452020-02-07 13:22:22 +000026#include "hf/mpool.h"
J-Alvescf6253e2024-01-03 13:48:48 +000027#include "hf/panic.h"
28#include "hf/plat/memory_protect.h"
Jose Marinho75509b42019-04-09 09:34:59 +010029#include "hf/std.h"
Andrew Scull3c257452019-11-26 13:32:50 +000030#include "hf/vm.h"
Daniel Boulby44e9b3b2024-01-17 12:21:44 +000031#include "hf/vm_ids.h"
Jose Marinho75509b42019-04-09 09:34:59 +010032
J-Alves2d8457f2022-10-05 11:06:41 +010033#include "vmapi/hf/ffa_v1_0.h"
34
J-Alves5da37d92022-10-24 16:33:48 +010035#define RECEIVERS_COUNT_IN_RETRIEVE_RESP 1
36
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000037/**
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010038 * All access to members of a `struct ffa_memory_share_state` must be guarded
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000039 * by this lock.
40 */
41static struct spinlock share_states_lock_instance = SPINLOCK_INIT;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010042static struct ffa_memory_share_state share_states[MAX_MEM_SHARES];
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000043
44/**
J-Alvesed508c82023-05-04 16:09:48 +010045 * Return the offset to the first constituent within the
46 * `ffa_composite_memory_region` for the given receiver from an
47 * `ffa_memory_region`. The caller must check that the receiver_index is within
48 * bounds, and that it has a composite memory region offset.
49 */
50static uint32_t ffa_composite_constituent_offset(
51 struct ffa_memory_region *memory_region, uint32_t receiver_index)
52{
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +000053 struct ffa_memory_access *receiver;
54 uint32_t composite_offset;
J-Alvesed508c82023-05-04 16:09:48 +010055
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +000056 CHECK(receiver_index < memory_region->receiver_count);
57
58 receiver =
59 ffa_memory_region_get_receiver(memory_region, receiver_index);
60 CHECK(receiver != NULL);
61
62 composite_offset = receiver->composite_memory_region_offset;
63
64 CHECK(composite_offset != 0);
65
66 return composite_offset + sizeof(struct ffa_composite_memory_region);
J-Alvesed508c82023-05-04 16:09:48 +010067}
68
69/**
J-Alves917d2f22020-10-30 18:39:30 +000070 * Extracts the index from a memory handle allocated by Hafnium's current world.
71 */
72uint64_t ffa_memory_handle_get_index(ffa_memory_handle_t handle)
73{
74 return handle & ~FFA_MEMORY_HANDLE_ALLOCATOR_MASK;
75}
76
77/**
Karl Meakin52cdfe72023-06-30 14:49:10 +010078 * Initialises the next available `struct ffa_memory_share_state`. If `handle`
79 * is `FFA_MEMORY_HANDLE_INVALID` then allocates an appropriate handle,
80 * otherwise uses the provided handle which is assumed to be globally unique.
Andrew Walbranca808b12020-05-15 17:22:28 +010081 *
Karl Meakin52cdfe72023-06-30 14:49:10 +010082 * Returns a pointer to the allocated `ffa_memory_share_state` on success or
83 * `NULL` if none are available.
Andrew Walbranca808b12020-05-15 17:22:28 +010084 */
Karl Meakin52cdfe72023-06-30 14:49:10 +010085struct ffa_memory_share_state *allocate_share_state(
86 struct share_states_locked share_states, uint32_t share_func,
87 struct ffa_memory_region *memory_region, uint32_t fragment_length,
88 ffa_memory_handle_t handle)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000089{
Daniel Boulbya2f8c662021-11-26 17:52:53 +000090 assert(share_states.share_states != NULL);
91 assert(memory_region != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000092
Karl Meakin52cdfe72023-06-30 14:49:10 +010093 for (uint64_t i = 0; i < MAX_MEM_SHARES; ++i) {
Andrew Walbranca808b12020-05-15 17:22:28 +010094 if (share_states.share_states[i].share_func == 0) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010095 struct ffa_memory_share_state *allocated_state =
Andrew Walbranca808b12020-05-15 17:22:28 +010096 &share_states.share_states[i];
97 struct ffa_composite_memory_region *composite =
98 ffa_memory_region_get_composite(memory_region,
99 0);
100
101 if (handle == FFA_MEMORY_HANDLE_INVALID) {
J-Alvesee68c542020-10-29 17:48:20 +0000102 memory_region->handle =
Olivier Deprez55a189e2021-06-09 15:45:27 +0200103 plat_ffa_memory_handle_make(i);
Andrew Walbranca808b12020-05-15 17:22:28 +0100104 } else {
J-Alvesee68c542020-10-29 17:48:20 +0000105 memory_region->handle = handle;
Andrew Walbranca808b12020-05-15 17:22:28 +0100106 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000107 allocated_state->share_func = share_func;
108 allocated_state->memory_region = memory_region;
Andrew Walbranca808b12020-05-15 17:22:28 +0100109 allocated_state->fragment_count = 1;
110 allocated_state->fragments[0] = composite->constituents;
111 allocated_state->fragment_constituent_counts[0] =
112 (fragment_length -
113 ffa_composite_constituent_offset(memory_region,
114 0)) /
115 sizeof(struct ffa_memory_region_constituent);
116 allocated_state->sending_complete = false;
Karl Meakin52cdfe72023-06-30 14:49:10 +0100117 for (uint32_t j = 0; j < MAX_MEM_SHARE_RECIPIENTS;
118 ++j) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100119 allocated_state->retrieved_fragment_count[j] =
120 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000121 }
Karl Meakin52cdfe72023-06-30 14:49:10 +0100122 return allocated_state;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000123 }
124 }
125
Karl Meakin52cdfe72023-06-30 14:49:10 +0100126 return NULL;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000127}
128
129/** Locks the share states lock. */
130struct share_states_locked share_states_lock(void)
131{
132 sl_lock(&share_states_lock_instance);
133
134 return (struct share_states_locked){.share_states = share_states};
135}
136
137/** Unlocks the share states lock. */
J-Alves66652252022-07-06 09:49:51 +0100138void share_states_unlock(struct share_states_locked *share_states)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000139{
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000140 assert(share_states->share_states != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000141 share_states->share_states = NULL;
142 sl_unlock(&share_states_lock_instance);
143}
144
145/**
Andrew Walbranca808b12020-05-15 17:22:28 +0100146 * If the given handle is a valid handle for an allocated share state then
Karl Meakin4a2854a2023-06-30 16:26:52 +0100147 * returns a pointer to the share state. Otherwise returns NULL.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000148 */
Karl Meakin4a2854a2023-06-30 16:26:52 +0100149struct ffa_memory_share_state *get_share_state(
150 struct share_states_locked share_states, ffa_memory_handle_t handle)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000151{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100152 struct ffa_memory_share_state *share_state;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000153
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000154 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100155
156 /*
157 * First look for a share_state allocated by us, in which case the
158 * handle is based on the index.
159 */
Olivier Deprez55a189e2021-06-09 15:45:27 +0200160 if (plat_ffa_memory_handle_allocated_by_current_world(handle)) {
Karl Meakin4a2854a2023-06-30 16:26:52 +0100161 uint64_t index = ffa_memory_handle_get_index(handle);
162
Andrew Walbranca808b12020-05-15 17:22:28 +0100163 if (index < MAX_MEM_SHARES) {
164 share_state = &share_states.share_states[index];
165 if (share_state->share_func != 0) {
Karl Meakin4a2854a2023-06-30 16:26:52 +0100166 return share_state;
Andrew Walbranca808b12020-05-15 17:22:28 +0100167 }
168 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000169 }
170
Andrew Walbranca808b12020-05-15 17:22:28 +0100171 /* Fall back to a linear scan. */
Karl Meakin4a2854a2023-06-30 16:26:52 +0100172 for (uint64_t index = 0; index < MAX_MEM_SHARES; ++index) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100173 share_state = &share_states.share_states[index];
J-Alvesee68c542020-10-29 17:48:20 +0000174 if (share_state->memory_region != NULL &&
175 share_state->memory_region->handle == handle &&
Andrew Walbranca808b12020-05-15 17:22:28 +0100176 share_state->share_func != 0) {
Karl Meakin4a2854a2023-06-30 16:26:52 +0100177 return share_state;
Andrew Walbranca808b12020-05-15 17:22:28 +0100178 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000179 }
180
Karl Meakin4a2854a2023-06-30 16:26:52 +0100181 return NULL;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000182}
183
184/** Marks a share state as unallocated. */
J-Alvesfdd29272022-07-19 13:16:31 +0100185void share_state_free(struct share_states_locked share_states,
186 struct ffa_memory_share_state *share_state,
187 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000188{
Andrew Walbranca808b12020-05-15 17:22:28 +0100189 uint32_t i;
190
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000191 assert(share_states.share_states != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000192 share_state->share_func = 0;
Andrew Walbranca808b12020-05-15 17:22:28 +0100193 share_state->sending_complete = false;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000194 mpool_free(page_pool, share_state->memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +0100195 /*
196 * First fragment is part of the same page as the `memory_region`, so it
197 * doesn't need to be freed separately.
198 */
199 share_state->fragments[0] = NULL;
200 share_state->fragment_constituent_counts[0] = 0;
201 for (i = 1; i < share_state->fragment_count; ++i) {
202 mpool_free(page_pool, share_state->fragments[i]);
203 share_state->fragments[i] = NULL;
204 share_state->fragment_constituent_counts[i] = 0;
205 }
206 share_state->fragment_count = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000207 share_state->memory_region = NULL;
J-Alvesa9cd7e32022-07-01 13:49:33 +0100208 share_state->hypervisor_fragment_count = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000209}
210
Andrew Walbranca808b12020-05-15 17:22:28 +0100211/** Checks whether the given share state has been fully sent. */
J-Alvesfdd29272022-07-19 13:16:31 +0100212bool share_state_sending_complete(struct share_states_locked share_states,
213 struct ffa_memory_share_state *share_state)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000214{
Andrew Walbranca808b12020-05-15 17:22:28 +0100215 struct ffa_composite_memory_region *composite;
216 uint32_t expected_constituent_count;
217 uint32_t fragment_constituent_count_total = 0;
218 uint32_t i;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000219
Andrew Walbranca808b12020-05-15 17:22:28 +0100220 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000221 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100222
223 /*
224 * Share state must already be valid, or it's not possible to get hold
225 * of it.
226 */
227 CHECK(share_state->memory_region != NULL &&
228 share_state->share_func != 0);
229
230 composite =
231 ffa_memory_region_get_composite(share_state->memory_region, 0);
232 expected_constituent_count = composite->constituent_count;
233 for (i = 0; i < share_state->fragment_count; ++i) {
234 fragment_constituent_count_total +=
235 share_state->fragment_constituent_counts[i];
236 }
237 dlog_verbose(
238 "Checking completion: constituent count %d/%d from %d "
239 "fragments.\n",
240 fragment_constituent_count_total, expected_constituent_count,
241 share_state->fragment_count);
242
243 return fragment_constituent_count_total == expected_constituent_count;
244}
245
246/**
247 * Calculates the offset of the next fragment expected for the given share
248 * state.
249 */
J-Alvesfdd29272022-07-19 13:16:31 +0100250uint32_t share_state_next_fragment_offset(
Andrew Walbranca808b12020-05-15 17:22:28 +0100251 struct share_states_locked share_states,
252 struct ffa_memory_share_state *share_state)
253{
254 uint32_t next_fragment_offset;
255 uint32_t i;
256
257 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000258 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100259
260 next_fragment_offset =
261 ffa_composite_constituent_offset(share_state->memory_region, 0);
262 for (i = 0; i < share_state->fragment_count; ++i) {
263 next_fragment_offset +=
264 share_state->fragment_constituent_counts[i] *
265 sizeof(struct ffa_memory_region_constituent);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000266 }
267
Andrew Walbranca808b12020-05-15 17:22:28 +0100268 return next_fragment_offset;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000269}
270
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100271static void dump_memory_region(struct ffa_memory_region *memory_region)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000272{
273 uint32_t i;
274
275 if (LOG_LEVEL < LOG_LEVEL_VERBOSE) {
276 return;
277 }
278
Karl Meakine8937d92024-03-19 16:04:25 +0000279 dlog("from VM %#x, attributes (shareability = %s, cacheability = %s, "
280 "type = %s, security = %s), flags %#x, handle %#lx "
281 "tag %lu, memory access descriptor size %u, to %u "
Andrew Walbrana65a1322020-04-06 19:32:32 +0100282 "recipients [",
Karl Meakine8937d92024-03-19 16:04:25 +0000283 memory_region->sender,
284 ffa_memory_shareability_name(
285 memory_region->attributes.shareability),
286 ffa_memory_cacheability_name(
287 memory_region->attributes.cacheability),
288 ffa_memory_type_name(memory_region->attributes.type),
289 ffa_memory_security_name(memory_region->attributes.security),
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000290 memory_region->flags, memory_region->handle, memory_region->tag,
291 memory_region->memory_access_desc_size,
Andrew Walbrana65a1322020-04-06 19:32:32 +0100292 memory_region->receiver_count);
293 for (i = 0; i < memory_region->receiver_count; ++i) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +0000294 struct ffa_memory_access *receiver =
295 ffa_memory_region_get_receiver(memory_region, i);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000296 if (i != 0) {
297 dlog(", ");
298 }
Karl Meakine8937d92024-03-19 16:04:25 +0000299 dlog("Receiver %#x: permissions (%s, %s) (offset %u)",
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +0000300 receiver->receiver_permissions.receiver,
Karl Meakine8937d92024-03-19 16:04:25 +0000301 ffa_data_access_name(receiver->receiver_permissions
302 .permissions.data_access),
303 ffa_instruction_access_name(
304 receiver->receiver_permissions.permissions
305 .instruction_access),
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +0000306 receiver->composite_memory_region_offset);
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000307 /* The impdef field is only present from v1.2 and later */
308 if (ffa_version_from_memory_access_desc_size(
309 memory_region->memory_access_desc_size) >=
Karl Meakin0e617d92024-04-05 12:55:22 +0100310 FFA_VERSION_1_2) {
Karl Meakine8937d92024-03-19 16:04:25 +0000311 dlog(", impdef: %#lx %#lx", receiver->impdef.val[0],
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000312 receiver->impdef.val[1]);
313 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000314 }
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000315 dlog("] at offset %u", memory_region->receivers_offset);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000316}
317
J-Alves66652252022-07-06 09:49:51 +0100318void dump_share_states(void)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000319{
320 uint32_t i;
321
322 if (LOG_LEVEL < LOG_LEVEL_VERBOSE) {
323 return;
324 }
325
326 dlog("Current share states:\n");
327 sl_lock(&share_states_lock_instance);
328 for (i = 0; i < MAX_MEM_SHARES; ++i) {
329 if (share_states[i].share_func != 0) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000330 switch (share_states[i].share_func) {
J-Alves95fbb312024-03-20 15:19:16 +0000331 case FFA_MEM_SHARE_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100332 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000333 dlog("SHARE");
334 break;
J-Alves95fbb312024-03-20 15:19:16 +0000335 case FFA_MEM_LEND_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100336 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000337 dlog("LEND");
338 break;
J-Alves95fbb312024-03-20 15:19:16 +0000339 case FFA_MEM_DONATE_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100340 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000341 dlog("DONATE");
342 break;
343 default:
344 dlog("invalid share_func %#x",
345 share_states[i].share_func);
346 }
Karl Meakine8937d92024-03-19 16:04:25 +0000347 dlog(" %#lx (", share_states[i].memory_region->handle);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000348 dump_memory_region(share_states[i].memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +0100349 if (share_states[i].sending_complete) {
350 dlog("): fully sent");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000351 } else {
Andrew Walbranca808b12020-05-15 17:22:28 +0100352 dlog("): partially sent");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000353 }
J-Alves2a0d2882020-10-29 14:49:50 +0000354 dlog(" with %d fragments, %d retrieved, "
355 " sender's original mode: %#x\n",
Andrew Walbranca808b12020-05-15 17:22:28 +0100356 share_states[i].fragment_count,
J-Alves2a0d2882020-10-29 14:49:50 +0000357 share_states[i].retrieved_fragment_count[0],
358 share_states[i].sender_orig_mode);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000359 }
360 }
361 sl_unlock(&share_states_lock_instance);
362}
363
Andrew Walbran475c1452020-02-07 13:22:22 +0000364/* TODO: Add device attributes: GRE, cacheability, shareability. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100365static inline uint32_t ffa_memory_permissions_to_mode(
J-Alves7cd5eb32020-10-16 19:06:10 +0100366 ffa_memory_access_permissions_t permissions, uint32_t default_mode)
Andrew Walbran475c1452020-02-07 13:22:22 +0000367{
368 uint32_t mode = 0;
369
Karl Meakin84710f32023-10-12 15:14:49 +0100370 switch (permissions.data_access) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100371 case FFA_DATA_ACCESS_RO:
Andrew Walbran475c1452020-02-07 13:22:22 +0000372 mode = MM_MODE_R;
373 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100374 case FFA_DATA_ACCESS_RW:
Andrew Walbran475c1452020-02-07 13:22:22 +0000375 mode = MM_MODE_R | MM_MODE_W;
376 break;
J-Alves7cd5eb32020-10-16 19:06:10 +0100377 case FFA_DATA_ACCESS_NOT_SPECIFIED:
378 mode = (default_mode & (MM_MODE_R | MM_MODE_W));
379 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100380 case FFA_DATA_ACCESS_RESERVED:
381 panic("Tried to convert FFA_DATA_ACCESS_RESERVED.");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100382 }
383
Karl Meakin84710f32023-10-12 15:14:49 +0100384 switch (permissions.instruction_access) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100385 case FFA_INSTRUCTION_ACCESS_NX:
Andrew Walbran475c1452020-02-07 13:22:22 +0000386 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100387 case FFA_INSTRUCTION_ACCESS_X:
Andrew Walbrana65a1322020-04-06 19:32:32 +0100388 mode |= MM_MODE_X;
389 break;
J-Alves7cd5eb32020-10-16 19:06:10 +0100390 case FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED:
391 mode |= (default_mode & MM_MODE_X);
392 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100393 case FFA_INSTRUCTION_ACCESS_RESERVED:
394 panic("Tried to convert FFA_INSTRUCTION_ACCESS_RESVERVED.");
Andrew Walbran475c1452020-02-07 13:22:22 +0000395 }
396
Olivier Deprez878bd5b2021-04-15 19:05:10 +0200397 /* Set the security state bit if necessary. */
398 if ((default_mode & plat_ffa_other_world_mode()) != 0) {
399 mode |= plat_ffa_other_world_mode();
400 }
401
Andrew Walbran475c1452020-02-07 13:22:22 +0000402 return mode;
403}
404
Jose Marinho75509b42019-04-09 09:34:59 +0100405/**
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000406 * Get the current mode in the stage-2 page table of the given vm of all the
407 * pages in the given constituents, if they all have the same mode, or return
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100408 * an appropriate FF-A error if not.
Jose Marinho75509b42019-04-09 09:34:59 +0100409 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100410static struct ffa_value constituents_get_mode(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000411 struct vm_locked vm, uint32_t *orig_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100412 struct ffa_memory_region_constituent **fragments,
413 const uint32_t *fragment_constituent_counts, uint32_t fragment_count)
Jose Marinho75509b42019-04-09 09:34:59 +0100414{
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100415 uint32_t i;
Andrew Walbranca808b12020-05-15 17:22:28 +0100416 uint32_t j;
Jose Marinho75509b42019-04-09 09:34:59 +0100417
Andrew Walbranca808b12020-05-15 17:22:28 +0100418 if (fragment_count == 0 || fragment_constituent_counts[0] == 0) {
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100419 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000420 * Fail if there are no constituents. Otherwise we would get an
421 * uninitialised *orig_mode.
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100422 */
Karl Meakin5df422c2023-07-11 17:31:38 +0100423 dlog_verbose("%s: no constituents\n", __func__);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100424 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho75509b42019-04-09 09:34:59 +0100425 }
426
Andrew Walbranca808b12020-05-15 17:22:28 +0100427 for (i = 0; i < fragment_count; ++i) {
428 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
429 ipaddr_t begin = ipa_init(fragments[i][j].address);
430 size_t size = fragments[i][j].page_count * PAGE_SIZE;
431 ipaddr_t end = ipa_add(begin, size);
432 uint32_t current_mode;
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100433
Andrew Walbranca808b12020-05-15 17:22:28 +0100434 /* Fail if addresses are not page-aligned. */
435 if (!is_aligned(ipa_addr(begin), PAGE_SIZE) ||
436 !is_aligned(ipa_addr(end), PAGE_SIZE)) {
Karl Meakin5df422c2023-07-11 17:31:38 +0100437 dlog_verbose("%s: addresses not page-aligned\n",
438 __func__);
Andrew Walbranca808b12020-05-15 17:22:28 +0100439 return ffa_error(FFA_INVALID_PARAMETERS);
440 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100441
Andrew Walbranca808b12020-05-15 17:22:28 +0100442 /*
443 * Ensure that this constituent memory range is all
444 * mapped with the same mode.
445 */
Raghu Krishnamurthy785d52f2021-02-13 00:02:40 -0800446 if (!vm_mem_get_mode(vm, begin, end, &current_mode)) {
Karl Meakin5df422c2023-07-11 17:31:38 +0100447 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +0000448 "%s: constituent memory range "
449 "%#lx..%#lx "
Karl Meakin5df422c2023-07-11 17:31:38 +0100450 "not mapped with the same mode\n",
Karl Meakine8937d92024-03-19 16:04:25 +0000451 __func__, begin.ipa, end.ipa);
Andrew Walbranca808b12020-05-15 17:22:28 +0100452 return ffa_error(FFA_DENIED);
453 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100454
Andrew Walbranca808b12020-05-15 17:22:28 +0100455 /*
456 * Ensure that all constituents are mapped with the same
457 * mode.
458 */
459 if (i == 0) {
460 *orig_mode = current_mode;
461 } else if (current_mode != *orig_mode) {
462 dlog_verbose(
Karl Meakin5df422c2023-07-11 17:31:38 +0100463 "%s: expected mode %#x but was %#x for "
Karl Meakine8937d92024-03-19 16:04:25 +0000464 "%d pages at %#lx.\n",
Karl Meakin5df422c2023-07-11 17:31:38 +0100465 __func__, *orig_mode, current_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100466 fragments[i][j].page_count,
467 ipa_addr(begin));
468 return ffa_error(FFA_DENIED);
469 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100470 }
Jose Marinho75509b42019-04-09 09:34:59 +0100471 }
472
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100473 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000474}
475
Karl Meakin0e617d92024-04-05 12:55:22 +0100476enum ffa_version ffa_version_from_memory_access_desc_size(
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100477 uint32_t memory_access_desc_size)
478{
479 switch (memory_access_desc_size) {
480 /*
481 * v1.0 and v1.1 memory access descriptors are the same size however
482 * v1.1 is the first version to include the memory access descriptor
483 * size field so return v1.1.
484 */
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000485 case sizeof(struct ffa_memory_access_v1_0):
Karl Meakin0e617d92024-04-05 12:55:22 +0100486 return FFA_VERSION_1_1;
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000487 case sizeof(struct ffa_memory_access):
Karl Meakin0e617d92024-04-05 12:55:22 +0100488 return FFA_VERSION_1_2;
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100489 }
490 return 0;
491}
492
493/**
494 * Check if the receivers size and offset given is valid for the senders
495 * FF-A version.
496 */
497static bool receiver_size_and_offset_valid_for_version(
498 uint32_t receivers_size, uint32_t receivers_offset,
Karl Meakin0e617d92024-04-05 12:55:22 +0100499 enum ffa_version ffa_version)
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100500{
501 /*
502 * Check that the version that the memory access descriptor size belongs
503 * to is compatible with the FF-A version we believe the sender to be.
504 */
Karl Meakin0e617d92024-04-05 12:55:22 +0100505 enum ffa_version expected_ffa_version =
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100506 ffa_version_from_memory_access_desc_size(receivers_size);
Karl Meakin0e617d92024-04-05 12:55:22 +0100507 if (!ffa_versions_are_compatible(expected_ffa_version, ffa_version)) {
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100508 return false;
509 }
510
511 /*
512 * Check the receivers_offset matches the version we found from
513 * memory access descriptor size.
514 */
515 switch (expected_ffa_version) {
Karl Meakin0e617d92024-04-05 12:55:22 +0100516 case FFA_VERSION_1_1:
517 case FFA_VERSION_1_2:
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100518 return receivers_offset == sizeof(struct ffa_memory_region);
519 default:
520 return false;
521 }
522}
523
524/**
525 * Check the values set for fields in the memory region are valid and safe.
526 * Offset values are within safe bounds, receiver count will not cause overflows
527 * and reserved fields are 0.
528 */
529bool ffa_memory_region_sanity_check(struct ffa_memory_region *memory_region,
Karl Meakin0e617d92024-04-05 12:55:22 +0100530 enum ffa_version ffa_version,
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100531 uint32_t fragment_length,
532 bool send_transaction)
533{
534 uint32_t receiver_count;
535 struct ffa_memory_access *receiver;
536 uint32_t composite_offset_0;
Daniel Boulbyf06b5232024-02-22 16:26:43 +0000537 struct ffa_memory_region_v1_0 *memory_region_v1_0 =
538 (struct ffa_memory_region_v1_0 *)memory_region;
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100539
Karl Meakin0e617d92024-04-05 12:55:22 +0100540 if (ffa_version == FFA_VERSION_1_0) {
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100541 /* Check the reserved fields are 0. */
542 if (memory_region_v1_0->reserved_0 != 0 ||
543 memory_region_v1_0->reserved_1 != 0) {
544 dlog_verbose("Reserved fields must be 0.\n");
545 return false;
546 }
547
548 receiver_count = memory_region_v1_0->receiver_count;
549 } else {
550 uint32_t receivers_size =
551 memory_region->memory_access_desc_size;
552 uint32_t receivers_offset = memory_region->receivers_offset;
553
554 /* Check the reserved field is 0. */
555 if (memory_region->reserved[0] != 0 ||
556 memory_region->reserved[1] != 0 ||
557 memory_region->reserved[2] != 0) {
558 dlog_verbose("Reserved fields must be 0.\n");
559 return false;
560 }
561
562 /*
563 * Check memory_access_desc_size matches the size of the struct
564 * for the senders FF-A version.
565 */
566 if (!receiver_size_and_offset_valid_for_version(
567 receivers_size, receivers_offset, ffa_version)) {
568 dlog_verbose(
569 "Invalid memory access descriptor size %d, "
570 " or receiver offset %d, "
571 "for FF-A version %#x\n",
572 receivers_size, receivers_offset, ffa_version);
573 return false;
574 }
575
576 receiver_count = memory_region->receiver_count;
577 }
578
579 /* Check receiver count is not too large. */
Daniel Boulbyf06b5232024-02-22 16:26:43 +0000580 if (receiver_count > MAX_MEM_SHARE_RECIPIENTS || receiver_count < 1) {
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100581 dlog_verbose(
Daniel Boulbyf06b5232024-02-22 16:26:43 +0000582 "Receiver count must be 0 < receiver_count < %u "
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100583 "specified %u\n",
584 MAX_MEM_SHARE_RECIPIENTS, receiver_count);
585 return false;
586 }
587
588 /* Check values in the memory access descriptors. */
589 /*
590 * The composite offset values must be the same for all recievers so
591 * check the first one is valid and then they are all the same.
592 */
Karl Meakin0e617d92024-04-05 12:55:22 +0100593 receiver = ffa_version == FFA_VERSION_1_0
Daniel Boulbyf06b5232024-02-22 16:26:43 +0000594 ? (struct ffa_memory_access *)&memory_region_v1_0
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100595 ->receivers[0]
596 : ffa_memory_region_get_receiver(memory_region, 0);
597 assert(receiver != NULL);
598 composite_offset_0 = receiver->composite_memory_region_offset;
599
600 if (!send_transaction) {
601 if (composite_offset_0 != 0) {
602 dlog_verbose(
603 "Composite offset memory region descriptor "
604 "offset must be 0 for retrieve requests. "
605 "Currently %d",
606 composite_offset_0);
607 return false;
608 }
609 } else {
610 bool comp_offset_is_zero = composite_offset_0 == 0U;
611 bool comp_offset_lt_transaction_descriptor_size =
612 composite_offset_0 <
613 (sizeof(struct ffa_memory_region) +
614 (uint32_t)(memory_region->memory_access_desc_size *
615 memory_region->receiver_count));
616 bool comp_offset_with_comp_gt_fragment_length =
617 composite_offset_0 +
618 sizeof(struct ffa_composite_memory_region) >
619 fragment_length;
620 if (comp_offset_is_zero ||
621 comp_offset_lt_transaction_descriptor_size ||
622 comp_offset_with_comp_gt_fragment_length) {
623 dlog_verbose(
624 "Invalid composite memory region descriptor "
625 "offset for send transaction %u\n",
626 composite_offset_0);
627 return false;
628 }
629 }
630
Karl Meakin824b63d2024-06-03 19:04:53 +0100631 for (size_t i = 0; i < memory_region->receiver_count; i++) {
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100632 uint32_t composite_offset;
633
Karl Meakin0e617d92024-04-05 12:55:22 +0100634 if (ffa_version == FFA_VERSION_1_0) {
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100635 struct ffa_memory_access_v1_0 *receiver_v1_0 =
636 &memory_region_v1_0->receivers[i];
637 /* Check reserved fields are 0 */
638 if (receiver_v1_0->reserved_0 != 0) {
639 dlog_verbose(
640 "Reserved field in the memory access "
Karl Meakine8937d92024-03-19 16:04:25 +0000641 "descriptor must be zero. Currently "
642 "reciever %zu has a reserved field "
643 "with a value of %lu\n",
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100644 i, receiver_v1_0->reserved_0);
645 return false;
646 }
647 /*
648 * We can cast to the current version receiver as the
649 * remaining fields we are checking have the same
650 * offsets for all versions since memory access
651 * descriptors are forwards compatible.
652 */
653 receiver = (struct ffa_memory_access *)receiver_v1_0;
654 } else {
655 receiver = ffa_memory_region_get_receiver(memory_region,
656 i);
657 assert(receiver != NULL);
658
659 if (receiver->reserved_0 != 0) {
660 dlog_verbose(
661 "Reserved field in the memory access "
Karl Meakine8937d92024-03-19 16:04:25 +0000662 "descriptor must be zero. Currently "
663 "reciever %zu has a reserved field "
664 "with a value of %lu\n",
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100665 i, receiver->reserved_0);
666 return false;
667 }
668 }
669
670 /* Check composite offset values are equal for all receivers. */
671 composite_offset = receiver->composite_memory_region_offset;
672 if (composite_offset != composite_offset_0) {
673 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +0000674 "Composite offset %x differs from %x in "
675 "index\n",
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100676 composite_offset, composite_offset_0);
677 return false;
678 }
679 }
680 return true;
681}
682
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000683/**
J-Alves460d36c2023-10-12 17:02:15 +0100684 * If the receivers for the memory management operation are all from the
685 * secure world and this isn't a FFA_MEM_SHARE, then request memory security
686 * state update by returning MAP_ACTION_CHECK_PROTECT.
687 */
688static enum ffa_map_action ffa_mem_send_get_map_action(
689 bool all_receivers_from_current_world, ffa_id_t sender_id,
690 uint32_t mem_func_id)
691{
J-Alves95fbb312024-03-20 15:19:16 +0000692 const bool is_memory_share_abi = mem_func_id == FFA_MEM_SHARE_32 ||
693 mem_func_id == FFA_MEM_SHARE_64;
694 const bool protect_memory =
695 (!is_memory_share_abi && all_receivers_from_current_world &&
696 ffa_is_vm_id(sender_id));
J-Alves460d36c2023-10-12 17:02:15 +0100697
698 return protect_memory ? MAP_ACTION_CHECK_PROTECT : MAP_ACTION_CHECK;
699}
700
701/**
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000702 * Verify that all pages have the same mode, that the starting mode
703 * constitutes a valid state and obtain the next mode to apply
J-Alves460d36c2023-10-12 17:02:15 +0100704 * to the sending VM. It outputs the mapping action that needs to be
705 * invoked for the given memory range. On memory lend/donate there
706 * could be a need to protect the memory from the normal world.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000707 *
708 * Returns:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100709 * 1) FFA_DENIED if a state transition was not found;
710 * 2) FFA_DENIED if the pages being shared do not have the same mode within
Andrew Walbrana65a1322020-04-06 19:32:32 +0100711 * the <from> VM;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100712 * 3) FFA_INVALID_PARAMETERS if the beginning and end IPAs are not page
Andrew Walbrana65a1322020-04-06 19:32:32 +0100713 * aligned;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100714 * 4) FFA_INVALID_PARAMETERS if the requested share type was not handled.
715 * Or FFA_SUCCESS on success.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000716 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100717static struct ffa_value ffa_send_check_transition(
Andrew Walbrana65a1322020-04-06 19:32:32 +0100718 struct vm_locked from, uint32_t share_func,
Daniel Boulbya76fd912024-02-22 14:22:15 +0000719 struct ffa_memory_region *memory_region, uint32_t *orig_from_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100720 struct ffa_memory_region_constituent **fragments,
721 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
Daniel Boulby4b846eb2024-05-23 17:32:23 +0100722 uint32_t *from_mode, enum ffa_map_action *map_action, bool zero)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000723{
724 const uint32_t state_mask =
725 MM_MODE_INVALID | MM_MODE_UNOWNED | MM_MODE_SHARED;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100726 struct ffa_value ret;
J-Alves460d36c2023-10-12 17:02:15 +0100727 bool all_receivers_from_current_world = true;
Daniel Boulbya76fd912024-02-22 14:22:15 +0000728 uint32_t receivers_count = memory_region->receiver_count;
J-Alves95fbb312024-03-20 15:19:16 +0000729 const bool is_memory_lend = (share_func == FFA_MEM_LEND_32) ||
730 (share_func == FFA_MEM_LEND_64);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000731
Andrew Walbranca808b12020-05-15 17:22:28 +0100732 ret = constituents_get_mode(from, orig_from_mode, fragments,
733 fragment_constituent_counts,
734 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100735 if (ret.func != FFA_SUCCESS_32) {
Olivier Depreze7eb1682022-03-16 17:09:03 +0100736 dlog_verbose("Inconsistent modes.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100737 return ret;
Andrew Scullb5f49e02019-10-02 13:20:47 +0100738 }
739
Daniel Boulby63af1fa2024-03-18 14:17:31 +0000740 /* Device memory regions can only be lent a single borrower. */
Daniel Boulby9764ff62024-01-30 17:47:39 +0000741 if ((*orig_from_mode & MM_MODE_D) != 0U &&
J-Alves95fbb312024-03-20 15:19:16 +0000742 !(is_memory_lend && receivers_count == 1)) {
Daniel Boulby9764ff62024-01-30 17:47:39 +0000743 dlog_verbose(
Daniel Boulby63af1fa2024-03-18 14:17:31 +0000744 "Device memory can only be lent to a single borrower "
745 "(mode is %#x).\n",
Daniel Boulby9764ff62024-01-30 17:47:39 +0000746 *orig_from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100747 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000748 }
749
750 /*
751 * Ensure the sender is the owner and has exclusive access to the
752 * memory.
753 */
754 if ((*orig_from_mode & state_mask) != 0) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100755 return ffa_error(FFA_DENIED);
Andrew Walbrana65a1322020-04-06 19:32:32 +0100756 }
757
Daniel Boulby4b846eb2024-05-23 17:32:23 +0100758 /*
759 * Memory cannot be zeroed during the lend/donate operation if the
760 * sender only has RO access.
761 */
762 if ((*orig_from_mode & MM_MODE_W) == 0 && zero == true) {
763 dlog_verbose(
764 "Cannot zero memory when the sender doesn't have "
765 "write access\n");
766 return ffa_error(FFA_DENIED);
767 }
768
Daniel Boulbya76fd912024-02-22 14:22:15 +0000769 assert(receivers_count > 0U);
J-Alves7cd5eb32020-10-16 19:06:10 +0100770
J-Alves363f5722022-04-25 17:37:37 +0100771 for (uint32_t i = 0U; i < receivers_count; i++) {
Daniel Boulbya76fd912024-02-22 14:22:15 +0000772 struct ffa_memory_access *receiver =
773 ffa_memory_region_get_receiver(memory_region, i);
774 assert(receiver != NULL);
J-Alves363f5722022-04-25 17:37:37 +0100775 ffa_memory_access_permissions_t permissions =
Daniel Boulbya76fd912024-02-22 14:22:15 +0000776 receiver->receiver_permissions.permissions;
J-Alves363f5722022-04-25 17:37:37 +0100777 uint32_t required_from_mode = ffa_memory_permissions_to_mode(
778 permissions, *orig_from_mode);
779
J-Alves788b4492023-04-18 14:01:23 +0100780 /*
781 * The assumption is that at this point, the operation from
782 * SP to a receiver VM, should have returned an FFA_ERROR
783 * already.
784 */
785 if (!ffa_is_vm_id(from.vm->id)) {
786 assert(!ffa_is_vm_id(
Daniel Boulbya76fd912024-02-22 14:22:15 +0000787 receiver->receiver_permissions.receiver));
J-Alves788b4492023-04-18 14:01:23 +0100788 }
789
J-Alves460d36c2023-10-12 17:02:15 +0100790 /* Track if all senders are from current world. */
791 all_receivers_from_current_world =
792 all_receivers_from_current_world &&
793 vm_id_is_current_world(
Daniel Boulbya76fd912024-02-22 14:22:15 +0000794 receiver->receiver_permissions.receiver);
J-Alves460d36c2023-10-12 17:02:15 +0100795
J-Alves363f5722022-04-25 17:37:37 +0100796 if ((*orig_from_mode & required_from_mode) !=
797 required_from_mode) {
798 dlog_verbose(
799 "Sender tried to send memory with permissions "
J-Alves788b4492023-04-18 14:01:23 +0100800 "which required mode %#x but only had %#x "
801 "itself.\n",
J-Alves363f5722022-04-25 17:37:37 +0100802 required_from_mode, *orig_from_mode);
803 return ffa_error(FFA_DENIED);
804 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000805 }
806
J-Alves460d36c2023-10-12 17:02:15 +0100807 *map_action = ffa_mem_send_get_map_action(
808 all_receivers_from_current_world, from.vm->id, share_func);
809
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000810 /* Find the appropriate new mode. */
811 *from_mode = ~state_mask & *orig_from_mode;
Andrew Walbrane7ad3c02019-12-24 17:03:04 +0000812 switch (share_func) {
J-Alves95fbb312024-03-20 15:19:16 +0000813 case FFA_MEM_DONATE_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100814 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000815 *from_mode |= MM_MODE_INVALID | MM_MODE_UNOWNED;
Jose Marinho75509b42019-04-09 09:34:59 +0100816 break;
J-Alves95fbb312024-03-20 15:19:16 +0000817 case FFA_MEM_LEND_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100818 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000819 *from_mode |= MM_MODE_INVALID;
Andrew Walbran648fc3e2019-10-22 16:23:05 +0100820 break;
J-Alves95fbb312024-03-20 15:19:16 +0000821 case FFA_MEM_SHARE_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100822 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000823 *from_mode |= MM_MODE_SHARED;
Jose Marinho56c25732019-05-20 09:48:53 +0100824 break;
825
Jose Marinho75509b42019-04-09 09:34:59 +0100826 default:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100827 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho75509b42019-04-09 09:34:59 +0100828 }
829
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100830 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000831}
832
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100833static struct ffa_value ffa_relinquish_check_transition(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000834 struct vm_locked from, uint32_t *orig_from_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100835 struct ffa_memory_region_constituent **fragments,
836 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
J-Alves69cdfd92024-04-26 11:40:59 +0100837 uint32_t *from_mode, enum ffa_map_action *map_action)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000838{
839 const uint32_t state_mask =
840 MM_MODE_INVALID | MM_MODE_UNOWNED | MM_MODE_SHARED;
841 uint32_t orig_from_state;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100842 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000843
J-Alves69cdfd92024-04-26 11:40:59 +0100844 assert(map_action != NULL);
845 if (vm_id_is_current_world(from.vm->id)) {
846 *map_action = MAP_ACTION_COMMIT;
847 } else {
848 /*
849 * No need to check the attributes of caller.
850 * The assumption is that the retrieve request of the receiver
851 * also used the MAP_ACTION_NONE, and no update was done to the
852 * page tables. When the receiver is not at the secure virtual
853 * instance SPMC doesn't manage its S2 translation (i.e. when
854 * the receiver is a VM).
855 */
856 *map_action = MAP_ACTION_NONE;
857
858 return (struct ffa_value){.func = FFA_SUCCESS_32};
859 }
860
Andrew Walbranca808b12020-05-15 17:22:28 +0100861 ret = constituents_get_mode(from, orig_from_mode, fragments,
862 fragment_constituent_counts,
863 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100864 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbrana65a1322020-04-06 19:32:32 +0100865 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000866 }
867
868 /* Ensure the address range is normal memory and not a device. */
869 if (*orig_from_mode & MM_MODE_D) {
870 dlog_verbose("Can't relinquish device memory (mode is %#x).\n",
871 *orig_from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100872 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000873 }
874
875 /*
876 * Ensure the relinquishing VM is not the owner but has access to the
877 * memory.
878 */
879 orig_from_state = *orig_from_mode & state_mask;
880 if ((orig_from_state & ~MM_MODE_SHARED) != MM_MODE_UNOWNED) {
881 dlog_verbose(
882 "Tried to relinquish memory in state %#x (masked %#x "
Andrew Walbranca808b12020-05-15 17:22:28 +0100883 "but should be %#x).\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000884 *orig_from_mode, orig_from_state, MM_MODE_UNOWNED);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100885 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000886 }
887
888 /* Find the appropriate new mode. */
889 *from_mode = (~state_mask & *orig_from_mode) | MM_MODE_UNMAPPED_MASK;
890
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100891 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000892}
893
894/**
895 * Verify that all pages have the same mode, that the starting mode
896 * constitutes a valid state and obtain the next mode to apply
897 * to the retrieving VM.
898 *
899 * Returns:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100900 * 1) FFA_DENIED if a state transition was not found;
901 * 2) FFA_DENIED if the pages being shared do not have the same mode within
Andrew Walbrana65a1322020-04-06 19:32:32 +0100902 * the <to> VM;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100903 * 3) FFA_INVALID_PARAMETERS if the beginning and end IPAs are not page
Andrew Walbrana65a1322020-04-06 19:32:32 +0100904 * aligned;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100905 * 4) FFA_INVALID_PARAMETERS if the requested share type was not handled.
906 * Or FFA_SUCCESS on success.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000907 */
J-Alvesfc19b372022-07-06 12:17:35 +0100908struct ffa_value ffa_retrieve_check_transition(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000909 struct vm_locked to, uint32_t share_func,
Andrew Walbranca808b12020-05-15 17:22:28 +0100910 struct ffa_memory_region_constituent **fragments,
911 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
J-Alvesfd206052023-05-22 16:45:00 +0100912 uint32_t memory_to_attributes, uint32_t *to_mode, bool memory_protected,
913 enum ffa_map_action *map_action)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000914{
915 uint32_t orig_to_mode;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100916 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000917
Andrew Walbranca808b12020-05-15 17:22:28 +0100918 ret = constituents_get_mode(to, &orig_to_mode, fragments,
919 fragment_constituent_counts,
920 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100921 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100922 dlog_verbose("Inconsistent modes.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100923 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000924 }
925
J-Alves460d36c2023-10-12 17:02:15 +0100926 /* Find the appropriate new mode. */
927 *to_mode = memory_to_attributes;
928
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100929 if (share_func == FFA_MEM_RECLAIM_32) {
J-Alves9256f162021-12-09 13:18:43 +0000930 /*
931 * If the original ffa memory send call has been processed
932 * successfully, it is expected the orig_to_mode would overlay
933 * with `state_mask`, as a result of the function
934 * `ffa_send_check_transition`.
J-Alvesfd206052023-05-22 16:45:00 +0100935 *
936 * If Hafnium is the SPMC:
937 * - Caller of the reclaim interface is an SP, the memory shall
938 * have been protected throughout the flow.
939 * - Caller of the reclaim is from the NWd, the memory may have
940 * been protected at the time of lending/donating the memory.
941 * In such case, set action to unprotect memory in the
942 * handling of reclaim operation.
943 * - If Hafnium is the hypervisor memory shall never have been
944 * protected in memory lend/share/donate.
945 *
946 * More details in the doc comment of the function
947 * `ffa_region_group_identity_map`.
J-Alves9256f162021-12-09 13:18:43 +0000948 */
J-Alves59ed0042022-07-28 18:26:41 +0100949 if (vm_id_is_current_world(to.vm->id)) {
950 assert((orig_to_mode &
951 (MM_MODE_INVALID | MM_MODE_UNOWNED |
952 MM_MODE_SHARED)) != 0U);
J-Alvesfd206052023-05-22 16:45:00 +0100953 assert(!memory_protected);
954 } else if (to.vm->id == HF_OTHER_WORLD_ID &&
955 map_action != NULL && memory_protected) {
956 *map_action = MAP_ACTION_COMMIT_UNPROTECT;
J-Alves59ed0042022-07-28 18:26:41 +0100957 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000958 } else {
J-Alves69cdfd92024-04-26 11:40:59 +0100959 if (!vm_id_is_current_world(to.vm->id)) {
960 assert(map_action != NULL);
961 *map_action = MAP_ACTION_NONE;
962 return (struct ffa_value){.func = FFA_SUCCESS_32};
963 }
964
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000965 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +0100966 * If the retriever is from virtual FF-A instance:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000967 * Ensure the retriever has the expected state. We don't care
968 * about the MM_MODE_SHARED bit; either with or without it set
969 * are both valid representations of the !O-NA state.
970 */
J-Alvesa9cd7e32022-07-01 13:49:33 +0100971 if (vm_id_is_current_world(to.vm->id) &&
Karl Meakin5e996992024-05-20 11:27:07 +0100972 !vm_is_primary(to.vm) &&
J-Alvesa9cd7e32022-07-01 13:49:33 +0100973 (orig_to_mode & MM_MODE_UNMAPPED_MASK) !=
974 MM_MODE_UNMAPPED_MASK) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100975 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000976 }
J-Alves460d36c2023-10-12 17:02:15 +0100977
978 /*
979 * If memory has been protected before, clear the NS bit to
980 * allow the secure access from the SP.
981 */
982 if (memory_protected) {
983 *to_mode &= ~plat_ffa_other_world_mode();
984 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000985 }
986
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000987 switch (share_func) {
J-Alves95fbb312024-03-20 15:19:16 +0000988 case FFA_MEM_DONATE_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100989 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000990 *to_mode |= 0;
991 break;
J-Alves95fbb312024-03-20 15:19:16 +0000992 case FFA_MEM_LEND_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100993 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000994 *to_mode |= MM_MODE_UNOWNED;
995 break;
J-Alves95fbb312024-03-20 15:19:16 +0000996 case FFA_MEM_SHARE_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100997 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000998 *to_mode |= MM_MODE_UNOWNED | MM_MODE_SHARED;
999 break;
1000
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001001 case FFA_MEM_RECLAIM_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001002 *to_mode |= 0;
1003 break;
1004
1005 default:
Andrew Walbranca808b12020-05-15 17:22:28 +01001006 dlog_error("Invalid share_func %#x.\n", share_func);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001007 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001008 }
1009
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001010 return (struct ffa_value){.func = FFA_SUCCESS_32};
Jose Marinho75509b42019-04-09 09:34:59 +01001011}
Jose Marinho09b1db82019-08-08 09:16:59 +01001012
J-Alvescf6253e2024-01-03 13:48:48 +00001013/*
1014 * Performs the operations related to the `action` MAP_ACTION_CHECK*.
1015 * Returns:
1016 * - FFA_SUCCESS_32: if all goes well.
1017 * - FFA_ERROR_32: with FFA_NO_MEMORY, if there is no memory to manage
1018 * the page table update. Or error code provided by the function
1019 * `arch_memory_protect`.
1020 */
1021static struct ffa_value ffa_region_group_check_actions(
1022 struct vm_locked vm_locked, paddr_t pa_begin, paddr_t pa_end,
1023 struct mpool *ppool, uint32_t mode, enum ffa_map_action action,
1024 bool *memory_protected)
1025{
1026 struct ffa_value ret;
1027 bool is_memory_protected;
1028
1029 if (!vm_identity_prepare(vm_locked, pa_begin, pa_end, mode, ppool)) {
1030 dlog_verbose(
1031 "%s: memory can't be mapped to %x due to lack of "
Karl Meakine8937d92024-03-19 16:04:25 +00001032 "memory. Base: %lx end: %lx\n",
J-Alvescf6253e2024-01-03 13:48:48 +00001033 __func__, vm_locked.vm->id, pa_addr(pa_begin),
1034 pa_addr(pa_end));
1035 return ffa_error(FFA_NO_MEMORY);
1036 }
1037
1038 switch (action) {
1039 case MAP_ACTION_CHECK:
1040 /* No protect requested. */
1041 is_memory_protected = false;
1042 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
1043 break;
1044 case MAP_ACTION_CHECK_PROTECT: {
1045 paddr_t last_protected_pa = pa_init(0);
1046
1047 ret = arch_memory_protect(pa_begin, pa_end, &last_protected_pa);
1048
1049 is_memory_protected = (ret.func == FFA_SUCCESS_32);
1050
1051 /*
1052 * - If protect memory has failed with FFA_DENIED, means some
1053 * range of memory was in the wrong state. In such case, SPM
1054 * reverts the state of the pages that were successfully
1055 * updated.
1056 * - If protect memory has failed with FFA_NOT_SUPPORTED, it
1057 * means the platform doesn't support the protection mechanism.
1058 * That said, it still permits the page table update to go
1059 * through. The variable
1060 * `is_memory_protected` will be equal to false.
1061 * - If protect memory has failed with FFA_INVALID_PARAMETERS,
1062 * break from switch and return the error.
1063 */
1064 if (ret.func == FFA_ERROR_32) {
1065 assert(!is_memory_protected);
1066 if (ffa_error_code(ret) == FFA_DENIED &&
1067 pa_addr(last_protected_pa) != (uintptr_t)0) {
1068 CHECK(arch_memory_unprotect(
1069 pa_begin,
1070 pa_add(last_protected_pa, PAGE_SIZE)));
1071 } else if (ffa_error_code(ret) == FFA_NOT_SUPPORTED) {
1072 ret = (struct ffa_value){
1073 .func = FFA_SUCCESS_32,
1074 };
1075 }
1076 }
1077 } break;
1078 default:
1079 panic("%s: invalid action to process %x\n", __func__, action);
1080 }
1081
1082 if (memory_protected != NULL) {
1083 *memory_protected = is_memory_protected;
1084 }
1085
1086 return ret;
1087}
1088
1089static void ffa_region_group_commit_actions(struct vm_locked vm_locked,
1090 paddr_t pa_begin, paddr_t pa_end,
1091 struct mpool *ppool, uint32_t mode,
1092 enum ffa_map_action action)
1093{
1094 switch (action) {
1095 case MAP_ACTION_COMMIT_UNPROTECT:
1096 /*
1097 * Checking that it should succeed because SPM should be
1098 * unprotecting memory that it had protected before.
1099 */
1100 CHECK(arch_memory_unprotect(pa_begin, pa_end));
1101 case MAP_ACTION_COMMIT:
1102 vm_identity_commit(vm_locked, pa_begin, pa_end, mode, ppool,
1103 NULL);
1104 break;
1105 default:
1106 panic("%s: invalid action to process %x\n", __func__, action);
1107 }
1108}
1109
Jose Marinho09b1db82019-08-08 09:16:59 +01001110/**
J-Alves063ad832023-10-03 18:05:40 +01001111 * Helper function to revert a failed "Protect" action from the SPMC:
1112 * - `fragment_count`: should specify the number of fragments to traverse from
1113 * `fragments`. This may not be the full amount of fragments that are part of
1114 * the share_state structure.
1115 * - `fragment_constituent_counts`: array holding the amount of constituents
1116 * per fragment.
1117 * - `end`: pointer to the constituent that failed the "protect" action. It
1118 * shall be part of the last fragment, and it shall make the loop below break.
1119 */
1120static void ffa_region_group_fragments_revert_protect(
1121 struct ffa_memory_region_constituent **fragments,
1122 const uint32_t *fragment_constituent_counts, uint32_t fragment_count,
1123 const struct ffa_memory_region_constituent *end)
1124{
1125 for (uint32_t i = 0; i < fragment_count; ++i) {
1126 for (uint32_t j = 0; j < fragment_constituent_counts[i]; ++j) {
1127 struct ffa_memory_region_constituent *constituent =
1128 &fragments[i][j];
1129 size_t size = constituent->page_count * PAGE_SIZE;
1130 paddr_t pa_begin =
1131 pa_from_ipa(ipa_init(constituent->address));
1132 paddr_t pa_end = pa_add(pa_begin, size);
1133
Karl Meakine8937d92024-03-19 16:04:25 +00001134 dlog_verbose("%s: reverting fragment %lx size %zx\n",
J-Alves063ad832023-10-03 18:05:40 +01001135 __func__, pa_addr(pa_begin), size);
1136
1137 if (constituent == end) {
1138 /*
1139 * The last constituent is expected to be in the
1140 * last fragment.
1141 */
1142 assert(i == fragment_count - 1);
1143 break;
1144 }
1145
1146 CHECK(arch_memory_unprotect(pa_begin, pa_end));
1147 }
1148 }
1149}
1150
1151/**
Jose Marinho09b1db82019-08-08 09:16:59 +01001152 * Updates a VM's page table such that the given set of physical address ranges
1153 * are mapped in the address space at the corresponding address ranges, in the
1154 * mode provided.
1155 *
J-Alves0a83dc22023-05-05 09:50:37 +01001156 * The enum ffa_map_action determines the action taken from a call to the
1157 * function below:
1158 * - If action is MAP_ACTION_CHECK, the page tables will be allocated from the
1159 * mpool but no mappings will actually be updated. This function must always
1160 * be called first with action set to MAP_ACTION_CHECK to check that it will
1161 * succeed before calling ffa_region_group_identity_map with whichever one of
1162 * the remaining actions, to avoid leaving the page table in a half-updated
1163 * state.
1164 * - The action MAP_ACTION_COMMIT allocates the page tables from the mpool, and
1165 * changes the memory mappings.
J-Alvescf6253e2024-01-03 13:48:48 +00001166 * - The action MAP_ACTION_CHECK_PROTECT extends the MAP_ACTION_CHECK with an
1167 * invocation to the monitor to update the security state of the memory,
1168 * to that of the SPMC.
1169 * - The action MAP_ACTION_COMMIT_UNPROTECT extends the MAP_ACTION_COMMIT
1170 * with a call into the monitor, to reset the security state of memory
1171 * that has priorly been mapped with the MAP_ACTION_CHECK_PROTECT action.
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001172 * vm_ptable_defrag should always be called after a series of page table
1173 * updates, whether they succeed or fail.
Jose Marinho09b1db82019-08-08 09:16:59 +01001174 *
J-Alvescf6253e2024-01-03 13:48:48 +00001175 * If all goes well, returns FFA_SUCCESS_32; or FFA_ERROR, with following
1176 * error codes:
1177 * - FFA_INVALID_PARAMETERS: invalid range of memory.
1178 * - FFA_DENIED:
1179 *
Jose Marinho09b1db82019-08-08 09:16:59 +01001180 * made to memory mappings.
1181 */
J-Alvescf6253e2024-01-03 13:48:48 +00001182struct ffa_value ffa_region_group_identity_map(
Andrew Walbranf4b51af2020-02-03 14:44:54 +00001183 struct vm_locked vm_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01001184 struct ffa_memory_region_constituent **fragments,
1185 const uint32_t *fragment_constituent_counts, uint32_t fragment_count,
J-Alvescf6253e2024-01-03 13:48:48 +00001186 uint32_t mode, struct mpool *ppool, enum ffa_map_action action,
1187 bool *memory_protected)
Jose Marinho09b1db82019-08-08 09:16:59 +01001188{
Andrew Walbranca808b12020-05-15 17:22:28 +01001189 uint32_t i;
1190 uint32_t j;
J-Alvescf6253e2024-01-03 13:48:48 +00001191 struct ffa_value ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01001192
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001193 if (vm_locked.vm->el0_partition) {
1194 mode |= MM_MODE_USER | MM_MODE_NG;
1195 }
1196
Andrew Walbranca808b12020-05-15 17:22:28 +01001197 /* Iterate over the memory region constituents within each fragment. */
1198 for (i = 0; i < fragment_count; ++i) {
1199 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
J-Alves063ad832023-10-03 18:05:40 +01001200 struct ffa_memory_region_constituent *constituent =
1201 &fragments[i][j];
1202 size_t size = constituent->page_count * PAGE_SIZE;
Andrew Walbranca808b12020-05-15 17:22:28 +01001203 paddr_t pa_begin =
J-Alves063ad832023-10-03 18:05:40 +01001204 pa_from_ipa(ipa_init(constituent->address));
Andrew Walbranca808b12020-05-15 17:22:28 +01001205 paddr_t pa_end = pa_add(pa_begin, size);
Jens Wiklander4f1880c2022-10-19 17:00:14 +02001206 uint32_t pa_bits =
1207 arch_mm_get_pa_bits(arch_mm_get_pa_range());
Federico Recanati4fd065d2021-12-13 20:06:23 +01001208
1209 /*
1210 * Ensure the requested region falls into system's PA
1211 * range.
1212 */
Jens Wiklander4f1880c2022-10-19 17:00:14 +02001213 if (((pa_addr(pa_begin) >> pa_bits) > 0) ||
1214 ((pa_addr(pa_end) >> pa_bits) > 0)) {
Federico Recanati4fd065d2021-12-13 20:06:23 +01001215 dlog_error("Region is outside of PA Range\n");
J-Alvescf6253e2024-01-03 13:48:48 +00001216 return ffa_error(FFA_INVALID_PARAMETERS);
Federico Recanati4fd065d2021-12-13 20:06:23 +01001217 }
Andrew Walbranca808b12020-05-15 17:22:28 +01001218
J-Alvescf6253e2024-01-03 13:48:48 +00001219 if (action <= MAP_ACTION_CHECK_PROTECT) {
1220 ret = ffa_region_group_check_actions(
1221 vm_locked, pa_begin, pa_end, ppool,
1222 mode, action, memory_protected);
J-Alves063ad832023-10-03 18:05:40 +01001223
1224 if (ret.func == FFA_ERROR_32 &&
1225 ffa_error_code(ret) == FFA_DENIED) {
1226 if (memory_protected != NULL) {
1227 assert(!*memory_protected);
1228 }
1229
1230 ffa_region_group_fragments_revert_protect(
1231 fragments,
1232 fragment_constituent_counts,
1233 i + 1, constituent);
1234 break;
1235 }
J-Alvescf6253e2024-01-03 13:48:48 +00001236 } else if (action >= MAP_ACTION_COMMIT &&
1237 action < MAP_ACTION_MAX) {
1238 ffa_region_group_commit_actions(
1239 vm_locked, pa_begin, pa_end, ppool,
1240 mode, action);
1241 ret = (struct ffa_value){
1242 .func = FFA_SUCCESS_32};
1243 } else {
1244 panic("%s: Unknown ffa_map_action.\n",
1245 __func__);
Andrew Walbranca808b12020-05-15 17:22:28 +01001246 }
Jose Marinho09b1db82019-08-08 09:16:59 +01001247 }
1248 }
1249
J-Alvescf6253e2024-01-03 13:48:48 +00001250 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01001251}
1252
1253/**
1254 * Clears a region of physical memory by overwriting it with zeros. The data is
1255 * flushed from the cache so the memory has been cleared across the system.
1256 */
J-Alves7db32002021-12-14 14:44:50 +00001257static bool clear_memory(paddr_t begin, paddr_t end, struct mpool *ppool,
1258 uint32_t extra_mode_attributes)
Jose Marinho09b1db82019-08-08 09:16:59 +01001259{
1260 /*
Fuad Tabbaed294af2019-12-20 10:43:01 +00001261 * TODO: change this to a CPU local single page window rather than a
Jose Marinho09b1db82019-08-08 09:16:59 +01001262 * global mapping of the whole range. Such an approach will limit
1263 * the changes to stage-1 tables and will allow only local
1264 * invalidation.
1265 */
1266 bool ret;
1267 struct mm_stage1_locked stage1_locked = mm_lock_stage1();
J-Alves7db32002021-12-14 14:44:50 +00001268 void *ptr = mm_identity_map(stage1_locked, begin, end,
1269 MM_MODE_W | (extra_mode_attributes &
1270 plat_ffa_other_world_mode()),
1271 ppool);
Jose Marinho09b1db82019-08-08 09:16:59 +01001272 size_t size = pa_difference(begin, end);
1273
1274 if (!ptr) {
Jose Marinho09b1db82019-08-08 09:16:59 +01001275 goto fail;
1276 }
1277
1278 memset_s(ptr, size, 0, size);
1279 arch_mm_flush_dcache(ptr, size);
1280 mm_unmap(stage1_locked, begin, end, ppool);
1281
1282 ret = true;
1283 goto out;
1284
1285fail:
1286 ret = false;
1287
1288out:
1289 mm_unlock_stage1(&stage1_locked);
1290
1291 return ret;
1292}
1293
1294/**
1295 * Clears a region of physical memory by overwriting it with zeros. The data is
1296 * flushed from the cache so the memory has been cleared across the system.
1297 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001298static bool ffa_clear_memory_constituents(
J-Alves7db32002021-12-14 14:44:50 +00001299 uint32_t security_state_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +01001300 struct ffa_memory_region_constituent **fragments,
1301 const uint32_t *fragment_constituent_counts, uint32_t fragment_count,
1302 struct mpool *page_pool)
Jose Marinho09b1db82019-08-08 09:16:59 +01001303{
1304 struct mpool local_page_pool;
Andrew Walbranca808b12020-05-15 17:22:28 +01001305 uint32_t i;
Jose Marinho09b1db82019-08-08 09:16:59 +01001306 bool ret = false;
1307
1308 /*
1309 * Create a local pool so any freed memory can't be used by another
1310 * thread. This is to ensure each constituent that is mapped can be
1311 * unmapped again afterwards.
1312 */
Andrew Walbran475c1452020-02-07 13:22:22 +00001313 mpool_init_with_fallback(&local_page_pool, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +01001314
Andrew Walbranca808b12020-05-15 17:22:28 +01001315 /* Iterate over the memory region constituents within each fragment. */
1316 for (i = 0; i < fragment_count; ++i) {
1317 uint32_t j;
Jose Marinho09b1db82019-08-08 09:16:59 +01001318
J-Alves8457f932023-10-11 16:41:45 +01001319 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001320 size_t size = fragments[i][j].page_count * PAGE_SIZE;
1321 paddr_t begin =
1322 pa_from_ipa(ipa_init(fragments[i][j].address));
1323 paddr_t end = pa_add(begin, size);
1324
J-Alves7db32002021-12-14 14:44:50 +00001325 if (!clear_memory(begin, end, &local_page_pool,
1326 security_state_mode)) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001327 /*
1328 * api_clear_memory will defrag on failure, so
1329 * no need to do it here.
1330 */
1331 goto out;
1332 }
Jose Marinho09b1db82019-08-08 09:16:59 +01001333 }
1334 }
1335
Jose Marinho09b1db82019-08-08 09:16:59 +01001336 ret = true;
1337
1338out:
1339 mpool_fini(&local_page_pool);
1340 return ret;
1341}
1342
J-Alves5952d942022-12-22 16:03:00 +00001343static bool is_memory_range_within(ipaddr_t begin, ipaddr_t end,
1344 ipaddr_t in_begin, ipaddr_t in_end)
1345{
1346 return (ipa_addr(begin) >= ipa_addr(in_begin) &&
1347 ipa_addr(begin) < ipa_addr(in_end)) ||
1348 (ipa_addr(end) <= ipa_addr(in_end) &&
1349 ipa_addr(end) > ipa_addr(in_begin));
1350}
1351
1352/**
1353 * Receives a memory range and looks for overlaps with the remainder
1354 * constituents of the memory share/lend/donate operation. Assumes they are
1355 * passed in order to avoid having to loop over all the elements at each call.
1356 * The function only compares the received memory ranges with those that follow
1357 * within the same fragment, and subsequent fragments from the same operation.
1358 */
1359static bool ffa_memory_check_overlap(
1360 struct ffa_memory_region_constituent **fragments,
1361 const uint32_t *fragment_constituent_counts,
1362 const uint32_t fragment_count, const uint32_t current_fragment,
1363 const uint32_t current_constituent)
1364{
1365 uint32_t i = current_fragment;
1366 uint32_t j = current_constituent;
1367 ipaddr_t current_begin = ipa_init(fragments[i][j].address);
1368 const uint32_t current_page_count = fragments[i][j].page_count;
1369 size_t current_size = current_page_count * PAGE_SIZE;
1370 ipaddr_t current_end = ipa_add(current_begin, current_size - 1);
1371
1372 if (current_size == 0 ||
1373 current_size > UINT64_MAX - ipa_addr(current_begin)) {
Karl Meakine8937d92024-03-19 16:04:25 +00001374 dlog_verbose("Invalid page count. Addr: %zx page_count: %x\n",
1375 current_begin.ipa, current_page_count);
J-Alves5952d942022-12-22 16:03:00 +00001376 return false;
1377 }
1378
1379 for (; i < fragment_count; i++) {
1380 j = (i == current_fragment) ? j + 1 : 0;
1381
1382 for (; j < fragment_constituent_counts[i]; j++) {
1383 ipaddr_t begin = ipa_init(fragments[i][j].address);
1384 const uint32_t page_count = fragments[i][j].page_count;
1385 size_t size = page_count * PAGE_SIZE;
1386 ipaddr_t end = ipa_add(begin, size - 1);
1387
1388 if (size == 0 || size > UINT64_MAX - ipa_addr(begin)) {
1389 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00001390 "Invalid page count. Addr: %lx "
J-Alves5952d942022-12-22 16:03:00 +00001391 "page_count: %x\n",
Karl Meakine8937d92024-03-19 16:04:25 +00001392 begin.ipa, page_count);
J-Alves5952d942022-12-22 16:03:00 +00001393 return false;
1394 }
1395
1396 /*
1397 * Check if current ranges is within begin and end, as
1398 * well as the reverse. This should help optimize the
1399 * loop, and reduce the number of iterations.
1400 */
1401 if (is_memory_range_within(begin, end, current_begin,
1402 current_end) ||
1403 is_memory_range_within(current_begin, current_end,
1404 begin, end)) {
1405 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00001406 "Overlapping memory ranges: %#lx - "
1407 "%#lx with %#lx - %#lx\n",
J-Alves5952d942022-12-22 16:03:00 +00001408 ipa_addr(begin), ipa_addr(end),
1409 ipa_addr(current_begin),
1410 ipa_addr(current_end));
1411 return true;
1412 }
1413 }
1414 }
1415
1416 return false;
1417}
1418
Jose Marinho09b1db82019-08-08 09:16:59 +01001419/**
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001420 * Validates and prepares memory to be sent from the calling VM to another.
Jose Marinho09b1db82019-08-08 09:16:59 +01001421 *
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001422 * This function requires the calling context to hold the <from> VM lock.
Jose Marinho09b1db82019-08-08 09:16:59 +01001423 *
1424 * Returns:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001425 * In case of error, one of the following values is returned:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001426 * 1) FFA_INVALID_PARAMETERS - The endpoint provided parameters were
Jose Marinho09b1db82019-08-08 09:16:59 +01001427 * erroneous;
Andrew Walbranf07f04d2020-05-01 18:09:00 +01001428 * 2) FFA_NO_MEMORY - Hafnium did not have sufficient memory to complete the
1429 * request.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001430 * 3) FFA_DENIED - The sender doesn't have sufficient access to send the
Andrew Walbrana65a1322020-04-06 19:32:32 +01001431 * memory with the given permissions.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001432 * Success is indicated by FFA_SUCCESS.
Jose Marinho09b1db82019-08-08 09:16:59 +01001433 */
Daniel Boulbya76fd912024-02-22 14:22:15 +00001434static struct ffa_value ffa_send_check_update(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001435 struct vm_locked from_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01001436 struct ffa_memory_region_constituent **fragments,
1437 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
J-Alves8f11cde2022-12-21 16:18:22 +00001438 uint32_t composite_total_page_count, uint32_t share_func,
Daniel Boulbya76fd912024-02-22 14:22:15 +00001439 struct ffa_memory_region *memory_region, struct mpool *page_pool,
1440 uint32_t *orig_from_mode_ret, bool *memory_protected)
Jose Marinho09b1db82019-08-08 09:16:59 +01001441{
Andrew Walbranca808b12020-05-15 17:22:28 +01001442 uint32_t i;
J-Alves8f11cde2022-12-21 16:18:22 +00001443 uint32_t j;
Jose Marinho09b1db82019-08-08 09:16:59 +01001444 uint32_t orig_from_mode;
J-Alves460d36c2023-10-12 17:02:15 +01001445 uint32_t clean_mode;
Jose Marinho09b1db82019-08-08 09:16:59 +01001446 uint32_t from_mode;
Jose Marinho09b1db82019-08-08 09:16:59 +01001447 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001448 struct ffa_value ret;
J-Alves8f11cde2022-12-21 16:18:22 +00001449 uint32_t constituents_total_page_count = 0;
J-Alves460d36c2023-10-12 17:02:15 +01001450 enum ffa_map_action map_action = MAP_ACTION_CHECK;
Daniel Boulbya76fd912024-02-22 14:22:15 +00001451 bool clear = memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR;
Jose Marinho09b1db82019-08-08 09:16:59 +01001452
1453 /*
Andrew Walbrana65a1322020-04-06 19:32:32 +01001454 * Make sure constituents are properly aligned to a 64-bit boundary. If
1455 * not we would get alignment faults trying to read (64-bit) values.
Jose Marinho09b1db82019-08-08 09:16:59 +01001456 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001457 for (i = 0; i < fragment_count; ++i) {
1458 if (!is_aligned(fragments[i], 8)) {
1459 dlog_verbose("Constituents not aligned.\n");
1460 return ffa_error(FFA_INVALID_PARAMETERS);
1461 }
J-Alves8f11cde2022-12-21 16:18:22 +00001462 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
1463 constituents_total_page_count +=
1464 fragments[i][j].page_count;
J-Alves5952d942022-12-22 16:03:00 +00001465 if (ffa_memory_check_overlap(
1466 fragments, fragment_constituent_counts,
1467 fragment_count, i, j)) {
1468 return ffa_error(FFA_INVALID_PARAMETERS);
1469 }
J-Alves8f11cde2022-12-21 16:18:22 +00001470 }
1471 }
1472
1473 if (constituents_total_page_count != composite_total_page_count) {
1474 dlog_verbose(
1475 "Composite page count differs from calculated page "
1476 "count from constituents.\n");
1477 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho09b1db82019-08-08 09:16:59 +01001478 }
1479
1480 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001481 * Check if the state transition is lawful for the sender, ensure that
1482 * all constituents of a memory region being shared are at the same
1483 * state.
Jose Marinho09b1db82019-08-08 09:16:59 +01001484 */
J-Alves460d36c2023-10-12 17:02:15 +01001485 ret = ffa_send_check_transition(
Daniel Boulbya76fd912024-02-22 14:22:15 +00001486 from_locked, share_func, memory_region, &orig_from_mode,
1487 fragments, fragment_constituent_counts, fragment_count,
Daniel Boulby4b846eb2024-05-23 17:32:23 +01001488 &from_mode, &map_action, clear);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001489 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001490 dlog_verbose("Invalid transition for send.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +01001491 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01001492 }
1493
Andrew Walbran37c574e2020-06-03 11:45:46 +01001494 if (orig_from_mode_ret != NULL) {
1495 *orig_from_mode_ret = orig_from_mode;
1496 }
1497
Jose Marinho09b1db82019-08-08 09:16:59 +01001498 /*
1499 * Create a local pool so any freed memory can't be used by another
1500 * thread. This is to ensure the original mapping can be restored if the
1501 * clear fails.
1502 */
Andrew Walbran475c1452020-02-07 13:22:22 +00001503 mpool_init_with_fallback(&local_page_pool, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +01001504
1505 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001506 * First reserve all required memory for the new page table entries
1507 * without committing, to make sure the entire operation will succeed
1508 * without exhausting the page pool.
J-Alves460d36c2023-10-12 17:02:15 +01001509 * Provide the map_action as populated by 'ffa_send_check_transition'.
1510 * It may request memory to be protected.
Jose Marinho09b1db82019-08-08 09:16:59 +01001511 */
J-Alvescf6253e2024-01-03 13:48:48 +00001512 ret = ffa_region_group_identity_map(
1513 from_locked, fragments, fragment_constituent_counts,
J-Alves460d36c2023-10-12 17:02:15 +01001514 fragment_count, from_mode, page_pool, map_action,
1515 memory_protected);
J-Alvescf6253e2024-01-03 13:48:48 +00001516 if (ret.func == FFA_ERROR_32) {
Jose Marinho09b1db82019-08-08 09:16:59 +01001517 goto out;
1518 }
1519
1520 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001521 * Update the mapping for the sender. This won't allocate because the
1522 * transaction was already prepared above, but may free pages in the
1523 * case that a whole block is being unmapped that was previously
1524 * partially mapped.
Jose Marinho09b1db82019-08-08 09:16:59 +01001525 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001526 CHECK(ffa_region_group_identity_map(
J-Alvescf6253e2024-01-03 13:48:48 +00001527 from_locked, fragments, fragment_constituent_counts,
1528 fragment_count, from_mode, &local_page_pool,
1529 MAP_ACTION_COMMIT, NULL)
1530 .func == FFA_SUCCESS_32);
Jose Marinho09b1db82019-08-08 09:16:59 +01001531
J-Alves460d36c2023-10-12 17:02:15 +01001532 /*
1533 * If memory has been protected, it is now part of the secure PAS
1534 * (happens for lend/donate from NWd to SWd), and the `orig_from_mode`
1535 * should have the MM_MODE_NS set, as such mask it in `clean_mode` for
1536 * SPM's S1 translation.
1537 * In case memory hasn't been protected, and it is in the non-secure
1538 * PAS (e.g. memory share from NWd to SWd), as such the SPM needs to
1539 * perform a non-secure memory access. In such case `clean_mode` takes
1540 * the same mode as `orig_from_mode`.
1541 */
1542 clean_mode = (memory_protected != NULL && *memory_protected)
1543 ? orig_from_mode & ~plat_ffa_other_world_mode()
1544 : orig_from_mode;
1545
Jose Marinho09b1db82019-08-08 09:16:59 +01001546 /* Clear the memory so no VM or device can see the previous contents. */
J-Alves460d36c2023-10-12 17:02:15 +01001547 if (clear && !ffa_clear_memory_constituents(
1548 clean_mode, fragments, fragment_constituent_counts,
1549 fragment_count, page_pool)) {
1550 map_action = (memory_protected != NULL && *memory_protected)
1551 ? MAP_ACTION_COMMIT_UNPROTECT
1552 : MAP_ACTION_COMMIT;
1553
Jose Marinho09b1db82019-08-08 09:16:59 +01001554 /*
1555 * On failure, roll back by returning memory to the sender. This
1556 * may allocate pages which were previously freed into
1557 * `local_page_pool` by the call above, but will never allocate
1558 * more pages than that so can never fail.
1559 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001560 CHECK(ffa_region_group_identity_map(
J-Alvescf6253e2024-01-03 13:48:48 +00001561 from_locked, fragments,
1562 fragment_constituent_counts, fragment_count,
1563 orig_from_mode, &local_page_pool,
1564 MAP_ACTION_COMMIT, NULL)
1565 .func == FFA_SUCCESS_32);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001566 ret = ffa_error(FFA_NO_MEMORY);
Jose Marinho09b1db82019-08-08 09:16:59 +01001567 goto out;
1568 }
1569
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001570 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001571
1572out:
1573 mpool_fini(&local_page_pool);
1574
1575 /*
1576 * Tidy up the page table by reclaiming failed mappings (if there was an
1577 * error) or merging entries into blocks where possible (on success).
1578 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001579 vm_ptable_defrag(from_locked, page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001580
1581 return ret;
1582}
1583
1584/**
1585 * Validates and maps memory shared from one VM to another.
1586 *
1587 * This function requires the calling context to hold the <to> lock.
1588 *
1589 * Returns:
1590 * In case of error, one of the following values is returned:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001591 * 1) FFA_INVALID_PARAMETERS - The endpoint provided parameters were
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001592 * erroneous;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001593 * 2) FFA_NO_MEMORY - Hafnium did not have sufficient memory to complete
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001594 * the request.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001595 * Success is indicated by FFA_SUCCESS.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001596 */
J-Alvesb5084cf2022-07-06 14:20:12 +01001597struct ffa_value ffa_retrieve_check_update(
J-Alves26483382023-04-20 12:01:49 +01001598 struct vm_locked to_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01001599 struct ffa_memory_region_constituent **fragments,
1600 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
J-Alves26483382023-04-20 12:01:49 +01001601 uint32_t sender_orig_mode, uint32_t share_func, bool clear,
J-Alves460d36c2023-10-12 17:02:15 +01001602 struct mpool *page_pool, uint32_t *response_mode, bool memory_protected)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001603{
Andrew Walbranca808b12020-05-15 17:22:28 +01001604 uint32_t i;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001605 uint32_t to_mode;
1606 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001607 struct ffa_value ret;
J-Alvesfd206052023-05-22 16:45:00 +01001608 enum ffa_map_action map_action = MAP_ACTION_COMMIT;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001609
1610 /*
Andrew Walbranca808b12020-05-15 17:22:28 +01001611 * Make sure constituents are properly aligned to a 64-bit boundary. If
1612 * not we would get alignment faults trying to read (64-bit) values.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001613 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001614 for (i = 0; i < fragment_count; ++i) {
1615 if (!is_aligned(fragments[i], 8)) {
J-Alvesb5084cf2022-07-06 14:20:12 +01001616 dlog_verbose("Fragment not properly aligned.\n");
Andrew Walbranca808b12020-05-15 17:22:28 +01001617 return ffa_error(FFA_INVALID_PARAMETERS);
1618 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001619 }
1620
1621 /*
Daniel Boulby4b846eb2024-05-23 17:32:23 +01001622 * Ensure the sender has write permissions if the memory needs to be
1623 * cleared.
1624 */
1625 if ((sender_orig_mode & MM_MODE_W) == 0 && clear == true) {
1626 dlog_verbose(
1627 "Cannot zero memory when the sender does not have "
1628 "write access\n");
1629 return ffa_error(FFA_DENIED);
1630 }
1631
1632 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001633 * Check if the state transition is lawful for the recipient, and ensure
1634 * that all constituents of the memory region being retrieved are at the
1635 * same state.
1636 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001637 ret = ffa_retrieve_check_transition(
1638 to_locked, share_func, fragments, fragment_constituent_counts,
J-Alvesfd206052023-05-22 16:45:00 +01001639 fragment_count, sender_orig_mode, &to_mode, memory_protected,
1640 &map_action);
J-Alves460d36c2023-10-12 17:02:15 +01001641
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001642 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001643 dlog_verbose("Invalid transition for retrieve.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +01001644 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001645 }
1646
1647 /*
J-Alves69cdfd92024-04-26 11:40:59 +01001648 * Create a local pool so any freed memory can't be used by
1649 * another thread. This is to ensure the original mapping can be
1650 * restored if the clear fails.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001651 */
1652 mpool_init_with_fallback(&local_page_pool, page_pool);
1653
1654 /*
J-Alves69cdfd92024-04-26 11:40:59 +01001655 * Memory retrieves from the NWd VMs don't require update to S2 PTs on
1656 * retrieve request.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001657 */
J-Alves69cdfd92024-04-26 11:40:59 +01001658 if (map_action != MAP_ACTION_NONE) {
1659 /*
1660 * First reserve all required memory for the new page table
1661 * entries in the recipient page tables without committing, to
1662 * make sure the entire operation will succeed without
1663 * exhausting the page pool.
1664 */
1665 ret = ffa_region_group_identity_map(
1666 to_locked, fragments, fragment_constituent_counts,
1667 fragment_count, to_mode, page_pool, MAP_ACTION_CHECK,
1668 NULL);
1669 if (ret.func == FFA_ERROR_32) {
1670 /* TODO: partial defrag of failed range. */
1671 goto out;
1672 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001673 }
1674
1675 /* Clear the memory so no VM or device can see the previous contents. */
J-Alves7db32002021-12-14 14:44:50 +00001676 if (clear &&
J-Alves26483382023-04-20 12:01:49 +01001677 !ffa_clear_memory_constituents(sender_orig_mode, fragments,
1678 fragment_constituent_counts,
1679 fragment_count, page_pool)) {
J-Alvesb5084cf2022-07-06 14:20:12 +01001680 dlog_verbose("Couldn't clear constituents.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001681 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001682 goto out;
1683 }
1684
J-Alves69cdfd92024-04-26 11:40:59 +01001685 if (map_action != MAP_ACTION_NONE) {
1686 /*
1687 * Complete the transfer by mapping the memory into the
1688 * recipient. This won't allocate because the transaction was
1689 * already prepared above, so it doesn't need to use the
1690 * `local_page_pool`.
1691 */
1692 CHECK(ffa_region_group_identity_map(to_locked, fragments,
1693 fragment_constituent_counts,
1694 fragment_count, to_mode,
1695 page_pool, map_action, NULL)
1696 .func == FFA_SUCCESS_32);
Jose Marinho09b1db82019-08-08 09:16:59 +01001697
J-Alves69cdfd92024-04-26 11:40:59 +01001698 /*
1699 * Return the mode used in mapping the memory in retriever's PT.
1700 */
1701 if (response_mode != NULL) {
1702 *response_mode = to_mode;
1703 }
J-Alves460d36c2023-10-12 17:02:15 +01001704 }
1705
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001706 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Jose Marinho09b1db82019-08-08 09:16:59 +01001707
1708out:
1709 mpool_fini(&local_page_pool);
1710
1711 /*
Andrew Walbranf07f04d2020-05-01 18:09:00 +01001712 * Tidy up the page table by reclaiming failed mappings (if there was an
1713 * error) or merging entries into blocks where possible (on success).
Jose Marinho09b1db82019-08-08 09:16:59 +01001714 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001715 vm_ptable_defrag(to_locked, page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001716
1717 return ret;
1718}
1719
Andrew Walbran996d1d12020-05-27 14:08:43 +01001720static struct ffa_value ffa_relinquish_check_update(
J-Alves26483382023-04-20 12:01:49 +01001721 struct vm_locked from_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01001722 struct ffa_memory_region_constituent **fragments,
1723 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
J-Alves69cdfd92024-04-26 11:40:59 +01001724 uint32_t sender_orig_mode, struct mpool *page_pool, bool clear)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001725{
1726 uint32_t orig_from_mode;
J-Alves69cdfd92024-04-26 11:40:59 +01001727 uint32_t clearing_mode;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001728 uint32_t from_mode;
1729 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001730 struct ffa_value ret;
J-Alves69cdfd92024-04-26 11:40:59 +01001731 enum ffa_map_action map_action;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001732
Andrew Walbranca808b12020-05-15 17:22:28 +01001733 ret = ffa_relinquish_check_transition(
1734 from_locked, &orig_from_mode, fragments,
J-Alves69cdfd92024-04-26 11:40:59 +01001735 fragment_constituent_counts, fragment_count, &from_mode,
1736 &map_action);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001737 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001738 dlog_verbose("Invalid transition for relinquish.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +01001739 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001740 }
1741
1742 /*
1743 * Create a local pool so any freed memory can't be used by another
1744 * thread. This is to ensure the original mapping can be restored if the
1745 * clear fails.
1746 */
1747 mpool_init_with_fallback(&local_page_pool, page_pool);
1748
J-Alves69cdfd92024-04-26 11:40:59 +01001749 if (map_action != MAP_ACTION_NONE) {
1750 clearing_mode = orig_from_mode;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001751
J-Alves69cdfd92024-04-26 11:40:59 +01001752 /*
1753 * First reserve all required memory for the new page table
1754 * entries without committing, to make sure the entire operation
1755 * will succeed without exhausting the page pool.
1756 */
1757 ret = ffa_region_group_identity_map(
1758 from_locked, fragments, fragment_constituent_counts,
1759 fragment_count, from_mode, page_pool, MAP_ACTION_CHECK,
1760 NULL);
1761 if (ret.func == FFA_ERROR_32) {
1762 goto out;
1763 }
1764
1765 /*
1766 * Update the mapping for the sender. This won't allocate
1767 * because the transaction was already prepared above, but may
1768 * free pages in the case that a whole block is being unmapped
1769 * that was previously partially mapped.
1770 */
1771 CHECK(ffa_region_group_identity_map(from_locked, fragments,
1772 fragment_constituent_counts,
1773 fragment_count, from_mode,
1774 &local_page_pool,
1775 MAP_ACTION_COMMIT, NULL)
1776 .func == FFA_SUCCESS_32);
1777 } else {
1778 /*
1779 * If the `map_action` is set to `MAP_ACTION_NONE`, S2 PTs
1780 * were not updated on retrieve/relinquish. These were updating
1781 * only the `share_state` structures. As such, use the sender's
1782 * original mode.
1783 */
1784 clearing_mode = sender_orig_mode;
1785 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001786
1787 /* Clear the memory so no VM or device can see the previous contents. */
J-Alves7db32002021-12-14 14:44:50 +00001788 if (clear &&
J-Alves69cdfd92024-04-26 11:40:59 +01001789 !ffa_clear_memory_constituents(clearing_mode, fragments,
J-Alves26483382023-04-20 12:01:49 +01001790 fragment_constituent_counts,
1791 fragment_count, page_pool)) {
J-Alves69cdfd92024-04-26 11:40:59 +01001792 if (map_action != MAP_ACTION_NONE) {
1793 /*
1794 * On failure, roll back by returning memory to the
1795 * sender. This may allocate pages which were previously
1796 * freed into `local_page_pool` by the call above, but
1797 * will never allocate more pages than that so can never
1798 * fail.
1799 */
1800 CHECK(ffa_region_group_identity_map(
1801 from_locked, fragments,
1802 fragment_constituent_counts,
1803 fragment_count, orig_from_mode,
1804 &local_page_pool, MAP_ACTION_COMMIT, NULL)
1805 .func == FFA_SUCCESS_32);
1806 }
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001807 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001808 goto out;
1809 }
1810
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001811 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001812
1813out:
1814 mpool_fini(&local_page_pool);
1815
1816 /*
1817 * Tidy up the page table by reclaiming failed mappings (if there was an
1818 * error) or merging entries into blocks where possible (on success).
1819 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001820 vm_ptable_defrag(from_locked, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +01001821
1822 return ret;
1823}
1824
1825/**
Andrew Walbranca808b12020-05-15 17:22:28 +01001826 * Complete a memory sending operation by checking that it is valid, updating
1827 * the sender page table, and then either marking the share state as having
1828 * completed sending (on success) or freeing it (on failure).
1829 *
1830 * Returns FFA_SUCCESS with the handle encoded, or the relevant FFA_ERROR.
1831 */
J-Alvesfdd29272022-07-19 13:16:31 +01001832struct ffa_value ffa_memory_send_complete(
Andrew Walbranca808b12020-05-15 17:22:28 +01001833 struct vm_locked from_locked, struct share_states_locked share_states,
Andrew Walbran37c574e2020-06-03 11:45:46 +01001834 struct ffa_memory_share_state *share_state, struct mpool *page_pool,
1835 uint32_t *orig_from_mode_ret)
Andrew Walbranca808b12020-05-15 17:22:28 +01001836{
1837 struct ffa_memory_region *memory_region = share_state->memory_region;
J-Alves8f11cde2022-12-21 16:18:22 +00001838 struct ffa_composite_memory_region *composite;
Andrew Walbranca808b12020-05-15 17:22:28 +01001839 struct ffa_value ret;
1840
1841 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +00001842 assert(share_states.share_states != NULL);
J-Alves8f11cde2022-12-21 16:18:22 +00001843 assert(memory_region != NULL);
1844 composite = ffa_memory_region_get_composite(memory_region, 0);
1845 assert(composite != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +01001846
1847 /* Check that state is valid in sender page table and update. */
1848 ret = ffa_send_check_update(
1849 from_locked, share_state->fragments,
1850 share_state->fragment_constituent_counts,
J-Alves8f11cde2022-12-21 16:18:22 +00001851 share_state->fragment_count, composite->page_count,
Daniel Boulbya76fd912024-02-22 14:22:15 +00001852 share_state->share_func, memory_region, page_pool,
J-Alves460d36c2023-10-12 17:02:15 +01001853 orig_from_mode_ret, &share_state->memory_protected);
Andrew Walbranca808b12020-05-15 17:22:28 +01001854 if (ret.func != FFA_SUCCESS_32) {
1855 /*
1856 * Free share state, it failed to send so it can't be retrieved.
1857 */
Karl Meakin4cec5e82023-06-30 16:30:22 +01001858 dlog_verbose("%s: failed to send check update: %s(%s)\n",
1859 __func__, ffa_func_name(ret.func),
1860 ffa_error_name(ffa_error_code(ret)));
Andrew Walbranca808b12020-05-15 17:22:28 +01001861 share_state_free(share_states, share_state, page_pool);
1862 return ret;
1863 }
1864
1865 share_state->sending_complete = true;
Karl Meakin4cec5e82023-06-30 16:30:22 +01001866 dlog_verbose("%s: marked sending complete.\n", __func__);
Andrew Walbranca808b12020-05-15 17:22:28 +01001867
J-Alvesee68c542020-10-29 17:48:20 +00001868 return ffa_mem_success(share_state->memory_region->handle);
Andrew Walbranca808b12020-05-15 17:22:28 +01001869}
1870
1871/**
Daniel Boulby9764ff62024-01-30 17:47:39 +00001872 * Check that the memory attributes match Hafnium expectations.
1873 * Cacheability:
1874 * - Normal Memory as `FFA_MEMORY_CACHE_WRITE_BACK`.
1875 * - Device memory as `FFA_MEMORY_DEV_NGNRNE`.
1876 *
1877 * Shareability:
1878 * - Inner Shareable.
Federico Recanatia98603a2021-12-20 18:04:03 +01001879 */
1880static struct ffa_value ffa_memory_attributes_validate(
J-Alves7a99d0d2023-02-08 13:49:48 +00001881 ffa_memory_attributes_t attributes)
Federico Recanatia98603a2021-12-20 18:04:03 +01001882{
1883 enum ffa_memory_type memory_type;
1884 enum ffa_memory_cacheability cacheability;
1885 enum ffa_memory_shareability shareability;
1886
Karl Meakin84710f32023-10-12 15:14:49 +01001887 memory_type = attributes.type;
Daniel Boulby9764ff62024-01-30 17:47:39 +00001888 cacheability = attributes.cacheability;
1889 if (memory_type == FFA_MEMORY_NORMAL_MEM &&
1890 cacheability != FFA_MEMORY_CACHE_WRITE_BACK) {
1891 dlog_verbose(
1892 "Normal Memory: Invalid cacheability %s, "
1893 "expected %s.\n",
1894 ffa_memory_cacheability_name(cacheability),
1895 ffa_memory_cacheability_name(
1896 FFA_MEMORY_CACHE_WRITE_BACK));
Federico Recanati3d953f32022-02-17 09:31:29 +01001897 return ffa_error(FFA_DENIED);
Federico Recanatia98603a2021-12-20 18:04:03 +01001898 }
Daniel Boulby9764ff62024-01-30 17:47:39 +00001899 if (memory_type == FFA_MEMORY_DEVICE_MEM &&
1900 cacheability != FFA_MEMORY_DEV_NGNRNE) {
1901 dlog_verbose(
1902 "Device Memory: Invalid cacheability %s, "
1903 "expected %s.\n",
1904 ffa_device_memory_cacheability_name(cacheability),
1905 ffa_device_memory_cacheability_name(
1906 FFA_MEMORY_DEV_NGNRNE));
Federico Recanati3d953f32022-02-17 09:31:29 +01001907 return ffa_error(FFA_DENIED);
Federico Recanatia98603a2021-12-20 18:04:03 +01001908 }
1909
Karl Meakin84710f32023-10-12 15:14:49 +01001910 shareability = attributes.shareability;
Federico Recanatia98603a2021-12-20 18:04:03 +01001911 if (shareability != FFA_MEMORY_INNER_SHAREABLE) {
Karl Meakinf98b2aa2023-10-12 16:09:59 +01001912 dlog_verbose("Invalid shareability %s, expected %s.\n",
1913 ffa_memory_shareability_name(shareability),
1914 ffa_memory_shareability_name(
1915 FFA_MEMORY_INNER_SHAREABLE));
Federico Recanati3d953f32022-02-17 09:31:29 +01001916 return ffa_error(FFA_DENIED);
Federico Recanatia98603a2021-12-20 18:04:03 +01001917 }
1918
1919 return (struct ffa_value){.func = FFA_SUCCESS_32};
1920}
1921
1922/**
Andrew Walbrana65a1322020-04-06 19:32:32 +01001923 * Check that the given `memory_region` represents a valid memory send request
1924 * of the given `share_func` type, return the clear flag and permissions via the
1925 * respective output parameters, and update the permissions if necessary.
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001926 *
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001927 * Returns FFA_SUCCESS if the request was valid, or the relevant FFA_ERROR if
Andrew Walbrana65a1322020-04-06 19:32:32 +01001928 * not.
1929 */
J-Alves66652252022-07-06 09:49:51 +01001930struct ffa_value ffa_memory_send_validate(
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001931 struct vm_locked from_locked, struct ffa_memory_region *memory_region,
1932 uint32_t memory_share_length, uint32_t fragment_length,
J-Alves363f5722022-04-25 17:37:37 +01001933 uint32_t share_func)
Andrew Walbrana65a1322020-04-06 19:32:32 +01001934{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001935 struct ffa_composite_memory_region *composite;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001936 struct ffa_memory_access *receiver =
1937 ffa_memory_region_get_receiver(memory_region, 0);
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001938 uint64_t receivers_end;
1939 uint64_t min_length;
Federico Recanati872cd692022-01-05 13:10:10 +01001940 uint32_t composite_memory_region_offset;
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001941 uint32_t constituents_start;
Andrew Walbran130a8ae2020-05-15 16:27:15 +01001942 uint32_t constituents_length;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001943 enum ffa_data_access data_access;
1944 enum ffa_instruction_access instruction_access;
Olivier Deprez4342a3c2022-02-28 09:37:25 +01001945 enum ffa_memory_security security_state;
Karl Meakinf98b2aa2023-10-12 16:09:59 +01001946 enum ffa_memory_type type;
Federico Recanatia98603a2021-12-20 18:04:03 +01001947 struct ffa_value ret;
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001948 const size_t minimum_first_fragment_length =
Daniel Boulby41ef8ba2023-10-13 17:01:22 +01001949 memory_region->receivers_offset +
1950 memory_region->memory_access_desc_size +
1951 sizeof(struct ffa_composite_memory_region);
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001952
1953 if (fragment_length < minimum_first_fragment_length) {
Karl Meakine8937d92024-03-19 16:04:25 +00001954 dlog_verbose("Fragment length %u too short (min %zu).\n",
1955 fragment_length, minimum_first_fragment_length);
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001956 return ffa_error(FFA_INVALID_PARAMETERS);
1957 }
1958
Demi Marie Obenour73a1e942023-02-04 14:09:18 -05001959 static_assert(sizeof(struct ffa_memory_region_constituent) == 16,
1960 "struct ffa_memory_region_constituent must be 16 bytes");
1961 if (!is_aligned(fragment_length,
1962 sizeof(struct ffa_memory_region_constituent)) ||
1963 !is_aligned(memory_share_length,
1964 sizeof(struct ffa_memory_region_constituent))) {
1965 dlog_verbose(
1966 "Fragment length %u or total length %u"
1967 " is not 16-byte aligned.\n",
1968 fragment_length, memory_share_length);
1969 return ffa_error(FFA_INVALID_PARAMETERS);
1970 }
1971
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001972 if (fragment_length > memory_share_length) {
1973 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00001974 "Fragment length %zu greater than total length %zu.\n",
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001975 (size_t)fragment_length, (size_t)memory_share_length);
1976 return ffa_error(FFA_INVALID_PARAMETERS);
1977 }
Andrew Walbrana65a1322020-04-06 19:32:32 +01001978
J-Alves95df0ef2022-12-07 10:09:48 +00001979 /* The sender must match the caller. */
1980 if ((!vm_id_is_current_world(from_locked.vm->id) &&
1981 vm_id_is_current_world(memory_region->sender)) ||
1982 (vm_id_is_current_world(from_locked.vm->id) &&
1983 memory_region->sender != from_locked.vm->id)) {
1984 dlog_verbose("Invalid memory sender ID.\n");
1985 return ffa_error(FFA_DENIED);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001986 }
1987
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001988 if (memory_region->receiver_count <= 0) {
1989 dlog_verbose("No receivers!\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001990 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001991 }
1992
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001993 /*
1994 * Ensure that the composite header is within the memory bounds and
1995 * doesn't overlap the first part of the message. Cast to uint64_t
1996 * to prevent overflow.
1997 */
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001998 receivers_end = ((uint64_t)memory_region->memory_access_desc_size *
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001999 (uint64_t)memory_region->receiver_count) +
Daniel Boulby41ef8ba2023-10-13 17:01:22 +01002000 memory_region->receivers_offset;
Demi Marie Obenourd4677412023-02-03 20:35:12 -05002001 min_length = receivers_end +
2002 sizeof(struct ffa_composite_memory_region) +
2003 sizeof(struct ffa_memory_region_constituent);
2004 if (min_length > memory_share_length) {
Karl Meakine8937d92024-03-19 16:04:25 +00002005 dlog_verbose("Share too short: got %zu but minimum is %zu.\n",
Demi Marie Obenourd4677412023-02-03 20:35:12 -05002006 (size_t)memory_share_length, (size_t)min_length);
2007 return ffa_error(FFA_INVALID_PARAMETERS);
2008 }
2009
2010 composite_memory_region_offset =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002011 receiver->composite_memory_region_offset;
Andrew Walbrana65a1322020-04-06 19:32:32 +01002012
2013 /*
Demi Marie Obenourd4677412023-02-03 20:35:12 -05002014 * Check that the composite memory region descriptor is after the access
2015 * descriptors, is at least 16-byte aligned, and fits in the first
2016 * fragment.
Andrew Walbrana65a1322020-04-06 19:32:32 +01002017 */
Demi Marie Obenourd4677412023-02-03 20:35:12 -05002018 if ((composite_memory_region_offset < receivers_end) ||
2019 (composite_memory_region_offset % 16 != 0) ||
2020 (composite_memory_region_offset >
2021 fragment_length - sizeof(struct ffa_composite_memory_region))) {
2022 dlog_verbose(
2023 "Invalid composite memory region descriptor offset "
Karl Meakine8937d92024-03-19 16:04:25 +00002024 "%zu.\n",
Demi Marie Obenourd4677412023-02-03 20:35:12 -05002025 (size_t)composite_memory_region_offset);
2026 return ffa_error(FFA_INVALID_PARAMETERS);
2027 }
2028
2029 /*
2030 * Compute the start of the constituent regions. Already checked
2031 * to be not more than fragment_length and thus not more than
2032 * memory_share_length.
2033 */
2034 constituents_start = composite_memory_region_offset +
2035 sizeof(struct ffa_composite_memory_region);
2036 constituents_length = memory_share_length - constituents_start;
2037
2038 /*
2039 * Check that the number of constituents is consistent with the length
2040 * of the constituent region.
2041 */
2042 composite = ffa_memory_region_get_composite(memory_region, 0);
2043 if ((constituents_length %
2044 sizeof(struct ffa_memory_region_constituent) !=
2045 0) ||
2046 ((constituents_length /
2047 sizeof(struct ffa_memory_region_constituent)) !=
2048 composite->constituent_count)) {
Karl Meakine8937d92024-03-19 16:04:25 +00002049 dlog_verbose("Invalid length %zu or composite offset %zu.\n",
Demi Marie Obenourd4677412023-02-03 20:35:12 -05002050 (size_t)memory_share_length,
2051 (size_t)composite_memory_region_offset);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002052 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01002053 }
Andrew Walbranca808b12020-05-15 17:22:28 +01002054 if (fragment_length < memory_share_length &&
2055 fragment_length < HF_MAILBOX_SIZE) {
2056 dlog_warning(
2057 "Initial fragment length %d smaller than mailbox "
2058 "size.\n",
2059 fragment_length);
2060 }
Andrew Walbrana65a1322020-04-06 19:32:32 +01002061
Andrew Walbrana65a1322020-04-06 19:32:32 +01002062 /*
2063 * Clear is not allowed for memory sharing, as the sender still has
2064 * access to the memory.
2065 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002066 if ((memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR) &&
J-Alves95fbb312024-03-20 15:19:16 +00002067 (share_func == FFA_MEM_SHARE_32 ||
2068 share_func == FFA_MEM_SHARE_64)) {
Andrew Walbrana65a1322020-04-06 19:32:32 +01002069 dlog_verbose("Memory can't be cleared while being shared.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002070 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01002071 }
2072
2073 /* No other flags are allowed/supported here. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002074 if (memory_region->flags & ~FFA_MEMORY_REGION_FLAG_CLEAR) {
Andrew Walbrana65a1322020-04-06 19:32:32 +01002075 dlog_verbose("Invalid flags %#x.\n", memory_region->flags);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002076 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01002077 }
2078
J-Alves363f5722022-04-25 17:37:37 +01002079 /* Check that the permissions are valid, for each specified receiver. */
2080 for (uint32_t i = 0U; i < memory_region->receiver_count; i++) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002081 struct ffa_memory_region_attributes receiver_permissions;
2082
2083 receiver = ffa_memory_region_get_receiver(memory_region, i);
2084 assert(receiver != NULL);
2085 receiver_permissions = receiver->receiver_permissions;
J-Alves363f5722022-04-25 17:37:37 +01002086 ffa_memory_access_permissions_t permissions =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002087 receiver_permissions.permissions;
2088 ffa_id_t receiver_id = receiver_permissions.receiver;
J-Alves363f5722022-04-25 17:37:37 +01002089
2090 if (memory_region->sender == receiver_id) {
2091 dlog_verbose("Can't share memory with itself.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002092 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01002093 }
Federico Recanati85090c42021-12-15 13:17:54 +01002094
J-Alves363f5722022-04-25 17:37:37 +01002095 for (uint32_t j = i + 1; j < memory_region->receiver_count;
2096 j++) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002097 struct ffa_memory_access *other_receiver =
2098 ffa_memory_region_get_receiver(memory_region,
2099 j);
2100 assert(other_receiver != NULL);
2101
J-Alves363f5722022-04-25 17:37:37 +01002102 if (receiver_id ==
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002103 other_receiver->receiver_permissions.receiver) {
J-Alves363f5722022-04-25 17:37:37 +01002104 dlog_verbose(
2105 "Repeated receiver(%x) in memory send "
2106 "operation.\n",
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002107 other_receiver->receiver_permissions
2108 .receiver);
J-Alves363f5722022-04-25 17:37:37 +01002109 return ffa_error(FFA_INVALID_PARAMETERS);
2110 }
2111 }
2112
2113 if (composite_memory_region_offset !=
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002114 receiver->composite_memory_region_offset) {
J-Alves363f5722022-04-25 17:37:37 +01002115 dlog_verbose(
2116 "All ffa_memory_access should point to the "
2117 "same composite memory region offset.\n");
2118 return ffa_error(FFA_INVALID_PARAMETERS);
2119 }
2120
Karl Meakin84710f32023-10-12 15:14:49 +01002121 data_access = permissions.data_access;
2122 instruction_access = permissions.instruction_access;
J-Alves363f5722022-04-25 17:37:37 +01002123 if (data_access == FFA_DATA_ACCESS_RESERVED ||
2124 instruction_access == FFA_INSTRUCTION_ACCESS_RESERVED) {
2125 dlog_verbose(
2126 "Reserved value for receiver permissions "
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002127 "(data_access = %s, instruction_access = %s)\n",
2128 ffa_data_access_name(data_access),
2129 ffa_instruction_access_name(
2130 instruction_access));
J-Alves363f5722022-04-25 17:37:37 +01002131 return ffa_error(FFA_INVALID_PARAMETERS);
2132 }
2133 if (instruction_access !=
2134 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
2135 dlog_verbose(
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002136 "Invalid instruction access permissions %s "
2137 "for sending memory, expected %s.\n",
2138 ffa_instruction_access_name(instruction_access),
2139 ffa_instruction_access_name(
Daniel Boulby91052c32024-05-21 14:09:48 +01002140 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED));
J-Alves363f5722022-04-25 17:37:37 +01002141 return ffa_error(FFA_INVALID_PARAMETERS);
2142 }
J-Alves95fbb312024-03-20 15:19:16 +00002143 if (share_func == FFA_MEM_SHARE_32 ||
2144 share_func == FFA_MEM_SHARE_64) {
J-Alves363f5722022-04-25 17:37:37 +01002145 if (data_access == FFA_DATA_ACCESS_NOT_SPECIFIED) {
2146 dlog_verbose(
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002147 "Invalid data access permissions %s "
2148 "for sharing memory, expected %s.\n",
2149 ffa_data_access_name(data_access),
2150 ffa_data_access_name(
2151 FFA_DATA_ACCESS_NOT_SPECIFIED));
J-Alves363f5722022-04-25 17:37:37 +01002152 return ffa_error(FFA_INVALID_PARAMETERS);
2153 }
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002154 /*
2155 * According to section 10.10.3 of the FF-A v1.1 EAC0
2156 * spec, NX is required for share operations (but must
2157 * not be specified by the sender) so set it in the
2158 * copy that we store, ready to be returned to the
2159 * retriever.
2160 */
2161 if (vm_id_is_current_world(receiver_id)) {
Karl Meakin84710f32023-10-12 15:14:49 +01002162 permissions.instruction_access =
2163 FFA_INSTRUCTION_ACCESS_NX;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002164 receiver_permissions.permissions = permissions;
2165 }
J-Alves363f5722022-04-25 17:37:37 +01002166 }
J-Alves95fbb312024-03-20 15:19:16 +00002167 if ((share_func == FFA_MEM_LEND_32 ||
2168 share_func == FFA_MEM_LEND_64) &&
J-Alves363f5722022-04-25 17:37:37 +01002169 data_access == FFA_DATA_ACCESS_NOT_SPECIFIED) {
2170 dlog_verbose(
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002171 "Invalid data access permissions %s for "
2172 "lending memory, expected %s.\n",
2173 ffa_data_access_name(data_access),
2174 ffa_data_access_name(
2175 FFA_DATA_ACCESS_NOT_SPECIFIED));
J-Alves363f5722022-04-25 17:37:37 +01002176 return ffa_error(FFA_INVALID_PARAMETERS);
2177 }
2178
J-Alves95fbb312024-03-20 15:19:16 +00002179 if ((share_func == FFA_MEM_DONATE_32 ||
2180 share_func == FFA_MEM_DONATE_64) &&
J-Alves363f5722022-04-25 17:37:37 +01002181 data_access != FFA_DATA_ACCESS_NOT_SPECIFIED) {
2182 dlog_verbose(
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002183 "Invalid data access permissions %s for "
2184 "donating memory, expected %s.\n",
2185 ffa_data_access_name(data_access),
2186 ffa_data_access_name(
2187 FFA_DATA_ACCESS_NOT_SPECIFIED));
J-Alves363f5722022-04-25 17:37:37 +01002188 return ffa_error(FFA_INVALID_PARAMETERS);
2189 }
Andrew Walbrana65a1322020-04-06 19:32:32 +01002190 }
2191
Olivier Deprez4342a3c2022-02-28 09:37:25 +01002192 /* Memory region attributes NS-Bit MBZ for FFA_MEM_SHARE/LEND/DONATE. */
Karl Meakin84710f32023-10-12 15:14:49 +01002193 security_state = memory_region->attributes.security;
Olivier Deprez4342a3c2022-02-28 09:37:25 +01002194 if (security_state != FFA_MEMORY_SECURITY_UNSPECIFIED) {
2195 dlog_verbose(
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002196 "Invalid security state %s for memory share operation, "
2197 "expected %s.\n",
2198 ffa_memory_security_name(security_state),
2199 ffa_memory_security_name(
2200 FFA_MEMORY_SECURITY_UNSPECIFIED));
Olivier Deprez4342a3c2022-02-28 09:37:25 +01002201 return ffa_error(FFA_INVALID_PARAMETERS);
2202 }
2203
Federico Recanatid937f5e2021-12-20 17:38:23 +01002204 /*
J-Alves807794e2022-06-16 13:42:47 +01002205 * If a memory donate or lend with single borrower, the memory type
2206 * shall not be specified by the sender.
Federico Recanatid937f5e2021-12-20 17:38:23 +01002207 */
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002208 type = memory_region->attributes.type;
J-Alves807794e2022-06-16 13:42:47 +01002209 if (share_func == FFA_MEM_DONATE_32 ||
J-Alves95fbb312024-03-20 15:19:16 +00002210 share_func == FFA_MEM_DONATE_64 ||
2211 ((share_func == FFA_MEM_LEND_32 || share_func == FFA_MEM_LEND_64) &&
J-Alves807794e2022-06-16 13:42:47 +01002212 memory_region->receiver_count == 1)) {
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002213 if (type != FFA_MEMORY_NOT_SPECIFIED_MEM) {
J-Alves807794e2022-06-16 13:42:47 +01002214 dlog_verbose(
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002215 "Invalid memory type %s for memory share "
2216 "operation, expected %s.\n",
2217 ffa_memory_type_name(type),
2218 ffa_memory_type_name(
2219 FFA_MEMORY_NOT_SPECIFIED_MEM));
J-Alves807794e2022-06-16 13:42:47 +01002220 return ffa_error(FFA_INVALID_PARAMETERS);
2221 }
2222 } else {
2223 /*
2224 * Check that sender's memory attributes match Hafnium
2225 * expectations: Normal Memory, Inner shareable, Write-Back
2226 * Read-Allocate Write-Allocate Cacheable.
2227 */
2228 ret = ffa_memory_attributes_validate(memory_region->attributes);
2229 if (ret.func != FFA_SUCCESS_32) {
2230 return ret;
2231 }
Federico Recanatid937f5e2021-12-20 17:38:23 +01002232 }
2233
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002234 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbrana65a1322020-04-06 19:32:32 +01002235}
2236
2237/**
Andrew Walbranca808b12020-05-15 17:22:28 +01002238 * Gets the share state for continuing an operation to donate, lend or share
2239 * memory, and checks that it is a valid request.
2240 *
2241 * Returns FFA_SUCCESS if the request was valid, or the relevant FFA_ERROR if
2242 * not.
2243 */
J-Alvesfdd29272022-07-19 13:16:31 +01002244struct ffa_value ffa_memory_send_continue_validate(
Andrew Walbranca808b12020-05-15 17:22:28 +01002245 struct share_states_locked share_states, ffa_memory_handle_t handle,
J-Alves19e20cf2023-08-02 12:48:55 +01002246 struct ffa_memory_share_state **share_state_ret, ffa_id_t from_vm_id,
Andrew Walbranca808b12020-05-15 17:22:28 +01002247 struct mpool *page_pool)
2248{
2249 struct ffa_memory_share_state *share_state;
2250 struct ffa_memory_region *memory_region;
2251
Daniel Boulbya2f8c662021-11-26 17:52:53 +00002252 assert(share_state_ret != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +01002253
2254 /*
2255 * Look up the share state by handle and make sure that the VM ID
2256 * matches.
2257 */
Karl Meakin4a2854a2023-06-30 16:26:52 +01002258 share_state = get_share_state(share_states, handle);
J-Alvesb56aac82023-11-10 09:44:43 +00002259 if (share_state == NULL) {
Andrew Walbranca808b12020-05-15 17:22:28 +01002260 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00002261 "Invalid handle %#lx for memory send continuation.\n",
Andrew Walbranca808b12020-05-15 17:22:28 +01002262 handle);
2263 return ffa_error(FFA_INVALID_PARAMETERS);
2264 }
2265 memory_region = share_state->memory_region;
2266
J-Alvesfdd29272022-07-19 13:16:31 +01002267 if (vm_id_is_current_world(from_vm_id) &&
2268 memory_region->sender != from_vm_id) {
Andrew Walbranca808b12020-05-15 17:22:28 +01002269 dlog_verbose("Invalid sender %d.\n", memory_region->sender);
2270 return ffa_error(FFA_INVALID_PARAMETERS);
2271 }
2272
2273 if (share_state->sending_complete) {
2274 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00002275 "Sending of memory handle %#lx is already complete.\n",
Andrew Walbranca808b12020-05-15 17:22:28 +01002276 handle);
2277 return ffa_error(FFA_INVALID_PARAMETERS);
2278 }
2279
2280 if (share_state->fragment_count == MAX_FRAGMENTS) {
2281 /*
2282 * Log a warning as this is a sign that MAX_FRAGMENTS should
2283 * probably be increased.
2284 */
2285 dlog_warning(
Karl Meakine8937d92024-03-19 16:04:25 +00002286 "Too many fragments for memory share with handle %#lx; "
Andrew Walbranca808b12020-05-15 17:22:28 +01002287 "only %d supported.\n",
2288 handle, MAX_FRAGMENTS);
2289 /* Free share state, as it's not possible to complete it. */
2290 share_state_free(share_states, share_state, page_pool);
2291 return ffa_error(FFA_NO_MEMORY);
2292 }
2293
2294 *share_state_ret = share_state;
2295
2296 return (struct ffa_value){.func = FFA_SUCCESS_32};
2297}
2298
2299/**
J-Alves95df0ef2022-12-07 10:09:48 +00002300 * Checks if there is at least one receiver from the other world.
2301 */
J-Alvesfdd29272022-07-19 13:16:31 +01002302bool memory_region_receivers_from_other_world(
J-Alves95df0ef2022-12-07 10:09:48 +00002303 struct ffa_memory_region *memory_region)
2304{
2305 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002306 struct ffa_memory_access *receiver =
2307 ffa_memory_region_get_receiver(memory_region, i);
2308 assert(receiver != NULL);
2309 ffa_id_t receiver_id = receiver->receiver_permissions.receiver;
2310
2311 if (!vm_id_is_current_world(receiver_id)) {
J-Alves95df0ef2022-12-07 10:09:48 +00002312 return true;
2313 }
2314 }
2315 return false;
2316}
2317
2318/**
J-Alves9da280b2022-12-21 14:55:39 +00002319 * Validates a call to donate, lend or share memory in which Hafnium is the
2320 * designated allocator of the memory handle. In practice, this also means
2321 * Hafnium is responsible for managing the state structures for the transaction.
2322 * If Hafnium is the SPMC, it should allocate the memory handle when either the
2323 * sender is an SP or there is at least one borrower that is an SP.
2324 * If Hafnium is the hypervisor, it should allocate the memory handle when
2325 * operation involves only NWd VMs.
2326 *
2327 * If validation goes well, Hafnium updates the stage-2 page tables of the
2328 * sender. Validation consists of checking if the message length and number of
2329 * memory region constituents match, and if the transition is valid for the
2330 * type of memory sending operation.
Andrew Walbran475c1452020-02-07 13:22:22 +00002331 *
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002332 * Assumes that the caller has already found and locked the sender VM and copied
2333 * the memory region descriptor from the sender's TX buffer to a freshly
2334 * allocated page from Hafnium's internal pool. The caller must have also
2335 * validated that the receiver VM ID is valid.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002336 *
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002337 * This function takes ownership of the `memory_region` passed in and will free
2338 * it when necessary; it must not be freed by the caller.
Jose Marinho09b1db82019-08-08 09:16:59 +01002339 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002340struct ffa_value ffa_memory_send(struct vm_locked from_locked,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002341 struct ffa_memory_region *memory_region,
Andrew Walbran130a8ae2020-05-15 16:27:15 +01002342 uint32_t memory_share_length,
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002343 uint32_t fragment_length, uint32_t share_func,
2344 struct mpool *page_pool)
Jose Marinho09b1db82019-08-08 09:16:59 +01002345{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002346 struct ffa_value ret;
Andrew Walbranca808b12020-05-15 17:22:28 +01002347 struct share_states_locked share_states;
2348 struct ffa_memory_share_state *share_state;
Jose Marinho09b1db82019-08-08 09:16:59 +01002349
2350 /*
Andrew Walbrana65a1322020-04-06 19:32:32 +01002351 * If there is an error validating the `memory_region` then we need to
2352 * free it because we own it but we won't be storing it in a share state
2353 * after all.
Jose Marinho09b1db82019-08-08 09:16:59 +01002354 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002355 ret = ffa_memory_send_validate(from_locked, memory_region,
2356 memory_share_length, fragment_length,
J-Alves363f5722022-04-25 17:37:37 +01002357 share_func);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002358 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002359 mpool_free(page_pool, memory_region);
Andrew Walbrana65a1322020-04-06 19:32:32 +01002360 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01002361 }
2362
Andrew Walbrana65a1322020-04-06 19:32:32 +01002363 /* Set flag for share function, ready to be retrieved later. */
2364 switch (share_func) {
J-Alves95fbb312024-03-20 15:19:16 +00002365 case FFA_MEM_SHARE_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002366 case FFA_MEM_SHARE_32:
Andrew Walbrana65a1322020-04-06 19:32:32 +01002367 memory_region->flags |=
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002368 FFA_MEMORY_REGION_TRANSACTION_TYPE_SHARE;
Andrew Walbrana65a1322020-04-06 19:32:32 +01002369 break;
J-Alves95fbb312024-03-20 15:19:16 +00002370 case FFA_MEM_LEND_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002371 case FFA_MEM_LEND_32:
2372 memory_region->flags |= FFA_MEMORY_REGION_TRANSACTION_TYPE_LEND;
Andrew Walbrana65a1322020-04-06 19:32:32 +01002373 break;
J-Alves95fbb312024-03-20 15:19:16 +00002374 case FFA_MEM_DONATE_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002375 case FFA_MEM_DONATE_32:
Andrew Walbrana65a1322020-04-06 19:32:32 +01002376 memory_region->flags |=
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002377 FFA_MEMORY_REGION_TRANSACTION_TYPE_DONATE;
Andrew Walbrana65a1322020-04-06 19:32:32 +01002378 break;
Jose Marinho09b1db82019-08-08 09:16:59 +01002379 }
2380
Andrew Walbranca808b12020-05-15 17:22:28 +01002381 share_states = share_states_lock();
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002382 /*
2383 * Allocate a share state before updating the page table. Otherwise if
2384 * updating the page table succeeded but allocating the share state
2385 * failed then it would leave the memory in a state where nobody could
2386 * get it back.
2387 */
Karl Meakin52cdfe72023-06-30 14:49:10 +01002388 share_state = allocate_share_state(share_states, share_func,
2389 memory_region, fragment_length,
2390 FFA_MEMORY_HANDLE_INVALID);
J-Alvesb56aac82023-11-10 09:44:43 +00002391 if (share_state == NULL) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002392 dlog_verbose("Failed to allocate share state.\n");
2393 mpool_free(page_pool, memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +01002394 ret = ffa_error(FFA_NO_MEMORY);
2395 goto out;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002396 }
2397
Andrew Walbranca808b12020-05-15 17:22:28 +01002398 if (fragment_length == memory_share_length) {
2399 /* No more fragments to come, everything fit in one message. */
J-Alves2a0d2882020-10-29 14:49:50 +00002400 ret = ffa_memory_send_complete(
2401 from_locked, share_states, share_state, page_pool,
2402 &(share_state->sender_orig_mode));
Andrew Walbranca808b12020-05-15 17:22:28 +01002403 } else {
J-Alvesfdd29272022-07-19 13:16:31 +01002404 /*
2405 * Use sender ID from 'memory_region' assuming
2406 * that at this point it has been validated:
2407 * - MBZ at virtual FF-A instance.
2408 */
J-Alves19e20cf2023-08-02 12:48:55 +01002409 ffa_id_t sender_to_ret =
J-Alvesfdd29272022-07-19 13:16:31 +01002410 (from_locked.vm->id == HF_OTHER_WORLD_ID)
2411 ? memory_region->sender
2412 : 0;
Andrew Walbranca808b12020-05-15 17:22:28 +01002413 ret = (struct ffa_value){
2414 .func = FFA_MEM_FRAG_RX_32,
J-Alvesee68c542020-10-29 17:48:20 +00002415 .arg1 = (uint32_t)memory_region->handle,
2416 .arg2 = (uint32_t)(memory_region->handle >> 32),
J-Alvesfdd29272022-07-19 13:16:31 +01002417 .arg3 = fragment_length,
2418 .arg4 = (uint32_t)(sender_to_ret & 0xffff) << 16};
Andrew Walbranca808b12020-05-15 17:22:28 +01002419 }
2420
2421out:
2422 share_states_unlock(&share_states);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002423 dump_share_states();
Andrew Walbranca808b12020-05-15 17:22:28 +01002424 return ret;
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002425}
2426
2427/**
J-Alves8505a8a2022-06-15 18:10:18 +01002428 * Continues an operation to donate, lend or share memory to a VM from current
2429 * world. If this is the last fragment then checks that the transition is valid
2430 * for the type of memory sending operation and updates the stage-2 page tables
2431 * of the sender.
Andrew Walbranca808b12020-05-15 17:22:28 +01002432 *
2433 * Assumes that the caller has already found and locked the sender VM and copied
2434 * the memory region descriptor from the sender's TX buffer to a freshly
2435 * allocated page from Hafnium's internal pool.
2436 *
2437 * This function takes ownership of the `fragment` passed in; it must not be
2438 * freed by the caller.
2439 */
2440struct ffa_value ffa_memory_send_continue(struct vm_locked from_locked,
2441 void *fragment,
2442 uint32_t fragment_length,
2443 ffa_memory_handle_t handle,
2444 struct mpool *page_pool)
2445{
2446 struct share_states_locked share_states = share_states_lock();
2447 struct ffa_memory_share_state *share_state;
2448 struct ffa_value ret;
2449 struct ffa_memory_region *memory_region;
2450
Demi Marie Obenour73a1e942023-02-04 14:09:18 -05002451 CHECK(is_aligned(fragment,
2452 alignof(struct ffa_memory_region_constituent)));
2453 if (fragment_length % sizeof(struct ffa_memory_region_constituent) !=
2454 0) {
2455 dlog_verbose("Fragment length %u misaligned.\n",
2456 fragment_length);
2457 ret = ffa_error(FFA_INVALID_PARAMETERS);
2458 goto out_free_fragment;
2459 }
2460
Andrew Walbranca808b12020-05-15 17:22:28 +01002461 ret = ffa_memory_send_continue_validate(share_states, handle,
2462 &share_state,
2463 from_locked.vm->id, page_pool);
2464 if (ret.func != FFA_SUCCESS_32) {
2465 goto out_free_fragment;
2466 }
2467 memory_region = share_state->memory_region;
2468
J-Alves95df0ef2022-12-07 10:09:48 +00002469 if (memory_region_receivers_from_other_world(memory_region)) {
Andrew Walbranca808b12020-05-15 17:22:28 +01002470 dlog_error(
2471 "Got hypervisor-allocated handle for memory send to "
J-Alves8505a8a2022-06-15 18:10:18 +01002472 "other world. This should never happen, and indicates "
2473 "a bug in "
Andrew Walbranca808b12020-05-15 17:22:28 +01002474 "EL3 code.\n");
2475 ret = ffa_error(FFA_INVALID_PARAMETERS);
2476 goto out_free_fragment;
2477 }
2478
2479 /* Add this fragment. */
2480 share_state->fragments[share_state->fragment_count] = fragment;
2481 share_state->fragment_constituent_counts[share_state->fragment_count] =
2482 fragment_length / sizeof(struct ffa_memory_region_constituent);
2483 share_state->fragment_count++;
2484
2485 /* Check whether the memory send operation is now ready to complete. */
2486 if (share_state_sending_complete(share_states, share_state)) {
J-Alves2a0d2882020-10-29 14:49:50 +00002487 ret = ffa_memory_send_complete(
2488 from_locked, share_states, share_state, page_pool,
2489 &(share_state->sender_orig_mode));
Andrew Walbranca808b12020-05-15 17:22:28 +01002490 } else {
2491 ret = (struct ffa_value){
2492 .func = FFA_MEM_FRAG_RX_32,
2493 .arg1 = (uint32_t)handle,
2494 .arg2 = (uint32_t)(handle >> 32),
2495 .arg3 = share_state_next_fragment_offset(share_states,
2496 share_state)};
2497 }
2498 goto out;
2499
2500out_free_fragment:
2501 mpool_free(page_pool, fragment);
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002502
2503out:
Andrew Walbranca808b12020-05-15 17:22:28 +01002504 share_states_unlock(&share_states);
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002505 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002506}
2507
Andrew Walbranca808b12020-05-15 17:22:28 +01002508/** Clean up after the receiver has finished retrieving a memory region. */
2509static void ffa_memory_retrieve_complete(
2510 struct share_states_locked share_states,
2511 struct ffa_memory_share_state *share_state, struct mpool *page_pool)
2512{
J-Alves95fbb312024-03-20 15:19:16 +00002513 if (share_state->share_func == FFA_MEM_DONATE_32 ||
2514 share_state->share_func == FFA_MEM_DONATE_64) {
Andrew Walbranca808b12020-05-15 17:22:28 +01002515 /*
2516 * Memory that has been donated can't be relinquished,
2517 * so no need to keep the share state around.
2518 */
2519 share_state_free(share_states, share_state, page_pool);
2520 dlog_verbose("Freed share state for donate.\n");
2521 }
2522}
2523
J-Alves2d8457f2022-10-05 11:06:41 +01002524/**
2525 * Initialises the given memory region descriptor to be used for an
2526 * `FFA_MEM_RETRIEVE_RESP`, including the given constituents for the first
2527 * fragment.
2528 * The memory region descriptor is initialized according to retriever's
2529 * FF-A version.
2530 *
2531 * Returns true on success, or false if the given constituents won't all fit in
2532 * the first fragment.
2533 */
2534static bool ffa_retrieved_memory_region_init(
Karl Meakin0e617d92024-04-05 12:55:22 +01002535 void *response, enum ffa_version ffa_version, size_t response_max_size,
J-Alves19e20cf2023-08-02 12:48:55 +01002536 ffa_id_t sender, ffa_memory_attributes_t attributes,
J-Alves2d8457f2022-10-05 11:06:41 +01002537 ffa_memory_region_flags_t flags, ffa_memory_handle_t handle,
Daniel Boulbyde974ca2023-12-12 13:53:31 +00002538 ffa_memory_access_permissions_t permissions,
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002539 struct ffa_memory_access *receivers, size_t receiver_count,
2540 uint32_t memory_access_desc_size, uint32_t page_count,
2541 uint32_t total_constituent_count,
J-Alves2d8457f2022-10-05 11:06:41 +01002542 const struct ffa_memory_region_constituent constituents[],
2543 uint32_t fragment_constituent_count, uint32_t *total_length,
2544 uint32_t *fragment_length)
2545{
2546 struct ffa_composite_memory_region *composite_memory_region;
J-Alves2d8457f2022-10-05 11:06:41 +01002547 uint32_t i;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002548 uint32_t composite_offset;
J-Alves2d8457f2022-10-05 11:06:41 +01002549 uint32_t constituents_offset;
J-Alves2d8457f2022-10-05 11:06:41 +01002550
2551 assert(response != NULL);
2552
Karl Meakin0e617d92024-04-05 12:55:22 +01002553 if (ffa_version == FFA_VERSION_1_0) {
J-Alves2d8457f2022-10-05 11:06:41 +01002554 struct ffa_memory_region_v1_0 *retrieve_response =
2555 (struct ffa_memory_region_v1_0 *)response;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002556 struct ffa_memory_access_v1_0 *receiver;
J-Alves2d8457f2022-10-05 11:06:41 +01002557
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002558 ffa_memory_region_init_header_v1_0(retrieve_response, sender,
2559 attributes, flags, handle, 0,
2560 receiver_count);
J-Alves2d8457f2022-10-05 11:06:41 +01002561
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002562 receiver = (struct ffa_memory_access_v1_0 *)
2563 retrieve_response->receivers;
J-Alves2d8457f2022-10-05 11:06:41 +01002564 receiver_count = retrieve_response->receiver_count;
2565
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002566 for (uint32_t i = 0; i < receiver_count; i++) {
2567 ffa_id_t receiver_id =
2568 receivers[i].receiver_permissions.receiver;
2569 ffa_memory_receiver_flags_t recv_flags =
2570 receivers[i].receiver_permissions.flags;
2571
2572 /*
2573 * Initialized here as in memory retrieve responses we
2574 * currently expect one borrower to be specified.
2575 */
2576 ffa_memory_access_init_v1_0(
Karl Meakin84710f32023-10-12 15:14:49 +01002577 receiver, receiver_id, permissions.data_access,
2578 permissions.instruction_access, recv_flags);
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002579 }
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002580
2581 composite_offset =
J-Alves2d8457f2022-10-05 11:06:41 +01002582 sizeof(struct ffa_memory_region_v1_0) +
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002583 receiver_count * sizeof(struct ffa_memory_access_v1_0);
2584 receiver->composite_memory_region_offset = composite_offset;
J-Alves2d8457f2022-10-05 11:06:41 +01002585
2586 composite_memory_region = ffa_memory_region_get_composite_v1_0(
2587 retrieve_response, 0);
2588 } else {
J-Alves2d8457f2022-10-05 11:06:41 +01002589 struct ffa_memory_region *retrieve_response =
2590 (struct ffa_memory_region *)response;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002591 struct ffa_memory_access *retrieve_response_receivers;
J-Alves2d8457f2022-10-05 11:06:41 +01002592
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002593 ffa_memory_region_init_header(
2594 retrieve_response, sender, attributes, flags, handle, 0,
2595 receiver_count, memory_access_desc_size);
J-Alves2d8457f2022-10-05 11:06:41 +01002596
2597 /*
2598 * Note that `sizeof(struct_ffa_memory_region)` and
2599 * `sizeof(struct ffa_memory_access)` must both be multiples of
2600 * 16 (as verified by the asserts in `ffa_memory.c`, so it is
2601 * guaranteed that the offset we calculate here is aligned to a
2602 * 64-bit boundary and so 64-bit values can be copied without
2603 * alignment faults.
2604 */
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002605 composite_offset =
Daniel Boulby41ef8ba2023-10-13 17:01:22 +01002606 retrieve_response->receivers_offset +
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002607 (uint32_t)(receiver_count *
2608 retrieve_response->memory_access_desc_size);
J-Alves2d8457f2022-10-05 11:06:41 +01002609
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002610 retrieve_response_receivers =
2611 ffa_memory_region_get_receiver(retrieve_response, 0);
2612 assert(retrieve_response_receivers != NULL);
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002613
2614 /*
2615 * Initialized here as in memory retrieve responses we currently
2616 * expect one borrower to be specified.
2617 */
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002618 memcpy_s(retrieve_response_receivers,
2619 sizeof(struct ffa_memory_access) * receiver_count,
2620 receivers,
2621 sizeof(struct ffa_memory_access) * receiver_count);
2622
2623 retrieve_response_receivers->composite_memory_region_offset =
2624 composite_offset;
2625
J-Alves2d8457f2022-10-05 11:06:41 +01002626 composite_memory_region =
2627 ffa_memory_region_get_composite(retrieve_response, 0);
2628 }
2629
J-Alves2d8457f2022-10-05 11:06:41 +01002630 assert(composite_memory_region != NULL);
2631
J-Alves2d8457f2022-10-05 11:06:41 +01002632 composite_memory_region->page_count = page_count;
2633 composite_memory_region->constituent_count = total_constituent_count;
2634 composite_memory_region->reserved_0 = 0;
2635
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002636 constituents_offset =
2637 composite_offset + sizeof(struct ffa_composite_memory_region);
J-Alves2d8457f2022-10-05 11:06:41 +01002638 if (constituents_offset +
2639 fragment_constituent_count *
2640 sizeof(struct ffa_memory_region_constituent) >
2641 response_max_size) {
2642 return false;
2643 }
2644
2645 for (i = 0; i < fragment_constituent_count; ++i) {
2646 composite_memory_region->constituents[i] = constituents[i];
2647 }
2648
2649 if (total_length != NULL) {
2650 *total_length =
2651 constituents_offset +
2652 composite_memory_region->constituent_count *
2653 sizeof(struct ffa_memory_region_constituent);
2654 }
2655 if (fragment_length != NULL) {
2656 *fragment_length =
2657 constituents_offset +
2658 fragment_constituent_count *
2659 sizeof(struct ffa_memory_region_constituent);
2660 }
2661
2662 return true;
2663}
2664
J-Alves96de29f2022-04-26 16:05:24 +01002665/**
2666 * Validates the retrieved permissions against those specified by the lender
2667 * of memory share operation. Optionally can help set the permissions to be used
2668 * for the S2 mapping, through the `permissions` argument.
J-Alvesdcad8992023-09-15 14:10:35 +01002669 * Returns FFA_SUCCESS if all the fields are valid. FFA_ERROR, with error code:
2670 * - FFA_INVALID_PARAMETERS -> if the fields have invalid values as per the
2671 * specification for each ABI.
2672 * - FFA_DENIED -> if the permissions specified by the retriever are not
2673 * less permissive than those provided by the sender.
J-Alves96de29f2022-04-26 16:05:24 +01002674 */
J-Alvesdcad8992023-09-15 14:10:35 +01002675static struct ffa_value ffa_memory_retrieve_is_memory_access_valid(
2676 uint32_t share_func, enum ffa_data_access sent_data_access,
J-Alves96de29f2022-04-26 16:05:24 +01002677 enum ffa_data_access requested_data_access,
2678 enum ffa_instruction_access sent_instruction_access,
2679 enum ffa_instruction_access requested_instruction_access,
J-Alvesdcad8992023-09-15 14:10:35 +01002680 ffa_memory_access_permissions_t *permissions, bool multiple_borrowers)
J-Alves96de29f2022-04-26 16:05:24 +01002681{
2682 switch (sent_data_access) {
2683 case FFA_DATA_ACCESS_NOT_SPECIFIED:
2684 case FFA_DATA_ACCESS_RW:
2685 if (requested_data_access == FFA_DATA_ACCESS_NOT_SPECIFIED ||
2686 requested_data_access == FFA_DATA_ACCESS_RW) {
2687 if (permissions != NULL) {
Karl Meakin84710f32023-10-12 15:14:49 +01002688 permissions->data_access = FFA_DATA_ACCESS_RW;
J-Alves96de29f2022-04-26 16:05:24 +01002689 }
2690 break;
2691 }
2692 /* Intentional fall-through. */
2693 case FFA_DATA_ACCESS_RO:
2694 if (requested_data_access == FFA_DATA_ACCESS_NOT_SPECIFIED ||
2695 requested_data_access == FFA_DATA_ACCESS_RO) {
2696 if (permissions != NULL) {
Karl Meakin84710f32023-10-12 15:14:49 +01002697 permissions->data_access = FFA_DATA_ACCESS_RO;
J-Alves96de29f2022-04-26 16:05:24 +01002698 }
2699 break;
2700 }
2701 dlog_verbose(
2702 "Invalid data access requested; sender specified "
2703 "permissions %#x but receiver requested %#x.\n",
2704 sent_data_access, requested_data_access);
J-Alvesdcad8992023-09-15 14:10:35 +01002705 return ffa_error(FFA_DENIED);
J-Alves96de29f2022-04-26 16:05:24 +01002706 case FFA_DATA_ACCESS_RESERVED:
2707 panic("Got unexpected FFA_DATA_ACCESS_RESERVED. Should be "
2708 "checked before this point.");
2709 }
2710
J-Alvesdcad8992023-09-15 14:10:35 +01002711 /*
2712 * For operations with a single borrower, If it is an FFA_MEMORY_LEND
2713 * or FFA_MEMORY_DONATE the retriever should have specifed the
2714 * instruction permissions it wishes to receive.
2715 */
2716 switch (share_func) {
J-Alves95fbb312024-03-20 15:19:16 +00002717 case FFA_MEM_SHARE_64:
J-Alvesdcad8992023-09-15 14:10:35 +01002718 case FFA_MEM_SHARE_32:
2719 if (requested_instruction_access !=
2720 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
2721 dlog_verbose(
2722 "%s: for share instruction permissions must "
2723 "NOT be specified.\n",
2724 __func__);
2725 return ffa_error(FFA_INVALID_PARAMETERS);
2726 }
2727 break;
J-Alves95fbb312024-03-20 15:19:16 +00002728 case FFA_MEM_LEND_64:
J-Alvesdcad8992023-09-15 14:10:35 +01002729 case FFA_MEM_LEND_32:
2730 /*
2731 * For operations with multiple borrowers only permit XN
2732 * permissions, and both Sender and borrower should have used
2733 * FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED.
2734 */
2735 if (multiple_borrowers) {
2736 if (requested_instruction_access !=
2737 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
2738 dlog_verbose(
2739 "%s: lend/share/donate with multiple "
2740 "borrowers "
2741 "instruction permissions must NOT be "
2742 "specified.\n",
2743 __func__);
2744 return ffa_error(FFA_INVALID_PARAMETERS);
2745 }
2746 break;
2747 }
2748 /* Fall through if the operation targets a single borrower. */
J-Alves95fbb312024-03-20 15:19:16 +00002749 case FFA_MEM_DONATE_64:
J-Alvesdcad8992023-09-15 14:10:35 +01002750 case FFA_MEM_DONATE_32:
2751 if (!multiple_borrowers &&
2752 requested_instruction_access ==
2753 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
2754 dlog_verbose(
2755 "%s: for lend/donate with single borrower "
2756 "instruction permissions must be speficified "
2757 "by borrower\n",
2758 __func__);
2759 return ffa_error(FFA_INVALID_PARAMETERS);
2760 }
2761 break;
2762 default:
2763 panic("%s: Wrong func id provided.\n", __func__);
2764 }
2765
J-Alves96de29f2022-04-26 16:05:24 +01002766 switch (sent_instruction_access) {
2767 case FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED:
2768 case FFA_INSTRUCTION_ACCESS_X:
J-Alvesdcad8992023-09-15 14:10:35 +01002769 if (requested_instruction_access == FFA_INSTRUCTION_ACCESS_X) {
J-Alves96de29f2022-04-26 16:05:24 +01002770 if (permissions != NULL) {
Karl Meakin84710f32023-10-12 15:14:49 +01002771 permissions->instruction_access =
2772 FFA_INSTRUCTION_ACCESS_X;
J-Alves96de29f2022-04-26 16:05:24 +01002773 }
2774 break;
2775 }
J-Alvesdcad8992023-09-15 14:10:35 +01002776 /*
2777 * Fall through if requested permissions are less
2778 * permissive than those provided by the sender.
2779 */
J-Alves96de29f2022-04-26 16:05:24 +01002780 case FFA_INSTRUCTION_ACCESS_NX:
2781 if (requested_instruction_access ==
2782 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED ||
2783 requested_instruction_access == FFA_INSTRUCTION_ACCESS_NX) {
2784 if (permissions != NULL) {
Karl Meakin84710f32023-10-12 15:14:49 +01002785 permissions->instruction_access =
2786 FFA_INSTRUCTION_ACCESS_NX;
J-Alves96de29f2022-04-26 16:05:24 +01002787 }
2788 break;
2789 }
2790 dlog_verbose(
2791 "Invalid instruction access requested; sender "
2792 "specified permissions %#x but receiver requested "
2793 "%#x.\n",
2794 sent_instruction_access, requested_instruction_access);
J-Alvesdcad8992023-09-15 14:10:35 +01002795 return ffa_error(FFA_DENIED);
J-Alves96de29f2022-04-26 16:05:24 +01002796 case FFA_INSTRUCTION_ACCESS_RESERVED:
2797 panic("Got unexpected FFA_INSTRUCTION_ACCESS_RESERVED. Should "
2798 "be checked before this point.");
2799 }
2800
J-Alvesdcad8992023-09-15 14:10:35 +01002801 return (struct ffa_value){.func = FFA_SUCCESS_32};
J-Alves96de29f2022-04-26 16:05:24 +01002802}
2803
2804/**
2805 * Validate the receivers' permissions in the retrieve request against those
2806 * specified by the lender.
2807 * In the `permissions` argument returns the permissions to set at S2 for the
2808 * caller to the FFA_MEMORY_RETRIEVE_REQ.
J-Alves3456e032023-07-20 12:20:05 +01002809 * The function looks into the flag to bypass multiple borrower checks:
2810 * - If not set returns FFA_SUCCESS if all specified permissions are valid.
2811 * - If set returns FFA_SUCCESS if the descriptor contains the permissions
2812 * to the caller of FFA_MEM_RETRIEVE_REQ and they are valid. Other permissions
2813 * are ignored, if provided.
J-Alves96de29f2022-04-26 16:05:24 +01002814 */
2815static struct ffa_value ffa_memory_retrieve_validate_memory_access_list(
2816 struct ffa_memory_region *memory_region,
J-Alves19e20cf2023-08-02 12:48:55 +01002817 struct ffa_memory_region *retrieve_request, ffa_id_t to_vm_id,
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002818 ffa_memory_access_permissions_t *permissions,
2819 struct ffa_memory_access **receiver_ret, uint32_t func_id)
J-Alves96de29f2022-04-26 16:05:24 +01002820{
2821 uint32_t retrieve_receiver_index;
J-Alves3456e032023-07-20 12:20:05 +01002822 bool bypass_multi_receiver_check =
2823 (retrieve_request->flags &
2824 FFA_MEMORY_REGION_FLAG_BYPASS_BORROWERS_CHECK) != 0U;
J-Alvesdcad8992023-09-15 14:10:35 +01002825 const uint32_t region_receiver_count = memory_region->receiver_count;
2826 struct ffa_value ret;
J-Alves96de29f2022-04-26 16:05:24 +01002827
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002828 assert(receiver_ret != NULL);
J-Alves96de29f2022-04-26 16:05:24 +01002829 assert(permissions != NULL);
2830
Karl Meakin84710f32023-10-12 15:14:49 +01002831 *permissions = (ffa_memory_access_permissions_t){0};
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002832
J-Alves3456e032023-07-20 12:20:05 +01002833 if (!bypass_multi_receiver_check) {
J-Alvesdcad8992023-09-15 14:10:35 +01002834 if (retrieve_request->receiver_count != region_receiver_count) {
J-Alves3456e032023-07-20 12:20:05 +01002835 dlog_verbose(
2836 "Retrieve request should contain same list of "
2837 "borrowers, as specified by the lender.\n");
2838 return ffa_error(FFA_INVALID_PARAMETERS);
2839 }
2840 } else {
2841 if (retrieve_request->receiver_count != 1) {
2842 dlog_verbose(
2843 "Set bypass multiple borrower check, receiver "
2844 "list must be sized 1 (%x)\n",
2845 memory_region->receiver_count);
2846 return ffa_error(FFA_INVALID_PARAMETERS);
2847 }
J-Alves96de29f2022-04-26 16:05:24 +01002848 }
2849
2850 retrieve_receiver_index = retrieve_request->receiver_count;
2851
J-Alves96de29f2022-04-26 16:05:24 +01002852 for (uint32_t i = 0U; i < retrieve_request->receiver_count; i++) {
2853 ffa_memory_access_permissions_t sent_permissions;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002854 struct ffa_memory_access *retrieve_request_receiver =
2855 ffa_memory_region_get_receiver(retrieve_request, i);
2856 assert(retrieve_request_receiver != NULL);
J-Alves96de29f2022-04-26 16:05:24 +01002857 ffa_memory_access_permissions_t requested_permissions =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002858 retrieve_request_receiver->receiver_permissions
2859 .permissions;
J-Alves19e20cf2023-08-02 12:48:55 +01002860 ffa_id_t current_receiver_id =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002861 retrieve_request_receiver->receiver_permissions
2862 .receiver;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002863 struct ffa_memory_access *receiver;
2864 uint32_t mem_region_receiver_index;
2865 bool permissions_RO;
2866 bool clear_memory_flags;
J-Alvesf220d572024-04-24 22:15:14 +01002867 /*
2868 * If the call is at the virtual FF-A instance the caller's
2869 * ID must match an entry in the memory access list.
2870 * In the SPMC, one of the specified receivers could be from
2871 * the NWd.
2872 */
2873 bool found_to_id = vm_id_is_current_world(to_vm_id)
2874 ? (current_receiver_id == to_vm_id)
2875 : (!vm_id_is_current_world(
2876 current_receiver_id));
J-Alves96de29f2022-04-26 16:05:24 +01002877
J-Alves3456e032023-07-20 12:20:05 +01002878 if (bypass_multi_receiver_check && !found_to_id) {
2879 dlog_verbose(
2880 "Bypass multiple borrower check for id %x.\n",
2881 current_receiver_id);
2882 continue;
2883 }
2884
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002885 if (retrieve_request_receiver->composite_memory_region_offset !=
2886 0U) {
2887 dlog_verbose(
2888 "Retriever specified address ranges not "
2889 "supported (got offset %d).\n",
2890 retrieve_request_receiver
2891 ->composite_memory_region_offset);
2892 return ffa_error(FFA_INVALID_PARAMETERS);
2893 }
2894
J-Alves96de29f2022-04-26 16:05:24 +01002895 /*
2896 * Find the current receiver in the transaction descriptor from
2897 * sender.
2898 */
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002899 mem_region_receiver_index =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002900 ffa_memory_region_get_receiver_index(
2901 memory_region, current_receiver_id);
J-Alves96de29f2022-04-26 16:05:24 +01002902
2903 if (mem_region_receiver_index ==
2904 memory_region->receiver_count) {
2905 dlog_verbose("%s: receiver %x not found\n", __func__,
2906 current_receiver_id);
2907 return ffa_error(FFA_DENIED);
2908 }
2909
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002910 receiver = ffa_memory_region_get_receiver(
2911 memory_region, mem_region_receiver_index);
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002912 assert(receiver != NULL);
2913
2914 sent_permissions = receiver->receiver_permissions.permissions;
J-Alves96de29f2022-04-26 16:05:24 +01002915
2916 if (found_to_id) {
2917 retrieve_receiver_index = i;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002918
2919 *receiver_ret = receiver;
J-Alves96de29f2022-04-26 16:05:24 +01002920 }
2921
2922 /*
J-Alvesdcad8992023-09-15 14:10:35 +01002923 * Check if retrieve request memory access list is valid:
2924 * - The retrieve request complies with the specification.
2925 * - Permissions are within those specified by the sender.
J-Alves96de29f2022-04-26 16:05:24 +01002926 */
J-Alvesdcad8992023-09-15 14:10:35 +01002927 ret = ffa_memory_retrieve_is_memory_access_valid(
Karl Meakin84710f32023-10-12 15:14:49 +01002928 func_id, sent_permissions.data_access,
2929 requested_permissions.data_access,
2930 sent_permissions.instruction_access,
2931 requested_permissions.instruction_access,
J-Alvesdcad8992023-09-15 14:10:35 +01002932 found_to_id ? permissions : NULL,
2933 region_receiver_count > 1);
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002934
J-Alvesdcad8992023-09-15 14:10:35 +01002935 if (ret.func != FFA_SUCCESS_32) {
2936 return ret;
J-Alves96de29f2022-04-26 16:05:24 +01002937 }
2938
Karl Meakin84710f32023-10-12 15:14:49 +01002939 permissions_RO =
2940 (permissions->data_access == FFA_DATA_ACCESS_RO);
J-Alvese5262372024-03-27 11:02:03 +00002941 clear_memory_flags =
2942 (retrieve_request->flags &
2943 (FFA_MEMORY_REGION_FLAG_CLEAR |
2944 FFA_MEMORY_REGION_FLAG_CLEAR_RELINQUISH)) != 0U;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002945
J-Alves96de29f2022-04-26 16:05:24 +01002946 /*
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002947 * Can't request PM to clear memory if only provided
2948 * with RO permissions.
J-Alves96de29f2022-04-26 16:05:24 +01002949 */
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002950 if (found_to_id && permissions_RO && clear_memory_flags) {
J-Alves96de29f2022-04-26 16:05:24 +01002951 dlog_verbose(
2952 "Receiver has RO permissions can not request "
2953 "clear.\n");
2954 return ffa_error(FFA_DENIED);
2955 }
Daniel Boulbyde974ca2023-12-12 13:53:31 +00002956
2957 /*
2958 * Check the impdef in the retrieve_request matches the value in
2959 * the original memory send.
2960 */
2961 if (ffa_version_from_memory_access_desc_size(
2962 memory_region->memory_access_desc_size) >=
Karl Meakin0e617d92024-04-05 12:55:22 +01002963 FFA_VERSION_1_2 &&
Daniel Boulbyde974ca2023-12-12 13:53:31 +00002964 ffa_version_from_memory_access_desc_size(
2965 retrieve_request->memory_access_desc_size) >=
Karl Meakin0e617d92024-04-05 12:55:22 +01002966 FFA_VERSION_1_2) {
Daniel Boulbyde974ca2023-12-12 13:53:31 +00002967 if (receiver->impdef.val[0] !=
2968 retrieve_request_receiver->impdef.val[0] ||
2969 receiver->impdef.val[1] !=
2970 retrieve_request_receiver->impdef.val[1]) {
2971 dlog_verbose(
2972 "Impdef value in memory send does not "
J-Alves0a824e92024-04-26 16:20:12 +01002973 "match retrieve request value send "
2974 "value %#lx %#lx retrieve request "
Karl Meakine8937d92024-03-19 16:04:25 +00002975 "value %#lx %#lx\n",
Daniel Boulbyde974ca2023-12-12 13:53:31 +00002976 receiver->impdef.val[0],
2977 receiver->impdef.val[1],
2978 retrieve_request_receiver->impdef
2979 .val[0],
2980 retrieve_request_receiver->impdef
2981 .val[1]);
2982 return ffa_error(FFA_INVALID_PARAMETERS);
2983 }
2984 }
J-Alves96de29f2022-04-26 16:05:24 +01002985 }
2986
2987 if (retrieve_receiver_index == retrieve_request->receiver_count) {
2988 dlog_verbose(
2989 "Retrieve request does not contain caller's (%x) "
2990 "permissions\n",
2991 to_vm_id);
2992 return ffa_error(FFA_INVALID_PARAMETERS);
2993 }
2994
2995 return (struct ffa_value){.func = FFA_SUCCESS_32};
2996}
2997
J-Alvesa9cd7e32022-07-01 13:49:33 +01002998/*
2999 * According to section 16.4.3 of FF-A v1.1 EAC0 specification, the hypervisor
3000 * may issue an FFA_MEM_RETRIEVE_REQ to obtain the memory region description
3001 * of a pending memory sharing operation whose allocator is the SPM, for
3002 * validation purposes before forwarding an FFA_MEM_RECLAIM call. In doing so
3003 * the memory region descriptor of the retrieve request must be zeroed with the
3004 * exception of the sender ID and handle.
3005 */
J-Alves4f0d9c12024-01-17 17:23:11 +00003006bool is_ffa_hypervisor_retrieve_request(struct ffa_memory_region *request,
3007 struct vm_locked to_locked)
J-Alvesa9cd7e32022-07-01 13:49:33 +01003008{
3009 return to_locked.vm->id == HF_HYPERVISOR_VM_ID &&
Karl Meakin84710f32023-10-12 15:14:49 +01003010 request->attributes.shareability == 0U &&
3011 request->attributes.cacheability == 0U &&
3012 request->attributes.type == 0U &&
3013 request->attributes.security == 0U && request->flags == 0U &&
J-Alvesa9cd7e32022-07-01 13:49:33 +01003014 request->tag == 0U && request->receiver_count == 0U &&
3015 plat_ffa_memory_handle_allocated_by_current_world(
3016 request->handle);
3017}
3018
3019/*
3020 * Helper to reset count of fragments retrieved by the hypervisor.
3021 */
3022static void ffa_memory_retrieve_complete_from_hyp(
3023 struct ffa_memory_share_state *share_state)
3024{
3025 if (share_state->hypervisor_fragment_count ==
3026 share_state->fragment_count) {
3027 share_state->hypervisor_fragment_count = 0;
3028 }
3029}
3030
J-Alves089004f2022-07-13 14:25:44 +01003031/**
J-Alves4f0d9c12024-01-17 17:23:11 +00003032 * Prepares the return of the ffa_value for the memory retrieve response.
3033 */
3034static struct ffa_value ffa_memory_retrieve_resp(uint32_t total_length,
3035 uint32_t fragment_length)
3036{
3037 return (struct ffa_value){.func = FFA_MEM_RETRIEVE_RESP_32,
3038 .arg1 = total_length,
3039 .arg2 = fragment_length};
3040}
3041
3042/**
J-Alves089004f2022-07-13 14:25:44 +01003043 * Validate that the memory region descriptor provided by the borrower on
3044 * FFA_MEM_RETRIEVE_REQ, against saved memory region provided by lender at the
3045 * memory sharing call.
3046 */
3047static struct ffa_value ffa_memory_retrieve_validate(
J-Alves4f0d9c12024-01-17 17:23:11 +00003048 ffa_id_t to_id, struct ffa_memory_region *retrieve_request,
3049 uint32_t retrieve_request_length,
J-Alves089004f2022-07-13 14:25:44 +01003050 struct ffa_memory_region *memory_region, uint32_t *receiver_index,
3051 uint32_t share_func)
3052{
3053 ffa_memory_region_flags_t transaction_type =
3054 retrieve_request->flags &
3055 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK;
Olivier Deprez4342a3c2022-02-28 09:37:25 +01003056 enum ffa_memory_security security_state;
J-Alves4f0d9c12024-01-17 17:23:11 +00003057 const uint64_t memory_access_desc_size =
3058 retrieve_request->memory_access_desc_size;
3059 const uint32_t expected_retrieve_request_length =
3060 retrieve_request->receivers_offset +
3061 (uint32_t)(retrieve_request->receiver_count *
3062 memory_access_desc_size);
J-Alves089004f2022-07-13 14:25:44 +01003063
3064 assert(retrieve_request != NULL);
3065 assert(memory_region != NULL);
3066 assert(receiver_index != NULL);
J-Alves089004f2022-07-13 14:25:44 +01003067
J-Alves4f0d9c12024-01-17 17:23:11 +00003068 if (retrieve_request_length != expected_retrieve_request_length) {
3069 dlog_verbose(
3070 "Invalid length for FFA_MEM_RETRIEVE_REQ, expected %d "
3071 "but was %d.\n",
3072 expected_retrieve_request_length,
3073 retrieve_request_length);
3074 return ffa_error(FFA_INVALID_PARAMETERS);
3075 }
3076
3077 if (retrieve_request->sender != memory_region->sender) {
3078 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003079 "Memory with handle %#lx not fully sent, can't "
J-Alves4f0d9c12024-01-17 17:23:11 +00003080 "retrieve.\n",
3081 memory_region->handle);
3082 return ffa_error(FFA_DENIED);
3083 }
3084
3085 /*
3086 * The SPMC can only process retrieve requests to memory share
3087 * operations with one borrower from the other world. It can't
3088 * determine the ID of the NWd VM that invoked the retrieve
3089 * request interface call. It relies on the hypervisor to
3090 * validate the caller's ID against that provided in the
3091 * `receivers` list of the retrieve response.
3092 * In case there is only one borrower from the NWd in the
3093 * transaction descriptor, record that in the `receiver_id` for
3094 * later use, and validate in the retrieve request message.
3095 * This limitation is due to the fact SPMC can't determine the
3096 * index in the memory share structures state to update.
3097 */
3098 if (to_id == HF_HYPERVISOR_VM_ID) {
3099 uint32_t other_world_count = 0;
3100
3101 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
3102 struct ffa_memory_access *receiver =
3103 ffa_memory_region_get_receiver(retrieve_request,
J-Alvesf220d572024-04-24 22:15:14 +01003104 i);
J-Alves4f0d9c12024-01-17 17:23:11 +00003105 assert(receiver != NULL);
3106
J-Alvesf220d572024-04-24 22:15:14 +01003107 if (!vm_id_is_current_world(
3108 receiver->receiver_permissions.receiver)) {
J-Alves4f0d9c12024-01-17 17:23:11 +00003109 other_world_count++;
J-Alvesf220d572024-04-24 22:15:14 +01003110 /* Set it to be used later. */
3111 to_id = receiver->receiver_permissions.receiver;
J-Alves4f0d9c12024-01-17 17:23:11 +00003112 }
3113 }
3114
3115 if (other_world_count > 1) {
3116 dlog_verbose(
J-Alves0a824e92024-04-26 16:20:12 +01003117 "Support one receiver from the other world.\n");
J-Alves4f0d9c12024-01-17 17:23:11 +00003118 return ffa_error(FFA_NOT_SUPPORTED);
3119 }
3120 }
J-Alves089004f2022-07-13 14:25:44 +01003121 /*
3122 * Check that the transaction type expected by the receiver is
3123 * correct, if it has been specified.
3124 */
3125 if (transaction_type !=
3126 FFA_MEMORY_REGION_TRANSACTION_TYPE_UNSPECIFIED &&
3127 transaction_type != (memory_region->flags &
3128 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK)) {
3129 dlog_verbose(
3130 "Incorrect transaction type %#x for "
Karl Meakine8937d92024-03-19 16:04:25 +00003131 "FFA_MEM_RETRIEVE_REQ, expected %#x for handle %#lx.\n",
J-Alves089004f2022-07-13 14:25:44 +01003132 transaction_type,
3133 memory_region->flags &
3134 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK,
3135 retrieve_request->handle);
3136 return ffa_error(FFA_INVALID_PARAMETERS);
3137 }
3138
3139 if (retrieve_request->tag != memory_region->tag) {
3140 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003141 "Incorrect tag %lu for FFA_MEM_RETRIEVE_REQ, expected "
3142 "%lu for handle %#lx.\n",
J-Alves089004f2022-07-13 14:25:44 +01003143 retrieve_request->tag, memory_region->tag,
3144 retrieve_request->handle);
3145 return ffa_error(FFA_INVALID_PARAMETERS);
3146 }
3147
J-Alves4f0d9c12024-01-17 17:23:11 +00003148 *receiver_index =
3149 ffa_memory_region_get_receiver_index(memory_region, to_id);
J-Alves089004f2022-07-13 14:25:44 +01003150
3151 if (*receiver_index == memory_region->receiver_count) {
3152 dlog_verbose(
3153 "Incorrect receiver VM ID %d for "
Karl Meakine8937d92024-03-19 16:04:25 +00003154 "FFA_MEM_RETRIEVE_REQ, for handle %#lx.\n",
J-Alves4f0d9c12024-01-17 17:23:11 +00003155 to_id, memory_region->handle);
J-Alves089004f2022-07-13 14:25:44 +01003156 return ffa_error(FFA_INVALID_PARAMETERS);
3157 }
3158
3159 if ((retrieve_request->flags &
3160 FFA_MEMORY_REGION_ADDRESS_RANGE_HINT_VALID) != 0U) {
3161 dlog_verbose(
3162 "Retriever specified 'address range alignment 'hint' "
3163 "not supported.\n");
3164 return ffa_error(FFA_INVALID_PARAMETERS);
3165 }
3166 if ((retrieve_request->flags &
3167 FFA_MEMORY_REGION_ADDRESS_RANGE_HINT_MASK) != 0) {
3168 dlog_verbose(
3169 "Bits 8-5 must be zero in memory region's flags "
3170 "(address range alignment hint not supported).\n");
3171 return ffa_error(FFA_INVALID_PARAMETERS);
3172 }
3173
3174 if ((retrieve_request->flags & ~0x7FF) != 0U) {
3175 dlog_verbose(
3176 "Bits 31-10 must be zero in memory region's flags.\n");
3177 return ffa_error(FFA_INVALID_PARAMETERS);
3178 }
3179
J-Alves95fbb312024-03-20 15:19:16 +00003180 if ((share_func == FFA_MEM_SHARE_32 ||
3181 share_func == FFA_MEM_SHARE_64) &&
J-Alves089004f2022-07-13 14:25:44 +01003182 (retrieve_request->flags &
3183 (FFA_MEMORY_REGION_FLAG_CLEAR |
3184 FFA_MEMORY_REGION_FLAG_CLEAR_RELINQUISH)) != 0U) {
3185 dlog_verbose(
3186 "Memory Share operation can't clean after relinquish "
3187 "memory region.\n");
3188 return ffa_error(FFA_INVALID_PARAMETERS);
3189 }
3190
3191 /*
3192 * If the borrower needs the memory to be cleared before mapping
3193 * to its address space, the sender should have set the flag
3194 * when calling FFA_MEM_LEND/FFA_MEM_DONATE, else return
3195 * FFA_DENIED.
3196 */
3197 if ((retrieve_request->flags & FFA_MEMORY_REGION_FLAG_CLEAR) != 0U &&
3198 (memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR) == 0U) {
3199 dlog_verbose(
3200 "Borrower needs memory cleared. Sender needs to set "
3201 "flag for clearing memory.\n");
3202 return ffa_error(FFA_DENIED);
3203 }
3204
Olivier Deprez4342a3c2022-02-28 09:37:25 +01003205 /* Memory region attributes NS-Bit MBZ for FFA_MEM_RETRIEVE_REQ. */
Karl Meakin84710f32023-10-12 15:14:49 +01003206 security_state = retrieve_request->attributes.security;
Olivier Deprez4342a3c2022-02-28 09:37:25 +01003207 if (security_state != FFA_MEMORY_SECURITY_UNSPECIFIED) {
3208 dlog_verbose(
3209 "Invalid security state for memory retrieve request "
3210 "operation.\n");
3211 return ffa_error(FFA_INVALID_PARAMETERS);
3212 }
3213
J-Alves089004f2022-07-13 14:25:44 +01003214 /*
3215 * If memory type is not specified, bypass validation of memory
3216 * attributes in the retrieve request. The retriever is expecting to
3217 * obtain this information from the SPMC.
3218 */
Karl Meakin84710f32023-10-12 15:14:49 +01003219 if (retrieve_request->attributes.type == FFA_MEMORY_NOT_SPECIFIED_MEM) {
J-Alves089004f2022-07-13 14:25:44 +01003220 return (struct ffa_value){.func = FFA_SUCCESS_32};
3221 }
3222
3223 /*
3224 * Ensure receiver's attributes are compatible with how
3225 * Hafnium maps memory: Normal Memory, Inner shareable,
3226 * Write-Back Read-Allocate Write-Allocate Cacheable.
3227 */
3228 return ffa_memory_attributes_validate(retrieve_request->attributes);
3229}
3230
J-Alves3f6527c2024-04-25 17:10:57 +01003231/**
3232 * Whilst processing the retrieve request, the operation could be aborted, and
3233 * changes to page tables and the share state structures need to be reverted.
3234 */
3235static void ffa_partition_memory_retrieve_request_undo(
3236 struct vm_locked from_locked,
3237 struct ffa_memory_share_state *share_state, uint32_t receiver_index)
3238{
3239 /*
3240 * Currently this operation is expected for operations involving the
3241 * 'other_world' vm.
3242 */
3243 assert(from_locked.vm->id == HF_OTHER_WORLD_ID);
3244 assert(share_state->retrieved_fragment_count[receiver_index] > 0);
3245
3246 /* Decrement the retrieved fragment count for the given receiver. */
3247 share_state->retrieved_fragment_count[receiver_index]--;
3248}
3249
3250/**
3251 * Whilst processing an hypervisor retrieve request the operation could be
3252 * aborted. There were no updates to PTs in this case, so decrementing the
3253 * fragment count retrieved by the hypervisor should be enough.
3254 */
3255static void ffa_hypervisor_memory_retrieve_request_undo(
3256 struct ffa_memory_share_state *share_state)
3257{
3258 assert(share_state->hypervisor_fragment_count > 0);
3259 share_state->hypervisor_fragment_count--;
3260}
3261
J-Alves4f0d9c12024-01-17 17:23:11 +00003262static struct ffa_value ffa_partition_retrieve_request(
3263 struct share_states_locked share_states,
3264 struct ffa_memory_share_state *share_state, struct vm_locked to_locked,
3265 struct ffa_memory_region *retrieve_request,
3266 uint32_t retrieve_request_length, struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003267{
Karl Meakin84710f32023-10-12 15:14:49 +01003268 ffa_memory_access_permissions_t permissions = {0};
Olivier Deprez878bd5b2021-04-15 19:05:10 +02003269 uint32_t memory_to_mode;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003270 struct ffa_value ret;
Andrew Walbranca808b12020-05-15 17:22:28 +01003271 struct ffa_composite_memory_region *composite;
3272 uint32_t total_length;
3273 uint32_t fragment_length;
J-Alves19e20cf2023-08-02 12:48:55 +01003274 ffa_id_t receiver_id = to_locked.vm->id;
J-Alves4f0d9c12024-01-17 17:23:11 +00003275 bool is_retrieve_complete = false;
J-Alves4f0d9c12024-01-17 17:23:11 +00003276 const uint64_t memory_access_desc_size =
Daniel Boulbyde974ca2023-12-12 13:53:31 +00003277 retrieve_request->memory_access_desc_size;
J-Alves4f0d9c12024-01-17 17:23:11 +00003278 uint32_t receiver_index;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003279 struct ffa_memory_access *receiver;
J-Alves4f0d9c12024-01-17 17:23:11 +00003280 ffa_memory_handle_t handle = retrieve_request->handle;
Karl Meakin84710f32023-10-12 15:14:49 +01003281 ffa_memory_attributes_t attributes = {0};
J-Alves460d36c2023-10-12 17:02:15 +01003282 uint32_t retrieve_mode = 0;
J-Alves4f0d9c12024-01-17 17:23:11 +00003283 struct ffa_memory_region *memory_region = share_state->memory_region;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003284
J-Alves96de29f2022-04-26 16:05:24 +01003285 if (!share_state->sending_complete) {
3286 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003287 "Memory with handle %#lx not fully sent, can't "
J-Alves96de29f2022-04-26 16:05:24 +01003288 "retrieve.\n",
3289 handle);
J-Alves4f0d9c12024-01-17 17:23:11 +00003290 return ffa_error(FFA_INVALID_PARAMETERS);
J-Alves96de29f2022-04-26 16:05:24 +01003291 }
3292
J-Alves4f0d9c12024-01-17 17:23:11 +00003293 /*
3294 * Validate retrieve request, according to what was sent by the
3295 * sender. Function will output the `receiver_index` from the
3296 * provided memory region.
3297 */
3298 ret = ffa_memory_retrieve_validate(
3299 receiver_id, retrieve_request, retrieve_request_length,
3300 memory_region, &receiver_index, share_state->share_func);
J-Alves089004f2022-07-13 14:25:44 +01003301
J-Alves4f0d9c12024-01-17 17:23:11 +00003302 if (ret.func != FFA_SUCCESS_32) {
3303 return ret;
J-Alves089004f2022-07-13 14:25:44 +01003304 }
J-Alves96de29f2022-04-26 16:05:24 +01003305
J-Alves4f0d9c12024-01-17 17:23:11 +00003306 /*
3307 * Validate the requested permissions against the sent
3308 * permissions.
3309 * Outputs the permissions to give to retriever at S2
3310 * PTs.
3311 */
3312 ret = ffa_memory_retrieve_validate_memory_access_list(
3313 memory_region, retrieve_request, receiver_id, &permissions,
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003314 &receiver, share_state->share_func);
J-Alves4f0d9c12024-01-17 17:23:11 +00003315 if (ret.func != FFA_SUCCESS_32) {
3316 return ret;
3317 }
3318
3319 memory_to_mode = ffa_memory_permissions_to_mode(
3320 permissions, share_state->sender_orig_mode);
3321
3322 ret = ffa_retrieve_check_update(
3323 to_locked, share_state->fragments,
3324 share_state->fragment_constituent_counts,
3325 share_state->fragment_count, memory_to_mode,
J-Alves460d36c2023-10-12 17:02:15 +01003326 share_state->share_func, false, page_pool, &retrieve_mode,
3327 share_state->memory_protected);
J-Alves4f0d9c12024-01-17 17:23:11 +00003328
3329 if (ret.func != FFA_SUCCESS_32) {
3330 return ret;
3331 }
3332
3333 share_state->retrieved_fragment_count[receiver_index] = 1;
3334
3335 is_retrieve_complete =
3336 share_state->retrieved_fragment_count[receiver_index] ==
3337 share_state->fragment_count;
3338
J-Alvesb5084cf2022-07-06 14:20:12 +01003339 /* VMs acquire the RX buffer from SPMC. */
3340 CHECK(plat_ffa_acquire_receiver_rx(to_locked, &ret));
3341
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003342 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01003343 * Copy response to RX buffer of caller and deliver the message.
3344 * This must be done before the share_state is (possibly) freed.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003345 */
Andrew Walbranca808b12020-05-15 17:22:28 +01003346 composite = ffa_memory_region_get_composite(memory_region, 0);
J-Alves4f0d9c12024-01-17 17:23:11 +00003347
Andrew Walbranca808b12020-05-15 17:22:28 +01003348 /*
J-Alves460d36c2023-10-12 17:02:15 +01003349 * Set the security state in the memory retrieve response attributes
3350 * if specified by the target mode.
3351 */
3352 attributes = plat_ffa_memory_security_mode(memory_region->attributes,
3353 retrieve_mode);
3354
3355 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01003356 * Constituents which we received in the first fragment should
3357 * always fit in the first fragment we are sending, because the
3358 * header is the same size in both cases and we have a fixed
3359 * message buffer size. So `ffa_retrieved_memory_region_init`
3360 * should never fail.
Andrew Walbranca808b12020-05-15 17:22:28 +01003361 */
Olivier Deprez878bd5b2021-04-15 19:05:10 +02003362
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003363 /* Provide the permissions that had been provided. */
3364 receiver->receiver_permissions.permissions = permissions;
3365
3366 /*
3367 * Prepare the memory region descriptor for the retrieve response.
3368 * Provide the pointer to the receiver tracked in the share state
J-Alves7b9cc432024-04-04 10:57:17 +01003369 * structures.
3370 * At this point the retrieve request descriptor from the partition
3371 * has been processed. The `retrieve_request` is expected to be in
3372 * a region that is handled by the SPMC/Hyp. Reuse the same buffer to
3373 * prepare the retrieve response before copying it to the RX buffer of
3374 * the caller.
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003375 */
Andrew Walbranca808b12020-05-15 17:22:28 +01003376 CHECK(ffa_retrieved_memory_region_init(
J-Alves7b9cc432024-04-04 10:57:17 +01003377 retrieve_request, to_locked.vm->ffa_version, HF_MAILBOX_SIZE,
3378 memory_region->sender, attributes, memory_region->flags, handle,
3379 permissions, receiver, 1, memory_access_desc_size,
3380 composite->page_count, composite->constituent_count,
3381 share_state->fragments[0],
Andrew Walbranca808b12020-05-15 17:22:28 +01003382 share_state->fragment_constituent_counts[0], &total_length,
3383 &fragment_length));
J-Alvesb5084cf2022-07-06 14:20:12 +01003384
J-Alves7b9cc432024-04-04 10:57:17 +01003385 /*
3386 * Copy the message from the buffer into the partition's mailbox.
3387 * The operation might fail unexpectedly due to change in PAS address
3388 * space, or improper values to the sizes of the structures.
3389 */
3390 if (!memcpy_trapped(to_locked.vm->mailbox.recv, HF_MAILBOX_SIZE,
3391 retrieve_request, fragment_length)) {
3392 dlog_error(
3393 "%s: aborted the copy of response to RX buffer of "
3394 "%x.\n",
3395 __func__, to_locked.vm->id);
J-Alves3f6527c2024-04-25 17:10:57 +01003396
3397 ffa_partition_memory_retrieve_request_undo(
3398 to_locked, share_state, receiver_index);
3399
J-Alves7b9cc432024-04-04 10:57:17 +01003400 return ffa_error(FFA_ABORTED);
3401 }
3402
J-Alves4f0d9c12024-01-17 17:23:11 +00003403 if (is_retrieve_complete) {
Andrew Walbranca808b12020-05-15 17:22:28 +01003404 ffa_memory_retrieve_complete(share_states, share_state,
3405 page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003406 }
J-Alves4f0d9c12024-01-17 17:23:11 +00003407
3408 return ffa_memory_retrieve_resp(total_length, fragment_length);
3409}
3410
3411static struct ffa_value ffa_hypervisor_retrieve_request(
3412 struct ffa_memory_share_state *share_state, struct vm_locked to_locked,
3413 struct ffa_memory_region *retrieve_request)
3414{
3415 struct ffa_value ret;
3416 struct ffa_composite_memory_region *composite;
3417 uint32_t total_length;
3418 uint32_t fragment_length;
J-Alves4f0d9c12024-01-17 17:23:11 +00003419 ffa_memory_attributes_t attributes;
J-Alves7b6ab612024-01-24 09:54:54 +00003420 uint64_t memory_access_desc_size;
J-Alves4f0d9c12024-01-17 17:23:11 +00003421 struct ffa_memory_region *memory_region;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003422 struct ffa_memory_access *receiver;
J-Alves4f0d9c12024-01-17 17:23:11 +00003423 ffa_memory_handle_t handle = retrieve_request->handle;
3424
J-Alves4f0d9c12024-01-17 17:23:11 +00003425 memory_region = share_state->memory_region;
3426
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003427 assert(to_locked.vm->id == HF_HYPERVISOR_VM_ID);
3428
J-Alves7b6ab612024-01-24 09:54:54 +00003429 switch (to_locked.vm->ffa_version) {
Karl Meakin0e617d92024-04-05 12:55:22 +01003430 case FFA_VERSION_1_2:
J-Alves7b6ab612024-01-24 09:54:54 +00003431 memory_access_desc_size = sizeof(struct ffa_memory_access);
3432 break;
Karl Meakin0e617d92024-04-05 12:55:22 +01003433 case FFA_VERSION_1_0:
3434 case FFA_VERSION_1_1:
J-Alves7b6ab612024-01-24 09:54:54 +00003435 memory_access_desc_size = sizeof(struct ffa_memory_access_v1_0);
3436 break;
3437 default:
3438 panic("version not supported: %x\n", to_locked.vm->ffa_version);
3439 }
3440
J-Alves4f0d9c12024-01-17 17:23:11 +00003441 if (share_state->hypervisor_fragment_count != 0U) {
3442 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003443 "Memory with handle %#lx already retrieved by "
J-Alves4f0d9c12024-01-17 17:23:11 +00003444 "the hypervisor.\n",
3445 handle);
3446 return ffa_error(FFA_DENIED);
3447 }
3448
3449 share_state->hypervisor_fragment_count = 1;
3450
J-Alves4f0d9c12024-01-17 17:23:11 +00003451 /* VMs acquire the RX buffer from SPMC. */
3452 CHECK(plat_ffa_acquire_receiver_rx(to_locked, &ret));
3453
3454 /*
3455 * Copy response to RX buffer of caller and deliver the message.
3456 * This must be done before the share_state is (possibly) freed.
3457 */
3458 composite = ffa_memory_region_get_composite(memory_region, 0);
3459
3460 /*
3461 * Constituents which we received in the first fragment should
3462 * always fit in the first fragment we are sending, because the
3463 * header is the same size in both cases and we have a fixed
3464 * message buffer size. So `ffa_retrieved_memory_region_init`
3465 * should never fail.
3466 */
3467
3468 /*
3469 * Set the security state in the memory retrieve response attributes
3470 * if specified by the target mode.
3471 */
3472 attributes = plat_ffa_memory_security_mode(
3473 memory_region->attributes, share_state->sender_orig_mode);
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003474
3475 receiver = ffa_memory_region_get_receiver(memory_region, 0);
3476
J-Alves7b9cc432024-04-04 10:57:17 +01003477 /*
3478 * At this point the `retrieve_request` is expected to be in a section
3479 * managed by the hypervisor.
3480 */
J-Alves4f0d9c12024-01-17 17:23:11 +00003481 CHECK(ffa_retrieved_memory_region_init(
J-Alves7b9cc432024-04-04 10:57:17 +01003482 retrieve_request, to_locked.vm->ffa_version, HF_MAILBOX_SIZE,
3483 memory_region->sender, attributes, memory_region->flags, handle,
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003484 receiver->receiver_permissions.permissions, receiver,
3485 memory_region->receiver_count, memory_access_desc_size,
J-Alves4f0d9c12024-01-17 17:23:11 +00003486 composite->page_count, composite->constituent_count,
3487 share_state->fragments[0],
3488 share_state->fragment_constituent_counts[0], &total_length,
3489 &fragment_length));
3490
J-Alves7b9cc432024-04-04 10:57:17 +01003491 /*
3492 * Copy the message from the buffer into the hypervisor's mailbox.
3493 * The operation might fail unexpectedly due to change in PAS, or
3494 * improper values for the sizes of the structures.
3495 */
3496 if (!memcpy_trapped(to_locked.vm->mailbox.recv, HF_MAILBOX_SIZE,
3497 retrieve_request, fragment_length)) {
3498 dlog_error(
3499 "%s: aborted the copy of response to RX buffer of "
3500 "%x.\n",
3501 __func__, to_locked.vm->id);
J-Alves3f6527c2024-04-25 17:10:57 +01003502
3503 ffa_hypervisor_memory_retrieve_request_undo(share_state);
3504
J-Alves7b9cc432024-04-04 10:57:17 +01003505 return ffa_error(FFA_ABORTED);
3506 }
3507
J-Alves3f6527c2024-04-25 17:10:57 +01003508 ffa_memory_retrieve_complete_from_hyp(share_state);
3509
J-Alves4f0d9c12024-01-17 17:23:11 +00003510 return ffa_memory_retrieve_resp(total_length, fragment_length);
3511}
3512
3513struct ffa_value ffa_memory_retrieve(struct vm_locked to_locked,
3514 struct ffa_memory_region *retrieve_request,
3515 uint32_t retrieve_request_length,
3516 struct mpool *page_pool)
3517{
3518 ffa_memory_handle_t handle = retrieve_request->handle;
3519 struct share_states_locked share_states;
3520 struct ffa_memory_share_state *share_state;
3521 struct ffa_value ret;
3522
3523 dump_share_states();
3524
3525 share_states = share_states_lock();
3526 share_state = get_share_state(share_states, handle);
3527 if (share_state == NULL) {
Karl Meakine8937d92024-03-19 16:04:25 +00003528 dlog_verbose("Invalid handle %#lx for FFA_MEM_RETRIEVE_REQ.\n",
J-Alves4f0d9c12024-01-17 17:23:11 +00003529 handle);
3530 ret = ffa_error(FFA_INVALID_PARAMETERS);
3531 goto out;
3532 }
3533
3534 if (is_ffa_hypervisor_retrieve_request(retrieve_request, to_locked)) {
3535 ret = ffa_hypervisor_retrieve_request(share_state, to_locked,
3536 retrieve_request);
3537 } else {
3538 ret = ffa_partition_retrieve_request(
3539 share_states, share_state, to_locked, retrieve_request,
3540 retrieve_request_length, page_pool);
3541 }
3542
3543 /* Track use of the RX buffer if the handling has succeeded. */
3544 if (ret.func == FFA_MEM_RETRIEVE_RESP_32) {
3545 to_locked.vm->mailbox.recv_func = FFA_MEM_RETRIEVE_RESP_32;
3546 to_locked.vm->mailbox.state = MAILBOX_STATE_FULL;
3547 }
3548
Andrew Walbranca808b12020-05-15 17:22:28 +01003549out:
3550 share_states_unlock(&share_states);
3551 dump_share_states();
3552 return ret;
3553}
3554
J-Alves5da37d92022-10-24 16:33:48 +01003555/**
3556 * Determine expected fragment offset according to the FF-A version of
3557 * the caller.
3558 */
3559static uint32_t ffa_memory_retrieve_expected_offset_per_ffa_version(
3560 struct ffa_memory_region *memory_region,
Karl Meakin0e617d92024-04-05 12:55:22 +01003561 uint32_t retrieved_constituents_count, enum ffa_version ffa_version)
J-Alves5da37d92022-10-24 16:33:48 +01003562{
3563 uint32_t expected_fragment_offset;
3564 uint32_t composite_constituents_offset;
3565
Karl Meakin0e617d92024-04-05 12:55:22 +01003566 if (ffa_version >= FFA_VERSION_1_1) {
J-Alves5da37d92022-10-24 16:33:48 +01003567 /*
3568 * Hafnium operates memory regions in FF-A v1.1 format, so we
3569 * can retrieve the constituents offset from descriptor.
3570 */
3571 composite_constituents_offset =
3572 ffa_composite_constituent_offset(memory_region, 0);
Karl Meakin0e617d92024-04-05 12:55:22 +01003573 } else if (ffa_version == FFA_VERSION_1_0) {
J-Alves5da37d92022-10-24 16:33:48 +01003574 /*
3575 * If retriever is FF-A v1.0, determine the composite offset
3576 * as it is expected to have been configured in the
3577 * retrieve response.
3578 */
3579 composite_constituents_offset =
3580 sizeof(struct ffa_memory_region_v1_0) +
3581 RECEIVERS_COUNT_IN_RETRIEVE_RESP *
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003582 sizeof(struct ffa_memory_access_v1_0) +
J-Alves5da37d92022-10-24 16:33:48 +01003583 sizeof(struct ffa_composite_memory_region);
3584 } else {
3585 panic("%s received an invalid FF-A version.\n", __func__);
3586 }
3587
3588 expected_fragment_offset =
3589 composite_constituents_offset +
3590 retrieved_constituents_count *
3591 sizeof(struct ffa_memory_region_constituent) -
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003592 (uint32_t)(memory_region->memory_access_desc_size *
3593 (memory_region->receiver_count - 1));
J-Alves5da37d92022-10-24 16:33:48 +01003594
3595 return expected_fragment_offset;
3596}
3597
Andrew Walbranca808b12020-05-15 17:22:28 +01003598struct ffa_value ffa_memory_retrieve_continue(struct vm_locked to_locked,
3599 ffa_memory_handle_t handle,
3600 uint32_t fragment_offset,
J-Alves19e20cf2023-08-02 12:48:55 +01003601 ffa_id_t sender_vm_id,
Andrew Walbranca808b12020-05-15 17:22:28 +01003602 struct mpool *page_pool)
3603{
3604 struct ffa_memory_region *memory_region;
3605 struct share_states_locked share_states;
3606 struct ffa_memory_share_state *share_state;
3607 struct ffa_value ret;
3608 uint32_t fragment_index;
3609 uint32_t retrieved_constituents_count;
3610 uint32_t i;
3611 uint32_t expected_fragment_offset;
3612 uint32_t remaining_constituent_count;
3613 uint32_t fragment_length;
J-Alvesc7484f12022-05-13 12:41:14 +01003614 uint32_t receiver_index;
J-Alves59ed0042022-07-28 18:26:41 +01003615 bool continue_ffa_hyp_mem_retrieve_req;
Andrew Walbranca808b12020-05-15 17:22:28 +01003616
3617 dump_share_states();
3618
3619 share_states = share_states_lock();
Karl Meakin4a2854a2023-06-30 16:26:52 +01003620 share_state = get_share_state(share_states, handle);
J-Alvesb56aac82023-11-10 09:44:43 +00003621 if (share_state == NULL) {
Karl Meakine8937d92024-03-19 16:04:25 +00003622 dlog_verbose("Invalid handle %#lx for FFA_MEM_FRAG_RX.\n",
Andrew Walbranca808b12020-05-15 17:22:28 +01003623 handle);
3624 ret = ffa_error(FFA_INVALID_PARAMETERS);
3625 goto out;
3626 }
3627
3628 memory_region = share_state->memory_region;
3629 CHECK(memory_region != NULL);
3630
Andrew Walbranca808b12020-05-15 17:22:28 +01003631 if (!share_state->sending_complete) {
3632 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003633 "Memory with handle %#lx not fully sent, can't "
Andrew Walbranca808b12020-05-15 17:22:28 +01003634 "retrieve.\n",
3635 handle);
3636 ret = ffa_error(FFA_INVALID_PARAMETERS);
3637 goto out;
3638 }
3639
J-Alves59ed0042022-07-28 18:26:41 +01003640 /*
3641 * If retrieve request from the hypervisor has been initiated in the
3642 * given share_state, continue it, else assume it is a continuation of
3643 * retrieve request from a NWd VM.
3644 */
3645 continue_ffa_hyp_mem_retrieve_req =
3646 (to_locked.vm->id == HF_HYPERVISOR_VM_ID) &&
3647 (share_state->hypervisor_fragment_count != 0U) &&
J-Alves661e1b72023-08-02 13:39:40 +01003648 ffa_is_vm_id(sender_vm_id);
Andrew Walbranca808b12020-05-15 17:22:28 +01003649
J-Alves59ed0042022-07-28 18:26:41 +01003650 if (!continue_ffa_hyp_mem_retrieve_req) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003651 receiver_index = ffa_memory_region_get_receiver_index(
J-Alves59ed0042022-07-28 18:26:41 +01003652 memory_region, to_locked.vm->id);
3653
3654 if (receiver_index == memory_region->receiver_count) {
3655 dlog_verbose(
3656 "Caller of FFA_MEM_FRAG_RX (%x) is not a "
Karl Meakine8937d92024-03-19 16:04:25 +00003657 "borrower to memory sharing transaction "
3658 "(%lx)\n",
J-Alves59ed0042022-07-28 18:26:41 +01003659 to_locked.vm->id, handle);
3660 ret = ffa_error(FFA_INVALID_PARAMETERS);
3661 goto out;
3662 }
3663
3664 if (share_state->retrieved_fragment_count[receiver_index] ==
3665 0 ||
3666 share_state->retrieved_fragment_count[receiver_index] >=
3667 share_state->fragment_count) {
3668 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003669 "Retrieval of memory with handle %#lx not yet "
J-Alves59ed0042022-07-28 18:26:41 +01003670 "started or already completed (%d/%d fragments "
3671 "retrieved).\n",
3672 handle,
3673 share_state->retrieved_fragment_count
3674 [receiver_index],
3675 share_state->fragment_count);
3676 ret = ffa_error(FFA_INVALID_PARAMETERS);
3677 goto out;
3678 }
3679
3680 fragment_index =
3681 share_state->retrieved_fragment_count[receiver_index];
3682 } else {
3683 if (share_state->hypervisor_fragment_count == 0 ||
3684 share_state->hypervisor_fragment_count >=
3685 share_state->fragment_count) {
3686 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003687 "Retrieve of memory with handle %lx not "
J-Alves59ed0042022-07-28 18:26:41 +01003688 "started from hypervisor.\n",
3689 handle);
3690 ret = ffa_error(FFA_INVALID_PARAMETERS);
3691 goto out;
3692 }
3693
3694 if (memory_region->sender != sender_vm_id) {
3695 dlog_verbose(
3696 "Sender ID (%x) is not as expected for memory "
Karl Meakine8937d92024-03-19 16:04:25 +00003697 "handle %lx\n",
J-Alves59ed0042022-07-28 18:26:41 +01003698 sender_vm_id, handle);
3699 ret = ffa_error(FFA_INVALID_PARAMETERS);
3700 goto out;
3701 }
3702
3703 fragment_index = share_state->hypervisor_fragment_count;
3704
3705 receiver_index = 0;
3706 }
Andrew Walbranca808b12020-05-15 17:22:28 +01003707
3708 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01003709 * Check that the given fragment offset is correct by counting
3710 * how many constituents were in the fragments previously sent.
Andrew Walbranca808b12020-05-15 17:22:28 +01003711 */
3712 retrieved_constituents_count = 0;
3713 for (i = 0; i < fragment_index; ++i) {
3714 retrieved_constituents_count +=
3715 share_state->fragment_constituent_counts[i];
3716 }
J-Alvesc7484f12022-05-13 12:41:14 +01003717
3718 CHECK(memory_region->receiver_count > 0);
3719
Andrew Walbranca808b12020-05-15 17:22:28 +01003720 expected_fragment_offset =
J-Alves5da37d92022-10-24 16:33:48 +01003721 ffa_memory_retrieve_expected_offset_per_ffa_version(
3722 memory_region, retrieved_constituents_count,
3723 to_locked.vm->ffa_version);
3724
Andrew Walbranca808b12020-05-15 17:22:28 +01003725 if (fragment_offset != expected_fragment_offset) {
3726 dlog_verbose("Fragment offset was %d but expected %d.\n",
3727 fragment_offset, expected_fragment_offset);
3728 ret = ffa_error(FFA_INVALID_PARAMETERS);
3729 goto out;
3730 }
3731
J-Alves4f0d9c12024-01-17 17:23:11 +00003732 /*
3733 * When hafnium is the hypervisor, acquire the RX buffer of a VM, that
3734 * is currently ownder by the SPMC.
3735 */
3736 assert(plat_ffa_acquire_receiver_rx(to_locked, &ret));
J-Alves59ed0042022-07-28 18:26:41 +01003737
Andrew Walbranca808b12020-05-15 17:22:28 +01003738 remaining_constituent_count = ffa_memory_fragment_init(
3739 to_locked.vm->mailbox.recv, HF_MAILBOX_SIZE,
3740 share_state->fragments[fragment_index],
3741 share_state->fragment_constituent_counts[fragment_index],
3742 &fragment_length);
3743 CHECK(remaining_constituent_count == 0);
J-Alves674e4de2024-01-17 16:20:32 +00003744
Andrew Walbranca808b12020-05-15 17:22:28 +01003745 to_locked.vm->mailbox.recv_func = FFA_MEM_FRAG_TX_32;
J-Alvese8c8c2b2022-12-16 15:34:48 +00003746 to_locked.vm->mailbox.state = MAILBOX_STATE_FULL;
Andrew Walbranca808b12020-05-15 17:22:28 +01003747
J-Alves59ed0042022-07-28 18:26:41 +01003748 if (!continue_ffa_hyp_mem_retrieve_req) {
3749 share_state->retrieved_fragment_count[receiver_index]++;
3750 if (share_state->retrieved_fragment_count[receiver_index] ==
3751 share_state->fragment_count) {
3752 ffa_memory_retrieve_complete(share_states, share_state,
3753 page_pool);
3754 }
3755 } else {
3756 share_state->hypervisor_fragment_count++;
3757
3758 ffa_memory_retrieve_complete_from_hyp(share_state);
3759 }
Andrew Walbranca808b12020-05-15 17:22:28 +01003760 ret = (struct ffa_value){.func = FFA_MEM_FRAG_TX_32,
3761 .arg1 = (uint32_t)handle,
3762 .arg2 = (uint32_t)(handle >> 32),
3763 .arg3 = fragment_length};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003764
3765out:
3766 share_states_unlock(&share_states);
3767 dump_share_states();
3768 return ret;
3769}
3770
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003771struct ffa_value ffa_memory_relinquish(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003772 struct vm_locked from_locked,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003773 struct ffa_mem_relinquish *relinquish_request, struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003774{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003775 ffa_memory_handle_t handle = relinquish_request->handle;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003776 struct share_states_locked share_states;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003777 struct ffa_memory_share_state *share_state;
3778 struct ffa_memory_region *memory_region;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003779 bool clear;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003780 struct ffa_value ret;
J-Alves8eb19162022-04-28 10:56:48 +01003781 uint32_t receiver_index;
J-Alves3c5b2072022-11-21 12:45:40 +00003782 bool receivers_relinquished_memory;
Karl Meakin84710f32023-10-12 15:14:49 +01003783 ffa_memory_access_permissions_t receiver_permissions = {0};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003784
Andrew Walbrana65a1322020-04-06 19:32:32 +01003785 if (relinquish_request->endpoint_count != 1) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003786 dlog_verbose(
J-Alves0a824e92024-04-26 16:20:12 +01003787 "Stream endpoints not supported (got %d endpoints on "
3788 "FFA_MEM_RELINQUISH, expected 1).\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003789 relinquish_request->endpoint_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003790 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003791 }
3792
J-Alvesbd060342024-04-26 18:44:31 +01003793 if (vm_id_is_current_world(from_locked.vm->id) &&
3794 relinquish_request->endpoints[0] != from_locked.vm->id) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003795 dlog_verbose(
J-Alves0a824e92024-04-26 16:20:12 +01003796 "VM ID %d in relinquish message doesn't match calling "
3797 "VM ID %d.\n",
Andrew Walbrana65a1322020-04-06 19:32:32 +01003798 relinquish_request->endpoints[0], from_locked.vm->id);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003799 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003800 }
3801
3802 dump_share_states();
3803
3804 share_states = share_states_lock();
Karl Meakin4a2854a2023-06-30 16:26:52 +01003805 share_state = get_share_state(share_states, handle);
J-Alvesb56aac82023-11-10 09:44:43 +00003806 if (share_state == NULL) {
Karl Meakine8937d92024-03-19 16:04:25 +00003807 dlog_verbose("Invalid handle %#lx for FFA_MEM_RELINQUISH.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003808 handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003809 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003810 goto out;
3811 }
3812
Andrew Walbranca808b12020-05-15 17:22:28 +01003813 if (!share_state->sending_complete) {
3814 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003815 "Memory with handle %#lx not fully sent, can't "
Andrew Walbranca808b12020-05-15 17:22:28 +01003816 "relinquish.\n",
3817 handle);
3818 ret = ffa_error(FFA_INVALID_PARAMETERS);
3819 goto out;
3820 }
3821
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003822 memory_region = share_state->memory_region;
3823 CHECK(memory_region != NULL);
3824
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003825 receiver_index = ffa_memory_region_get_receiver_index(
J-Alvesbd060342024-04-26 18:44:31 +01003826 memory_region, relinquish_request->endpoints[0]);
J-Alves8eb19162022-04-28 10:56:48 +01003827
3828 if (receiver_index == memory_region->receiver_count) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003829 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01003830 "VM ID %d tried to relinquish memory region "
Karl Meakine8937d92024-03-19 16:04:25 +00003831 "with handle %#lx and it is not a valid borrower.\n",
J-Alves8eb19162022-04-28 10:56:48 +01003832 from_locked.vm->id, handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003833 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003834 goto out;
3835 }
3836
J-Alves8eb19162022-04-28 10:56:48 +01003837 if (share_state->retrieved_fragment_count[receiver_index] !=
Andrew Walbranca808b12020-05-15 17:22:28 +01003838 share_state->fragment_count) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003839 dlog_verbose(
J-Alves0a824e92024-04-26 16:20:12 +01003840 "Memory with handle %#lx not yet fully retrieved, "
J-Alves8eb19162022-04-28 10:56:48 +01003841 "receiver %x can't relinquish.\n",
3842 handle, from_locked.vm->id);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003843 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003844 goto out;
3845 }
3846
J-Alves3c5b2072022-11-21 12:45:40 +00003847 /*
3848 * Either clear if requested in relinquish call, or in a retrieve
3849 * request from one of the borrowers.
3850 */
3851 receivers_relinquished_memory = true;
3852
3853 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
3854 struct ffa_memory_access *receiver =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003855 ffa_memory_region_get_receiver(memory_region, i);
3856 assert(receiver != NULL);
J-Alves3c5b2072022-11-21 12:45:40 +00003857 if (receiver->receiver_permissions.receiver ==
3858 from_locked.vm->id) {
J-Alves639ddfc2023-11-21 14:17:26 +00003859 receiver_permissions =
3860 receiver->receiver_permissions.permissions;
J-Alves3c5b2072022-11-21 12:45:40 +00003861 continue;
3862 }
3863
3864 if (share_state->retrieved_fragment_count[i] != 0U) {
3865 receivers_relinquished_memory = false;
3866 break;
3867 }
3868 }
3869
3870 clear = receivers_relinquished_memory &&
Daniel Boulby2e14ebe2024-01-15 16:21:44 +00003871 ((relinquish_request->flags & FFA_MEMORY_REGION_FLAG_CLEAR) !=
3872 0U);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003873
3874 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01003875 * Clear is not allowed for memory that was shared, as the
3876 * original sender still has access to the memory.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003877 */
J-Alves95fbb312024-03-20 15:19:16 +00003878 if (clear && (share_state->share_func == FFA_MEM_SHARE_32 ||
3879 share_state->share_func == FFA_MEM_SHARE_64)) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003880 dlog_verbose("Memory which was shared can't be cleared.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003881 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003882 goto out;
3883 }
3884
J-Alvesb886d492024-04-15 10:55:29 +01003885 if (clear && receiver_permissions.data_access == FFA_DATA_ACCESS_RO) {
J-Alves639ddfc2023-11-21 14:17:26 +00003886 dlog_verbose("%s: RO memory can't use clear memory flag.\n",
3887 __func__);
3888 ret = ffa_error(FFA_DENIED);
3889 goto out;
3890 }
3891
Andrew Walbranca808b12020-05-15 17:22:28 +01003892 ret = ffa_relinquish_check_update(
J-Alves26483382023-04-20 12:01:49 +01003893 from_locked, share_state->fragments,
Andrew Walbranca808b12020-05-15 17:22:28 +01003894 share_state->fragment_constituent_counts,
J-Alves69cdfd92024-04-26 11:40:59 +01003895 share_state->fragment_count, share_state->sender_orig_mode,
3896 page_pool, clear);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003897
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003898 if (ret.func == FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003899 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01003900 * Mark memory handle as not retrieved, so it can be
3901 * reclaimed (or retrieved again).
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003902 */
J-Alves8eb19162022-04-28 10:56:48 +01003903 share_state->retrieved_fragment_count[receiver_index] = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003904 }
3905
3906out:
3907 share_states_unlock(&share_states);
3908 dump_share_states();
3909 return ret;
3910}
3911
3912/**
J-Alvesa9cd7e32022-07-01 13:49:33 +01003913 * Validates that the reclaim transition is allowed for the given
3914 * handle, updates the page table of the reclaiming VM, and frees the
3915 * internal state associated with the handle.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003916 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003917struct ffa_value ffa_memory_reclaim(struct vm_locked to_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01003918 ffa_memory_handle_t handle,
3919 ffa_memory_region_flags_t flags,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003920 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003921{
3922 struct share_states_locked share_states;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003923 struct ffa_memory_share_state *share_state;
3924 struct ffa_memory_region *memory_region;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003925 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003926
3927 dump_share_states();
3928
3929 share_states = share_states_lock();
Karl Meakin52cdfe72023-06-30 14:49:10 +01003930
Karl Meakin4a2854a2023-06-30 16:26:52 +01003931 share_state = get_share_state(share_states, handle);
J-Alvesb56aac82023-11-10 09:44:43 +00003932 if (share_state == NULL) {
Karl Meakine8937d92024-03-19 16:04:25 +00003933 dlog_verbose("Invalid handle %#lx for FFA_MEM_RECLAIM.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003934 handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003935 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003936 goto out;
3937 }
Karl Meakin4a2854a2023-06-30 16:26:52 +01003938 memory_region = share_state->memory_region;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003939
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003940 CHECK(memory_region != NULL);
3941
J-Alvesa9cd7e32022-07-01 13:49:33 +01003942 if (vm_id_is_current_world(to_locked.vm->id) &&
3943 to_locked.vm->id != memory_region->sender) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003944 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003945 "VM %#x attempted to reclaim memory handle %#lx "
Olivier Deprezf92e5d42020-11-13 16:00:54 +01003946 "originally sent by VM %#x.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003947 to_locked.vm->id, handle, memory_region->sender);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003948 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003949 goto out;
3950 }
3951
Andrew Walbranca808b12020-05-15 17:22:28 +01003952 if (!share_state->sending_complete) {
3953 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003954 "Memory with handle %#lx not fully sent, can't "
Andrew Walbranca808b12020-05-15 17:22:28 +01003955 "reclaim.\n",
3956 handle);
3957 ret = ffa_error(FFA_INVALID_PARAMETERS);
3958 goto out;
3959 }
3960
J-Alves752236c2022-04-28 11:07:47 +01003961 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
3962 if (share_state->retrieved_fragment_count[i] != 0) {
J-Alves9bbcb872024-04-25 17:19:00 +01003963 struct ffa_memory_access *receiver =
3964 ffa_memory_region_get_receiver(memory_region,
3965 i);
3966
3967 assert(receiver != NULL);
3968 (void)receiver;
J-Alves752236c2022-04-28 11:07:47 +01003969 dlog_verbose(
J-Alves0a824e92024-04-26 16:20:12 +01003970 "Tried to reclaim memory handle %#lx that has "
3971 "not been relinquished by all borrowers(%x).\n",
J-Alves752236c2022-04-28 11:07:47 +01003972 handle,
J-Alves9bbcb872024-04-25 17:19:00 +01003973 receiver->receiver_permissions.receiver);
J-Alves752236c2022-04-28 11:07:47 +01003974 ret = ffa_error(FFA_DENIED);
3975 goto out;
3976 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003977 }
3978
Andrew Walbranca808b12020-05-15 17:22:28 +01003979 ret = ffa_retrieve_check_update(
J-Alves26483382023-04-20 12:01:49 +01003980 to_locked, share_state->fragments,
Andrew Walbranca808b12020-05-15 17:22:28 +01003981 share_state->fragment_constituent_counts,
J-Alves2a0d2882020-10-29 14:49:50 +00003982 share_state->fragment_count, share_state->sender_orig_mode,
J-Alves460d36c2023-10-12 17:02:15 +01003983 FFA_MEM_RECLAIM_32, flags & FFA_MEM_RECLAIM_CLEAR, page_pool,
J-Alvesfd206052023-05-22 16:45:00 +01003984 NULL, share_state->memory_protected);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003985
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003986 if (ret.func == FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003987 share_state_free(share_states, share_state, page_pool);
J-Alves3c5b2072022-11-21 12:45:40 +00003988 dlog_verbose("Freed share state after successful reclaim.\n");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003989 }
3990
3991out:
3992 share_states_unlock(&share_states);
3993 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01003994}