blob: 3932201ddff3807e186bc20713d8a7211011b631 [file] [log] [blame]
Jose Marinho75509b42019-04-09 09:34:59 +01001/*
2 * Copyright 2019 The Hafnium Authors.
3 *
Andrew Walbrane959ec12020-06-17 15:01:09 +01004 * Use of this source code is governed by a BSD-style
5 * license that can be found in the LICENSE file or at
6 * https://opensource.org/licenses/BSD-3-Clause.
Jose Marinho75509b42019-04-09 09:34:59 +01007 */
8
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01009#include "hf/ffa_memory.h"
Andrew Walbran475c1452020-02-07 13:22:22 +000010
J-Alves7b9cc432024-04-04 10:57:17 +010011#include "hf/arch/memcpy_trapped.h"
Federico Recanati4fd065d2021-12-13 20:06:23 +010012#include "hf/arch/mm.h"
Olivier Deprez112d2b52020-09-30 07:39:23 +020013#include "hf/arch/other_world.h"
Olivier Deprez55a189e2021-06-09 15:45:27 +020014#include "hf/arch/plat/ffa.h"
Andrew Walbran290b0c92020-02-03 16:37:14 +000015
J-Alves5952d942022-12-22 16:03:00 +000016#include "hf/addr.h"
Jose Marinho75509b42019-04-09 09:34:59 +010017#include "hf/api.h"
Daniel Boulbya2f8c662021-11-26 17:52:53 +000018#include "hf/assert.h"
Jose Marinho09b1db82019-08-08 09:16:59 +010019#include "hf/check.h"
Jose Marinho75509b42019-04-09 09:34:59 +010020#include "hf/dlog.h"
J-Alves3456e032023-07-20 12:20:05 +010021#include "hf/ffa.h"
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010022#include "hf/ffa_internal.h"
J-Alves66652252022-07-06 09:49:51 +010023#include "hf/ffa_memory_internal.h"
J-Alves3456e032023-07-20 12:20:05 +010024#include "hf/ffa_partition_manifest.h"
J-Alves5952d942022-12-22 16:03:00 +000025#include "hf/mm.h"
Andrew Walbran475c1452020-02-07 13:22:22 +000026#include "hf/mpool.h"
J-Alvescf6253e2024-01-03 13:48:48 +000027#include "hf/panic.h"
28#include "hf/plat/memory_protect.h"
Jose Marinho75509b42019-04-09 09:34:59 +010029#include "hf/std.h"
Andrew Scull3c257452019-11-26 13:32:50 +000030#include "hf/vm.h"
Daniel Boulby44e9b3b2024-01-17 12:21:44 +000031#include "hf/vm_ids.h"
Jose Marinho75509b42019-04-09 09:34:59 +010032
J-Alves2d8457f2022-10-05 11:06:41 +010033#include "vmapi/hf/ffa_v1_0.h"
34
J-Alves5da37d92022-10-24 16:33:48 +010035#define RECEIVERS_COUNT_IN_RETRIEVE_RESP 1
36
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000037/**
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010038 * All access to members of a `struct ffa_memory_share_state` must be guarded
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000039 * by this lock.
40 */
41static struct spinlock share_states_lock_instance = SPINLOCK_INIT;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010042static struct ffa_memory_share_state share_states[MAX_MEM_SHARES];
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000043
44/**
J-Alvesed508c82023-05-04 16:09:48 +010045 * Return the offset to the first constituent within the
46 * `ffa_composite_memory_region` for the given receiver from an
47 * `ffa_memory_region`. The caller must check that the receiver_index is within
48 * bounds, and that it has a composite memory region offset.
49 */
50static uint32_t ffa_composite_constituent_offset(
51 struct ffa_memory_region *memory_region, uint32_t receiver_index)
52{
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +000053 struct ffa_memory_access *receiver;
54 uint32_t composite_offset;
J-Alvesed508c82023-05-04 16:09:48 +010055
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +000056 CHECK(receiver_index < memory_region->receiver_count);
57
58 receiver =
59 ffa_memory_region_get_receiver(memory_region, receiver_index);
60 CHECK(receiver != NULL);
61
62 composite_offset = receiver->composite_memory_region_offset;
63
64 CHECK(composite_offset != 0);
65
66 return composite_offset + sizeof(struct ffa_composite_memory_region);
J-Alvesed508c82023-05-04 16:09:48 +010067}
68
69/**
J-Alves917d2f22020-10-30 18:39:30 +000070 * Extracts the index from a memory handle allocated by Hafnium's current world.
71 */
72uint64_t ffa_memory_handle_get_index(ffa_memory_handle_t handle)
73{
74 return handle & ~FFA_MEMORY_HANDLE_ALLOCATOR_MASK;
75}
76
77/**
Karl Meakin52cdfe72023-06-30 14:49:10 +010078 * Initialises the next available `struct ffa_memory_share_state`. If `handle`
79 * is `FFA_MEMORY_HANDLE_INVALID` then allocates an appropriate handle,
80 * otherwise uses the provided handle which is assumed to be globally unique.
Andrew Walbranca808b12020-05-15 17:22:28 +010081 *
Karl Meakin52cdfe72023-06-30 14:49:10 +010082 * Returns a pointer to the allocated `ffa_memory_share_state` on success or
83 * `NULL` if none are available.
Andrew Walbranca808b12020-05-15 17:22:28 +010084 */
Karl Meakin52cdfe72023-06-30 14:49:10 +010085struct ffa_memory_share_state *allocate_share_state(
86 struct share_states_locked share_states, uint32_t share_func,
87 struct ffa_memory_region *memory_region, uint32_t fragment_length,
88 ffa_memory_handle_t handle)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000089{
Daniel Boulbya2f8c662021-11-26 17:52:53 +000090 assert(share_states.share_states != NULL);
91 assert(memory_region != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000092
Karl Meakin52cdfe72023-06-30 14:49:10 +010093 for (uint64_t i = 0; i < MAX_MEM_SHARES; ++i) {
Andrew Walbranca808b12020-05-15 17:22:28 +010094 if (share_states.share_states[i].share_func == 0) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010095 struct ffa_memory_share_state *allocated_state =
Andrew Walbranca808b12020-05-15 17:22:28 +010096 &share_states.share_states[i];
97 struct ffa_composite_memory_region *composite =
98 ffa_memory_region_get_composite(memory_region,
99 0);
100
101 if (handle == FFA_MEMORY_HANDLE_INVALID) {
J-Alvesee68c542020-10-29 17:48:20 +0000102 memory_region->handle =
Olivier Deprez55a189e2021-06-09 15:45:27 +0200103 plat_ffa_memory_handle_make(i);
Andrew Walbranca808b12020-05-15 17:22:28 +0100104 } else {
J-Alvesee68c542020-10-29 17:48:20 +0000105 memory_region->handle = handle;
Andrew Walbranca808b12020-05-15 17:22:28 +0100106 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000107 allocated_state->share_func = share_func;
108 allocated_state->memory_region = memory_region;
Andrew Walbranca808b12020-05-15 17:22:28 +0100109 allocated_state->fragment_count = 1;
110 allocated_state->fragments[0] = composite->constituents;
111 allocated_state->fragment_constituent_counts[0] =
112 (fragment_length -
113 ffa_composite_constituent_offset(memory_region,
114 0)) /
115 sizeof(struct ffa_memory_region_constituent);
116 allocated_state->sending_complete = false;
Karl Meakin52cdfe72023-06-30 14:49:10 +0100117 for (uint32_t j = 0; j < MAX_MEM_SHARE_RECIPIENTS;
118 ++j) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100119 allocated_state->retrieved_fragment_count[j] =
120 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000121 }
Karl Meakin52cdfe72023-06-30 14:49:10 +0100122 return allocated_state;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000123 }
124 }
125
Karl Meakin52cdfe72023-06-30 14:49:10 +0100126 return NULL;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000127}
128
129/** Locks the share states lock. */
130struct share_states_locked share_states_lock(void)
131{
132 sl_lock(&share_states_lock_instance);
133
134 return (struct share_states_locked){.share_states = share_states};
135}
136
137/** Unlocks the share states lock. */
J-Alves66652252022-07-06 09:49:51 +0100138void share_states_unlock(struct share_states_locked *share_states)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000139{
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000140 assert(share_states->share_states != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000141 share_states->share_states = NULL;
142 sl_unlock(&share_states_lock_instance);
143}
144
145/**
Andrew Walbranca808b12020-05-15 17:22:28 +0100146 * If the given handle is a valid handle for an allocated share state then
Karl Meakin4a2854a2023-06-30 16:26:52 +0100147 * returns a pointer to the share state. Otherwise returns NULL.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000148 */
Karl Meakin4a2854a2023-06-30 16:26:52 +0100149struct ffa_memory_share_state *get_share_state(
150 struct share_states_locked share_states, ffa_memory_handle_t handle)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000151{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100152 struct ffa_memory_share_state *share_state;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000153
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000154 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100155
156 /*
157 * First look for a share_state allocated by us, in which case the
158 * handle is based on the index.
159 */
Olivier Deprez55a189e2021-06-09 15:45:27 +0200160 if (plat_ffa_memory_handle_allocated_by_current_world(handle)) {
Karl Meakin4a2854a2023-06-30 16:26:52 +0100161 uint64_t index = ffa_memory_handle_get_index(handle);
162
Andrew Walbranca808b12020-05-15 17:22:28 +0100163 if (index < MAX_MEM_SHARES) {
164 share_state = &share_states.share_states[index];
165 if (share_state->share_func != 0) {
Karl Meakin4a2854a2023-06-30 16:26:52 +0100166 return share_state;
Andrew Walbranca808b12020-05-15 17:22:28 +0100167 }
168 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000169 }
170
Andrew Walbranca808b12020-05-15 17:22:28 +0100171 /* Fall back to a linear scan. */
Karl Meakin4a2854a2023-06-30 16:26:52 +0100172 for (uint64_t index = 0; index < MAX_MEM_SHARES; ++index) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100173 share_state = &share_states.share_states[index];
J-Alvesee68c542020-10-29 17:48:20 +0000174 if (share_state->memory_region != NULL &&
175 share_state->memory_region->handle == handle &&
Andrew Walbranca808b12020-05-15 17:22:28 +0100176 share_state->share_func != 0) {
Karl Meakin4a2854a2023-06-30 16:26:52 +0100177 return share_state;
Andrew Walbranca808b12020-05-15 17:22:28 +0100178 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000179 }
180
Karl Meakin4a2854a2023-06-30 16:26:52 +0100181 return NULL;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000182}
183
184/** Marks a share state as unallocated. */
J-Alvesfdd29272022-07-19 13:16:31 +0100185void share_state_free(struct share_states_locked share_states,
186 struct ffa_memory_share_state *share_state,
187 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000188{
Andrew Walbranca808b12020-05-15 17:22:28 +0100189 uint32_t i;
190
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000191 assert(share_states.share_states != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000192 share_state->share_func = 0;
Andrew Walbranca808b12020-05-15 17:22:28 +0100193 share_state->sending_complete = false;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000194 mpool_free(page_pool, share_state->memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +0100195 /*
196 * First fragment is part of the same page as the `memory_region`, so it
197 * doesn't need to be freed separately.
198 */
199 share_state->fragments[0] = NULL;
200 share_state->fragment_constituent_counts[0] = 0;
201 for (i = 1; i < share_state->fragment_count; ++i) {
202 mpool_free(page_pool, share_state->fragments[i]);
203 share_state->fragments[i] = NULL;
204 share_state->fragment_constituent_counts[i] = 0;
205 }
206 share_state->fragment_count = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000207 share_state->memory_region = NULL;
J-Alvesa9cd7e32022-07-01 13:49:33 +0100208 share_state->hypervisor_fragment_count = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000209}
210
Andrew Walbranca808b12020-05-15 17:22:28 +0100211/** Checks whether the given share state has been fully sent. */
J-Alvesfdd29272022-07-19 13:16:31 +0100212bool share_state_sending_complete(struct share_states_locked share_states,
213 struct ffa_memory_share_state *share_state)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000214{
Andrew Walbranca808b12020-05-15 17:22:28 +0100215 struct ffa_composite_memory_region *composite;
216 uint32_t expected_constituent_count;
217 uint32_t fragment_constituent_count_total = 0;
218 uint32_t i;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000219
Andrew Walbranca808b12020-05-15 17:22:28 +0100220 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000221 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100222
223 /*
224 * Share state must already be valid, or it's not possible to get hold
225 * of it.
226 */
227 CHECK(share_state->memory_region != NULL &&
228 share_state->share_func != 0);
229
230 composite =
231 ffa_memory_region_get_composite(share_state->memory_region, 0);
232 expected_constituent_count = composite->constituent_count;
233 for (i = 0; i < share_state->fragment_count; ++i) {
234 fragment_constituent_count_total +=
235 share_state->fragment_constituent_counts[i];
236 }
237 dlog_verbose(
238 "Checking completion: constituent count %d/%d from %d "
239 "fragments.\n",
240 fragment_constituent_count_total, expected_constituent_count,
241 share_state->fragment_count);
242
243 return fragment_constituent_count_total == expected_constituent_count;
244}
245
246/**
247 * Calculates the offset of the next fragment expected for the given share
248 * state.
249 */
J-Alvesfdd29272022-07-19 13:16:31 +0100250uint32_t share_state_next_fragment_offset(
Andrew Walbranca808b12020-05-15 17:22:28 +0100251 struct share_states_locked share_states,
252 struct ffa_memory_share_state *share_state)
253{
254 uint32_t next_fragment_offset;
255 uint32_t i;
256
257 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000258 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100259
260 next_fragment_offset =
261 ffa_composite_constituent_offset(share_state->memory_region, 0);
262 for (i = 0; i < share_state->fragment_count; ++i) {
263 next_fragment_offset +=
264 share_state->fragment_constituent_counts[i] *
265 sizeof(struct ffa_memory_region_constituent);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000266 }
267
Andrew Walbranca808b12020-05-15 17:22:28 +0100268 return next_fragment_offset;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000269}
270
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100271static void dump_memory_region(struct ffa_memory_region *memory_region)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000272{
273 uint32_t i;
274
275 if (LOG_LEVEL < LOG_LEVEL_VERBOSE) {
276 return;
277 }
278
Karl Meakine8937d92024-03-19 16:04:25 +0000279 dlog("from VM %#x, attributes (shareability = %s, cacheability = %s, "
280 "type = %s, security = %s), flags %#x, handle %#lx "
281 "tag %lu, memory access descriptor size %u, to %u "
Andrew Walbrana65a1322020-04-06 19:32:32 +0100282 "recipients [",
Karl Meakine8937d92024-03-19 16:04:25 +0000283 memory_region->sender,
284 ffa_memory_shareability_name(
285 memory_region->attributes.shareability),
286 ffa_memory_cacheability_name(
287 memory_region->attributes.cacheability),
288 ffa_memory_type_name(memory_region->attributes.type),
289 ffa_memory_security_name(memory_region->attributes.security),
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000290 memory_region->flags, memory_region->handle, memory_region->tag,
291 memory_region->memory_access_desc_size,
Andrew Walbrana65a1322020-04-06 19:32:32 +0100292 memory_region->receiver_count);
293 for (i = 0; i < memory_region->receiver_count; ++i) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +0000294 struct ffa_memory_access *receiver =
295 ffa_memory_region_get_receiver(memory_region, i);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000296 if (i != 0) {
297 dlog(", ");
298 }
Karl Meakine8937d92024-03-19 16:04:25 +0000299 dlog("Receiver %#x: permissions (%s, %s) (offset %u)",
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +0000300 receiver->receiver_permissions.receiver,
Karl Meakine8937d92024-03-19 16:04:25 +0000301 ffa_data_access_name(receiver->receiver_permissions
302 .permissions.data_access),
303 ffa_instruction_access_name(
304 receiver->receiver_permissions.permissions
305 .instruction_access),
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +0000306 receiver->composite_memory_region_offset);
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000307 /* The impdef field is only present from v1.2 and later */
308 if (ffa_version_from_memory_access_desc_size(
309 memory_region->memory_access_desc_size) >=
Karl Meakin0e617d92024-04-05 12:55:22 +0100310 FFA_VERSION_1_2) {
Karl Meakine8937d92024-03-19 16:04:25 +0000311 dlog(", impdef: %#lx %#lx", receiver->impdef.val[0],
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000312 receiver->impdef.val[1]);
313 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000314 }
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000315 dlog("] at offset %u", memory_region->receivers_offset);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000316}
317
J-Alves66652252022-07-06 09:49:51 +0100318void dump_share_states(void)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000319{
320 uint32_t i;
321
322 if (LOG_LEVEL < LOG_LEVEL_VERBOSE) {
323 return;
324 }
325
326 dlog("Current share states:\n");
327 sl_lock(&share_states_lock_instance);
328 for (i = 0; i < MAX_MEM_SHARES; ++i) {
329 if (share_states[i].share_func != 0) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000330 switch (share_states[i].share_func) {
J-Alves95fbb312024-03-20 15:19:16 +0000331 case FFA_MEM_SHARE_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100332 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000333 dlog("SHARE");
334 break;
J-Alves95fbb312024-03-20 15:19:16 +0000335 case FFA_MEM_LEND_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100336 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000337 dlog("LEND");
338 break;
J-Alves95fbb312024-03-20 15:19:16 +0000339 case FFA_MEM_DONATE_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100340 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000341 dlog("DONATE");
342 break;
343 default:
344 dlog("invalid share_func %#x",
345 share_states[i].share_func);
346 }
Karl Meakine8937d92024-03-19 16:04:25 +0000347 dlog(" %#lx (", share_states[i].memory_region->handle);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000348 dump_memory_region(share_states[i].memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +0100349 if (share_states[i].sending_complete) {
350 dlog("): fully sent");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000351 } else {
Andrew Walbranca808b12020-05-15 17:22:28 +0100352 dlog("): partially sent");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000353 }
J-Alves2a0d2882020-10-29 14:49:50 +0000354 dlog(" with %d fragments, %d retrieved, "
355 " sender's original mode: %#x\n",
Andrew Walbranca808b12020-05-15 17:22:28 +0100356 share_states[i].fragment_count,
J-Alves2a0d2882020-10-29 14:49:50 +0000357 share_states[i].retrieved_fragment_count[0],
358 share_states[i].sender_orig_mode);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000359 }
360 }
361 sl_unlock(&share_states_lock_instance);
362}
363
Andrew Walbran475c1452020-02-07 13:22:22 +0000364/* TODO: Add device attributes: GRE, cacheability, shareability. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100365static inline uint32_t ffa_memory_permissions_to_mode(
J-Alves7cd5eb32020-10-16 19:06:10 +0100366 ffa_memory_access_permissions_t permissions, uint32_t default_mode)
Andrew Walbran475c1452020-02-07 13:22:22 +0000367{
368 uint32_t mode = 0;
369
Karl Meakin84710f32023-10-12 15:14:49 +0100370 switch (permissions.data_access) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100371 case FFA_DATA_ACCESS_RO:
Andrew Walbran475c1452020-02-07 13:22:22 +0000372 mode = MM_MODE_R;
373 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100374 case FFA_DATA_ACCESS_RW:
Andrew Walbran475c1452020-02-07 13:22:22 +0000375 mode = MM_MODE_R | MM_MODE_W;
376 break;
J-Alves7cd5eb32020-10-16 19:06:10 +0100377 case FFA_DATA_ACCESS_NOT_SPECIFIED:
378 mode = (default_mode & (MM_MODE_R | MM_MODE_W));
379 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100380 case FFA_DATA_ACCESS_RESERVED:
381 panic("Tried to convert FFA_DATA_ACCESS_RESERVED.");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100382 }
383
Karl Meakin84710f32023-10-12 15:14:49 +0100384 switch (permissions.instruction_access) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100385 case FFA_INSTRUCTION_ACCESS_NX:
Andrew Walbran475c1452020-02-07 13:22:22 +0000386 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100387 case FFA_INSTRUCTION_ACCESS_X:
Andrew Walbrana65a1322020-04-06 19:32:32 +0100388 mode |= MM_MODE_X;
389 break;
J-Alves7cd5eb32020-10-16 19:06:10 +0100390 case FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED:
391 mode |= (default_mode & MM_MODE_X);
392 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100393 case FFA_INSTRUCTION_ACCESS_RESERVED:
394 panic("Tried to convert FFA_INSTRUCTION_ACCESS_RESVERVED.");
Andrew Walbran475c1452020-02-07 13:22:22 +0000395 }
396
Olivier Deprez878bd5b2021-04-15 19:05:10 +0200397 /* Set the security state bit if necessary. */
398 if ((default_mode & plat_ffa_other_world_mode()) != 0) {
399 mode |= plat_ffa_other_world_mode();
400 }
401
Andrew Walbran475c1452020-02-07 13:22:22 +0000402 return mode;
403}
404
Jose Marinho75509b42019-04-09 09:34:59 +0100405/**
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000406 * Get the current mode in the stage-2 page table of the given vm of all the
407 * pages in the given constituents, if they all have the same mode, or return
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100408 * an appropriate FF-A error if not.
Jose Marinho75509b42019-04-09 09:34:59 +0100409 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100410static struct ffa_value constituents_get_mode(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000411 struct vm_locked vm, uint32_t *orig_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100412 struct ffa_memory_region_constituent **fragments,
413 const uint32_t *fragment_constituent_counts, uint32_t fragment_count)
Jose Marinho75509b42019-04-09 09:34:59 +0100414{
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100415 uint32_t i;
Andrew Walbranca808b12020-05-15 17:22:28 +0100416 uint32_t j;
Jose Marinho75509b42019-04-09 09:34:59 +0100417
Andrew Walbranca808b12020-05-15 17:22:28 +0100418 if (fragment_count == 0 || fragment_constituent_counts[0] == 0) {
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100419 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000420 * Fail if there are no constituents. Otherwise we would get an
421 * uninitialised *orig_mode.
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100422 */
Karl Meakin5df422c2023-07-11 17:31:38 +0100423 dlog_verbose("%s: no constituents\n", __func__);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100424 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho75509b42019-04-09 09:34:59 +0100425 }
426
Andrew Walbranca808b12020-05-15 17:22:28 +0100427 for (i = 0; i < fragment_count; ++i) {
428 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
429 ipaddr_t begin = ipa_init(fragments[i][j].address);
430 size_t size = fragments[i][j].page_count * PAGE_SIZE;
431 ipaddr_t end = ipa_add(begin, size);
432 uint32_t current_mode;
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100433
Andrew Walbranca808b12020-05-15 17:22:28 +0100434 /* Fail if addresses are not page-aligned. */
435 if (!is_aligned(ipa_addr(begin), PAGE_SIZE) ||
436 !is_aligned(ipa_addr(end), PAGE_SIZE)) {
Karl Meakin5df422c2023-07-11 17:31:38 +0100437 dlog_verbose("%s: addresses not page-aligned\n",
438 __func__);
Andrew Walbranca808b12020-05-15 17:22:28 +0100439 return ffa_error(FFA_INVALID_PARAMETERS);
440 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100441
Andrew Walbranca808b12020-05-15 17:22:28 +0100442 /*
443 * Ensure that this constituent memory range is all
444 * mapped with the same mode.
445 */
Raghu Krishnamurthy785d52f2021-02-13 00:02:40 -0800446 if (!vm_mem_get_mode(vm, begin, end, &current_mode)) {
Karl Meakin5df422c2023-07-11 17:31:38 +0100447 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +0000448 "%s: constituent memory range "
449 "%#lx..%#lx "
Karl Meakin5df422c2023-07-11 17:31:38 +0100450 "not mapped with the same mode\n",
Karl Meakine8937d92024-03-19 16:04:25 +0000451 __func__, begin.ipa, end.ipa);
Andrew Walbranca808b12020-05-15 17:22:28 +0100452 return ffa_error(FFA_DENIED);
453 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100454
Andrew Walbranca808b12020-05-15 17:22:28 +0100455 /*
456 * Ensure that all constituents are mapped with the same
457 * mode.
458 */
459 if (i == 0) {
460 *orig_mode = current_mode;
461 } else if (current_mode != *orig_mode) {
462 dlog_verbose(
Karl Meakin5df422c2023-07-11 17:31:38 +0100463 "%s: expected mode %#x but was %#x for "
Karl Meakine8937d92024-03-19 16:04:25 +0000464 "%d pages at %#lx.\n",
Karl Meakin5df422c2023-07-11 17:31:38 +0100465 __func__, *orig_mode, current_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100466 fragments[i][j].page_count,
467 ipa_addr(begin));
468 return ffa_error(FFA_DENIED);
469 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100470 }
Jose Marinho75509b42019-04-09 09:34:59 +0100471 }
472
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100473 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000474}
475
Karl Meakin0e617d92024-04-05 12:55:22 +0100476enum ffa_version ffa_version_from_memory_access_desc_size(
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100477 uint32_t memory_access_desc_size)
478{
479 switch (memory_access_desc_size) {
480 /*
481 * v1.0 and v1.1 memory access descriptors are the same size however
482 * v1.1 is the first version to include the memory access descriptor
483 * size field so return v1.1.
484 */
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000485 case sizeof(struct ffa_memory_access_v1_0):
Karl Meakin0e617d92024-04-05 12:55:22 +0100486 return FFA_VERSION_1_1;
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000487 case sizeof(struct ffa_memory_access):
Karl Meakin0e617d92024-04-05 12:55:22 +0100488 return FFA_VERSION_1_2;
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100489 }
490 return 0;
491}
492
493/**
494 * Check if the receivers size and offset given is valid for the senders
495 * FF-A version.
496 */
497static bool receiver_size_and_offset_valid_for_version(
498 uint32_t receivers_size, uint32_t receivers_offset,
Karl Meakin0e617d92024-04-05 12:55:22 +0100499 enum ffa_version ffa_version)
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100500{
501 /*
502 * Check that the version that the memory access descriptor size belongs
503 * to is compatible with the FF-A version we believe the sender to be.
504 */
Karl Meakin0e617d92024-04-05 12:55:22 +0100505 enum ffa_version expected_ffa_version =
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100506 ffa_version_from_memory_access_desc_size(receivers_size);
Karl Meakin0e617d92024-04-05 12:55:22 +0100507 if (!ffa_versions_are_compatible(expected_ffa_version, ffa_version)) {
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100508 return false;
509 }
510
511 /*
512 * Check the receivers_offset matches the version we found from
513 * memory access descriptor size.
514 */
515 switch (expected_ffa_version) {
Karl Meakin0e617d92024-04-05 12:55:22 +0100516 case FFA_VERSION_1_1:
517 case FFA_VERSION_1_2:
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100518 return receivers_offset == sizeof(struct ffa_memory_region);
519 default:
520 return false;
521 }
522}
523
524/**
525 * Check the values set for fields in the memory region are valid and safe.
526 * Offset values are within safe bounds, receiver count will not cause overflows
527 * and reserved fields are 0.
528 */
529bool ffa_memory_region_sanity_check(struct ffa_memory_region *memory_region,
Karl Meakin0e617d92024-04-05 12:55:22 +0100530 enum ffa_version ffa_version,
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100531 uint32_t fragment_length,
532 bool send_transaction)
533{
534 uint32_t receiver_count;
535 struct ffa_memory_access *receiver;
536 uint32_t composite_offset_0;
Daniel Boulbyf06b5232024-02-22 16:26:43 +0000537 struct ffa_memory_region_v1_0 *memory_region_v1_0 =
538 (struct ffa_memory_region_v1_0 *)memory_region;
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100539
Karl Meakin0e617d92024-04-05 12:55:22 +0100540 if (ffa_version == FFA_VERSION_1_0) {
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100541 /* Check the reserved fields are 0. */
542 if (memory_region_v1_0->reserved_0 != 0 ||
543 memory_region_v1_0->reserved_1 != 0) {
544 dlog_verbose("Reserved fields must be 0.\n");
545 return false;
546 }
547
548 receiver_count = memory_region_v1_0->receiver_count;
549 } else {
550 uint32_t receivers_size =
551 memory_region->memory_access_desc_size;
552 uint32_t receivers_offset = memory_region->receivers_offset;
553
554 /* Check the reserved field is 0. */
555 if (memory_region->reserved[0] != 0 ||
556 memory_region->reserved[1] != 0 ||
557 memory_region->reserved[2] != 0) {
558 dlog_verbose("Reserved fields must be 0.\n");
559 return false;
560 }
561
562 /*
563 * Check memory_access_desc_size matches the size of the struct
564 * for the senders FF-A version.
565 */
566 if (!receiver_size_and_offset_valid_for_version(
567 receivers_size, receivers_offset, ffa_version)) {
568 dlog_verbose(
569 "Invalid memory access descriptor size %d, "
570 " or receiver offset %d, "
571 "for FF-A version %#x\n",
572 receivers_size, receivers_offset, ffa_version);
573 return false;
574 }
575
576 receiver_count = memory_region->receiver_count;
577 }
578
579 /* Check receiver count is not too large. */
Daniel Boulbyf06b5232024-02-22 16:26:43 +0000580 if (receiver_count > MAX_MEM_SHARE_RECIPIENTS || receiver_count < 1) {
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100581 dlog_verbose(
Daniel Boulbyf06b5232024-02-22 16:26:43 +0000582 "Receiver count must be 0 < receiver_count < %u "
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100583 "specified %u\n",
584 MAX_MEM_SHARE_RECIPIENTS, receiver_count);
585 return false;
586 }
587
588 /* Check values in the memory access descriptors. */
589 /*
590 * The composite offset values must be the same for all recievers so
591 * check the first one is valid and then they are all the same.
592 */
Karl Meakin0e617d92024-04-05 12:55:22 +0100593 receiver = ffa_version == FFA_VERSION_1_0
Daniel Boulbyf06b5232024-02-22 16:26:43 +0000594 ? (struct ffa_memory_access *)&memory_region_v1_0
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100595 ->receivers[0]
596 : ffa_memory_region_get_receiver(memory_region, 0);
597 assert(receiver != NULL);
598 composite_offset_0 = receiver->composite_memory_region_offset;
599
600 if (!send_transaction) {
601 if (composite_offset_0 != 0) {
602 dlog_verbose(
603 "Composite offset memory region descriptor "
604 "offset must be 0 for retrieve requests. "
605 "Currently %d",
606 composite_offset_0);
607 return false;
608 }
609 } else {
610 bool comp_offset_is_zero = composite_offset_0 == 0U;
611 bool comp_offset_lt_transaction_descriptor_size =
612 composite_offset_0 <
613 (sizeof(struct ffa_memory_region) +
614 (uint32_t)(memory_region->memory_access_desc_size *
615 memory_region->receiver_count));
616 bool comp_offset_with_comp_gt_fragment_length =
617 composite_offset_0 +
618 sizeof(struct ffa_composite_memory_region) >
619 fragment_length;
620 if (comp_offset_is_zero ||
621 comp_offset_lt_transaction_descriptor_size ||
622 comp_offset_with_comp_gt_fragment_length) {
623 dlog_verbose(
624 "Invalid composite memory region descriptor "
625 "offset for send transaction %u\n",
626 composite_offset_0);
627 return false;
628 }
629 }
630
Karl Meakin824b63d2024-06-03 19:04:53 +0100631 for (size_t i = 0; i < memory_region->receiver_count; i++) {
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100632 uint32_t composite_offset;
633
Karl Meakin0e617d92024-04-05 12:55:22 +0100634 if (ffa_version == FFA_VERSION_1_0) {
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100635 struct ffa_memory_access_v1_0 *receiver_v1_0 =
636 &memory_region_v1_0->receivers[i];
637 /* Check reserved fields are 0 */
638 if (receiver_v1_0->reserved_0 != 0) {
639 dlog_verbose(
640 "Reserved field in the memory access "
Karl Meakine8937d92024-03-19 16:04:25 +0000641 "descriptor must be zero. Currently "
642 "reciever %zu has a reserved field "
643 "with a value of %lu\n",
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100644 i, receiver_v1_0->reserved_0);
645 return false;
646 }
647 /*
648 * We can cast to the current version receiver as the
649 * remaining fields we are checking have the same
650 * offsets for all versions since memory access
651 * descriptors are forwards compatible.
652 */
653 receiver = (struct ffa_memory_access *)receiver_v1_0;
654 } else {
655 receiver = ffa_memory_region_get_receiver(memory_region,
656 i);
657 assert(receiver != NULL);
658
659 if (receiver->reserved_0 != 0) {
660 dlog_verbose(
661 "Reserved field in the memory access "
Karl Meakine8937d92024-03-19 16:04:25 +0000662 "descriptor must be zero. Currently "
663 "reciever %zu has a reserved field "
664 "with a value of %lu\n",
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100665 i, receiver->reserved_0);
666 return false;
667 }
668 }
669
670 /* Check composite offset values are equal for all receivers. */
671 composite_offset = receiver->composite_memory_region_offset;
672 if (composite_offset != composite_offset_0) {
673 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +0000674 "Composite offset %x differs from %x in "
675 "index\n",
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100676 composite_offset, composite_offset_0);
677 return false;
678 }
679 }
680 return true;
681}
682
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000683/**
J-Alves460d36c2023-10-12 17:02:15 +0100684 * If the receivers for the memory management operation are all from the
685 * secure world and this isn't a FFA_MEM_SHARE, then request memory security
686 * state update by returning MAP_ACTION_CHECK_PROTECT.
687 */
688static enum ffa_map_action ffa_mem_send_get_map_action(
689 bool all_receivers_from_current_world, ffa_id_t sender_id,
690 uint32_t mem_func_id)
691{
J-Alves95fbb312024-03-20 15:19:16 +0000692 const bool is_memory_share_abi = mem_func_id == FFA_MEM_SHARE_32 ||
693 mem_func_id == FFA_MEM_SHARE_64;
694 const bool protect_memory =
695 (!is_memory_share_abi && all_receivers_from_current_world &&
696 ffa_is_vm_id(sender_id));
J-Alves460d36c2023-10-12 17:02:15 +0100697
698 return protect_memory ? MAP_ACTION_CHECK_PROTECT : MAP_ACTION_CHECK;
699}
700
701/**
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000702 * Verify that all pages have the same mode, that the starting mode
703 * constitutes a valid state and obtain the next mode to apply
J-Alves460d36c2023-10-12 17:02:15 +0100704 * to the sending VM. It outputs the mapping action that needs to be
705 * invoked for the given memory range. On memory lend/donate there
706 * could be a need to protect the memory from the normal world.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000707 *
708 * Returns:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100709 * 1) FFA_DENIED if a state transition was not found;
710 * 2) FFA_DENIED if the pages being shared do not have the same mode within
Andrew Walbrana65a1322020-04-06 19:32:32 +0100711 * the <from> VM;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100712 * 3) FFA_INVALID_PARAMETERS if the beginning and end IPAs are not page
Andrew Walbrana65a1322020-04-06 19:32:32 +0100713 * aligned;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100714 * 4) FFA_INVALID_PARAMETERS if the requested share type was not handled.
715 * Or FFA_SUCCESS on success.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000716 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100717static struct ffa_value ffa_send_check_transition(
Andrew Walbrana65a1322020-04-06 19:32:32 +0100718 struct vm_locked from, uint32_t share_func,
Daniel Boulbya76fd912024-02-22 14:22:15 +0000719 struct ffa_memory_region *memory_region, uint32_t *orig_from_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100720 struct ffa_memory_region_constituent **fragments,
721 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
Daniel Boulby4b846eb2024-05-23 17:32:23 +0100722 uint32_t *from_mode, enum ffa_map_action *map_action, bool zero)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000723{
724 const uint32_t state_mask =
725 MM_MODE_INVALID | MM_MODE_UNOWNED | MM_MODE_SHARED;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100726 struct ffa_value ret;
J-Alves460d36c2023-10-12 17:02:15 +0100727 bool all_receivers_from_current_world = true;
Daniel Boulbya76fd912024-02-22 14:22:15 +0000728 uint32_t receivers_count = memory_region->receiver_count;
J-Alves95fbb312024-03-20 15:19:16 +0000729 const bool is_memory_lend = (share_func == FFA_MEM_LEND_32) ||
730 (share_func == FFA_MEM_LEND_64);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000731
Andrew Walbranca808b12020-05-15 17:22:28 +0100732 ret = constituents_get_mode(from, orig_from_mode, fragments,
733 fragment_constituent_counts,
734 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100735 if (ret.func != FFA_SUCCESS_32) {
Olivier Depreze7eb1682022-03-16 17:09:03 +0100736 dlog_verbose("Inconsistent modes.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100737 return ret;
Andrew Scullb5f49e02019-10-02 13:20:47 +0100738 }
739
Daniel Boulby63af1fa2024-03-18 14:17:31 +0000740 /* Device memory regions can only be lent a single borrower. */
Daniel Boulby9764ff62024-01-30 17:47:39 +0000741 if ((*orig_from_mode & MM_MODE_D) != 0U &&
J-Alves95fbb312024-03-20 15:19:16 +0000742 !(is_memory_lend && receivers_count == 1)) {
Daniel Boulby9764ff62024-01-30 17:47:39 +0000743 dlog_verbose(
Daniel Boulby63af1fa2024-03-18 14:17:31 +0000744 "Device memory can only be lent to a single borrower "
745 "(mode is %#x).\n",
Daniel Boulby9764ff62024-01-30 17:47:39 +0000746 *orig_from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100747 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000748 }
749
750 /*
751 * Ensure the sender is the owner and has exclusive access to the
752 * memory.
753 */
754 if ((*orig_from_mode & state_mask) != 0) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100755 return ffa_error(FFA_DENIED);
Andrew Walbrana65a1322020-04-06 19:32:32 +0100756 }
757
Daniel Boulby4b846eb2024-05-23 17:32:23 +0100758 /*
759 * Memory cannot be zeroed during the lend/donate operation if the
760 * sender only has RO access.
761 */
762 if ((*orig_from_mode & MM_MODE_W) == 0 && zero == true) {
763 dlog_verbose(
764 "Cannot zero memory when the sender doesn't have "
765 "write access\n");
766 return ffa_error(FFA_DENIED);
767 }
768
Daniel Boulbya76fd912024-02-22 14:22:15 +0000769 assert(receivers_count > 0U);
J-Alves7cd5eb32020-10-16 19:06:10 +0100770
J-Alves363f5722022-04-25 17:37:37 +0100771 for (uint32_t i = 0U; i < receivers_count; i++) {
Daniel Boulbya76fd912024-02-22 14:22:15 +0000772 struct ffa_memory_access *receiver =
773 ffa_memory_region_get_receiver(memory_region, i);
774 assert(receiver != NULL);
J-Alves363f5722022-04-25 17:37:37 +0100775 ffa_memory_access_permissions_t permissions =
Daniel Boulbya76fd912024-02-22 14:22:15 +0000776 receiver->receiver_permissions.permissions;
J-Alves363f5722022-04-25 17:37:37 +0100777 uint32_t required_from_mode = ffa_memory_permissions_to_mode(
778 permissions, *orig_from_mode);
779
J-Alves788b4492023-04-18 14:01:23 +0100780 /*
781 * The assumption is that at this point, the operation from
782 * SP to a receiver VM, should have returned an FFA_ERROR
783 * already.
784 */
785 if (!ffa_is_vm_id(from.vm->id)) {
786 assert(!ffa_is_vm_id(
Daniel Boulbya76fd912024-02-22 14:22:15 +0000787 receiver->receiver_permissions.receiver));
J-Alves788b4492023-04-18 14:01:23 +0100788 }
789
J-Alves460d36c2023-10-12 17:02:15 +0100790 /* Track if all senders are from current world. */
791 all_receivers_from_current_world =
792 all_receivers_from_current_world &&
793 vm_id_is_current_world(
Daniel Boulbya76fd912024-02-22 14:22:15 +0000794 receiver->receiver_permissions.receiver);
J-Alves460d36c2023-10-12 17:02:15 +0100795
J-Alves363f5722022-04-25 17:37:37 +0100796 if ((*orig_from_mode & required_from_mode) !=
797 required_from_mode) {
798 dlog_verbose(
799 "Sender tried to send memory with permissions "
J-Alves788b4492023-04-18 14:01:23 +0100800 "which required mode %#x but only had %#x "
801 "itself.\n",
J-Alves363f5722022-04-25 17:37:37 +0100802 required_from_mode, *orig_from_mode);
803 return ffa_error(FFA_DENIED);
804 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000805 }
806
J-Alves460d36c2023-10-12 17:02:15 +0100807 *map_action = ffa_mem_send_get_map_action(
808 all_receivers_from_current_world, from.vm->id, share_func);
809
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000810 /* Find the appropriate new mode. */
811 *from_mode = ~state_mask & *orig_from_mode;
Andrew Walbrane7ad3c02019-12-24 17:03:04 +0000812 switch (share_func) {
J-Alves95fbb312024-03-20 15:19:16 +0000813 case FFA_MEM_DONATE_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100814 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000815 *from_mode |= MM_MODE_INVALID | MM_MODE_UNOWNED;
Jose Marinho75509b42019-04-09 09:34:59 +0100816 break;
J-Alves95fbb312024-03-20 15:19:16 +0000817 case FFA_MEM_LEND_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100818 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000819 *from_mode |= MM_MODE_INVALID;
Andrew Walbran648fc3e2019-10-22 16:23:05 +0100820 break;
J-Alves95fbb312024-03-20 15:19:16 +0000821 case FFA_MEM_SHARE_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100822 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000823 *from_mode |= MM_MODE_SHARED;
Jose Marinho56c25732019-05-20 09:48:53 +0100824 break;
825
Jose Marinho75509b42019-04-09 09:34:59 +0100826 default:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100827 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho75509b42019-04-09 09:34:59 +0100828 }
829
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100830 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000831}
832
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100833static struct ffa_value ffa_relinquish_check_transition(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000834 struct vm_locked from, uint32_t *orig_from_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100835 struct ffa_memory_region_constituent **fragments,
836 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
J-Alves69cdfd92024-04-26 11:40:59 +0100837 uint32_t *from_mode, enum ffa_map_action *map_action)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000838{
839 const uint32_t state_mask =
840 MM_MODE_INVALID | MM_MODE_UNOWNED | MM_MODE_SHARED;
841 uint32_t orig_from_state;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100842 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000843
J-Alves69cdfd92024-04-26 11:40:59 +0100844 assert(map_action != NULL);
845 if (vm_id_is_current_world(from.vm->id)) {
846 *map_action = MAP_ACTION_COMMIT;
847 } else {
848 /*
849 * No need to check the attributes of caller.
850 * The assumption is that the retrieve request of the receiver
851 * also used the MAP_ACTION_NONE, and no update was done to the
852 * page tables. When the receiver is not at the secure virtual
853 * instance SPMC doesn't manage its S2 translation (i.e. when
854 * the receiver is a VM).
855 */
856 *map_action = MAP_ACTION_NONE;
857
858 return (struct ffa_value){.func = FFA_SUCCESS_32};
859 }
860
Andrew Walbranca808b12020-05-15 17:22:28 +0100861 ret = constituents_get_mode(from, orig_from_mode, fragments,
862 fragment_constituent_counts,
863 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100864 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbrana65a1322020-04-06 19:32:32 +0100865 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000866 }
867
868 /* Ensure the address range is normal memory and not a device. */
869 if (*orig_from_mode & MM_MODE_D) {
870 dlog_verbose("Can't relinquish device memory (mode is %#x).\n",
871 *orig_from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100872 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000873 }
874
875 /*
876 * Ensure the relinquishing VM is not the owner but has access to the
877 * memory.
878 */
879 orig_from_state = *orig_from_mode & state_mask;
880 if ((orig_from_state & ~MM_MODE_SHARED) != MM_MODE_UNOWNED) {
881 dlog_verbose(
882 "Tried to relinquish memory in state %#x (masked %#x "
Andrew Walbranca808b12020-05-15 17:22:28 +0100883 "but should be %#x).\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000884 *orig_from_mode, orig_from_state, MM_MODE_UNOWNED);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100885 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000886 }
887
888 /* Find the appropriate new mode. */
889 *from_mode = (~state_mask & *orig_from_mode) | MM_MODE_UNMAPPED_MASK;
890
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100891 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000892}
893
894/**
895 * Verify that all pages have the same mode, that the starting mode
896 * constitutes a valid state and obtain the next mode to apply
897 * to the retrieving VM.
898 *
899 * Returns:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100900 * 1) FFA_DENIED if a state transition was not found;
901 * 2) FFA_DENIED if the pages being shared do not have the same mode within
Andrew Walbrana65a1322020-04-06 19:32:32 +0100902 * the <to> VM;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100903 * 3) FFA_INVALID_PARAMETERS if the beginning and end IPAs are not page
Andrew Walbrana65a1322020-04-06 19:32:32 +0100904 * aligned;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100905 * 4) FFA_INVALID_PARAMETERS if the requested share type was not handled.
906 * Or FFA_SUCCESS on success.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000907 */
J-Alvesfc19b372022-07-06 12:17:35 +0100908struct ffa_value ffa_retrieve_check_transition(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000909 struct vm_locked to, uint32_t share_func,
Andrew Walbranca808b12020-05-15 17:22:28 +0100910 struct ffa_memory_region_constituent **fragments,
911 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
J-Alvesfd206052023-05-22 16:45:00 +0100912 uint32_t memory_to_attributes, uint32_t *to_mode, bool memory_protected,
913 enum ffa_map_action *map_action)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000914{
915 uint32_t orig_to_mode;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100916 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000917
Andrew Walbranca808b12020-05-15 17:22:28 +0100918 ret = constituents_get_mode(to, &orig_to_mode, fragments,
919 fragment_constituent_counts,
920 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100921 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100922 dlog_verbose("Inconsistent modes.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100923 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000924 }
925
J-Alves460d36c2023-10-12 17:02:15 +0100926 /* Find the appropriate new mode. */
927 *to_mode = memory_to_attributes;
928
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100929 if (share_func == FFA_MEM_RECLAIM_32) {
J-Alves9256f162021-12-09 13:18:43 +0000930 /*
931 * If the original ffa memory send call has been processed
932 * successfully, it is expected the orig_to_mode would overlay
933 * with `state_mask`, as a result of the function
934 * `ffa_send_check_transition`.
J-Alvesfd206052023-05-22 16:45:00 +0100935 *
936 * If Hafnium is the SPMC:
937 * - Caller of the reclaim interface is an SP, the memory shall
938 * have been protected throughout the flow.
939 * - Caller of the reclaim is from the NWd, the memory may have
940 * been protected at the time of lending/donating the memory.
941 * In such case, set action to unprotect memory in the
942 * handling of reclaim operation.
943 * - If Hafnium is the hypervisor memory shall never have been
944 * protected in memory lend/share/donate.
945 *
946 * More details in the doc comment of the function
947 * `ffa_region_group_identity_map`.
J-Alves9256f162021-12-09 13:18:43 +0000948 */
J-Alves59ed0042022-07-28 18:26:41 +0100949 if (vm_id_is_current_world(to.vm->id)) {
950 assert((orig_to_mode &
951 (MM_MODE_INVALID | MM_MODE_UNOWNED |
952 MM_MODE_SHARED)) != 0U);
J-Alvesfd206052023-05-22 16:45:00 +0100953 assert(!memory_protected);
954 } else if (to.vm->id == HF_OTHER_WORLD_ID &&
955 map_action != NULL && memory_protected) {
956 *map_action = MAP_ACTION_COMMIT_UNPROTECT;
J-Alves59ed0042022-07-28 18:26:41 +0100957 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000958 } else {
J-Alves69cdfd92024-04-26 11:40:59 +0100959 if (!vm_id_is_current_world(to.vm->id)) {
960 assert(map_action != NULL);
961 *map_action = MAP_ACTION_NONE;
962 return (struct ffa_value){.func = FFA_SUCCESS_32};
963 }
964
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000965 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +0100966 * If the retriever is from virtual FF-A instance:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000967 * Ensure the retriever has the expected state. We don't care
968 * about the MM_MODE_SHARED bit; either with or without it set
969 * are both valid representations of the !O-NA state.
970 */
J-Alvesa9cd7e32022-07-01 13:49:33 +0100971 if (vm_id_is_current_world(to.vm->id) &&
972 to.vm->id != HF_PRIMARY_VM_ID &&
973 (orig_to_mode & MM_MODE_UNMAPPED_MASK) !=
974 MM_MODE_UNMAPPED_MASK) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100975 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000976 }
J-Alves460d36c2023-10-12 17:02:15 +0100977
978 /*
979 * If memory has been protected before, clear the NS bit to
980 * allow the secure access from the SP.
981 */
982 if (memory_protected) {
983 *to_mode &= ~plat_ffa_other_world_mode();
984 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000985 }
986
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000987 switch (share_func) {
J-Alves95fbb312024-03-20 15:19:16 +0000988 case FFA_MEM_DONATE_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100989 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000990 *to_mode |= 0;
991 break;
J-Alves95fbb312024-03-20 15:19:16 +0000992 case FFA_MEM_LEND_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100993 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000994 *to_mode |= MM_MODE_UNOWNED;
995 break;
J-Alves95fbb312024-03-20 15:19:16 +0000996 case FFA_MEM_SHARE_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100997 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000998 *to_mode |= MM_MODE_UNOWNED | MM_MODE_SHARED;
999 break;
1000
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001001 case FFA_MEM_RECLAIM_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001002 *to_mode |= 0;
1003 break;
1004
1005 default:
Andrew Walbranca808b12020-05-15 17:22:28 +01001006 dlog_error("Invalid share_func %#x.\n", share_func);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001007 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001008 }
1009
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001010 return (struct ffa_value){.func = FFA_SUCCESS_32};
Jose Marinho75509b42019-04-09 09:34:59 +01001011}
Jose Marinho09b1db82019-08-08 09:16:59 +01001012
J-Alvescf6253e2024-01-03 13:48:48 +00001013/*
1014 * Performs the operations related to the `action` MAP_ACTION_CHECK*.
1015 * Returns:
1016 * - FFA_SUCCESS_32: if all goes well.
1017 * - FFA_ERROR_32: with FFA_NO_MEMORY, if there is no memory to manage
1018 * the page table update. Or error code provided by the function
1019 * `arch_memory_protect`.
1020 */
1021static struct ffa_value ffa_region_group_check_actions(
1022 struct vm_locked vm_locked, paddr_t pa_begin, paddr_t pa_end,
1023 struct mpool *ppool, uint32_t mode, enum ffa_map_action action,
1024 bool *memory_protected)
1025{
1026 struct ffa_value ret;
1027 bool is_memory_protected;
1028
1029 if (!vm_identity_prepare(vm_locked, pa_begin, pa_end, mode, ppool)) {
1030 dlog_verbose(
1031 "%s: memory can't be mapped to %x due to lack of "
Karl Meakine8937d92024-03-19 16:04:25 +00001032 "memory. Base: %lx end: %lx\n",
J-Alvescf6253e2024-01-03 13:48:48 +00001033 __func__, vm_locked.vm->id, pa_addr(pa_begin),
1034 pa_addr(pa_end));
1035 return ffa_error(FFA_NO_MEMORY);
1036 }
1037
1038 switch (action) {
1039 case MAP_ACTION_CHECK:
1040 /* No protect requested. */
1041 is_memory_protected = false;
1042 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
1043 break;
1044 case MAP_ACTION_CHECK_PROTECT: {
1045 paddr_t last_protected_pa = pa_init(0);
1046
1047 ret = arch_memory_protect(pa_begin, pa_end, &last_protected_pa);
1048
1049 is_memory_protected = (ret.func == FFA_SUCCESS_32);
1050
1051 /*
1052 * - If protect memory has failed with FFA_DENIED, means some
1053 * range of memory was in the wrong state. In such case, SPM
1054 * reverts the state of the pages that were successfully
1055 * updated.
1056 * - If protect memory has failed with FFA_NOT_SUPPORTED, it
1057 * means the platform doesn't support the protection mechanism.
1058 * That said, it still permits the page table update to go
1059 * through. The variable
1060 * `is_memory_protected` will be equal to false.
1061 * - If protect memory has failed with FFA_INVALID_PARAMETERS,
1062 * break from switch and return the error.
1063 */
1064 if (ret.func == FFA_ERROR_32) {
1065 assert(!is_memory_protected);
1066 if (ffa_error_code(ret) == FFA_DENIED &&
1067 pa_addr(last_protected_pa) != (uintptr_t)0) {
1068 CHECK(arch_memory_unprotect(
1069 pa_begin,
1070 pa_add(last_protected_pa, PAGE_SIZE)));
1071 } else if (ffa_error_code(ret) == FFA_NOT_SUPPORTED) {
1072 ret = (struct ffa_value){
1073 .func = FFA_SUCCESS_32,
1074 };
1075 }
1076 }
1077 } break;
1078 default:
1079 panic("%s: invalid action to process %x\n", __func__, action);
1080 }
1081
1082 if (memory_protected != NULL) {
1083 *memory_protected = is_memory_protected;
1084 }
1085
1086 return ret;
1087}
1088
1089static void ffa_region_group_commit_actions(struct vm_locked vm_locked,
1090 paddr_t pa_begin, paddr_t pa_end,
1091 struct mpool *ppool, uint32_t mode,
1092 enum ffa_map_action action)
1093{
1094 switch (action) {
1095 case MAP_ACTION_COMMIT_UNPROTECT:
1096 /*
1097 * Checking that it should succeed because SPM should be
1098 * unprotecting memory that it had protected before.
1099 */
1100 CHECK(arch_memory_unprotect(pa_begin, pa_end));
1101 case MAP_ACTION_COMMIT:
1102 vm_identity_commit(vm_locked, pa_begin, pa_end, mode, ppool,
1103 NULL);
1104 break;
1105 default:
1106 panic("%s: invalid action to process %x\n", __func__, action);
1107 }
1108}
1109
Jose Marinho09b1db82019-08-08 09:16:59 +01001110/**
J-Alves063ad832023-10-03 18:05:40 +01001111 * Helper function to revert a failed "Protect" action from the SPMC:
1112 * - `fragment_count`: should specify the number of fragments to traverse from
1113 * `fragments`. This may not be the full amount of fragments that are part of
1114 * the share_state structure.
1115 * - `fragment_constituent_counts`: array holding the amount of constituents
1116 * per fragment.
1117 * - `end`: pointer to the constituent that failed the "protect" action. It
1118 * shall be part of the last fragment, and it shall make the loop below break.
1119 */
1120static void ffa_region_group_fragments_revert_protect(
1121 struct ffa_memory_region_constituent **fragments,
1122 const uint32_t *fragment_constituent_counts, uint32_t fragment_count,
1123 const struct ffa_memory_region_constituent *end)
1124{
1125 for (uint32_t i = 0; i < fragment_count; ++i) {
1126 for (uint32_t j = 0; j < fragment_constituent_counts[i]; ++j) {
1127 struct ffa_memory_region_constituent *constituent =
1128 &fragments[i][j];
1129 size_t size = constituent->page_count * PAGE_SIZE;
1130 paddr_t pa_begin =
1131 pa_from_ipa(ipa_init(constituent->address));
1132 paddr_t pa_end = pa_add(pa_begin, size);
1133
Karl Meakine8937d92024-03-19 16:04:25 +00001134 dlog_verbose("%s: reverting fragment %lx size %zx\n",
J-Alves063ad832023-10-03 18:05:40 +01001135 __func__, pa_addr(pa_begin), size);
1136
1137 if (constituent == end) {
1138 /*
1139 * The last constituent is expected to be in the
1140 * last fragment.
1141 */
1142 assert(i == fragment_count - 1);
1143 break;
1144 }
1145
1146 CHECK(arch_memory_unprotect(pa_begin, pa_end));
1147 }
1148 }
1149}
1150
1151/**
Jose Marinho09b1db82019-08-08 09:16:59 +01001152 * Updates a VM's page table such that the given set of physical address ranges
1153 * are mapped in the address space at the corresponding address ranges, in the
1154 * mode provided.
1155 *
J-Alves0a83dc22023-05-05 09:50:37 +01001156 * The enum ffa_map_action determines the action taken from a call to the
1157 * function below:
1158 * - If action is MAP_ACTION_CHECK, the page tables will be allocated from the
1159 * mpool but no mappings will actually be updated. This function must always
1160 * be called first with action set to MAP_ACTION_CHECK to check that it will
1161 * succeed before calling ffa_region_group_identity_map with whichever one of
1162 * the remaining actions, to avoid leaving the page table in a half-updated
1163 * state.
1164 * - The action MAP_ACTION_COMMIT allocates the page tables from the mpool, and
1165 * changes the memory mappings.
J-Alvescf6253e2024-01-03 13:48:48 +00001166 * - The action MAP_ACTION_CHECK_PROTECT extends the MAP_ACTION_CHECK with an
1167 * invocation to the monitor to update the security state of the memory,
1168 * to that of the SPMC.
1169 * - The action MAP_ACTION_COMMIT_UNPROTECT extends the MAP_ACTION_COMMIT
1170 * with a call into the monitor, to reset the security state of memory
1171 * that has priorly been mapped with the MAP_ACTION_CHECK_PROTECT action.
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001172 * vm_ptable_defrag should always be called after a series of page table
1173 * updates, whether they succeed or fail.
Jose Marinho09b1db82019-08-08 09:16:59 +01001174 *
J-Alvescf6253e2024-01-03 13:48:48 +00001175 * If all goes well, returns FFA_SUCCESS_32; or FFA_ERROR, with following
1176 * error codes:
1177 * - FFA_INVALID_PARAMETERS: invalid range of memory.
1178 * - FFA_DENIED:
1179 *
Jose Marinho09b1db82019-08-08 09:16:59 +01001180 * made to memory mappings.
1181 */
J-Alvescf6253e2024-01-03 13:48:48 +00001182struct ffa_value ffa_region_group_identity_map(
Andrew Walbranf4b51af2020-02-03 14:44:54 +00001183 struct vm_locked vm_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01001184 struct ffa_memory_region_constituent **fragments,
1185 const uint32_t *fragment_constituent_counts, uint32_t fragment_count,
J-Alvescf6253e2024-01-03 13:48:48 +00001186 uint32_t mode, struct mpool *ppool, enum ffa_map_action action,
1187 bool *memory_protected)
Jose Marinho09b1db82019-08-08 09:16:59 +01001188{
Andrew Walbranca808b12020-05-15 17:22:28 +01001189 uint32_t i;
1190 uint32_t j;
J-Alvescf6253e2024-01-03 13:48:48 +00001191 struct ffa_value ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01001192
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001193 if (vm_locked.vm->el0_partition) {
1194 mode |= MM_MODE_USER | MM_MODE_NG;
1195 }
1196
Andrew Walbranca808b12020-05-15 17:22:28 +01001197 /* Iterate over the memory region constituents within each fragment. */
1198 for (i = 0; i < fragment_count; ++i) {
1199 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
J-Alves063ad832023-10-03 18:05:40 +01001200 struct ffa_memory_region_constituent *constituent =
1201 &fragments[i][j];
1202 size_t size = constituent->page_count * PAGE_SIZE;
Andrew Walbranca808b12020-05-15 17:22:28 +01001203 paddr_t pa_begin =
J-Alves063ad832023-10-03 18:05:40 +01001204 pa_from_ipa(ipa_init(constituent->address));
Andrew Walbranca808b12020-05-15 17:22:28 +01001205 paddr_t pa_end = pa_add(pa_begin, size);
Jens Wiklander4f1880c2022-10-19 17:00:14 +02001206 uint32_t pa_bits =
1207 arch_mm_get_pa_bits(arch_mm_get_pa_range());
Federico Recanati4fd065d2021-12-13 20:06:23 +01001208
1209 /*
1210 * Ensure the requested region falls into system's PA
1211 * range.
1212 */
Jens Wiklander4f1880c2022-10-19 17:00:14 +02001213 if (((pa_addr(pa_begin) >> pa_bits) > 0) ||
1214 ((pa_addr(pa_end) >> pa_bits) > 0)) {
Federico Recanati4fd065d2021-12-13 20:06:23 +01001215 dlog_error("Region is outside of PA Range\n");
J-Alvescf6253e2024-01-03 13:48:48 +00001216 return ffa_error(FFA_INVALID_PARAMETERS);
Federico Recanati4fd065d2021-12-13 20:06:23 +01001217 }
Andrew Walbranca808b12020-05-15 17:22:28 +01001218
J-Alvescf6253e2024-01-03 13:48:48 +00001219 if (action <= MAP_ACTION_CHECK_PROTECT) {
1220 ret = ffa_region_group_check_actions(
1221 vm_locked, pa_begin, pa_end, ppool,
1222 mode, action, memory_protected);
J-Alves063ad832023-10-03 18:05:40 +01001223
1224 if (ret.func == FFA_ERROR_32 &&
1225 ffa_error_code(ret) == FFA_DENIED) {
1226 if (memory_protected != NULL) {
1227 assert(!*memory_protected);
1228 }
1229
1230 ffa_region_group_fragments_revert_protect(
1231 fragments,
1232 fragment_constituent_counts,
1233 i + 1, constituent);
1234 break;
1235 }
J-Alvescf6253e2024-01-03 13:48:48 +00001236 } else if (action >= MAP_ACTION_COMMIT &&
1237 action < MAP_ACTION_MAX) {
1238 ffa_region_group_commit_actions(
1239 vm_locked, pa_begin, pa_end, ppool,
1240 mode, action);
1241 ret = (struct ffa_value){
1242 .func = FFA_SUCCESS_32};
1243 } else {
1244 panic("%s: Unknown ffa_map_action.\n",
1245 __func__);
Andrew Walbranca808b12020-05-15 17:22:28 +01001246 }
Jose Marinho09b1db82019-08-08 09:16:59 +01001247 }
1248 }
1249
J-Alvescf6253e2024-01-03 13:48:48 +00001250 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01001251}
1252
1253/**
1254 * Clears a region of physical memory by overwriting it with zeros. The data is
1255 * flushed from the cache so the memory has been cleared across the system.
1256 */
J-Alves7db32002021-12-14 14:44:50 +00001257static bool clear_memory(paddr_t begin, paddr_t end, struct mpool *ppool,
1258 uint32_t extra_mode_attributes)
Jose Marinho09b1db82019-08-08 09:16:59 +01001259{
1260 /*
Fuad Tabbaed294af2019-12-20 10:43:01 +00001261 * TODO: change this to a CPU local single page window rather than a
Jose Marinho09b1db82019-08-08 09:16:59 +01001262 * global mapping of the whole range. Such an approach will limit
1263 * the changes to stage-1 tables and will allow only local
1264 * invalidation.
1265 */
1266 bool ret;
1267 struct mm_stage1_locked stage1_locked = mm_lock_stage1();
J-Alves7db32002021-12-14 14:44:50 +00001268 void *ptr = mm_identity_map(stage1_locked, begin, end,
1269 MM_MODE_W | (extra_mode_attributes &
1270 plat_ffa_other_world_mode()),
1271 ppool);
Jose Marinho09b1db82019-08-08 09:16:59 +01001272 size_t size = pa_difference(begin, end);
1273
1274 if (!ptr) {
Jose Marinho09b1db82019-08-08 09:16:59 +01001275 goto fail;
1276 }
1277
1278 memset_s(ptr, size, 0, size);
1279 arch_mm_flush_dcache(ptr, size);
1280 mm_unmap(stage1_locked, begin, end, ppool);
1281
1282 ret = true;
1283 goto out;
1284
1285fail:
1286 ret = false;
1287
1288out:
1289 mm_unlock_stage1(&stage1_locked);
1290
1291 return ret;
1292}
1293
1294/**
1295 * Clears a region of physical memory by overwriting it with zeros. The data is
1296 * flushed from the cache so the memory has been cleared across the system.
1297 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001298static bool ffa_clear_memory_constituents(
J-Alves7db32002021-12-14 14:44:50 +00001299 uint32_t security_state_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +01001300 struct ffa_memory_region_constituent **fragments,
1301 const uint32_t *fragment_constituent_counts, uint32_t fragment_count,
1302 struct mpool *page_pool)
Jose Marinho09b1db82019-08-08 09:16:59 +01001303{
1304 struct mpool local_page_pool;
Andrew Walbranca808b12020-05-15 17:22:28 +01001305 uint32_t i;
Jose Marinho09b1db82019-08-08 09:16:59 +01001306 bool ret = false;
1307
1308 /*
1309 * Create a local pool so any freed memory can't be used by another
1310 * thread. This is to ensure each constituent that is mapped can be
1311 * unmapped again afterwards.
1312 */
Andrew Walbran475c1452020-02-07 13:22:22 +00001313 mpool_init_with_fallback(&local_page_pool, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +01001314
Andrew Walbranca808b12020-05-15 17:22:28 +01001315 /* Iterate over the memory region constituents within each fragment. */
1316 for (i = 0; i < fragment_count; ++i) {
1317 uint32_t j;
Jose Marinho09b1db82019-08-08 09:16:59 +01001318
J-Alves8457f932023-10-11 16:41:45 +01001319 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001320 size_t size = fragments[i][j].page_count * PAGE_SIZE;
1321 paddr_t begin =
1322 pa_from_ipa(ipa_init(fragments[i][j].address));
1323 paddr_t end = pa_add(begin, size);
1324
J-Alves7db32002021-12-14 14:44:50 +00001325 if (!clear_memory(begin, end, &local_page_pool,
1326 security_state_mode)) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001327 /*
1328 * api_clear_memory will defrag on failure, so
1329 * no need to do it here.
1330 */
1331 goto out;
1332 }
Jose Marinho09b1db82019-08-08 09:16:59 +01001333 }
1334 }
1335
Jose Marinho09b1db82019-08-08 09:16:59 +01001336 ret = true;
1337
1338out:
1339 mpool_fini(&local_page_pool);
1340 return ret;
1341}
1342
J-Alves5952d942022-12-22 16:03:00 +00001343static bool is_memory_range_within(ipaddr_t begin, ipaddr_t end,
1344 ipaddr_t in_begin, ipaddr_t in_end)
1345{
1346 return (ipa_addr(begin) >= ipa_addr(in_begin) &&
1347 ipa_addr(begin) < ipa_addr(in_end)) ||
1348 (ipa_addr(end) <= ipa_addr(in_end) &&
1349 ipa_addr(end) > ipa_addr(in_begin));
1350}
1351
1352/**
1353 * Receives a memory range and looks for overlaps with the remainder
1354 * constituents of the memory share/lend/donate operation. Assumes they are
1355 * passed in order to avoid having to loop over all the elements at each call.
1356 * The function only compares the received memory ranges with those that follow
1357 * within the same fragment, and subsequent fragments from the same operation.
1358 */
1359static bool ffa_memory_check_overlap(
1360 struct ffa_memory_region_constituent **fragments,
1361 const uint32_t *fragment_constituent_counts,
1362 const uint32_t fragment_count, const uint32_t current_fragment,
1363 const uint32_t current_constituent)
1364{
1365 uint32_t i = current_fragment;
1366 uint32_t j = current_constituent;
1367 ipaddr_t current_begin = ipa_init(fragments[i][j].address);
1368 const uint32_t current_page_count = fragments[i][j].page_count;
1369 size_t current_size = current_page_count * PAGE_SIZE;
1370 ipaddr_t current_end = ipa_add(current_begin, current_size - 1);
1371
1372 if (current_size == 0 ||
1373 current_size > UINT64_MAX - ipa_addr(current_begin)) {
Karl Meakine8937d92024-03-19 16:04:25 +00001374 dlog_verbose("Invalid page count. Addr: %zx page_count: %x\n",
1375 current_begin.ipa, current_page_count);
J-Alves5952d942022-12-22 16:03:00 +00001376 return false;
1377 }
1378
1379 for (; i < fragment_count; i++) {
1380 j = (i == current_fragment) ? j + 1 : 0;
1381
1382 for (; j < fragment_constituent_counts[i]; j++) {
1383 ipaddr_t begin = ipa_init(fragments[i][j].address);
1384 const uint32_t page_count = fragments[i][j].page_count;
1385 size_t size = page_count * PAGE_SIZE;
1386 ipaddr_t end = ipa_add(begin, size - 1);
1387
1388 if (size == 0 || size > UINT64_MAX - ipa_addr(begin)) {
1389 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00001390 "Invalid page count. Addr: %lx "
J-Alves5952d942022-12-22 16:03:00 +00001391 "page_count: %x\n",
Karl Meakine8937d92024-03-19 16:04:25 +00001392 begin.ipa, page_count);
J-Alves5952d942022-12-22 16:03:00 +00001393 return false;
1394 }
1395
1396 /*
1397 * Check if current ranges is within begin and end, as
1398 * well as the reverse. This should help optimize the
1399 * loop, and reduce the number of iterations.
1400 */
1401 if (is_memory_range_within(begin, end, current_begin,
1402 current_end) ||
1403 is_memory_range_within(current_begin, current_end,
1404 begin, end)) {
1405 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00001406 "Overlapping memory ranges: %#lx - "
1407 "%#lx with %#lx - %#lx\n",
J-Alves5952d942022-12-22 16:03:00 +00001408 ipa_addr(begin), ipa_addr(end),
1409 ipa_addr(current_begin),
1410 ipa_addr(current_end));
1411 return true;
1412 }
1413 }
1414 }
1415
1416 return false;
1417}
1418
Jose Marinho09b1db82019-08-08 09:16:59 +01001419/**
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001420 * Validates and prepares memory to be sent from the calling VM to another.
Jose Marinho09b1db82019-08-08 09:16:59 +01001421 *
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001422 * This function requires the calling context to hold the <from> VM lock.
Jose Marinho09b1db82019-08-08 09:16:59 +01001423 *
1424 * Returns:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001425 * In case of error, one of the following values is returned:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001426 * 1) FFA_INVALID_PARAMETERS - The endpoint provided parameters were
Jose Marinho09b1db82019-08-08 09:16:59 +01001427 * erroneous;
Andrew Walbranf07f04d2020-05-01 18:09:00 +01001428 * 2) FFA_NO_MEMORY - Hafnium did not have sufficient memory to complete the
1429 * request.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001430 * 3) FFA_DENIED - The sender doesn't have sufficient access to send the
Andrew Walbrana65a1322020-04-06 19:32:32 +01001431 * memory with the given permissions.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001432 * Success is indicated by FFA_SUCCESS.
Jose Marinho09b1db82019-08-08 09:16:59 +01001433 */
Daniel Boulbya76fd912024-02-22 14:22:15 +00001434static struct ffa_value ffa_send_check_update(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001435 struct vm_locked from_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01001436 struct ffa_memory_region_constituent **fragments,
1437 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
J-Alves8f11cde2022-12-21 16:18:22 +00001438 uint32_t composite_total_page_count, uint32_t share_func,
Daniel Boulbya76fd912024-02-22 14:22:15 +00001439 struct ffa_memory_region *memory_region, struct mpool *page_pool,
1440 uint32_t *orig_from_mode_ret, bool *memory_protected)
Jose Marinho09b1db82019-08-08 09:16:59 +01001441{
Andrew Walbranca808b12020-05-15 17:22:28 +01001442 uint32_t i;
J-Alves8f11cde2022-12-21 16:18:22 +00001443 uint32_t j;
Jose Marinho09b1db82019-08-08 09:16:59 +01001444 uint32_t orig_from_mode;
J-Alves460d36c2023-10-12 17:02:15 +01001445 uint32_t clean_mode;
Jose Marinho09b1db82019-08-08 09:16:59 +01001446 uint32_t from_mode;
Jose Marinho09b1db82019-08-08 09:16:59 +01001447 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001448 struct ffa_value ret;
J-Alves8f11cde2022-12-21 16:18:22 +00001449 uint32_t constituents_total_page_count = 0;
J-Alves460d36c2023-10-12 17:02:15 +01001450 enum ffa_map_action map_action = MAP_ACTION_CHECK;
Daniel Boulbya76fd912024-02-22 14:22:15 +00001451 bool clear = memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR;
Jose Marinho09b1db82019-08-08 09:16:59 +01001452
1453 /*
Andrew Walbrana65a1322020-04-06 19:32:32 +01001454 * Make sure constituents are properly aligned to a 64-bit boundary. If
1455 * not we would get alignment faults trying to read (64-bit) values.
Jose Marinho09b1db82019-08-08 09:16:59 +01001456 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001457 for (i = 0; i < fragment_count; ++i) {
1458 if (!is_aligned(fragments[i], 8)) {
1459 dlog_verbose("Constituents not aligned.\n");
1460 return ffa_error(FFA_INVALID_PARAMETERS);
1461 }
J-Alves8f11cde2022-12-21 16:18:22 +00001462 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
1463 constituents_total_page_count +=
1464 fragments[i][j].page_count;
J-Alves5952d942022-12-22 16:03:00 +00001465 if (ffa_memory_check_overlap(
1466 fragments, fragment_constituent_counts,
1467 fragment_count, i, j)) {
1468 return ffa_error(FFA_INVALID_PARAMETERS);
1469 }
J-Alves8f11cde2022-12-21 16:18:22 +00001470 }
1471 }
1472
1473 if (constituents_total_page_count != composite_total_page_count) {
1474 dlog_verbose(
1475 "Composite page count differs from calculated page "
1476 "count from constituents.\n");
1477 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho09b1db82019-08-08 09:16:59 +01001478 }
1479
1480 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001481 * Check if the state transition is lawful for the sender, ensure that
1482 * all constituents of a memory region being shared are at the same
1483 * state.
Jose Marinho09b1db82019-08-08 09:16:59 +01001484 */
J-Alves460d36c2023-10-12 17:02:15 +01001485 ret = ffa_send_check_transition(
Daniel Boulbya76fd912024-02-22 14:22:15 +00001486 from_locked, share_func, memory_region, &orig_from_mode,
1487 fragments, fragment_constituent_counts, fragment_count,
Daniel Boulby4b846eb2024-05-23 17:32:23 +01001488 &from_mode, &map_action, clear);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001489 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001490 dlog_verbose("Invalid transition for send.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +01001491 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01001492 }
1493
Andrew Walbran37c574e2020-06-03 11:45:46 +01001494 if (orig_from_mode_ret != NULL) {
1495 *orig_from_mode_ret = orig_from_mode;
1496 }
1497
Jose Marinho09b1db82019-08-08 09:16:59 +01001498 /*
1499 * Create a local pool so any freed memory can't be used by another
1500 * thread. This is to ensure the original mapping can be restored if the
1501 * clear fails.
1502 */
Andrew Walbran475c1452020-02-07 13:22:22 +00001503 mpool_init_with_fallback(&local_page_pool, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +01001504
1505 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001506 * First reserve all required memory for the new page table entries
1507 * without committing, to make sure the entire operation will succeed
1508 * without exhausting the page pool.
J-Alves460d36c2023-10-12 17:02:15 +01001509 * Provide the map_action as populated by 'ffa_send_check_transition'.
1510 * It may request memory to be protected.
Jose Marinho09b1db82019-08-08 09:16:59 +01001511 */
J-Alvescf6253e2024-01-03 13:48:48 +00001512 ret = ffa_region_group_identity_map(
1513 from_locked, fragments, fragment_constituent_counts,
J-Alves460d36c2023-10-12 17:02:15 +01001514 fragment_count, from_mode, page_pool, map_action,
1515 memory_protected);
J-Alvescf6253e2024-01-03 13:48:48 +00001516 if (ret.func == FFA_ERROR_32) {
Jose Marinho09b1db82019-08-08 09:16:59 +01001517 goto out;
1518 }
1519
1520 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001521 * Update the mapping for the sender. This won't allocate because the
1522 * transaction was already prepared above, but may free pages in the
1523 * case that a whole block is being unmapped that was previously
1524 * partially mapped.
Jose Marinho09b1db82019-08-08 09:16:59 +01001525 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001526 CHECK(ffa_region_group_identity_map(
J-Alvescf6253e2024-01-03 13:48:48 +00001527 from_locked, fragments, fragment_constituent_counts,
1528 fragment_count, from_mode, &local_page_pool,
1529 MAP_ACTION_COMMIT, NULL)
1530 .func == FFA_SUCCESS_32);
Jose Marinho09b1db82019-08-08 09:16:59 +01001531
J-Alves460d36c2023-10-12 17:02:15 +01001532 /*
1533 * If memory has been protected, it is now part of the secure PAS
1534 * (happens for lend/donate from NWd to SWd), and the `orig_from_mode`
1535 * should have the MM_MODE_NS set, as such mask it in `clean_mode` for
1536 * SPM's S1 translation.
1537 * In case memory hasn't been protected, and it is in the non-secure
1538 * PAS (e.g. memory share from NWd to SWd), as such the SPM needs to
1539 * perform a non-secure memory access. In such case `clean_mode` takes
1540 * the same mode as `orig_from_mode`.
1541 */
1542 clean_mode = (memory_protected != NULL && *memory_protected)
1543 ? orig_from_mode & ~plat_ffa_other_world_mode()
1544 : orig_from_mode;
1545
Jose Marinho09b1db82019-08-08 09:16:59 +01001546 /* Clear the memory so no VM or device can see the previous contents. */
J-Alves460d36c2023-10-12 17:02:15 +01001547 if (clear && !ffa_clear_memory_constituents(
1548 clean_mode, fragments, fragment_constituent_counts,
1549 fragment_count, page_pool)) {
1550 map_action = (memory_protected != NULL && *memory_protected)
1551 ? MAP_ACTION_COMMIT_UNPROTECT
1552 : MAP_ACTION_COMMIT;
1553
Jose Marinho09b1db82019-08-08 09:16:59 +01001554 /*
1555 * On failure, roll back by returning memory to the sender. This
1556 * may allocate pages which were previously freed into
1557 * `local_page_pool` by the call above, but will never allocate
1558 * more pages than that so can never fail.
1559 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001560 CHECK(ffa_region_group_identity_map(
J-Alvescf6253e2024-01-03 13:48:48 +00001561 from_locked, fragments,
1562 fragment_constituent_counts, fragment_count,
1563 orig_from_mode, &local_page_pool,
1564 MAP_ACTION_COMMIT, NULL)
1565 .func == FFA_SUCCESS_32);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001566 ret = ffa_error(FFA_NO_MEMORY);
Jose Marinho09b1db82019-08-08 09:16:59 +01001567 goto out;
1568 }
1569
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001570 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001571
1572out:
1573 mpool_fini(&local_page_pool);
1574
1575 /*
1576 * Tidy up the page table by reclaiming failed mappings (if there was an
1577 * error) or merging entries into blocks where possible (on success).
1578 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001579 vm_ptable_defrag(from_locked, page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001580
1581 return ret;
1582}
1583
1584/**
1585 * Validates and maps memory shared from one VM to another.
1586 *
1587 * This function requires the calling context to hold the <to> lock.
1588 *
1589 * Returns:
1590 * In case of error, one of the following values is returned:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001591 * 1) FFA_INVALID_PARAMETERS - The endpoint provided parameters were
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001592 * erroneous;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001593 * 2) FFA_NO_MEMORY - Hafnium did not have sufficient memory to complete
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001594 * the request.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001595 * Success is indicated by FFA_SUCCESS.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001596 */
J-Alvesb5084cf2022-07-06 14:20:12 +01001597struct ffa_value ffa_retrieve_check_update(
J-Alves26483382023-04-20 12:01:49 +01001598 struct vm_locked to_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01001599 struct ffa_memory_region_constituent **fragments,
1600 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
J-Alves26483382023-04-20 12:01:49 +01001601 uint32_t sender_orig_mode, uint32_t share_func, bool clear,
J-Alves460d36c2023-10-12 17:02:15 +01001602 struct mpool *page_pool, uint32_t *response_mode, bool memory_protected)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001603{
Andrew Walbranca808b12020-05-15 17:22:28 +01001604 uint32_t i;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001605 uint32_t to_mode;
1606 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001607 struct ffa_value ret;
J-Alvesfd206052023-05-22 16:45:00 +01001608 enum ffa_map_action map_action = MAP_ACTION_COMMIT;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001609
1610 /*
Andrew Walbranca808b12020-05-15 17:22:28 +01001611 * Make sure constituents are properly aligned to a 64-bit boundary. If
1612 * not we would get alignment faults trying to read (64-bit) values.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001613 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001614 for (i = 0; i < fragment_count; ++i) {
1615 if (!is_aligned(fragments[i], 8)) {
J-Alvesb5084cf2022-07-06 14:20:12 +01001616 dlog_verbose("Fragment not properly aligned.\n");
Andrew Walbranca808b12020-05-15 17:22:28 +01001617 return ffa_error(FFA_INVALID_PARAMETERS);
1618 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001619 }
1620
1621 /*
Daniel Boulby4b846eb2024-05-23 17:32:23 +01001622 * Ensure the sender has write permissions if the memory needs to be
1623 * cleared.
1624 */
1625 if ((sender_orig_mode & MM_MODE_W) == 0 && clear == true) {
1626 dlog_verbose(
1627 "Cannot zero memory when the sender does not have "
1628 "write access\n");
1629 return ffa_error(FFA_DENIED);
1630 }
1631
1632 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001633 * Check if the state transition is lawful for the recipient, and ensure
1634 * that all constituents of the memory region being retrieved are at the
1635 * same state.
1636 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001637 ret = ffa_retrieve_check_transition(
1638 to_locked, share_func, fragments, fragment_constituent_counts,
J-Alvesfd206052023-05-22 16:45:00 +01001639 fragment_count, sender_orig_mode, &to_mode, memory_protected,
1640 &map_action);
J-Alves460d36c2023-10-12 17:02:15 +01001641
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001642 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001643 dlog_verbose("Invalid transition for retrieve.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +01001644 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001645 }
1646
1647 /*
J-Alves69cdfd92024-04-26 11:40:59 +01001648 * Create a local pool so any freed memory can't be used by
1649 * another thread. This is to ensure the original mapping can be
1650 * restored if the clear fails.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001651 */
1652 mpool_init_with_fallback(&local_page_pool, page_pool);
1653
1654 /*
J-Alves69cdfd92024-04-26 11:40:59 +01001655 * Memory retrieves from the NWd VMs don't require update to S2 PTs on
1656 * retrieve request.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001657 */
J-Alves69cdfd92024-04-26 11:40:59 +01001658 if (map_action != MAP_ACTION_NONE) {
1659 /*
1660 * First reserve all required memory for the new page table
1661 * entries in the recipient page tables without committing, to
1662 * make sure the entire operation will succeed without
1663 * exhausting the page pool.
1664 */
1665 ret = ffa_region_group_identity_map(
1666 to_locked, fragments, fragment_constituent_counts,
1667 fragment_count, to_mode, page_pool, MAP_ACTION_CHECK,
1668 NULL);
1669 if (ret.func == FFA_ERROR_32) {
1670 /* TODO: partial defrag of failed range. */
1671 goto out;
1672 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001673 }
1674
1675 /* Clear the memory so no VM or device can see the previous contents. */
J-Alves7db32002021-12-14 14:44:50 +00001676 if (clear &&
J-Alves26483382023-04-20 12:01:49 +01001677 !ffa_clear_memory_constituents(sender_orig_mode, fragments,
1678 fragment_constituent_counts,
1679 fragment_count, page_pool)) {
J-Alvesb5084cf2022-07-06 14:20:12 +01001680 dlog_verbose("Couldn't clear constituents.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001681 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001682 goto out;
1683 }
1684
J-Alves69cdfd92024-04-26 11:40:59 +01001685 if (map_action != MAP_ACTION_NONE) {
1686 /*
1687 * Complete the transfer by mapping the memory into the
1688 * recipient. This won't allocate because the transaction was
1689 * already prepared above, so it doesn't need to use the
1690 * `local_page_pool`.
1691 */
1692 CHECK(ffa_region_group_identity_map(to_locked, fragments,
1693 fragment_constituent_counts,
1694 fragment_count, to_mode,
1695 page_pool, map_action, NULL)
1696 .func == FFA_SUCCESS_32);
Jose Marinho09b1db82019-08-08 09:16:59 +01001697
J-Alves69cdfd92024-04-26 11:40:59 +01001698 /*
1699 * Return the mode used in mapping the memory in retriever's PT.
1700 */
1701 if (response_mode != NULL) {
1702 *response_mode = to_mode;
1703 }
J-Alves460d36c2023-10-12 17:02:15 +01001704 }
1705
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001706 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Jose Marinho09b1db82019-08-08 09:16:59 +01001707
1708out:
1709 mpool_fini(&local_page_pool);
1710
1711 /*
Andrew Walbranf07f04d2020-05-01 18:09:00 +01001712 * Tidy up the page table by reclaiming failed mappings (if there was an
1713 * error) or merging entries into blocks where possible (on success).
Jose Marinho09b1db82019-08-08 09:16:59 +01001714 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001715 vm_ptable_defrag(to_locked, page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001716
1717 return ret;
1718}
1719
Andrew Walbran996d1d12020-05-27 14:08:43 +01001720static struct ffa_value ffa_relinquish_check_update(
J-Alves26483382023-04-20 12:01:49 +01001721 struct vm_locked from_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01001722 struct ffa_memory_region_constituent **fragments,
1723 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
J-Alves69cdfd92024-04-26 11:40:59 +01001724 uint32_t sender_orig_mode, struct mpool *page_pool, bool clear)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001725{
1726 uint32_t orig_from_mode;
J-Alves69cdfd92024-04-26 11:40:59 +01001727 uint32_t clearing_mode;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001728 uint32_t from_mode;
1729 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001730 struct ffa_value ret;
J-Alves69cdfd92024-04-26 11:40:59 +01001731 enum ffa_map_action map_action;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001732
Andrew Walbranca808b12020-05-15 17:22:28 +01001733 ret = ffa_relinquish_check_transition(
1734 from_locked, &orig_from_mode, fragments,
J-Alves69cdfd92024-04-26 11:40:59 +01001735 fragment_constituent_counts, fragment_count, &from_mode,
1736 &map_action);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001737 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001738 dlog_verbose("Invalid transition for relinquish.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +01001739 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001740 }
1741
1742 /*
1743 * Create a local pool so any freed memory can't be used by another
1744 * thread. This is to ensure the original mapping can be restored if the
1745 * clear fails.
1746 */
1747 mpool_init_with_fallback(&local_page_pool, page_pool);
1748
J-Alves69cdfd92024-04-26 11:40:59 +01001749 if (map_action != MAP_ACTION_NONE) {
1750 clearing_mode = orig_from_mode;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001751
J-Alves69cdfd92024-04-26 11:40:59 +01001752 /*
1753 * First reserve all required memory for the new page table
1754 * entries without committing, to make sure the entire operation
1755 * will succeed without exhausting the page pool.
1756 */
1757 ret = ffa_region_group_identity_map(
1758 from_locked, fragments, fragment_constituent_counts,
1759 fragment_count, from_mode, page_pool, MAP_ACTION_CHECK,
1760 NULL);
1761 if (ret.func == FFA_ERROR_32) {
1762 goto out;
1763 }
1764
1765 /*
1766 * Update the mapping for the sender. This won't allocate
1767 * because the transaction was already prepared above, but may
1768 * free pages in the case that a whole block is being unmapped
1769 * that was previously partially mapped.
1770 */
1771 CHECK(ffa_region_group_identity_map(from_locked, fragments,
1772 fragment_constituent_counts,
1773 fragment_count, from_mode,
1774 &local_page_pool,
1775 MAP_ACTION_COMMIT, NULL)
1776 .func == FFA_SUCCESS_32);
1777 } else {
1778 /*
1779 * If the `map_action` is set to `MAP_ACTION_NONE`, S2 PTs
1780 * were not updated on retrieve/relinquish. These were updating
1781 * only the `share_state` structures. As such, use the sender's
1782 * original mode.
1783 */
1784 clearing_mode = sender_orig_mode;
1785 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001786
1787 /* Clear the memory so no VM or device can see the previous contents. */
J-Alves7db32002021-12-14 14:44:50 +00001788 if (clear &&
J-Alves69cdfd92024-04-26 11:40:59 +01001789 !ffa_clear_memory_constituents(clearing_mode, fragments,
J-Alves26483382023-04-20 12:01:49 +01001790 fragment_constituent_counts,
1791 fragment_count, page_pool)) {
J-Alves69cdfd92024-04-26 11:40:59 +01001792 if (map_action != MAP_ACTION_NONE) {
1793 /*
1794 * On failure, roll back by returning memory to the
1795 * sender. This may allocate pages which were previously
1796 * freed into `local_page_pool` by the call above, but
1797 * will never allocate more pages than that so can never
1798 * fail.
1799 */
1800 CHECK(ffa_region_group_identity_map(
1801 from_locked, fragments,
1802 fragment_constituent_counts,
1803 fragment_count, orig_from_mode,
1804 &local_page_pool, MAP_ACTION_COMMIT, NULL)
1805 .func == FFA_SUCCESS_32);
1806 }
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001807 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001808 goto out;
1809 }
1810
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001811 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001812
1813out:
1814 mpool_fini(&local_page_pool);
1815
1816 /*
1817 * Tidy up the page table by reclaiming failed mappings (if there was an
1818 * error) or merging entries into blocks where possible (on success).
1819 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001820 vm_ptable_defrag(from_locked, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +01001821
1822 return ret;
1823}
1824
1825/**
Andrew Walbranca808b12020-05-15 17:22:28 +01001826 * Complete a memory sending operation by checking that it is valid, updating
1827 * the sender page table, and then either marking the share state as having
1828 * completed sending (on success) or freeing it (on failure).
1829 *
1830 * Returns FFA_SUCCESS with the handle encoded, or the relevant FFA_ERROR.
1831 */
J-Alvesfdd29272022-07-19 13:16:31 +01001832struct ffa_value ffa_memory_send_complete(
Andrew Walbranca808b12020-05-15 17:22:28 +01001833 struct vm_locked from_locked, struct share_states_locked share_states,
Andrew Walbran37c574e2020-06-03 11:45:46 +01001834 struct ffa_memory_share_state *share_state, struct mpool *page_pool,
1835 uint32_t *orig_from_mode_ret)
Andrew Walbranca808b12020-05-15 17:22:28 +01001836{
1837 struct ffa_memory_region *memory_region = share_state->memory_region;
J-Alves8f11cde2022-12-21 16:18:22 +00001838 struct ffa_composite_memory_region *composite;
Andrew Walbranca808b12020-05-15 17:22:28 +01001839 struct ffa_value ret;
1840
1841 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +00001842 assert(share_states.share_states != NULL);
J-Alves8f11cde2022-12-21 16:18:22 +00001843 assert(memory_region != NULL);
1844 composite = ffa_memory_region_get_composite(memory_region, 0);
1845 assert(composite != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +01001846
1847 /* Check that state is valid in sender page table and update. */
1848 ret = ffa_send_check_update(
1849 from_locked, share_state->fragments,
1850 share_state->fragment_constituent_counts,
J-Alves8f11cde2022-12-21 16:18:22 +00001851 share_state->fragment_count, composite->page_count,
Daniel Boulbya76fd912024-02-22 14:22:15 +00001852 share_state->share_func, memory_region, page_pool,
J-Alves460d36c2023-10-12 17:02:15 +01001853 orig_from_mode_ret, &share_state->memory_protected);
Andrew Walbranca808b12020-05-15 17:22:28 +01001854 if (ret.func != FFA_SUCCESS_32) {
1855 /*
1856 * Free share state, it failed to send so it can't be retrieved.
1857 */
Karl Meakin4cec5e82023-06-30 16:30:22 +01001858 dlog_verbose("%s: failed to send check update: %s(%s)\n",
1859 __func__, ffa_func_name(ret.func),
1860 ffa_error_name(ffa_error_code(ret)));
Andrew Walbranca808b12020-05-15 17:22:28 +01001861 share_state_free(share_states, share_state, page_pool);
1862 return ret;
1863 }
1864
1865 share_state->sending_complete = true;
Karl Meakin4cec5e82023-06-30 16:30:22 +01001866 dlog_verbose("%s: marked sending complete.\n", __func__);
Andrew Walbranca808b12020-05-15 17:22:28 +01001867
J-Alvesee68c542020-10-29 17:48:20 +00001868 return ffa_mem_success(share_state->memory_region->handle);
Andrew Walbranca808b12020-05-15 17:22:28 +01001869}
1870
1871/**
Daniel Boulby9764ff62024-01-30 17:47:39 +00001872 * Check that the memory attributes match Hafnium expectations.
1873 * Cacheability:
1874 * - Normal Memory as `FFA_MEMORY_CACHE_WRITE_BACK`.
1875 * - Device memory as `FFA_MEMORY_DEV_NGNRNE`.
1876 *
1877 * Shareability:
1878 * - Inner Shareable.
Federico Recanatia98603a2021-12-20 18:04:03 +01001879 */
1880static struct ffa_value ffa_memory_attributes_validate(
J-Alves7a99d0d2023-02-08 13:49:48 +00001881 ffa_memory_attributes_t attributes)
Federico Recanatia98603a2021-12-20 18:04:03 +01001882{
1883 enum ffa_memory_type memory_type;
1884 enum ffa_memory_cacheability cacheability;
1885 enum ffa_memory_shareability shareability;
1886
Karl Meakin84710f32023-10-12 15:14:49 +01001887 memory_type = attributes.type;
Daniel Boulby9764ff62024-01-30 17:47:39 +00001888 cacheability = attributes.cacheability;
1889 if (memory_type == FFA_MEMORY_NORMAL_MEM &&
1890 cacheability != FFA_MEMORY_CACHE_WRITE_BACK) {
1891 dlog_verbose(
1892 "Normal Memory: Invalid cacheability %s, "
1893 "expected %s.\n",
1894 ffa_memory_cacheability_name(cacheability),
1895 ffa_memory_cacheability_name(
1896 FFA_MEMORY_CACHE_WRITE_BACK));
Federico Recanati3d953f32022-02-17 09:31:29 +01001897 return ffa_error(FFA_DENIED);
Federico Recanatia98603a2021-12-20 18:04:03 +01001898 }
Daniel Boulby9764ff62024-01-30 17:47:39 +00001899 if (memory_type == FFA_MEMORY_DEVICE_MEM &&
1900 cacheability != FFA_MEMORY_DEV_NGNRNE) {
1901 dlog_verbose(
1902 "Device Memory: Invalid cacheability %s, "
1903 "expected %s.\n",
1904 ffa_device_memory_cacheability_name(cacheability),
1905 ffa_device_memory_cacheability_name(
1906 FFA_MEMORY_DEV_NGNRNE));
Federico Recanati3d953f32022-02-17 09:31:29 +01001907 return ffa_error(FFA_DENIED);
Federico Recanatia98603a2021-12-20 18:04:03 +01001908 }
1909
Karl Meakin84710f32023-10-12 15:14:49 +01001910 shareability = attributes.shareability;
Federico Recanatia98603a2021-12-20 18:04:03 +01001911 if (shareability != FFA_MEMORY_INNER_SHAREABLE) {
Karl Meakinf98b2aa2023-10-12 16:09:59 +01001912 dlog_verbose("Invalid shareability %s, expected %s.\n",
1913 ffa_memory_shareability_name(shareability),
1914 ffa_memory_shareability_name(
1915 FFA_MEMORY_INNER_SHAREABLE));
Federico Recanati3d953f32022-02-17 09:31:29 +01001916 return ffa_error(FFA_DENIED);
Federico Recanatia98603a2021-12-20 18:04:03 +01001917 }
1918
1919 return (struct ffa_value){.func = FFA_SUCCESS_32};
1920}
1921
1922/**
Andrew Walbrana65a1322020-04-06 19:32:32 +01001923 * Check that the given `memory_region` represents a valid memory send request
1924 * of the given `share_func` type, return the clear flag and permissions via the
1925 * respective output parameters, and update the permissions if necessary.
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001926 *
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001927 * Returns FFA_SUCCESS if the request was valid, or the relevant FFA_ERROR if
Andrew Walbrana65a1322020-04-06 19:32:32 +01001928 * not.
1929 */
J-Alves66652252022-07-06 09:49:51 +01001930struct ffa_value ffa_memory_send_validate(
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001931 struct vm_locked from_locked, struct ffa_memory_region *memory_region,
1932 uint32_t memory_share_length, uint32_t fragment_length,
J-Alves363f5722022-04-25 17:37:37 +01001933 uint32_t share_func)
Andrew Walbrana65a1322020-04-06 19:32:32 +01001934{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001935 struct ffa_composite_memory_region *composite;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001936 struct ffa_memory_access *receiver =
1937 ffa_memory_region_get_receiver(memory_region, 0);
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001938 uint64_t receivers_end;
1939 uint64_t min_length;
Federico Recanati872cd692022-01-05 13:10:10 +01001940 uint32_t composite_memory_region_offset;
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001941 uint32_t constituents_start;
Andrew Walbran130a8ae2020-05-15 16:27:15 +01001942 uint32_t constituents_length;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001943 enum ffa_data_access data_access;
1944 enum ffa_instruction_access instruction_access;
Olivier Deprez4342a3c2022-02-28 09:37:25 +01001945 enum ffa_memory_security security_state;
Karl Meakinf98b2aa2023-10-12 16:09:59 +01001946 enum ffa_memory_type type;
Federico Recanatia98603a2021-12-20 18:04:03 +01001947 struct ffa_value ret;
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001948 const size_t minimum_first_fragment_length =
Daniel Boulby41ef8ba2023-10-13 17:01:22 +01001949 memory_region->receivers_offset +
1950 memory_region->memory_access_desc_size +
1951 sizeof(struct ffa_composite_memory_region);
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001952
1953 if (fragment_length < minimum_first_fragment_length) {
Karl Meakine8937d92024-03-19 16:04:25 +00001954 dlog_verbose("Fragment length %u too short (min %zu).\n",
1955 fragment_length, minimum_first_fragment_length);
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001956 return ffa_error(FFA_INVALID_PARAMETERS);
1957 }
1958
Demi Marie Obenour73a1e942023-02-04 14:09:18 -05001959 static_assert(sizeof(struct ffa_memory_region_constituent) == 16,
1960 "struct ffa_memory_region_constituent must be 16 bytes");
1961 if (!is_aligned(fragment_length,
1962 sizeof(struct ffa_memory_region_constituent)) ||
1963 !is_aligned(memory_share_length,
1964 sizeof(struct ffa_memory_region_constituent))) {
1965 dlog_verbose(
1966 "Fragment length %u or total length %u"
1967 " is not 16-byte aligned.\n",
1968 fragment_length, memory_share_length);
1969 return ffa_error(FFA_INVALID_PARAMETERS);
1970 }
1971
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001972 if (fragment_length > memory_share_length) {
1973 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00001974 "Fragment length %zu greater than total length %zu.\n",
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001975 (size_t)fragment_length, (size_t)memory_share_length);
1976 return ffa_error(FFA_INVALID_PARAMETERS);
1977 }
Andrew Walbrana65a1322020-04-06 19:32:32 +01001978
J-Alves95df0ef2022-12-07 10:09:48 +00001979 /* The sender must match the caller. */
1980 if ((!vm_id_is_current_world(from_locked.vm->id) &&
1981 vm_id_is_current_world(memory_region->sender)) ||
1982 (vm_id_is_current_world(from_locked.vm->id) &&
1983 memory_region->sender != from_locked.vm->id)) {
1984 dlog_verbose("Invalid memory sender ID.\n");
1985 return ffa_error(FFA_DENIED);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001986 }
1987
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001988 if (memory_region->receiver_count <= 0) {
1989 dlog_verbose("No receivers!\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001990 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001991 }
1992
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001993 /*
1994 * Ensure that the composite header is within the memory bounds and
1995 * doesn't overlap the first part of the message. Cast to uint64_t
1996 * to prevent overflow.
1997 */
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001998 receivers_end = ((uint64_t)memory_region->memory_access_desc_size *
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001999 (uint64_t)memory_region->receiver_count) +
Daniel Boulby41ef8ba2023-10-13 17:01:22 +01002000 memory_region->receivers_offset;
Demi Marie Obenourd4677412023-02-03 20:35:12 -05002001 min_length = receivers_end +
2002 sizeof(struct ffa_composite_memory_region) +
2003 sizeof(struct ffa_memory_region_constituent);
2004 if (min_length > memory_share_length) {
Karl Meakine8937d92024-03-19 16:04:25 +00002005 dlog_verbose("Share too short: got %zu but minimum is %zu.\n",
Demi Marie Obenourd4677412023-02-03 20:35:12 -05002006 (size_t)memory_share_length, (size_t)min_length);
2007 return ffa_error(FFA_INVALID_PARAMETERS);
2008 }
2009
2010 composite_memory_region_offset =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002011 receiver->composite_memory_region_offset;
Andrew Walbrana65a1322020-04-06 19:32:32 +01002012
2013 /*
Demi Marie Obenourd4677412023-02-03 20:35:12 -05002014 * Check that the composite memory region descriptor is after the access
2015 * descriptors, is at least 16-byte aligned, and fits in the first
2016 * fragment.
Andrew Walbrana65a1322020-04-06 19:32:32 +01002017 */
Demi Marie Obenourd4677412023-02-03 20:35:12 -05002018 if ((composite_memory_region_offset < receivers_end) ||
2019 (composite_memory_region_offset % 16 != 0) ||
2020 (composite_memory_region_offset >
2021 fragment_length - sizeof(struct ffa_composite_memory_region))) {
2022 dlog_verbose(
2023 "Invalid composite memory region descriptor offset "
Karl Meakine8937d92024-03-19 16:04:25 +00002024 "%zu.\n",
Demi Marie Obenourd4677412023-02-03 20:35:12 -05002025 (size_t)composite_memory_region_offset);
2026 return ffa_error(FFA_INVALID_PARAMETERS);
2027 }
2028
2029 /*
2030 * Compute the start of the constituent regions. Already checked
2031 * to be not more than fragment_length and thus not more than
2032 * memory_share_length.
2033 */
2034 constituents_start = composite_memory_region_offset +
2035 sizeof(struct ffa_composite_memory_region);
2036 constituents_length = memory_share_length - constituents_start;
2037
2038 /*
2039 * Check that the number of constituents is consistent with the length
2040 * of the constituent region.
2041 */
2042 composite = ffa_memory_region_get_composite(memory_region, 0);
2043 if ((constituents_length %
2044 sizeof(struct ffa_memory_region_constituent) !=
2045 0) ||
2046 ((constituents_length /
2047 sizeof(struct ffa_memory_region_constituent)) !=
2048 composite->constituent_count)) {
Karl Meakine8937d92024-03-19 16:04:25 +00002049 dlog_verbose("Invalid length %zu or composite offset %zu.\n",
Demi Marie Obenourd4677412023-02-03 20:35:12 -05002050 (size_t)memory_share_length,
2051 (size_t)composite_memory_region_offset);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002052 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01002053 }
Andrew Walbranca808b12020-05-15 17:22:28 +01002054 if (fragment_length < memory_share_length &&
2055 fragment_length < HF_MAILBOX_SIZE) {
2056 dlog_warning(
2057 "Initial fragment length %d smaller than mailbox "
2058 "size.\n",
2059 fragment_length);
2060 }
Andrew Walbrana65a1322020-04-06 19:32:32 +01002061
Andrew Walbrana65a1322020-04-06 19:32:32 +01002062 /*
2063 * Clear is not allowed for memory sharing, as the sender still has
2064 * access to the memory.
2065 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002066 if ((memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR) &&
J-Alves95fbb312024-03-20 15:19:16 +00002067 (share_func == FFA_MEM_SHARE_32 ||
2068 share_func == FFA_MEM_SHARE_64)) {
Andrew Walbrana65a1322020-04-06 19:32:32 +01002069 dlog_verbose("Memory can't be cleared while being shared.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002070 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01002071 }
2072
2073 /* No other flags are allowed/supported here. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002074 if (memory_region->flags & ~FFA_MEMORY_REGION_FLAG_CLEAR) {
Andrew Walbrana65a1322020-04-06 19:32:32 +01002075 dlog_verbose("Invalid flags %#x.\n", memory_region->flags);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002076 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01002077 }
2078
J-Alves363f5722022-04-25 17:37:37 +01002079 /* Check that the permissions are valid, for each specified receiver. */
2080 for (uint32_t i = 0U; i < memory_region->receiver_count; i++) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002081 struct ffa_memory_region_attributes receiver_permissions;
2082
2083 receiver = ffa_memory_region_get_receiver(memory_region, i);
2084 assert(receiver != NULL);
2085 receiver_permissions = receiver->receiver_permissions;
J-Alves363f5722022-04-25 17:37:37 +01002086 ffa_memory_access_permissions_t permissions =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002087 receiver_permissions.permissions;
2088 ffa_id_t receiver_id = receiver_permissions.receiver;
J-Alves363f5722022-04-25 17:37:37 +01002089
2090 if (memory_region->sender == receiver_id) {
2091 dlog_verbose("Can't share memory with itself.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002092 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01002093 }
Federico Recanati85090c42021-12-15 13:17:54 +01002094
J-Alves363f5722022-04-25 17:37:37 +01002095 for (uint32_t j = i + 1; j < memory_region->receiver_count;
2096 j++) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002097 struct ffa_memory_access *other_receiver =
2098 ffa_memory_region_get_receiver(memory_region,
2099 j);
2100 assert(other_receiver != NULL);
2101
J-Alves363f5722022-04-25 17:37:37 +01002102 if (receiver_id ==
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002103 other_receiver->receiver_permissions.receiver) {
J-Alves363f5722022-04-25 17:37:37 +01002104 dlog_verbose(
2105 "Repeated receiver(%x) in memory send "
2106 "operation.\n",
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002107 other_receiver->receiver_permissions
2108 .receiver);
J-Alves363f5722022-04-25 17:37:37 +01002109 return ffa_error(FFA_INVALID_PARAMETERS);
2110 }
2111 }
2112
2113 if (composite_memory_region_offset !=
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002114 receiver->composite_memory_region_offset) {
J-Alves363f5722022-04-25 17:37:37 +01002115 dlog_verbose(
2116 "All ffa_memory_access should point to the "
2117 "same composite memory region offset.\n");
2118 return ffa_error(FFA_INVALID_PARAMETERS);
2119 }
2120
Karl Meakin84710f32023-10-12 15:14:49 +01002121 data_access = permissions.data_access;
2122 instruction_access = permissions.instruction_access;
J-Alves363f5722022-04-25 17:37:37 +01002123 if (data_access == FFA_DATA_ACCESS_RESERVED ||
2124 instruction_access == FFA_INSTRUCTION_ACCESS_RESERVED) {
2125 dlog_verbose(
2126 "Reserved value for receiver permissions "
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002127 "(data_access = %s, instruction_access = %s)\n",
2128 ffa_data_access_name(data_access),
2129 ffa_instruction_access_name(
2130 instruction_access));
J-Alves363f5722022-04-25 17:37:37 +01002131 return ffa_error(FFA_INVALID_PARAMETERS);
2132 }
2133 if (instruction_access !=
2134 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
2135 dlog_verbose(
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002136 "Invalid instruction access permissions %s "
2137 "for sending memory, expected %s.\n",
2138 ffa_instruction_access_name(instruction_access),
2139 ffa_instruction_access_name(
2140 FFA_INSTRUCTION_ACCESS_RESERVED));
J-Alves363f5722022-04-25 17:37:37 +01002141 return ffa_error(FFA_INVALID_PARAMETERS);
2142 }
J-Alves95fbb312024-03-20 15:19:16 +00002143 if (share_func == FFA_MEM_SHARE_32 ||
2144 share_func == FFA_MEM_SHARE_64) {
J-Alves363f5722022-04-25 17:37:37 +01002145 if (data_access == FFA_DATA_ACCESS_NOT_SPECIFIED) {
2146 dlog_verbose(
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002147 "Invalid data access permissions %s "
2148 "for sharing memory, expected %s.\n",
2149 ffa_data_access_name(data_access),
2150 ffa_data_access_name(
2151 FFA_DATA_ACCESS_NOT_SPECIFIED));
J-Alves363f5722022-04-25 17:37:37 +01002152 return ffa_error(FFA_INVALID_PARAMETERS);
2153 }
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002154 /*
2155 * According to section 10.10.3 of the FF-A v1.1 EAC0
2156 * spec, NX is required for share operations (but must
2157 * not be specified by the sender) so set it in the
2158 * copy that we store, ready to be returned to the
2159 * retriever.
2160 */
2161 if (vm_id_is_current_world(receiver_id)) {
Karl Meakin84710f32023-10-12 15:14:49 +01002162 permissions.instruction_access =
2163 FFA_INSTRUCTION_ACCESS_NX;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002164 receiver_permissions.permissions = permissions;
2165 }
J-Alves363f5722022-04-25 17:37:37 +01002166 }
J-Alves95fbb312024-03-20 15:19:16 +00002167 if ((share_func == FFA_MEM_LEND_32 ||
2168 share_func == FFA_MEM_LEND_64) &&
J-Alves363f5722022-04-25 17:37:37 +01002169 data_access == FFA_DATA_ACCESS_NOT_SPECIFIED) {
2170 dlog_verbose(
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002171 "Invalid data access permissions %s for "
2172 "lending memory, expected %s.\n",
2173 ffa_data_access_name(data_access),
2174 ffa_data_access_name(
2175 FFA_DATA_ACCESS_NOT_SPECIFIED));
J-Alves363f5722022-04-25 17:37:37 +01002176 return ffa_error(FFA_INVALID_PARAMETERS);
2177 }
2178
J-Alves95fbb312024-03-20 15:19:16 +00002179 if ((share_func == FFA_MEM_DONATE_32 ||
2180 share_func == FFA_MEM_DONATE_64) &&
J-Alves363f5722022-04-25 17:37:37 +01002181 data_access != FFA_DATA_ACCESS_NOT_SPECIFIED) {
2182 dlog_verbose(
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002183 "Invalid data access permissions %s for "
2184 "donating memory, expected %s.\n",
2185 ffa_data_access_name(data_access),
2186 ffa_data_access_name(
2187 FFA_DATA_ACCESS_NOT_SPECIFIED));
J-Alves363f5722022-04-25 17:37:37 +01002188 return ffa_error(FFA_INVALID_PARAMETERS);
2189 }
Andrew Walbrana65a1322020-04-06 19:32:32 +01002190 }
2191
Olivier Deprez4342a3c2022-02-28 09:37:25 +01002192 /* Memory region attributes NS-Bit MBZ for FFA_MEM_SHARE/LEND/DONATE. */
Karl Meakin84710f32023-10-12 15:14:49 +01002193 security_state = memory_region->attributes.security;
Olivier Deprez4342a3c2022-02-28 09:37:25 +01002194 if (security_state != FFA_MEMORY_SECURITY_UNSPECIFIED) {
2195 dlog_verbose(
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002196 "Invalid security state %s for memory share operation, "
2197 "expected %s.\n",
2198 ffa_memory_security_name(security_state),
2199 ffa_memory_security_name(
2200 FFA_MEMORY_SECURITY_UNSPECIFIED));
Olivier Deprez4342a3c2022-02-28 09:37:25 +01002201 return ffa_error(FFA_INVALID_PARAMETERS);
2202 }
2203
Federico Recanatid937f5e2021-12-20 17:38:23 +01002204 /*
J-Alves807794e2022-06-16 13:42:47 +01002205 * If a memory donate or lend with single borrower, the memory type
2206 * shall not be specified by the sender.
Federico Recanatid937f5e2021-12-20 17:38:23 +01002207 */
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002208 type = memory_region->attributes.type;
J-Alves807794e2022-06-16 13:42:47 +01002209 if (share_func == FFA_MEM_DONATE_32 ||
J-Alves95fbb312024-03-20 15:19:16 +00002210 share_func == FFA_MEM_DONATE_64 ||
2211 ((share_func == FFA_MEM_LEND_32 || share_func == FFA_MEM_LEND_64) &&
J-Alves807794e2022-06-16 13:42:47 +01002212 memory_region->receiver_count == 1)) {
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002213 if (type != FFA_MEMORY_NOT_SPECIFIED_MEM) {
J-Alves807794e2022-06-16 13:42:47 +01002214 dlog_verbose(
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002215 "Invalid memory type %s for memory share "
2216 "operation, expected %s.\n",
2217 ffa_memory_type_name(type),
2218 ffa_memory_type_name(
2219 FFA_MEMORY_NOT_SPECIFIED_MEM));
J-Alves807794e2022-06-16 13:42:47 +01002220 return ffa_error(FFA_INVALID_PARAMETERS);
2221 }
2222 } else {
2223 /*
2224 * Check that sender's memory attributes match Hafnium
2225 * expectations: Normal Memory, Inner shareable, Write-Back
2226 * Read-Allocate Write-Allocate Cacheable.
2227 */
2228 ret = ffa_memory_attributes_validate(memory_region->attributes);
2229 if (ret.func != FFA_SUCCESS_32) {
2230 return ret;
2231 }
Federico Recanatid937f5e2021-12-20 17:38:23 +01002232 }
2233
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002234 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbrana65a1322020-04-06 19:32:32 +01002235}
2236
2237/**
Andrew Walbranca808b12020-05-15 17:22:28 +01002238 * Gets the share state for continuing an operation to donate, lend or share
2239 * memory, and checks that it is a valid request.
2240 *
2241 * Returns FFA_SUCCESS if the request was valid, or the relevant FFA_ERROR if
2242 * not.
2243 */
J-Alvesfdd29272022-07-19 13:16:31 +01002244struct ffa_value ffa_memory_send_continue_validate(
Andrew Walbranca808b12020-05-15 17:22:28 +01002245 struct share_states_locked share_states, ffa_memory_handle_t handle,
J-Alves19e20cf2023-08-02 12:48:55 +01002246 struct ffa_memory_share_state **share_state_ret, ffa_id_t from_vm_id,
Andrew Walbranca808b12020-05-15 17:22:28 +01002247 struct mpool *page_pool)
2248{
2249 struct ffa_memory_share_state *share_state;
2250 struct ffa_memory_region *memory_region;
2251
Daniel Boulbya2f8c662021-11-26 17:52:53 +00002252 assert(share_state_ret != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +01002253
2254 /*
2255 * Look up the share state by handle and make sure that the VM ID
2256 * matches.
2257 */
Karl Meakin4a2854a2023-06-30 16:26:52 +01002258 share_state = get_share_state(share_states, handle);
J-Alvesb56aac82023-11-10 09:44:43 +00002259 if (share_state == NULL) {
Andrew Walbranca808b12020-05-15 17:22:28 +01002260 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00002261 "Invalid handle %#lx for memory send continuation.\n",
Andrew Walbranca808b12020-05-15 17:22:28 +01002262 handle);
2263 return ffa_error(FFA_INVALID_PARAMETERS);
2264 }
2265 memory_region = share_state->memory_region;
2266
J-Alvesfdd29272022-07-19 13:16:31 +01002267 if (vm_id_is_current_world(from_vm_id) &&
2268 memory_region->sender != from_vm_id) {
Andrew Walbranca808b12020-05-15 17:22:28 +01002269 dlog_verbose("Invalid sender %d.\n", memory_region->sender);
2270 return ffa_error(FFA_INVALID_PARAMETERS);
2271 }
2272
2273 if (share_state->sending_complete) {
2274 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00002275 "Sending of memory handle %#lx is already complete.\n",
Andrew Walbranca808b12020-05-15 17:22:28 +01002276 handle);
2277 return ffa_error(FFA_INVALID_PARAMETERS);
2278 }
2279
2280 if (share_state->fragment_count == MAX_FRAGMENTS) {
2281 /*
2282 * Log a warning as this is a sign that MAX_FRAGMENTS should
2283 * probably be increased.
2284 */
2285 dlog_warning(
Karl Meakine8937d92024-03-19 16:04:25 +00002286 "Too many fragments for memory share with handle %#lx; "
Andrew Walbranca808b12020-05-15 17:22:28 +01002287 "only %d supported.\n",
2288 handle, MAX_FRAGMENTS);
2289 /* Free share state, as it's not possible to complete it. */
2290 share_state_free(share_states, share_state, page_pool);
2291 return ffa_error(FFA_NO_MEMORY);
2292 }
2293
2294 *share_state_ret = share_state;
2295
2296 return (struct ffa_value){.func = FFA_SUCCESS_32};
2297}
2298
2299/**
J-Alves95df0ef2022-12-07 10:09:48 +00002300 * Checks if there is at least one receiver from the other world.
2301 */
J-Alvesfdd29272022-07-19 13:16:31 +01002302bool memory_region_receivers_from_other_world(
J-Alves95df0ef2022-12-07 10:09:48 +00002303 struct ffa_memory_region *memory_region)
2304{
2305 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002306 struct ffa_memory_access *receiver =
2307 ffa_memory_region_get_receiver(memory_region, i);
2308 assert(receiver != NULL);
2309 ffa_id_t receiver_id = receiver->receiver_permissions.receiver;
2310
2311 if (!vm_id_is_current_world(receiver_id)) {
J-Alves95df0ef2022-12-07 10:09:48 +00002312 return true;
2313 }
2314 }
2315 return false;
2316}
2317
2318/**
J-Alves9da280b2022-12-21 14:55:39 +00002319 * Validates a call to donate, lend or share memory in which Hafnium is the
2320 * designated allocator of the memory handle. In practice, this also means
2321 * Hafnium is responsible for managing the state structures for the transaction.
2322 * If Hafnium is the SPMC, it should allocate the memory handle when either the
2323 * sender is an SP or there is at least one borrower that is an SP.
2324 * If Hafnium is the hypervisor, it should allocate the memory handle when
2325 * operation involves only NWd VMs.
2326 *
2327 * If validation goes well, Hafnium updates the stage-2 page tables of the
2328 * sender. Validation consists of checking if the message length and number of
2329 * memory region constituents match, and if the transition is valid for the
2330 * type of memory sending operation.
Andrew Walbran475c1452020-02-07 13:22:22 +00002331 *
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002332 * Assumes that the caller has already found and locked the sender VM and copied
2333 * the memory region descriptor from the sender's TX buffer to a freshly
2334 * allocated page from Hafnium's internal pool. The caller must have also
2335 * validated that the receiver VM ID is valid.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002336 *
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002337 * This function takes ownership of the `memory_region` passed in and will free
2338 * it when necessary; it must not be freed by the caller.
Jose Marinho09b1db82019-08-08 09:16:59 +01002339 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002340struct ffa_value ffa_memory_send(struct vm_locked from_locked,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002341 struct ffa_memory_region *memory_region,
Andrew Walbran130a8ae2020-05-15 16:27:15 +01002342 uint32_t memory_share_length,
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002343 uint32_t fragment_length, uint32_t share_func,
2344 struct mpool *page_pool)
Jose Marinho09b1db82019-08-08 09:16:59 +01002345{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002346 struct ffa_value ret;
Andrew Walbranca808b12020-05-15 17:22:28 +01002347 struct share_states_locked share_states;
2348 struct ffa_memory_share_state *share_state;
Jose Marinho09b1db82019-08-08 09:16:59 +01002349
2350 /*
Andrew Walbrana65a1322020-04-06 19:32:32 +01002351 * If there is an error validating the `memory_region` then we need to
2352 * free it because we own it but we won't be storing it in a share state
2353 * after all.
Jose Marinho09b1db82019-08-08 09:16:59 +01002354 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002355 ret = ffa_memory_send_validate(from_locked, memory_region,
2356 memory_share_length, fragment_length,
J-Alves363f5722022-04-25 17:37:37 +01002357 share_func);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002358 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002359 mpool_free(page_pool, memory_region);
Andrew Walbrana65a1322020-04-06 19:32:32 +01002360 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01002361 }
2362
Andrew Walbrana65a1322020-04-06 19:32:32 +01002363 /* Set flag for share function, ready to be retrieved later. */
2364 switch (share_func) {
J-Alves95fbb312024-03-20 15:19:16 +00002365 case FFA_MEM_SHARE_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002366 case FFA_MEM_SHARE_32:
Andrew Walbrana65a1322020-04-06 19:32:32 +01002367 memory_region->flags |=
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002368 FFA_MEMORY_REGION_TRANSACTION_TYPE_SHARE;
Andrew Walbrana65a1322020-04-06 19:32:32 +01002369 break;
J-Alves95fbb312024-03-20 15:19:16 +00002370 case FFA_MEM_LEND_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002371 case FFA_MEM_LEND_32:
2372 memory_region->flags |= FFA_MEMORY_REGION_TRANSACTION_TYPE_LEND;
Andrew Walbrana65a1322020-04-06 19:32:32 +01002373 break;
J-Alves95fbb312024-03-20 15:19:16 +00002374 case FFA_MEM_DONATE_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002375 case FFA_MEM_DONATE_32:
Andrew Walbrana65a1322020-04-06 19:32:32 +01002376 memory_region->flags |=
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002377 FFA_MEMORY_REGION_TRANSACTION_TYPE_DONATE;
Andrew Walbrana65a1322020-04-06 19:32:32 +01002378 break;
Jose Marinho09b1db82019-08-08 09:16:59 +01002379 }
2380
Andrew Walbranca808b12020-05-15 17:22:28 +01002381 share_states = share_states_lock();
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002382 /*
2383 * Allocate a share state before updating the page table. Otherwise if
2384 * updating the page table succeeded but allocating the share state
2385 * failed then it would leave the memory in a state where nobody could
2386 * get it back.
2387 */
Karl Meakin52cdfe72023-06-30 14:49:10 +01002388 share_state = allocate_share_state(share_states, share_func,
2389 memory_region, fragment_length,
2390 FFA_MEMORY_HANDLE_INVALID);
J-Alvesb56aac82023-11-10 09:44:43 +00002391 if (share_state == NULL) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002392 dlog_verbose("Failed to allocate share state.\n");
2393 mpool_free(page_pool, memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +01002394 ret = ffa_error(FFA_NO_MEMORY);
2395 goto out;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002396 }
2397
Andrew Walbranca808b12020-05-15 17:22:28 +01002398 if (fragment_length == memory_share_length) {
2399 /* No more fragments to come, everything fit in one message. */
J-Alves2a0d2882020-10-29 14:49:50 +00002400 ret = ffa_memory_send_complete(
2401 from_locked, share_states, share_state, page_pool,
2402 &(share_state->sender_orig_mode));
Andrew Walbranca808b12020-05-15 17:22:28 +01002403 } else {
J-Alvesfdd29272022-07-19 13:16:31 +01002404 /*
2405 * Use sender ID from 'memory_region' assuming
2406 * that at this point it has been validated:
2407 * - MBZ at virtual FF-A instance.
2408 */
J-Alves19e20cf2023-08-02 12:48:55 +01002409 ffa_id_t sender_to_ret =
J-Alvesfdd29272022-07-19 13:16:31 +01002410 (from_locked.vm->id == HF_OTHER_WORLD_ID)
2411 ? memory_region->sender
2412 : 0;
Andrew Walbranca808b12020-05-15 17:22:28 +01002413 ret = (struct ffa_value){
2414 .func = FFA_MEM_FRAG_RX_32,
J-Alvesee68c542020-10-29 17:48:20 +00002415 .arg1 = (uint32_t)memory_region->handle,
2416 .arg2 = (uint32_t)(memory_region->handle >> 32),
J-Alvesfdd29272022-07-19 13:16:31 +01002417 .arg3 = fragment_length,
2418 .arg4 = (uint32_t)(sender_to_ret & 0xffff) << 16};
Andrew Walbranca808b12020-05-15 17:22:28 +01002419 }
2420
2421out:
2422 share_states_unlock(&share_states);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002423 dump_share_states();
Andrew Walbranca808b12020-05-15 17:22:28 +01002424 return ret;
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002425}
2426
2427/**
J-Alves8505a8a2022-06-15 18:10:18 +01002428 * Continues an operation to donate, lend or share memory to a VM from current
2429 * world. If this is the last fragment then checks that the transition is valid
2430 * for the type of memory sending operation and updates the stage-2 page tables
2431 * of the sender.
Andrew Walbranca808b12020-05-15 17:22:28 +01002432 *
2433 * Assumes that the caller has already found and locked the sender VM and copied
2434 * the memory region descriptor from the sender's TX buffer to a freshly
2435 * allocated page from Hafnium's internal pool.
2436 *
2437 * This function takes ownership of the `fragment` passed in; it must not be
2438 * freed by the caller.
2439 */
2440struct ffa_value ffa_memory_send_continue(struct vm_locked from_locked,
2441 void *fragment,
2442 uint32_t fragment_length,
2443 ffa_memory_handle_t handle,
2444 struct mpool *page_pool)
2445{
2446 struct share_states_locked share_states = share_states_lock();
2447 struct ffa_memory_share_state *share_state;
2448 struct ffa_value ret;
2449 struct ffa_memory_region *memory_region;
2450
Demi Marie Obenour73a1e942023-02-04 14:09:18 -05002451 CHECK(is_aligned(fragment,
2452 alignof(struct ffa_memory_region_constituent)));
2453 if (fragment_length % sizeof(struct ffa_memory_region_constituent) !=
2454 0) {
2455 dlog_verbose("Fragment length %u misaligned.\n",
2456 fragment_length);
2457 ret = ffa_error(FFA_INVALID_PARAMETERS);
2458 goto out_free_fragment;
2459 }
2460
Andrew Walbranca808b12020-05-15 17:22:28 +01002461 ret = ffa_memory_send_continue_validate(share_states, handle,
2462 &share_state,
2463 from_locked.vm->id, page_pool);
2464 if (ret.func != FFA_SUCCESS_32) {
2465 goto out_free_fragment;
2466 }
2467 memory_region = share_state->memory_region;
2468
J-Alves95df0ef2022-12-07 10:09:48 +00002469 if (memory_region_receivers_from_other_world(memory_region)) {
Andrew Walbranca808b12020-05-15 17:22:28 +01002470 dlog_error(
2471 "Got hypervisor-allocated handle for memory send to "
J-Alves8505a8a2022-06-15 18:10:18 +01002472 "other world. This should never happen, and indicates "
2473 "a bug in "
Andrew Walbranca808b12020-05-15 17:22:28 +01002474 "EL3 code.\n");
2475 ret = ffa_error(FFA_INVALID_PARAMETERS);
2476 goto out_free_fragment;
2477 }
2478
2479 /* Add this fragment. */
2480 share_state->fragments[share_state->fragment_count] = fragment;
2481 share_state->fragment_constituent_counts[share_state->fragment_count] =
2482 fragment_length / sizeof(struct ffa_memory_region_constituent);
2483 share_state->fragment_count++;
2484
2485 /* Check whether the memory send operation is now ready to complete. */
2486 if (share_state_sending_complete(share_states, share_state)) {
J-Alves2a0d2882020-10-29 14:49:50 +00002487 ret = ffa_memory_send_complete(
2488 from_locked, share_states, share_state, page_pool,
2489 &(share_state->sender_orig_mode));
Andrew Walbranca808b12020-05-15 17:22:28 +01002490 } else {
2491 ret = (struct ffa_value){
2492 .func = FFA_MEM_FRAG_RX_32,
2493 .arg1 = (uint32_t)handle,
2494 .arg2 = (uint32_t)(handle >> 32),
2495 .arg3 = share_state_next_fragment_offset(share_states,
2496 share_state)};
2497 }
2498 goto out;
2499
2500out_free_fragment:
2501 mpool_free(page_pool, fragment);
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002502
2503out:
Andrew Walbranca808b12020-05-15 17:22:28 +01002504 share_states_unlock(&share_states);
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002505 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002506}
2507
Andrew Walbranca808b12020-05-15 17:22:28 +01002508/** Clean up after the receiver has finished retrieving a memory region. */
2509static void ffa_memory_retrieve_complete(
2510 struct share_states_locked share_states,
2511 struct ffa_memory_share_state *share_state, struct mpool *page_pool)
2512{
J-Alves95fbb312024-03-20 15:19:16 +00002513 if (share_state->share_func == FFA_MEM_DONATE_32 ||
2514 share_state->share_func == FFA_MEM_DONATE_64) {
Andrew Walbranca808b12020-05-15 17:22:28 +01002515 /*
2516 * Memory that has been donated can't be relinquished,
2517 * so no need to keep the share state around.
2518 */
2519 share_state_free(share_states, share_state, page_pool);
2520 dlog_verbose("Freed share state for donate.\n");
2521 }
2522}
2523
J-Alves2d8457f2022-10-05 11:06:41 +01002524/**
2525 * Initialises the given memory region descriptor to be used for an
2526 * `FFA_MEM_RETRIEVE_RESP`, including the given constituents for the first
2527 * fragment.
2528 * The memory region descriptor is initialized according to retriever's
2529 * FF-A version.
2530 *
2531 * Returns true on success, or false if the given constituents won't all fit in
2532 * the first fragment.
2533 */
2534static bool ffa_retrieved_memory_region_init(
Karl Meakin0e617d92024-04-05 12:55:22 +01002535 void *response, enum ffa_version ffa_version, size_t response_max_size,
J-Alves19e20cf2023-08-02 12:48:55 +01002536 ffa_id_t sender, ffa_memory_attributes_t attributes,
J-Alves2d8457f2022-10-05 11:06:41 +01002537 ffa_memory_region_flags_t flags, ffa_memory_handle_t handle,
Daniel Boulbyde974ca2023-12-12 13:53:31 +00002538 ffa_memory_access_permissions_t permissions,
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002539 struct ffa_memory_access *receivers, size_t receiver_count,
2540 uint32_t memory_access_desc_size, uint32_t page_count,
2541 uint32_t total_constituent_count,
J-Alves2d8457f2022-10-05 11:06:41 +01002542 const struct ffa_memory_region_constituent constituents[],
2543 uint32_t fragment_constituent_count, uint32_t *total_length,
2544 uint32_t *fragment_length)
2545{
2546 struct ffa_composite_memory_region *composite_memory_region;
J-Alves2d8457f2022-10-05 11:06:41 +01002547 uint32_t i;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002548 uint32_t composite_offset;
J-Alves2d8457f2022-10-05 11:06:41 +01002549 uint32_t constituents_offset;
J-Alves2d8457f2022-10-05 11:06:41 +01002550
2551 assert(response != NULL);
2552
Karl Meakin0e617d92024-04-05 12:55:22 +01002553 if (ffa_version == FFA_VERSION_1_0) {
J-Alves2d8457f2022-10-05 11:06:41 +01002554 struct ffa_memory_region_v1_0 *retrieve_response =
2555 (struct ffa_memory_region_v1_0 *)response;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002556 struct ffa_memory_access_v1_0 *receiver;
J-Alves2d8457f2022-10-05 11:06:41 +01002557
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002558 ffa_memory_region_init_header_v1_0(retrieve_response, sender,
2559 attributes, flags, handle, 0,
2560 receiver_count);
J-Alves2d8457f2022-10-05 11:06:41 +01002561
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002562 receiver = (struct ffa_memory_access_v1_0 *)
2563 retrieve_response->receivers;
J-Alves2d8457f2022-10-05 11:06:41 +01002564 receiver_count = retrieve_response->receiver_count;
2565
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002566 for (uint32_t i = 0; i < receiver_count; i++) {
2567 ffa_id_t receiver_id =
2568 receivers[i].receiver_permissions.receiver;
2569 ffa_memory_receiver_flags_t recv_flags =
2570 receivers[i].receiver_permissions.flags;
2571
2572 /*
2573 * Initialized here as in memory retrieve responses we
2574 * currently expect one borrower to be specified.
2575 */
2576 ffa_memory_access_init_v1_0(
Karl Meakin84710f32023-10-12 15:14:49 +01002577 receiver, receiver_id, permissions.data_access,
2578 permissions.instruction_access, recv_flags);
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002579 }
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002580
2581 composite_offset =
J-Alves2d8457f2022-10-05 11:06:41 +01002582 sizeof(struct ffa_memory_region_v1_0) +
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002583 receiver_count * sizeof(struct ffa_memory_access_v1_0);
2584 receiver->composite_memory_region_offset = composite_offset;
J-Alves2d8457f2022-10-05 11:06:41 +01002585
2586 composite_memory_region = ffa_memory_region_get_composite_v1_0(
2587 retrieve_response, 0);
2588 } else {
J-Alves2d8457f2022-10-05 11:06:41 +01002589 struct ffa_memory_region *retrieve_response =
2590 (struct ffa_memory_region *)response;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002591 struct ffa_memory_access *retrieve_response_receivers;
J-Alves2d8457f2022-10-05 11:06:41 +01002592
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002593 ffa_memory_region_init_header(
2594 retrieve_response, sender, attributes, flags, handle, 0,
2595 receiver_count, memory_access_desc_size);
J-Alves2d8457f2022-10-05 11:06:41 +01002596
2597 /*
2598 * Note that `sizeof(struct_ffa_memory_region)` and
2599 * `sizeof(struct ffa_memory_access)` must both be multiples of
2600 * 16 (as verified by the asserts in `ffa_memory.c`, so it is
2601 * guaranteed that the offset we calculate here is aligned to a
2602 * 64-bit boundary and so 64-bit values can be copied without
2603 * alignment faults.
2604 */
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002605 composite_offset =
Daniel Boulby41ef8ba2023-10-13 17:01:22 +01002606 retrieve_response->receivers_offset +
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002607 (uint32_t)(receiver_count *
2608 retrieve_response->memory_access_desc_size);
J-Alves2d8457f2022-10-05 11:06:41 +01002609
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002610 retrieve_response_receivers =
2611 ffa_memory_region_get_receiver(retrieve_response, 0);
2612 assert(retrieve_response_receivers != NULL);
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002613
2614 /*
2615 * Initialized here as in memory retrieve responses we currently
2616 * expect one borrower to be specified.
2617 */
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002618 memcpy_s(retrieve_response_receivers,
2619 sizeof(struct ffa_memory_access) * receiver_count,
2620 receivers,
2621 sizeof(struct ffa_memory_access) * receiver_count);
2622
2623 retrieve_response_receivers->composite_memory_region_offset =
2624 composite_offset;
2625
J-Alves2d8457f2022-10-05 11:06:41 +01002626 composite_memory_region =
2627 ffa_memory_region_get_composite(retrieve_response, 0);
2628 }
2629
J-Alves2d8457f2022-10-05 11:06:41 +01002630 assert(composite_memory_region != NULL);
2631
J-Alves2d8457f2022-10-05 11:06:41 +01002632 composite_memory_region->page_count = page_count;
2633 composite_memory_region->constituent_count = total_constituent_count;
2634 composite_memory_region->reserved_0 = 0;
2635
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002636 constituents_offset =
2637 composite_offset + sizeof(struct ffa_composite_memory_region);
J-Alves2d8457f2022-10-05 11:06:41 +01002638 if (constituents_offset +
2639 fragment_constituent_count *
2640 sizeof(struct ffa_memory_region_constituent) >
2641 response_max_size) {
2642 return false;
2643 }
2644
2645 for (i = 0; i < fragment_constituent_count; ++i) {
2646 composite_memory_region->constituents[i] = constituents[i];
2647 }
2648
2649 if (total_length != NULL) {
2650 *total_length =
2651 constituents_offset +
2652 composite_memory_region->constituent_count *
2653 sizeof(struct ffa_memory_region_constituent);
2654 }
2655 if (fragment_length != NULL) {
2656 *fragment_length =
2657 constituents_offset +
2658 fragment_constituent_count *
2659 sizeof(struct ffa_memory_region_constituent);
2660 }
2661
2662 return true;
2663}
2664
J-Alves96de29f2022-04-26 16:05:24 +01002665/**
2666 * Validates the retrieved permissions against those specified by the lender
2667 * of memory share operation. Optionally can help set the permissions to be used
2668 * for the S2 mapping, through the `permissions` argument.
J-Alvesdcad8992023-09-15 14:10:35 +01002669 * Returns FFA_SUCCESS if all the fields are valid. FFA_ERROR, with error code:
2670 * - FFA_INVALID_PARAMETERS -> if the fields have invalid values as per the
2671 * specification for each ABI.
2672 * - FFA_DENIED -> if the permissions specified by the retriever are not
2673 * less permissive than those provided by the sender.
J-Alves96de29f2022-04-26 16:05:24 +01002674 */
J-Alvesdcad8992023-09-15 14:10:35 +01002675static struct ffa_value ffa_memory_retrieve_is_memory_access_valid(
2676 uint32_t share_func, enum ffa_data_access sent_data_access,
J-Alves96de29f2022-04-26 16:05:24 +01002677 enum ffa_data_access requested_data_access,
2678 enum ffa_instruction_access sent_instruction_access,
2679 enum ffa_instruction_access requested_instruction_access,
J-Alvesdcad8992023-09-15 14:10:35 +01002680 ffa_memory_access_permissions_t *permissions, bool multiple_borrowers)
J-Alves96de29f2022-04-26 16:05:24 +01002681{
2682 switch (sent_data_access) {
2683 case FFA_DATA_ACCESS_NOT_SPECIFIED:
2684 case FFA_DATA_ACCESS_RW:
2685 if (requested_data_access == FFA_DATA_ACCESS_NOT_SPECIFIED ||
2686 requested_data_access == FFA_DATA_ACCESS_RW) {
2687 if (permissions != NULL) {
Karl Meakin84710f32023-10-12 15:14:49 +01002688 permissions->data_access = FFA_DATA_ACCESS_RW;
J-Alves96de29f2022-04-26 16:05:24 +01002689 }
2690 break;
2691 }
2692 /* Intentional fall-through. */
2693 case FFA_DATA_ACCESS_RO:
2694 if (requested_data_access == FFA_DATA_ACCESS_NOT_SPECIFIED ||
2695 requested_data_access == FFA_DATA_ACCESS_RO) {
2696 if (permissions != NULL) {
Karl Meakin84710f32023-10-12 15:14:49 +01002697 permissions->data_access = FFA_DATA_ACCESS_RO;
J-Alves96de29f2022-04-26 16:05:24 +01002698 }
2699 break;
2700 }
2701 dlog_verbose(
2702 "Invalid data access requested; sender specified "
2703 "permissions %#x but receiver requested %#x.\n",
2704 sent_data_access, requested_data_access);
J-Alvesdcad8992023-09-15 14:10:35 +01002705 return ffa_error(FFA_DENIED);
J-Alves96de29f2022-04-26 16:05:24 +01002706 case FFA_DATA_ACCESS_RESERVED:
2707 panic("Got unexpected FFA_DATA_ACCESS_RESERVED. Should be "
2708 "checked before this point.");
2709 }
2710
J-Alvesdcad8992023-09-15 14:10:35 +01002711 /*
2712 * For operations with a single borrower, If it is an FFA_MEMORY_LEND
2713 * or FFA_MEMORY_DONATE the retriever should have specifed the
2714 * instruction permissions it wishes to receive.
2715 */
2716 switch (share_func) {
J-Alves95fbb312024-03-20 15:19:16 +00002717 case FFA_MEM_SHARE_64:
J-Alvesdcad8992023-09-15 14:10:35 +01002718 case FFA_MEM_SHARE_32:
2719 if (requested_instruction_access !=
2720 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
2721 dlog_verbose(
2722 "%s: for share instruction permissions must "
2723 "NOT be specified.\n",
2724 __func__);
2725 return ffa_error(FFA_INVALID_PARAMETERS);
2726 }
2727 break;
J-Alves95fbb312024-03-20 15:19:16 +00002728 case FFA_MEM_LEND_64:
J-Alvesdcad8992023-09-15 14:10:35 +01002729 case FFA_MEM_LEND_32:
2730 /*
2731 * For operations with multiple borrowers only permit XN
2732 * permissions, and both Sender and borrower should have used
2733 * FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED.
2734 */
2735 if (multiple_borrowers) {
2736 if (requested_instruction_access !=
2737 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
2738 dlog_verbose(
2739 "%s: lend/share/donate with multiple "
2740 "borrowers "
2741 "instruction permissions must NOT be "
2742 "specified.\n",
2743 __func__);
2744 return ffa_error(FFA_INVALID_PARAMETERS);
2745 }
2746 break;
2747 }
2748 /* Fall through if the operation targets a single borrower. */
J-Alves95fbb312024-03-20 15:19:16 +00002749 case FFA_MEM_DONATE_64:
J-Alvesdcad8992023-09-15 14:10:35 +01002750 case FFA_MEM_DONATE_32:
2751 if (!multiple_borrowers &&
2752 requested_instruction_access ==
2753 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
2754 dlog_verbose(
2755 "%s: for lend/donate with single borrower "
2756 "instruction permissions must be speficified "
2757 "by borrower\n",
2758 __func__);
2759 return ffa_error(FFA_INVALID_PARAMETERS);
2760 }
2761 break;
2762 default:
2763 panic("%s: Wrong func id provided.\n", __func__);
2764 }
2765
J-Alves96de29f2022-04-26 16:05:24 +01002766 switch (sent_instruction_access) {
2767 case FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED:
2768 case FFA_INSTRUCTION_ACCESS_X:
J-Alvesdcad8992023-09-15 14:10:35 +01002769 if (requested_instruction_access == FFA_INSTRUCTION_ACCESS_X) {
J-Alves96de29f2022-04-26 16:05:24 +01002770 if (permissions != NULL) {
Karl Meakin84710f32023-10-12 15:14:49 +01002771 permissions->instruction_access =
2772 FFA_INSTRUCTION_ACCESS_X;
J-Alves96de29f2022-04-26 16:05:24 +01002773 }
2774 break;
2775 }
J-Alvesdcad8992023-09-15 14:10:35 +01002776 /*
2777 * Fall through if requested permissions are less
2778 * permissive than those provided by the sender.
2779 */
J-Alves96de29f2022-04-26 16:05:24 +01002780 case FFA_INSTRUCTION_ACCESS_NX:
2781 if (requested_instruction_access ==
2782 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED ||
2783 requested_instruction_access == FFA_INSTRUCTION_ACCESS_NX) {
2784 if (permissions != NULL) {
Karl Meakin84710f32023-10-12 15:14:49 +01002785 permissions->instruction_access =
2786 FFA_INSTRUCTION_ACCESS_NX;
J-Alves96de29f2022-04-26 16:05:24 +01002787 }
2788 break;
2789 }
2790 dlog_verbose(
2791 "Invalid instruction access requested; sender "
2792 "specified permissions %#x but receiver requested "
2793 "%#x.\n",
2794 sent_instruction_access, requested_instruction_access);
J-Alvesdcad8992023-09-15 14:10:35 +01002795 return ffa_error(FFA_DENIED);
J-Alves96de29f2022-04-26 16:05:24 +01002796 case FFA_INSTRUCTION_ACCESS_RESERVED:
2797 panic("Got unexpected FFA_INSTRUCTION_ACCESS_RESERVED. Should "
2798 "be checked before this point.");
2799 }
2800
J-Alvesdcad8992023-09-15 14:10:35 +01002801 return (struct ffa_value){.func = FFA_SUCCESS_32};
J-Alves96de29f2022-04-26 16:05:24 +01002802}
2803
2804/**
2805 * Validate the receivers' permissions in the retrieve request against those
2806 * specified by the lender.
2807 * In the `permissions` argument returns the permissions to set at S2 for the
2808 * caller to the FFA_MEMORY_RETRIEVE_REQ.
J-Alves3456e032023-07-20 12:20:05 +01002809 * The function looks into the flag to bypass multiple borrower checks:
2810 * - If not set returns FFA_SUCCESS if all specified permissions are valid.
2811 * - If set returns FFA_SUCCESS if the descriptor contains the permissions
2812 * to the caller of FFA_MEM_RETRIEVE_REQ and they are valid. Other permissions
2813 * are ignored, if provided.
J-Alves96de29f2022-04-26 16:05:24 +01002814 */
2815static struct ffa_value ffa_memory_retrieve_validate_memory_access_list(
2816 struct ffa_memory_region *memory_region,
J-Alves19e20cf2023-08-02 12:48:55 +01002817 struct ffa_memory_region *retrieve_request, ffa_id_t to_vm_id,
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002818 ffa_memory_access_permissions_t *permissions,
2819 struct ffa_memory_access **receiver_ret, uint32_t func_id)
J-Alves96de29f2022-04-26 16:05:24 +01002820{
2821 uint32_t retrieve_receiver_index;
J-Alves3456e032023-07-20 12:20:05 +01002822 bool bypass_multi_receiver_check =
2823 (retrieve_request->flags &
2824 FFA_MEMORY_REGION_FLAG_BYPASS_BORROWERS_CHECK) != 0U;
J-Alvesdcad8992023-09-15 14:10:35 +01002825 const uint32_t region_receiver_count = memory_region->receiver_count;
2826 struct ffa_value ret;
J-Alves96de29f2022-04-26 16:05:24 +01002827
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002828 assert(receiver_ret != NULL);
J-Alves96de29f2022-04-26 16:05:24 +01002829 assert(permissions != NULL);
2830
Karl Meakin84710f32023-10-12 15:14:49 +01002831 *permissions = (ffa_memory_access_permissions_t){0};
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002832
J-Alves3456e032023-07-20 12:20:05 +01002833 if (!bypass_multi_receiver_check) {
J-Alvesdcad8992023-09-15 14:10:35 +01002834 if (retrieve_request->receiver_count != region_receiver_count) {
J-Alves3456e032023-07-20 12:20:05 +01002835 dlog_verbose(
2836 "Retrieve request should contain same list of "
2837 "borrowers, as specified by the lender.\n");
2838 return ffa_error(FFA_INVALID_PARAMETERS);
2839 }
2840 } else {
2841 if (retrieve_request->receiver_count != 1) {
2842 dlog_verbose(
2843 "Set bypass multiple borrower check, receiver "
2844 "list must be sized 1 (%x)\n",
2845 memory_region->receiver_count);
2846 return ffa_error(FFA_INVALID_PARAMETERS);
2847 }
J-Alves96de29f2022-04-26 16:05:24 +01002848 }
2849
2850 retrieve_receiver_index = retrieve_request->receiver_count;
2851
J-Alves96de29f2022-04-26 16:05:24 +01002852 for (uint32_t i = 0U; i < retrieve_request->receiver_count; i++) {
2853 ffa_memory_access_permissions_t sent_permissions;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002854 struct ffa_memory_access *retrieve_request_receiver =
2855 ffa_memory_region_get_receiver(retrieve_request, i);
2856 assert(retrieve_request_receiver != NULL);
J-Alves96de29f2022-04-26 16:05:24 +01002857 ffa_memory_access_permissions_t requested_permissions =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002858 retrieve_request_receiver->receiver_permissions
2859 .permissions;
J-Alves19e20cf2023-08-02 12:48:55 +01002860 ffa_id_t current_receiver_id =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002861 retrieve_request_receiver->receiver_permissions
2862 .receiver;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002863 struct ffa_memory_access *receiver;
2864 uint32_t mem_region_receiver_index;
2865 bool permissions_RO;
2866 bool clear_memory_flags;
J-Alvesf220d572024-04-24 22:15:14 +01002867 /*
2868 * If the call is at the virtual FF-A instance the caller's
2869 * ID must match an entry in the memory access list.
2870 * In the SPMC, one of the specified receivers could be from
2871 * the NWd.
2872 */
2873 bool found_to_id = vm_id_is_current_world(to_vm_id)
2874 ? (current_receiver_id == to_vm_id)
2875 : (!vm_id_is_current_world(
2876 current_receiver_id));
J-Alves96de29f2022-04-26 16:05:24 +01002877
J-Alves3456e032023-07-20 12:20:05 +01002878 if (bypass_multi_receiver_check && !found_to_id) {
2879 dlog_verbose(
2880 "Bypass multiple borrower check for id %x.\n",
2881 current_receiver_id);
2882 continue;
2883 }
2884
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002885 if (retrieve_request_receiver->composite_memory_region_offset !=
2886 0U) {
2887 dlog_verbose(
2888 "Retriever specified address ranges not "
2889 "supported (got offset %d).\n",
2890 retrieve_request_receiver
2891 ->composite_memory_region_offset);
2892 return ffa_error(FFA_INVALID_PARAMETERS);
2893 }
2894
J-Alves96de29f2022-04-26 16:05:24 +01002895 /*
2896 * Find the current receiver in the transaction descriptor from
2897 * sender.
2898 */
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002899 mem_region_receiver_index =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002900 ffa_memory_region_get_receiver_index(
2901 memory_region, current_receiver_id);
J-Alves96de29f2022-04-26 16:05:24 +01002902
2903 if (mem_region_receiver_index ==
2904 memory_region->receiver_count) {
2905 dlog_verbose("%s: receiver %x not found\n", __func__,
2906 current_receiver_id);
2907 return ffa_error(FFA_DENIED);
2908 }
2909
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002910 receiver = ffa_memory_region_get_receiver(
2911 memory_region, mem_region_receiver_index);
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002912 assert(receiver != NULL);
2913
2914 sent_permissions = receiver->receiver_permissions.permissions;
J-Alves96de29f2022-04-26 16:05:24 +01002915
2916 if (found_to_id) {
2917 retrieve_receiver_index = i;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002918
2919 *receiver_ret = receiver;
J-Alves96de29f2022-04-26 16:05:24 +01002920 }
2921
2922 /*
J-Alvesdcad8992023-09-15 14:10:35 +01002923 * Check if retrieve request memory access list is valid:
2924 * - The retrieve request complies with the specification.
2925 * - Permissions are within those specified by the sender.
J-Alves96de29f2022-04-26 16:05:24 +01002926 */
J-Alvesdcad8992023-09-15 14:10:35 +01002927 ret = ffa_memory_retrieve_is_memory_access_valid(
Karl Meakin84710f32023-10-12 15:14:49 +01002928 func_id, sent_permissions.data_access,
2929 requested_permissions.data_access,
2930 sent_permissions.instruction_access,
2931 requested_permissions.instruction_access,
J-Alvesdcad8992023-09-15 14:10:35 +01002932 found_to_id ? permissions : NULL,
2933 region_receiver_count > 1);
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002934
J-Alvesdcad8992023-09-15 14:10:35 +01002935 if (ret.func != FFA_SUCCESS_32) {
2936 return ret;
J-Alves96de29f2022-04-26 16:05:24 +01002937 }
2938
Karl Meakin84710f32023-10-12 15:14:49 +01002939 permissions_RO =
2940 (permissions->data_access == FFA_DATA_ACCESS_RO);
J-Alvese5262372024-03-27 11:02:03 +00002941 clear_memory_flags =
2942 (retrieve_request->flags &
2943 (FFA_MEMORY_REGION_FLAG_CLEAR |
2944 FFA_MEMORY_REGION_FLAG_CLEAR_RELINQUISH)) != 0U;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002945
J-Alves96de29f2022-04-26 16:05:24 +01002946 /*
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002947 * Can't request PM to clear memory if only provided
2948 * with RO permissions.
J-Alves96de29f2022-04-26 16:05:24 +01002949 */
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002950 if (found_to_id && permissions_RO && clear_memory_flags) {
J-Alves96de29f2022-04-26 16:05:24 +01002951 dlog_verbose(
2952 "Receiver has RO permissions can not request "
2953 "clear.\n");
2954 return ffa_error(FFA_DENIED);
2955 }
Daniel Boulbyde974ca2023-12-12 13:53:31 +00002956
2957 /*
2958 * Check the impdef in the retrieve_request matches the value in
2959 * the original memory send.
2960 */
2961 if (ffa_version_from_memory_access_desc_size(
2962 memory_region->memory_access_desc_size) >=
Karl Meakin0e617d92024-04-05 12:55:22 +01002963 FFA_VERSION_1_2 &&
Daniel Boulbyde974ca2023-12-12 13:53:31 +00002964 ffa_version_from_memory_access_desc_size(
2965 retrieve_request->memory_access_desc_size) >=
Karl Meakin0e617d92024-04-05 12:55:22 +01002966 FFA_VERSION_1_2) {
Daniel Boulbyde974ca2023-12-12 13:53:31 +00002967 if (receiver->impdef.val[0] !=
2968 retrieve_request_receiver->impdef.val[0] ||
2969 receiver->impdef.val[1] !=
2970 retrieve_request_receiver->impdef.val[1]) {
2971 dlog_verbose(
2972 "Impdef value in memory send does not "
2973 "match retrieve request value "
Karl Meakine8937d92024-03-19 16:04:25 +00002974 "send value %#lx %#lx retrieve request "
2975 "value %#lx %#lx\n",
Daniel Boulbyde974ca2023-12-12 13:53:31 +00002976 receiver->impdef.val[0],
2977 receiver->impdef.val[1],
2978 retrieve_request_receiver->impdef
2979 .val[0],
2980 retrieve_request_receiver->impdef
2981 .val[1]);
2982 return ffa_error(FFA_INVALID_PARAMETERS);
2983 }
2984 }
J-Alves96de29f2022-04-26 16:05:24 +01002985 }
2986
2987 if (retrieve_receiver_index == retrieve_request->receiver_count) {
2988 dlog_verbose(
2989 "Retrieve request does not contain caller's (%x) "
2990 "permissions\n",
2991 to_vm_id);
2992 return ffa_error(FFA_INVALID_PARAMETERS);
2993 }
2994
2995 return (struct ffa_value){.func = FFA_SUCCESS_32};
2996}
2997
J-Alvesa9cd7e32022-07-01 13:49:33 +01002998/*
2999 * According to section 16.4.3 of FF-A v1.1 EAC0 specification, the hypervisor
3000 * may issue an FFA_MEM_RETRIEVE_REQ to obtain the memory region description
3001 * of a pending memory sharing operation whose allocator is the SPM, for
3002 * validation purposes before forwarding an FFA_MEM_RECLAIM call. In doing so
3003 * the memory region descriptor of the retrieve request must be zeroed with the
3004 * exception of the sender ID and handle.
3005 */
J-Alves4f0d9c12024-01-17 17:23:11 +00003006bool is_ffa_hypervisor_retrieve_request(struct ffa_memory_region *request,
3007 struct vm_locked to_locked)
J-Alvesa9cd7e32022-07-01 13:49:33 +01003008{
3009 return to_locked.vm->id == HF_HYPERVISOR_VM_ID &&
Karl Meakin84710f32023-10-12 15:14:49 +01003010 request->attributes.shareability == 0U &&
3011 request->attributes.cacheability == 0U &&
3012 request->attributes.type == 0U &&
3013 request->attributes.security == 0U && request->flags == 0U &&
J-Alvesa9cd7e32022-07-01 13:49:33 +01003014 request->tag == 0U && request->receiver_count == 0U &&
3015 plat_ffa_memory_handle_allocated_by_current_world(
3016 request->handle);
3017}
3018
3019/*
3020 * Helper to reset count of fragments retrieved by the hypervisor.
3021 */
3022static void ffa_memory_retrieve_complete_from_hyp(
3023 struct ffa_memory_share_state *share_state)
3024{
3025 if (share_state->hypervisor_fragment_count ==
3026 share_state->fragment_count) {
3027 share_state->hypervisor_fragment_count = 0;
3028 }
3029}
3030
J-Alves089004f2022-07-13 14:25:44 +01003031/**
J-Alves4f0d9c12024-01-17 17:23:11 +00003032 * Prepares the return of the ffa_value for the memory retrieve response.
3033 */
3034static struct ffa_value ffa_memory_retrieve_resp(uint32_t total_length,
3035 uint32_t fragment_length)
3036{
3037 return (struct ffa_value){.func = FFA_MEM_RETRIEVE_RESP_32,
3038 .arg1 = total_length,
3039 .arg2 = fragment_length};
3040}
3041
3042/**
J-Alves089004f2022-07-13 14:25:44 +01003043 * Validate that the memory region descriptor provided by the borrower on
3044 * FFA_MEM_RETRIEVE_REQ, against saved memory region provided by lender at the
3045 * memory sharing call.
3046 */
3047static struct ffa_value ffa_memory_retrieve_validate(
J-Alves4f0d9c12024-01-17 17:23:11 +00003048 ffa_id_t to_id, struct ffa_memory_region *retrieve_request,
3049 uint32_t retrieve_request_length,
J-Alves089004f2022-07-13 14:25:44 +01003050 struct ffa_memory_region *memory_region, uint32_t *receiver_index,
3051 uint32_t share_func)
3052{
3053 ffa_memory_region_flags_t transaction_type =
3054 retrieve_request->flags &
3055 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK;
Olivier Deprez4342a3c2022-02-28 09:37:25 +01003056 enum ffa_memory_security security_state;
J-Alves4f0d9c12024-01-17 17:23:11 +00003057 const uint64_t memory_access_desc_size =
3058 retrieve_request->memory_access_desc_size;
3059 const uint32_t expected_retrieve_request_length =
3060 retrieve_request->receivers_offset +
3061 (uint32_t)(retrieve_request->receiver_count *
3062 memory_access_desc_size);
J-Alves089004f2022-07-13 14:25:44 +01003063
3064 assert(retrieve_request != NULL);
3065 assert(memory_region != NULL);
3066 assert(receiver_index != NULL);
J-Alves089004f2022-07-13 14:25:44 +01003067
J-Alves4f0d9c12024-01-17 17:23:11 +00003068 if (retrieve_request_length != expected_retrieve_request_length) {
3069 dlog_verbose(
3070 "Invalid length for FFA_MEM_RETRIEVE_REQ, expected %d "
3071 "but was %d.\n",
3072 expected_retrieve_request_length,
3073 retrieve_request_length);
3074 return ffa_error(FFA_INVALID_PARAMETERS);
3075 }
3076
3077 if (retrieve_request->sender != memory_region->sender) {
3078 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003079 "Memory with handle %#lx not fully sent, can't "
J-Alves4f0d9c12024-01-17 17:23:11 +00003080 "retrieve.\n",
3081 memory_region->handle);
3082 return ffa_error(FFA_DENIED);
3083 }
3084
3085 /*
3086 * The SPMC can only process retrieve requests to memory share
3087 * operations with one borrower from the other world. It can't
3088 * determine the ID of the NWd VM that invoked the retrieve
3089 * request interface call. It relies on the hypervisor to
3090 * validate the caller's ID against that provided in the
3091 * `receivers` list of the retrieve response.
3092 * In case there is only one borrower from the NWd in the
3093 * transaction descriptor, record that in the `receiver_id` for
3094 * later use, and validate in the retrieve request message.
3095 * This limitation is due to the fact SPMC can't determine the
3096 * index in the memory share structures state to update.
3097 */
3098 if (to_id == HF_HYPERVISOR_VM_ID) {
3099 uint32_t other_world_count = 0;
3100
3101 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
3102 struct ffa_memory_access *receiver =
3103 ffa_memory_region_get_receiver(retrieve_request,
J-Alvesf220d572024-04-24 22:15:14 +01003104 i);
J-Alves4f0d9c12024-01-17 17:23:11 +00003105 assert(receiver != NULL);
3106
J-Alvesf220d572024-04-24 22:15:14 +01003107 if (!vm_id_is_current_world(
3108 receiver->receiver_permissions.receiver)) {
J-Alves4f0d9c12024-01-17 17:23:11 +00003109 other_world_count++;
J-Alvesf220d572024-04-24 22:15:14 +01003110 /* Set it to be used later. */
3111 to_id = receiver->receiver_permissions.receiver;
J-Alves4f0d9c12024-01-17 17:23:11 +00003112 }
3113 }
3114
3115 if (other_world_count > 1) {
3116 dlog_verbose(
3117 "Support one receiver from the other "
3118 "world.\n");
3119 return ffa_error(FFA_NOT_SUPPORTED);
3120 }
3121 }
J-Alves089004f2022-07-13 14:25:44 +01003122 /*
3123 * Check that the transaction type expected by the receiver is
3124 * correct, if it has been specified.
3125 */
3126 if (transaction_type !=
3127 FFA_MEMORY_REGION_TRANSACTION_TYPE_UNSPECIFIED &&
3128 transaction_type != (memory_region->flags &
3129 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK)) {
3130 dlog_verbose(
3131 "Incorrect transaction type %#x for "
Karl Meakine8937d92024-03-19 16:04:25 +00003132 "FFA_MEM_RETRIEVE_REQ, expected %#x for handle %#lx.\n",
J-Alves089004f2022-07-13 14:25:44 +01003133 transaction_type,
3134 memory_region->flags &
3135 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK,
3136 retrieve_request->handle);
3137 return ffa_error(FFA_INVALID_PARAMETERS);
3138 }
3139
3140 if (retrieve_request->tag != memory_region->tag) {
3141 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003142 "Incorrect tag %lu for FFA_MEM_RETRIEVE_REQ, expected "
3143 "%lu for handle %#lx.\n",
J-Alves089004f2022-07-13 14:25:44 +01003144 retrieve_request->tag, memory_region->tag,
3145 retrieve_request->handle);
3146 return ffa_error(FFA_INVALID_PARAMETERS);
3147 }
3148
J-Alves4f0d9c12024-01-17 17:23:11 +00003149 *receiver_index =
3150 ffa_memory_region_get_receiver_index(memory_region, to_id);
J-Alves089004f2022-07-13 14:25:44 +01003151
3152 if (*receiver_index == memory_region->receiver_count) {
3153 dlog_verbose(
3154 "Incorrect receiver VM ID %d for "
Karl Meakine8937d92024-03-19 16:04:25 +00003155 "FFA_MEM_RETRIEVE_REQ, for handle %#lx.\n",
J-Alves4f0d9c12024-01-17 17:23:11 +00003156 to_id, memory_region->handle);
J-Alves089004f2022-07-13 14:25:44 +01003157 return ffa_error(FFA_INVALID_PARAMETERS);
3158 }
3159
3160 if ((retrieve_request->flags &
3161 FFA_MEMORY_REGION_ADDRESS_RANGE_HINT_VALID) != 0U) {
3162 dlog_verbose(
3163 "Retriever specified 'address range alignment 'hint' "
3164 "not supported.\n");
3165 return ffa_error(FFA_INVALID_PARAMETERS);
3166 }
3167 if ((retrieve_request->flags &
3168 FFA_MEMORY_REGION_ADDRESS_RANGE_HINT_MASK) != 0) {
3169 dlog_verbose(
3170 "Bits 8-5 must be zero in memory region's flags "
3171 "(address range alignment hint not supported).\n");
3172 return ffa_error(FFA_INVALID_PARAMETERS);
3173 }
3174
3175 if ((retrieve_request->flags & ~0x7FF) != 0U) {
3176 dlog_verbose(
3177 "Bits 31-10 must be zero in memory region's flags.\n");
3178 return ffa_error(FFA_INVALID_PARAMETERS);
3179 }
3180
J-Alves95fbb312024-03-20 15:19:16 +00003181 if ((share_func == FFA_MEM_SHARE_32 ||
3182 share_func == FFA_MEM_SHARE_64) &&
J-Alves089004f2022-07-13 14:25:44 +01003183 (retrieve_request->flags &
3184 (FFA_MEMORY_REGION_FLAG_CLEAR |
3185 FFA_MEMORY_REGION_FLAG_CLEAR_RELINQUISH)) != 0U) {
3186 dlog_verbose(
3187 "Memory Share operation can't clean after relinquish "
3188 "memory region.\n");
3189 return ffa_error(FFA_INVALID_PARAMETERS);
3190 }
3191
3192 /*
3193 * If the borrower needs the memory to be cleared before mapping
3194 * to its address space, the sender should have set the flag
3195 * when calling FFA_MEM_LEND/FFA_MEM_DONATE, else return
3196 * FFA_DENIED.
3197 */
3198 if ((retrieve_request->flags & FFA_MEMORY_REGION_FLAG_CLEAR) != 0U &&
3199 (memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR) == 0U) {
3200 dlog_verbose(
3201 "Borrower needs memory cleared. Sender needs to set "
3202 "flag for clearing memory.\n");
3203 return ffa_error(FFA_DENIED);
3204 }
3205
Olivier Deprez4342a3c2022-02-28 09:37:25 +01003206 /* Memory region attributes NS-Bit MBZ for FFA_MEM_RETRIEVE_REQ. */
Karl Meakin84710f32023-10-12 15:14:49 +01003207 security_state = retrieve_request->attributes.security;
Olivier Deprez4342a3c2022-02-28 09:37:25 +01003208 if (security_state != FFA_MEMORY_SECURITY_UNSPECIFIED) {
3209 dlog_verbose(
3210 "Invalid security state for memory retrieve request "
3211 "operation.\n");
3212 return ffa_error(FFA_INVALID_PARAMETERS);
3213 }
3214
J-Alves089004f2022-07-13 14:25:44 +01003215 /*
3216 * If memory type is not specified, bypass validation of memory
3217 * attributes in the retrieve request. The retriever is expecting to
3218 * obtain this information from the SPMC.
3219 */
Karl Meakin84710f32023-10-12 15:14:49 +01003220 if (retrieve_request->attributes.type == FFA_MEMORY_NOT_SPECIFIED_MEM) {
J-Alves089004f2022-07-13 14:25:44 +01003221 return (struct ffa_value){.func = FFA_SUCCESS_32};
3222 }
3223
3224 /*
3225 * Ensure receiver's attributes are compatible with how
3226 * Hafnium maps memory: Normal Memory, Inner shareable,
3227 * Write-Back Read-Allocate Write-Allocate Cacheable.
3228 */
3229 return ffa_memory_attributes_validate(retrieve_request->attributes);
3230}
3231
J-Alves4f0d9c12024-01-17 17:23:11 +00003232static struct ffa_value ffa_partition_retrieve_request(
3233 struct share_states_locked share_states,
3234 struct ffa_memory_share_state *share_state, struct vm_locked to_locked,
3235 struct ffa_memory_region *retrieve_request,
3236 uint32_t retrieve_request_length, struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003237{
Karl Meakin84710f32023-10-12 15:14:49 +01003238 ffa_memory_access_permissions_t permissions = {0};
Olivier Deprez878bd5b2021-04-15 19:05:10 +02003239 uint32_t memory_to_mode;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003240 struct ffa_value ret;
Andrew Walbranca808b12020-05-15 17:22:28 +01003241 struct ffa_composite_memory_region *composite;
3242 uint32_t total_length;
3243 uint32_t fragment_length;
J-Alves19e20cf2023-08-02 12:48:55 +01003244 ffa_id_t receiver_id = to_locked.vm->id;
J-Alves4f0d9c12024-01-17 17:23:11 +00003245 bool is_retrieve_complete = false;
J-Alves4f0d9c12024-01-17 17:23:11 +00003246 const uint64_t memory_access_desc_size =
Daniel Boulbyde974ca2023-12-12 13:53:31 +00003247 retrieve_request->memory_access_desc_size;
J-Alves4f0d9c12024-01-17 17:23:11 +00003248 uint32_t receiver_index;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003249 struct ffa_memory_access *receiver;
J-Alves4f0d9c12024-01-17 17:23:11 +00003250 ffa_memory_handle_t handle = retrieve_request->handle;
Karl Meakin84710f32023-10-12 15:14:49 +01003251 ffa_memory_attributes_t attributes = {0};
J-Alves460d36c2023-10-12 17:02:15 +01003252 uint32_t retrieve_mode = 0;
J-Alves4f0d9c12024-01-17 17:23:11 +00003253 struct ffa_memory_region *memory_region = share_state->memory_region;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003254
J-Alves96de29f2022-04-26 16:05:24 +01003255 if (!share_state->sending_complete) {
3256 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003257 "Memory with handle %#lx not fully sent, can't "
J-Alves96de29f2022-04-26 16:05:24 +01003258 "retrieve.\n",
3259 handle);
J-Alves4f0d9c12024-01-17 17:23:11 +00003260 return ffa_error(FFA_INVALID_PARAMETERS);
J-Alves96de29f2022-04-26 16:05:24 +01003261 }
3262
J-Alves4f0d9c12024-01-17 17:23:11 +00003263 /*
3264 * Validate retrieve request, according to what was sent by the
3265 * sender. Function will output the `receiver_index` from the
3266 * provided memory region.
3267 */
3268 ret = ffa_memory_retrieve_validate(
3269 receiver_id, retrieve_request, retrieve_request_length,
3270 memory_region, &receiver_index, share_state->share_func);
J-Alves089004f2022-07-13 14:25:44 +01003271
J-Alves4f0d9c12024-01-17 17:23:11 +00003272 if (ret.func != FFA_SUCCESS_32) {
3273 return ret;
J-Alves089004f2022-07-13 14:25:44 +01003274 }
J-Alves96de29f2022-04-26 16:05:24 +01003275
J-Alves4f0d9c12024-01-17 17:23:11 +00003276 /*
3277 * Validate the requested permissions against the sent
3278 * permissions.
3279 * Outputs the permissions to give to retriever at S2
3280 * PTs.
3281 */
3282 ret = ffa_memory_retrieve_validate_memory_access_list(
3283 memory_region, retrieve_request, receiver_id, &permissions,
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003284 &receiver, share_state->share_func);
J-Alves4f0d9c12024-01-17 17:23:11 +00003285 if (ret.func != FFA_SUCCESS_32) {
3286 return ret;
3287 }
3288
3289 memory_to_mode = ffa_memory_permissions_to_mode(
3290 permissions, share_state->sender_orig_mode);
3291
3292 ret = ffa_retrieve_check_update(
3293 to_locked, share_state->fragments,
3294 share_state->fragment_constituent_counts,
3295 share_state->fragment_count, memory_to_mode,
J-Alves460d36c2023-10-12 17:02:15 +01003296 share_state->share_func, false, page_pool, &retrieve_mode,
3297 share_state->memory_protected);
J-Alves4f0d9c12024-01-17 17:23:11 +00003298
3299 if (ret.func != FFA_SUCCESS_32) {
3300 return ret;
3301 }
3302
3303 share_state->retrieved_fragment_count[receiver_index] = 1;
3304
3305 is_retrieve_complete =
3306 share_state->retrieved_fragment_count[receiver_index] ==
3307 share_state->fragment_count;
3308
J-Alvesb5084cf2022-07-06 14:20:12 +01003309 /* VMs acquire the RX buffer from SPMC. */
3310 CHECK(plat_ffa_acquire_receiver_rx(to_locked, &ret));
3311
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003312 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01003313 * Copy response to RX buffer of caller and deliver the message.
3314 * This must be done before the share_state is (possibly) freed.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003315 */
Andrew Walbranca808b12020-05-15 17:22:28 +01003316 composite = ffa_memory_region_get_composite(memory_region, 0);
J-Alves4f0d9c12024-01-17 17:23:11 +00003317
Andrew Walbranca808b12020-05-15 17:22:28 +01003318 /*
J-Alves460d36c2023-10-12 17:02:15 +01003319 * Set the security state in the memory retrieve response attributes
3320 * if specified by the target mode.
3321 */
3322 attributes = plat_ffa_memory_security_mode(memory_region->attributes,
3323 retrieve_mode);
3324
3325 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01003326 * Constituents which we received in the first fragment should
3327 * always fit in the first fragment we are sending, because the
3328 * header is the same size in both cases and we have a fixed
3329 * message buffer size. So `ffa_retrieved_memory_region_init`
3330 * should never fail.
Andrew Walbranca808b12020-05-15 17:22:28 +01003331 */
Olivier Deprez878bd5b2021-04-15 19:05:10 +02003332
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003333 /* Provide the permissions that had been provided. */
3334 receiver->receiver_permissions.permissions = permissions;
3335
3336 /*
3337 * Prepare the memory region descriptor for the retrieve response.
3338 * Provide the pointer to the receiver tracked in the share state
J-Alves7b9cc432024-04-04 10:57:17 +01003339 * structures.
3340 * At this point the retrieve request descriptor from the partition
3341 * has been processed. The `retrieve_request` is expected to be in
3342 * a region that is handled by the SPMC/Hyp. Reuse the same buffer to
3343 * prepare the retrieve response before copying it to the RX buffer of
3344 * the caller.
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003345 */
Andrew Walbranca808b12020-05-15 17:22:28 +01003346 CHECK(ffa_retrieved_memory_region_init(
J-Alves7b9cc432024-04-04 10:57:17 +01003347 retrieve_request, to_locked.vm->ffa_version, HF_MAILBOX_SIZE,
3348 memory_region->sender, attributes, memory_region->flags, handle,
3349 permissions, receiver, 1, memory_access_desc_size,
3350 composite->page_count, composite->constituent_count,
3351 share_state->fragments[0],
Andrew Walbranca808b12020-05-15 17:22:28 +01003352 share_state->fragment_constituent_counts[0], &total_length,
3353 &fragment_length));
J-Alvesb5084cf2022-07-06 14:20:12 +01003354
J-Alves7b9cc432024-04-04 10:57:17 +01003355 /*
3356 * Copy the message from the buffer into the partition's mailbox.
3357 * The operation might fail unexpectedly due to change in PAS address
3358 * space, or improper values to the sizes of the structures.
3359 */
3360 if (!memcpy_trapped(to_locked.vm->mailbox.recv, HF_MAILBOX_SIZE,
3361 retrieve_request, fragment_length)) {
3362 dlog_error(
3363 "%s: aborted the copy of response to RX buffer of "
3364 "%x.\n",
3365 __func__, to_locked.vm->id);
3366 return ffa_error(FFA_ABORTED);
3367 }
3368
J-Alves4f0d9c12024-01-17 17:23:11 +00003369 if (is_retrieve_complete) {
Andrew Walbranca808b12020-05-15 17:22:28 +01003370 ffa_memory_retrieve_complete(share_states, share_state,
3371 page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003372 }
J-Alves4f0d9c12024-01-17 17:23:11 +00003373
3374 return ffa_memory_retrieve_resp(total_length, fragment_length);
3375}
3376
3377static struct ffa_value ffa_hypervisor_retrieve_request(
3378 struct ffa_memory_share_state *share_state, struct vm_locked to_locked,
3379 struct ffa_memory_region *retrieve_request)
3380{
3381 struct ffa_value ret;
3382 struct ffa_composite_memory_region *composite;
3383 uint32_t total_length;
3384 uint32_t fragment_length;
J-Alves4f0d9c12024-01-17 17:23:11 +00003385 ffa_memory_attributes_t attributes;
J-Alves7b6ab612024-01-24 09:54:54 +00003386 uint64_t memory_access_desc_size;
J-Alves4f0d9c12024-01-17 17:23:11 +00003387 struct ffa_memory_region *memory_region;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003388 struct ffa_memory_access *receiver;
J-Alves4f0d9c12024-01-17 17:23:11 +00003389 ffa_memory_handle_t handle = retrieve_request->handle;
3390
J-Alves4f0d9c12024-01-17 17:23:11 +00003391 memory_region = share_state->memory_region;
3392
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003393 assert(to_locked.vm->id == HF_HYPERVISOR_VM_ID);
3394
J-Alves7b6ab612024-01-24 09:54:54 +00003395 switch (to_locked.vm->ffa_version) {
Karl Meakin0e617d92024-04-05 12:55:22 +01003396 case FFA_VERSION_1_2:
J-Alves7b6ab612024-01-24 09:54:54 +00003397 memory_access_desc_size = sizeof(struct ffa_memory_access);
3398 break;
Karl Meakin0e617d92024-04-05 12:55:22 +01003399 case FFA_VERSION_1_0:
3400 case FFA_VERSION_1_1:
J-Alves7b6ab612024-01-24 09:54:54 +00003401 memory_access_desc_size = sizeof(struct ffa_memory_access_v1_0);
3402 break;
3403 default:
3404 panic("version not supported: %x\n", to_locked.vm->ffa_version);
3405 }
3406
J-Alves4f0d9c12024-01-17 17:23:11 +00003407 if (share_state->hypervisor_fragment_count != 0U) {
3408 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003409 "Memory with handle %#lx already retrieved by "
J-Alves4f0d9c12024-01-17 17:23:11 +00003410 "the hypervisor.\n",
3411 handle);
3412 return ffa_error(FFA_DENIED);
3413 }
3414
3415 share_state->hypervisor_fragment_count = 1;
3416
3417 ffa_memory_retrieve_complete_from_hyp(share_state);
3418
3419 /* VMs acquire the RX buffer from SPMC. */
3420 CHECK(plat_ffa_acquire_receiver_rx(to_locked, &ret));
3421
3422 /*
3423 * Copy response to RX buffer of caller and deliver the message.
3424 * This must be done before the share_state is (possibly) freed.
3425 */
3426 composite = ffa_memory_region_get_composite(memory_region, 0);
3427
3428 /*
3429 * Constituents which we received in the first fragment should
3430 * always fit in the first fragment we are sending, because the
3431 * header is the same size in both cases and we have a fixed
3432 * message buffer size. So `ffa_retrieved_memory_region_init`
3433 * should never fail.
3434 */
3435
3436 /*
3437 * Set the security state in the memory retrieve response attributes
3438 * if specified by the target mode.
3439 */
3440 attributes = plat_ffa_memory_security_mode(
3441 memory_region->attributes, share_state->sender_orig_mode);
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003442
3443 receiver = ffa_memory_region_get_receiver(memory_region, 0);
3444
J-Alves7b9cc432024-04-04 10:57:17 +01003445 /*
3446 * At this point the `retrieve_request` is expected to be in a section
3447 * managed by the hypervisor.
3448 */
J-Alves4f0d9c12024-01-17 17:23:11 +00003449 CHECK(ffa_retrieved_memory_region_init(
J-Alves7b9cc432024-04-04 10:57:17 +01003450 retrieve_request, to_locked.vm->ffa_version, HF_MAILBOX_SIZE,
3451 memory_region->sender, attributes, memory_region->flags, handle,
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003452 receiver->receiver_permissions.permissions, receiver,
3453 memory_region->receiver_count, memory_access_desc_size,
J-Alves4f0d9c12024-01-17 17:23:11 +00003454 composite->page_count, composite->constituent_count,
3455 share_state->fragments[0],
3456 share_state->fragment_constituent_counts[0], &total_length,
3457 &fragment_length));
3458
J-Alves7b9cc432024-04-04 10:57:17 +01003459 /*
3460 * Copy the message from the buffer into the hypervisor's mailbox.
3461 * The operation might fail unexpectedly due to change in PAS, or
3462 * improper values for the sizes of the structures.
3463 */
3464 if (!memcpy_trapped(to_locked.vm->mailbox.recv, HF_MAILBOX_SIZE,
3465 retrieve_request, fragment_length)) {
3466 dlog_error(
3467 "%s: aborted the copy of response to RX buffer of "
3468 "%x.\n",
3469 __func__, to_locked.vm->id);
3470 return ffa_error(FFA_ABORTED);
3471 }
3472
J-Alves4f0d9c12024-01-17 17:23:11 +00003473 return ffa_memory_retrieve_resp(total_length, fragment_length);
3474}
3475
3476struct ffa_value ffa_memory_retrieve(struct vm_locked to_locked,
3477 struct ffa_memory_region *retrieve_request,
3478 uint32_t retrieve_request_length,
3479 struct mpool *page_pool)
3480{
3481 ffa_memory_handle_t handle = retrieve_request->handle;
3482 struct share_states_locked share_states;
3483 struct ffa_memory_share_state *share_state;
3484 struct ffa_value ret;
3485
3486 dump_share_states();
3487
3488 share_states = share_states_lock();
3489 share_state = get_share_state(share_states, handle);
3490 if (share_state == NULL) {
Karl Meakine8937d92024-03-19 16:04:25 +00003491 dlog_verbose("Invalid handle %#lx for FFA_MEM_RETRIEVE_REQ.\n",
J-Alves4f0d9c12024-01-17 17:23:11 +00003492 handle);
3493 ret = ffa_error(FFA_INVALID_PARAMETERS);
3494 goto out;
3495 }
3496
3497 if (is_ffa_hypervisor_retrieve_request(retrieve_request, to_locked)) {
3498 ret = ffa_hypervisor_retrieve_request(share_state, to_locked,
3499 retrieve_request);
3500 } else {
3501 ret = ffa_partition_retrieve_request(
3502 share_states, share_state, to_locked, retrieve_request,
3503 retrieve_request_length, page_pool);
3504 }
3505
3506 /* Track use of the RX buffer if the handling has succeeded. */
3507 if (ret.func == FFA_MEM_RETRIEVE_RESP_32) {
3508 to_locked.vm->mailbox.recv_func = FFA_MEM_RETRIEVE_RESP_32;
3509 to_locked.vm->mailbox.state = MAILBOX_STATE_FULL;
3510 }
3511
Andrew Walbranca808b12020-05-15 17:22:28 +01003512out:
3513 share_states_unlock(&share_states);
3514 dump_share_states();
3515 return ret;
3516}
3517
J-Alves5da37d92022-10-24 16:33:48 +01003518/**
3519 * Determine expected fragment offset according to the FF-A version of
3520 * the caller.
3521 */
3522static uint32_t ffa_memory_retrieve_expected_offset_per_ffa_version(
3523 struct ffa_memory_region *memory_region,
Karl Meakin0e617d92024-04-05 12:55:22 +01003524 uint32_t retrieved_constituents_count, enum ffa_version ffa_version)
J-Alves5da37d92022-10-24 16:33:48 +01003525{
3526 uint32_t expected_fragment_offset;
3527 uint32_t composite_constituents_offset;
3528
Karl Meakin0e617d92024-04-05 12:55:22 +01003529 if (ffa_version >= FFA_VERSION_1_1) {
J-Alves5da37d92022-10-24 16:33:48 +01003530 /*
3531 * Hafnium operates memory regions in FF-A v1.1 format, so we
3532 * can retrieve the constituents offset from descriptor.
3533 */
3534 composite_constituents_offset =
3535 ffa_composite_constituent_offset(memory_region, 0);
Karl Meakin0e617d92024-04-05 12:55:22 +01003536 } else if (ffa_version == FFA_VERSION_1_0) {
J-Alves5da37d92022-10-24 16:33:48 +01003537 /*
3538 * If retriever is FF-A v1.0, determine the composite offset
3539 * as it is expected to have been configured in the
3540 * retrieve response.
3541 */
3542 composite_constituents_offset =
3543 sizeof(struct ffa_memory_region_v1_0) +
3544 RECEIVERS_COUNT_IN_RETRIEVE_RESP *
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003545 sizeof(struct ffa_memory_access_v1_0) +
J-Alves5da37d92022-10-24 16:33:48 +01003546 sizeof(struct ffa_composite_memory_region);
3547 } else {
3548 panic("%s received an invalid FF-A version.\n", __func__);
3549 }
3550
3551 expected_fragment_offset =
3552 composite_constituents_offset +
3553 retrieved_constituents_count *
3554 sizeof(struct ffa_memory_region_constituent) -
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003555 (uint32_t)(memory_region->memory_access_desc_size *
3556 (memory_region->receiver_count - 1));
J-Alves5da37d92022-10-24 16:33:48 +01003557
3558 return expected_fragment_offset;
3559}
3560
Andrew Walbranca808b12020-05-15 17:22:28 +01003561struct ffa_value ffa_memory_retrieve_continue(struct vm_locked to_locked,
3562 ffa_memory_handle_t handle,
3563 uint32_t fragment_offset,
J-Alves19e20cf2023-08-02 12:48:55 +01003564 ffa_id_t sender_vm_id,
Andrew Walbranca808b12020-05-15 17:22:28 +01003565 struct mpool *page_pool)
3566{
3567 struct ffa_memory_region *memory_region;
3568 struct share_states_locked share_states;
3569 struct ffa_memory_share_state *share_state;
3570 struct ffa_value ret;
3571 uint32_t fragment_index;
3572 uint32_t retrieved_constituents_count;
3573 uint32_t i;
3574 uint32_t expected_fragment_offset;
3575 uint32_t remaining_constituent_count;
3576 uint32_t fragment_length;
J-Alvesc7484f12022-05-13 12:41:14 +01003577 uint32_t receiver_index;
J-Alves59ed0042022-07-28 18:26:41 +01003578 bool continue_ffa_hyp_mem_retrieve_req;
Andrew Walbranca808b12020-05-15 17:22:28 +01003579
3580 dump_share_states();
3581
3582 share_states = share_states_lock();
Karl Meakin4a2854a2023-06-30 16:26:52 +01003583 share_state = get_share_state(share_states, handle);
J-Alvesb56aac82023-11-10 09:44:43 +00003584 if (share_state == NULL) {
Karl Meakine8937d92024-03-19 16:04:25 +00003585 dlog_verbose("Invalid handle %#lx for FFA_MEM_FRAG_RX.\n",
Andrew Walbranca808b12020-05-15 17:22:28 +01003586 handle);
3587 ret = ffa_error(FFA_INVALID_PARAMETERS);
3588 goto out;
3589 }
3590
3591 memory_region = share_state->memory_region;
3592 CHECK(memory_region != NULL);
3593
Andrew Walbranca808b12020-05-15 17:22:28 +01003594 if (!share_state->sending_complete) {
3595 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003596 "Memory with handle %#lx not fully sent, can't "
Andrew Walbranca808b12020-05-15 17:22:28 +01003597 "retrieve.\n",
3598 handle);
3599 ret = ffa_error(FFA_INVALID_PARAMETERS);
3600 goto out;
3601 }
3602
J-Alves59ed0042022-07-28 18:26:41 +01003603 /*
3604 * If retrieve request from the hypervisor has been initiated in the
3605 * given share_state, continue it, else assume it is a continuation of
3606 * retrieve request from a NWd VM.
3607 */
3608 continue_ffa_hyp_mem_retrieve_req =
3609 (to_locked.vm->id == HF_HYPERVISOR_VM_ID) &&
3610 (share_state->hypervisor_fragment_count != 0U) &&
J-Alves661e1b72023-08-02 13:39:40 +01003611 ffa_is_vm_id(sender_vm_id);
Andrew Walbranca808b12020-05-15 17:22:28 +01003612
J-Alves59ed0042022-07-28 18:26:41 +01003613 if (!continue_ffa_hyp_mem_retrieve_req) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003614 receiver_index = ffa_memory_region_get_receiver_index(
J-Alves59ed0042022-07-28 18:26:41 +01003615 memory_region, to_locked.vm->id);
3616
3617 if (receiver_index == memory_region->receiver_count) {
3618 dlog_verbose(
3619 "Caller of FFA_MEM_FRAG_RX (%x) is not a "
Karl Meakine8937d92024-03-19 16:04:25 +00003620 "borrower to memory sharing transaction "
3621 "(%lx)\n",
J-Alves59ed0042022-07-28 18:26:41 +01003622 to_locked.vm->id, handle);
3623 ret = ffa_error(FFA_INVALID_PARAMETERS);
3624 goto out;
3625 }
3626
3627 if (share_state->retrieved_fragment_count[receiver_index] ==
3628 0 ||
3629 share_state->retrieved_fragment_count[receiver_index] >=
3630 share_state->fragment_count) {
3631 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003632 "Retrieval of memory with handle %#lx not yet "
J-Alves59ed0042022-07-28 18:26:41 +01003633 "started or already completed (%d/%d fragments "
3634 "retrieved).\n",
3635 handle,
3636 share_state->retrieved_fragment_count
3637 [receiver_index],
3638 share_state->fragment_count);
3639 ret = ffa_error(FFA_INVALID_PARAMETERS);
3640 goto out;
3641 }
3642
3643 fragment_index =
3644 share_state->retrieved_fragment_count[receiver_index];
3645 } else {
3646 if (share_state->hypervisor_fragment_count == 0 ||
3647 share_state->hypervisor_fragment_count >=
3648 share_state->fragment_count) {
3649 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003650 "Retrieve of memory with handle %lx not "
J-Alves59ed0042022-07-28 18:26:41 +01003651 "started from hypervisor.\n",
3652 handle);
3653 ret = ffa_error(FFA_INVALID_PARAMETERS);
3654 goto out;
3655 }
3656
3657 if (memory_region->sender != sender_vm_id) {
3658 dlog_verbose(
3659 "Sender ID (%x) is not as expected for memory "
Karl Meakine8937d92024-03-19 16:04:25 +00003660 "handle %lx\n",
J-Alves59ed0042022-07-28 18:26:41 +01003661 sender_vm_id, handle);
3662 ret = ffa_error(FFA_INVALID_PARAMETERS);
3663 goto out;
3664 }
3665
3666 fragment_index = share_state->hypervisor_fragment_count;
3667
3668 receiver_index = 0;
3669 }
Andrew Walbranca808b12020-05-15 17:22:28 +01003670
3671 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01003672 * Check that the given fragment offset is correct by counting
3673 * how many constituents were in the fragments previously sent.
Andrew Walbranca808b12020-05-15 17:22:28 +01003674 */
3675 retrieved_constituents_count = 0;
3676 for (i = 0; i < fragment_index; ++i) {
3677 retrieved_constituents_count +=
3678 share_state->fragment_constituent_counts[i];
3679 }
J-Alvesc7484f12022-05-13 12:41:14 +01003680
3681 CHECK(memory_region->receiver_count > 0);
3682
Andrew Walbranca808b12020-05-15 17:22:28 +01003683 expected_fragment_offset =
J-Alves5da37d92022-10-24 16:33:48 +01003684 ffa_memory_retrieve_expected_offset_per_ffa_version(
3685 memory_region, retrieved_constituents_count,
3686 to_locked.vm->ffa_version);
3687
Andrew Walbranca808b12020-05-15 17:22:28 +01003688 if (fragment_offset != expected_fragment_offset) {
3689 dlog_verbose("Fragment offset was %d but expected %d.\n",
3690 fragment_offset, expected_fragment_offset);
3691 ret = ffa_error(FFA_INVALID_PARAMETERS);
3692 goto out;
3693 }
3694
J-Alves4f0d9c12024-01-17 17:23:11 +00003695 /*
3696 * When hafnium is the hypervisor, acquire the RX buffer of a VM, that
3697 * is currently ownder by the SPMC.
3698 */
3699 assert(plat_ffa_acquire_receiver_rx(to_locked, &ret));
J-Alves59ed0042022-07-28 18:26:41 +01003700
Andrew Walbranca808b12020-05-15 17:22:28 +01003701 remaining_constituent_count = ffa_memory_fragment_init(
3702 to_locked.vm->mailbox.recv, HF_MAILBOX_SIZE,
3703 share_state->fragments[fragment_index],
3704 share_state->fragment_constituent_counts[fragment_index],
3705 &fragment_length);
3706 CHECK(remaining_constituent_count == 0);
J-Alves674e4de2024-01-17 16:20:32 +00003707
Andrew Walbranca808b12020-05-15 17:22:28 +01003708 to_locked.vm->mailbox.recv_func = FFA_MEM_FRAG_TX_32;
J-Alvese8c8c2b2022-12-16 15:34:48 +00003709 to_locked.vm->mailbox.state = MAILBOX_STATE_FULL;
Andrew Walbranca808b12020-05-15 17:22:28 +01003710
J-Alves59ed0042022-07-28 18:26:41 +01003711 if (!continue_ffa_hyp_mem_retrieve_req) {
3712 share_state->retrieved_fragment_count[receiver_index]++;
3713 if (share_state->retrieved_fragment_count[receiver_index] ==
3714 share_state->fragment_count) {
3715 ffa_memory_retrieve_complete(share_states, share_state,
3716 page_pool);
3717 }
3718 } else {
3719 share_state->hypervisor_fragment_count++;
3720
3721 ffa_memory_retrieve_complete_from_hyp(share_state);
3722 }
Andrew Walbranca808b12020-05-15 17:22:28 +01003723 ret = (struct ffa_value){.func = FFA_MEM_FRAG_TX_32,
3724 .arg1 = (uint32_t)handle,
3725 .arg2 = (uint32_t)(handle >> 32),
3726 .arg3 = fragment_length};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003727
3728out:
3729 share_states_unlock(&share_states);
3730 dump_share_states();
3731 return ret;
3732}
3733
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003734struct ffa_value ffa_memory_relinquish(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003735 struct vm_locked from_locked,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003736 struct ffa_mem_relinquish *relinquish_request, struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003737{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003738 ffa_memory_handle_t handle = relinquish_request->handle;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003739 struct share_states_locked share_states;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003740 struct ffa_memory_share_state *share_state;
3741 struct ffa_memory_region *memory_region;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003742 bool clear;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003743 struct ffa_value ret;
J-Alves8eb19162022-04-28 10:56:48 +01003744 uint32_t receiver_index;
J-Alves3c5b2072022-11-21 12:45:40 +00003745 bool receivers_relinquished_memory;
Karl Meakin84710f32023-10-12 15:14:49 +01003746 ffa_memory_access_permissions_t receiver_permissions = {0};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003747
Andrew Walbrana65a1322020-04-06 19:32:32 +01003748 if (relinquish_request->endpoint_count != 1) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003749 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01003750 "Stream endpoints not supported (got %d "
J-Alves668a86e2023-05-10 11:53:25 +01003751 "endpoints on FFA_MEM_RELINQUISH, expected 1).\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003752 relinquish_request->endpoint_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003753 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003754 }
3755
Andrew Walbrana65a1322020-04-06 19:32:32 +01003756 if (relinquish_request->endpoints[0] != from_locked.vm->id) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003757 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01003758 "VM ID %d in relinquish message doesn't match "
J-Alves668a86e2023-05-10 11:53:25 +01003759 "calling VM ID %d.\n",
Andrew Walbrana65a1322020-04-06 19:32:32 +01003760 relinquish_request->endpoints[0], from_locked.vm->id);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003761 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003762 }
3763
3764 dump_share_states();
3765
3766 share_states = share_states_lock();
Karl Meakin4a2854a2023-06-30 16:26:52 +01003767 share_state = get_share_state(share_states, handle);
J-Alvesb56aac82023-11-10 09:44:43 +00003768 if (share_state == NULL) {
Karl Meakine8937d92024-03-19 16:04:25 +00003769 dlog_verbose("Invalid handle %#lx for FFA_MEM_RELINQUISH.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003770 handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003771 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003772 goto out;
3773 }
3774
Andrew Walbranca808b12020-05-15 17:22:28 +01003775 if (!share_state->sending_complete) {
3776 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003777 "Memory with handle %#lx not fully sent, can't "
Andrew Walbranca808b12020-05-15 17:22:28 +01003778 "relinquish.\n",
3779 handle);
3780 ret = ffa_error(FFA_INVALID_PARAMETERS);
3781 goto out;
3782 }
3783
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003784 memory_region = share_state->memory_region;
3785 CHECK(memory_region != NULL);
3786
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003787 receiver_index = ffa_memory_region_get_receiver_index(
3788 memory_region, from_locked.vm->id);
J-Alves8eb19162022-04-28 10:56:48 +01003789
3790 if (receiver_index == memory_region->receiver_count) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003791 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01003792 "VM ID %d tried to relinquish memory region "
Karl Meakine8937d92024-03-19 16:04:25 +00003793 "with handle %#lx and it is not a valid borrower.\n",
J-Alves8eb19162022-04-28 10:56:48 +01003794 from_locked.vm->id, handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003795 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003796 goto out;
3797 }
3798
J-Alves8eb19162022-04-28 10:56:48 +01003799 if (share_state->retrieved_fragment_count[receiver_index] !=
Andrew Walbranca808b12020-05-15 17:22:28 +01003800 share_state->fragment_count) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003801 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003802 "Memory with handle %#lx not yet fully "
J-Alvesa9cd7e32022-07-01 13:49:33 +01003803 "retrieved, "
J-Alves8eb19162022-04-28 10:56:48 +01003804 "receiver %x can't relinquish.\n",
3805 handle, from_locked.vm->id);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003806 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003807 goto out;
3808 }
3809
J-Alves3c5b2072022-11-21 12:45:40 +00003810 /*
3811 * Either clear if requested in relinquish call, or in a retrieve
3812 * request from one of the borrowers.
3813 */
3814 receivers_relinquished_memory = true;
3815
3816 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
3817 struct ffa_memory_access *receiver =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003818 ffa_memory_region_get_receiver(memory_region, i);
3819 assert(receiver != NULL);
J-Alves3c5b2072022-11-21 12:45:40 +00003820 if (receiver->receiver_permissions.receiver ==
3821 from_locked.vm->id) {
J-Alves639ddfc2023-11-21 14:17:26 +00003822 receiver_permissions =
3823 receiver->receiver_permissions.permissions;
J-Alves3c5b2072022-11-21 12:45:40 +00003824 continue;
3825 }
3826
3827 if (share_state->retrieved_fragment_count[i] != 0U) {
3828 receivers_relinquished_memory = false;
3829 break;
3830 }
3831 }
3832
3833 clear = receivers_relinquished_memory &&
Daniel Boulby2e14ebe2024-01-15 16:21:44 +00003834 ((relinquish_request->flags & FFA_MEMORY_REGION_FLAG_CLEAR) !=
3835 0U);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003836
3837 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01003838 * Clear is not allowed for memory that was shared, as the
3839 * original sender still has access to the memory.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003840 */
J-Alves95fbb312024-03-20 15:19:16 +00003841 if (clear && (share_state->share_func == FFA_MEM_SHARE_32 ||
3842 share_state->share_func == FFA_MEM_SHARE_64)) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003843 dlog_verbose("Memory which was shared can't be cleared.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003844 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003845 goto out;
3846 }
3847
J-Alvesb886d492024-04-15 10:55:29 +01003848 if (clear && receiver_permissions.data_access == FFA_DATA_ACCESS_RO) {
J-Alves639ddfc2023-11-21 14:17:26 +00003849 dlog_verbose("%s: RO memory can't use clear memory flag.\n",
3850 __func__);
3851 ret = ffa_error(FFA_DENIED);
3852 goto out;
3853 }
3854
Andrew Walbranca808b12020-05-15 17:22:28 +01003855 ret = ffa_relinquish_check_update(
J-Alves26483382023-04-20 12:01:49 +01003856 from_locked, share_state->fragments,
Andrew Walbranca808b12020-05-15 17:22:28 +01003857 share_state->fragment_constituent_counts,
J-Alves69cdfd92024-04-26 11:40:59 +01003858 share_state->fragment_count, share_state->sender_orig_mode,
3859 page_pool, clear);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003860
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003861 if (ret.func == FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003862 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01003863 * Mark memory handle as not retrieved, so it can be
3864 * reclaimed (or retrieved again).
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003865 */
J-Alves8eb19162022-04-28 10:56:48 +01003866 share_state->retrieved_fragment_count[receiver_index] = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003867 }
3868
3869out:
3870 share_states_unlock(&share_states);
3871 dump_share_states();
3872 return ret;
3873}
3874
3875/**
J-Alvesa9cd7e32022-07-01 13:49:33 +01003876 * Validates that the reclaim transition is allowed for the given
3877 * handle, updates the page table of the reclaiming VM, and frees the
3878 * internal state associated with the handle.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003879 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003880struct ffa_value ffa_memory_reclaim(struct vm_locked to_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01003881 ffa_memory_handle_t handle,
3882 ffa_memory_region_flags_t flags,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003883 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003884{
3885 struct share_states_locked share_states;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003886 struct ffa_memory_share_state *share_state;
3887 struct ffa_memory_region *memory_region;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003888 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003889
3890 dump_share_states();
3891
3892 share_states = share_states_lock();
Karl Meakin52cdfe72023-06-30 14:49:10 +01003893
Karl Meakin4a2854a2023-06-30 16:26:52 +01003894 share_state = get_share_state(share_states, handle);
J-Alvesb56aac82023-11-10 09:44:43 +00003895 if (share_state == NULL) {
Karl Meakine8937d92024-03-19 16:04:25 +00003896 dlog_verbose("Invalid handle %#lx for FFA_MEM_RECLAIM.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003897 handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003898 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003899 goto out;
3900 }
Karl Meakin4a2854a2023-06-30 16:26:52 +01003901 memory_region = share_state->memory_region;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003902
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003903 CHECK(memory_region != NULL);
3904
J-Alvesa9cd7e32022-07-01 13:49:33 +01003905 if (vm_id_is_current_world(to_locked.vm->id) &&
3906 to_locked.vm->id != memory_region->sender) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003907 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003908 "VM %#x attempted to reclaim memory handle %#lx "
Olivier Deprezf92e5d42020-11-13 16:00:54 +01003909 "originally sent by VM %#x.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003910 to_locked.vm->id, handle, memory_region->sender);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003911 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003912 goto out;
3913 }
3914
Andrew Walbranca808b12020-05-15 17:22:28 +01003915 if (!share_state->sending_complete) {
3916 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003917 "Memory with handle %#lx not fully sent, can't "
Andrew Walbranca808b12020-05-15 17:22:28 +01003918 "reclaim.\n",
3919 handle);
3920 ret = ffa_error(FFA_INVALID_PARAMETERS);
3921 goto out;
3922 }
3923
J-Alves752236c2022-04-28 11:07:47 +01003924 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
3925 if (share_state->retrieved_fragment_count[i] != 0) {
3926 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003927 "Tried to reclaim memory handle %#lx "
J-Alves3c5b2072022-11-21 12:45:40 +00003928 "that has not been relinquished by all "
J-Alvesa9cd7e32022-07-01 13:49:33 +01003929 "borrowers(%x).\n",
J-Alves752236c2022-04-28 11:07:47 +01003930 handle,
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003931 ffa_memory_region_get_receiver(memory_region, i)
3932 ->receiver_permissions.receiver);
J-Alves752236c2022-04-28 11:07:47 +01003933 ret = ffa_error(FFA_DENIED);
3934 goto out;
3935 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003936 }
3937
Andrew Walbranca808b12020-05-15 17:22:28 +01003938 ret = ffa_retrieve_check_update(
J-Alves26483382023-04-20 12:01:49 +01003939 to_locked, share_state->fragments,
Andrew Walbranca808b12020-05-15 17:22:28 +01003940 share_state->fragment_constituent_counts,
J-Alves2a0d2882020-10-29 14:49:50 +00003941 share_state->fragment_count, share_state->sender_orig_mode,
J-Alves460d36c2023-10-12 17:02:15 +01003942 FFA_MEM_RECLAIM_32, flags & FFA_MEM_RECLAIM_CLEAR, page_pool,
J-Alvesfd206052023-05-22 16:45:00 +01003943 NULL, share_state->memory_protected);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003944
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003945 if (ret.func == FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003946 share_state_free(share_states, share_state, page_pool);
J-Alves3c5b2072022-11-21 12:45:40 +00003947 dlog_verbose("Freed share state after successful reclaim.\n");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003948 }
3949
3950out:
3951 share_states_unlock(&share_states);
3952 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01003953}