blob: 11864cafd1f139dfe8f1dc2aff3c72db68e12d9c [file] [log] [blame]
Jose Marinho75509b42019-04-09 09:34:59 +01001/*
2 * Copyright 2019 The Hafnium Authors.
3 *
Andrew Walbrane959ec12020-06-17 15:01:09 +01004 * Use of this source code is governed by a BSD-style
5 * license that can be found in the LICENSE file or at
6 * https://opensource.org/licenses/BSD-3-Clause.
Jose Marinho75509b42019-04-09 09:34:59 +01007 */
8
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01009#include "hf/ffa_memory.h"
Andrew Walbran475c1452020-02-07 13:22:22 +000010
J-Alves460d36c2023-10-12 17:02:15 +010011#include <stdint.h>
12
Federico Recanati4fd065d2021-12-13 20:06:23 +010013#include "hf/arch/mm.h"
Olivier Deprez112d2b52020-09-30 07:39:23 +020014#include "hf/arch/other_world.h"
Olivier Deprez55a189e2021-06-09 15:45:27 +020015#include "hf/arch/plat/ffa.h"
Andrew Walbran290b0c92020-02-03 16:37:14 +000016
J-Alves5952d942022-12-22 16:03:00 +000017#include "hf/addr.h"
Jose Marinho75509b42019-04-09 09:34:59 +010018#include "hf/api.h"
Daniel Boulbya2f8c662021-11-26 17:52:53 +000019#include "hf/assert.h"
Jose Marinho09b1db82019-08-08 09:16:59 +010020#include "hf/check.h"
Jose Marinho75509b42019-04-09 09:34:59 +010021#include "hf/dlog.h"
J-Alves3456e032023-07-20 12:20:05 +010022#include "hf/ffa.h"
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010023#include "hf/ffa_internal.h"
J-Alves66652252022-07-06 09:49:51 +010024#include "hf/ffa_memory_internal.h"
J-Alves3456e032023-07-20 12:20:05 +010025#include "hf/ffa_partition_manifest.h"
J-Alves5952d942022-12-22 16:03:00 +000026#include "hf/mm.h"
Andrew Walbran475c1452020-02-07 13:22:22 +000027#include "hf/mpool.h"
J-Alvescf6253e2024-01-03 13:48:48 +000028#include "hf/panic.h"
29#include "hf/plat/memory_protect.h"
Jose Marinho75509b42019-04-09 09:34:59 +010030#include "hf/std.h"
Andrew Scull3c257452019-11-26 13:32:50 +000031#include "hf/vm.h"
Daniel Boulby44e9b3b2024-01-17 12:21:44 +000032#include "hf/vm_ids.h"
Jose Marinho75509b42019-04-09 09:34:59 +010033
J-Alves2d8457f2022-10-05 11:06:41 +010034#include "vmapi/hf/ffa_v1_0.h"
35
J-Alves5da37d92022-10-24 16:33:48 +010036#define RECEIVERS_COUNT_IN_RETRIEVE_RESP 1
37
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000038/**
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010039 * All access to members of a `struct ffa_memory_share_state` must be guarded
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000040 * by this lock.
41 */
42static struct spinlock share_states_lock_instance = SPINLOCK_INIT;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010043static struct ffa_memory_share_state share_states[MAX_MEM_SHARES];
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000044
45/**
J-Alvesed508c82023-05-04 16:09:48 +010046 * Return the offset to the first constituent within the
47 * `ffa_composite_memory_region` for the given receiver from an
48 * `ffa_memory_region`. The caller must check that the receiver_index is within
49 * bounds, and that it has a composite memory region offset.
50 */
51static uint32_t ffa_composite_constituent_offset(
52 struct ffa_memory_region *memory_region, uint32_t receiver_index)
53{
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +000054 struct ffa_memory_access *receiver;
55 uint32_t composite_offset;
J-Alvesed508c82023-05-04 16:09:48 +010056
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +000057 CHECK(receiver_index < memory_region->receiver_count);
58
59 receiver =
60 ffa_memory_region_get_receiver(memory_region, receiver_index);
61 CHECK(receiver != NULL);
62
63 composite_offset = receiver->composite_memory_region_offset;
64
65 CHECK(composite_offset != 0);
66
67 return composite_offset + sizeof(struct ffa_composite_memory_region);
J-Alvesed508c82023-05-04 16:09:48 +010068}
69
70/**
J-Alves917d2f22020-10-30 18:39:30 +000071 * Extracts the index from a memory handle allocated by Hafnium's current world.
72 */
73uint64_t ffa_memory_handle_get_index(ffa_memory_handle_t handle)
74{
75 return handle & ~FFA_MEMORY_HANDLE_ALLOCATOR_MASK;
76}
77
78/**
Karl Meakin52cdfe72023-06-30 14:49:10 +010079 * Initialises the next available `struct ffa_memory_share_state`. If `handle`
80 * is `FFA_MEMORY_HANDLE_INVALID` then allocates an appropriate handle,
81 * otherwise uses the provided handle which is assumed to be globally unique.
Andrew Walbranca808b12020-05-15 17:22:28 +010082 *
Karl Meakin52cdfe72023-06-30 14:49:10 +010083 * Returns a pointer to the allocated `ffa_memory_share_state` on success or
84 * `NULL` if none are available.
Andrew Walbranca808b12020-05-15 17:22:28 +010085 */
Karl Meakin52cdfe72023-06-30 14:49:10 +010086struct ffa_memory_share_state *allocate_share_state(
87 struct share_states_locked share_states, uint32_t share_func,
88 struct ffa_memory_region *memory_region, uint32_t fragment_length,
89 ffa_memory_handle_t handle)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000090{
Daniel Boulbya2f8c662021-11-26 17:52:53 +000091 assert(share_states.share_states != NULL);
92 assert(memory_region != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000093
Karl Meakin52cdfe72023-06-30 14:49:10 +010094 for (uint64_t i = 0; i < MAX_MEM_SHARES; ++i) {
Andrew Walbranca808b12020-05-15 17:22:28 +010095 if (share_states.share_states[i].share_func == 0) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010096 struct ffa_memory_share_state *allocated_state =
Andrew Walbranca808b12020-05-15 17:22:28 +010097 &share_states.share_states[i];
98 struct ffa_composite_memory_region *composite =
99 ffa_memory_region_get_composite(memory_region,
100 0);
101
102 if (handle == FFA_MEMORY_HANDLE_INVALID) {
J-Alvesee68c542020-10-29 17:48:20 +0000103 memory_region->handle =
Olivier Deprez55a189e2021-06-09 15:45:27 +0200104 plat_ffa_memory_handle_make(i);
Andrew Walbranca808b12020-05-15 17:22:28 +0100105 } else {
J-Alvesee68c542020-10-29 17:48:20 +0000106 memory_region->handle = handle;
Andrew Walbranca808b12020-05-15 17:22:28 +0100107 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000108 allocated_state->share_func = share_func;
109 allocated_state->memory_region = memory_region;
Andrew Walbranca808b12020-05-15 17:22:28 +0100110 allocated_state->fragment_count = 1;
111 allocated_state->fragments[0] = composite->constituents;
112 allocated_state->fragment_constituent_counts[0] =
113 (fragment_length -
114 ffa_composite_constituent_offset(memory_region,
115 0)) /
116 sizeof(struct ffa_memory_region_constituent);
117 allocated_state->sending_complete = false;
Karl Meakin52cdfe72023-06-30 14:49:10 +0100118 for (uint32_t j = 0; j < MAX_MEM_SHARE_RECIPIENTS;
119 ++j) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100120 allocated_state->retrieved_fragment_count[j] =
121 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000122 }
Karl Meakin52cdfe72023-06-30 14:49:10 +0100123 return allocated_state;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000124 }
125 }
126
Karl Meakin52cdfe72023-06-30 14:49:10 +0100127 return NULL;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000128}
129
130/** Locks the share states lock. */
131struct share_states_locked share_states_lock(void)
132{
133 sl_lock(&share_states_lock_instance);
134
135 return (struct share_states_locked){.share_states = share_states};
136}
137
138/** Unlocks the share states lock. */
J-Alves66652252022-07-06 09:49:51 +0100139void share_states_unlock(struct share_states_locked *share_states)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000140{
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000141 assert(share_states->share_states != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000142 share_states->share_states = NULL;
143 sl_unlock(&share_states_lock_instance);
144}
145
146/**
Andrew Walbranca808b12020-05-15 17:22:28 +0100147 * If the given handle is a valid handle for an allocated share state then
Karl Meakin4a2854a2023-06-30 16:26:52 +0100148 * returns a pointer to the share state. Otherwise returns NULL.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000149 */
Karl Meakin4a2854a2023-06-30 16:26:52 +0100150struct ffa_memory_share_state *get_share_state(
151 struct share_states_locked share_states, ffa_memory_handle_t handle)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000152{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100153 struct ffa_memory_share_state *share_state;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000154
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000155 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100156
157 /*
158 * First look for a share_state allocated by us, in which case the
159 * handle is based on the index.
160 */
Olivier Deprez55a189e2021-06-09 15:45:27 +0200161 if (plat_ffa_memory_handle_allocated_by_current_world(handle)) {
Karl Meakin4a2854a2023-06-30 16:26:52 +0100162 uint64_t index = ffa_memory_handle_get_index(handle);
163
Andrew Walbranca808b12020-05-15 17:22:28 +0100164 if (index < MAX_MEM_SHARES) {
165 share_state = &share_states.share_states[index];
166 if (share_state->share_func != 0) {
Karl Meakin4a2854a2023-06-30 16:26:52 +0100167 return share_state;
Andrew Walbranca808b12020-05-15 17:22:28 +0100168 }
169 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000170 }
171
Andrew Walbranca808b12020-05-15 17:22:28 +0100172 /* Fall back to a linear scan. */
Karl Meakin4a2854a2023-06-30 16:26:52 +0100173 for (uint64_t index = 0; index < MAX_MEM_SHARES; ++index) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100174 share_state = &share_states.share_states[index];
J-Alvesee68c542020-10-29 17:48:20 +0000175 if (share_state->memory_region != NULL &&
176 share_state->memory_region->handle == handle &&
Andrew Walbranca808b12020-05-15 17:22:28 +0100177 share_state->share_func != 0) {
Karl Meakin4a2854a2023-06-30 16:26:52 +0100178 return share_state;
Andrew Walbranca808b12020-05-15 17:22:28 +0100179 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000180 }
181
Karl Meakin4a2854a2023-06-30 16:26:52 +0100182 return NULL;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000183}
184
185/** Marks a share state as unallocated. */
J-Alvesfdd29272022-07-19 13:16:31 +0100186void share_state_free(struct share_states_locked share_states,
187 struct ffa_memory_share_state *share_state,
188 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000189{
Andrew Walbranca808b12020-05-15 17:22:28 +0100190 uint32_t i;
191
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000192 assert(share_states.share_states != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000193 share_state->share_func = 0;
Andrew Walbranca808b12020-05-15 17:22:28 +0100194 share_state->sending_complete = false;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000195 mpool_free(page_pool, share_state->memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +0100196 /*
197 * First fragment is part of the same page as the `memory_region`, so it
198 * doesn't need to be freed separately.
199 */
200 share_state->fragments[0] = NULL;
201 share_state->fragment_constituent_counts[0] = 0;
202 for (i = 1; i < share_state->fragment_count; ++i) {
203 mpool_free(page_pool, share_state->fragments[i]);
204 share_state->fragments[i] = NULL;
205 share_state->fragment_constituent_counts[i] = 0;
206 }
207 share_state->fragment_count = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000208 share_state->memory_region = NULL;
J-Alvesa9cd7e32022-07-01 13:49:33 +0100209 share_state->hypervisor_fragment_count = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000210}
211
Andrew Walbranca808b12020-05-15 17:22:28 +0100212/** Checks whether the given share state has been fully sent. */
J-Alvesfdd29272022-07-19 13:16:31 +0100213bool share_state_sending_complete(struct share_states_locked share_states,
214 struct ffa_memory_share_state *share_state)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000215{
Andrew Walbranca808b12020-05-15 17:22:28 +0100216 struct ffa_composite_memory_region *composite;
217 uint32_t expected_constituent_count;
218 uint32_t fragment_constituent_count_total = 0;
219 uint32_t i;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000220
Andrew Walbranca808b12020-05-15 17:22:28 +0100221 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000222 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100223
224 /*
225 * Share state must already be valid, or it's not possible to get hold
226 * of it.
227 */
228 CHECK(share_state->memory_region != NULL &&
229 share_state->share_func != 0);
230
231 composite =
232 ffa_memory_region_get_composite(share_state->memory_region, 0);
233 expected_constituent_count = composite->constituent_count;
234 for (i = 0; i < share_state->fragment_count; ++i) {
235 fragment_constituent_count_total +=
236 share_state->fragment_constituent_counts[i];
237 }
238 dlog_verbose(
239 "Checking completion: constituent count %d/%d from %d "
240 "fragments.\n",
241 fragment_constituent_count_total, expected_constituent_count,
242 share_state->fragment_count);
243
244 return fragment_constituent_count_total == expected_constituent_count;
245}
246
247/**
248 * Calculates the offset of the next fragment expected for the given share
249 * state.
250 */
J-Alvesfdd29272022-07-19 13:16:31 +0100251uint32_t share_state_next_fragment_offset(
Andrew Walbranca808b12020-05-15 17:22:28 +0100252 struct share_states_locked share_states,
253 struct ffa_memory_share_state *share_state)
254{
255 uint32_t next_fragment_offset;
256 uint32_t i;
257
258 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000259 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100260
261 next_fragment_offset =
262 ffa_composite_constituent_offset(share_state->memory_region, 0);
263 for (i = 0; i < share_state->fragment_count; ++i) {
264 next_fragment_offset +=
265 share_state->fragment_constituent_counts[i] *
266 sizeof(struct ffa_memory_region_constituent);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000267 }
268
Andrew Walbranca808b12020-05-15 17:22:28 +0100269 return next_fragment_offset;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000270}
271
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100272static void dump_memory_region(struct ffa_memory_region *memory_region)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000273{
274 uint32_t i;
275
276 if (LOG_LEVEL < LOG_LEVEL_VERBOSE) {
277 return;
278 }
279
Karl Meakine8937d92024-03-19 16:04:25 +0000280 dlog("from VM %#x, attributes (shareability = %s, cacheability = %s, "
281 "type = %s, security = %s), flags %#x, handle %#lx "
282 "tag %lu, memory access descriptor size %u, to %u "
Andrew Walbrana65a1322020-04-06 19:32:32 +0100283 "recipients [",
Karl Meakine8937d92024-03-19 16:04:25 +0000284 memory_region->sender,
285 ffa_memory_shareability_name(
286 memory_region->attributes.shareability),
287 ffa_memory_cacheability_name(
288 memory_region->attributes.cacheability),
289 ffa_memory_type_name(memory_region->attributes.type),
290 ffa_memory_security_name(memory_region->attributes.security),
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000291 memory_region->flags, memory_region->handle, memory_region->tag,
292 memory_region->memory_access_desc_size,
Andrew Walbrana65a1322020-04-06 19:32:32 +0100293 memory_region->receiver_count);
294 for (i = 0; i < memory_region->receiver_count; ++i) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +0000295 struct ffa_memory_access *receiver =
296 ffa_memory_region_get_receiver(memory_region, i);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000297 if (i != 0) {
298 dlog(", ");
299 }
Karl Meakine8937d92024-03-19 16:04:25 +0000300 dlog("Receiver %#x: permissions (%s, %s) (offset %u)",
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +0000301 receiver->receiver_permissions.receiver,
Karl Meakine8937d92024-03-19 16:04:25 +0000302 ffa_data_access_name(receiver->receiver_permissions
303 .permissions.data_access),
304 ffa_instruction_access_name(
305 receiver->receiver_permissions.permissions
306 .instruction_access),
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +0000307 receiver->composite_memory_region_offset);
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000308 /* The impdef field is only present from v1.2 and later */
309 if (ffa_version_from_memory_access_desc_size(
310 memory_region->memory_access_desc_size) >=
311 MAKE_FFA_VERSION(1, 2)) {
Karl Meakine8937d92024-03-19 16:04:25 +0000312 dlog(", impdef: %#lx %#lx", receiver->impdef.val[0],
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000313 receiver->impdef.val[1]);
314 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000315 }
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000316 dlog("] at offset %u", memory_region->receivers_offset);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000317}
318
J-Alves66652252022-07-06 09:49:51 +0100319void dump_share_states(void)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000320{
321 uint32_t i;
322
323 if (LOG_LEVEL < LOG_LEVEL_VERBOSE) {
324 return;
325 }
326
327 dlog("Current share states:\n");
328 sl_lock(&share_states_lock_instance);
329 for (i = 0; i < MAX_MEM_SHARES; ++i) {
330 if (share_states[i].share_func != 0) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000331 switch (share_states[i].share_func) {
J-Alves95fbb312024-03-20 15:19:16 +0000332 case FFA_MEM_SHARE_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100333 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000334 dlog("SHARE");
335 break;
J-Alves95fbb312024-03-20 15:19:16 +0000336 case FFA_MEM_LEND_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100337 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000338 dlog("LEND");
339 break;
J-Alves95fbb312024-03-20 15:19:16 +0000340 case FFA_MEM_DONATE_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100341 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000342 dlog("DONATE");
343 break;
344 default:
345 dlog("invalid share_func %#x",
346 share_states[i].share_func);
347 }
Karl Meakine8937d92024-03-19 16:04:25 +0000348 dlog(" %#lx (", share_states[i].memory_region->handle);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000349 dump_memory_region(share_states[i].memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +0100350 if (share_states[i].sending_complete) {
351 dlog("): fully sent");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000352 } else {
Andrew Walbranca808b12020-05-15 17:22:28 +0100353 dlog("): partially sent");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000354 }
J-Alves2a0d2882020-10-29 14:49:50 +0000355 dlog(" with %d fragments, %d retrieved, "
356 " sender's original mode: %#x\n",
Andrew Walbranca808b12020-05-15 17:22:28 +0100357 share_states[i].fragment_count,
J-Alves2a0d2882020-10-29 14:49:50 +0000358 share_states[i].retrieved_fragment_count[0],
359 share_states[i].sender_orig_mode);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000360 }
361 }
362 sl_unlock(&share_states_lock_instance);
363}
364
Andrew Walbran475c1452020-02-07 13:22:22 +0000365/* TODO: Add device attributes: GRE, cacheability, shareability. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100366static inline uint32_t ffa_memory_permissions_to_mode(
J-Alves7cd5eb32020-10-16 19:06:10 +0100367 ffa_memory_access_permissions_t permissions, uint32_t default_mode)
Andrew Walbran475c1452020-02-07 13:22:22 +0000368{
369 uint32_t mode = 0;
370
Karl Meakin84710f32023-10-12 15:14:49 +0100371 switch (permissions.data_access) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100372 case FFA_DATA_ACCESS_RO:
Andrew Walbran475c1452020-02-07 13:22:22 +0000373 mode = MM_MODE_R;
374 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100375 case FFA_DATA_ACCESS_RW:
Andrew Walbran475c1452020-02-07 13:22:22 +0000376 mode = MM_MODE_R | MM_MODE_W;
377 break;
J-Alves7cd5eb32020-10-16 19:06:10 +0100378 case FFA_DATA_ACCESS_NOT_SPECIFIED:
379 mode = (default_mode & (MM_MODE_R | MM_MODE_W));
380 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100381 case FFA_DATA_ACCESS_RESERVED:
382 panic("Tried to convert FFA_DATA_ACCESS_RESERVED.");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100383 }
384
Karl Meakin84710f32023-10-12 15:14:49 +0100385 switch (permissions.instruction_access) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100386 case FFA_INSTRUCTION_ACCESS_NX:
Andrew Walbran475c1452020-02-07 13:22:22 +0000387 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100388 case FFA_INSTRUCTION_ACCESS_X:
Andrew Walbrana65a1322020-04-06 19:32:32 +0100389 mode |= MM_MODE_X;
390 break;
J-Alves7cd5eb32020-10-16 19:06:10 +0100391 case FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED:
392 mode |= (default_mode & MM_MODE_X);
393 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100394 case FFA_INSTRUCTION_ACCESS_RESERVED:
395 panic("Tried to convert FFA_INSTRUCTION_ACCESS_RESVERVED.");
Andrew Walbran475c1452020-02-07 13:22:22 +0000396 }
397
Olivier Deprez878bd5b2021-04-15 19:05:10 +0200398 /* Set the security state bit if necessary. */
399 if ((default_mode & plat_ffa_other_world_mode()) != 0) {
400 mode |= plat_ffa_other_world_mode();
401 }
402
Andrew Walbran475c1452020-02-07 13:22:22 +0000403 return mode;
404}
405
Jose Marinho75509b42019-04-09 09:34:59 +0100406/**
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000407 * Get the current mode in the stage-2 page table of the given vm of all the
408 * pages in the given constituents, if they all have the same mode, or return
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100409 * an appropriate FF-A error if not.
Jose Marinho75509b42019-04-09 09:34:59 +0100410 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100411static struct ffa_value constituents_get_mode(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000412 struct vm_locked vm, uint32_t *orig_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100413 struct ffa_memory_region_constituent **fragments,
414 const uint32_t *fragment_constituent_counts, uint32_t fragment_count)
Jose Marinho75509b42019-04-09 09:34:59 +0100415{
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100416 uint32_t i;
Andrew Walbranca808b12020-05-15 17:22:28 +0100417 uint32_t j;
Jose Marinho75509b42019-04-09 09:34:59 +0100418
Andrew Walbranca808b12020-05-15 17:22:28 +0100419 if (fragment_count == 0 || fragment_constituent_counts[0] == 0) {
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100420 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000421 * Fail if there are no constituents. Otherwise we would get an
422 * uninitialised *orig_mode.
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100423 */
Karl Meakin5df422c2023-07-11 17:31:38 +0100424 dlog_verbose("%s: no constituents\n", __func__);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100425 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho75509b42019-04-09 09:34:59 +0100426 }
427
Andrew Walbranca808b12020-05-15 17:22:28 +0100428 for (i = 0; i < fragment_count; ++i) {
429 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
430 ipaddr_t begin = ipa_init(fragments[i][j].address);
431 size_t size = fragments[i][j].page_count * PAGE_SIZE;
432 ipaddr_t end = ipa_add(begin, size);
433 uint32_t current_mode;
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100434
Andrew Walbranca808b12020-05-15 17:22:28 +0100435 /* Fail if addresses are not page-aligned. */
436 if (!is_aligned(ipa_addr(begin), PAGE_SIZE) ||
437 !is_aligned(ipa_addr(end), PAGE_SIZE)) {
Karl Meakin5df422c2023-07-11 17:31:38 +0100438 dlog_verbose("%s: addresses not page-aligned\n",
439 __func__);
Andrew Walbranca808b12020-05-15 17:22:28 +0100440 return ffa_error(FFA_INVALID_PARAMETERS);
441 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100442
Andrew Walbranca808b12020-05-15 17:22:28 +0100443 /*
444 * Ensure that this constituent memory range is all
445 * mapped with the same mode.
446 */
Raghu Krishnamurthy785d52f2021-02-13 00:02:40 -0800447 if (!vm_mem_get_mode(vm, begin, end, &current_mode)) {
Karl Meakin5df422c2023-07-11 17:31:38 +0100448 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +0000449 "%s: constituent memory range "
450 "%#lx..%#lx "
Karl Meakin5df422c2023-07-11 17:31:38 +0100451 "not mapped with the same mode\n",
Karl Meakine8937d92024-03-19 16:04:25 +0000452 __func__, begin.ipa, end.ipa);
Andrew Walbranca808b12020-05-15 17:22:28 +0100453 return ffa_error(FFA_DENIED);
454 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100455
Andrew Walbranca808b12020-05-15 17:22:28 +0100456 /*
457 * Ensure that all constituents are mapped with the same
458 * mode.
459 */
460 if (i == 0) {
461 *orig_mode = current_mode;
462 } else if (current_mode != *orig_mode) {
463 dlog_verbose(
Karl Meakin5df422c2023-07-11 17:31:38 +0100464 "%s: expected mode %#x but was %#x for "
Karl Meakine8937d92024-03-19 16:04:25 +0000465 "%d pages at %#lx.\n",
Karl Meakin5df422c2023-07-11 17:31:38 +0100466 __func__, *orig_mode, current_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100467 fragments[i][j].page_count,
468 ipa_addr(begin));
469 return ffa_error(FFA_DENIED);
470 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100471 }
Jose Marinho75509b42019-04-09 09:34:59 +0100472 }
473
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100474 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000475}
476
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100477uint32_t ffa_version_from_memory_access_desc_size(
478 uint32_t memory_access_desc_size)
479{
480 switch (memory_access_desc_size) {
481 /*
482 * v1.0 and v1.1 memory access descriptors are the same size however
483 * v1.1 is the first version to include the memory access descriptor
484 * size field so return v1.1.
485 */
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000486 case sizeof(struct ffa_memory_access_v1_0):
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100487 return MAKE_FFA_VERSION(1, 1);
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000488 case sizeof(struct ffa_memory_access):
489 return MAKE_FFA_VERSION(1, 2);
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100490 }
491 return 0;
492}
493
494/**
495 * Check if the receivers size and offset given is valid for the senders
496 * FF-A version.
497 */
498static bool receiver_size_and_offset_valid_for_version(
499 uint32_t receivers_size, uint32_t receivers_offset,
500 uint32_t ffa_version)
501{
502 /*
503 * Check that the version that the memory access descriptor size belongs
504 * to is compatible with the FF-A version we believe the sender to be.
505 */
506 uint32_t expected_ffa_version =
507 ffa_version_from_memory_access_desc_size(receivers_size);
508 if (!FFA_VERSIONS_ARE_COMPATIBLE(expected_ffa_version, ffa_version)) {
509 return false;
510 }
511
512 /*
513 * Check the receivers_offset matches the version we found from
514 * memory access descriptor size.
515 */
516 switch (expected_ffa_version) {
517 case MAKE_FFA_VERSION(1, 1):
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000518 case MAKE_FFA_VERSION(1, 2):
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100519 return receivers_offset == sizeof(struct ffa_memory_region);
520 default:
521 return false;
522 }
523}
524
525/**
526 * Check the values set for fields in the memory region are valid and safe.
527 * Offset values are within safe bounds, receiver count will not cause overflows
528 * and reserved fields are 0.
529 */
530bool ffa_memory_region_sanity_check(struct ffa_memory_region *memory_region,
531 uint32_t ffa_version,
532 uint32_t fragment_length,
533 bool send_transaction)
534{
535 uint32_t receiver_count;
536 struct ffa_memory_access *receiver;
537 uint32_t composite_offset_0;
Daniel Boulbyf06b5232024-02-22 16:26:43 +0000538 struct ffa_memory_region_v1_0 *memory_region_v1_0 =
539 (struct ffa_memory_region_v1_0 *)memory_region;
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100540
541 if (ffa_version == MAKE_FFA_VERSION(1, 0)) {
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100542 /* Check the reserved fields are 0. */
543 if (memory_region_v1_0->reserved_0 != 0 ||
544 memory_region_v1_0->reserved_1 != 0) {
545 dlog_verbose("Reserved fields must be 0.\n");
546 return false;
547 }
548
549 receiver_count = memory_region_v1_0->receiver_count;
550 } else {
551 uint32_t receivers_size =
552 memory_region->memory_access_desc_size;
553 uint32_t receivers_offset = memory_region->receivers_offset;
554
555 /* Check the reserved field is 0. */
556 if (memory_region->reserved[0] != 0 ||
557 memory_region->reserved[1] != 0 ||
558 memory_region->reserved[2] != 0) {
559 dlog_verbose("Reserved fields must be 0.\n");
560 return false;
561 }
562
563 /*
564 * Check memory_access_desc_size matches the size of the struct
565 * for the senders FF-A version.
566 */
567 if (!receiver_size_and_offset_valid_for_version(
568 receivers_size, receivers_offset, ffa_version)) {
569 dlog_verbose(
570 "Invalid memory access descriptor size %d, "
571 " or receiver offset %d, "
572 "for FF-A version %#x\n",
573 receivers_size, receivers_offset, ffa_version);
574 return false;
575 }
576
577 receiver_count = memory_region->receiver_count;
578 }
579
580 /* Check receiver count is not too large. */
Daniel Boulbyf06b5232024-02-22 16:26:43 +0000581 if (receiver_count > MAX_MEM_SHARE_RECIPIENTS || receiver_count < 1) {
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100582 dlog_verbose(
Daniel Boulbyf06b5232024-02-22 16:26:43 +0000583 "Receiver count must be 0 < receiver_count < %u "
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100584 "specified %u\n",
585 MAX_MEM_SHARE_RECIPIENTS, receiver_count);
586 return false;
587 }
588
589 /* Check values in the memory access descriptors. */
590 /*
591 * The composite offset values must be the same for all recievers so
592 * check the first one is valid and then they are all the same.
593 */
594 receiver = ffa_version == MAKE_FFA_VERSION(1, 0)
Daniel Boulbyf06b5232024-02-22 16:26:43 +0000595 ? (struct ffa_memory_access *)&memory_region_v1_0
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100596 ->receivers[0]
597 : ffa_memory_region_get_receiver(memory_region, 0);
598 assert(receiver != NULL);
599 composite_offset_0 = receiver->composite_memory_region_offset;
600
601 if (!send_transaction) {
602 if (composite_offset_0 != 0) {
603 dlog_verbose(
604 "Composite offset memory region descriptor "
605 "offset must be 0 for retrieve requests. "
606 "Currently %d",
607 composite_offset_0);
608 return false;
609 }
610 } else {
611 bool comp_offset_is_zero = composite_offset_0 == 0U;
612 bool comp_offset_lt_transaction_descriptor_size =
613 composite_offset_0 <
614 (sizeof(struct ffa_memory_region) +
615 (uint32_t)(memory_region->memory_access_desc_size *
616 memory_region->receiver_count));
617 bool comp_offset_with_comp_gt_fragment_length =
618 composite_offset_0 +
619 sizeof(struct ffa_composite_memory_region) >
620 fragment_length;
621 if (comp_offset_is_zero ||
622 comp_offset_lt_transaction_descriptor_size ||
623 comp_offset_with_comp_gt_fragment_length) {
624 dlog_verbose(
625 "Invalid composite memory region descriptor "
626 "offset for send transaction %u\n",
627 composite_offset_0);
628 return false;
629 }
630 }
631
Karl Meakin824b63d2024-06-03 19:04:53 +0100632 for (size_t i = 0; i < memory_region->receiver_count; i++) {
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100633 uint32_t composite_offset;
634
635 if (ffa_version == MAKE_FFA_VERSION(1, 0)) {
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100636 struct ffa_memory_access_v1_0 *receiver_v1_0 =
637 &memory_region_v1_0->receivers[i];
638 /* Check reserved fields are 0 */
639 if (receiver_v1_0->reserved_0 != 0) {
640 dlog_verbose(
641 "Reserved field in the memory access "
Karl Meakine8937d92024-03-19 16:04:25 +0000642 "descriptor must be zero. Currently "
643 "reciever %zu has a reserved field "
644 "with a value of %lu\n",
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100645 i, receiver_v1_0->reserved_0);
646 return false;
647 }
648 /*
649 * We can cast to the current version receiver as the
650 * remaining fields we are checking have the same
651 * offsets for all versions since memory access
652 * descriptors are forwards compatible.
653 */
654 receiver = (struct ffa_memory_access *)receiver_v1_0;
655 } else {
656 receiver = ffa_memory_region_get_receiver(memory_region,
657 i);
658 assert(receiver != NULL);
659
660 if (receiver->reserved_0 != 0) {
661 dlog_verbose(
662 "Reserved field in the memory access "
Karl Meakine8937d92024-03-19 16:04:25 +0000663 "descriptor must be zero. Currently "
664 "reciever %zu has a reserved field "
665 "with a value of %lu\n",
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100666 i, receiver->reserved_0);
667 return false;
668 }
669 }
670
671 /* Check composite offset values are equal for all receivers. */
672 composite_offset = receiver->composite_memory_region_offset;
673 if (composite_offset != composite_offset_0) {
674 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +0000675 "Composite offset %x differs from %x in "
676 "index\n",
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100677 composite_offset, composite_offset_0);
678 return false;
679 }
680 }
681 return true;
682}
683
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000684/**
J-Alves460d36c2023-10-12 17:02:15 +0100685 * If the receivers for the memory management operation are all from the
686 * secure world and this isn't a FFA_MEM_SHARE, then request memory security
687 * state update by returning MAP_ACTION_CHECK_PROTECT.
688 */
689static enum ffa_map_action ffa_mem_send_get_map_action(
690 bool all_receivers_from_current_world, ffa_id_t sender_id,
691 uint32_t mem_func_id)
692{
J-Alves95fbb312024-03-20 15:19:16 +0000693 const bool is_memory_share_abi = mem_func_id == FFA_MEM_SHARE_32 ||
694 mem_func_id == FFA_MEM_SHARE_64;
695 const bool protect_memory =
696 (!is_memory_share_abi && all_receivers_from_current_world &&
697 ffa_is_vm_id(sender_id));
J-Alves460d36c2023-10-12 17:02:15 +0100698
699 return protect_memory ? MAP_ACTION_CHECK_PROTECT : MAP_ACTION_CHECK;
700}
701
702/**
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000703 * Verify that all pages have the same mode, that the starting mode
704 * constitutes a valid state and obtain the next mode to apply
J-Alves460d36c2023-10-12 17:02:15 +0100705 * to the sending VM. It outputs the mapping action that needs to be
706 * invoked for the given memory range. On memory lend/donate there
707 * could be a need to protect the memory from the normal world.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000708 *
709 * Returns:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100710 * 1) FFA_DENIED if a state transition was not found;
711 * 2) FFA_DENIED if the pages being shared do not have the same mode within
Andrew Walbrana65a1322020-04-06 19:32:32 +0100712 * the <from> VM;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100713 * 3) FFA_INVALID_PARAMETERS if the beginning and end IPAs are not page
Andrew Walbrana65a1322020-04-06 19:32:32 +0100714 * aligned;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100715 * 4) FFA_INVALID_PARAMETERS if the requested share type was not handled.
716 * Or FFA_SUCCESS on success.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000717 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100718static struct ffa_value ffa_send_check_transition(
Andrew Walbrana65a1322020-04-06 19:32:32 +0100719 struct vm_locked from, uint32_t share_func,
Daniel Boulbya76fd912024-02-22 14:22:15 +0000720 struct ffa_memory_region *memory_region, uint32_t *orig_from_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100721 struct ffa_memory_region_constituent **fragments,
722 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
Daniel Boulby4b846eb2024-05-23 17:32:23 +0100723 uint32_t *from_mode, enum ffa_map_action *map_action, bool zero)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000724{
725 const uint32_t state_mask =
726 MM_MODE_INVALID | MM_MODE_UNOWNED | MM_MODE_SHARED;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100727 struct ffa_value ret;
J-Alves460d36c2023-10-12 17:02:15 +0100728 bool all_receivers_from_current_world = true;
Daniel Boulbya76fd912024-02-22 14:22:15 +0000729 uint32_t receivers_count = memory_region->receiver_count;
J-Alves95fbb312024-03-20 15:19:16 +0000730 const bool is_memory_lend = (share_func == FFA_MEM_LEND_32) ||
731 (share_func == FFA_MEM_LEND_64);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000732
Andrew Walbranca808b12020-05-15 17:22:28 +0100733 ret = constituents_get_mode(from, orig_from_mode, fragments,
734 fragment_constituent_counts,
735 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100736 if (ret.func != FFA_SUCCESS_32) {
Olivier Depreze7eb1682022-03-16 17:09:03 +0100737 dlog_verbose("Inconsistent modes.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100738 return ret;
Andrew Scullb5f49e02019-10-02 13:20:47 +0100739 }
740
Daniel Boulby63af1fa2024-03-18 14:17:31 +0000741 /* Device memory regions can only be lent a single borrower. */
Daniel Boulby9764ff62024-01-30 17:47:39 +0000742 if ((*orig_from_mode & MM_MODE_D) != 0U &&
J-Alves95fbb312024-03-20 15:19:16 +0000743 !(is_memory_lend && receivers_count == 1)) {
Daniel Boulby9764ff62024-01-30 17:47:39 +0000744 dlog_verbose(
Daniel Boulby63af1fa2024-03-18 14:17:31 +0000745 "Device memory can only be lent to a single borrower "
746 "(mode is %#x).\n",
Daniel Boulby9764ff62024-01-30 17:47:39 +0000747 *orig_from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100748 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000749 }
750
751 /*
752 * Ensure the sender is the owner and has exclusive access to the
753 * memory.
754 */
755 if ((*orig_from_mode & state_mask) != 0) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100756 return ffa_error(FFA_DENIED);
Andrew Walbrana65a1322020-04-06 19:32:32 +0100757 }
758
Daniel Boulby4b846eb2024-05-23 17:32:23 +0100759 /*
760 * Memory cannot be zeroed during the lend/donate operation if the
761 * sender only has RO access.
762 */
763 if ((*orig_from_mode & MM_MODE_W) == 0 && zero == true) {
764 dlog_verbose(
765 "Cannot zero memory when the sender doesn't have "
766 "write access\n");
767 return ffa_error(FFA_DENIED);
768 }
769
Daniel Boulbya76fd912024-02-22 14:22:15 +0000770 assert(receivers_count > 0U);
J-Alves7cd5eb32020-10-16 19:06:10 +0100771
J-Alves363f5722022-04-25 17:37:37 +0100772 for (uint32_t i = 0U; i < receivers_count; i++) {
Daniel Boulbya76fd912024-02-22 14:22:15 +0000773 struct ffa_memory_access *receiver =
774 ffa_memory_region_get_receiver(memory_region, i);
775 assert(receiver != NULL);
J-Alves363f5722022-04-25 17:37:37 +0100776 ffa_memory_access_permissions_t permissions =
Daniel Boulbya76fd912024-02-22 14:22:15 +0000777 receiver->receiver_permissions.permissions;
J-Alves363f5722022-04-25 17:37:37 +0100778 uint32_t required_from_mode = ffa_memory_permissions_to_mode(
779 permissions, *orig_from_mode);
780
J-Alves788b4492023-04-18 14:01:23 +0100781 /*
782 * The assumption is that at this point, the operation from
783 * SP to a receiver VM, should have returned an FFA_ERROR
784 * already.
785 */
786 if (!ffa_is_vm_id(from.vm->id)) {
787 assert(!ffa_is_vm_id(
Daniel Boulbya76fd912024-02-22 14:22:15 +0000788 receiver->receiver_permissions.receiver));
J-Alves788b4492023-04-18 14:01:23 +0100789 }
790
J-Alves460d36c2023-10-12 17:02:15 +0100791 /* Track if all senders are from current world. */
792 all_receivers_from_current_world =
793 all_receivers_from_current_world &&
794 vm_id_is_current_world(
Daniel Boulbya76fd912024-02-22 14:22:15 +0000795 receiver->receiver_permissions.receiver);
J-Alves460d36c2023-10-12 17:02:15 +0100796
J-Alves363f5722022-04-25 17:37:37 +0100797 if ((*orig_from_mode & required_from_mode) !=
798 required_from_mode) {
799 dlog_verbose(
800 "Sender tried to send memory with permissions "
J-Alves788b4492023-04-18 14:01:23 +0100801 "which required mode %#x but only had %#x "
802 "itself.\n",
J-Alves363f5722022-04-25 17:37:37 +0100803 required_from_mode, *orig_from_mode);
804 return ffa_error(FFA_DENIED);
805 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000806 }
807
J-Alves460d36c2023-10-12 17:02:15 +0100808 *map_action = ffa_mem_send_get_map_action(
809 all_receivers_from_current_world, from.vm->id, share_func);
810
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000811 /* Find the appropriate new mode. */
812 *from_mode = ~state_mask & *orig_from_mode;
Andrew Walbrane7ad3c02019-12-24 17:03:04 +0000813 switch (share_func) {
J-Alves95fbb312024-03-20 15:19:16 +0000814 case FFA_MEM_DONATE_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100815 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000816 *from_mode |= MM_MODE_INVALID | MM_MODE_UNOWNED;
Jose Marinho75509b42019-04-09 09:34:59 +0100817 break;
J-Alves95fbb312024-03-20 15:19:16 +0000818 case FFA_MEM_LEND_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100819 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000820 *from_mode |= MM_MODE_INVALID;
Andrew Walbran648fc3e2019-10-22 16:23:05 +0100821 break;
J-Alves95fbb312024-03-20 15:19:16 +0000822 case FFA_MEM_SHARE_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100823 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000824 *from_mode |= MM_MODE_SHARED;
Jose Marinho56c25732019-05-20 09:48:53 +0100825 break;
826
Jose Marinho75509b42019-04-09 09:34:59 +0100827 default:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100828 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho75509b42019-04-09 09:34:59 +0100829 }
830
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100831 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000832}
833
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100834static struct ffa_value ffa_relinquish_check_transition(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000835 struct vm_locked from, uint32_t *orig_from_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100836 struct ffa_memory_region_constituent **fragments,
837 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
838 uint32_t *from_mode)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000839{
840 const uint32_t state_mask =
841 MM_MODE_INVALID | MM_MODE_UNOWNED | MM_MODE_SHARED;
842 uint32_t orig_from_state;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100843 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000844
Andrew Walbranca808b12020-05-15 17:22:28 +0100845 ret = constituents_get_mode(from, orig_from_mode, fragments,
846 fragment_constituent_counts,
847 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100848 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbrana65a1322020-04-06 19:32:32 +0100849 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000850 }
851
852 /* Ensure the address range is normal memory and not a device. */
853 if (*orig_from_mode & MM_MODE_D) {
854 dlog_verbose("Can't relinquish device memory (mode is %#x).\n",
855 *orig_from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100856 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000857 }
858
859 /*
860 * Ensure the relinquishing VM is not the owner but has access to the
861 * memory.
862 */
863 orig_from_state = *orig_from_mode & state_mask;
864 if ((orig_from_state & ~MM_MODE_SHARED) != MM_MODE_UNOWNED) {
865 dlog_verbose(
866 "Tried to relinquish memory in state %#x (masked %#x "
Andrew Walbranca808b12020-05-15 17:22:28 +0100867 "but should be %#x).\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000868 *orig_from_mode, orig_from_state, MM_MODE_UNOWNED);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100869 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000870 }
871
872 /* Find the appropriate new mode. */
873 *from_mode = (~state_mask & *orig_from_mode) | MM_MODE_UNMAPPED_MASK;
874
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100875 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000876}
877
878/**
879 * Verify that all pages have the same mode, that the starting mode
880 * constitutes a valid state and obtain the next mode to apply
881 * to the retrieving VM.
882 *
883 * Returns:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100884 * 1) FFA_DENIED if a state transition was not found;
885 * 2) FFA_DENIED if the pages being shared do not have the same mode within
Andrew Walbrana65a1322020-04-06 19:32:32 +0100886 * the <to> VM;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100887 * 3) FFA_INVALID_PARAMETERS if the beginning and end IPAs are not page
Andrew Walbrana65a1322020-04-06 19:32:32 +0100888 * aligned;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100889 * 4) FFA_INVALID_PARAMETERS if the requested share type was not handled.
890 * Or FFA_SUCCESS on success.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000891 */
J-Alvesfc19b372022-07-06 12:17:35 +0100892struct ffa_value ffa_retrieve_check_transition(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000893 struct vm_locked to, uint32_t share_func,
Andrew Walbranca808b12020-05-15 17:22:28 +0100894 struct ffa_memory_region_constituent **fragments,
895 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
J-Alvesfd206052023-05-22 16:45:00 +0100896 uint32_t memory_to_attributes, uint32_t *to_mode, bool memory_protected,
897 enum ffa_map_action *map_action)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000898{
899 uint32_t orig_to_mode;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100900 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000901
Andrew Walbranca808b12020-05-15 17:22:28 +0100902 ret = constituents_get_mode(to, &orig_to_mode, fragments,
903 fragment_constituent_counts,
904 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100905 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100906 dlog_verbose("Inconsistent modes.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100907 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000908 }
909
J-Alves460d36c2023-10-12 17:02:15 +0100910 /* Find the appropriate new mode. */
911 *to_mode = memory_to_attributes;
912
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100913 if (share_func == FFA_MEM_RECLAIM_32) {
J-Alves9256f162021-12-09 13:18:43 +0000914 /*
915 * If the original ffa memory send call has been processed
916 * successfully, it is expected the orig_to_mode would overlay
917 * with `state_mask`, as a result of the function
918 * `ffa_send_check_transition`.
J-Alvesfd206052023-05-22 16:45:00 +0100919 *
920 * If Hafnium is the SPMC:
921 * - Caller of the reclaim interface is an SP, the memory shall
922 * have been protected throughout the flow.
923 * - Caller of the reclaim is from the NWd, the memory may have
924 * been protected at the time of lending/donating the memory.
925 * In such case, set action to unprotect memory in the
926 * handling of reclaim operation.
927 * - If Hafnium is the hypervisor memory shall never have been
928 * protected in memory lend/share/donate.
929 *
930 * More details in the doc comment of the function
931 * `ffa_region_group_identity_map`.
J-Alves9256f162021-12-09 13:18:43 +0000932 */
J-Alves59ed0042022-07-28 18:26:41 +0100933 if (vm_id_is_current_world(to.vm->id)) {
934 assert((orig_to_mode &
935 (MM_MODE_INVALID | MM_MODE_UNOWNED |
936 MM_MODE_SHARED)) != 0U);
J-Alvesfd206052023-05-22 16:45:00 +0100937 assert(!memory_protected);
938 } else if (to.vm->id == HF_OTHER_WORLD_ID &&
939 map_action != NULL && memory_protected) {
940 *map_action = MAP_ACTION_COMMIT_UNPROTECT;
J-Alves59ed0042022-07-28 18:26:41 +0100941 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000942 } else {
943 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +0100944 * If the retriever is from virtual FF-A instance:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000945 * Ensure the retriever has the expected state. We don't care
946 * about the MM_MODE_SHARED bit; either with or without it set
947 * are both valid representations of the !O-NA state.
948 */
J-Alvesa9cd7e32022-07-01 13:49:33 +0100949 if (vm_id_is_current_world(to.vm->id) &&
950 to.vm->id != HF_PRIMARY_VM_ID &&
951 (orig_to_mode & MM_MODE_UNMAPPED_MASK) !=
952 MM_MODE_UNMAPPED_MASK) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100953 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000954 }
J-Alves460d36c2023-10-12 17:02:15 +0100955
956 /*
957 * If memory has been protected before, clear the NS bit to
958 * allow the secure access from the SP.
959 */
960 if (memory_protected) {
961 *to_mode &= ~plat_ffa_other_world_mode();
962 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000963 }
964
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000965 switch (share_func) {
J-Alves95fbb312024-03-20 15:19:16 +0000966 case FFA_MEM_DONATE_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100967 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000968 *to_mode |= 0;
969 break;
J-Alves95fbb312024-03-20 15:19:16 +0000970 case FFA_MEM_LEND_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100971 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000972 *to_mode |= MM_MODE_UNOWNED;
973 break;
J-Alves95fbb312024-03-20 15:19:16 +0000974 case FFA_MEM_SHARE_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100975 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000976 *to_mode |= MM_MODE_UNOWNED | MM_MODE_SHARED;
977 break;
978
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100979 case FFA_MEM_RECLAIM_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000980 *to_mode |= 0;
981 break;
982
983 default:
Andrew Walbranca808b12020-05-15 17:22:28 +0100984 dlog_error("Invalid share_func %#x.\n", share_func);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100985 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000986 }
987
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100988 return (struct ffa_value){.func = FFA_SUCCESS_32};
Jose Marinho75509b42019-04-09 09:34:59 +0100989}
Jose Marinho09b1db82019-08-08 09:16:59 +0100990
J-Alvescf6253e2024-01-03 13:48:48 +0000991/*
992 * Performs the operations related to the `action` MAP_ACTION_CHECK*.
993 * Returns:
994 * - FFA_SUCCESS_32: if all goes well.
995 * - FFA_ERROR_32: with FFA_NO_MEMORY, if there is no memory to manage
996 * the page table update. Or error code provided by the function
997 * `arch_memory_protect`.
998 */
999static struct ffa_value ffa_region_group_check_actions(
1000 struct vm_locked vm_locked, paddr_t pa_begin, paddr_t pa_end,
1001 struct mpool *ppool, uint32_t mode, enum ffa_map_action action,
1002 bool *memory_protected)
1003{
1004 struct ffa_value ret;
1005 bool is_memory_protected;
1006
1007 if (!vm_identity_prepare(vm_locked, pa_begin, pa_end, mode, ppool)) {
1008 dlog_verbose(
1009 "%s: memory can't be mapped to %x due to lack of "
Karl Meakine8937d92024-03-19 16:04:25 +00001010 "memory. Base: %lx end: %lx\n",
J-Alvescf6253e2024-01-03 13:48:48 +00001011 __func__, vm_locked.vm->id, pa_addr(pa_begin),
1012 pa_addr(pa_end));
1013 return ffa_error(FFA_NO_MEMORY);
1014 }
1015
1016 switch (action) {
1017 case MAP_ACTION_CHECK:
1018 /* No protect requested. */
1019 is_memory_protected = false;
1020 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
1021 break;
1022 case MAP_ACTION_CHECK_PROTECT: {
1023 paddr_t last_protected_pa = pa_init(0);
1024
1025 ret = arch_memory_protect(pa_begin, pa_end, &last_protected_pa);
1026
1027 is_memory_protected = (ret.func == FFA_SUCCESS_32);
1028
1029 /*
1030 * - If protect memory has failed with FFA_DENIED, means some
1031 * range of memory was in the wrong state. In such case, SPM
1032 * reverts the state of the pages that were successfully
1033 * updated.
1034 * - If protect memory has failed with FFA_NOT_SUPPORTED, it
1035 * means the platform doesn't support the protection mechanism.
1036 * That said, it still permits the page table update to go
1037 * through. The variable
1038 * `is_memory_protected` will be equal to false.
1039 * - If protect memory has failed with FFA_INVALID_PARAMETERS,
1040 * break from switch and return the error.
1041 */
1042 if (ret.func == FFA_ERROR_32) {
1043 assert(!is_memory_protected);
1044 if (ffa_error_code(ret) == FFA_DENIED &&
1045 pa_addr(last_protected_pa) != (uintptr_t)0) {
1046 CHECK(arch_memory_unprotect(
1047 pa_begin,
1048 pa_add(last_protected_pa, PAGE_SIZE)));
1049 } else if (ffa_error_code(ret) == FFA_NOT_SUPPORTED) {
1050 ret = (struct ffa_value){
1051 .func = FFA_SUCCESS_32,
1052 };
1053 }
1054 }
1055 } break;
1056 default:
1057 panic("%s: invalid action to process %x\n", __func__, action);
1058 }
1059
1060 if (memory_protected != NULL) {
1061 *memory_protected = is_memory_protected;
1062 }
1063
1064 return ret;
1065}
1066
1067static void ffa_region_group_commit_actions(struct vm_locked vm_locked,
1068 paddr_t pa_begin, paddr_t pa_end,
1069 struct mpool *ppool, uint32_t mode,
1070 enum ffa_map_action action)
1071{
1072 switch (action) {
1073 case MAP_ACTION_COMMIT_UNPROTECT:
1074 /*
1075 * Checking that it should succeed because SPM should be
1076 * unprotecting memory that it had protected before.
1077 */
1078 CHECK(arch_memory_unprotect(pa_begin, pa_end));
1079 case MAP_ACTION_COMMIT:
1080 vm_identity_commit(vm_locked, pa_begin, pa_end, mode, ppool,
1081 NULL);
1082 break;
1083 default:
1084 panic("%s: invalid action to process %x\n", __func__, action);
1085 }
1086}
1087
Jose Marinho09b1db82019-08-08 09:16:59 +01001088/**
J-Alves063ad832023-10-03 18:05:40 +01001089 * Helper function to revert a failed "Protect" action from the SPMC:
1090 * - `fragment_count`: should specify the number of fragments to traverse from
1091 * `fragments`. This may not be the full amount of fragments that are part of
1092 * the share_state structure.
1093 * - `fragment_constituent_counts`: array holding the amount of constituents
1094 * per fragment.
1095 * - `end`: pointer to the constituent that failed the "protect" action. It
1096 * shall be part of the last fragment, and it shall make the loop below break.
1097 */
1098static void ffa_region_group_fragments_revert_protect(
1099 struct ffa_memory_region_constituent **fragments,
1100 const uint32_t *fragment_constituent_counts, uint32_t fragment_count,
1101 const struct ffa_memory_region_constituent *end)
1102{
1103 for (uint32_t i = 0; i < fragment_count; ++i) {
1104 for (uint32_t j = 0; j < fragment_constituent_counts[i]; ++j) {
1105 struct ffa_memory_region_constituent *constituent =
1106 &fragments[i][j];
1107 size_t size = constituent->page_count * PAGE_SIZE;
1108 paddr_t pa_begin =
1109 pa_from_ipa(ipa_init(constituent->address));
1110 paddr_t pa_end = pa_add(pa_begin, size);
1111
Karl Meakine8937d92024-03-19 16:04:25 +00001112 dlog_verbose("%s: reverting fragment %lx size %zx\n",
J-Alves063ad832023-10-03 18:05:40 +01001113 __func__, pa_addr(pa_begin), size);
1114
1115 if (constituent == end) {
1116 /*
1117 * The last constituent is expected to be in the
1118 * last fragment.
1119 */
1120 assert(i == fragment_count - 1);
1121 break;
1122 }
1123
1124 CHECK(arch_memory_unprotect(pa_begin, pa_end));
1125 }
1126 }
1127}
1128
1129/**
Jose Marinho09b1db82019-08-08 09:16:59 +01001130 * Updates a VM's page table such that the given set of physical address ranges
1131 * are mapped in the address space at the corresponding address ranges, in the
1132 * mode provided.
1133 *
J-Alves0a83dc22023-05-05 09:50:37 +01001134 * The enum ffa_map_action determines the action taken from a call to the
1135 * function below:
1136 * - If action is MAP_ACTION_CHECK, the page tables will be allocated from the
1137 * mpool but no mappings will actually be updated. This function must always
1138 * be called first with action set to MAP_ACTION_CHECK to check that it will
1139 * succeed before calling ffa_region_group_identity_map with whichever one of
1140 * the remaining actions, to avoid leaving the page table in a half-updated
1141 * state.
1142 * - The action MAP_ACTION_COMMIT allocates the page tables from the mpool, and
1143 * changes the memory mappings.
J-Alvescf6253e2024-01-03 13:48:48 +00001144 * - The action MAP_ACTION_CHECK_PROTECT extends the MAP_ACTION_CHECK with an
1145 * invocation to the monitor to update the security state of the memory,
1146 * to that of the SPMC.
1147 * - The action MAP_ACTION_COMMIT_UNPROTECT extends the MAP_ACTION_COMMIT
1148 * with a call into the monitor, to reset the security state of memory
1149 * that has priorly been mapped with the MAP_ACTION_CHECK_PROTECT action.
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001150 * vm_ptable_defrag should always be called after a series of page table
1151 * updates, whether they succeed or fail.
Jose Marinho09b1db82019-08-08 09:16:59 +01001152 *
J-Alvescf6253e2024-01-03 13:48:48 +00001153 * If all goes well, returns FFA_SUCCESS_32; or FFA_ERROR, with following
1154 * error codes:
1155 * - FFA_INVALID_PARAMETERS: invalid range of memory.
1156 * - FFA_DENIED:
1157 *
Jose Marinho09b1db82019-08-08 09:16:59 +01001158 * made to memory mappings.
1159 */
J-Alvescf6253e2024-01-03 13:48:48 +00001160struct ffa_value ffa_region_group_identity_map(
Andrew Walbranf4b51af2020-02-03 14:44:54 +00001161 struct vm_locked vm_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01001162 struct ffa_memory_region_constituent **fragments,
1163 const uint32_t *fragment_constituent_counts, uint32_t fragment_count,
J-Alvescf6253e2024-01-03 13:48:48 +00001164 uint32_t mode, struct mpool *ppool, enum ffa_map_action action,
1165 bool *memory_protected)
Jose Marinho09b1db82019-08-08 09:16:59 +01001166{
Andrew Walbranca808b12020-05-15 17:22:28 +01001167 uint32_t i;
1168 uint32_t j;
J-Alvescf6253e2024-01-03 13:48:48 +00001169 struct ffa_value ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01001170
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001171 if (vm_locked.vm->el0_partition) {
1172 mode |= MM_MODE_USER | MM_MODE_NG;
1173 }
1174
Andrew Walbranca808b12020-05-15 17:22:28 +01001175 /* Iterate over the memory region constituents within each fragment. */
1176 for (i = 0; i < fragment_count; ++i) {
1177 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
J-Alves063ad832023-10-03 18:05:40 +01001178 struct ffa_memory_region_constituent *constituent =
1179 &fragments[i][j];
1180 size_t size = constituent->page_count * PAGE_SIZE;
Andrew Walbranca808b12020-05-15 17:22:28 +01001181 paddr_t pa_begin =
J-Alves063ad832023-10-03 18:05:40 +01001182 pa_from_ipa(ipa_init(constituent->address));
Andrew Walbranca808b12020-05-15 17:22:28 +01001183 paddr_t pa_end = pa_add(pa_begin, size);
Jens Wiklander4f1880c2022-10-19 17:00:14 +02001184 uint32_t pa_bits =
1185 arch_mm_get_pa_bits(arch_mm_get_pa_range());
Federico Recanati4fd065d2021-12-13 20:06:23 +01001186
1187 /*
1188 * Ensure the requested region falls into system's PA
1189 * range.
1190 */
Jens Wiklander4f1880c2022-10-19 17:00:14 +02001191 if (((pa_addr(pa_begin) >> pa_bits) > 0) ||
1192 ((pa_addr(pa_end) >> pa_bits) > 0)) {
Federico Recanati4fd065d2021-12-13 20:06:23 +01001193 dlog_error("Region is outside of PA Range\n");
J-Alvescf6253e2024-01-03 13:48:48 +00001194 return ffa_error(FFA_INVALID_PARAMETERS);
Federico Recanati4fd065d2021-12-13 20:06:23 +01001195 }
Andrew Walbranca808b12020-05-15 17:22:28 +01001196
J-Alvescf6253e2024-01-03 13:48:48 +00001197 if (action <= MAP_ACTION_CHECK_PROTECT) {
1198 ret = ffa_region_group_check_actions(
1199 vm_locked, pa_begin, pa_end, ppool,
1200 mode, action, memory_protected);
J-Alves063ad832023-10-03 18:05:40 +01001201
1202 if (ret.func == FFA_ERROR_32 &&
1203 ffa_error_code(ret) == FFA_DENIED) {
1204 if (memory_protected != NULL) {
1205 assert(!*memory_protected);
1206 }
1207
1208 ffa_region_group_fragments_revert_protect(
1209 fragments,
1210 fragment_constituent_counts,
1211 i + 1, constituent);
1212 break;
1213 }
J-Alvescf6253e2024-01-03 13:48:48 +00001214 } else if (action >= MAP_ACTION_COMMIT &&
1215 action < MAP_ACTION_MAX) {
1216 ffa_region_group_commit_actions(
1217 vm_locked, pa_begin, pa_end, ppool,
1218 mode, action);
1219 ret = (struct ffa_value){
1220 .func = FFA_SUCCESS_32};
1221 } else {
1222 panic("%s: Unknown ffa_map_action.\n",
1223 __func__);
Andrew Walbranca808b12020-05-15 17:22:28 +01001224 }
Jose Marinho09b1db82019-08-08 09:16:59 +01001225 }
1226 }
1227
J-Alvescf6253e2024-01-03 13:48:48 +00001228 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01001229}
1230
1231/**
1232 * Clears a region of physical memory by overwriting it with zeros. The data is
1233 * flushed from the cache so the memory has been cleared across the system.
1234 */
J-Alves7db32002021-12-14 14:44:50 +00001235static bool clear_memory(paddr_t begin, paddr_t end, struct mpool *ppool,
1236 uint32_t extra_mode_attributes)
Jose Marinho09b1db82019-08-08 09:16:59 +01001237{
1238 /*
Fuad Tabbaed294af2019-12-20 10:43:01 +00001239 * TODO: change this to a CPU local single page window rather than a
Jose Marinho09b1db82019-08-08 09:16:59 +01001240 * global mapping of the whole range. Such an approach will limit
1241 * the changes to stage-1 tables and will allow only local
1242 * invalidation.
1243 */
1244 bool ret;
1245 struct mm_stage1_locked stage1_locked = mm_lock_stage1();
J-Alves7db32002021-12-14 14:44:50 +00001246 void *ptr = mm_identity_map(stage1_locked, begin, end,
1247 MM_MODE_W | (extra_mode_attributes &
1248 plat_ffa_other_world_mode()),
1249 ppool);
Jose Marinho09b1db82019-08-08 09:16:59 +01001250 size_t size = pa_difference(begin, end);
1251
1252 if (!ptr) {
Jose Marinho09b1db82019-08-08 09:16:59 +01001253 goto fail;
1254 }
1255
1256 memset_s(ptr, size, 0, size);
1257 arch_mm_flush_dcache(ptr, size);
1258 mm_unmap(stage1_locked, begin, end, ppool);
1259
1260 ret = true;
1261 goto out;
1262
1263fail:
1264 ret = false;
1265
1266out:
1267 mm_unlock_stage1(&stage1_locked);
1268
1269 return ret;
1270}
1271
1272/**
1273 * Clears a region of physical memory by overwriting it with zeros. The data is
1274 * flushed from the cache so the memory has been cleared across the system.
1275 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001276static bool ffa_clear_memory_constituents(
J-Alves7db32002021-12-14 14:44:50 +00001277 uint32_t security_state_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +01001278 struct ffa_memory_region_constituent **fragments,
1279 const uint32_t *fragment_constituent_counts, uint32_t fragment_count,
1280 struct mpool *page_pool)
Jose Marinho09b1db82019-08-08 09:16:59 +01001281{
1282 struct mpool local_page_pool;
Andrew Walbranca808b12020-05-15 17:22:28 +01001283 uint32_t i;
Jose Marinho09b1db82019-08-08 09:16:59 +01001284 bool ret = false;
1285
1286 /*
1287 * Create a local pool so any freed memory can't be used by another
1288 * thread. This is to ensure each constituent that is mapped can be
1289 * unmapped again afterwards.
1290 */
Andrew Walbran475c1452020-02-07 13:22:22 +00001291 mpool_init_with_fallback(&local_page_pool, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +01001292
Andrew Walbranca808b12020-05-15 17:22:28 +01001293 /* Iterate over the memory region constituents within each fragment. */
1294 for (i = 0; i < fragment_count; ++i) {
1295 uint32_t j;
Jose Marinho09b1db82019-08-08 09:16:59 +01001296
J-Alves8457f932023-10-11 16:41:45 +01001297 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001298 size_t size = fragments[i][j].page_count * PAGE_SIZE;
1299 paddr_t begin =
1300 pa_from_ipa(ipa_init(fragments[i][j].address));
1301 paddr_t end = pa_add(begin, size);
1302
J-Alves7db32002021-12-14 14:44:50 +00001303 if (!clear_memory(begin, end, &local_page_pool,
1304 security_state_mode)) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001305 /*
1306 * api_clear_memory will defrag on failure, so
1307 * no need to do it here.
1308 */
1309 goto out;
1310 }
Jose Marinho09b1db82019-08-08 09:16:59 +01001311 }
1312 }
1313
Jose Marinho09b1db82019-08-08 09:16:59 +01001314 ret = true;
1315
1316out:
1317 mpool_fini(&local_page_pool);
1318 return ret;
1319}
1320
J-Alves5952d942022-12-22 16:03:00 +00001321static bool is_memory_range_within(ipaddr_t begin, ipaddr_t end,
1322 ipaddr_t in_begin, ipaddr_t in_end)
1323{
1324 return (ipa_addr(begin) >= ipa_addr(in_begin) &&
1325 ipa_addr(begin) < ipa_addr(in_end)) ||
1326 (ipa_addr(end) <= ipa_addr(in_end) &&
1327 ipa_addr(end) > ipa_addr(in_begin));
1328}
1329
1330/**
1331 * Receives a memory range and looks for overlaps with the remainder
1332 * constituents of the memory share/lend/donate operation. Assumes they are
1333 * passed in order to avoid having to loop over all the elements at each call.
1334 * The function only compares the received memory ranges with those that follow
1335 * within the same fragment, and subsequent fragments from the same operation.
1336 */
1337static bool ffa_memory_check_overlap(
1338 struct ffa_memory_region_constituent **fragments,
1339 const uint32_t *fragment_constituent_counts,
1340 const uint32_t fragment_count, const uint32_t current_fragment,
1341 const uint32_t current_constituent)
1342{
1343 uint32_t i = current_fragment;
1344 uint32_t j = current_constituent;
1345 ipaddr_t current_begin = ipa_init(fragments[i][j].address);
1346 const uint32_t current_page_count = fragments[i][j].page_count;
1347 size_t current_size = current_page_count * PAGE_SIZE;
1348 ipaddr_t current_end = ipa_add(current_begin, current_size - 1);
1349
1350 if (current_size == 0 ||
1351 current_size > UINT64_MAX - ipa_addr(current_begin)) {
Karl Meakine8937d92024-03-19 16:04:25 +00001352 dlog_verbose("Invalid page count. Addr: %zx page_count: %x\n",
1353 current_begin.ipa, current_page_count);
J-Alves5952d942022-12-22 16:03:00 +00001354 return false;
1355 }
1356
1357 for (; i < fragment_count; i++) {
1358 j = (i == current_fragment) ? j + 1 : 0;
1359
1360 for (; j < fragment_constituent_counts[i]; j++) {
1361 ipaddr_t begin = ipa_init(fragments[i][j].address);
1362 const uint32_t page_count = fragments[i][j].page_count;
1363 size_t size = page_count * PAGE_SIZE;
1364 ipaddr_t end = ipa_add(begin, size - 1);
1365
1366 if (size == 0 || size > UINT64_MAX - ipa_addr(begin)) {
1367 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00001368 "Invalid page count. Addr: %lx "
J-Alves5952d942022-12-22 16:03:00 +00001369 "page_count: %x\n",
Karl Meakine8937d92024-03-19 16:04:25 +00001370 begin.ipa, page_count);
J-Alves5952d942022-12-22 16:03:00 +00001371 return false;
1372 }
1373
1374 /*
1375 * Check if current ranges is within begin and end, as
1376 * well as the reverse. This should help optimize the
1377 * loop, and reduce the number of iterations.
1378 */
1379 if (is_memory_range_within(begin, end, current_begin,
1380 current_end) ||
1381 is_memory_range_within(current_begin, current_end,
1382 begin, end)) {
1383 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00001384 "Overlapping memory ranges: %#lx - "
1385 "%#lx with %#lx - %#lx\n",
J-Alves5952d942022-12-22 16:03:00 +00001386 ipa_addr(begin), ipa_addr(end),
1387 ipa_addr(current_begin),
1388 ipa_addr(current_end));
1389 return true;
1390 }
1391 }
1392 }
1393
1394 return false;
1395}
1396
Jose Marinho09b1db82019-08-08 09:16:59 +01001397/**
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001398 * Validates and prepares memory to be sent from the calling VM to another.
Jose Marinho09b1db82019-08-08 09:16:59 +01001399 *
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001400 * This function requires the calling context to hold the <from> VM lock.
Jose Marinho09b1db82019-08-08 09:16:59 +01001401 *
1402 * Returns:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001403 * In case of error, one of the following values is returned:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001404 * 1) FFA_INVALID_PARAMETERS - The endpoint provided parameters were
Jose Marinho09b1db82019-08-08 09:16:59 +01001405 * erroneous;
Andrew Walbranf07f04d2020-05-01 18:09:00 +01001406 * 2) FFA_NO_MEMORY - Hafnium did not have sufficient memory to complete the
1407 * request.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001408 * 3) FFA_DENIED - The sender doesn't have sufficient access to send the
Andrew Walbrana65a1322020-04-06 19:32:32 +01001409 * memory with the given permissions.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001410 * Success is indicated by FFA_SUCCESS.
Jose Marinho09b1db82019-08-08 09:16:59 +01001411 */
Daniel Boulbya76fd912024-02-22 14:22:15 +00001412static struct ffa_value ffa_send_check_update(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001413 struct vm_locked from_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01001414 struct ffa_memory_region_constituent **fragments,
1415 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
J-Alves8f11cde2022-12-21 16:18:22 +00001416 uint32_t composite_total_page_count, uint32_t share_func,
Daniel Boulbya76fd912024-02-22 14:22:15 +00001417 struct ffa_memory_region *memory_region, struct mpool *page_pool,
1418 uint32_t *orig_from_mode_ret, bool *memory_protected)
Jose Marinho09b1db82019-08-08 09:16:59 +01001419{
Andrew Walbranca808b12020-05-15 17:22:28 +01001420 uint32_t i;
J-Alves8f11cde2022-12-21 16:18:22 +00001421 uint32_t j;
Jose Marinho09b1db82019-08-08 09:16:59 +01001422 uint32_t orig_from_mode;
J-Alves460d36c2023-10-12 17:02:15 +01001423 uint32_t clean_mode;
Jose Marinho09b1db82019-08-08 09:16:59 +01001424 uint32_t from_mode;
Jose Marinho09b1db82019-08-08 09:16:59 +01001425 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001426 struct ffa_value ret;
J-Alves8f11cde2022-12-21 16:18:22 +00001427 uint32_t constituents_total_page_count = 0;
J-Alves460d36c2023-10-12 17:02:15 +01001428 enum ffa_map_action map_action = MAP_ACTION_CHECK;
Daniel Boulbya76fd912024-02-22 14:22:15 +00001429 bool clear = memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR;
Jose Marinho09b1db82019-08-08 09:16:59 +01001430
1431 /*
Andrew Walbrana65a1322020-04-06 19:32:32 +01001432 * Make sure constituents are properly aligned to a 64-bit boundary. If
1433 * not we would get alignment faults trying to read (64-bit) values.
Jose Marinho09b1db82019-08-08 09:16:59 +01001434 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001435 for (i = 0; i < fragment_count; ++i) {
1436 if (!is_aligned(fragments[i], 8)) {
1437 dlog_verbose("Constituents not aligned.\n");
1438 return ffa_error(FFA_INVALID_PARAMETERS);
1439 }
J-Alves8f11cde2022-12-21 16:18:22 +00001440 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
1441 constituents_total_page_count +=
1442 fragments[i][j].page_count;
J-Alves5952d942022-12-22 16:03:00 +00001443 if (ffa_memory_check_overlap(
1444 fragments, fragment_constituent_counts,
1445 fragment_count, i, j)) {
1446 return ffa_error(FFA_INVALID_PARAMETERS);
1447 }
J-Alves8f11cde2022-12-21 16:18:22 +00001448 }
1449 }
1450
1451 if (constituents_total_page_count != composite_total_page_count) {
1452 dlog_verbose(
1453 "Composite page count differs from calculated page "
1454 "count from constituents.\n");
1455 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho09b1db82019-08-08 09:16:59 +01001456 }
1457
1458 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001459 * Check if the state transition is lawful for the sender, ensure that
1460 * all constituents of a memory region being shared are at the same
1461 * state.
Jose Marinho09b1db82019-08-08 09:16:59 +01001462 */
J-Alves460d36c2023-10-12 17:02:15 +01001463 ret = ffa_send_check_transition(
Daniel Boulbya76fd912024-02-22 14:22:15 +00001464 from_locked, share_func, memory_region, &orig_from_mode,
1465 fragments, fragment_constituent_counts, fragment_count,
Daniel Boulby4b846eb2024-05-23 17:32:23 +01001466 &from_mode, &map_action, clear);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001467 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001468 dlog_verbose("Invalid transition for send.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +01001469 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01001470 }
1471
Andrew Walbran37c574e2020-06-03 11:45:46 +01001472 if (orig_from_mode_ret != NULL) {
1473 *orig_from_mode_ret = orig_from_mode;
1474 }
1475
Jose Marinho09b1db82019-08-08 09:16:59 +01001476 /*
1477 * Create a local pool so any freed memory can't be used by another
1478 * thread. This is to ensure the original mapping can be restored if the
1479 * clear fails.
1480 */
Andrew Walbran475c1452020-02-07 13:22:22 +00001481 mpool_init_with_fallback(&local_page_pool, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +01001482
1483 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001484 * First reserve all required memory for the new page table entries
1485 * without committing, to make sure the entire operation will succeed
1486 * without exhausting the page pool.
J-Alves460d36c2023-10-12 17:02:15 +01001487 * Provide the map_action as populated by 'ffa_send_check_transition'.
1488 * It may request memory to be protected.
Jose Marinho09b1db82019-08-08 09:16:59 +01001489 */
J-Alvescf6253e2024-01-03 13:48:48 +00001490 ret = ffa_region_group_identity_map(
1491 from_locked, fragments, fragment_constituent_counts,
J-Alves460d36c2023-10-12 17:02:15 +01001492 fragment_count, from_mode, page_pool, map_action,
1493 memory_protected);
J-Alvescf6253e2024-01-03 13:48:48 +00001494 if (ret.func == FFA_ERROR_32) {
Jose Marinho09b1db82019-08-08 09:16:59 +01001495 goto out;
1496 }
1497
1498 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001499 * Update the mapping for the sender. This won't allocate because the
1500 * transaction was already prepared above, but may free pages in the
1501 * case that a whole block is being unmapped that was previously
1502 * partially mapped.
Jose Marinho09b1db82019-08-08 09:16:59 +01001503 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001504 CHECK(ffa_region_group_identity_map(
J-Alvescf6253e2024-01-03 13:48:48 +00001505 from_locked, fragments, fragment_constituent_counts,
1506 fragment_count, from_mode, &local_page_pool,
1507 MAP_ACTION_COMMIT, NULL)
1508 .func == FFA_SUCCESS_32);
Jose Marinho09b1db82019-08-08 09:16:59 +01001509
J-Alves460d36c2023-10-12 17:02:15 +01001510 /*
1511 * If memory has been protected, it is now part of the secure PAS
1512 * (happens for lend/donate from NWd to SWd), and the `orig_from_mode`
1513 * should have the MM_MODE_NS set, as such mask it in `clean_mode` for
1514 * SPM's S1 translation.
1515 * In case memory hasn't been protected, and it is in the non-secure
1516 * PAS (e.g. memory share from NWd to SWd), as such the SPM needs to
1517 * perform a non-secure memory access. In such case `clean_mode` takes
1518 * the same mode as `orig_from_mode`.
1519 */
1520 clean_mode = (memory_protected != NULL && *memory_protected)
1521 ? orig_from_mode & ~plat_ffa_other_world_mode()
1522 : orig_from_mode;
1523
Jose Marinho09b1db82019-08-08 09:16:59 +01001524 /* Clear the memory so no VM or device can see the previous contents. */
J-Alves460d36c2023-10-12 17:02:15 +01001525 if (clear && !ffa_clear_memory_constituents(
1526 clean_mode, fragments, fragment_constituent_counts,
1527 fragment_count, page_pool)) {
1528 map_action = (memory_protected != NULL && *memory_protected)
1529 ? MAP_ACTION_COMMIT_UNPROTECT
1530 : MAP_ACTION_COMMIT;
1531
Jose Marinho09b1db82019-08-08 09:16:59 +01001532 /*
1533 * On failure, roll back by returning memory to the sender. This
1534 * may allocate pages which were previously freed into
1535 * `local_page_pool` by the call above, but will never allocate
1536 * more pages than that so can never fail.
1537 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001538 CHECK(ffa_region_group_identity_map(
J-Alvescf6253e2024-01-03 13:48:48 +00001539 from_locked, fragments,
1540 fragment_constituent_counts, fragment_count,
1541 orig_from_mode, &local_page_pool,
1542 MAP_ACTION_COMMIT, NULL)
1543 .func == FFA_SUCCESS_32);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001544 ret = ffa_error(FFA_NO_MEMORY);
Jose Marinho09b1db82019-08-08 09:16:59 +01001545 goto out;
1546 }
1547
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001548 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001549
1550out:
1551 mpool_fini(&local_page_pool);
1552
1553 /*
1554 * Tidy up the page table by reclaiming failed mappings (if there was an
1555 * error) or merging entries into blocks where possible (on success).
1556 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001557 vm_ptable_defrag(from_locked, page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001558
1559 return ret;
1560}
1561
1562/**
1563 * Validates and maps memory shared from one VM to another.
1564 *
1565 * This function requires the calling context to hold the <to> lock.
1566 *
1567 * Returns:
1568 * In case of error, one of the following values is returned:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001569 * 1) FFA_INVALID_PARAMETERS - The endpoint provided parameters were
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001570 * erroneous;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001571 * 2) FFA_NO_MEMORY - Hafnium did not have sufficient memory to complete
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001572 * the request.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001573 * Success is indicated by FFA_SUCCESS.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001574 */
J-Alvesb5084cf2022-07-06 14:20:12 +01001575struct ffa_value ffa_retrieve_check_update(
J-Alves26483382023-04-20 12:01:49 +01001576 struct vm_locked to_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01001577 struct ffa_memory_region_constituent **fragments,
1578 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
J-Alves26483382023-04-20 12:01:49 +01001579 uint32_t sender_orig_mode, uint32_t share_func, bool clear,
J-Alves460d36c2023-10-12 17:02:15 +01001580 struct mpool *page_pool, uint32_t *response_mode, bool memory_protected)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001581{
Andrew Walbranca808b12020-05-15 17:22:28 +01001582 uint32_t i;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001583 uint32_t to_mode;
1584 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001585 struct ffa_value ret;
J-Alvesfd206052023-05-22 16:45:00 +01001586 enum ffa_map_action map_action = MAP_ACTION_COMMIT;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001587
1588 /*
Andrew Walbranca808b12020-05-15 17:22:28 +01001589 * Make sure constituents are properly aligned to a 64-bit boundary. If
1590 * not we would get alignment faults trying to read (64-bit) values.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001591 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001592 for (i = 0; i < fragment_count; ++i) {
1593 if (!is_aligned(fragments[i], 8)) {
J-Alvesb5084cf2022-07-06 14:20:12 +01001594 dlog_verbose("Fragment not properly aligned.\n");
Andrew Walbranca808b12020-05-15 17:22:28 +01001595 return ffa_error(FFA_INVALID_PARAMETERS);
1596 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001597 }
1598
1599 /*
Daniel Boulby4b846eb2024-05-23 17:32:23 +01001600 * Ensure the sender has write permissions if the memory needs to be
1601 * cleared.
1602 */
1603 if ((sender_orig_mode & MM_MODE_W) == 0 && clear == true) {
1604 dlog_verbose(
1605 "Cannot zero memory when the sender does not have "
1606 "write access\n");
1607 return ffa_error(FFA_DENIED);
1608 }
1609
1610 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001611 * Check if the state transition is lawful for the recipient, and ensure
1612 * that all constituents of the memory region being retrieved are at the
1613 * same state.
1614 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001615 ret = ffa_retrieve_check_transition(
1616 to_locked, share_func, fragments, fragment_constituent_counts,
J-Alvesfd206052023-05-22 16:45:00 +01001617 fragment_count, sender_orig_mode, &to_mode, memory_protected,
1618 &map_action);
J-Alves460d36c2023-10-12 17:02:15 +01001619
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001620 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001621 dlog_verbose("Invalid transition for retrieve.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +01001622 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001623 }
1624
1625 /*
1626 * Create a local pool so any freed memory can't be used by another
1627 * thread. This is to ensure the original mapping can be restored if the
1628 * clear fails.
1629 */
1630 mpool_init_with_fallback(&local_page_pool, page_pool);
1631
1632 /*
1633 * First reserve all required memory for the new page table entries in
1634 * the recipient page tables without committing, to make sure the entire
1635 * operation will succeed without exhausting the page pool.
1636 */
J-Alvescf6253e2024-01-03 13:48:48 +00001637 ret = ffa_region_group_identity_map(
1638 to_locked, fragments, fragment_constituent_counts,
1639 fragment_count, to_mode, page_pool, MAP_ACTION_CHECK, NULL);
1640 if (ret.func == FFA_ERROR_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001641 /* TODO: partial defrag of failed range. */
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001642 goto out;
1643 }
1644
1645 /* Clear the memory so no VM or device can see the previous contents. */
J-Alves7db32002021-12-14 14:44:50 +00001646 if (clear &&
J-Alves26483382023-04-20 12:01:49 +01001647 !ffa_clear_memory_constituents(sender_orig_mode, fragments,
1648 fragment_constituent_counts,
1649 fragment_count, page_pool)) {
J-Alvesb5084cf2022-07-06 14:20:12 +01001650 dlog_verbose("Couldn't clear constituents.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001651 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001652 goto out;
1653 }
1654
Jose Marinho09b1db82019-08-08 09:16:59 +01001655 /*
1656 * Complete the transfer by mapping the memory into the recipient. This
1657 * won't allocate because the transaction was already prepared above, so
1658 * it doesn't need to use the `local_page_pool`.
1659 */
J-Alvesfd206052023-05-22 16:45:00 +01001660 CHECK(ffa_region_group_identity_map(
1661 to_locked, fragments, fragment_constituent_counts,
1662 fragment_count, to_mode, page_pool, map_action, NULL)
J-Alvescf6253e2024-01-03 13:48:48 +00001663 .func == FFA_SUCCESS_32);
Jose Marinho09b1db82019-08-08 09:16:59 +01001664
J-Alves460d36c2023-10-12 17:02:15 +01001665 /* Return the mode used in mapping the memory in retriever's PT. */
1666 if (response_mode != NULL) {
1667 *response_mode = to_mode;
1668 }
1669
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001670 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Jose Marinho09b1db82019-08-08 09:16:59 +01001671
1672out:
1673 mpool_fini(&local_page_pool);
1674
1675 /*
Andrew Walbranf07f04d2020-05-01 18:09:00 +01001676 * Tidy up the page table by reclaiming failed mappings (if there was an
1677 * error) or merging entries into blocks where possible (on success).
Jose Marinho09b1db82019-08-08 09:16:59 +01001678 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001679 vm_ptable_defrag(to_locked, page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001680
1681 return ret;
1682}
1683
Andrew Walbran996d1d12020-05-27 14:08:43 +01001684static struct ffa_value ffa_relinquish_check_update(
J-Alves26483382023-04-20 12:01:49 +01001685 struct vm_locked from_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01001686 struct ffa_memory_region_constituent **fragments,
1687 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
1688 struct mpool *page_pool, bool clear)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001689{
1690 uint32_t orig_from_mode;
1691 uint32_t from_mode;
1692 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001693 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001694
Andrew Walbranca808b12020-05-15 17:22:28 +01001695 ret = ffa_relinquish_check_transition(
1696 from_locked, &orig_from_mode, fragments,
1697 fragment_constituent_counts, fragment_count, &from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001698 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001699 dlog_verbose("Invalid transition for relinquish.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +01001700 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001701 }
1702
1703 /*
1704 * Create a local pool so any freed memory can't be used by another
1705 * thread. This is to ensure the original mapping can be restored if the
1706 * clear fails.
1707 */
1708 mpool_init_with_fallback(&local_page_pool, page_pool);
1709
1710 /*
1711 * First reserve all required memory for the new page table entries
1712 * without committing, to make sure the entire operation will succeed
1713 * without exhausting the page pool.
1714 */
J-Alvescf6253e2024-01-03 13:48:48 +00001715 ret = ffa_region_group_identity_map(
1716 from_locked, fragments, fragment_constituent_counts,
1717 fragment_count, from_mode, page_pool, MAP_ACTION_CHECK, NULL);
1718 if (ret.func == FFA_ERROR_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001719 goto out;
1720 }
1721
1722 /*
1723 * Update the mapping for the sender. This won't allocate because the
1724 * transaction was already prepared above, but may free pages in the
1725 * case that a whole block is being unmapped that was previously
1726 * partially mapped.
1727 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001728 CHECK(ffa_region_group_identity_map(
J-Alvescf6253e2024-01-03 13:48:48 +00001729 from_locked, fragments, fragment_constituent_counts,
1730 fragment_count, from_mode, &local_page_pool,
1731 MAP_ACTION_COMMIT, NULL)
1732 .func == FFA_SUCCESS_32);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001733
1734 /* Clear the memory so no VM or device can see the previous contents. */
J-Alves7db32002021-12-14 14:44:50 +00001735 if (clear &&
J-Alves26483382023-04-20 12:01:49 +01001736 !ffa_clear_memory_constituents(orig_from_mode, fragments,
1737 fragment_constituent_counts,
1738 fragment_count, page_pool)) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001739 /*
1740 * On failure, roll back by returning memory to the sender. This
1741 * may allocate pages which were previously freed into
1742 * `local_page_pool` by the call above, but will never allocate
1743 * more pages than that so can never fail.
1744 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001745 CHECK(ffa_region_group_identity_map(
J-Alvescf6253e2024-01-03 13:48:48 +00001746 from_locked, fragments,
1747 fragment_constituent_counts, fragment_count,
1748 orig_from_mode, &local_page_pool,
1749 MAP_ACTION_COMMIT, NULL)
1750 .func == FFA_SUCCESS_32);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001751
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001752 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001753 goto out;
1754 }
1755
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001756 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001757
1758out:
1759 mpool_fini(&local_page_pool);
1760
1761 /*
1762 * Tidy up the page table by reclaiming failed mappings (if there was an
1763 * error) or merging entries into blocks where possible (on success).
1764 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001765 vm_ptable_defrag(from_locked, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +01001766
1767 return ret;
1768}
1769
1770/**
Andrew Walbranca808b12020-05-15 17:22:28 +01001771 * Complete a memory sending operation by checking that it is valid, updating
1772 * the sender page table, and then either marking the share state as having
1773 * completed sending (on success) or freeing it (on failure).
1774 *
1775 * Returns FFA_SUCCESS with the handle encoded, or the relevant FFA_ERROR.
1776 */
J-Alvesfdd29272022-07-19 13:16:31 +01001777struct ffa_value ffa_memory_send_complete(
Andrew Walbranca808b12020-05-15 17:22:28 +01001778 struct vm_locked from_locked, struct share_states_locked share_states,
Andrew Walbran37c574e2020-06-03 11:45:46 +01001779 struct ffa_memory_share_state *share_state, struct mpool *page_pool,
1780 uint32_t *orig_from_mode_ret)
Andrew Walbranca808b12020-05-15 17:22:28 +01001781{
1782 struct ffa_memory_region *memory_region = share_state->memory_region;
J-Alves8f11cde2022-12-21 16:18:22 +00001783 struct ffa_composite_memory_region *composite;
Andrew Walbranca808b12020-05-15 17:22:28 +01001784 struct ffa_value ret;
1785
1786 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +00001787 assert(share_states.share_states != NULL);
J-Alves8f11cde2022-12-21 16:18:22 +00001788 assert(memory_region != NULL);
1789 composite = ffa_memory_region_get_composite(memory_region, 0);
1790 assert(composite != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +01001791
1792 /* Check that state is valid in sender page table and update. */
1793 ret = ffa_send_check_update(
1794 from_locked, share_state->fragments,
1795 share_state->fragment_constituent_counts,
J-Alves8f11cde2022-12-21 16:18:22 +00001796 share_state->fragment_count, composite->page_count,
Daniel Boulbya76fd912024-02-22 14:22:15 +00001797 share_state->share_func, memory_region, page_pool,
J-Alves460d36c2023-10-12 17:02:15 +01001798 orig_from_mode_ret, &share_state->memory_protected);
Andrew Walbranca808b12020-05-15 17:22:28 +01001799 if (ret.func != FFA_SUCCESS_32) {
1800 /*
1801 * Free share state, it failed to send so it can't be retrieved.
1802 */
Karl Meakin4cec5e82023-06-30 16:30:22 +01001803 dlog_verbose("%s: failed to send check update: %s(%s)\n",
1804 __func__, ffa_func_name(ret.func),
1805 ffa_error_name(ffa_error_code(ret)));
Andrew Walbranca808b12020-05-15 17:22:28 +01001806 share_state_free(share_states, share_state, page_pool);
1807 return ret;
1808 }
1809
1810 share_state->sending_complete = true;
Karl Meakin4cec5e82023-06-30 16:30:22 +01001811 dlog_verbose("%s: marked sending complete.\n", __func__);
Andrew Walbranca808b12020-05-15 17:22:28 +01001812
J-Alvesee68c542020-10-29 17:48:20 +00001813 return ffa_mem_success(share_state->memory_region->handle);
Andrew Walbranca808b12020-05-15 17:22:28 +01001814}
1815
1816/**
Daniel Boulby9764ff62024-01-30 17:47:39 +00001817 * Check that the memory attributes match Hafnium expectations.
1818 * Cacheability:
1819 * - Normal Memory as `FFA_MEMORY_CACHE_WRITE_BACK`.
1820 * - Device memory as `FFA_MEMORY_DEV_NGNRNE`.
1821 *
1822 * Shareability:
1823 * - Inner Shareable.
Federico Recanatia98603a2021-12-20 18:04:03 +01001824 */
1825static struct ffa_value ffa_memory_attributes_validate(
J-Alves7a99d0d2023-02-08 13:49:48 +00001826 ffa_memory_attributes_t attributes)
Federico Recanatia98603a2021-12-20 18:04:03 +01001827{
1828 enum ffa_memory_type memory_type;
1829 enum ffa_memory_cacheability cacheability;
1830 enum ffa_memory_shareability shareability;
1831
Karl Meakin84710f32023-10-12 15:14:49 +01001832 memory_type = attributes.type;
Daniel Boulby9764ff62024-01-30 17:47:39 +00001833 cacheability = attributes.cacheability;
1834 if (memory_type == FFA_MEMORY_NORMAL_MEM &&
1835 cacheability != FFA_MEMORY_CACHE_WRITE_BACK) {
1836 dlog_verbose(
1837 "Normal Memory: Invalid cacheability %s, "
1838 "expected %s.\n",
1839 ffa_memory_cacheability_name(cacheability),
1840 ffa_memory_cacheability_name(
1841 FFA_MEMORY_CACHE_WRITE_BACK));
Federico Recanati3d953f32022-02-17 09:31:29 +01001842 return ffa_error(FFA_DENIED);
Federico Recanatia98603a2021-12-20 18:04:03 +01001843 }
Daniel Boulby9764ff62024-01-30 17:47:39 +00001844 if (memory_type == FFA_MEMORY_DEVICE_MEM &&
1845 cacheability != FFA_MEMORY_DEV_NGNRNE) {
1846 dlog_verbose(
1847 "Device Memory: Invalid cacheability %s, "
1848 "expected %s.\n",
1849 ffa_device_memory_cacheability_name(cacheability),
1850 ffa_device_memory_cacheability_name(
1851 FFA_MEMORY_DEV_NGNRNE));
Federico Recanati3d953f32022-02-17 09:31:29 +01001852 return ffa_error(FFA_DENIED);
Federico Recanatia98603a2021-12-20 18:04:03 +01001853 }
1854
Karl Meakin84710f32023-10-12 15:14:49 +01001855 shareability = attributes.shareability;
Federico Recanatia98603a2021-12-20 18:04:03 +01001856 if (shareability != FFA_MEMORY_INNER_SHAREABLE) {
Karl Meakinf98b2aa2023-10-12 16:09:59 +01001857 dlog_verbose("Invalid shareability %s, expected %s.\n",
1858 ffa_memory_shareability_name(shareability),
1859 ffa_memory_shareability_name(
1860 FFA_MEMORY_INNER_SHAREABLE));
Federico Recanati3d953f32022-02-17 09:31:29 +01001861 return ffa_error(FFA_DENIED);
Federico Recanatia98603a2021-12-20 18:04:03 +01001862 }
1863
1864 return (struct ffa_value){.func = FFA_SUCCESS_32};
1865}
1866
1867/**
Andrew Walbrana65a1322020-04-06 19:32:32 +01001868 * Check that the given `memory_region` represents a valid memory send request
1869 * of the given `share_func` type, return the clear flag and permissions via the
1870 * respective output parameters, and update the permissions if necessary.
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001871 *
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001872 * Returns FFA_SUCCESS if the request was valid, or the relevant FFA_ERROR if
Andrew Walbrana65a1322020-04-06 19:32:32 +01001873 * not.
1874 */
J-Alves66652252022-07-06 09:49:51 +01001875struct ffa_value ffa_memory_send_validate(
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001876 struct vm_locked from_locked, struct ffa_memory_region *memory_region,
1877 uint32_t memory_share_length, uint32_t fragment_length,
J-Alves363f5722022-04-25 17:37:37 +01001878 uint32_t share_func)
Andrew Walbrana65a1322020-04-06 19:32:32 +01001879{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001880 struct ffa_composite_memory_region *composite;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001881 struct ffa_memory_access *receiver =
1882 ffa_memory_region_get_receiver(memory_region, 0);
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001883 uint64_t receivers_end;
1884 uint64_t min_length;
Federico Recanati872cd692022-01-05 13:10:10 +01001885 uint32_t composite_memory_region_offset;
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001886 uint32_t constituents_start;
Andrew Walbran130a8ae2020-05-15 16:27:15 +01001887 uint32_t constituents_length;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001888 enum ffa_data_access data_access;
1889 enum ffa_instruction_access instruction_access;
Olivier Deprez4342a3c2022-02-28 09:37:25 +01001890 enum ffa_memory_security security_state;
Karl Meakinf98b2aa2023-10-12 16:09:59 +01001891 enum ffa_memory_type type;
Federico Recanatia98603a2021-12-20 18:04:03 +01001892 struct ffa_value ret;
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001893 const size_t minimum_first_fragment_length =
Daniel Boulby41ef8ba2023-10-13 17:01:22 +01001894 memory_region->receivers_offset +
1895 memory_region->memory_access_desc_size +
1896 sizeof(struct ffa_composite_memory_region);
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001897
1898 if (fragment_length < minimum_first_fragment_length) {
Karl Meakine8937d92024-03-19 16:04:25 +00001899 dlog_verbose("Fragment length %u too short (min %zu).\n",
1900 fragment_length, minimum_first_fragment_length);
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001901 return ffa_error(FFA_INVALID_PARAMETERS);
1902 }
1903
Demi Marie Obenour73a1e942023-02-04 14:09:18 -05001904 static_assert(sizeof(struct ffa_memory_region_constituent) == 16,
1905 "struct ffa_memory_region_constituent must be 16 bytes");
1906 if (!is_aligned(fragment_length,
1907 sizeof(struct ffa_memory_region_constituent)) ||
1908 !is_aligned(memory_share_length,
1909 sizeof(struct ffa_memory_region_constituent))) {
1910 dlog_verbose(
1911 "Fragment length %u or total length %u"
1912 " is not 16-byte aligned.\n",
1913 fragment_length, memory_share_length);
1914 return ffa_error(FFA_INVALID_PARAMETERS);
1915 }
1916
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001917 if (fragment_length > memory_share_length) {
1918 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00001919 "Fragment length %zu greater than total length %zu.\n",
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001920 (size_t)fragment_length, (size_t)memory_share_length);
1921 return ffa_error(FFA_INVALID_PARAMETERS);
1922 }
Andrew Walbrana65a1322020-04-06 19:32:32 +01001923
J-Alves95df0ef2022-12-07 10:09:48 +00001924 /* The sender must match the caller. */
1925 if ((!vm_id_is_current_world(from_locked.vm->id) &&
1926 vm_id_is_current_world(memory_region->sender)) ||
1927 (vm_id_is_current_world(from_locked.vm->id) &&
1928 memory_region->sender != from_locked.vm->id)) {
1929 dlog_verbose("Invalid memory sender ID.\n");
1930 return ffa_error(FFA_DENIED);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001931 }
1932
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001933 if (memory_region->receiver_count <= 0) {
1934 dlog_verbose("No receivers!\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001935 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001936 }
1937
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001938 /*
1939 * Ensure that the composite header is within the memory bounds and
1940 * doesn't overlap the first part of the message. Cast to uint64_t
1941 * to prevent overflow.
1942 */
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001943 receivers_end = ((uint64_t)memory_region->memory_access_desc_size *
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001944 (uint64_t)memory_region->receiver_count) +
Daniel Boulby41ef8ba2023-10-13 17:01:22 +01001945 memory_region->receivers_offset;
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001946 min_length = receivers_end +
1947 sizeof(struct ffa_composite_memory_region) +
1948 sizeof(struct ffa_memory_region_constituent);
1949 if (min_length > memory_share_length) {
Karl Meakine8937d92024-03-19 16:04:25 +00001950 dlog_verbose("Share too short: got %zu but minimum is %zu.\n",
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001951 (size_t)memory_share_length, (size_t)min_length);
1952 return ffa_error(FFA_INVALID_PARAMETERS);
1953 }
1954
1955 composite_memory_region_offset =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001956 receiver->composite_memory_region_offset;
Andrew Walbrana65a1322020-04-06 19:32:32 +01001957
1958 /*
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001959 * Check that the composite memory region descriptor is after the access
1960 * descriptors, is at least 16-byte aligned, and fits in the first
1961 * fragment.
Andrew Walbrana65a1322020-04-06 19:32:32 +01001962 */
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001963 if ((composite_memory_region_offset < receivers_end) ||
1964 (composite_memory_region_offset % 16 != 0) ||
1965 (composite_memory_region_offset >
1966 fragment_length - sizeof(struct ffa_composite_memory_region))) {
1967 dlog_verbose(
1968 "Invalid composite memory region descriptor offset "
Karl Meakine8937d92024-03-19 16:04:25 +00001969 "%zu.\n",
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001970 (size_t)composite_memory_region_offset);
1971 return ffa_error(FFA_INVALID_PARAMETERS);
1972 }
1973
1974 /*
1975 * Compute the start of the constituent regions. Already checked
1976 * to be not more than fragment_length and thus not more than
1977 * memory_share_length.
1978 */
1979 constituents_start = composite_memory_region_offset +
1980 sizeof(struct ffa_composite_memory_region);
1981 constituents_length = memory_share_length - constituents_start;
1982
1983 /*
1984 * Check that the number of constituents is consistent with the length
1985 * of the constituent region.
1986 */
1987 composite = ffa_memory_region_get_composite(memory_region, 0);
1988 if ((constituents_length %
1989 sizeof(struct ffa_memory_region_constituent) !=
1990 0) ||
1991 ((constituents_length /
1992 sizeof(struct ffa_memory_region_constituent)) !=
1993 composite->constituent_count)) {
Karl Meakine8937d92024-03-19 16:04:25 +00001994 dlog_verbose("Invalid length %zu or composite offset %zu.\n",
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001995 (size_t)memory_share_length,
1996 (size_t)composite_memory_region_offset);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001997 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001998 }
Andrew Walbranca808b12020-05-15 17:22:28 +01001999 if (fragment_length < memory_share_length &&
2000 fragment_length < HF_MAILBOX_SIZE) {
2001 dlog_warning(
2002 "Initial fragment length %d smaller than mailbox "
2003 "size.\n",
2004 fragment_length);
2005 }
Andrew Walbrana65a1322020-04-06 19:32:32 +01002006
Andrew Walbrana65a1322020-04-06 19:32:32 +01002007 /*
2008 * Clear is not allowed for memory sharing, as the sender still has
2009 * access to the memory.
2010 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002011 if ((memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR) &&
J-Alves95fbb312024-03-20 15:19:16 +00002012 (share_func == FFA_MEM_SHARE_32 ||
2013 share_func == FFA_MEM_SHARE_64)) {
Andrew Walbrana65a1322020-04-06 19:32:32 +01002014 dlog_verbose("Memory can't be cleared while being shared.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002015 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01002016 }
2017
2018 /* No other flags are allowed/supported here. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002019 if (memory_region->flags & ~FFA_MEMORY_REGION_FLAG_CLEAR) {
Andrew Walbrana65a1322020-04-06 19:32:32 +01002020 dlog_verbose("Invalid flags %#x.\n", memory_region->flags);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002021 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01002022 }
2023
J-Alves363f5722022-04-25 17:37:37 +01002024 /* Check that the permissions are valid, for each specified receiver. */
2025 for (uint32_t i = 0U; i < memory_region->receiver_count; i++) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002026 struct ffa_memory_region_attributes receiver_permissions;
2027
2028 receiver = ffa_memory_region_get_receiver(memory_region, i);
2029 assert(receiver != NULL);
2030 receiver_permissions = receiver->receiver_permissions;
J-Alves363f5722022-04-25 17:37:37 +01002031 ffa_memory_access_permissions_t permissions =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002032 receiver_permissions.permissions;
2033 ffa_id_t receiver_id = receiver_permissions.receiver;
J-Alves363f5722022-04-25 17:37:37 +01002034
2035 if (memory_region->sender == receiver_id) {
2036 dlog_verbose("Can't share memory with itself.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002037 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01002038 }
Federico Recanati85090c42021-12-15 13:17:54 +01002039
J-Alves363f5722022-04-25 17:37:37 +01002040 for (uint32_t j = i + 1; j < memory_region->receiver_count;
2041 j++) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002042 struct ffa_memory_access *other_receiver =
2043 ffa_memory_region_get_receiver(memory_region,
2044 j);
2045 assert(other_receiver != NULL);
2046
J-Alves363f5722022-04-25 17:37:37 +01002047 if (receiver_id ==
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002048 other_receiver->receiver_permissions.receiver) {
J-Alves363f5722022-04-25 17:37:37 +01002049 dlog_verbose(
2050 "Repeated receiver(%x) in memory send "
2051 "operation.\n",
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002052 other_receiver->receiver_permissions
2053 .receiver);
J-Alves363f5722022-04-25 17:37:37 +01002054 return ffa_error(FFA_INVALID_PARAMETERS);
2055 }
2056 }
2057
2058 if (composite_memory_region_offset !=
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002059 receiver->composite_memory_region_offset) {
J-Alves363f5722022-04-25 17:37:37 +01002060 dlog_verbose(
2061 "All ffa_memory_access should point to the "
2062 "same composite memory region offset.\n");
2063 return ffa_error(FFA_INVALID_PARAMETERS);
2064 }
2065
Karl Meakin84710f32023-10-12 15:14:49 +01002066 data_access = permissions.data_access;
2067 instruction_access = permissions.instruction_access;
J-Alves363f5722022-04-25 17:37:37 +01002068 if (data_access == FFA_DATA_ACCESS_RESERVED ||
2069 instruction_access == FFA_INSTRUCTION_ACCESS_RESERVED) {
2070 dlog_verbose(
2071 "Reserved value for receiver permissions "
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002072 "(data_access = %s, instruction_access = %s)\n",
2073 ffa_data_access_name(data_access),
2074 ffa_instruction_access_name(
2075 instruction_access));
J-Alves363f5722022-04-25 17:37:37 +01002076 return ffa_error(FFA_INVALID_PARAMETERS);
2077 }
2078 if (instruction_access !=
2079 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
2080 dlog_verbose(
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002081 "Invalid instruction access permissions %s "
2082 "for sending memory, expected %s.\n",
2083 ffa_instruction_access_name(instruction_access),
2084 ffa_instruction_access_name(
2085 FFA_INSTRUCTION_ACCESS_RESERVED));
J-Alves363f5722022-04-25 17:37:37 +01002086 return ffa_error(FFA_INVALID_PARAMETERS);
2087 }
J-Alves95fbb312024-03-20 15:19:16 +00002088 if (share_func == FFA_MEM_SHARE_32 ||
2089 share_func == FFA_MEM_SHARE_64) {
J-Alves363f5722022-04-25 17:37:37 +01002090 if (data_access == FFA_DATA_ACCESS_NOT_SPECIFIED) {
2091 dlog_verbose(
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002092 "Invalid data access permissions %s "
2093 "for sharing memory, expected %s.\n",
2094 ffa_data_access_name(data_access),
2095 ffa_data_access_name(
2096 FFA_DATA_ACCESS_NOT_SPECIFIED));
J-Alves363f5722022-04-25 17:37:37 +01002097 return ffa_error(FFA_INVALID_PARAMETERS);
2098 }
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002099 /*
2100 * According to section 10.10.3 of the FF-A v1.1 EAC0
2101 * spec, NX is required for share operations (but must
2102 * not be specified by the sender) so set it in the
2103 * copy that we store, ready to be returned to the
2104 * retriever.
2105 */
2106 if (vm_id_is_current_world(receiver_id)) {
Karl Meakin84710f32023-10-12 15:14:49 +01002107 permissions.instruction_access =
2108 FFA_INSTRUCTION_ACCESS_NX;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002109 receiver_permissions.permissions = permissions;
2110 }
J-Alves363f5722022-04-25 17:37:37 +01002111 }
J-Alves95fbb312024-03-20 15:19:16 +00002112 if ((share_func == FFA_MEM_LEND_32 ||
2113 share_func == FFA_MEM_LEND_64) &&
J-Alves363f5722022-04-25 17:37:37 +01002114 data_access == FFA_DATA_ACCESS_NOT_SPECIFIED) {
2115 dlog_verbose(
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002116 "Invalid data access permissions %s for "
2117 "lending memory, expected %s.\n",
2118 ffa_data_access_name(data_access),
2119 ffa_data_access_name(
2120 FFA_DATA_ACCESS_NOT_SPECIFIED));
J-Alves363f5722022-04-25 17:37:37 +01002121 return ffa_error(FFA_INVALID_PARAMETERS);
2122 }
2123
J-Alves95fbb312024-03-20 15:19:16 +00002124 if ((share_func == FFA_MEM_DONATE_32 ||
2125 share_func == FFA_MEM_DONATE_64) &&
J-Alves363f5722022-04-25 17:37:37 +01002126 data_access != FFA_DATA_ACCESS_NOT_SPECIFIED) {
2127 dlog_verbose(
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002128 "Invalid data access permissions %s for "
2129 "donating memory, expected %s.\n",
2130 ffa_data_access_name(data_access),
2131 ffa_data_access_name(
2132 FFA_DATA_ACCESS_NOT_SPECIFIED));
J-Alves363f5722022-04-25 17:37:37 +01002133 return ffa_error(FFA_INVALID_PARAMETERS);
2134 }
Andrew Walbrana65a1322020-04-06 19:32:32 +01002135 }
2136
Olivier Deprez4342a3c2022-02-28 09:37:25 +01002137 /* Memory region attributes NS-Bit MBZ for FFA_MEM_SHARE/LEND/DONATE. */
Karl Meakin84710f32023-10-12 15:14:49 +01002138 security_state = memory_region->attributes.security;
Olivier Deprez4342a3c2022-02-28 09:37:25 +01002139 if (security_state != FFA_MEMORY_SECURITY_UNSPECIFIED) {
2140 dlog_verbose(
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002141 "Invalid security state %s for memory share operation, "
2142 "expected %s.\n",
2143 ffa_memory_security_name(security_state),
2144 ffa_memory_security_name(
2145 FFA_MEMORY_SECURITY_UNSPECIFIED));
Olivier Deprez4342a3c2022-02-28 09:37:25 +01002146 return ffa_error(FFA_INVALID_PARAMETERS);
2147 }
2148
Federico Recanatid937f5e2021-12-20 17:38:23 +01002149 /*
J-Alves807794e2022-06-16 13:42:47 +01002150 * If a memory donate or lend with single borrower, the memory type
2151 * shall not be specified by the sender.
Federico Recanatid937f5e2021-12-20 17:38:23 +01002152 */
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002153 type = memory_region->attributes.type;
J-Alves807794e2022-06-16 13:42:47 +01002154 if (share_func == FFA_MEM_DONATE_32 ||
J-Alves95fbb312024-03-20 15:19:16 +00002155 share_func == FFA_MEM_DONATE_64 ||
2156 ((share_func == FFA_MEM_LEND_32 || share_func == FFA_MEM_LEND_64) &&
J-Alves807794e2022-06-16 13:42:47 +01002157 memory_region->receiver_count == 1)) {
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002158 if (type != FFA_MEMORY_NOT_SPECIFIED_MEM) {
J-Alves807794e2022-06-16 13:42:47 +01002159 dlog_verbose(
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002160 "Invalid memory type %s for memory share "
2161 "operation, expected %s.\n",
2162 ffa_memory_type_name(type),
2163 ffa_memory_type_name(
2164 FFA_MEMORY_NOT_SPECIFIED_MEM));
J-Alves807794e2022-06-16 13:42:47 +01002165 return ffa_error(FFA_INVALID_PARAMETERS);
2166 }
2167 } else {
2168 /*
2169 * Check that sender's memory attributes match Hafnium
2170 * expectations: Normal Memory, Inner shareable, Write-Back
2171 * Read-Allocate Write-Allocate Cacheable.
2172 */
2173 ret = ffa_memory_attributes_validate(memory_region->attributes);
2174 if (ret.func != FFA_SUCCESS_32) {
2175 return ret;
2176 }
Federico Recanatid937f5e2021-12-20 17:38:23 +01002177 }
2178
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002179 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbrana65a1322020-04-06 19:32:32 +01002180}
2181
2182/**
Andrew Walbranca808b12020-05-15 17:22:28 +01002183 * Gets the share state for continuing an operation to donate, lend or share
2184 * memory, and checks that it is a valid request.
2185 *
2186 * Returns FFA_SUCCESS if the request was valid, or the relevant FFA_ERROR if
2187 * not.
2188 */
J-Alvesfdd29272022-07-19 13:16:31 +01002189struct ffa_value ffa_memory_send_continue_validate(
Andrew Walbranca808b12020-05-15 17:22:28 +01002190 struct share_states_locked share_states, ffa_memory_handle_t handle,
J-Alves19e20cf2023-08-02 12:48:55 +01002191 struct ffa_memory_share_state **share_state_ret, ffa_id_t from_vm_id,
Andrew Walbranca808b12020-05-15 17:22:28 +01002192 struct mpool *page_pool)
2193{
2194 struct ffa_memory_share_state *share_state;
2195 struct ffa_memory_region *memory_region;
2196
Daniel Boulbya2f8c662021-11-26 17:52:53 +00002197 assert(share_state_ret != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +01002198
2199 /*
2200 * Look up the share state by handle and make sure that the VM ID
2201 * matches.
2202 */
Karl Meakin4a2854a2023-06-30 16:26:52 +01002203 share_state = get_share_state(share_states, handle);
J-Alvesb56aac82023-11-10 09:44:43 +00002204 if (share_state == NULL) {
Andrew Walbranca808b12020-05-15 17:22:28 +01002205 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00002206 "Invalid handle %#lx for memory send continuation.\n",
Andrew Walbranca808b12020-05-15 17:22:28 +01002207 handle);
2208 return ffa_error(FFA_INVALID_PARAMETERS);
2209 }
2210 memory_region = share_state->memory_region;
2211
J-Alvesfdd29272022-07-19 13:16:31 +01002212 if (vm_id_is_current_world(from_vm_id) &&
2213 memory_region->sender != from_vm_id) {
Andrew Walbranca808b12020-05-15 17:22:28 +01002214 dlog_verbose("Invalid sender %d.\n", memory_region->sender);
2215 return ffa_error(FFA_INVALID_PARAMETERS);
2216 }
2217
2218 if (share_state->sending_complete) {
2219 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00002220 "Sending of memory handle %#lx is already complete.\n",
Andrew Walbranca808b12020-05-15 17:22:28 +01002221 handle);
2222 return ffa_error(FFA_INVALID_PARAMETERS);
2223 }
2224
2225 if (share_state->fragment_count == MAX_FRAGMENTS) {
2226 /*
2227 * Log a warning as this is a sign that MAX_FRAGMENTS should
2228 * probably be increased.
2229 */
2230 dlog_warning(
Karl Meakine8937d92024-03-19 16:04:25 +00002231 "Too many fragments for memory share with handle %#lx; "
Andrew Walbranca808b12020-05-15 17:22:28 +01002232 "only %d supported.\n",
2233 handle, MAX_FRAGMENTS);
2234 /* Free share state, as it's not possible to complete it. */
2235 share_state_free(share_states, share_state, page_pool);
2236 return ffa_error(FFA_NO_MEMORY);
2237 }
2238
2239 *share_state_ret = share_state;
2240
2241 return (struct ffa_value){.func = FFA_SUCCESS_32};
2242}
2243
2244/**
J-Alves95df0ef2022-12-07 10:09:48 +00002245 * Checks if there is at least one receiver from the other world.
2246 */
J-Alvesfdd29272022-07-19 13:16:31 +01002247bool memory_region_receivers_from_other_world(
J-Alves95df0ef2022-12-07 10:09:48 +00002248 struct ffa_memory_region *memory_region)
2249{
2250 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002251 struct ffa_memory_access *receiver =
2252 ffa_memory_region_get_receiver(memory_region, i);
2253 assert(receiver != NULL);
2254 ffa_id_t receiver_id = receiver->receiver_permissions.receiver;
2255
2256 if (!vm_id_is_current_world(receiver_id)) {
J-Alves95df0ef2022-12-07 10:09:48 +00002257 return true;
2258 }
2259 }
2260 return false;
2261}
2262
2263/**
J-Alves9da280b2022-12-21 14:55:39 +00002264 * Validates a call to donate, lend or share memory in which Hafnium is the
2265 * designated allocator of the memory handle. In practice, this also means
2266 * Hafnium is responsible for managing the state structures for the transaction.
2267 * If Hafnium is the SPMC, it should allocate the memory handle when either the
2268 * sender is an SP or there is at least one borrower that is an SP.
2269 * If Hafnium is the hypervisor, it should allocate the memory handle when
2270 * operation involves only NWd VMs.
2271 *
2272 * If validation goes well, Hafnium updates the stage-2 page tables of the
2273 * sender. Validation consists of checking if the message length and number of
2274 * memory region constituents match, and if the transition is valid for the
2275 * type of memory sending operation.
Andrew Walbran475c1452020-02-07 13:22:22 +00002276 *
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002277 * Assumes that the caller has already found and locked the sender VM and copied
2278 * the memory region descriptor from the sender's TX buffer to a freshly
2279 * allocated page from Hafnium's internal pool. The caller must have also
2280 * validated that the receiver VM ID is valid.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002281 *
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002282 * This function takes ownership of the `memory_region` passed in and will free
2283 * it when necessary; it must not be freed by the caller.
Jose Marinho09b1db82019-08-08 09:16:59 +01002284 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002285struct ffa_value ffa_memory_send(struct vm_locked from_locked,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002286 struct ffa_memory_region *memory_region,
Andrew Walbran130a8ae2020-05-15 16:27:15 +01002287 uint32_t memory_share_length,
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002288 uint32_t fragment_length, uint32_t share_func,
2289 struct mpool *page_pool)
Jose Marinho09b1db82019-08-08 09:16:59 +01002290{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002291 struct ffa_value ret;
Andrew Walbranca808b12020-05-15 17:22:28 +01002292 struct share_states_locked share_states;
2293 struct ffa_memory_share_state *share_state;
Jose Marinho09b1db82019-08-08 09:16:59 +01002294
2295 /*
Andrew Walbrana65a1322020-04-06 19:32:32 +01002296 * If there is an error validating the `memory_region` then we need to
2297 * free it because we own it but we won't be storing it in a share state
2298 * after all.
Jose Marinho09b1db82019-08-08 09:16:59 +01002299 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002300 ret = ffa_memory_send_validate(from_locked, memory_region,
2301 memory_share_length, fragment_length,
J-Alves363f5722022-04-25 17:37:37 +01002302 share_func);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002303 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002304 mpool_free(page_pool, memory_region);
Andrew Walbrana65a1322020-04-06 19:32:32 +01002305 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01002306 }
2307
Andrew Walbrana65a1322020-04-06 19:32:32 +01002308 /* Set flag for share function, ready to be retrieved later. */
2309 switch (share_func) {
J-Alves95fbb312024-03-20 15:19:16 +00002310 case FFA_MEM_SHARE_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002311 case FFA_MEM_SHARE_32:
Andrew Walbrana65a1322020-04-06 19:32:32 +01002312 memory_region->flags |=
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002313 FFA_MEMORY_REGION_TRANSACTION_TYPE_SHARE;
Andrew Walbrana65a1322020-04-06 19:32:32 +01002314 break;
J-Alves95fbb312024-03-20 15:19:16 +00002315 case FFA_MEM_LEND_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002316 case FFA_MEM_LEND_32:
2317 memory_region->flags |= FFA_MEMORY_REGION_TRANSACTION_TYPE_LEND;
Andrew Walbrana65a1322020-04-06 19:32:32 +01002318 break;
J-Alves95fbb312024-03-20 15:19:16 +00002319 case FFA_MEM_DONATE_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002320 case FFA_MEM_DONATE_32:
Andrew Walbrana65a1322020-04-06 19:32:32 +01002321 memory_region->flags |=
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002322 FFA_MEMORY_REGION_TRANSACTION_TYPE_DONATE;
Andrew Walbrana65a1322020-04-06 19:32:32 +01002323 break;
Jose Marinho09b1db82019-08-08 09:16:59 +01002324 }
2325
Andrew Walbranca808b12020-05-15 17:22:28 +01002326 share_states = share_states_lock();
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002327 /*
2328 * Allocate a share state before updating the page table. Otherwise if
2329 * updating the page table succeeded but allocating the share state
2330 * failed then it would leave the memory in a state where nobody could
2331 * get it back.
2332 */
Karl Meakin52cdfe72023-06-30 14:49:10 +01002333 share_state = allocate_share_state(share_states, share_func,
2334 memory_region, fragment_length,
2335 FFA_MEMORY_HANDLE_INVALID);
J-Alvesb56aac82023-11-10 09:44:43 +00002336 if (share_state == NULL) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002337 dlog_verbose("Failed to allocate share state.\n");
2338 mpool_free(page_pool, memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +01002339 ret = ffa_error(FFA_NO_MEMORY);
2340 goto out;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002341 }
2342
Andrew Walbranca808b12020-05-15 17:22:28 +01002343 if (fragment_length == memory_share_length) {
2344 /* No more fragments to come, everything fit in one message. */
J-Alves2a0d2882020-10-29 14:49:50 +00002345 ret = ffa_memory_send_complete(
2346 from_locked, share_states, share_state, page_pool,
2347 &(share_state->sender_orig_mode));
Andrew Walbranca808b12020-05-15 17:22:28 +01002348 } else {
J-Alvesfdd29272022-07-19 13:16:31 +01002349 /*
2350 * Use sender ID from 'memory_region' assuming
2351 * that at this point it has been validated:
2352 * - MBZ at virtual FF-A instance.
2353 */
J-Alves19e20cf2023-08-02 12:48:55 +01002354 ffa_id_t sender_to_ret =
J-Alvesfdd29272022-07-19 13:16:31 +01002355 (from_locked.vm->id == HF_OTHER_WORLD_ID)
2356 ? memory_region->sender
2357 : 0;
Andrew Walbranca808b12020-05-15 17:22:28 +01002358 ret = (struct ffa_value){
2359 .func = FFA_MEM_FRAG_RX_32,
J-Alvesee68c542020-10-29 17:48:20 +00002360 .arg1 = (uint32_t)memory_region->handle,
2361 .arg2 = (uint32_t)(memory_region->handle >> 32),
J-Alvesfdd29272022-07-19 13:16:31 +01002362 .arg3 = fragment_length,
2363 .arg4 = (uint32_t)(sender_to_ret & 0xffff) << 16};
Andrew Walbranca808b12020-05-15 17:22:28 +01002364 }
2365
2366out:
2367 share_states_unlock(&share_states);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002368 dump_share_states();
Andrew Walbranca808b12020-05-15 17:22:28 +01002369 return ret;
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002370}
2371
2372/**
J-Alves8505a8a2022-06-15 18:10:18 +01002373 * Continues an operation to donate, lend or share memory to a VM from current
2374 * world. If this is the last fragment then checks that the transition is valid
2375 * for the type of memory sending operation and updates the stage-2 page tables
2376 * of the sender.
Andrew Walbranca808b12020-05-15 17:22:28 +01002377 *
2378 * Assumes that the caller has already found and locked the sender VM and copied
2379 * the memory region descriptor from the sender's TX buffer to a freshly
2380 * allocated page from Hafnium's internal pool.
2381 *
2382 * This function takes ownership of the `fragment` passed in; it must not be
2383 * freed by the caller.
2384 */
2385struct ffa_value ffa_memory_send_continue(struct vm_locked from_locked,
2386 void *fragment,
2387 uint32_t fragment_length,
2388 ffa_memory_handle_t handle,
2389 struct mpool *page_pool)
2390{
2391 struct share_states_locked share_states = share_states_lock();
2392 struct ffa_memory_share_state *share_state;
2393 struct ffa_value ret;
2394 struct ffa_memory_region *memory_region;
2395
Demi Marie Obenour73a1e942023-02-04 14:09:18 -05002396 CHECK(is_aligned(fragment,
2397 alignof(struct ffa_memory_region_constituent)));
2398 if (fragment_length % sizeof(struct ffa_memory_region_constituent) !=
2399 0) {
2400 dlog_verbose("Fragment length %u misaligned.\n",
2401 fragment_length);
2402 ret = ffa_error(FFA_INVALID_PARAMETERS);
2403 goto out_free_fragment;
2404 }
2405
Andrew Walbranca808b12020-05-15 17:22:28 +01002406 ret = ffa_memory_send_continue_validate(share_states, handle,
2407 &share_state,
2408 from_locked.vm->id, page_pool);
2409 if (ret.func != FFA_SUCCESS_32) {
2410 goto out_free_fragment;
2411 }
2412 memory_region = share_state->memory_region;
2413
J-Alves95df0ef2022-12-07 10:09:48 +00002414 if (memory_region_receivers_from_other_world(memory_region)) {
Andrew Walbranca808b12020-05-15 17:22:28 +01002415 dlog_error(
2416 "Got hypervisor-allocated handle for memory send to "
J-Alves8505a8a2022-06-15 18:10:18 +01002417 "other world. This should never happen, and indicates "
2418 "a bug in "
Andrew Walbranca808b12020-05-15 17:22:28 +01002419 "EL3 code.\n");
2420 ret = ffa_error(FFA_INVALID_PARAMETERS);
2421 goto out_free_fragment;
2422 }
2423
2424 /* Add this fragment. */
2425 share_state->fragments[share_state->fragment_count] = fragment;
2426 share_state->fragment_constituent_counts[share_state->fragment_count] =
2427 fragment_length / sizeof(struct ffa_memory_region_constituent);
2428 share_state->fragment_count++;
2429
2430 /* Check whether the memory send operation is now ready to complete. */
2431 if (share_state_sending_complete(share_states, share_state)) {
J-Alves2a0d2882020-10-29 14:49:50 +00002432 ret = ffa_memory_send_complete(
2433 from_locked, share_states, share_state, page_pool,
2434 &(share_state->sender_orig_mode));
Andrew Walbranca808b12020-05-15 17:22:28 +01002435 } else {
2436 ret = (struct ffa_value){
2437 .func = FFA_MEM_FRAG_RX_32,
2438 .arg1 = (uint32_t)handle,
2439 .arg2 = (uint32_t)(handle >> 32),
2440 .arg3 = share_state_next_fragment_offset(share_states,
2441 share_state)};
2442 }
2443 goto out;
2444
2445out_free_fragment:
2446 mpool_free(page_pool, fragment);
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002447
2448out:
Andrew Walbranca808b12020-05-15 17:22:28 +01002449 share_states_unlock(&share_states);
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002450 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002451}
2452
Andrew Walbranca808b12020-05-15 17:22:28 +01002453/** Clean up after the receiver has finished retrieving a memory region. */
2454static void ffa_memory_retrieve_complete(
2455 struct share_states_locked share_states,
2456 struct ffa_memory_share_state *share_state, struct mpool *page_pool)
2457{
J-Alves95fbb312024-03-20 15:19:16 +00002458 if (share_state->share_func == FFA_MEM_DONATE_32 ||
2459 share_state->share_func == FFA_MEM_DONATE_64) {
Andrew Walbranca808b12020-05-15 17:22:28 +01002460 /*
2461 * Memory that has been donated can't be relinquished,
2462 * so no need to keep the share state around.
2463 */
2464 share_state_free(share_states, share_state, page_pool);
2465 dlog_verbose("Freed share state for donate.\n");
2466 }
2467}
2468
J-Alves2d8457f2022-10-05 11:06:41 +01002469/**
2470 * Initialises the given memory region descriptor to be used for an
2471 * `FFA_MEM_RETRIEVE_RESP`, including the given constituents for the first
2472 * fragment.
2473 * The memory region descriptor is initialized according to retriever's
2474 * FF-A version.
2475 *
2476 * Returns true on success, or false if the given constituents won't all fit in
2477 * the first fragment.
2478 */
2479static bool ffa_retrieved_memory_region_init(
2480 void *response, uint32_t ffa_version, size_t response_max_size,
J-Alves19e20cf2023-08-02 12:48:55 +01002481 ffa_id_t sender, ffa_memory_attributes_t attributes,
J-Alves2d8457f2022-10-05 11:06:41 +01002482 ffa_memory_region_flags_t flags, ffa_memory_handle_t handle,
Daniel Boulbyde974ca2023-12-12 13:53:31 +00002483 ffa_memory_access_permissions_t permissions,
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002484 struct ffa_memory_access *receivers, size_t receiver_count,
2485 uint32_t memory_access_desc_size, uint32_t page_count,
2486 uint32_t total_constituent_count,
J-Alves2d8457f2022-10-05 11:06:41 +01002487 const struct ffa_memory_region_constituent constituents[],
2488 uint32_t fragment_constituent_count, uint32_t *total_length,
2489 uint32_t *fragment_length)
2490{
2491 struct ffa_composite_memory_region *composite_memory_region;
J-Alves2d8457f2022-10-05 11:06:41 +01002492 uint32_t i;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002493 uint32_t composite_offset;
J-Alves2d8457f2022-10-05 11:06:41 +01002494 uint32_t constituents_offset;
J-Alves2d8457f2022-10-05 11:06:41 +01002495
2496 assert(response != NULL);
2497
2498 if (ffa_version == MAKE_FFA_VERSION(1, 0)) {
2499 struct ffa_memory_region_v1_0 *retrieve_response =
2500 (struct ffa_memory_region_v1_0 *)response;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002501 struct ffa_memory_access_v1_0 *receiver;
J-Alves2d8457f2022-10-05 11:06:41 +01002502
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002503 ffa_memory_region_init_header_v1_0(retrieve_response, sender,
2504 attributes, flags, handle, 0,
2505 receiver_count);
J-Alves2d8457f2022-10-05 11:06:41 +01002506
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002507 receiver = (struct ffa_memory_access_v1_0 *)
2508 retrieve_response->receivers;
J-Alves2d8457f2022-10-05 11:06:41 +01002509 receiver_count = retrieve_response->receiver_count;
2510
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002511 for (uint32_t i = 0; i < receiver_count; i++) {
2512 ffa_id_t receiver_id =
2513 receivers[i].receiver_permissions.receiver;
2514 ffa_memory_receiver_flags_t recv_flags =
2515 receivers[i].receiver_permissions.flags;
2516
2517 /*
2518 * Initialized here as in memory retrieve responses we
2519 * currently expect one borrower to be specified.
2520 */
2521 ffa_memory_access_init_v1_0(
Karl Meakin84710f32023-10-12 15:14:49 +01002522 receiver, receiver_id, permissions.data_access,
2523 permissions.instruction_access, recv_flags);
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002524 }
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002525
2526 composite_offset =
J-Alves2d8457f2022-10-05 11:06:41 +01002527 sizeof(struct ffa_memory_region_v1_0) +
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002528 receiver_count * sizeof(struct ffa_memory_access_v1_0);
2529 receiver->composite_memory_region_offset = composite_offset;
J-Alves2d8457f2022-10-05 11:06:41 +01002530
2531 composite_memory_region = ffa_memory_region_get_composite_v1_0(
2532 retrieve_response, 0);
2533 } else {
J-Alves2d8457f2022-10-05 11:06:41 +01002534 struct ffa_memory_region *retrieve_response =
2535 (struct ffa_memory_region *)response;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002536 struct ffa_memory_access *retrieve_response_receivers;
J-Alves2d8457f2022-10-05 11:06:41 +01002537
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002538 ffa_memory_region_init_header(
2539 retrieve_response, sender, attributes, flags, handle, 0,
2540 receiver_count, memory_access_desc_size);
J-Alves2d8457f2022-10-05 11:06:41 +01002541
2542 /*
2543 * Note that `sizeof(struct_ffa_memory_region)` and
2544 * `sizeof(struct ffa_memory_access)` must both be multiples of
2545 * 16 (as verified by the asserts in `ffa_memory.c`, so it is
2546 * guaranteed that the offset we calculate here is aligned to a
2547 * 64-bit boundary and so 64-bit values can be copied without
2548 * alignment faults.
2549 */
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002550 composite_offset =
Daniel Boulby41ef8ba2023-10-13 17:01:22 +01002551 retrieve_response->receivers_offset +
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002552 (uint32_t)(receiver_count *
2553 retrieve_response->memory_access_desc_size);
J-Alves2d8457f2022-10-05 11:06:41 +01002554
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002555 retrieve_response_receivers =
2556 ffa_memory_region_get_receiver(retrieve_response, 0);
2557 assert(retrieve_response_receivers != NULL);
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002558
2559 /*
2560 * Initialized here as in memory retrieve responses we currently
2561 * expect one borrower to be specified.
2562 */
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002563 memcpy_s(retrieve_response_receivers,
2564 sizeof(struct ffa_memory_access) * receiver_count,
2565 receivers,
2566 sizeof(struct ffa_memory_access) * receiver_count);
2567
2568 retrieve_response_receivers->composite_memory_region_offset =
2569 composite_offset;
2570
J-Alves2d8457f2022-10-05 11:06:41 +01002571 composite_memory_region =
2572 ffa_memory_region_get_composite(retrieve_response, 0);
2573 }
2574
J-Alves2d8457f2022-10-05 11:06:41 +01002575 assert(composite_memory_region != NULL);
2576
J-Alves2d8457f2022-10-05 11:06:41 +01002577 composite_memory_region->page_count = page_count;
2578 composite_memory_region->constituent_count = total_constituent_count;
2579 composite_memory_region->reserved_0 = 0;
2580
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002581 constituents_offset =
2582 composite_offset + sizeof(struct ffa_composite_memory_region);
J-Alves2d8457f2022-10-05 11:06:41 +01002583 if (constituents_offset +
2584 fragment_constituent_count *
2585 sizeof(struct ffa_memory_region_constituent) >
2586 response_max_size) {
2587 return false;
2588 }
2589
2590 for (i = 0; i < fragment_constituent_count; ++i) {
2591 composite_memory_region->constituents[i] = constituents[i];
2592 }
2593
2594 if (total_length != NULL) {
2595 *total_length =
2596 constituents_offset +
2597 composite_memory_region->constituent_count *
2598 sizeof(struct ffa_memory_region_constituent);
2599 }
2600 if (fragment_length != NULL) {
2601 *fragment_length =
2602 constituents_offset +
2603 fragment_constituent_count *
2604 sizeof(struct ffa_memory_region_constituent);
2605 }
2606
2607 return true;
2608}
2609
J-Alves96de29f2022-04-26 16:05:24 +01002610/**
2611 * Validates the retrieved permissions against those specified by the lender
2612 * of memory share operation. Optionally can help set the permissions to be used
2613 * for the S2 mapping, through the `permissions` argument.
J-Alvesdcad8992023-09-15 14:10:35 +01002614 * Returns FFA_SUCCESS if all the fields are valid. FFA_ERROR, with error code:
2615 * - FFA_INVALID_PARAMETERS -> if the fields have invalid values as per the
2616 * specification for each ABI.
2617 * - FFA_DENIED -> if the permissions specified by the retriever are not
2618 * less permissive than those provided by the sender.
J-Alves96de29f2022-04-26 16:05:24 +01002619 */
J-Alvesdcad8992023-09-15 14:10:35 +01002620static struct ffa_value ffa_memory_retrieve_is_memory_access_valid(
2621 uint32_t share_func, enum ffa_data_access sent_data_access,
J-Alves96de29f2022-04-26 16:05:24 +01002622 enum ffa_data_access requested_data_access,
2623 enum ffa_instruction_access sent_instruction_access,
2624 enum ffa_instruction_access requested_instruction_access,
J-Alvesdcad8992023-09-15 14:10:35 +01002625 ffa_memory_access_permissions_t *permissions, bool multiple_borrowers)
J-Alves96de29f2022-04-26 16:05:24 +01002626{
2627 switch (sent_data_access) {
2628 case FFA_DATA_ACCESS_NOT_SPECIFIED:
2629 case FFA_DATA_ACCESS_RW:
2630 if (requested_data_access == FFA_DATA_ACCESS_NOT_SPECIFIED ||
2631 requested_data_access == FFA_DATA_ACCESS_RW) {
2632 if (permissions != NULL) {
Karl Meakin84710f32023-10-12 15:14:49 +01002633 permissions->data_access = FFA_DATA_ACCESS_RW;
J-Alves96de29f2022-04-26 16:05:24 +01002634 }
2635 break;
2636 }
2637 /* Intentional fall-through. */
2638 case FFA_DATA_ACCESS_RO:
2639 if (requested_data_access == FFA_DATA_ACCESS_NOT_SPECIFIED ||
2640 requested_data_access == FFA_DATA_ACCESS_RO) {
2641 if (permissions != NULL) {
Karl Meakin84710f32023-10-12 15:14:49 +01002642 permissions->data_access = FFA_DATA_ACCESS_RO;
J-Alves96de29f2022-04-26 16:05:24 +01002643 }
2644 break;
2645 }
2646 dlog_verbose(
2647 "Invalid data access requested; sender specified "
2648 "permissions %#x but receiver requested %#x.\n",
2649 sent_data_access, requested_data_access);
J-Alvesdcad8992023-09-15 14:10:35 +01002650 return ffa_error(FFA_DENIED);
J-Alves96de29f2022-04-26 16:05:24 +01002651 case FFA_DATA_ACCESS_RESERVED:
2652 panic("Got unexpected FFA_DATA_ACCESS_RESERVED. Should be "
2653 "checked before this point.");
2654 }
2655
J-Alvesdcad8992023-09-15 14:10:35 +01002656 /*
2657 * For operations with a single borrower, If it is an FFA_MEMORY_LEND
2658 * or FFA_MEMORY_DONATE the retriever should have specifed the
2659 * instruction permissions it wishes to receive.
2660 */
2661 switch (share_func) {
J-Alves95fbb312024-03-20 15:19:16 +00002662 case FFA_MEM_SHARE_64:
J-Alvesdcad8992023-09-15 14:10:35 +01002663 case FFA_MEM_SHARE_32:
2664 if (requested_instruction_access !=
2665 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
2666 dlog_verbose(
2667 "%s: for share instruction permissions must "
2668 "NOT be specified.\n",
2669 __func__);
2670 return ffa_error(FFA_INVALID_PARAMETERS);
2671 }
2672 break;
J-Alves95fbb312024-03-20 15:19:16 +00002673 case FFA_MEM_LEND_64:
J-Alvesdcad8992023-09-15 14:10:35 +01002674 case FFA_MEM_LEND_32:
2675 /*
2676 * For operations with multiple borrowers only permit XN
2677 * permissions, and both Sender and borrower should have used
2678 * FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED.
2679 */
2680 if (multiple_borrowers) {
2681 if (requested_instruction_access !=
2682 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
2683 dlog_verbose(
2684 "%s: lend/share/donate with multiple "
2685 "borrowers "
2686 "instruction permissions must NOT be "
2687 "specified.\n",
2688 __func__);
2689 return ffa_error(FFA_INVALID_PARAMETERS);
2690 }
2691 break;
2692 }
2693 /* Fall through if the operation targets a single borrower. */
J-Alves95fbb312024-03-20 15:19:16 +00002694 case FFA_MEM_DONATE_64:
J-Alvesdcad8992023-09-15 14:10:35 +01002695 case FFA_MEM_DONATE_32:
2696 if (!multiple_borrowers &&
2697 requested_instruction_access ==
2698 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
2699 dlog_verbose(
2700 "%s: for lend/donate with single borrower "
2701 "instruction permissions must be speficified "
2702 "by borrower\n",
2703 __func__);
2704 return ffa_error(FFA_INVALID_PARAMETERS);
2705 }
2706 break;
2707 default:
2708 panic("%s: Wrong func id provided.\n", __func__);
2709 }
2710
J-Alves96de29f2022-04-26 16:05:24 +01002711 switch (sent_instruction_access) {
2712 case FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED:
2713 case FFA_INSTRUCTION_ACCESS_X:
J-Alvesdcad8992023-09-15 14:10:35 +01002714 if (requested_instruction_access == FFA_INSTRUCTION_ACCESS_X) {
J-Alves96de29f2022-04-26 16:05:24 +01002715 if (permissions != NULL) {
Karl Meakin84710f32023-10-12 15:14:49 +01002716 permissions->instruction_access =
2717 FFA_INSTRUCTION_ACCESS_X;
J-Alves96de29f2022-04-26 16:05:24 +01002718 }
2719 break;
2720 }
J-Alvesdcad8992023-09-15 14:10:35 +01002721 /*
2722 * Fall through if requested permissions are less
2723 * permissive than those provided by the sender.
2724 */
J-Alves96de29f2022-04-26 16:05:24 +01002725 case FFA_INSTRUCTION_ACCESS_NX:
2726 if (requested_instruction_access ==
2727 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED ||
2728 requested_instruction_access == FFA_INSTRUCTION_ACCESS_NX) {
2729 if (permissions != NULL) {
Karl Meakin84710f32023-10-12 15:14:49 +01002730 permissions->instruction_access =
2731 FFA_INSTRUCTION_ACCESS_NX;
J-Alves96de29f2022-04-26 16:05:24 +01002732 }
2733 break;
2734 }
2735 dlog_verbose(
2736 "Invalid instruction access requested; sender "
2737 "specified permissions %#x but receiver requested "
2738 "%#x.\n",
2739 sent_instruction_access, requested_instruction_access);
J-Alvesdcad8992023-09-15 14:10:35 +01002740 return ffa_error(FFA_DENIED);
J-Alves96de29f2022-04-26 16:05:24 +01002741 case FFA_INSTRUCTION_ACCESS_RESERVED:
2742 panic("Got unexpected FFA_INSTRUCTION_ACCESS_RESERVED. Should "
2743 "be checked before this point.");
2744 }
2745
J-Alvesdcad8992023-09-15 14:10:35 +01002746 return (struct ffa_value){.func = FFA_SUCCESS_32};
J-Alves96de29f2022-04-26 16:05:24 +01002747}
2748
2749/**
2750 * Validate the receivers' permissions in the retrieve request against those
2751 * specified by the lender.
2752 * In the `permissions` argument returns the permissions to set at S2 for the
2753 * caller to the FFA_MEMORY_RETRIEVE_REQ.
J-Alves3456e032023-07-20 12:20:05 +01002754 * The function looks into the flag to bypass multiple borrower checks:
2755 * - If not set returns FFA_SUCCESS if all specified permissions are valid.
2756 * - If set returns FFA_SUCCESS if the descriptor contains the permissions
2757 * to the caller of FFA_MEM_RETRIEVE_REQ and they are valid. Other permissions
2758 * are ignored, if provided.
J-Alves96de29f2022-04-26 16:05:24 +01002759 */
2760static struct ffa_value ffa_memory_retrieve_validate_memory_access_list(
2761 struct ffa_memory_region *memory_region,
J-Alves19e20cf2023-08-02 12:48:55 +01002762 struct ffa_memory_region *retrieve_request, ffa_id_t to_vm_id,
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002763 ffa_memory_access_permissions_t *permissions,
2764 struct ffa_memory_access **receiver_ret, uint32_t func_id)
J-Alves96de29f2022-04-26 16:05:24 +01002765{
2766 uint32_t retrieve_receiver_index;
J-Alves3456e032023-07-20 12:20:05 +01002767 bool bypass_multi_receiver_check =
2768 (retrieve_request->flags &
2769 FFA_MEMORY_REGION_FLAG_BYPASS_BORROWERS_CHECK) != 0U;
J-Alvesdcad8992023-09-15 14:10:35 +01002770 const uint32_t region_receiver_count = memory_region->receiver_count;
2771 struct ffa_value ret;
J-Alves96de29f2022-04-26 16:05:24 +01002772
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002773 assert(receiver_ret != NULL);
J-Alves96de29f2022-04-26 16:05:24 +01002774 assert(permissions != NULL);
2775
Karl Meakin84710f32023-10-12 15:14:49 +01002776 *permissions = (ffa_memory_access_permissions_t){0};
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002777
J-Alves3456e032023-07-20 12:20:05 +01002778 if (!bypass_multi_receiver_check) {
J-Alvesdcad8992023-09-15 14:10:35 +01002779 if (retrieve_request->receiver_count != region_receiver_count) {
J-Alves3456e032023-07-20 12:20:05 +01002780 dlog_verbose(
2781 "Retrieve request should contain same list of "
2782 "borrowers, as specified by the lender.\n");
2783 return ffa_error(FFA_INVALID_PARAMETERS);
2784 }
2785 } else {
2786 if (retrieve_request->receiver_count != 1) {
2787 dlog_verbose(
2788 "Set bypass multiple borrower check, receiver "
2789 "list must be sized 1 (%x)\n",
2790 memory_region->receiver_count);
2791 return ffa_error(FFA_INVALID_PARAMETERS);
2792 }
J-Alves96de29f2022-04-26 16:05:24 +01002793 }
2794
2795 retrieve_receiver_index = retrieve_request->receiver_count;
2796
J-Alves96de29f2022-04-26 16:05:24 +01002797 for (uint32_t i = 0U; i < retrieve_request->receiver_count; i++) {
2798 ffa_memory_access_permissions_t sent_permissions;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002799 struct ffa_memory_access *retrieve_request_receiver =
2800 ffa_memory_region_get_receiver(retrieve_request, i);
2801 assert(retrieve_request_receiver != NULL);
J-Alves96de29f2022-04-26 16:05:24 +01002802 ffa_memory_access_permissions_t requested_permissions =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002803 retrieve_request_receiver->receiver_permissions
2804 .permissions;
J-Alves19e20cf2023-08-02 12:48:55 +01002805 ffa_id_t current_receiver_id =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002806 retrieve_request_receiver->receiver_permissions
2807 .receiver;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002808 struct ffa_memory_access *receiver;
2809 uint32_t mem_region_receiver_index;
2810 bool permissions_RO;
2811 bool clear_memory_flags;
J-Alves96de29f2022-04-26 16:05:24 +01002812 bool found_to_id = current_receiver_id == to_vm_id;
2813
J-Alves3456e032023-07-20 12:20:05 +01002814 if (bypass_multi_receiver_check && !found_to_id) {
2815 dlog_verbose(
2816 "Bypass multiple borrower check for id %x.\n",
2817 current_receiver_id);
2818 continue;
2819 }
2820
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002821 if (retrieve_request_receiver->composite_memory_region_offset !=
2822 0U) {
2823 dlog_verbose(
2824 "Retriever specified address ranges not "
2825 "supported (got offset %d).\n",
2826 retrieve_request_receiver
2827 ->composite_memory_region_offset);
2828 return ffa_error(FFA_INVALID_PARAMETERS);
2829 }
2830
J-Alves96de29f2022-04-26 16:05:24 +01002831 /*
2832 * Find the current receiver in the transaction descriptor from
2833 * sender.
2834 */
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002835 mem_region_receiver_index =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002836 ffa_memory_region_get_receiver_index(
2837 memory_region, current_receiver_id);
J-Alves96de29f2022-04-26 16:05:24 +01002838
2839 if (mem_region_receiver_index ==
2840 memory_region->receiver_count) {
2841 dlog_verbose("%s: receiver %x not found\n", __func__,
2842 current_receiver_id);
2843 return ffa_error(FFA_DENIED);
2844 }
2845
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002846 receiver = ffa_memory_region_get_receiver(
2847 memory_region, mem_region_receiver_index);
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002848 assert(receiver != NULL);
2849
2850 sent_permissions = receiver->receiver_permissions.permissions;
J-Alves96de29f2022-04-26 16:05:24 +01002851
2852 if (found_to_id) {
2853 retrieve_receiver_index = i;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002854
2855 *receiver_ret = receiver;
J-Alves96de29f2022-04-26 16:05:24 +01002856 }
2857
2858 /*
J-Alvesdcad8992023-09-15 14:10:35 +01002859 * Check if retrieve request memory access list is valid:
2860 * - The retrieve request complies with the specification.
2861 * - Permissions are within those specified by the sender.
J-Alves96de29f2022-04-26 16:05:24 +01002862 */
J-Alvesdcad8992023-09-15 14:10:35 +01002863 ret = ffa_memory_retrieve_is_memory_access_valid(
Karl Meakin84710f32023-10-12 15:14:49 +01002864 func_id, sent_permissions.data_access,
2865 requested_permissions.data_access,
2866 sent_permissions.instruction_access,
2867 requested_permissions.instruction_access,
J-Alvesdcad8992023-09-15 14:10:35 +01002868 found_to_id ? permissions : NULL,
2869 region_receiver_count > 1);
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002870
J-Alvesdcad8992023-09-15 14:10:35 +01002871 if (ret.func != FFA_SUCCESS_32) {
2872 return ret;
J-Alves96de29f2022-04-26 16:05:24 +01002873 }
2874
Karl Meakin84710f32023-10-12 15:14:49 +01002875 permissions_RO =
2876 (permissions->data_access == FFA_DATA_ACCESS_RO);
J-Alvese5262372024-03-27 11:02:03 +00002877 clear_memory_flags =
2878 (retrieve_request->flags &
2879 (FFA_MEMORY_REGION_FLAG_CLEAR |
2880 FFA_MEMORY_REGION_FLAG_CLEAR_RELINQUISH)) != 0U;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002881
J-Alves96de29f2022-04-26 16:05:24 +01002882 /*
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002883 * Can't request PM to clear memory if only provided
2884 * with RO permissions.
J-Alves96de29f2022-04-26 16:05:24 +01002885 */
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002886 if (found_to_id && permissions_RO && clear_memory_flags) {
J-Alves96de29f2022-04-26 16:05:24 +01002887 dlog_verbose(
2888 "Receiver has RO permissions can not request "
2889 "clear.\n");
2890 return ffa_error(FFA_DENIED);
2891 }
Daniel Boulbyde974ca2023-12-12 13:53:31 +00002892
2893 /*
2894 * Check the impdef in the retrieve_request matches the value in
2895 * the original memory send.
2896 */
2897 if (ffa_version_from_memory_access_desc_size(
2898 memory_region->memory_access_desc_size) >=
2899 MAKE_FFA_VERSION(1, 2) &&
2900 ffa_version_from_memory_access_desc_size(
2901 retrieve_request->memory_access_desc_size) >=
2902 MAKE_FFA_VERSION(1, 2)) {
Daniel Boulbyde974ca2023-12-12 13:53:31 +00002903 if (receiver->impdef.val[0] !=
2904 retrieve_request_receiver->impdef.val[0] ||
2905 receiver->impdef.val[1] !=
2906 retrieve_request_receiver->impdef.val[1]) {
2907 dlog_verbose(
2908 "Impdef value in memory send does not "
2909 "match retrieve request value "
Karl Meakine8937d92024-03-19 16:04:25 +00002910 "send value %#lx %#lx retrieve request "
2911 "value %#lx %#lx\n",
Daniel Boulbyde974ca2023-12-12 13:53:31 +00002912 receiver->impdef.val[0],
2913 receiver->impdef.val[1],
2914 retrieve_request_receiver->impdef
2915 .val[0],
2916 retrieve_request_receiver->impdef
2917 .val[1]);
2918 return ffa_error(FFA_INVALID_PARAMETERS);
2919 }
2920 }
J-Alves96de29f2022-04-26 16:05:24 +01002921 }
2922
2923 if (retrieve_receiver_index == retrieve_request->receiver_count) {
2924 dlog_verbose(
2925 "Retrieve request does not contain caller's (%x) "
2926 "permissions\n",
2927 to_vm_id);
2928 return ffa_error(FFA_INVALID_PARAMETERS);
2929 }
2930
2931 return (struct ffa_value){.func = FFA_SUCCESS_32};
2932}
2933
J-Alvesa9cd7e32022-07-01 13:49:33 +01002934/*
2935 * According to section 16.4.3 of FF-A v1.1 EAC0 specification, the hypervisor
2936 * may issue an FFA_MEM_RETRIEVE_REQ to obtain the memory region description
2937 * of a pending memory sharing operation whose allocator is the SPM, for
2938 * validation purposes before forwarding an FFA_MEM_RECLAIM call. In doing so
2939 * the memory region descriptor of the retrieve request must be zeroed with the
2940 * exception of the sender ID and handle.
2941 */
J-Alves4f0d9c12024-01-17 17:23:11 +00002942bool is_ffa_hypervisor_retrieve_request(struct ffa_memory_region *request,
2943 struct vm_locked to_locked)
J-Alvesa9cd7e32022-07-01 13:49:33 +01002944{
2945 return to_locked.vm->id == HF_HYPERVISOR_VM_ID &&
Karl Meakin84710f32023-10-12 15:14:49 +01002946 request->attributes.shareability == 0U &&
2947 request->attributes.cacheability == 0U &&
2948 request->attributes.type == 0U &&
2949 request->attributes.security == 0U && request->flags == 0U &&
J-Alvesa9cd7e32022-07-01 13:49:33 +01002950 request->tag == 0U && request->receiver_count == 0U &&
2951 plat_ffa_memory_handle_allocated_by_current_world(
2952 request->handle);
2953}
2954
2955/*
2956 * Helper to reset count of fragments retrieved by the hypervisor.
2957 */
2958static void ffa_memory_retrieve_complete_from_hyp(
2959 struct ffa_memory_share_state *share_state)
2960{
2961 if (share_state->hypervisor_fragment_count ==
2962 share_state->fragment_count) {
2963 share_state->hypervisor_fragment_count = 0;
2964 }
2965}
2966
J-Alves089004f2022-07-13 14:25:44 +01002967/**
J-Alves4f0d9c12024-01-17 17:23:11 +00002968 * Prepares the return of the ffa_value for the memory retrieve response.
2969 */
2970static struct ffa_value ffa_memory_retrieve_resp(uint32_t total_length,
2971 uint32_t fragment_length)
2972{
2973 return (struct ffa_value){.func = FFA_MEM_RETRIEVE_RESP_32,
2974 .arg1 = total_length,
2975 .arg2 = fragment_length};
2976}
2977
2978/**
J-Alves089004f2022-07-13 14:25:44 +01002979 * Validate that the memory region descriptor provided by the borrower on
2980 * FFA_MEM_RETRIEVE_REQ, against saved memory region provided by lender at the
2981 * memory sharing call.
2982 */
2983static struct ffa_value ffa_memory_retrieve_validate(
J-Alves4f0d9c12024-01-17 17:23:11 +00002984 ffa_id_t to_id, struct ffa_memory_region *retrieve_request,
2985 uint32_t retrieve_request_length,
J-Alves089004f2022-07-13 14:25:44 +01002986 struct ffa_memory_region *memory_region, uint32_t *receiver_index,
2987 uint32_t share_func)
2988{
2989 ffa_memory_region_flags_t transaction_type =
2990 retrieve_request->flags &
2991 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK;
Olivier Deprez4342a3c2022-02-28 09:37:25 +01002992 enum ffa_memory_security security_state;
J-Alves4f0d9c12024-01-17 17:23:11 +00002993 const uint64_t memory_access_desc_size =
2994 retrieve_request->memory_access_desc_size;
2995 const uint32_t expected_retrieve_request_length =
2996 retrieve_request->receivers_offset +
2997 (uint32_t)(retrieve_request->receiver_count *
2998 memory_access_desc_size);
J-Alves089004f2022-07-13 14:25:44 +01002999
3000 assert(retrieve_request != NULL);
3001 assert(memory_region != NULL);
3002 assert(receiver_index != NULL);
J-Alves089004f2022-07-13 14:25:44 +01003003
J-Alves4f0d9c12024-01-17 17:23:11 +00003004 if (retrieve_request_length != expected_retrieve_request_length) {
3005 dlog_verbose(
3006 "Invalid length for FFA_MEM_RETRIEVE_REQ, expected %d "
3007 "but was %d.\n",
3008 expected_retrieve_request_length,
3009 retrieve_request_length);
3010 return ffa_error(FFA_INVALID_PARAMETERS);
3011 }
3012
3013 if (retrieve_request->sender != memory_region->sender) {
3014 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003015 "Memory with handle %#lx not fully sent, can't "
J-Alves4f0d9c12024-01-17 17:23:11 +00003016 "retrieve.\n",
3017 memory_region->handle);
3018 return ffa_error(FFA_DENIED);
3019 }
3020
3021 /*
3022 * The SPMC can only process retrieve requests to memory share
3023 * operations with one borrower from the other world. It can't
3024 * determine the ID of the NWd VM that invoked the retrieve
3025 * request interface call. It relies on the hypervisor to
3026 * validate the caller's ID against that provided in the
3027 * `receivers` list of the retrieve response.
3028 * In case there is only one borrower from the NWd in the
3029 * transaction descriptor, record that in the `receiver_id` for
3030 * later use, and validate in the retrieve request message.
3031 * This limitation is due to the fact SPMC can't determine the
3032 * index in the memory share structures state to update.
3033 */
3034 if (to_id == HF_HYPERVISOR_VM_ID) {
3035 uint32_t other_world_count = 0;
3036
3037 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
3038 struct ffa_memory_access *receiver =
3039 ffa_memory_region_get_receiver(retrieve_request,
3040 0);
3041 assert(receiver != NULL);
3042
3043 to_id = receiver->receiver_permissions.receiver;
3044
3045 if (!vm_id_is_current_world(to_id)) {
3046 other_world_count++;
3047 }
3048 }
3049
3050 if (other_world_count > 1) {
3051 dlog_verbose(
3052 "Support one receiver from the other "
3053 "world.\n");
3054 return ffa_error(FFA_NOT_SUPPORTED);
3055 }
3056 }
J-Alves089004f2022-07-13 14:25:44 +01003057 /*
3058 * Check that the transaction type expected by the receiver is
3059 * correct, if it has been specified.
3060 */
3061 if (transaction_type !=
3062 FFA_MEMORY_REGION_TRANSACTION_TYPE_UNSPECIFIED &&
3063 transaction_type != (memory_region->flags &
3064 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK)) {
3065 dlog_verbose(
3066 "Incorrect transaction type %#x for "
Karl Meakine8937d92024-03-19 16:04:25 +00003067 "FFA_MEM_RETRIEVE_REQ, expected %#x for handle %#lx.\n",
J-Alves089004f2022-07-13 14:25:44 +01003068 transaction_type,
3069 memory_region->flags &
3070 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK,
3071 retrieve_request->handle);
3072 return ffa_error(FFA_INVALID_PARAMETERS);
3073 }
3074
3075 if (retrieve_request->tag != memory_region->tag) {
3076 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003077 "Incorrect tag %lu for FFA_MEM_RETRIEVE_REQ, expected "
3078 "%lu for handle %#lx.\n",
J-Alves089004f2022-07-13 14:25:44 +01003079 retrieve_request->tag, memory_region->tag,
3080 retrieve_request->handle);
3081 return ffa_error(FFA_INVALID_PARAMETERS);
3082 }
3083
J-Alves4f0d9c12024-01-17 17:23:11 +00003084 *receiver_index =
3085 ffa_memory_region_get_receiver_index(memory_region, to_id);
J-Alves089004f2022-07-13 14:25:44 +01003086
3087 if (*receiver_index == memory_region->receiver_count) {
3088 dlog_verbose(
3089 "Incorrect receiver VM ID %d for "
Karl Meakine8937d92024-03-19 16:04:25 +00003090 "FFA_MEM_RETRIEVE_REQ, for handle %#lx.\n",
J-Alves4f0d9c12024-01-17 17:23:11 +00003091 to_id, memory_region->handle);
J-Alves089004f2022-07-13 14:25:44 +01003092 return ffa_error(FFA_INVALID_PARAMETERS);
3093 }
3094
3095 if ((retrieve_request->flags &
3096 FFA_MEMORY_REGION_ADDRESS_RANGE_HINT_VALID) != 0U) {
3097 dlog_verbose(
3098 "Retriever specified 'address range alignment 'hint' "
3099 "not supported.\n");
3100 return ffa_error(FFA_INVALID_PARAMETERS);
3101 }
3102 if ((retrieve_request->flags &
3103 FFA_MEMORY_REGION_ADDRESS_RANGE_HINT_MASK) != 0) {
3104 dlog_verbose(
3105 "Bits 8-5 must be zero in memory region's flags "
3106 "(address range alignment hint not supported).\n");
3107 return ffa_error(FFA_INVALID_PARAMETERS);
3108 }
3109
3110 if ((retrieve_request->flags & ~0x7FF) != 0U) {
3111 dlog_verbose(
3112 "Bits 31-10 must be zero in memory region's flags.\n");
3113 return ffa_error(FFA_INVALID_PARAMETERS);
3114 }
3115
J-Alves95fbb312024-03-20 15:19:16 +00003116 if ((share_func == FFA_MEM_SHARE_32 ||
3117 share_func == FFA_MEM_SHARE_64) &&
J-Alves089004f2022-07-13 14:25:44 +01003118 (retrieve_request->flags &
3119 (FFA_MEMORY_REGION_FLAG_CLEAR |
3120 FFA_MEMORY_REGION_FLAG_CLEAR_RELINQUISH)) != 0U) {
3121 dlog_verbose(
3122 "Memory Share operation can't clean after relinquish "
3123 "memory region.\n");
3124 return ffa_error(FFA_INVALID_PARAMETERS);
3125 }
3126
3127 /*
3128 * If the borrower needs the memory to be cleared before mapping
3129 * to its address space, the sender should have set the flag
3130 * when calling FFA_MEM_LEND/FFA_MEM_DONATE, else return
3131 * FFA_DENIED.
3132 */
3133 if ((retrieve_request->flags & FFA_MEMORY_REGION_FLAG_CLEAR) != 0U &&
3134 (memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR) == 0U) {
3135 dlog_verbose(
3136 "Borrower needs memory cleared. Sender needs to set "
3137 "flag for clearing memory.\n");
3138 return ffa_error(FFA_DENIED);
3139 }
3140
Olivier Deprez4342a3c2022-02-28 09:37:25 +01003141 /* Memory region attributes NS-Bit MBZ for FFA_MEM_RETRIEVE_REQ. */
Karl Meakin84710f32023-10-12 15:14:49 +01003142 security_state = retrieve_request->attributes.security;
Olivier Deprez4342a3c2022-02-28 09:37:25 +01003143 if (security_state != FFA_MEMORY_SECURITY_UNSPECIFIED) {
3144 dlog_verbose(
3145 "Invalid security state for memory retrieve request "
3146 "operation.\n");
3147 return ffa_error(FFA_INVALID_PARAMETERS);
3148 }
3149
J-Alves089004f2022-07-13 14:25:44 +01003150 /*
3151 * If memory type is not specified, bypass validation of memory
3152 * attributes in the retrieve request. The retriever is expecting to
3153 * obtain this information from the SPMC.
3154 */
Karl Meakin84710f32023-10-12 15:14:49 +01003155 if (retrieve_request->attributes.type == FFA_MEMORY_NOT_SPECIFIED_MEM) {
J-Alves089004f2022-07-13 14:25:44 +01003156 return (struct ffa_value){.func = FFA_SUCCESS_32};
3157 }
3158
3159 /*
3160 * Ensure receiver's attributes are compatible with how
3161 * Hafnium maps memory: Normal Memory, Inner shareable,
3162 * Write-Back Read-Allocate Write-Allocate Cacheable.
3163 */
3164 return ffa_memory_attributes_validate(retrieve_request->attributes);
3165}
3166
J-Alves4f0d9c12024-01-17 17:23:11 +00003167static struct ffa_value ffa_partition_retrieve_request(
3168 struct share_states_locked share_states,
3169 struct ffa_memory_share_state *share_state, struct vm_locked to_locked,
3170 struct ffa_memory_region *retrieve_request,
3171 uint32_t retrieve_request_length, struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003172{
Karl Meakin84710f32023-10-12 15:14:49 +01003173 ffa_memory_access_permissions_t permissions = {0};
Olivier Deprez878bd5b2021-04-15 19:05:10 +02003174 uint32_t memory_to_mode;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003175 struct ffa_value ret;
Andrew Walbranca808b12020-05-15 17:22:28 +01003176 struct ffa_composite_memory_region *composite;
3177 uint32_t total_length;
3178 uint32_t fragment_length;
J-Alves19e20cf2023-08-02 12:48:55 +01003179 ffa_id_t receiver_id = to_locked.vm->id;
J-Alves4f0d9c12024-01-17 17:23:11 +00003180 bool is_retrieve_complete = false;
J-Alves4f0d9c12024-01-17 17:23:11 +00003181 const uint64_t memory_access_desc_size =
Daniel Boulbyde974ca2023-12-12 13:53:31 +00003182 retrieve_request->memory_access_desc_size;
J-Alves4f0d9c12024-01-17 17:23:11 +00003183 uint32_t receiver_index;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003184 struct ffa_memory_access *receiver;
J-Alves4f0d9c12024-01-17 17:23:11 +00003185 ffa_memory_handle_t handle = retrieve_request->handle;
Karl Meakin84710f32023-10-12 15:14:49 +01003186 ffa_memory_attributes_t attributes = {0};
J-Alves460d36c2023-10-12 17:02:15 +01003187 uint32_t retrieve_mode = 0;
J-Alves4f0d9c12024-01-17 17:23:11 +00003188 struct ffa_memory_region *memory_region = share_state->memory_region;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003189
J-Alves96de29f2022-04-26 16:05:24 +01003190 if (!share_state->sending_complete) {
3191 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003192 "Memory with handle %#lx not fully sent, can't "
J-Alves96de29f2022-04-26 16:05:24 +01003193 "retrieve.\n",
3194 handle);
J-Alves4f0d9c12024-01-17 17:23:11 +00003195 return ffa_error(FFA_INVALID_PARAMETERS);
J-Alves96de29f2022-04-26 16:05:24 +01003196 }
3197
J-Alves4f0d9c12024-01-17 17:23:11 +00003198 /*
3199 * Validate retrieve request, according to what was sent by the
3200 * sender. Function will output the `receiver_index` from the
3201 * provided memory region.
3202 */
3203 ret = ffa_memory_retrieve_validate(
3204 receiver_id, retrieve_request, retrieve_request_length,
3205 memory_region, &receiver_index, share_state->share_func);
J-Alves089004f2022-07-13 14:25:44 +01003206
J-Alves4f0d9c12024-01-17 17:23:11 +00003207 if (ret.func != FFA_SUCCESS_32) {
3208 return ret;
J-Alves089004f2022-07-13 14:25:44 +01003209 }
J-Alves96de29f2022-04-26 16:05:24 +01003210
J-Alves4f0d9c12024-01-17 17:23:11 +00003211 /*
3212 * Validate the requested permissions against the sent
3213 * permissions.
3214 * Outputs the permissions to give to retriever at S2
3215 * PTs.
3216 */
3217 ret = ffa_memory_retrieve_validate_memory_access_list(
3218 memory_region, retrieve_request, receiver_id, &permissions,
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003219 &receiver, share_state->share_func);
J-Alves4f0d9c12024-01-17 17:23:11 +00003220 if (ret.func != FFA_SUCCESS_32) {
3221 return ret;
3222 }
3223
3224 memory_to_mode = ffa_memory_permissions_to_mode(
3225 permissions, share_state->sender_orig_mode);
3226
3227 ret = ffa_retrieve_check_update(
3228 to_locked, share_state->fragments,
3229 share_state->fragment_constituent_counts,
3230 share_state->fragment_count, memory_to_mode,
J-Alves460d36c2023-10-12 17:02:15 +01003231 share_state->share_func, false, page_pool, &retrieve_mode,
3232 share_state->memory_protected);
J-Alves4f0d9c12024-01-17 17:23:11 +00003233
3234 if (ret.func != FFA_SUCCESS_32) {
3235 return ret;
3236 }
3237
3238 share_state->retrieved_fragment_count[receiver_index] = 1;
3239
3240 is_retrieve_complete =
3241 share_state->retrieved_fragment_count[receiver_index] ==
3242 share_state->fragment_count;
3243
J-Alvesb5084cf2022-07-06 14:20:12 +01003244 /* VMs acquire the RX buffer from SPMC. */
3245 CHECK(plat_ffa_acquire_receiver_rx(to_locked, &ret));
3246
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003247 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01003248 * Copy response to RX buffer of caller and deliver the message.
3249 * This must be done before the share_state is (possibly) freed.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003250 */
Andrew Walbranca808b12020-05-15 17:22:28 +01003251 composite = ffa_memory_region_get_composite(memory_region, 0);
J-Alves4f0d9c12024-01-17 17:23:11 +00003252
Andrew Walbranca808b12020-05-15 17:22:28 +01003253 /*
J-Alves460d36c2023-10-12 17:02:15 +01003254 * Set the security state in the memory retrieve response attributes
3255 * if specified by the target mode.
3256 */
3257 attributes = plat_ffa_memory_security_mode(memory_region->attributes,
3258 retrieve_mode);
3259
3260 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01003261 * Constituents which we received in the first fragment should
3262 * always fit in the first fragment we are sending, because the
3263 * header is the same size in both cases and we have a fixed
3264 * message buffer size. So `ffa_retrieved_memory_region_init`
3265 * should never fail.
Andrew Walbranca808b12020-05-15 17:22:28 +01003266 */
Olivier Deprez878bd5b2021-04-15 19:05:10 +02003267
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003268 /* Provide the permissions that had been provided. */
3269 receiver->receiver_permissions.permissions = permissions;
3270
3271 /*
3272 * Prepare the memory region descriptor for the retrieve response.
3273 * Provide the pointer to the receiver tracked in the share state
3274 * strucutures.
3275 */
Andrew Walbranca808b12020-05-15 17:22:28 +01003276 CHECK(ffa_retrieved_memory_region_init(
J-Alves2d8457f2022-10-05 11:06:41 +01003277 to_locked.vm->mailbox.recv, to_locked.vm->ffa_version,
Olivier Deprez878bd5b2021-04-15 19:05:10 +02003278 HF_MAILBOX_SIZE, memory_region->sender, attributes,
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003279 memory_region->flags, handle, permissions, receiver, 1,
3280 memory_access_desc_size, composite->page_count,
3281 composite->constituent_count, share_state->fragments[0],
Andrew Walbranca808b12020-05-15 17:22:28 +01003282 share_state->fragment_constituent_counts[0], &total_length,
3283 &fragment_length));
J-Alvesb5084cf2022-07-06 14:20:12 +01003284
J-Alves4f0d9c12024-01-17 17:23:11 +00003285 if (is_retrieve_complete) {
Andrew Walbranca808b12020-05-15 17:22:28 +01003286 ffa_memory_retrieve_complete(share_states, share_state,
3287 page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003288 }
J-Alves4f0d9c12024-01-17 17:23:11 +00003289
3290 return ffa_memory_retrieve_resp(total_length, fragment_length);
3291}
3292
3293static struct ffa_value ffa_hypervisor_retrieve_request(
3294 struct ffa_memory_share_state *share_state, struct vm_locked to_locked,
3295 struct ffa_memory_region *retrieve_request)
3296{
3297 struct ffa_value ret;
3298 struct ffa_composite_memory_region *composite;
3299 uint32_t total_length;
3300 uint32_t fragment_length;
J-Alves4f0d9c12024-01-17 17:23:11 +00003301 ffa_memory_attributes_t attributes;
J-Alves7b6ab612024-01-24 09:54:54 +00003302 uint64_t memory_access_desc_size;
J-Alves4f0d9c12024-01-17 17:23:11 +00003303 struct ffa_memory_region *memory_region;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003304 struct ffa_memory_access *receiver;
J-Alves4f0d9c12024-01-17 17:23:11 +00003305 ffa_memory_handle_t handle = retrieve_request->handle;
3306
J-Alves4f0d9c12024-01-17 17:23:11 +00003307 memory_region = share_state->memory_region;
3308
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003309 assert(to_locked.vm->id == HF_HYPERVISOR_VM_ID);
3310
J-Alves7b6ab612024-01-24 09:54:54 +00003311 switch (to_locked.vm->ffa_version) {
3312 case MAKE_FFA_VERSION(1, 2):
3313 memory_access_desc_size = sizeof(struct ffa_memory_access);
3314 break;
3315 case MAKE_FFA_VERSION(1, 0):
3316 case MAKE_FFA_VERSION(1, 1):
3317 memory_access_desc_size = sizeof(struct ffa_memory_access_v1_0);
3318 break;
3319 default:
3320 panic("version not supported: %x\n", to_locked.vm->ffa_version);
3321 }
3322
J-Alves4f0d9c12024-01-17 17:23:11 +00003323 if (share_state->hypervisor_fragment_count != 0U) {
3324 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003325 "Memory with handle %#lx already retrieved by "
J-Alves4f0d9c12024-01-17 17:23:11 +00003326 "the hypervisor.\n",
3327 handle);
3328 return ffa_error(FFA_DENIED);
3329 }
3330
3331 share_state->hypervisor_fragment_count = 1;
3332
3333 ffa_memory_retrieve_complete_from_hyp(share_state);
3334
3335 /* VMs acquire the RX buffer from SPMC. */
3336 CHECK(plat_ffa_acquire_receiver_rx(to_locked, &ret));
3337
3338 /*
3339 * Copy response to RX buffer of caller and deliver the message.
3340 * This must be done before the share_state is (possibly) freed.
3341 */
3342 composite = ffa_memory_region_get_composite(memory_region, 0);
3343
3344 /*
3345 * Constituents which we received in the first fragment should
3346 * always fit in the first fragment we are sending, because the
3347 * header is the same size in both cases and we have a fixed
3348 * message buffer size. So `ffa_retrieved_memory_region_init`
3349 * should never fail.
3350 */
3351
3352 /*
3353 * Set the security state in the memory retrieve response attributes
3354 * if specified by the target mode.
3355 */
3356 attributes = plat_ffa_memory_security_mode(
3357 memory_region->attributes, share_state->sender_orig_mode);
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003358
3359 receiver = ffa_memory_region_get_receiver(memory_region, 0);
3360
J-Alves4f0d9c12024-01-17 17:23:11 +00003361 CHECK(ffa_retrieved_memory_region_init(
3362 to_locked.vm->mailbox.recv, to_locked.vm->ffa_version,
3363 HF_MAILBOX_SIZE, memory_region->sender, attributes,
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003364 memory_region->flags, handle,
3365 receiver->receiver_permissions.permissions, receiver,
3366 memory_region->receiver_count, memory_access_desc_size,
J-Alves4f0d9c12024-01-17 17:23:11 +00003367 composite->page_count, composite->constituent_count,
3368 share_state->fragments[0],
3369 share_state->fragment_constituent_counts[0], &total_length,
3370 &fragment_length));
3371
3372 return ffa_memory_retrieve_resp(total_length, fragment_length);
3373}
3374
3375struct ffa_value ffa_memory_retrieve(struct vm_locked to_locked,
3376 struct ffa_memory_region *retrieve_request,
3377 uint32_t retrieve_request_length,
3378 struct mpool *page_pool)
3379{
3380 ffa_memory_handle_t handle = retrieve_request->handle;
3381 struct share_states_locked share_states;
3382 struct ffa_memory_share_state *share_state;
3383 struct ffa_value ret;
3384
3385 dump_share_states();
3386
3387 share_states = share_states_lock();
3388 share_state = get_share_state(share_states, handle);
3389 if (share_state == NULL) {
Karl Meakine8937d92024-03-19 16:04:25 +00003390 dlog_verbose("Invalid handle %#lx for FFA_MEM_RETRIEVE_REQ.\n",
J-Alves4f0d9c12024-01-17 17:23:11 +00003391 handle);
3392 ret = ffa_error(FFA_INVALID_PARAMETERS);
3393 goto out;
3394 }
3395
3396 if (is_ffa_hypervisor_retrieve_request(retrieve_request, to_locked)) {
3397 ret = ffa_hypervisor_retrieve_request(share_state, to_locked,
3398 retrieve_request);
3399 } else {
3400 ret = ffa_partition_retrieve_request(
3401 share_states, share_state, to_locked, retrieve_request,
3402 retrieve_request_length, page_pool);
3403 }
3404
3405 /* Track use of the RX buffer if the handling has succeeded. */
3406 if (ret.func == FFA_MEM_RETRIEVE_RESP_32) {
3407 to_locked.vm->mailbox.recv_func = FFA_MEM_RETRIEVE_RESP_32;
3408 to_locked.vm->mailbox.state = MAILBOX_STATE_FULL;
3409 }
3410
Andrew Walbranca808b12020-05-15 17:22:28 +01003411out:
3412 share_states_unlock(&share_states);
3413 dump_share_states();
3414 return ret;
3415}
3416
J-Alves5da37d92022-10-24 16:33:48 +01003417/**
3418 * Determine expected fragment offset according to the FF-A version of
3419 * the caller.
3420 */
3421static uint32_t ffa_memory_retrieve_expected_offset_per_ffa_version(
3422 struct ffa_memory_region *memory_region,
3423 uint32_t retrieved_constituents_count, uint32_t ffa_version)
3424{
3425 uint32_t expected_fragment_offset;
3426 uint32_t composite_constituents_offset;
3427
Kathleen Capellae4fe2962023-09-01 17:08:47 -04003428 if (ffa_version >= MAKE_FFA_VERSION(1, 1)) {
J-Alves5da37d92022-10-24 16:33:48 +01003429 /*
3430 * Hafnium operates memory regions in FF-A v1.1 format, so we
3431 * can retrieve the constituents offset from descriptor.
3432 */
3433 composite_constituents_offset =
3434 ffa_composite_constituent_offset(memory_region, 0);
3435 } else if (ffa_version == MAKE_FFA_VERSION(1, 0)) {
3436 /*
3437 * If retriever is FF-A v1.0, determine the composite offset
3438 * as it is expected to have been configured in the
3439 * retrieve response.
3440 */
3441 composite_constituents_offset =
3442 sizeof(struct ffa_memory_region_v1_0) +
3443 RECEIVERS_COUNT_IN_RETRIEVE_RESP *
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003444 sizeof(struct ffa_memory_access_v1_0) +
J-Alves5da37d92022-10-24 16:33:48 +01003445 sizeof(struct ffa_composite_memory_region);
3446 } else {
3447 panic("%s received an invalid FF-A version.\n", __func__);
3448 }
3449
3450 expected_fragment_offset =
3451 composite_constituents_offset +
3452 retrieved_constituents_count *
3453 sizeof(struct ffa_memory_region_constituent) -
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003454 (uint32_t)(memory_region->memory_access_desc_size *
3455 (memory_region->receiver_count - 1));
J-Alves5da37d92022-10-24 16:33:48 +01003456
3457 return expected_fragment_offset;
3458}
3459
Andrew Walbranca808b12020-05-15 17:22:28 +01003460struct ffa_value ffa_memory_retrieve_continue(struct vm_locked to_locked,
3461 ffa_memory_handle_t handle,
3462 uint32_t fragment_offset,
J-Alves19e20cf2023-08-02 12:48:55 +01003463 ffa_id_t sender_vm_id,
Andrew Walbranca808b12020-05-15 17:22:28 +01003464 struct mpool *page_pool)
3465{
3466 struct ffa_memory_region *memory_region;
3467 struct share_states_locked share_states;
3468 struct ffa_memory_share_state *share_state;
3469 struct ffa_value ret;
3470 uint32_t fragment_index;
3471 uint32_t retrieved_constituents_count;
3472 uint32_t i;
3473 uint32_t expected_fragment_offset;
3474 uint32_t remaining_constituent_count;
3475 uint32_t fragment_length;
J-Alvesc7484f12022-05-13 12:41:14 +01003476 uint32_t receiver_index;
J-Alves59ed0042022-07-28 18:26:41 +01003477 bool continue_ffa_hyp_mem_retrieve_req;
Andrew Walbranca808b12020-05-15 17:22:28 +01003478
3479 dump_share_states();
3480
3481 share_states = share_states_lock();
Karl Meakin4a2854a2023-06-30 16:26:52 +01003482 share_state = get_share_state(share_states, handle);
J-Alvesb56aac82023-11-10 09:44:43 +00003483 if (share_state == NULL) {
Karl Meakine8937d92024-03-19 16:04:25 +00003484 dlog_verbose("Invalid handle %#lx for FFA_MEM_FRAG_RX.\n",
Andrew Walbranca808b12020-05-15 17:22:28 +01003485 handle);
3486 ret = ffa_error(FFA_INVALID_PARAMETERS);
3487 goto out;
3488 }
3489
3490 memory_region = share_state->memory_region;
3491 CHECK(memory_region != NULL);
3492
Andrew Walbranca808b12020-05-15 17:22:28 +01003493 if (!share_state->sending_complete) {
3494 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003495 "Memory with handle %#lx not fully sent, can't "
Andrew Walbranca808b12020-05-15 17:22:28 +01003496 "retrieve.\n",
3497 handle);
3498 ret = ffa_error(FFA_INVALID_PARAMETERS);
3499 goto out;
3500 }
3501
J-Alves59ed0042022-07-28 18:26:41 +01003502 /*
3503 * If retrieve request from the hypervisor has been initiated in the
3504 * given share_state, continue it, else assume it is a continuation of
3505 * retrieve request from a NWd VM.
3506 */
3507 continue_ffa_hyp_mem_retrieve_req =
3508 (to_locked.vm->id == HF_HYPERVISOR_VM_ID) &&
3509 (share_state->hypervisor_fragment_count != 0U) &&
J-Alves661e1b72023-08-02 13:39:40 +01003510 ffa_is_vm_id(sender_vm_id);
Andrew Walbranca808b12020-05-15 17:22:28 +01003511
J-Alves59ed0042022-07-28 18:26:41 +01003512 if (!continue_ffa_hyp_mem_retrieve_req) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003513 receiver_index = ffa_memory_region_get_receiver_index(
J-Alves59ed0042022-07-28 18:26:41 +01003514 memory_region, to_locked.vm->id);
3515
3516 if (receiver_index == memory_region->receiver_count) {
3517 dlog_verbose(
3518 "Caller of FFA_MEM_FRAG_RX (%x) is not a "
Karl Meakine8937d92024-03-19 16:04:25 +00003519 "borrower to memory sharing transaction "
3520 "(%lx)\n",
J-Alves59ed0042022-07-28 18:26:41 +01003521 to_locked.vm->id, handle);
3522 ret = ffa_error(FFA_INVALID_PARAMETERS);
3523 goto out;
3524 }
3525
3526 if (share_state->retrieved_fragment_count[receiver_index] ==
3527 0 ||
3528 share_state->retrieved_fragment_count[receiver_index] >=
3529 share_state->fragment_count) {
3530 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003531 "Retrieval of memory with handle %#lx not yet "
J-Alves59ed0042022-07-28 18:26:41 +01003532 "started or already completed (%d/%d fragments "
3533 "retrieved).\n",
3534 handle,
3535 share_state->retrieved_fragment_count
3536 [receiver_index],
3537 share_state->fragment_count);
3538 ret = ffa_error(FFA_INVALID_PARAMETERS);
3539 goto out;
3540 }
3541
3542 fragment_index =
3543 share_state->retrieved_fragment_count[receiver_index];
3544 } else {
3545 if (share_state->hypervisor_fragment_count == 0 ||
3546 share_state->hypervisor_fragment_count >=
3547 share_state->fragment_count) {
3548 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003549 "Retrieve of memory with handle %lx not "
J-Alves59ed0042022-07-28 18:26:41 +01003550 "started from hypervisor.\n",
3551 handle);
3552 ret = ffa_error(FFA_INVALID_PARAMETERS);
3553 goto out;
3554 }
3555
3556 if (memory_region->sender != sender_vm_id) {
3557 dlog_verbose(
3558 "Sender ID (%x) is not as expected for memory "
Karl Meakine8937d92024-03-19 16:04:25 +00003559 "handle %lx\n",
J-Alves59ed0042022-07-28 18:26:41 +01003560 sender_vm_id, handle);
3561 ret = ffa_error(FFA_INVALID_PARAMETERS);
3562 goto out;
3563 }
3564
3565 fragment_index = share_state->hypervisor_fragment_count;
3566
3567 receiver_index = 0;
3568 }
Andrew Walbranca808b12020-05-15 17:22:28 +01003569
3570 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01003571 * Check that the given fragment offset is correct by counting
3572 * how many constituents were in the fragments previously sent.
Andrew Walbranca808b12020-05-15 17:22:28 +01003573 */
3574 retrieved_constituents_count = 0;
3575 for (i = 0; i < fragment_index; ++i) {
3576 retrieved_constituents_count +=
3577 share_state->fragment_constituent_counts[i];
3578 }
J-Alvesc7484f12022-05-13 12:41:14 +01003579
3580 CHECK(memory_region->receiver_count > 0);
3581
Andrew Walbranca808b12020-05-15 17:22:28 +01003582 expected_fragment_offset =
J-Alves5da37d92022-10-24 16:33:48 +01003583 ffa_memory_retrieve_expected_offset_per_ffa_version(
3584 memory_region, retrieved_constituents_count,
3585 to_locked.vm->ffa_version);
3586
Andrew Walbranca808b12020-05-15 17:22:28 +01003587 if (fragment_offset != expected_fragment_offset) {
3588 dlog_verbose("Fragment offset was %d but expected %d.\n",
3589 fragment_offset, expected_fragment_offset);
3590 ret = ffa_error(FFA_INVALID_PARAMETERS);
3591 goto out;
3592 }
3593
J-Alves4f0d9c12024-01-17 17:23:11 +00003594 /*
3595 * When hafnium is the hypervisor, acquire the RX buffer of a VM, that
3596 * is currently ownder by the SPMC.
3597 */
3598 assert(plat_ffa_acquire_receiver_rx(to_locked, &ret));
J-Alves59ed0042022-07-28 18:26:41 +01003599
Andrew Walbranca808b12020-05-15 17:22:28 +01003600 remaining_constituent_count = ffa_memory_fragment_init(
3601 to_locked.vm->mailbox.recv, HF_MAILBOX_SIZE,
3602 share_state->fragments[fragment_index],
3603 share_state->fragment_constituent_counts[fragment_index],
3604 &fragment_length);
3605 CHECK(remaining_constituent_count == 0);
J-Alves674e4de2024-01-17 16:20:32 +00003606
Andrew Walbranca808b12020-05-15 17:22:28 +01003607 to_locked.vm->mailbox.recv_func = FFA_MEM_FRAG_TX_32;
J-Alvese8c8c2b2022-12-16 15:34:48 +00003608 to_locked.vm->mailbox.state = MAILBOX_STATE_FULL;
Andrew Walbranca808b12020-05-15 17:22:28 +01003609
J-Alves59ed0042022-07-28 18:26:41 +01003610 if (!continue_ffa_hyp_mem_retrieve_req) {
3611 share_state->retrieved_fragment_count[receiver_index]++;
3612 if (share_state->retrieved_fragment_count[receiver_index] ==
3613 share_state->fragment_count) {
3614 ffa_memory_retrieve_complete(share_states, share_state,
3615 page_pool);
3616 }
3617 } else {
3618 share_state->hypervisor_fragment_count++;
3619
3620 ffa_memory_retrieve_complete_from_hyp(share_state);
3621 }
Andrew Walbranca808b12020-05-15 17:22:28 +01003622 ret = (struct ffa_value){.func = FFA_MEM_FRAG_TX_32,
3623 .arg1 = (uint32_t)handle,
3624 .arg2 = (uint32_t)(handle >> 32),
3625 .arg3 = fragment_length};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003626
3627out:
3628 share_states_unlock(&share_states);
3629 dump_share_states();
3630 return ret;
3631}
3632
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003633struct ffa_value ffa_memory_relinquish(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003634 struct vm_locked from_locked,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003635 struct ffa_mem_relinquish *relinquish_request, struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003636{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003637 ffa_memory_handle_t handle = relinquish_request->handle;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003638 struct share_states_locked share_states;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003639 struct ffa_memory_share_state *share_state;
3640 struct ffa_memory_region *memory_region;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003641 bool clear;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003642 struct ffa_value ret;
J-Alves8eb19162022-04-28 10:56:48 +01003643 uint32_t receiver_index;
J-Alves3c5b2072022-11-21 12:45:40 +00003644 bool receivers_relinquished_memory;
Karl Meakin84710f32023-10-12 15:14:49 +01003645 ffa_memory_access_permissions_t receiver_permissions = {0};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003646
Andrew Walbrana65a1322020-04-06 19:32:32 +01003647 if (relinquish_request->endpoint_count != 1) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003648 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01003649 "Stream endpoints not supported (got %d "
J-Alves668a86e2023-05-10 11:53:25 +01003650 "endpoints on FFA_MEM_RELINQUISH, expected 1).\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003651 relinquish_request->endpoint_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003652 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003653 }
3654
Andrew Walbrana65a1322020-04-06 19:32:32 +01003655 if (relinquish_request->endpoints[0] != from_locked.vm->id) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003656 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01003657 "VM ID %d in relinquish message doesn't match "
J-Alves668a86e2023-05-10 11:53:25 +01003658 "calling VM ID %d.\n",
Andrew Walbrana65a1322020-04-06 19:32:32 +01003659 relinquish_request->endpoints[0], from_locked.vm->id);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003660 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003661 }
3662
3663 dump_share_states();
3664
3665 share_states = share_states_lock();
Karl Meakin4a2854a2023-06-30 16:26:52 +01003666 share_state = get_share_state(share_states, handle);
J-Alvesb56aac82023-11-10 09:44:43 +00003667 if (share_state == NULL) {
Karl Meakine8937d92024-03-19 16:04:25 +00003668 dlog_verbose("Invalid handle %#lx for FFA_MEM_RELINQUISH.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003669 handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003670 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003671 goto out;
3672 }
3673
Andrew Walbranca808b12020-05-15 17:22:28 +01003674 if (!share_state->sending_complete) {
3675 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003676 "Memory with handle %#lx not fully sent, can't "
Andrew Walbranca808b12020-05-15 17:22:28 +01003677 "relinquish.\n",
3678 handle);
3679 ret = ffa_error(FFA_INVALID_PARAMETERS);
3680 goto out;
3681 }
3682
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003683 memory_region = share_state->memory_region;
3684 CHECK(memory_region != NULL);
3685
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003686 receiver_index = ffa_memory_region_get_receiver_index(
3687 memory_region, from_locked.vm->id);
J-Alves8eb19162022-04-28 10:56:48 +01003688
3689 if (receiver_index == memory_region->receiver_count) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003690 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01003691 "VM ID %d tried to relinquish memory region "
Karl Meakine8937d92024-03-19 16:04:25 +00003692 "with handle %#lx and it is not a valid borrower.\n",
J-Alves8eb19162022-04-28 10:56:48 +01003693 from_locked.vm->id, handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003694 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003695 goto out;
3696 }
3697
J-Alves8eb19162022-04-28 10:56:48 +01003698 if (share_state->retrieved_fragment_count[receiver_index] !=
Andrew Walbranca808b12020-05-15 17:22:28 +01003699 share_state->fragment_count) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003700 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003701 "Memory with handle %#lx not yet fully "
J-Alvesa9cd7e32022-07-01 13:49:33 +01003702 "retrieved, "
J-Alves8eb19162022-04-28 10:56:48 +01003703 "receiver %x can't relinquish.\n",
3704 handle, from_locked.vm->id);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003705 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003706 goto out;
3707 }
3708
J-Alves3c5b2072022-11-21 12:45:40 +00003709 /*
3710 * Either clear if requested in relinquish call, or in a retrieve
3711 * request from one of the borrowers.
3712 */
3713 receivers_relinquished_memory = true;
3714
3715 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
3716 struct ffa_memory_access *receiver =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003717 ffa_memory_region_get_receiver(memory_region, i);
3718 assert(receiver != NULL);
J-Alves3c5b2072022-11-21 12:45:40 +00003719 if (receiver->receiver_permissions.receiver ==
3720 from_locked.vm->id) {
J-Alves639ddfc2023-11-21 14:17:26 +00003721 receiver_permissions =
3722 receiver->receiver_permissions.permissions;
J-Alves3c5b2072022-11-21 12:45:40 +00003723 continue;
3724 }
3725
3726 if (share_state->retrieved_fragment_count[i] != 0U) {
3727 receivers_relinquished_memory = false;
3728 break;
3729 }
3730 }
3731
3732 clear = receivers_relinquished_memory &&
Daniel Boulby2e14ebe2024-01-15 16:21:44 +00003733 ((relinquish_request->flags & FFA_MEMORY_REGION_FLAG_CLEAR) !=
3734 0U);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003735
3736 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01003737 * Clear is not allowed for memory that was shared, as the
3738 * original sender still has access to the memory.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003739 */
J-Alves95fbb312024-03-20 15:19:16 +00003740 if (clear && (share_state->share_func == FFA_MEM_SHARE_32 ||
3741 share_state->share_func == FFA_MEM_SHARE_64)) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003742 dlog_verbose("Memory which was shared can't be cleared.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003743 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003744 goto out;
3745 }
3746
J-Alvesb886d492024-04-15 10:55:29 +01003747 if (clear && receiver_permissions.data_access == FFA_DATA_ACCESS_RO) {
J-Alves639ddfc2023-11-21 14:17:26 +00003748 dlog_verbose("%s: RO memory can't use clear memory flag.\n",
3749 __func__);
3750 ret = ffa_error(FFA_DENIED);
3751 goto out;
3752 }
3753
Andrew Walbranca808b12020-05-15 17:22:28 +01003754 ret = ffa_relinquish_check_update(
J-Alves26483382023-04-20 12:01:49 +01003755 from_locked, share_state->fragments,
Andrew Walbranca808b12020-05-15 17:22:28 +01003756 share_state->fragment_constituent_counts,
3757 share_state->fragment_count, page_pool, clear);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003758
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003759 if (ret.func == FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003760 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01003761 * Mark memory handle as not retrieved, so it can be
3762 * reclaimed (or retrieved again).
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003763 */
J-Alves8eb19162022-04-28 10:56:48 +01003764 share_state->retrieved_fragment_count[receiver_index] = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003765 }
3766
3767out:
3768 share_states_unlock(&share_states);
3769 dump_share_states();
3770 return ret;
3771}
3772
3773/**
J-Alvesa9cd7e32022-07-01 13:49:33 +01003774 * Validates that the reclaim transition is allowed for the given
3775 * handle, updates the page table of the reclaiming VM, and frees the
3776 * internal state associated with the handle.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003777 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003778struct ffa_value ffa_memory_reclaim(struct vm_locked to_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01003779 ffa_memory_handle_t handle,
3780 ffa_memory_region_flags_t flags,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003781 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003782{
3783 struct share_states_locked share_states;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003784 struct ffa_memory_share_state *share_state;
3785 struct ffa_memory_region *memory_region;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003786 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003787
3788 dump_share_states();
3789
3790 share_states = share_states_lock();
Karl Meakin52cdfe72023-06-30 14:49:10 +01003791
Karl Meakin4a2854a2023-06-30 16:26:52 +01003792 share_state = get_share_state(share_states, handle);
J-Alvesb56aac82023-11-10 09:44:43 +00003793 if (share_state == NULL) {
Karl Meakine8937d92024-03-19 16:04:25 +00003794 dlog_verbose("Invalid handle %#lx for FFA_MEM_RECLAIM.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003795 handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003796 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003797 goto out;
3798 }
Karl Meakin4a2854a2023-06-30 16:26:52 +01003799 memory_region = share_state->memory_region;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003800
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003801 CHECK(memory_region != NULL);
3802
J-Alvesa9cd7e32022-07-01 13:49:33 +01003803 if (vm_id_is_current_world(to_locked.vm->id) &&
3804 to_locked.vm->id != memory_region->sender) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003805 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003806 "VM %#x attempted to reclaim memory handle %#lx "
Olivier Deprezf92e5d42020-11-13 16:00:54 +01003807 "originally sent by VM %#x.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003808 to_locked.vm->id, handle, memory_region->sender);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003809 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003810 goto out;
3811 }
3812
Andrew Walbranca808b12020-05-15 17:22:28 +01003813 if (!share_state->sending_complete) {
3814 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003815 "Memory with handle %#lx not fully sent, can't "
Andrew Walbranca808b12020-05-15 17:22:28 +01003816 "reclaim.\n",
3817 handle);
3818 ret = ffa_error(FFA_INVALID_PARAMETERS);
3819 goto out;
3820 }
3821
J-Alves752236c2022-04-28 11:07:47 +01003822 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
3823 if (share_state->retrieved_fragment_count[i] != 0) {
3824 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003825 "Tried to reclaim memory handle %#lx "
J-Alves3c5b2072022-11-21 12:45:40 +00003826 "that has not been relinquished by all "
J-Alvesa9cd7e32022-07-01 13:49:33 +01003827 "borrowers(%x).\n",
J-Alves752236c2022-04-28 11:07:47 +01003828 handle,
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003829 ffa_memory_region_get_receiver(memory_region, i)
3830 ->receiver_permissions.receiver);
J-Alves752236c2022-04-28 11:07:47 +01003831 ret = ffa_error(FFA_DENIED);
3832 goto out;
3833 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003834 }
3835
Andrew Walbranca808b12020-05-15 17:22:28 +01003836 ret = ffa_retrieve_check_update(
J-Alves26483382023-04-20 12:01:49 +01003837 to_locked, share_state->fragments,
Andrew Walbranca808b12020-05-15 17:22:28 +01003838 share_state->fragment_constituent_counts,
J-Alves2a0d2882020-10-29 14:49:50 +00003839 share_state->fragment_count, share_state->sender_orig_mode,
J-Alves460d36c2023-10-12 17:02:15 +01003840 FFA_MEM_RECLAIM_32, flags & FFA_MEM_RECLAIM_CLEAR, page_pool,
J-Alvesfd206052023-05-22 16:45:00 +01003841 NULL, share_state->memory_protected);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003842
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003843 if (ret.func == FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003844 share_state_free(share_states, share_state, page_pool);
J-Alves3c5b2072022-11-21 12:45:40 +00003845 dlog_verbose("Freed share state after successful reclaim.\n");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003846 }
3847
3848out:
3849 share_states_unlock(&share_states);
3850 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01003851}