blob: d188f3d8b63be82ad334eb1e6fe49d92b4c85348 [file] [log] [blame]
Jose Marinho75509b42019-04-09 09:34:59 +01001/*
2 * Copyright 2019 The Hafnium Authors.
3 *
Andrew Walbrane959ec12020-06-17 15:01:09 +01004 * Use of this source code is governed by a BSD-style
5 * license that can be found in the LICENSE file or at
6 * https://opensource.org/licenses/BSD-3-Clause.
Jose Marinho75509b42019-04-09 09:34:59 +01007 */
8
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01009#include "hf/ffa_memory.h"
Andrew Walbran475c1452020-02-07 13:22:22 +000010
J-Alves460d36c2023-10-12 17:02:15 +010011#include <stdint.h>
12
Federico Recanati4fd065d2021-12-13 20:06:23 +010013#include "hf/arch/mm.h"
Olivier Deprez112d2b52020-09-30 07:39:23 +020014#include "hf/arch/other_world.h"
Olivier Deprez55a189e2021-06-09 15:45:27 +020015#include "hf/arch/plat/ffa.h"
Andrew Walbran290b0c92020-02-03 16:37:14 +000016
J-Alves5952d942022-12-22 16:03:00 +000017#include "hf/addr.h"
Jose Marinho75509b42019-04-09 09:34:59 +010018#include "hf/api.h"
Daniel Boulbya2f8c662021-11-26 17:52:53 +000019#include "hf/assert.h"
Jose Marinho09b1db82019-08-08 09:16:59 +010020#include "hf/check.h"
Jose Marinho75509b42019-04-09 09:34:59 +010021#include "hf/dlog.h"
J-Alves3456e032023-07-20 12:20:05 +010022#include "hf/ffa.h"
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010023#include "hf/ffa_internal.h"
J-Alves66652252022-07-06 09:49:51 +010024#include "hf/ffa_memory_internal.h"
J-Alves3456e032023-07-20 12:20:05 +010025#include "hf/ffa_partition_manifest.h"
J-Alves5952d942022-12-22 16:03:00 +000026#include "hf/mm.h"
Andrew Walbran475c1452020-02-07 13:22:22 +000027#include "hf/mpool.h"
J-Alvescf6253e2024-01-03 13:48:48 +000028#include "hf/panic.h"
29#include "hf/plat/memory_protect.h"
Jose Marinho75509b42019-04-09 09:34:59 +010030#include "hf/std.h"
Andrew Scull3c257452019-11-26 13:32:50 +000031#include "hf/vm.h"
Daniel Boulby44e9b3b2024-01-17 12:21:44 +000032#include "hf/vm_ids.h"
Jose Marinho75509b42019-04-09 09:34:59 +010033
J-Alves2d8457f2022-10-05 11:06:41 +010034#include "vmapi/hf/ffa_v1_0.h"
35
J-Alves5da37d92022-10-24 16:33:48 +010036#define RECEIVERS_COUNT_IN_RETRIEVE_RESP 1
37
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000038/**
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010039 * All access to members of a `struct ffa_memory_share_state` must be guarded
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000040 * by this lock.
41 */
42static struct spinlock share_states_lock_instance = SPINLOCK_INIT;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010043static struct ffa_memory_share_state share_states[MAX_MEM_SHARES];
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000044
45/**
J-Alvesed508c82023-05-04 16:09:48 +010046 * Return the offset to the first constituent within the
47 * `ffa_composite_memory_region` for the given receiver from an
48 * `ffa_memory_region`. The caller must check that the receiver_index is within
49 * bounds, and that it has a composite memory region offset.
50 */
51static uint32_t ffa_composite_constituent_offset(
52 struct ffa_memory_region *memory_region, uint32_t receiver_index)
53{
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +000054 struct ffa_memory_access *receiver;
55 uint32_t composite_offset;
J-Alvesed508c82023-05-04 16:09:48 +010056
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +000057 CHECK(receiver_index < memory_region->receiver_count);
58
59 receiver =
60 ffa_memory_region_get_receiver(memory_region, receiver_index);
61 CHECK(receiver != NULL);
62
63 composite_offset = receiver->composite_memory_region_offset;
64
65 CHECK(composite_offset != 0);
66
67 return composite_offset + sizeof(struct ffa_composite_memory_region);
J-Alvesed508c82023-05-04 16:09:48 +010068}
69
70/**
J-Alves917d2f22020-10-30 18:39:30 +000071 * Extracts the index from a memory handle allocated by Hafnium's current world.
72 */
73uint64_t ffa_memory_handle_get_index(ffa_memory_handle_t handle)
74{
75 return handle & ~FFA_MEMORY_HANDLE_ALLOCATOR_MASK;
76}
77
78/**
Karl Meakin52cdfe72023-06-30 14:49:10 +010079 * Initialises the next available `struct ffa_memory_share_state`. If `handle`
80 * is `FFA_MEMORY_HANDLE_INVALID` then allocates an appropriate handle,
81 * otherwise uses the provided handle which is assumed to be globally unique.
Andrew Walbranca808b12020-05-15 17:22:28 +010082 *
Karl Meakin52cdfe72023-06-30 14:49:10 +010083 * Returns a pointer to the allocated `ffa_memory_share_state` on success or
84 * `NULL` if none are available.
Andrew Walbranca808b12020-05-15 17:22:28 +010085 */
Karl Meakin52cdfe72023-06-30 14:49:10 +010086struct ffa_memory_share_state *allocate_share_state(
87 struct share_states_locked share_states, uint32_t share_func,
88 struct ffa_memory_region *memory_region, uint32_t fragment_length,
89 ffa_memory_handle_t handle)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000090{
Daniel Boulbya2f8c662021-11-26 17:52:53 +000091 assert(share_states.share_states != NULL);
92 assert(memory_region != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000093
Karl Meakin52cdfe72023-06-30 14:49:10 +010094 for (uint64_t i = 0; i < MAX_MEM_SHARES; ++i) {
Andrew Walbranca808b12020-05-15 17:22:28 +010095 if (share_states.share_states[i].share_func == 0) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010096 struct ffa_memory_share_state *allocated_state =
Andrew Walbranca808b12020-05-15 17:22:28 +010097 &share_states.share_states[i];
98 struct ffa_composite_memory_region *composite =
99 ffa_memory_region_get_composite(memory_region,
100 0);
101
102 if (handle == FFA_MEMORY_HANDLE_INVALID) {
J-Alvesee68c542020-10-29 17:48:20 +0000103 memory_region->handle =
Olivier Deprez55a189e2021-06-09 15:45:27 +0200104 plat_ffa_memory_handle_make(i);
Andrew Walbranca808b12020-05-15 17:22:28 +0100105 } else {
J-Alvesee68c542020-10-29 17:48:20 +0000106 memory_region->handle = handle;
Andrew Walbranca808b12020-05-15 17:22:28 +0100107 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000108 allocated_state->share_func = share_func;
109 allocated_state->memory_region = memory_region;
Andrew Walbranca808b12020-05-15 17:22:28 +0100110 allocated_state->fragment_count = 1;
111 allocated_state->fragments[0] = composite->constituents;
112 allocated_state->fragment_constituent_counts[0] =
113 (fragment_length -
114 ffa_composite_constituent_offset(memory_region,
115 0)) /
116 sizeof(struct ffa_memory_region_constituent);
117 allocated_state->sending_complete = false;
Karl Meakin52cdfe72023-06-30 14:49:10 +0100118 for (uint32_t j = 0; j < MAX_MEM_SHARE_RECIPIENTS;
119 ++j) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100120 allocated_state->retrieved_fragment_count[j] =
121 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000122 }
Karl Meakin52cdfe72023-06-30 14:49:10 +0100123 return allocated_state;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000124 }
125 }
126
Karl Meakin52cdfe72023-06-30 14:49:10 +0100127 return NULL;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000128}
129
130/** Locks the share states lock. */
131struct share_states_locked share_states_lock(void)
132{
133 sl_lock(&share_states_lock_instance);
134
135 return (struct share_states_locked){.share_states = share_states};
136}
137
138/** Unlocks the share states lock. */
J-Alves66652252022-07-06 09:49:51 +0100139void share_states_unlock(struct share_states_locked *share_states)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000140{
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000141 assert(share_states->share_states != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000142 share_states->share_states = NULL;
143 sl_unlock(&share_states_lock_instance);
144}
145
146/**
Andrew Walbranca808b12020-05-15 17:22:28 +0100147 * If the given handle is a valid handle for an allocated share state then
Karl Meakin4a2854a2023-06-30 16:26:52 +0100148 * returns a pointer to the share state. Otherwise returns NULL.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000149 */
Karl Meakin4a2854a2023-06-30 16:26:52 +0100150struct ffa_memory_share_state *get_share_state(
151 struct share_states_locked share_states, ffa_memory_handle_t handle)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000152{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100153 struct ffa_memory_share_state *share_state;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000154
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000155 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100156
157 /*
158 * First look for a share_state allocated by us, in which case the
159 * handle is based on the index.
160 */
Olivier Deprez55a189e2021-06-09 15:45:27 +0200161 if (plat_ffa_memory_handle_allocated_by_current_world(handle)) {
Karl Meakin4a2854a2023-06-30 16:26:52 +0100162 uint64_t index = ffa_memory_handle_get_index(handle);
163
Andrew Walbranca808b12020-05-15 17:22:28 +0100164 if (index < MAX_MEM_SHARES) {
165 share_state = &share_states.share_states[index];
166 if (share_state->share_func != 0) {
Karl Meakin4a2854a2023-06-30 16:26:52 +0100167 return share_state;
Andrew Walbranca808b12020-05-15 17:22:28 +0100168 }
169 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000170 }
171
Andrew Walbranca808b12020-05-15 17:22:28 +0100172 /* Fall back to a linear scan. */
Karl Meakin4a2854a2023-06-30 16:26:52 +0100173 for (uint64_t index = 0; index < MAX_MEM_SHARES; ++index) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100174 share_state = &share_states.share_states[index];
J-Alvesee68c542020-10-29 17:48:20 +0000175 if (share_state->memory_region != NULL &&
176 share_state->memory_region->handle == handle &&
Andrew Walbranca808b12020-05-15 17:22:28 +0100177 share_state->share_func != 0) {
Karl Meakin4a2854a2023-06-30 16:26:52 +0100178 return share_state;
Andrew Walbranca808b12020-05-15 17:22:28 +0100179 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000180 }
181
Karl Meakin4a2854a2023-06-30 16:26:52 +0100182 return NULL;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000183}
184
185/** Marks a share state as unallocated. */
J-Alvesfdd29272022-07-19 13:16:31 +0100186void share_state_free(struct share_states_locked share_states,
187 struct ffa_memory_share_state *share_state,
188 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000189{
Andrew Walbranca808b12020-05-15 17:22:28 +0100190 uint32_t i;
191
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000192 assert(share_states.share_states != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000193 share_state->share_func = 0;
Andrew Walbranca808b12020-05-15 17:22:28 +0100194 share_state->sending_complete = false;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000195 mpool_free(page_pool, share_state->memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +0100196 /*
197 * First fragment is part of the same page as the `memory_region`, so it
198 * doesn't need to be freed separately.
199 */
200 share_state->fragments[0] = NULL;
201 share_state->fragment_constituent_counts[0] = 0;
202 for (i = 1; i < share_state->fragment_count; ++i) {
203 mpool_free(page_pool, share_state->fragments[i]);
204 share_state->fragments[i] = NULL;
205 share_state->fragment_constituent_counts[i] = 0;
206 }
207 share_state->fragment_count = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000208 share_state->memory_region = NULL;
J-Alvesa9cd7e32022-07-01 13:49:33 +0100209 share_state->hypervisor_fragment_count = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000210}
211
Andrew Walbranca808b12020-05-15 17:22:28 +0100212/** Checks whether the given share state has been fully sent. */
J-Alvesfdd29272022-07-19 13:16:31 +0100213bool share_state_sending_complete(struct share_states_locked share_states,
214 struct ffa_memory_share_state *share_state)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000215{
Andrew Walbranca808b12020-05-15 17:22:28 +0100216 struct ffa_composite_memory_region *composite;
217 uint32_t expected_constituent_count;
218 uint32_t fragment_constituent_count_total = 0;
219 uint32_t i;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000220
Andrew Walbranca808b12020-05-15 17:22:28 +0100221 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000222 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100223
224 /*
225 * Share state must already be valid, or it's not possible to get hold
226 * of it.
227 */
228 CHECK(share_state->memory_region != NULL &&
229 share_state->share_func != 0);
230
231 composite =
232 ffa_memory_region_get_composite(share_state->memory_region, 0);
233 expected_constituent_count = composite->constituent_count;
234 for (i = 0; i < share_state->fragment_count; ++i) {
235 fragment_constituent_count_total +=
236 share_state->fragment_constituent_counts[i];
237 }
238 dlog_verbose(
239 "Checking completion: constituent count %d/%d from %d "
240 "fragments.\n",
241 fragment_constituent_count_total, expected_constituent_count,
242 share_state->fragment_count);
243
244 return fragment_constituent_count_total == expected_constituent_count;
245}
246
247/**
248 * Calculates the offset of the next fragment expected for the given share
249 * state.
250 */
J-Alvesfdd29272022-07-19 13:16:31 +0100251uint32_t share_state_next_fragment_offset(
Andrew Walbranca808b12020-05-15 17:22:28 +0100252 struct share_states_locked share_states,
253 struct ffa_memory_share_state *share_state)
254{
255 uint32_t next_fragment_offset;
256 uint32_t i;
257
258 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000259 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100260
261 next_fragment_offset =
262 ffa_composite_constituent_offset(share_state->memory_region, 0);
263 for (i = 0; i < share_state->fragment_count; ++i) {
264 next_fragment_offset +=
265 share_state->fragment_constituent_counts[i] *
266 sizeof(struct ffa_memory_region_constituent);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000267 }
268
Andrew Walbranca808b12020-05-15 17:22:28 +0100269 return next_fragment_offset;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000270}
271
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100272static void dump_memory_region(struct ffa_memory_region *memory_region)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000273{
274 uint32_t i;
275
276 if (LOG_LEVEL < LOG_LEVEL_VERBOSE) {
277 return;
278 }
279
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000280 dlog("from VM %#x, attributes %#x, flags %#x, handle %#x "
281 "tag %u, memory access descriptor size %u, to %u "
Andrew Walbrana65a1322020-04-06 19:32:32 +0100282 "recipients [",
283 memory_region->sender, memory_region->attributes,
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000284 memory_region->flags, memory_region->handle, memory_region->tag,
285 memory_region->memory_access_desc_size,
Andrew Walbrana65a1322020-04-06 19:32:32 +0100286 memory_region->receiver_count);
287 for (i = 0; i < memory_region->receiver_count; ++i) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +0000288 struct ffa_memory_access *receiver =
289 ffa_memory_region_get_receiver(memory_region, i);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000290 if (i != 0) {
291 dlog(", ");
292 }
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +0000293 dlog("Receiver %#x: %#x (offset %u)",
294 receiver->receiver_permissions.receiver,
295 receiver->receiver_permissions.permissions,
296 receiver->composite_memory_region_offset);
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000297 /* The impdef field is only present from v1.2 and later */
298 if (ffa_version_from_memory_access_desc_size(
299 memory_region->memory_access_desc_size) >=
300 MAKE_FFA_VERSION(1, 2)) {
301 dlog(", impdef: %#x %#x", receiver->impdef.val[0],
302 receiver->impdef.val[1]);
303 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000304 }
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000305 dlog("] at offset %u", memory_region->receivers_offset);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000306}
307
J-Alves66652252022-07-06 09:49:51 +0100308void dump_share_states(void)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000309{
310 uint32_t i;
311
312 if (LOG_LEVEL < LOG_LEVEL_VERBOSE) {
313 return;
314 }
315
316 dlog("Current share states:\n");
317 sl_lock(&share_states_lock_instance);
318 for (i = 0; i < MAX_MEM_SHARES; ++i) {
319 if (share_states[i].share_func != 0) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000320 switch (share_states[i].share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100321 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000322 dlog("SHARE");
323 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100324 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000325 dlog("LEND");
326 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100327 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000328 dlog("DONATE");
329 break;
330 default:
331 dlog("invalid share_func %#x",
332 share_states[i].share_func);
333 }
Olivier Deprez935e1b12020-12-22 18:01:29 +0100334 dlog(" %#x (", share_states[i].memory_region->handle);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000335 dump_memory_region(share_states[i].memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +0100336 if (share_states[i].sending_complete) {
337 dlog("): fully sent");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000338 } else {
Andrew Walbranca808b12020-05-15 17:22:28 +0100339 dlog("): partially sent");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000340 }
J-Alves2a0d2882020-10-29 14:49:50 +0000341 dlog(" with %d fragments, %d retrieved, "
342 " sender's original mode: %#x\n",
Andrew Walbranca808b12020-05-15 17:22:28 +0100343 share_states[i].fragment_count,
J-Alves2a0d2882020-10-29 14:49:50 +0000344 share_states[i].retrieved_fragment_count[0],
345 share_states[i].sender_orig_mode);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000346 }
347 }
348 sl_unlock(&share_states_lock_instance);
349}
350
Andrew Walbran475c1452020-02-07 13:22:22 +0000351/* TODO: Add device attributes: GRE, cacheability, shareability. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100352static inline uint32_t ffa_memory_permissions_to_mode(
J-Alves7cd5eb32020-10-16 19:06:10 +0100353 ffa_memory_access_permissions_t permissions, uint32_t default_mode)
Andrew Walbran475c1452020-02-07 13:22:22 +0000354{
355 uint32_t mode = 0;
356
Karl Meakin84710f32023-10-12 15:14:49 +0100357 switch (permissions.data_access) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100358 case FFA_DATA_ACCESS_RO:
Andrew Walbran475c1452020-02-07 13:22:22 +0000359 mode = MM_MODE_R;
360 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100361 case FFA_DATA_ACCESS_RW:
Andrew Walbran475c1452020-02-07 13:22:22 +0000362 mode = MM_MODE_R | MM_MODE_W;
363 break;
J-Alves7cd5eb32020-10-16 19:06:10 +0100364 case FFA_DATA_ACCESS_NOT_SPECIFIED:
365 mode = (default_mode & (MM_MODE_R | MM_MODE_W));
366 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100367 case FFA_DATA_ACCESS_RESERVED:
368 panic("Tried to convert FFA_DATA_ACCESS_RESERVED.");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100369 }
370
Karl Meakin84710f32023-10-12 15:14:49 +0100371 switch (permissions.instruction_access) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100372 case FFA_INSTRUCTION_ACCESS_NX:
Andrew Walbran475c1452020-02-07 13:22:22 +0000373 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100374 case FFA_INSTRUCTION_ACCESS_X:
Andrew Walbrana65a1322020-04-06 19:32:32 +0100375 mode |= MM_MODE_X;
376 break;
J-Alves7cd5eb32020-10-16 19:06:10 +0100377 case FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED:
378 mode |= (default_mode & MM_MODE_X);
379 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100380 case FFA_INSTRUCTION_ACCESS_RESERVED:
381 panic("Tried to convert FFA_INSTRUCTION_ACCESS_RESVERVED.");
Andrew Walbran475c1452020-02-07 13:22:22 +0000382 }
383
Olivier Deprez878bd5b2021-04-15 19:05:10 +0200384 /* Set the security state bit if necessary. */
385 if ((default_mode & plat_ffa_other_world_mode()) != 0) {
386 mode |= plat_ffa_other_world_mode();
387 }
388
Andrew Walbran475c1452020-02-07 13:22:22 +0000389 return mode;
390}
391
Jose Marinho75509b42019-04-09 09:34:59 +0100392/**
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000393 * Get the current mode in the stage-2 page table of the given vm of all the
394 * pages in the given constituents, if they all have the same mode, or return
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100395 * an appropriate FF-A error if not.
Jose Marinho75509b42019-04-09 09:34:59 +0100396 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100397static struct ffa_value constituents_get_mode(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000398 struct vm_locked vm, uint32_t *orig_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100399 struct ffa_memory_region_constituent **fragments,
400 const uint32_t *fragment_constituent_counts, uint32_t fragment_count)
Jose Marinho75509b42019-04-09 09:34:59 +0100401{
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100402 uint32_t i;
Andrew Walbranca808b12020-05-15 17:22:28 +0100403 uint32_t j;
Jose Marinho75509b42019-04-09 09:34:59 +0100404
Andrew Walbranca808b12020-05-15 17:22:28 +0100405 if (fragment_count == 0 || fragment_constituent_counts[0] == 0) {
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100406 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000407 * Fail if there are no constituents. Otherwise we would get an
408 * uninitialised *orig_mode.
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100409 */
Karl Meakin5df422c2023-07-11 17:31:38 +0100410 dlog_verbose("%s: no constituents\n", __func__);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100411 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho75509b42019-04-09 09:34:59 +0100412 }
413
Andrew Walbranca808b12020-05-15 17:22:28 +0100414 for (i = 0; i < fragment_count; ++i) {
415 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
416 ipaddr_t begin = ipa_init(fragments[i][j].address);
417 size_t size = fragments[i][j].page_count * PAGE_SIZE;
418 ipaddr_t end = ipa_add(begin, size);
419 uint32_t current_mode;
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100420
Andrew Walbranca808b12020-05-15 17:22:28 +0100421 /* Fail if addresses are not page-aligned. */
422 if (!is_aligned(ipa_addr(begin), PAGE_SIZE) ||
423 !is_aligned(ipa_addr(end), PAGE_SIZE)) {
Karl Meakin5df422c2023-07-11 17:31:38 +0100424 dlog_verbose("%s: addresses not page-aligned\n",
425 __func__);
Andrew Walbranca808b12020-05-15 17:22:28 +0100426 return ffa_error(FFA_INVALID_PARAMETERS);
427 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100428
Andrew Walbranca808b12020-05-15 17:22:28 +0100429 /*
430 * Ensure that this constituent memory range is all
431 * mapped with the same mode.
432 */
Raghu Krishnamurthy785d52f2021-02-13 00:02:40 -0800433 if (!vm_mem_get_mode(vm, begin, end, &current_mode)) {
Karl Meakin5df422c2023-07-11 17:31:38 +0100434 dlog_verbose(
435 "%s: constituent memory range %#x..%#x "
436 "not mapped with the same mode\n",
437 __func__, begin, end);
Andrew Walbranca808b12020-05-15 17:22:28 +0100438 return ffa_error(FFA_DENIED);
439 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100440
Andrew Walbranca808b12020-05-15 17:22:28 +0100441 /*
442 * Ensure that all constituents are mapped with the same
443 * mode.
444 */
445 if (i == 0) {
446 *orig_mode = current_mode;
447 } else if (current_mode != *orig_mode) {
448 dlog_verbose(
Karl Meakin5df422c2023-07-11 17:31:38 +0100449 "%s: expected mode %#x but was %#x for "
450 "%d pages at %#x.\n",
451 __func__, *orig_mode, current_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100452 fragments[i][j].page_count,
453 ipa_addr(begin));
454 return ffa_error(FFA_DENIED);
455 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100456 }
Jose Marinho75509b42019-04-09 09:34:59 +0100457 }
458
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100459 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000460}
461
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100462uint32_t ffa_version_from_memory_access_desc_size(
463 uint32_t memory_access_desc_size)
464{
465 switch (memory_access_desc_size) {
466 /*
467 * v1.0 and v1.1 memory access descriptors are the same size however
468 * v1.1 is the first version to include the memory access descriptor
469 * size field so return v1.1.
470 */
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000471 case sizeof(struct ffa_memory_access_v1_0):
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100472 return MAKE_FFA_VERSION(1, 1);
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000473 case sizeof(struct ffa_memory_access):
474 return MAKE_FFA_VERSION(1, 2);
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100475 }
476 return 0;
477}
478
479/**
480 * Check if the receivers size and offset given is valid for the senders
481 * FF-A version.
482 */
483static bool receiver_size_and_offset_valid_for_version(
484 uint32_t receivers_size, uint32_t receivers_offset,
485 uint32_t ffa_version)
486{
487 /*
488 * Check that the version that the memory access descriptor size belongs
489 * to is compatible with the FF-A version we believe the sender to be.
490 */
491 uint32_t expected_ffa_version =
492 ffa_version_from_memory_access_desc_size(receivers_size);
493 if (!FFA_VERSIONS_ARE_COMPATIBLE(expected_ffa_version, ffa_version)) {
494 return false;
495 }
496
497 /*
498 * Check the receivers_offset matches the version we found from
499 * memory access descriptor size.
500 */
501 switch (expected_ffa_version) {
502 case MAKE_FFA_VERSION(1, 1):
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000503 case MAKE_FFA_VERSION(1, 2):
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100504 return receivers_offset == sizeof(struct ffa_memory_region);
505 default:
506 return false;
507 }
508}
509
510/**
511 * Check the values set for fields in the memory region are valid and safe.
512 * Offset values are within safe bounds, receiver count will not cause overflows
513 * and reserved fields are 0.
514 */
515bool ffa_memory_region_sanity_check(struct ffa_memory_region *memory_region,
516 uint32_t ffa_version,
517 uint32_t fragment_length,
518 bool send_transaction)
519{
520 uint32_t receiver_count;
521 struct ffa_memory_access *receiver;
522 uint32_t composite_offset_0;
Daniel Boulbyf06b5232024-02-22 16:26:43 +0000523 struct ffa_memory_region_v1_0 *memory_region_v1_0 =
524 (struct ffa_memory_region_v1_0 *)memory_region;
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100525
526 if (ffa_version == MAKE_FFA_VERSION(1, 0)) {
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100527 /* Check the reserved fields are 0. */
528 if (memory_region_v1_0->reserved_0 != 0 ||
529 memory_region_v1_0->reserved_1 != 0) {
530 dlog_verbose("Reserved fields must be 0.\n");
531 return false;
532 }
533
534 receiver_count = memory_region_v1_0->receiver_count;
535 } else {
536 uint32_t receivers_size =
537 memory_region->memory_access_desc_size;
538 uint32_t receivers_offset = memory_region->receivers_offset;
539
540 /* Check the reserved field is 0. */
541 if (memory_region->reserved[0] != 0 ||
542 memory_region->reserved[1] != 0 ||
543 memory_region->reserved[2] != 0) {
544 dlog_verbose("Reserved fields must be 0.\n");
545 return false;
546 }
547
548 /*
549 * Check memory_access_desc_size matches the size of the struct
550 * for the senders FF-A version.
551 */
552 if (!receiver_size_and_offset_valid_for_version(
553 receivers_size, receivers_offset, ffa_version)) {
554 dlog_verbose(
555 "Invalid memory access descriptor size %d, "
556 " or receiver offset %d, "
557 "for FF-A version %#x\n",
558 receivers_size, receivers_offset, ffa_version);
559 return false;
560 }
561
562 receiver_count = memory_region->receiver_count;
563 }
564
565 /* Check receiver count is not too large. */
Daniel Boulbyf06b5232024-02-22 16:26:43 +0000566 if (receiver_count > MAX_MEM_SHARE_RECIPIENTS || receiver_count < 1) {
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100567 dlog_verbose(
Daniel Boulbyf06b5232024-02-22 16:26:43 +0000568 "Receiver count must be 0 < receiver_count < %u "
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100569 "specified %u\n",
570 MAX_MEM_SHARE_RECIPIENTS, receiver_count);
571 return false;
572 }
573
574 /* Check values in the memory access descriptors. */
575 /*
576 * The composite offset values must be the same for all recievers so
577 * check the first one is valid and then they are all the same.
578 */
579 receiver = ffa_version == MAKE_FFA_VERSION(1, 0)
Daniel Boulbyf06b5232024-02-22 16:26:43 +0000580 ? (struct ffa_memory_access *)&memory_region_v1_0
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100581 ->receivers[0]
582 : ffa_memory_region_get_receiver(memory_region, 0);
583 assert(receiver != NULL);
584 composite_offset_0 = receiver->composite_memory_region_offset;
585
586 if (!send_transaction) {
587 if (composite_offset_0 != 0) {
588 dlog_verbose(
589 "Composite offset memory region descriptor "
590 "offset must be 0 for retrieve requests. "
591 "Currently %d",
592 composite_offset_0);
593 return false;
594 }
595 } else {
596 bool comp_offset_is_zero = composite_offset_0 == 0U;
597 bool comp_offset_lt_transaction_descriptor_size =
598 composite_offset_0 <
599 (sizeof(struct ffa_memory_region) +
600 (uint32_t)(memory_region->memory_access_desc_size *
601 memory_region->receiver_count));
602 bool comp_offset_with_comp_gt_fragment_length =
603 composite_offset_0 +
604 sizeof(struct ffa_composite_memory_region) >
605 fragment_length;
606 if (comp_offset_is_zero ||
607 comp_offset_lt_transaction_descriptor_size ||
608 comp_offset_with_comp_gt_fragment_length) {
609 dlog_verbose(
610 "Invalid composite memory region descriptor "
611 "offset for send transaction %u\n",
612 composite_offset_0);
613 return false;
614 }
615 }
616
Karl Meakin824b63d2024-06-03 19:04:53 +0100617 for (size_t i = 0; i < memory_region->receiver_count; i++) {
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100618 uint32_t composite_offset;
619
620 if (ffa_version == MAKE_FFA_VERSION(1, 0)) {
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100621 struct ffa_memory_access_v1_0 *receiver_v1_0 =
622 &memory_region_v1_0->receivers[i];
623 /* Check reserved fields are 0 */
624 if (receiver_v1_0->reserved_0 != 0) {
625 dlog_verbose(
626 "Reserved field in the memory access "
627 " descriptor must be zero "
628 " Currently reciever %d has a reserved "
629 " field with a value of %d\n",
630 i, receiver_v1_0->reserved_0);
631 return false;
632 }
633 /*
634 * We can cast to the current version receiver as the
635 * remaining fields we are checking have the same
636 * offsets for all versions since memory access
637 * descriptors are forwards compatible.
638 */
639 receiver = (struct ffa_memory_access *)receiver_v1_0;
640 } else {
641 receiver = ffa_memory_region_get_receiver(memory_region,
642 i);
643 assert(receiver != NULL);
644
645 if (receiver->reserved_0 != 0) {
646 dlog_verbose(
647 "Reserved field in the memory access "
648 " descriptor must be zero "
649 " Currently reciever %d has a reserved "
650 " field with a value of %d\n",
651 i, receiver->reserved_0);
652 return false;
653 }
654 }
655
656 /* Check composite offset values are equal for all receivers. */
657 composite_offset = receiver->composite_memory_region_offset;
658 if (composite_offset != composite_offset_0) {
659 dlog_verbose(
660 "Composite offset %x differs from %x in index "
661 "%u\n",
662 composite_offset, composite_offset_0);
663 return false;
664 }
665 }
666 return true;
667}
668
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000669/**
J-Alves460d36c2023-10-12 17:02:15 +0100670 * If the receivers for the memory management operation are all from the
671 * secure world and this isn't a FFA_MEM_SHARE, then request memory security
672 * state update by returning MAP_ACTION_CHECK_PROTECT.
673 */
674static enum ffa_map_action ffa_mem_send_get_map_action(
675 bool all_receivers_from_current_world, ffa_id_t sender_id,
676 uint32_t mem_func_id)
677{
678 bool protect_memory =
679 (mem_func_id != FFA_MEM_SHARE_32 &&
680 all_receivers_from_current_world && ffa_is_vm_id(sender_id));
681
682 return protect_memory ? MAP_ACTION_CHECK_PROTECT : MAP_ACTION_CHECK;
683}
684
685/**
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000686 * Verify that all pages have the same mode, that the starting mode
687 * constitutes a valid state and obtain the next mode to apply
J-Alves460d36c2023-10-12 17:02:15 +0100688 * to the sending VM. It outputs the mapping action that needs to be
689 * invoked for the given memory range. On memory lend/donate there
690 * could be a need to protect the memory from the normal world.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000691 *
692 * Returns:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100693 * 1) FFA_DENIED if a state transition was not found;
694 * 2) FFA_DENIED if the pages being shared do not have the same mode within
Andrew Walbrana65a1322020-04-06 19:32:32 +0100695 * the <from> VM;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100696 * 3) FFA_INVALID_PARAMETERS if the beginning and end IPAs are not page
Andrew Walbrana65a1322020-04-06 19:32:32 +0100697 * aligned;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100698 * 4) FFA_INVALID_PARAMETERS if the requested share type was not handled.
699 * Or FFA_SUCCESS on success.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000700 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100701static struct ffa_value ffa_send_check_transition(
Andrew Walbrana65a1322020-04-06 19:32:32 +0100702 struct vm_locked from, uint32_t share_func,
Daniel Boulbya76fd912024-02-22 14:22:15 +0000703 struct ffa_memory_region *memory_region, uint32_t *orig_from_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100704 struct ffa_memory_region_constituent **fragments,
705 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
J-Alves460d36c2023-10-12 17:02:15 +0100706 uint32_t *from_mode, enum ffa_map_action *map_action)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000707{
708 const uint32_t state_mask =
709 MM_MODE_INVALID | MM_MODE_UNOWNED | MM_MODE_SHARED;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100710 struct ffa_value ret;
J-Alves460d36c2023-10-12 17:02:15 +0100711 bool all_receivers_from_current_world = true;
Daniel Boulbya76fd912024-02-22 14:22:15 +0000712 uint32_t receivers_count = memory_region->receiver_count;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000713
Andrew Walbranca808b12020-05-15 17:22:28 +0100714 ret = constituents_get_mode(from, orig_from_mode, fragments,
715 fragment_constituent_counts,
716 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100717 if (ret.func != FFA_SUCCESS_32) {
Olivier Depreze7eb1682022-03-16 17:09:03 +0100718 dlog_verbose("Inconsistent modes.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100719 return ret;
Andrew Scullb5f49e02019-10-02 13:20:47 +0100720 }
721
Daniel Boulby63af1fa2024-03-18 14:17:31 +0000722 /* Device memory regions can only be lent a single borrower. */
Daniel Boulby9764ff62024-01-30 17:47:39 +0000723 if ((*orig_from_mode & MM_MODE_D) != 0U &&
Daniel Boulby63af1fa2024-03-18 14:17:31 +0000724 !(share_func == FFA_MEM_LEND_32 && receivers_count == 1)) {
Daniel Boulby9764ff62024-01-30 17:47:39 +0000725 dlog_verbose(
Daniel Boulby63af1fa2024-03-18 14:17:31 +0000726 "Device memory can only be lent to a single borrower "
727 "(mode is %#x).\n",
Daniel Boulby9764ff62024-01-30 17:47:39 +0000728 *orig_from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100729 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000730 }
731
732 /*
733 * Ensure the sender is the owner and has exclusive access to the
734 * memory.
735 */
736 if ((*orig_from_mode & state_mask) != 0) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100737 return ffa_error(FFA_DENIED);
Andrew Walbrana65a1322020-04-06 19:32:32 +0100738 }
739
Daniel Boulbya76fd912024-02-22 14:22:15 +0000740 assert(receivers_count > 0U);
J-Alves7cd5eb32020-10-16 19:06:10 +0100741
J-Alves363f5722022-04-25 17:37:37 +0100742 for (uint32_t i = 0U; i < receivers_count; i++) {
Daniel Boulbya76fd912024-02-22 14:22:15 +0000743 struct ffa_memory_access *receiver =
744 ffa_memory_region_get_receiver(memory_region, i);
745 assert(receiver != NULL);
J-Alves363f5722022-04-25 17:37:37 +0100746 ffa_memory_access_permissions_t permissions =
Daniel Boulbya76fd912024-02-22 14:22:15 +0000747 receiver->receiver_permissions.permissions;
J-Alves363f5722022-04-25 17:37:37 +0100748 uint32_t required_from_mode = ffa_memory_permissions_to_mode(
749 permissions, *orig_from_mode);
750
J-Alves788b4492023-04-18 14:01:23 +0100751 /*
752 * The assumption is that at this point, the operation from
753 * SP to a receiver VM, should have returned an FFA_ERROR
754 * already.
755 */
756 if (!ffa_is_vm_id(from.vm->id)) {
757 assert(!ffa_is_vm_id(
Daniel Boulbya76fd912024-02-22 14:22:15 +0000758 receiver->receiver_permissions.receiver));
J-Alves788b4492023-04-18 14:01:23 +0100759 }
760
J-Alves460d36c2023-10-12 17:02:15 +0100761 /* Track if all senders are from current world. */
762 all_receivers_from_current_world =
763 all_receivers_from_current_world &&
764 vm_id_is_current_world(
Daniel Boulbya76fd912024-02-22 14:22:15 +0000765 receiver->receiver_permissions.receiver);
J-Alves460d36c2023-10-12 17:02:15 +0100766
J-Alves363f5722022-04-25 17:37:37 +0100767 if ((*orig_from_mode & required_from_mode) !=
768 required_from_mode) {
769 dlog_verbose(
770 "Sender tried to send memory with permissions "
J-Alves788b4492023-04-18 14:01:23 +0100771 "which required mode %#x but only had %#x "
772 "itself.\n",
J-Alves363f5722022-04-25 17:37:37 +0100773 required_from_mode, *orig_from_mode);
774 return ffa_error(FFA_DENIED);
775 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000776 }
777
J-Alves460d36c2023-10-12 17:02:15 +0100778 *map_action = ffa_mem_send_get_map_action(
779 all_receivers_from_current_world, from.vm->id, share_func);
780
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000781 /* Find the appropriate new mode. */
782 *from_mode = ~state_mask & *orig_from_mode;
Andrew Walbrane7ad3c02019-12-24 17:03:04 +0000783 switch (share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100784 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000785 *from_mode |= MM_MODE_INVALID | MM_MODE_UNOWNED;
Jose Marinho75509b42019-04-09 09:34:59 +0100786 break;
787
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100788 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000789 *from_mode |= MM_MODE_INVALID;
Andrew Walbran648fc3e2019-10-22 16:23:05 +0100790 break;
791
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100792 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000793 *from_mode |= MM_MODE_SHARED;
Jose Marinho56c25732019-05-20 09:48:53 +0100794 break;
795
Jose Marinho75509b42019-04-09 09:34:59 +0100796 default:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100797 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho75509b42019-04-09 09:34:59 +0100798 }
799
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100800 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000801}
802
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100803static struct ffa_value ffa_relinquish_check_transition(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000804 struct vm_locked from, uint32_t *orig_from_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100805 struct ffa_memory_region_constituent **fragments,
806 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
807 uint32_t *from_mode)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000808{
809 const uint32_t state_mask =
810 MM_MODE_INVALID | MM_MODE_UNOWNED | MM_MODE_SHARED;
811 uint32_t orig_from_state;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100812 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000813
Andrew Walbranca808b12020-05-15 17:22:28 +0100814 ret = constituents_get_mode(from, orig_from_mode, fragments,
815 fragment_constituent_counts,
816 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100817 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbrana65a1322020-04-06 19:32:32 +0100818 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000819 }
820
821 /* Ensure the address range is normal memory and not a device. */
822 if (*orig_from_mode & MM_MODE_D) {
823 dlog_verbose("Can't relinquish device memory (mode is %#x).\n",
824 *orig_from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100825 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000826 }
827
828 /*
829 * Ensure the relinquishing VM is not the owner but has access to the
830 * memory.
831 */
832 orig_from_state = *orig_from_mode & state_mask;
833 if ((orig_from_state & ~MM_MODE_SHARED) != MM_MODE_UNOWNED) {
834 dlog_verbose(
835 "Tried to relinquish memory in state %#x (masked %#x "
Andrew Walbranca808b12020-05-15 17:22:28 +0100836 "but should be %#x).\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000837 *orig_from_mode, orig_from_state, MM_MODE_UNOWNED);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100838 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000839 }
840
841 /* Find the appropriate new mode. */
842 *from_mode = (~state_mask & *orig_from_mode) | MM_MODE_UNMAPPED_MASK;
843
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100844 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000845}
846
847/**
848 * Verify that all pages have the same mode, that the starting mode
849 * constitutes a valid state and obtain the next mode to apply
850 * to the retrieving VM.
851 *
852 * Returns:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100853 * 1) FFA_DENIED if a state transition was not found;
854 * 2) FFA_DENIED if the pages being shared do not have the same mode within
Andrew Walbrana65a1322020-04-06 19:32:32 +0100855 * the <to> VM;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100856 * 3) FFA_INVALID_PARAMETERS if the beginning and end IPAs are not page
Andrew Walbrana65a1322020-04-06 19:32:32 +0100857 * aligned;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100858 * 4) FFA_INVALID_PARAMETERS if the requested share type was not handled.
859 * Or FFA_SUCCESS on success.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000860 */
J-Alvesfc19b372022-07-06 12:17:35 +0100861struct ffa_value ffa_retrieve_check_transition(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000862 struct vm_locked to, uint32_t share_func,
Andrew Walbranca808b12020-05-15 17:22:28 +0100863 struct ffa_memory_region_constituent **fragments,
864 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
J-Alvesfd206052023-05-22 16:45:00 +0100865 uint32_t memory_to_attributes, uint32_t *to_mode, bool memory_protected,
866 enum ffa_map_action *map_action)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000867{
868 uint32_t orig_to_mode;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100869 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000870
Andrew Walbranca808b12020-05-15 17:22:28 +0100871 ret = constituents_get_mode(to, &orig_to_mode, fragments,
872 fragment_constituent_counts,
873 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100874 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100875 dlog_verbose("Inconsistent modes.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100876 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000877 }
878
J-Alves460d36c2023-10-12 17:02:15 +0100879 /* Find the appropriate new mode. */
880 *to_mode = memory_to_attributes;
881
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100882 if (share_func == FFA_MEM_RECLAIM_32) {
J-Alves9256f162021-12-09 13:18:43 +0000883 /*
884 * If the original ffa memory send call has been processed
885 * successfully, it is expected the orig_to_mode would overlay
886 * with `state_mask`, as a result of the function
887 * `ffa_send_check_transition`.
J-Alvesfd206052023-05-22 16:45:00 +0100888 *
889 * If Hafnium is the SPMC:
890 * - Caller of the reclaim interface is an SP, the memory shall
891 * have been protected throughout the flow.
892 * - Caller of the reclaim is from the NWd, the memory may have
893 * been protected at the time of lending/donating the memory.
894 * In such case, set action to unprotect memory in the
895 * handling of reclaim operation.
896 * - If Hafnium is the hypervisor memory shall never have been
897 * protected in memory lend/share/donate.
898 *
899 * More details in the doc comment of the function
900 * `ffa_region_group_identity_map`.
J-Alves9256f162021-12-09 13:18:43 +0000901 */
J-Alves59ed0042022-07-28 18:26:41 +0100902 if (vm_id_is_current_world(to.vm->id)) {
903 assert((orig_to_mode &
904 (MM_MODE_INVALID | MM_MODE_UNOWNED |
905 MM_MODE_SHARED)) != 0U);
J-Alvesfd206052023-05-22 16:45:00 +0100906 assert(!memory_protected);
907 } else if (to.vm->id == HF_OTHER_WORLD_ID &&
908 map_action != NULL && memory_protected) {
909 *map_action = MAP_ACTION_COMMIT_UNPROTECT;
J-Alves59ed0042022-07-28 18:26:41 +0100910 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000911 } else {
912 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +0100913 * If the retriever is from virtual FF-A instance:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000914 * Ensure the retriever has the expected state. We don't care
915 * about the MM_MODE_SHARED bit; either with or without it set
916 * are both valid representations of the !O-NA state.
917 */
J-Alvesa9cd7e32022-07-01 13:49:33 +0100918 if (vm_id_is_current_world(to.vm->id) &&
919 to.vm->id != HF_PRIMARY_VM_ID &&
920 (orig_to_mode & MM_MODE_UNMAPPED_MASK) !=
921 MM_MODE_UNMAPPED_MASK) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100922 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000923 }
J-Alves460d36c2023-10-12 17:02:15 +0100924
925 /*
926 * If memory has been protected before, clear the NS bit to
927 * allow the secure access from the SP.
928 */
929 if (memory_protected) {
930 *to_mode &= ~plat_ffa_other_world_mode();
931 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000932 }
933
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000934 switch (share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100935 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000936 *to_mode |= 0;
937 break;
938
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100939 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000940 *to_mode |= MM_MODE_UNOWNED;
941 break;
942
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100943 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000944 *to_mode |= MM_MODE_UNOWNED | MM_MODE_SHARED;
945 break;
946
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100947 case FFA_MEM_RECLAIM_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000948 *to_mode |= 0;
949 break;
950
951 default:
Andrew Walbranca808b12020-05-15 17:22:28 +0100952 dlog_error("Invalid share_func %#x.\n", share_func);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100953 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000954 }
955
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100956 return (struct ffa_value){.func = FFA_SUCCESS_32};
Jose Marinho75509b42019-04-09 09:34:59 +0100957}
Jose Marinho09b1db82019-08-08 09:16:59 +0100958
J-Alvescf6253e2024-01-03 13:48:48 +0000959/*
960 * Performs the operations related to the `action` MAP_ACTION_CHECK*.
961 * Returns:
962 * - FFA_SUCCESS_32: if all goes well.
963 * - FFA_ERROR_32: with FFA_NO_MEMORY, if there is no memory to manage
964 * the page table update. Or error code provided by the function
965 * `arch_memory_protect`.
966 */
967static struct ffa_value ffa_region_group_check_actions(
968 struct vm_locked vm_locked, paddr_t pa_begin, paddr_t pa_end,
969 struct mpool *ppool, uint32_t mode, enum ffa_map_action action,
970 bool *memory_protected)
971{
972 struct ffa_value ret;
973 bool is_memory_protected;
974
975 if (!vm_identity_prepare(vm_locked, pa_begin, pa_end, mode, ppool)) {
976 dlog_verbose(
977 "%s: memory can't be mapped to %x due to lack of "
978 "memory. Base: %lx end: %x\n",
979 __func__, vm_locked.vm->id, pa_addr(pa_begin),
980 pa_addr(pa_end));
981 return ffa_error(FFA_NO_MEMORY);
982 }
983
984 switch (action) {
985 case MAP_ACTION_CHECK:
986 /* No protect requested. */
987 is_memory_protected = false;
988 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
989 break;
990 case MAP_ACTION_CHECK_PROTECT: {
991 paddr_t last_protected_pa = pa_init(0);
992
993 ret = arch_memory_protect(pa_begin, pa_end, &last_protected_pa);
994
995 is_memory_protected = (ret.func == FFA_SUCCESS_32);
996
997 /*
998 * - If protect memory has failed with FFA_DENIED, means some
999 * range of memory was in the wrong state. In such case, SPM
1000 * reverts the state of the pages that were successfully
1001 * updated.
1002 * - If protect memory has failed with FFA_NOT_SUPPORTED, it
1003 * means the platform doesn't support the protection mechanism.
1004 * That said, it still permits the page table update to go
1005 * through. The variable
1006 * `is_memory_protected` will be equal to false.
1007 * - If protect memory has failed with FFA_INVALID_PARAMETERS,
1008 * break from switch and return the error.
1009 */
1010 if (ret.func == FFA_ERROR_32) {
1011 assert(!is_memory_protected);
1012 if (ffa_error_code(ret) == FFA_DENIED &&
1013 pa_addr(last_protected_pa) != (uintptr_t)0) {
1014 CHECK(arch_memory_unprotect(
1015 pa_begin,
1016 pa_add(last_protected_pa, PAGE_SIZE)));
1017 } else if (ffa_error_code(ret) == FFA_NOT_SUPPORTED) {
1018 ret = (struct ffa_value){
1019 .func = FFA_SUCCESS_32,
1020 };
1021 }
1022 }
1023 } break;
1024 default:
1025 panic("%s: invalid action to process %x\n", __func__, action);
1026 }
1027
1028 if (memory_protected != NULL) {
1029 *memory_protected = is_memory_protected;
1030 }
1031
1032 return ret;
1033}
1034
1035static void ffa_region_group_commit_actions(struct vm_locked vm_locked,
1036 paddr_t pa_begin, paddr_t pa_end,
1037 struct mpool *ppool, uint32_t mode,
1038 enum ffa_map_action action)
1039{
1040 switch (action) {
1041 case MAP_ACTION_COMMIT_UNPROTECT:
1042 /*
1043 * Checking that it should succeed because SPM should be
1044 * unprotecting memory that it had protected before.
1045 */
1046 CHECK(arch_memory_unprotect(pa_begin, pa_end));
1047 case MAP_ACTION_COMMIT:
1048 vm_identity_commit(vm_locked, pa_begin, pa_end, mode, ppool,
1049 NULL);
1050 break;
1051 default:
1052 panic("%s: invalid action to process %x\n", __func__, action);
1053 }
1054}
1055
Jose Marinho09b1db82019-08-08 09:16:59 +01001056/**
J-Alves063ad832023-10-03 18:05:40 +01001057 * Helper function to revert a failed "Protect" action from the SPMC:
1058 * - `fragment_count`: should specify the number of fragments to traverse from
1059 * `fragments`. This may not be the full amount of fragments that are part of
1060 * the share_state structure.
1061 * - `fragment_constituent_counts`: array holding the amount of constituents
1062 * per fragment.
1063 * - `end`: pointer to the constituent that failed the "protect" action. It
1064 * shall be part of the last fragment, and it shall make the loop below break.
1065 */
1066static void ffa_region_group_fragments_revert_protect(
1067 struct ffa_memory_region_constituent **fragments,
1068 const uint32_t *fragment_constituent_counts, uint32_t fragment_count,
1069 const struct ffa_memory_region_constituent *end)
1070{
1071 for (uint32_t i = 0; i < fragment_count; ++i) {
1072 for (uint32_t j = 0; j < fragment_constituent_counts[i]; ++j) {
1073 struct ffa_memory_region_constituent *constituent =
1074 &fragments[i][j];
1075 size_t size = constituent->page_count * PAGE_SIZE;
1076 paddr_t pa_begin =
1077 pa_from_ipa(ipa_init(constituent->address));
1078 paddr_t pa_end = pa_add(pa_begin, size);
1079
1080 dlog_verbose("%s: reverting fragment %x size %x\n",
1081 __func__, pa_addr(pa_begin), size);
1082
1083 if (constituent == end) {
1084 /*
1085 * The last constituent is expected to be in the
1086 * last fragment.
1087 */
1088 assert(i == fragment_count - 1);
1089 break;
1090 }
1091
1092 CHECK(arch_memory_unprotect(pa_begin, pa_end));
1093 }
1094 }
1095}
1096
1097/**
Jose Marinho09b1db82019-08-08 09:16:59 +01001098 * Updates a VM's page table such that the given set of physical address ranges
1099 * are mapped in the address space at the corresponding address ranges, in the
1100 * mode provided.
1101 *
J-Alves0a83dc22023-05-05 09:50:37 +01001102 * The enum ffa_map_action determines the action taken from a call to the
1103 * function below:
1104 * - If action is MAP_ACTION_CHECK, the page tables will be allocated from the
1105 * mpool but no mappings will actually be updated. This function must always
1106 * be called first with action set to MAP_ACTION_CHECK to check that it will
1107 * succeed before calling ffa_region_group_identity_map with whichever one of
1108 * the remaining actions, to avoid leaving the page table in a half-updated
1109 * state.
1110 * - The action MAP_ACTION_COMMIT allocates the page tables from the mpool, and
1111 * changes the memory mappings.
J-Alvescf6253e2024-01-03 13:48:48 +00001112 * - The action MAP_ACTION_CHECK_PROTECT extends the MAP_ACTION_CHECK with an
1113 * invocation to the monitor to update the security state of the memory,
1114 * to that of the SPMC.
1115 * - The action MAP_ACTION_COMMIT_UNPROTECT extends the MAP_ACTION_COMMIT
1116 * with a call into the monitor, to reset the security state of memory
1117 * that has priorly been mapped with the MAP_ACTION_CHECK_PROTECT action.
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001118 * vm_ptable_defrag should always be called after a series of page table
1119 * updates, whether they succeed or fail.
Jose Marinho09b1db82019-08-08 09:16:59 +01001120 *
J-Alvescf6253e2024-01-03 13:48:48 +00001121 * If all goes well, returns FFA_SUCCESS_32; or FFA_ERROR, with following
1122 * error codes:
1123 * - FFA_INVALID_PARAMETERS: invalid range of memory.
1124 * - FFA_DENIED:
1125 *
Jose Marinho09b1db82019-08-08 09:16:59 +01001126 * made to memory mappings.
1127 */
J-Alvescf6253e2024-01-03 13:48:48 +00001128struct ffa_value ffa_region_group_identity_map(
Andrew Walbranf4b51af2020-02-03 14:44:54 +00001129 struct vm_locked vm_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01001130 struct ffa_memory_region_constituent **fragments,
1131 const uint32_t *fragment_constituent_counts, uint32_t fragment_count,
J-Alvescf6253e2024-01-03 13:48:48 +00001132 uint32_t mode, struct mpool *ppool, enum ffa_map_action action,
1133 bool *memory_protected)
Jose Marinho09b1db82019-08-08 09:16:59 +01001134{
Andrew Walbranca808b12020-05-15 17:22:28 +01001135 uint32_t i;
1136 uint32_t j;
J-Alvescf6253e2024-01-03 13:48:48 +00001137 struct ffa_value ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01001138
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001139 if (vm_locked.vm->el0_partition) {
1140 mode |= MM_MODE_USER | MM_MODE_NG;
1141 }
1142
Andrew Walbranca808b12020-05-15 17:22:28 +01001143 /* Iterate over the memory region constituents within each fragment. */
1144 for (i = 0; i < fragment_count; ++i) {
1145 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
J-Alves063ad832023-10-03 18:05:40 +01001146 struct ffa_memory_region_constituent *constituent =
1147 &fragments[i][j];
1148 size_t size = constituent->page_count * PAGE_SIZE;
Andrew Walbranca808b12020-05-15 17:22:28 +01001149 paddr_t pa_begin =
J-Alves063ad832023-10-03 18:05:40 +01001150 pa_from_ipa(ipa_init(constituent->address));
Andrew Walbranca808b12020-05-15 17:22:28 +01001151 paddr_t pa_end = pa_add(pa_begin, size);
Jens Wiklander4f1880c2022-10-19 17:00:14 +02001152 uint32_t pa_bits =
1153 arch_mm_get_pa_bits(arch_mm_get_pa_range());
Federico Recanati4fd065d2021-12-13 20:06:23 +01001154
1155 /*
1156 * Ensure the requested region falls into system's PA
1157 * range.
1158 */
Jens Wiklander4f1880c2022-10-19 17:00:14 +02001159 if (((pa_addr(pa_begin) >> pa_bits) > 0) ||
1160 ((pa_addr(pa_end) >> pa_bits) > 0)) {
Federico Recanati4fd065d2021-12-13 20:06:23 +01001161 dlog_error("Region is outside of PA Range\n");
J-Alvescf6253e2024-01-03 13:48:48 +00001162 return ffa_error(FFA_INVALID_PARAMETERS);
Federico Recanati4fd065d2021-12-13 20:06:23 +01001163 }
Andrew Walbranca808b12020-05-15 17:22:28 +01001164
J-Alvescf6253e2024-01-03 13:48:48 +00001165 if (action <= MAP_ACTION_CHECK_PROTECT) {
1166 ret = ffa_region_group_check_actions(
1167 vm_locked, pa_begin, pa_end, ppool,
1168 mode, action, memory_protected);
J-Alves063ad832023-10-03 18:05:40 +01001169
1170 if (ret.func == FFA_ERROR_32 &&
1171 ffa_error_code(ret) == FFA_DENIED) {
1172 if (memory_protected != NULL) {
1173 assert(!*memory_protected);
1174 }
1175
1176 ffa_region_group_fragments_revert_protect(
1177 fragments,
1178 fragment_constituent_counts,
1179 i + 1, constituent);
1180 break;
1181 }
J-Alvescf6253e2024-01-03 13:48:48 +00001182 } else if (action >= MAP_ACTION_COMMIT &&
1183 action < MAP_ACTION_MAX) {
1184 ffa_region_group_commit_actions(
1185 vm_locked, pa_begin, pa_end, ppool,
1186 mode, action);
1187 ret = (struct ffa_value){
1188 .func = FFA_SUCCESS_32};
1189 } else {
1190 panic("%s: Unknown ffa_map_action.\n",
1191 __func__);
Andrew Walbranca808b12020-05-15 17:22:28 +01001192 }
Jose Marinho09b1db82019-08-08 09:16:59 +01001193 }
1194 }
1195
J-Alvescf6253e2024-01-03 13:48:48 +00001196 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01001197}
1198
1199/**
1200 * Clears a region of physical memory by overwriting it with zeros. The data is
1201 * flushed from the cache so the memory has been cleared across the system.
1202 */
J-Alves7db32002021-12-14 14:44:50 +00001203static bool clear_memory(paddr_t begin, paddr_t end, struct mpool *ppool,
1204 uint32_t extra_mode_attributes)
Jose Marinho09b1db82019-08-08 09:16:59 +01001205{
1206 /*
Fuad Tabbaed294af2019-12-20 10:43:01 +00001207 * TODO: change this to a CPU local single page window rather than a
Jose Marinho09b1db82019-08-08 09:16:59 +01001208 * global mapping of the whole range. Such an approach will limit
1209 * the changes to stage-1 tables and will allow only local
1210 * invalidation.
1211 */
1212 bool ret;
1213 struct mm_stage1_locked stage1_locked = mm_lock_stage1();
J-Alves7db32002021-12-14 14:44:50 +00001214 void *ptr = mm_identity_map(stage1_locked, begin, end,
1215 MM_MODE_W | (extra_mode_attributes &
1216 plat_ffa_other_world_mode()),
1217 ppool);
Jose Marinho09b1db82019-08-08 09:16:59 +01001218 size_t size = pa_difference(begin, end);
1219
1220 if (!ptr) {
Jose Marinho09b1db82019-08-08 09:16:59 +01001221 goto fail;
1222 }
1223
1224 memset_s(ptr, size, 0, size);
1225 arch_mm_flush_dcache(ptr, size);
1226 mm_unmap(stage1_locked, begin, end, ppool);
1227
1228 ret = true;
1229 goto out;
1230
1231fail:
1232 ret = false;
1233
1234out:
1235 mm_unlock_stage1(&stage1_locked);
1236
1237 return ret;
1238}
1239
1240/**
1241 * Clears a region of physical memory by overwriting it with zeros. The data is
1242 * flushed from the cache so the memory has been cleared across the system.
1243 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001244static bool ffa_clear_memory_constituents(
J-Alves7db32002021-12-14 14:44:50 +00001245 uint32_t security_state_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +01001246 struct ffa_memory_region_constituent **fragments,
1247 const uint32_t *fragment_constituent_counts, uint32_t fragment_count,
1248 struct mpool *page_pool)
Jose Marinho09b1db82019-08-08 09:16:59 +01001249{
1250 struct mpool local_page_pool;
Andrew Walbranca808b12020-05-15 17:22:28 +01001251 uint32_t i;
Jose Marinho09b1db82019-08-08 09:16:59 +01001252 bool ret = false;
1253
1254 /*
1255 * Create a local pool so any freed memory can't be used by another
1256 * thread. This is to ensure each constituent that is mapped can be
1257 * unmapped again afterwards.
1258 */
Andrew Walbran475c1452020-02-07 13:22:22 +00001259 mpool_init_with_fallback(&local_page_pool, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +01001260
Andrew Walbranca808b12020-05-15 17:22:28 +01001261 /* Iterate over the memory region constituents within each fragment. */
1262 for (i = 0; i < fragment_count; ++i) {
1263 uint32_t j;
Jose Marinho09b1db82019-08-08 09:16:59 +01001264
J-Alves8457f932023-10-11 16:41:45 +01001265 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001266 size_t size = fragments[i][j].page_count * PAGE_SIZE;
1267 paddr_t begin =
1268 pa_from_ipa(ipa_init(fragments[i][j].address));
1269 paddr_t end = pa_add(begin, size);
1270
J-Alves7db32002021-12-14 14:44:50 +00001271 if (!clear_memory(begin, end, &local_page_pool,
1272 security_state_mode)) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001273 /*
1274 * api_clear_memory will defrag on failure, so
1275 * no need to do it here.
1276 */
1277 goto out;
1278 }
Jose Marinho09b1db82019-08-08 09:16:59 +01001279 }
1280 }
1281
Jose Marinho09b1db82019-08-08 09:16:59 +01001282 ret = true;
1283
1284out:
1285 mpool_fini(&local_page_pool);
1286 return ret;
1287}
1288
J-Alves5952d942022-12-22 16:03:00 +00001289static bool is_memory_range_within(ipaddr_t begin, ipaddr_t end,
1290 ipaddr_t in_begin, ipaddr_t in_end)
1291{
1292 return (ipa_addr(begin) >= ipa_addr(in_begin) &&
1293 ipa_addr(begin) < ipa_addr(in_end)) ||
1294 (ipa_addr(end) <= ipa_addr(in_end) &&
1295 ipa_addr(end) > ipa_addr(in_begin));
1296}
1297
1298/**
1299 * Receives a memory range and looks for overlaps with the remainder
1300 * constituents of the memory share/lend/donate operation. Assumes they are
1301 * passed in order to avoid having to loop over all the elements at each call.
1302 * The function only compares the received memory ranges with those that follow
1303 * within the same fragment, and subsequent fragments from the same operation.
1304 */
1305static bool ffa_memory_check_overlap(
1306 struct ffa_memory_region_constituent **fragments,
1307 const uint32_t *fragment_constituent_counts,
1308 const uint32_t fragment_count, const uint32_t current_fragment,
1309 const uint32_t current_constituent)
1310{
1311 uint32_t i = current_fragment;
1312 uint32_t j = current_constituent;
1313 ipaddr_t current_begin = ipa_init(fragments[i][j].address);
1314 const uint32_t current_page_count = fragments[i][j].page_count;
1315 size_t current_size = current_page_count * PAGE_SIZE;
1316 ipaddr_t current_end = ipa_add(current_begin, current_size - 1);
1317
1318 if (current_size == 0 ||
1319 current_size > UINT64_MAX - ipa_addr(current_begin)) {
1320 dlog_verbose("Invalid page count. Addr: %x page_count: %x\n",
1321 current_begin, current_page_count);
1322 return false;
1323 }
1324
1325 for (; i < fragment_count; i++) {
1326 j = (i == current_fragment) ? j + 1 : 0;
1327
1328 for (; j < fragment_constituent_counts[i]; j++) {
1329 ipaddr_t begin = ipa_init(fragments[i][j].address);
1330 const uint32_t page_count = fragments[i][j].page_count;
1331 size_t size = page_count * PAGE_SIZE;
1332 ipaddr_t end = ipa_add(begin, size - 1);
1333
1334 if (size == 0 || size > UINT64_MAX - ipa_addr(begin)) {
1335 dlog_verbose(
1336 "Invalid page count. Addr: %x "
1337 "page_count: %x\n",
1338 begin, page_count);
1339 return false;
1340 }
1341
1342 /*
1343 * Check if current ranges is within begin and end, as
1344 * well as the reverse. This should help optimize the
1345 * loop, and reduce the number of iterations.
1346 */
1347 if (is_memory_range_within(begin, end, current_begin,
1348 current_end) ||
1349 is_memory_range_within(current_begin, current_end,
1350 begin, end)) {
1351 dlog_verbose(
1352 "Overlapping memory ranges: %#x - %#x "
1353 "with %#x - %#x\n",
1354 ipa_addr(begin), ipa_addr(end),
1355 ipa_addr(current_begin),
1356 ipa_addr(current_end));
1357 return true;
1358 }
1359 }
1360 }
1361
1362 return false;
1363}
1364
Jose Marinho09b1db82019-08-08 09:16:59 +01001365/**
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001366 * Validates and prepares memory to be sent from the calling VM to another.
Jose Marinho09b1db82019-08-08 09:16:59 +01001367 *
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001368 * This function requires the calling context to hold the <from> VM lock.
Jose Marinho09b1db82019-08-08 09:16:59 +01001369 *
1370 * Returns:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001371 * In case of error, one of the following values is returned:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001372 * 1) FFA_INVALID_PARAMETERS - The endpoint provided parameters were
Jose Marinho09b1db82019-08-08 09:16:59 +01001373 * erroneous;
Andrew Walbranf07f04d2020-05-01 18:09:00 +01001374 * 2) FFA_NO_MEMORY - Hafnium did not have sufficient memory to complete the
1375 * request.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001376 * 3) FFA_DENIED - The sender doesn't have sufficient access to send the
Andrew Walbrana65a1322020-04-06 19:32:32 +01001377 * memory with the given permissions.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001378 * Success is indicated by FFA_SUCCESS.
Jose Marinho09b1db82019-08-08 09:16:59 +01001379 */
Daniel Boulbya76fd912024-02-22 14:22:15 +00001380static struct ffa_value ffa_send_check_update(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001381 struct vm_locked from_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01001382 struct ffa_memory_region_constituent **fragments,
1383 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
J-Alves8f11cde2022-12-21 16:18:22 +00001384 uint32_t composite_total_page_count, uint32_t share_func,
Daniel Boulbya76fd912024-02-22 14:22:15 +00001385 struct ffa_memory_region *memory_region, struct mpool *page_pool,
1386 uint32_t *orig_from_mode_ret, bool *memory_protected)
Jose Marinho09b1db82019-08-08 09:16:59 +01001387{
Andrew Walbranca808b12020-05-15 17:22:28 +01001388 uint32_t i;
J-Alves8f11cde2022-12-21 16:18:22 +00001389 uint32_t j;
Jose Marinho09b1db82019-08-08 09:16:59 +01001390 uint32_t orig_from_mode;
J-Alves460d36c2023-10-12 17:02:15 +01001391 uint32_t clean_mode;
Jose Marinho09b1db82019-08-08 09:16:59 +01001392 uint32_t from_mode;
Jose Marinho09b1db82019-08-08 09:16:59 +01001393 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001394 struct ffa_value ret;
J-Alves8f11cde2022-12-21 16:18:22 +00001395 uint32_t constituents_total_page_count = 0;
J-Alves460d36c2023-10-12 17:02:15 +01001396 enum ffa_map_action map_action = MAP_ACTION_CHECK;
Daniel Boulbya76fd912024-02-22 14:22:15 +00001397 bool clear = memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR;
Jose Marinho09b1db82019-08-08 09:16:59 +01001398
1399 /*
Andrew Walbrana65a1322020-04-06 19:32:32 +01001400 * Make sure constituents are properly aligned to a 64-bit boundary. If
1401 * not we would get alignment faults trying to read (64-bit) values.
Jose Marinho09b1db82019-08-08 09:16:59 +01001402 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001403 for (i = 0; i < fragment_count; ++i) {
1404 if (!is_aligned(fragments[i], 8)) {
1405 dlog_verbose("Constituents not aligned.\n");
1406 return ffa_error(FFA_INVALID_PARAMETERS);
1407 }
J-Alves8f11cde2022-12-21 16:18:22 +00001408 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
1409 constituents_total_page_count +=
1410 fragments[i][j].page_count;
J-Alves5952d942022-12-22 16:03:00 +00001411 if (ffa_memory_check_overlap(
1412 fragments, fragment_constituent_counts,
1413 fragment_count, i, j)) {
1414 return ffa_error(FFA_INVALID_PARAMETERS);
1415 }
J-Alves8f11cde2022-12-21 16:18:22 +00001416 }
1417 }
1418
1419 if (constituents_total_page_count != composite_total_page_count) {
1420 dlog_verbose(
1421 "Composite page count differs from calculated page "
1422 "count from constituents.\n");
1423 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho09b1db82019-08-08 09:16:59 +01001424 }
1425
1426 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001427 * Check if the state transition is lawful for the sender, ensure that
1428 * all constituents of a memory region being shared are at the same
1429 * state.
Jose Marinho09b1db82019-08-08 09:16:59 +01001430 */
J-Alves460d36c2023-10-12 17:02:15 +01001431 ret = ffa_send_check_transition(
Daniel Boulbya76fd912024-02-22 14:22:15 +00001432 from_locked, share_func, memory_region, &orig_from_mode,
1433 fragments, fragment_constituent_counts, fragment_count,
1434 &from_mode, &map_action);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001435 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001436 dlog_verbose("Invalid transition for send.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +01001437 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01001438 }
1439
Andrew Walbran37c574e2020-06-03 11:45:46 +01001440 if (orig_from_mode_ret != NULL) {
1441 *orig_from_mode_ret = orig_from_mode;
1442 }
1443
Jose Marinho09b1db82019-08-08 09:16:59 +01001444 /*
1445 * Create a local pool so any freed memory can't be used by another
1446 * thread. This is to ensure the original mapping can be restored if the
1447 * clear fails.
1448 */
Andrew Walbran475c1452020-02-07 13:22:22 +00001449 mpool_init_with_fallback(&local_page_pool, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +01001450
1451 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001452 * First reserve all required memory for the new page table entries
1453 * without committing, to make sure the entire operation will succeed
1454 * without exhausting the page pool.
J-Alves460d36c2023-10-12 17:02:15 +01001455 * Provide the map_action as populated by 'ffa_send_check_transition'.
1456 * It may request memory to be protected.
Jose Marinho09b1db82019-08-08 09:16:59 +01001457 */
J-Alvescf6253e2024-01-03 13:48:48 +00001458 ret = ffa_region_group_identity_map(
1459 from_locked, fragments, fragment_constituent_counts,
J-Alves460d36c2023-10-12 17:02:15 +01001460 fragment_count, from_mode, page_pool, map_action,
1461 memory_protected);
J-Alvescf6253e2024-01-03 13:48:48 +00001462 if (ret.func == FFA_ERROR_32) {
Jose Marinho09b1db82019-08-08 09:16:59 +01001463 goto out;
1464 }
1465
1466 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001467 * Update the mapping for the sender. This won't allocate because the
1468 * transaction was already prepared above, but may free pages in the
1469 * case that a whole block is being unmapped that was previously
1470 * partially mapped.
Jose Marinho09b1db82019-08-08 09:16:59 +01001471 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001472 CHECK(ffa_region_group_identity_map(
J-Alvescf6253e2024-01-03 13:48:48 +00001473 from_locked, fragments, fragment_constituent_counts,
1474 fragment_count, from_mode, &local_page_pool,
1475 MAP_ACTION_COMMIT, NULL)
1476 .func == FFA_SUCCESS_32);
Jose Marinho09b1db82019-08-08 09:16:59 +01001477
J-Alves460d36c2023-10-12 17:02:15 +01001478 /*
1479 * If memory has been protected, it is now part of the secure PAS
1480 * (happens for lend/donate from NWd to SWd), and the `orig_from_mode`
1481 * should have the MM_MODE_NS set, as such mask it in `clean_mode` for
1482 * SPM's S1 translation.
1483 * In case memory hasn't been protected, and it is in the non-secure
1484 * PAS (e.g. memory share from NWd to SWd), as such the SPM needs to
1485 * perform a non-secure memory access. In such case `clean_mode` takes
1486 * the same mode as `orig_from_mode`.
1487 */
1488 clean_mode = (memory_protected != NULL && *memory_protected)
1489 ? orig_from_mode & ~plat_ffa_other_world_mode()
1490 : orig_from_mode;
1491
Jose Marinho09b1db82019-08-08 09:16:59 +01001492 /* Clear the memory so no VM or device can see the previous contents. */
J-Alves460d36c2023-10-12 17:02:15 +01001493 if (clear && !ffa_clear_memory_constituents(
1494 clean_mode, fragments, fragment_constituent_counts,
1495 fragment_count, page_pool)) {
1496 map_action = (memory_protected != NULL && *memory_protected)
1497 ? MAP_ACTION_COMMIT_UNPROTECT
1498 : MAP_ACTION_COMMIT;
1499
Jose Marinho09b1db82019-08-08 09:16:59 +01001500 /*
1501 * On failure, roll back by returning memory to the sender. This
1502 * may allocate pages which were previously freed into
1503 * `local_page_pool` by the call above, but will never allocate
1504 * more pages than that so can never fail.
1505 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001506 CHECK(ffa_region_group_identity_map(
J-Alvescf6253e2024-01-03 13:48:48 +00001507 from_locked, fragments,
1508 fragment_constituent_counts, fragment_count,
1509 orig_from_mode, &local_page_pool,
1510 MAP_ACTION_COMMIT, NULL)
1511 .func == FFA_SUCCESS_32);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001512 ret = ffa_error(FFA_NO_MEMORY);
Jose Marinho09b1db82019-08-08 09:16:59 +01001513 goto out;
1514 }
1515
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001516 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001517
1518out:
1519 mpool_fini(&local_page_pool);
1520
1521 /*
1522 * Tidy up the page table by reclaiming failed mappings (if there was an
1523 * error) or merging entries into blocks where possible (on success).
1524 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001525 vm_ptable_defrag(from_locked, page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001526
1527 return ret;
1528}
1529
1530/**
1531 * Validates and maps memory shared from one VM to another.
1532 *
1533 * This function requires the calling context to hold the <to> lock.
1534 *
1535 * Returns:
1536 * In case of error, one of the following values is returned:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001537 * 1) FFA_INVALID_PARAMETERS - The endpoint provided parameters were
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001538 * erroneous;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001539 * 2) FFA_NO_MEMORY - Hafnium did not have sufficient memory to complete
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001540 * the request.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001541 * Success is indicated by FFA_SUCCESS.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001542 */
J-Alvesb5084cf2022-07-06 14:20:12 +01001543struct ffa_value ffa_retrieve_check_update(
J-Alves26483382023-04-20 12:01:49 +01001544 struct vm_locked to_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01001545 struct ffa_memory_region_constituent **fragments,
1546 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
J-Alves26483382023-04-20 12:01:49 +01001547 uint32_t sender_orig_mode, uint32_t share_func, bool clear,
J-Alves460d36c2023-10-12 17:02:15 +01001548 struct mpool *page_pool, uint32_t *response_mode, bool memory_protected)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001549{
Andrew Walbranca808b12020-05-15 17:22:28 +01001550 uint32_t i;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001551 uint32_t to_mode;
1552 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001553 struct ffa_value ret;
J-Alvesfd206052023-05-22 16:45:00 +01001554 enum ffa_map_action map_action = MAP_ACTION_COMMIT;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001555
1556 /*
Andrew Walbranca808b12020-05-15 17:22:28 +01001557 * Make sure constituents are properly aligned to a 64-bit boundary. If
1558 * not we would get alignment faults trying to read (64-bit) values.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001559 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001560 for (i = 0; i < fragment_count; ++i) {
1561 if (!is_aligned(fragments[i], 8)) {
J-Alvesb5084cf2022-07-06 14:20:12 +01001562 dlog_verbose("Fragment not properly aligned.\n");
Andrew Walbranca808b12020-05-15 17:22:28 +01001563 return ffa_error(FFA_INVALID_PARAMETERS);
1564 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001565 }
1566
1567 /*
1568 * Check if the state transition is lawful for the recipient, and ensure
1569 * that all constituents of the memory region being retrieved are at the
1570 * same state.
1571 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001572 ret = ffa_retrieve_check_transition(
1573 to_locked, share_func, fragments, fragment_constituent_counts,
J-Alvesfd206052023-05-22 16:45:00 +01001574 fragment_count, sender_orig_mode, &to_mode, memory_protected,
1575 &map_action);
J-Alves460d36c2023-10-12 17:02:15 +01001576
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001577 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001578 dlog_verbose("Invalid transition for retrieve.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +01001579 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001580 }
1581
1582 /*
1583 * Create a local pool so any freed memory can't be used by another
1584 * thread. This is to ensure the original mapping can be restored if the
1585 * clear fails.
1586 */
1587 mpool_init_with_fallback(&local_page_pool, page_pool);
1588
1589 /*
1590 * First reserve all required memory for the new page table entries in
1591 * the recipient page tables without committing, to make sure the entire
1592 * operation will succeed without exhausting the page pool.
1593 */
J-Alvescf6253e2024-01-03 13:48:48 +00001594 ret = ffa_region_group_identity_map(
1595 to_locked, fragments, fragment_constituent_counts,
1596 fragment_count, to_mode, page_pool, MAP_ACTION_CHECK, NULL);
1597 if (ret.func == FFA_ERROR_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001598 /* TODO: partial defrag of failed range. */
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001599 goto out;
1600 }
1601
1602 /* Clear the memory so no VM or device can see the previous contents. */
J-Alves7db32002021-12-14 14:44:50 +00001603 if (clear &&
J-Alves26483382023-04-20 12:01:49 +01001604 !ffa_clear_memory_constituents(sender_orig_mode, fragments,
1605 fragment_constituent_counts,
1606 fragment_count, page_pool)) {
J-Alvesb5084cf2022-07-06 14:20:12 +01001607 dlog_verbose("Couldn't clear constituents.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001608 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001609 goto out;
1610 }
1611
Jose Marinho09b1db82019-08-08 09:16:59 +01001612 /*
1613 * Complete the transfer by mapping the memory into the recipient. This
1614 * won't allocate because the transaction was already prepared above, so
1615 * it doesn't need to use the `local_page_pool`.
1616 */
J-Alvesfd206052023-05-22 16:45:00 +01001617 CHECK(ffa_region_group_identity_map(
1618 to_locked, fragments, fragment_constituent_counts,
1619 fragment_count, to_mode, page_pool, map_action, NULL)
J-Alvescf6253e2024-01-03 13:48:48 +00001620 .func == FFA_SUCCESS_32);
Jose Marinho09b1db82019-08-08 09:16:59 +01001621
J-Alves460d36c2023-10-12 17:02:15 +01001622 /* Return the mode used in mapping the memory in retriever's PT. */
1623 if (response_mode != NULL) {
1624 *response_mode = to_mode;
1625 }
1626
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001627 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Jose Marinho09b1db82019-08-08 09:16:59 +01001628
1629out:
1630 mpool_fini(&local_page_pool);
1631
1632 /*
Andrew Walbranf07f04d2020-05-01 18:09:00 +01001633 * Tidy up the page table by reclaiming failed mappings (if there was an
1634 * error) or merging entries into blocks where possible (on success).
Jose Marinho09b1db82019-08-08 09:16:59 +01001635 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001636 vm_ptable_defrag(to_locked, page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001637
1638 return ret;
1639}
1640
Andrew Walbran996d1d12020-05-27 14:08:43 +01001641static struct ffa_value ffa_relinquish_check_update(
J-Alves26483382023-04-20 12:01:49 +01001642 struct vm_locked from_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01001643 struct ffa_memory_region_constituent **fragments,
1644 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
1645 struct mpool *page_pool, bool clear)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001646{
1647 uint32_t orig_from_mode;
1648 uint32_t from_mode;
1649 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001650 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001651
Andrew Walbranca808b12020-05-15 17:22:28 +01001652 ret = ffa_relinquish_check_transition(
1653 from_locked, &orig_from_mode, fragments,
1654 fragment_constituent_counts, fragment_count, &from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001655 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001656 dlog_verbose("Invalid transition for relinquish.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +01001657 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001658 }
1659
1660 /*
1661 * Create a local pool so any freed memory can't be used by another
1662 * thread. This is to ensure the original mapping can be restored if the
1663 * clear fails.
1664 */
1665 mpool_init_with_fallback(&local_page_pool, page_pool);
1666
1667 /*
1668 * First reserve all required memory for the new page table entries
1669 * without committing, to make sure the entire operation will succeed
1670 * without exhausting the page pool.
1671 */
J-Alvescf6253e2024-01-03 13:48:48 +00001672 ret = ffa_region_group_identity_map(
1673 from_locked, fragments, fragment_constituent_counts,
1674 fragment_count, from_mode, page_pool, MAP_ACTION_CHECK, NULL);
1675 if (ret.func == FFA_ERROR_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001676 goto out;
1677 }
1678
1679 /*
1680 * Update the mapping for the sender. This won't allocate because the
1681 * transaction was already prepared above, but may free pages in the
1682 * case that a whole block is being unmapped that was previously
1683 * partially mapped.
1684 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001685 CHECK(ffa_region_group_identity_map(
J-Alvescf6253e2024-01-03 13:48:48 +00001686 from_locked, fragments, fragment_constituent_counts,
1687 fragment_count, from_mode, &local_page_pool,
1688 MAP_ACTION_COMMIT, NULL)
1689 .func == FFA_SUCCESS_32);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001690
1691 /* Clear the memory so no VM or device can see the previous contents. */
J-Alves7db32002021-12-14 14:44:50 +00001692 if (clear &&
J-Alves26483382023-04-20 12:01:49 +01001693 !ffa_clear_memory_constituents(orig_from_mode, fragments,
1694 fragment_constituent_counts,
1695 fragment_count, page_pool)) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001696 /*
1697 * On failure, roll back by returning memory to the sender. This
1698 * may allocate pages which were previously freed into
1699 * `local_page_pool` by the call above, but will never allocate
1700 * more pages than that so can never fail.
1701 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001702 CHECK(ffa_region_group_identity_map(
J-Alvescf6253e2024-01-03 13:48:48 +00001703 from_locked, fragments,
1704 fragment_constituent_counts, fragment_count,
1705 orig_from_mode, &local_page_pool,
1706 MAP_ACTION_COMMIT, NULL)
1707 .func == FFA_SUCCESS_32);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001708
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001709 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001710 goto out;
1711 }
1712
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001713 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001714
1715out:
1716 mpool_fini(&local_page_pool);
1717
1718 /*
1719 * Tidy up the page table by reclaiming failed mappings (if there was an
1720 * error) or merging entries into blocks where possible (on success).
1721 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001722 vm_ptable_defrag(from_locked, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +01001723
1724 return ret;
1725}
1726
1727/**
Andrew Walbranca808b12020-05-15 17:22:28 +01001728 * Complete a memory sending operation by checking that it is valid, updating
1729 * the sender page table, and then either marking the share state as having
1730 * completed sending (on success) or freeing it (on failure).
1731 *
1732 * Returns FFA_SUCCESS with the handle encoded, or the relevant FFA_ERROR.
1733 */
J-Alvesfdd29272022-07-19 13:16:31 +01001734struct ffa_value ffa_memory_send_complete(
Andrew Walbranca808b12020-05-15 17:22:28 +01001735 struct vm_locked from_locked, struct share_states_locked share_states,
Andrew Walbran37c574e2020-06-03 11:45:46 +01001736 struct ffa_memory_share_state *share_state, struct mpool *page_pool,
1737 uint32_t *orig_from_mode_ret)
Andrew Walbranca808b12020-05-15 17:22:28 +01001738{
1739 struct ffa_memory_region *memory_region = share_state->memory_region;
J-Alves8f11cde2022-12-21 16:18:22 +00001740 struct ffa_composite_memory_region *composite;
Andrew Walbranca808b12020-05-15 17:22:28 +01001741 struct ffa_value ret;
1742
1743 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +00001744 assert(share_states.share_states != NULL);
J-Alves8f11cde2022-12-21 16:18:22 +00001745 assert(memory_region != NULL);
1746 composite = ffa_memory_region_get_composite(memory_region, 0);
1747 assert(composite != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +01001748
1749 /* Check that state is valid in sender page table and update. */
1750 ret = ffa_send_check_update(
1751 from_locked, share_state->fragments,
1752 share_state->fragment_constituent_counts,
J-Alves8f11cde2022-12-21 16:18:22 +00001753 share_state->fragment_count, composite->page_count,
Daniel Boulbya76fd912024-02-22 14:22:15 +00001754 share_state->share_func, memory_region, page_pool,
J-Alves460d36c2023-10-12 17:02:15 +01001755 orig_from_mode_ret, &share_state->memory_protected);
Andrew Walbranca808b12020-05-15 17:22:28 +01001756 if (ret.func != FFA_SUCCESS_32) {
1757 /*
1758 * Free share state, it failed to send so it can't be retrieved.
1759 */
Karl Meakin4cec5e82023-06-30 16:30:22 +01001760 dlog_verbose("%s: failed to send check update: %s(%s)\n",
1761 __func__, ffa_func_name(ret.func),
1762 ffa_error_name(ffa_error_code(ret)));
Andrew Walbranca808b12020-05-15 17:22:28 +01001763 share_state_free(share_states, share_state, page_pool);
1764 return ret;
1765 }
1766
1767 share_state->sending_complete = true;
Karl Meakin4cec5e82023-06-30 16:30:22 +01001768 dlog_verbose("%s: marked sending complete.\n", __func__);
Andrew Walbranca808b12020-05-15 17:22:28 +01001769
J-Alvesee68c542020-10-29 17:48:20 +00001770 return ffa_mem_success(share_state->memory_region->handle);
Andrew Walbranca808b12020-05-15 17:22:28 +01001771}
1772
1773/**
Daniel Boulby9764ff62024-01-30 17:47:39 +00001774 * Check that the memory attributes match Hafnium expectations.
1775 * Cacheability:
1776 * - Normal Memory as `FFA_MEMORY_CACHE_WRITE_BACK`.
1777 * - Device memory as `FFA_MEMORY_DEV_NGNRNE`.
1778 *
1779 * Shareability:
1780 * - Inner Shareable.
Federico Recanatia98603a2021-12-20 18:04:03 +01001781 */
1782static struct ffa_value ffa_memory_attributes_validate(
J-Alves7a99d0d2023-02-08 13:49:48 +00001783 ffa_memory_attributes_t attributes)
Federico Recanatia98603a2021-12-20 18:04:03 +01001784{
1785 enum ffa_memory_type memory_type;
1786 enum ffa_memory_cacheability cacheability;
1787 enum ffa_memory_shareability shareability;
1788
Karl Meakin84710f32023-10-12 15:14:49 +01001789 memory_type = attributes.type;
Daniel Boulby9764ff62024-01-30 17:47:39 +00001790 cacheability = attributes.cacheability;
1791 if (memory_type == FFA_MEMORY_NORMAL_MEM &&
1792 cacheability != FFA_MEMORY_CACHE_WRITE_BACK) {
1793 dlog_verbose(
1794 "Normal Memory: Invalid cacheability %s, "
1795 "expected %s.\n",
1796 ffa_memory_cacheability_name(cacheability),
1797 ffa_memory_cacheability_name(
1798 FFA_MEMORY_CACHE_WRITE_BACK));
Federico Recanati3d953f32022-02-17 09:31:29 +01001799 return ffa_error(FFA_DENIED);
Federico Recanatia98603a2021-12-20 18:04:03 +01001800 }
Daniel Boulby9764ff62024-01-30 17:47:39 +00001801 if (memory_type == FFA_MEMORY_DEVICE_MEM &&
1802 cacheability != FFA_MEMORY_DEV_NGNRNE) {
1803 dlog_verbose(
1804 "Device Memory: Invalid cacheability %s, "
1805 "expected %s.\n",
1806 ffa_device_memory_cacheability_name(cacheability),
1807 ffa_device_memory_cacheability_name(
1808 FFA_MEMORY_DEV_NGNRNE));
Federico Recanati3d953f32022-02-17 09:31:29 +01001809 return ffa_error(FFA_DENIED);
Federico Recanatia98603a2021-12-20 18:04:03 +01001810 }
1811
Karl Meakin84710f32023-10-12 15:14:49 +01001812 shareability = attributes.shareability;
Federico Recanatia98603a2021-12-20 18:04:03 +01001813 if (shareability != FFA_MEMORY_INNER_SHAREABLE) {
Karl Meakinf98b2aa2023-10-12 16:09:59 +01001814 dlog_verbose("Invalid shareability %s, expected %s.\n",
1815 ffa_memory_shareability_name(shareability),
1816 ffa_memory_shareability_name(
1817 FFA_MEMORY_INNER_SHAREABLE));
Federico Recanati3d953f32022-02-17 09:31:29 +01001818 return ffa_error(FFA_DENIED);
Federico Recanatia98603a2021-12-20 18:04:03 +01001819 }
1820
1821 return (struct ffa_value){.func = FFA_SUCCESS_32};
1822}
1823
1824/**
Andrew Walbrana65a1322020-04-06 19:32:32 +01001825 * Check that the given `memory_region` represents a valid memory send request
1826 * of the given `share_func` type, return the clear flag and permissions via the
1827 * respective output parameters, and update the permissions if necessary.
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001828 *
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001829 * Returns FFA_SUCCESS if the request was valid, or the relevant FFA_ERROR if
Andrew Walbrana65a1322020-04-06 19:32:32 +01001830 * not.
1831 */
J-Alves66652252022-07-06 09:49:51 +01001832struct ffa_value ffa_memory_send_validate(
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001833 struct vm_locked from_locked, struct ffa_memory_region *memory_region,
1834 uint32_t memory_share_length, uint32_t fragment_length,
J-Alves363f5722022-04-25 17:37:37 +01001835 uint32_t share_func)
Andrew Walbrana65a1322020-04-06 19:32:32 +01001836{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001837 struct ffa_composite_memory_region *composite;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001838 struct ffa_memory_access *receiver =
1839 ffa_memory_region_get_receiver(memory_region, 0);
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001840 uint64_t receivers_end;
1841 uint64_t min_length;
Federico Recanati872cd692022-01-05 13:10:10 +01001842 uint32_t composite_memory_region_offset;
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001843 uint32_t constituents_start;
Andrew Walbran130a8ae2020-05-15 16:27:15 +01001844 uint32_t constituents_length;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001845 enum ffa_data_access data_access;
1846 enum ffa_instruction_access instruction_access;
Olivier Deprez4342a3c2022-02-28 09:37:25 +01001847 enum ffa_memory_security security_state;
Karl Meakinf98b2aa2023-10-12 16:09:59 +01001848 enum ffa_memory_type type;
Federico Recanatia98603a2021-12-20 18:04:03 +01001849 struct ffa_value ret;
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001850 const size_t minimum_first_fragment_length =
Daniel Boulby41ef8ba2023-10-13 17:01:22 +01001851 memory_region->receivers_offset +
1852 memory_region->memory_access_desc_size +
1853 sizeof(struct ffa_composite_memory_region);
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001854
1855 if (fragment_length < minimum_first_fragment_length) {
1856 dlog_verbose("Fragment length %u too short (min %u).\n",
1857 (size_t)fragment_length,
1858 minimum_first_fragment_length);
1859 return ffa_error(FFA_INVALID_PARAMETERS);
1860 }
1861
Demi Marie Obenour73a1e942023-02-04 14:09:18 -05001862 static_assert(sizeof(struct ffa_memory_region_constituent) == 16,
1863 "struct ffa_memory_region_constituent must be 16 bytes");
1864 if (!is_aligned(fragment_length,
1865 sizeof(struct ffa_memory_region_constituent)) ||
1866 !is_aligned(memory_share_length,
1867 sizeof(struct ffa_memory_region_constituent))) {
1868 dlog_verbose(
1869 "Fragment length %u or total length %u"
1870 " is not 16-byte aligned.\n",
1871 fragment_length, memory_share_length);
1872 return ffa_error(FFA_INVALID_PARAMETERS);
1873 }
1874
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001875 if (fragment_length > memory_share_length) {
1876 dlog_verbose(
1877 "Fragment length %u greater than total length %u.\n",
1878 (size_t)fragment_length, (size_t)memory_share_length);
1879 return ffa_error(FFA_INVALID_PARAMETERS);
1880 }
Andrew Walbrana65a1322020-04-06 19:32:32 +01001881
J-Alves95df0ef2022-12-07 10:09:48 +00001882 /* The sender must match the caller. */
1883 if ((!vm_id_is_current_world(from_locked.vm->id) &&
1884 vm_id_is_current_world(memory_region->sender)) ||
1885 (vm_id_is_current_world(from_locked.vm->id) &&
1886 memory_region->sender != from_locked.vm->id)) {
1887 dlog_verbose("Invalid memory sender ID.\n");
1888 return ffa_error(FFA_DENIED);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001889 }
1890
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001891 if (memory_region->receiver_count <= 0) {
1892 dlog_verbose("No receivers!\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001893 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001894 }
1895
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001896 /*
1897 * Ensure that the composite header is within the memory bounds and
1898 * doesn't overlap the first part of the message. Cast to uint64_t
1899 * to prevent overflow.
1900 */
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001901 receivers_end = ((uint64_t)memory_region->memory_access_desc_size *
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001902 (uint64_t)memory_region->receiver_count) +
Daniel Boulby41ef8ba2023-10-13 17:01:22 +01001903 memory_region->receivers_offset;
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001904 min_length = receivers_end +
1905 sizeof(struct ffa_composite_memory_region) +
1906 sizeof(struct ffa_memory_region_constituent);
1907 if (min_length > memory_share_length) {
1908 dlog_verbose("Share too short: got %u but minimum is %u.\n",
1909 (size_t)memory_share_length, (size_t)min_length);
1910 return ffa_error(FFA_INVALID_PARAMETERS);
1911 }
1912
1913 composite_memory_region_offset =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001914 receiver->composite_memory_region_offset;
Andrew Walbrana65a1322020-04-06 19:32:32 +01001915
1916 /*
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001917 * Check that the composite memory region descriptor is after the access
1918 * descriptors, is at least 16-byte aligned, and fits in the first
1919 * fragment.
Andrew Walbrana65a1322020-04-06 19:32:32 +01001920 */
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001921 if ((composite_memory_region_offset < receivers_end) ||
1922 (composite_memory_region_offset % 16 != 0) ||
1923 (composite_memory_region_offset >
1924 fragment_length - sizeof(struct ffa_composite_memory_region))) {
1925 dlog_verbose(
1926 "Invalid composite memory region descriptor offset "
1927 "%u.\n",
1928 (size_t)composite_memory_region_offset);
1929 return ffa_error(FFA_INVALID_PARAMETERS);
1930 }
1931
1932 /*
1933 * Compute the start of the constituent regions. Already checked
1934 * to be not more than fragment_length and thus not more than
1935 * memory_share_length.
1936 */
1937 constituents_start = composite_memory_region_offset +
1938 sizeof(struct ffa_composite_memory_region);
1939 constituents_length = memory_share_length - constituents_start;
1940
1941 /*
1942 * Check that the number of constituents is consistent with the length
1943 * of the constituent region.
1944 */
1945 composite = ffa_memory_region_get_composite(memory_region, 0);
1946 if ((constituents_length %
1947 sizeof(struct ffa_memory_region_constituent) !=
1948 0) ||
1949 ((constituents_length /
1950 sizeof(struct ffa_memory_region_constituent)) !=
1951 composite->constituent_count)) {
1952 dlog_verbose("Invalid length %u or composite offset %u.\n",
1953 (size_t)memory_share_length,
1954 (size_t)composite_memory_region_offset);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001955 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001956 }
Andrew Walbranca808b12020-05-15 17:22:28 +01001957 if (fragment_length < memory_share_length &&
1958 fragment_length < HF_MAILBOX_SIZE) {
1959 dlog_warning(
1960 "Initial fragment length %d smaller than mailbox "
1961 "size.\n",
1962 fragment_length);
1963 }
Andrew Walbrana65a1322020-04-06 19:32:32 +01001964
Andrew Walbrana65a1322020-04-06 19:32:32 +01001965 /*
1966 * Clear is not allowed for memory sharing, as the sender still has
1967 * access to the memory.
1968 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001969 if ((memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR) &&
1970 share_func == FFA_MEM_SHARE_32) {
Andrew Walbrana65a1322020-04-06 19:32:32 +01001971 dlog_verbose("Memory can't be cleared while being shared.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001972 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001973 }
1974
1975 /* No other flags are allowed/supported here. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001976 if (memory_region->flags & ~FFA_MEMORY_REGION_FLAG_CLEAR) {
Andrew Walbrana65a1322020-04-06 19:32:32 +01001977 dlog_verbose("Invalid flags %#x.\n", memory_region->flags);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001978 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001979 }
1980
J-Alves363f5722022-04-25 17:37:37 +01001981 /* Check that the permissions are valid, for each specified receiver. */
1982 for (uint32_t i = 0U; i < memory_region->receiver_count; i++) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001983 struct ffa_memory_region_attributes receiver_permissions;
1984
1985 receiver = ffa_memory_region_get_receiver(memory_region, i);
1986 assert(receiver != NULL);
1987 receiver_permissions = receiver->receiver_permissions;
J-Alves363f5722022-04-25 17:37:37 +01001988 ffa_memory_access_permissions_t permissions =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001989 receiver_permissions.permissions;
1990 ffa_id_t receiver_id = receiver_permissions.receiver;
J-Alves363f5722022-04-25 17:37:37 +01001991
1992 if (memory_region->sender == receiver_id) {
1993 dlog_verbose("Can't share memory with itself.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001994 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001995 }
Federico Recanati85090c42021-12-15 13:17:54 +01001996
J-Alves363f5722022-04-25 17:37:37 +01001997 for (uint32_t j = i + 1; j < memory_region->receiver_count;
1998 j++) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001999 struct ffa_memory_access *other_receiver =
2000 ffa_memory_region_get_receiver(memory_region,
2001 j);
2002 assert(other_receiver != NULL);
2003
J-Alves363f5722022-04-25 17:37:37 +01002004 if (receiver_id ==
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002005 other_receiver->receiver_permissions.receiver) {
J-Alves363f5722022-04-25 17:37:37 +01002006 dlog_verbose(
2007 "Repeated receiver(%x) in memory send "
2008 "operation.\n",
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002009 other_receiver->receiver_permissions
2010 .receiver);
J-Alves363f5722022-04-25 17:37:37 +01002011 return ffa_error(FFA_INVALID_PARAMETERS);
2012 }
2013 }
2014
2015 if (composite_memory_region_offset !=
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002016 receiver->composite_memory_region_offset) {
J-Alves363f5722022-04-25 17:37:37 +01002017 dlog_verbose(
2018 "All ffa_memory_access should point to the "
2019 "same composite memory region offset.\n");
2020 return ffa_error(FFA_INVALID_PARAMETERS);
2021 }
2022
Karl Meakin84710f32023-10-12 15:14:49 +01002023 data_access = permissions.data_access;
2024 instruction_access = permissions.instruction_access;
J-Alves363f5722022-04-25 17:37:37 +01002025 if (data_access == FFA_DATA_ACCESS_RESERVED ||
2026 instruction_access == FFA_INSTRUCTION_ACCESS_RESERVED) {
2027 dlog_verbose(
2028 "Reserved value for receiver permissions "
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002029 "(data_access = %s, instruction_access = %s)\n",
2030 ffa_data_access_name(data_access),
2031 ffa_instruction_access_name(
2032 instruction_access));
J-Alves363f5722022-04-25 17:37:37 +01002033 return ffa_error(FFA_INVALID_PARAMETERS);
2034 }
2035 if (instruction_access !=
2036 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
2037 dlog_verbose(
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002038 "Invalid instruction access permissions %s "
2039 "for sending memory, expected %s.\n",
2040 ffa_instruction_access_name(instruction_access),
2041 ffa_instruction_access_name(
2042 FFA_INSTRUCTION_ACCESS_RESERVED));
J-Alves363f5722022-04-25 17:37:37 +01002043 return ffa_error(FFA_INVALID_PARAMETERS);
2044 }
2045 if (share_func == FFA_MEM_SHARE_32) {
2046 if (data_access == FFA_DATA_ACCESS_NOT_SPECIFIED) {
2047 dlog_verbose(
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002048 "Invalid data access permissions %s "
2049 "for sharing memory, expected %s.\n",
2050 ffa_data_access_name(data_access),
2051 ffa_data_access_name(
2052 FFA_DATA_ACCESS_NOT_SPECIFIED));
J-Alves363f5722022-04-25 17:37:37 +01002053 return ffa_error(FFA_INVALID_PARAMETERS);
2054 }
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002055 /*
2056 * According to section 10.10.3 of the FF-A v1.1 EAC0
2057 * spec, NX is required for share operations (but must
2058 * not be specified by the sender) so set it in the
2059 * copy that we store, ready to be returned to the
2060 * retriever.
2061 */
2062 if (vm_id_is_current_world(receiver_id)) {
Karl Meakin84710f32023-10-12 15:14:49 +01002063 permissions.instruction_access =
2064 FFA_INSTRUCTION_ACCESS_NX;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002065 receiver_permissions.permissions = permissions;
2066 }
J-Alves363f5722022-04-25 17:37:37 +01002067 }
2068 if (share_func == FFA_MEM_LEND_32 &&
2069 data_access == FFA_DATA_ACCESS_NOT_SPECIFIED) {
2070 dlog_verbose(
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002071 "Invalid data access permissions %s for "
2072 "lending memory, expected %s.\n",
2073 ffa_data_access_name(data_access),
2074 ffa_data_access_name(
2075 FFA_DATA_ACCESS_NOT_SPECIFIED));
J-Alves363f5722022-04-25 17:37:37 +01002076 return ffa_error(FFA_INVALID_PARAMETERS);
2077 }
2078
2079 if (share_func == FFA_MEM_DONATE_32 &&
2080 data_access != FFA_DATA_ACCESS_NOT_SPECIFIED) {
2081 dlog_verbose(
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002082 "Invalid data access permissions %s for "
2083 "donating memory, expected %s.\n",
2084 ffa_data_access_name(data_access),
2085 ffa_data_access_name(
2086 FFA_DATA_ACCESS_NOT_SPECIFIED));
J-Alves363f5722022-04-25 17:37:37 +01002087 return ffa_error(FFA_INVALID_PARAMETERS);
2088 }
Andrew Walbrana65a1322020-04-06 19:32:32 +01002089 }
2090
Olivier Deprez4342a3c2022-02-28 09:37:25 +01002091 /* Memory region attributes NS-Bit MBZ for FFA_MEM_SHARE/LEND/DONATE. */
Karl Meakin84710f32023-10-12 15:14:49 +01002092 security_state = memory_region->attributes.security;
Olivier Deprez4342a3c2022-02-28 09:37:25 +01002093 if (security_state != FFA_MEMORY_SECURITY_UNSPECIFIED) {
2094 dlog_verbose(
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002095 "Invalid security state %s for memory share operation, "
2096 "expected %s.\n",
2097 ffa_memory_security_name(security_state),
2098 ffa_memory_security_name(
2099 FFA_MEMORY_SECURITY_UNSPECIFIED));
Olivier Deprez4342a3c2022-02-28 09:37:25 +01002100 return ffa_error(FFA_INVALID_PARAMETERS);
2101 }
2102
Federico Recanatid937f5e2021-12-20 17:38:23 +01002103 /*
J-Alves807794e2022-06-16 13:42:47 +01002104 * If a memory donate or lend with single borrower, the memory type
2105 * shall not be specified by the sender.
Federico Recanatid937f5e2021-12-20 17:38:23 +01002106 */
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002107 type = memory_region->attributes.type;
J-Alves807794e2022-06-16 13:42:47 +01002108 if (share_func == FFA_MEM_DONATE_32 ||
2109 (share_func == FFA_MEM_LEND_32 &&
2110 memory_region->receiver_count == 1)) {
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002111 if (type != FFA_MEMORY_NOT_SPECIFIED_MEM) {
J-Alves807794e2022-06-16 13:42:47 +01002112 dlog_verbose(
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002113 "Invalid memory type %s for memory share "
2114 "operation, expected %s.\n",
2115 ffa_memory_type_name(type),
2116 ffa_memory_type_name(
2117 FFA_MEMORY_NOT_SPECIFIED_MEM));
J-Alves807794e2022-06-16 13:42:47 +01002118 return ffa_error(FFA_INVALID_PARAMETERS);
2119 }
2120 } else {
2121 /*
2122 * Check that sender's memory attributes match Hafnium
2123 * expectations: Normal Memory, Inner shareable, Write-Back
2124 * Read-Allocate Write-Allocate Cacheable.
2125 */
2126 ret = ffa_memory_attributes_validate(memory_region->attributes);
2127 if (ret.func != FFA_SUCCESS_32) {
2128 return ret;
2129 }
Federico Recanatid937f5e2021-12-20 17:38:23 +01002130 }
2131
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002132 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbrana65a1322020-04-06 19:32:32 +01002133}
2134
2135/**
Andrew Walbranca808b12020-05-15 17:22:28 +01002136 * Gets the share state for continuing an operation to donate, lend or share
2137 * memory, and checks that it is a valid request.
2138 *
2139 * Returns FFA_SUCCESS if the request was valid, or the relevant FFA_ERROR if
2140 * not.
2141 */
J-Alvesfdd29272022-07-19 13:16:31 +01002142struct ffa_value ffa_memory_send_continue_validate(
Andrew Walbranca808b12020-05-15 17:22:28 +01002143 struct share_states_locked share_states, ffa_memory_handle_t handle,
J-Alves19e20cf2023-08-02 12:48:55 +01002144 struct ffa_memory_share_state **share_state_ret, ffa_id_t from_vm_id,
Andrew Walbranca808b12020-05-15 17:22:28 +01002145 struct mpool *page_pool)
2146{
2147 struct ffa_memory_share_state *share_state;
2148 struct ffa_memory_region *memory_region;
2149
Daniel Boulbya2f8c662021-11-26 17:52:53 +00002150 assert(share_state_ret != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +01002151
2152 /*
2153 * Look up the share state by handle and make sure that the VM ID
2154 * matches.
2155 */
Karl Meakin4a2854a2023-06-30 16:26:52 +01002156 share_state = get_share_state(share_states, handle);
J-Alvesb56aac82023-11-10 09:44:43 +00002157 if (share_state == NULL) {
Andrew Walbranca808b12020-05-15 17:22:28 +01002158 dlog_verbose(
2159 "Invalid handle %#x for memory send continuation.\n",
2160 handle);
2161 return ffa_error(FFA_INVALID_PARAMETERS);
2162 }
2163 memory_region = share_state->memory_region;
2164
J-Alvesfdd29272022-07-19 13:16:31 +01002165 if (vm_id_is_current_world(from_vm_id) &&
2166 memory_region->sender != from_vm_id) {
Andrew Walbranca808b12020-05-15 17:22:28 +01002167 dlog_verbose("Invalid sender %d.\n", memory_region->sender);
2168 return ffa_error(FFA_INVALID_PARAMETERS);
2169 }
2170
2171 if (share_state->sending_complete) {
2172 dlog_verbose(
2173 "Sending of memory handle %#x is already complete.\n",
2174 handle);
2175 return ffa_error(FFA_INVALID_PARAMETERS);
2176 }
2177
2178 if (share_state->fragment_count == MAX_FRAGMENTS) {
2179 /*
2180 * Log a warning as this is a sign that MAX_FRAGMENTS should
2181 * probably be increased.
2182 */
2183 dlog_warning(
2184 "Too many fragments for memory share with handle %#x; "
2185 "only %d supported.\n",
2186 handle, MAX_FRAGMENTS);
2187 /* Free share state, as it's not possible to complete it. */
2188 share_state_free(share_states, share_state, page_pool);
2189 return ffa_error(FFA_NO_MEMORY);
2190 }
2191
2192 *share_state_ret = share_state;
2193
2194 return (struct ffa_value){.func = FFA_SUCCESS_32};
2195}
2196
2197/**
J-Alves95df0ef2022-12-07 10:09:48 +00002198 * Checks if there is at least one receiver from the other world.
2199 */
J-Alvesfdd29272022-07-19 13:16:31 +01002200bool memory_region_receivers_from_other_world(
J-Alves95df0ef2022-12-07 10:09:48 +00002201 struct ffa_memory_region *memory_region)
2202{
2203 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002204 struct ffa_memory_access *receiver =
2205 ffa_memory_region_get_receiver(memory_region, i);
2206 assert(receiver != NULL);
2207 ffa_id_t receiver_id = receiver->receiver_permissions.receiver;
2208
2209 if (!vm_id_is_current_world(receiver_id)) {
J-Alves95df0ef2022-12-07 10:09:48 +00002210 return true;
2211 }
2212 }
2213 return false;
2214}
2215
2216/**
J-Alves9da280b2022-12-21 14:55:39 +00002217 * Validates a call to donate, lend or share memory in which Hafnium is the
2218 * designated allocator of the memory handle. In practice, this also means
2219 * Hafnium is responsible for managing the state structures for the transaction.
2220 * If Hafnium is the SPMC, it should allocate the memory handle when either the
2221 * sender is an SP or there is at least one borrower that is an SP.
2222 * If Hafnium is the hypervisor, it should allocate the memory handle when
2223 * operation involves only NWd VMs.
2224 *
2225 * If validation goes well, Hafnium updates the stage-2 page tables of the
2226 * sender. Validation consists of checking if the message length and number of
2227 * memory region constituents match, and if the transition is valid for the
2228 * type of memory sending operation.
Andrew Walbran475c1452020-02-07 13:22:22 +00002229 *
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002230 * Assumes that the caller has already found and locked the sender VM and copied
2231 * the memory region descriptor from the sender's TX buffer to a freshly
2232 * allocated page from Hafnium's internal pool. The caller must have also
2233 * validated that the receiver VM ID is valid.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002234 *
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002235 * This function takes ownership of the `memory_region` passed in and will free
2236 * it when necessary; it must not be freed by the caller.
Jose Marinho09b1db82019-08-08 09:16:59 +01002237 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002238struct ffa_value ffa_memory_send(struct vm_locked from_locked,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002239 struct ffa_memory_region *memory_region,
Andrew Walbran130a8ae2020-05-15 16:27:15 +01002240 uint32_t memory_share_length,
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002241 uint32_t fragment_length, uint32_t share_func,
2242 struct mpool *page_pool)
Jose Marinho09b1db82019-08-08 09:16:59 +01002243{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002244 struct ffa_value ret;
Andrew Walbranca808b12020-05-15 17:22:28 +01002245 struct share_states_locked share_states;
2246 struct ffa_memory_share_state *share_state;
Jose Marinho09b1db82019-08-08 09:16:59 +01002247
2248 /*
Andrew Walbrana65a1322020-04-06 19:32:32 +01002249 * If there is an error validating the `memory_region` then we need to
2250 * free it because we own it but we won't be storing it in a share state
2251 * after all.
Jose Marinho09b1db82019-08-08 09:16:59 +01002252 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002253 ret = ffa_memory_send_validate(from_locked, memory_region,
2254 memory_share_length, fragment_length,
J-Alves363f5722022-04-25 17:37:37 +01002255 share_func);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002256 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002257 mpool_free(page_pool, memory_region);
Andrew Walbrana65a1322020-04-06 19:32:32 +01002258 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01002259 }
2260
Andrew Walbrana65a1322020-04-06 19:32:32 +01002261 /* Set flag for share function, ready to be retrieved later. */
2262 switch (share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002263 case FFA_MEM_SHARE_32:
Andrew Walbrana65a1322020-04-06 19:32:32 +01002264 memory_region->flags |=
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002265 FFA_MEMORY_REGION_TRANSACTION_TYPE_SHARE;
Andrew Walbrana65a1322020-04-06 19:32:32 +01002266 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002267 case FFA_MEM_LEND_32:
2268 memory_region->flags |= FFA_MEMORY_REGION_TRANSACTION_TYPE_LEND;
Andrew Walbrana65a1322020-04-06 19:32:32 +01002269 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002270 case FFA_MEM_DONATE_32:
Andrew Walbrana65a1322020-04-06 19:32:32 +01002271 memory_region->flags |=
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002272 FFA_MEMORY_REGION_TRANSACTION_TYPE_DONATE;
Andrew Walbrana65a1322020-04-06 19:32:32 +01002273 break;
Jose Marinho09b1db82019-08-08 09:16:59 +01002274 }
2275
Andrew Walbranca808b12020-05-15 17:22:28 +01002276 share_states = share_states_lock();
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002277 /*
2278 * Allocate a share state before updating the page table. Otherwise if
2279 * updating the page table succeeded but allocating the share state
2280 * failed then it would leave the memory in a state where nobody could
2281 * get it back.
2282 */
Karl Meakin52cdfe72023-06-30 14:49:10 +01002283 share_state = allocate_share_state(share_states, share_func,
2284 memory_region, fragment_length,
2285 FFA_MEMORY_HANDLE_INVALID);
J-Alvesb56aac82023-11-10 09:44:43 +00002286 if (share_state == NULL) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002287 dlog_verbose("Failed to allocate share state.\n");
2288 mpool_free(page_pool, memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +01002289 ret = ffa_error(FFA_NO_MEMORY);
2290 goto out;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002291 }
2292
Andrew Walbranca808b12020-05-15 17:22:28 +01002293 if (fragment_length == memory_share_length) {
2294 /* No more fragments to come, everything fit in one message. */
J-Alves2a0d2882020-10-29 14:49:50 +00002295 ret = ffa_memory_send_complete(
2296 from_locked, share_states, share_state, page_pool,
2297 &(share_state->sender_orig_mode));
Andrew Walbranca808b12020-05-15 17:22:28 +01002298 } else {
J-Alvesfdd29272022-07-19 13:16:31 +01002299 /*
2300 * Use sender ID from 'memory_region' assuming
2301 * that at this point it has been validated:
2302 * - MBZ at virtual FF-A instance.
2303 */
J-Alves19e20cf2023-08-02 12:48:55 +01002304 ffa_id_t sender_to_ret =
J-Alvesfdd29272022-07-19 13:16:31 +01002305 (from_locked.vm->id == HF_OTHER_WORLD_ID)
2306 ? memory_region->sender
2307 : 0;
Andrew Walbranca808b12020-05-15 17:22:28 +01002308 ret = (struct ffa_value){
2309 .func = FFA_MEM_FRAG_RX_32,
J-Alvesee68c542020-10-29 17:48:20 +00002310 .arg1 = (uint32_t)memory_region->handle,
2311 .arg2 = (uint32_t)(memory_region->handle >> 32),
J-Alvesfdd29272022-07-19 13:16:31 +01002312 .arg3 = fragment_length,
2313 .arg4 = (uint32_t)(sender_to_ret & 0xffff) << 16};
Andrew Walbranca808b12020-05-15 17:22:28 +01002314 }
2315
2316out:
2317 share_states_unlock(&share_states);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002318 dump_share_states();
Andrew Walbranca808b12020-05-15 17:22:28 +01002319 return ret;
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002320}
2321
2322/**
J-Alves8505a8a2022-06-15 18:10:18 +01002323 * Continues an operation to donate, lend or share memory to a VM from current
2324 * world. If this is the last fragment then checks that the transition is valid
2325 * for the type of memory sending operation and updates the stage-2 page tables
2326 * of the sender.
Andrew Walbranca808b12020-05-15 17:22:28 +01002327 *
2328 * Assumes that the caller has already found and locked the sender VM and copied
2329 * the memory region descriptor from the sender's TX buffer to a freshly
2330 * allocated page from Hafnium's internal pool.
2331 *
2332 * This function takes ownership of the `fragment` passed in; it must not be
2333 * freed by the caller.
2334 */
2335struct ffa_value ffa_memory_send_continue(struct vm_locked from_locked,
2336 void *fragment,
2337 uint32_t fragment_length,
2338 ffa_memory_handle_t handle,
2339 struct mpool *page_pool)
2340{
2341 struct share_states_locked share_states = share_states_lock();
2342 struct ffa_memory_share_state *share_state;
2343 struct ffa_value ret;
2344 struct ffa_memory_region *memory_region;
2345
Demi Marie Obenour73a1e942023-02-04 14:09:18 -05002346 CHECK(is_aligned(fragment,
2347 alignof(struct ffa_memory_region_constituent)));
2348 if (fragment_length % sizeof(struct ffa_memory_region_constituent) !=
2349 0) {
2350 dlog_verbose("Fragment length %u misaligned.\n",
2351 fragment_length);
2352 ret = ffa_error(FFA_INVALID_PARAMETERS);
2353 goto out_free_fragment;
2354 }
2355
Andrew Walbranca808b12020-05-15 17:22:28 +01002356 ret = ffa_memory_send_continue_validate(share_states, handle,
2357 &share_state,
2358 from_locked.vm->id, page_pool);
2359 if (ret.func != FFA_SUCCESS_32) {
2360 goto out_free_fragment;
2361 }
2362 memory_region = share_state->memory_region;
2363
J-Alves95df0ef2022-12-07 10:09:48 +00002364 if (memory_region_receivers_from_other_world(memory_region)) {
Andrew Walbranca808b12020-05-15 17:22:28 +01002365 dlog_error(
2366 "Got hypervisor-allocated handle for memory send to "
J-Alves8505a8a2022-06-15 18:10:18 +01002367 "other world. This should never happen, and indicates "
2368 "a bug in "
Andrew Walbranca808b12020-05-15 17:22:28 +01002369 "EL3 code.\n");
2370 ret = ffa_error(FFA_INVALID_PARAMETERS);
2371 goto out_free_fragment;
2372 }
2373
2374 /* Add this fragment. */
2375 share_state->fragments[share_state->fragment_count] = fragment;
2376 share_state->fragment_constituent_counts[share_state->fragment_count] =
2377 fragment_length / sizeof(struct ffa_memory_region_constituent);
2378 share_state->fragment_count++;
2379
2380 /* Check whether the memory send operation is now ready to complete. */
2381 if (share_state_sending_complete(share_states, share_state)) {
J-Alves2a0d2882020-10-29 14:49:50 +00002382 ret = ffa_memory_send_complete(
2383 from_locked, share_states, share_state, page_pool,
2384 &(share_state->sender_orig_mode));
Andrew Walbranca808b12020-05-15 17:22:28 +01002385 } else {
2386 ret = (struct ffa_value){
2387 .func = FFA_MEM_FRAG_RX_32,
2388 .arg1 = (uint32_t)handle,
2389 .arg2 = (uint32_t)(handle >> 32),
2390 .arg3 = share_state_next_fragment_offset(share_states,
2391 share_state)};
2392 }
2393 goto out;
2394
2395out_free_fragment:
2396 mpool_free(page_pool, fragment);
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002397
2398out:
Andrew Walbranca808b12020-05-15 17:22:28 +01002399 share_states_unlock(&share_states);
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002400 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002401}
2402
Andrew Walbranca808b12020-05-15 17:22:28 +01002403/** Clean up after the receiver has finished retrieving a memory region. */
2404static void ffa_memory_retrieve_complete(
2405 struct share_states_locked share_states,
2406 struct ffa_memory_share_state *share_state, struct mpool *page_pool)
2407{
2408 if (share_state->share_func == FFA_MEM_DONATE_32) {
2409 /*
2410 * Memory that has been donated can't be relinquished,
2411 * so no need to keep the share state around.
2412 */
2413 share_state_free(share_states, share_state, page_pool);
2414 dlog_verbose("Freed share state for donate.\n");
2415 }
2416}
2417
J-Alves2d8457f2022-10-05 11:06:41 +01002418/**
2419 * Initialises the given memory region descriptor to be used for an
2420 * `FFA_MEM_RETRIEVE_RESP`, including the given constituents for the first
2421 * fragment.
2422 * The memory region descriptor is initialized according to retriever's
2423 * FF-A version.
2424 *
2425 * Returns true on success, or false if the given constituents won't all fit in
2426 * the first fragment.
2427 */
2428static bool ffa_retrieved_memory_region_init(
2429 void *response, uint32_t ffa_version, size_t response_max_size,
J-Alves19e20cf2023-08-02 12:48:55 +01002430 ffa_id_t sender, ffa_memory_attributes_t attributes,
J-Alves2d8457f2022-10-05 11:06:41 +01002431 ffa_memory_region_flags_t flags, ffa_memory_handle_t handle,
Daniel Boulbyde974ca2023-12-12 13:53:31 +00002432 ffa_memory_access_permissions_t permissions,
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002433 struct ffa_memory_access *receivers, size_t receiver_count,
2434 uint32_t memory_access_desc_size, uint32_t page_count,
2435 uint32_t total_constituent_count,
J-Alves2d8457f2022-10-05 11:06:41 +01002436 const struct ffa_memory_region_constituent constituents[],
2437 uint32_t fragment_constituent_count, uint32_t *total_length,
2438 uint32_t *fragment_length)
2439{
2440 struct ffa_composite_memory_region *composite_memory_region;
J-Alves2d8457f2022-10-05 11:06:41 +01002441 uint32_t i;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002442 uint32_t composite_offset;
J-Alves2d8457f2022-10-05 11:06:41 +01002443 uint32_t constituents_offset;
J-Alves2d8457f2022-10-05 11:06:41 +01002444
2445 assert(response != NULL);
2446
2447 if (ffa_version == MAKE_FFA_VERSION(1, 0)) {
2448 struct ffa_memory_region_v1_0 *retrieve_response =
2449 (struct ffa_memory_region_v1_0 *)response;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002450 struct ffa_memory_access_v1_0 *receiver;
J-Alves2d8457f2022-10-05 11:06:41 +01002451
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002452 ffa_memory_region_init_header_v1_0(retrieve_response, sender,
2453 attributes, flags, handle, 0,
2454 receiver_count);
J-Alves2d8457f2022-10-05 11:06:41 +01002455
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002456 receiver = (struct ffa_memory_access_v1_0 *)
2457 retrieve_response->receivers;
J-Alves2d8457f2022-10-05 11:06:41 +01002458 receiver_count = retrieve_response->receiver_count;
2459
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002460 for (uint32_t i = 0; i < receiver_count; i++) {
2461 ffa_id_t receiver_id =
2462 receivers[i].receiver_permissions.receiver;
2463 ffa_memory_receiver_flags_t recv_flags =
2464 receivers[i].receiver_permissions.flags;
2465
2466 /*
2467 * Initialized here as in memory retrieve responses we
2468 * currently expect one borrower to be specified.
2469 */
2470 ffa_memory_access_init_v1_0(
Karl Meakin84710f32023-10-12 15:14:49 +01002471 receiver, receiver_id, permissions.data_access,
2472 permissions.instruction_access, recv_flags);
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002473 }
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002474
2475 composite_offset =
J-Alves2d8457f2022-10-05 11:06:41 +01002476 sizeof(struct ffa_memory_region_v1_0) +
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002477 receiver_count * sizeof(struct ffa_memory_access_v1_0);
2478 receiver->composite_memory_region_offset = composite_offset;
J-Alves2d8457f2022-10-05 11:06:41 +01002479
2480 composite_memory_region = ffa_memory_region_get_composite_v1_0(
2481 retrieve_response, 0);
2482 } else {
J-Alves2d8457f2022-10-05 11:06:41 +01002483 struct ffa_memory_region *retrieve_response =
2484 (struct ffa_memory_region *)response;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002485 struct ffa_memory_access *retrieve_response_receivers;
J-Alves2d8457f2022-10-05 11:06:41 +01002486
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002487 ffa_memory_region_init_header(
2488 retrieve_response, sender, attributes, flags, handle, 0,
2489 receiver_count, memory_access_desc_size);
J-Alves2d8457f2022-10-05 11:06:41 +01002490
2491 /*
2492 * Note that `sizeof(struct_ffa_memory_region)` and
2493 * `sizeof(struct ffa_memory_access)` must both be multiples of
2494 * 16 (as verified by the asserts in `ffa_memory.c`, so it is
2495 * guaranteed that the offset we calculate here is aligned to a
2496 * 64-bit boundary and so 64-bit values can be copied without
2497 * alignment faults.
2498 */
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002499 composite_offset =
Daniel Boulby41ef8ba2023-10-13 17:01:22 +01002500 retrieve_response->receivers_offset +
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002501 (uint32_t)(receiver_count *
2502 retrieve_response->memory_access_desc_size);
J-Alves2d8457f2022-10-05 11:06:41 +01002503
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002504 retrieve_response_receivers =
2505 ffa_memory_region_get_receiver(retrieve_response, 0);
2506 assert(retrieve_response_receivers != NULL);
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002507
2508 /*
2509 * Initialized here as in memory retrieve responses we currently
2510 * expect one borrower to be specified.
2511 */
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002512 memcpy_s(retrieve_response_receivers,
2513 sizeof(struct ffa_memory_access) * receiver_count,
2514 receivers,
2515 sizeof(struct ffa_memory_access) * receiver_count);
2516
2517 retrieve_response_receivers->composite_memory_region_offset =
2518 composite_offset;
2519
J-Alves2d8457f2022-10-05 11:06:41 +01002520 composite_memory_region =
2521 ffa_memory_region_get_composite(retrieve_response, 0);
2522 }
2523
J-Alves2d8457f2022-10-05 11:06:41 +01002524 assert(composite_memory_region != NULL);
2525
J-Alves2d8457f2022-10-05 11:06:41 +01002526 composite_memory_region->page_count = page_count;
2527 composite_memory_region->constituent_count = total_constituent_count;
2528 composite_memory_region->reserved_0 = 0;
2529
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002530 constituents_offset =
2531 composite_offset + sizeof(struct ffa_composite_memory_region);
J-Alves2d8457f2022-10-05 11:06:41 +01002532 if (constituents_offset +
2533 fragment_constituent_count *
2534 sizeof(struct ffa_memory_region_constituent) >
2535 response_max_size) {
2536 return false;
2537 }
2538
2539 for (i = 0; i < fragment_constituent_count; ++i) {
2540 composite_memory_region->constituents[i] = constituents[i];
2541 }
2542
2543 if (total_length != NULL) {
2544 *total_length =
2545 constituents_offset +
2546 composite_memory_region->constituent_count *
2547 sizeof(struct ffa_memory_region_constituent);
2548 }
2549 if (fragment_length != NULL) {
2550 *fragment_length =
2551 constituents_offset +
2552 fragment_constituent_count *
2553 sizeof(struct ffa_memory_region_constituent);
2554 }
2555
2556 return true;
2557}
2558
J-Alves96de29f2022-04-26 16:05:24 +01002559/**
2560 * Validates the retrieved permissions against those specified by the lender
2561 * of memory share operation. Optionally can help set the permissions to be used
2562 * for the S2 mapping, through the `permissions` argument.
J-Alvesdcad8992023-09-15 14:10:35 +01002563 * Returns FFA_SUCCESS if all the fields are valid. FFA_ERROR, with error code:
2564 * - FFA_INVALID_PARAMETERS -> if the fields have invalid values as per the
2565 * specification for each ABI.
2566 * - FFA_DENIED -> if the permissions specified by the retriever are not
2567 * less permissive than those provided by the sender.
J-Alves96de29f2022-04-26 16:05:24 +01002568 */
J-Alvesdcad8992023-09-15 14:10:35 +01002569static struct ffa_value ffa_memory_retrieve_is_memory_access_valid(
2570 uint32_t share_func, enum ffa_data_access sent_data_access,
J-Alves96de29f2022-04-26 16:05:24 +01002571 enum ffa_data_access requested_data_access,
2572 enum ffa_instruction_access sent_instruction_access,
2573 enum ffa_instruction_access requested_instruction_access,
J-Alvesdcad8992023-09-15 14:10:35 +01002574 ffa_memory_access_permissions_t *permissions, bool multiple_borrowers)
J-Alves96de29f2022-04-26 16:05:24 +01002575{
2576 switch (sent_data_access) {
2577 case FFA_DATA_ACCESS_NOT_SPECIFIED:
2578 case FFA_DATA_ACCESS_RW:
2579 if (requested_data_access == FFA_DATA_ACCESS_NOT_SPECIFIED ||
2580 requested_data_access == FFA_DATA_ACCESS_RW) {
2581 if (permissions != NULL) {
Karl Meakin84710f32023-10-12 15:14:49 +01002582 permissions->data_access = FFA_DATA_ACCESS_RW;
J-Alves96de29f2022-04-26 16:05:24 +01002583 }
2584 break;
2585 }
2586 /* Intentional fall-through. */
2587 case FFA_DATA_ACCESS_RO:
2588 if (requested_data_access == FFA_DATA_ACCESS_NOT_SPECIFIED ||
2589 requested_data_access == FFA_DATA_ACCESS_RO) {
2590 if (permissions != NULL) {
Karl Meakin84710f32023-10-12 15:14:49 +01002591 permissions->data_access = FFA_DATA_ACCESS_RO;
J-Alves96de29f2022-04-26 16:05:24 +01002592 }
2593 break;
2594 }
2595 dlog_verbose(
2596 "Invalid data access requested; sender specified "
2597 "permissions %#x but receiver requested %#x.\n",
2598 sent_data_access, requested_data_access);
J-Alvesdcad8992023-09-15 14:10:35 +01002599 return ffa_error(FFA_DENIED);
J-Alves96de29f2022-04-26 16:05:24 +01002600 case FFA_DATA_ACCESS_RESERVED:
2601 panic("Got unexpected FFA_DATA_ACCESS_RESERVED. Should be "
2602 "checked before this point.");
2603 }
2604
J-Alvesdcad8992023-09-15 14:10:35 +01002605 /*
2606 * For operations with a single borrower, If it is an FFA_MEMORY_LEND
2607 * or FFA_MEMORY_DONATE the retriever should have specifed the
2608 * instruction permissions it wishes to receive.
2609 */
2610 switch (share_func) {
2611 case FFA_MEM_SHARE_32:
2612 if (requested_instruction_access !=
2613 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
2614 dlog_verbose(
2615 "%s: for share instruction permissions must "
2616 "NOT be specified.\n",
2617 __func__);
2618 return ffa_error(FFA_INVALID_PARAMETERS);
2619 }
2620 break;
2621 case FFA_MEM_LEND_32:
2622 /*
2623 * For operations with multiple borrowers only permit XN
2624 * permissions, and both Sender and borrower should have used
2625 * FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED.
2626 */
2627 if (multiple_borrowers) {
2628 if (requested_instruction_access !=
2629 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
2630 dlog_verbose(
2631 "%s: lend/share/donate with multiple "
2632 "borrowers "
2633 "instruction permissions must NOT be "
2634 "specified.\n",
2635 __func__);
2636 return ffa_error(FFA_INVALID_PARAMETERS);
2637 }
2638 break;
2639 }
2640 /* Fall through if the operation targets a single borrower. */
2641 case FFA_MEM_DONATE_32:
2642 if (!multiple_borrowers &&
2643 requested_instruction_access ==
2644 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
2645 dlog_verbose(
2646 "%s: for lend/donate with single borrower "
2647 "instruction permissions must be speficified "
2648 "by borrower\n",
2649 __func__);
2650 return ffa_error(FFA_INVALID_PARAMETERS);
2651 }
2652 break;
2653 default:
2654 panic("%s: Wrong func id provided.\n", __func__);
2655 }
2656
J-Alves96de29f2022-04-26 16:05:24 +01002657 switch (sent_instruction_access) {
2658 case FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED:
2659 case FFA_INSTRUCTION_ACCESS_X:
J-Alvesdcad8992023-09-15 14:10:35 +01002660 if (requested_instruction_access == FFA_INSTRUCTION_ACCESS_X) {
J-Alves96de29f2022-04-26 16:05:24 +01002661 if (permissions != NULL) {
Karl Meakin84710f32023-10-12 15:14:49 +01002662 permissions->instruction_access =
2663 FFA_INSTRUCTION_ACCESS_X;
J-Alves96de29f2022-04-26 16:05:24 +01002664 }
2665 break;
2666 }
J-Alvesdcad8992023-09-15 14:10:35 +01002667 /*
2668 * Fall through if requested permissions are less
2669 * permissive than those provided by the sender.
2670 */
J-Alves96de29f2022-04-26 16:05:24 +01002671 case FFA_INSTRUCTION_ACCESS_NX:
2672 if (requested_instruction_access ==
2673 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED ||
2674 requested_instruction_access == FFA_INSTRUCTION_ACCESS_NX) {
2675 if (permissions != NULL) {
Karl Meakin84710f32023-10-12 15:14:49 +01002676 permissions->instruction_access =
2677 FFA_INSTRUCTION_ACCESS_NX;
J-Alves96de29f2022-04-26 16:05:24 +01002678 }
2679 break;
2680 }
2681 dlog_verbose(
2682 "Invalid instruction access requested; sender "
2683 "specified permissions %#x but receiver requested "
2684 "%#x.\n",
2685 sent_instruction_access, requested_instruction_access);
J-Alvesdcad8992023-09-15 14:10:35 +01002686 return ffa_error(FFA_DENIED);
J-Alves96de29f2022-04-26 16:05:24 +01002687 case FFA_INSTRUCTION_ACCESS_RESERVED:
2688 panic("Got unexpected FFA_INSTRUCTION_ACCESS_RESERVED. Should "
2689 "be checked before this point.");
2690 }
2691
J-Alvesdcad8992023-09-15 14:10:35 +01002692 return (struct ffa_value){.func = FFA_SUCCESS_32};
J-Alves96de29f2022-04-26 16:05:24 +01002693}
2694
2695/**
2696 * Validate the receivers' permissions in the retrieve request against those
2697 * specified by the lender.
2698 * In the `permissions` argument returns the permissions to set at S2 for the
2699 * caller to the FFA_MEMORY_RETRIEVE_REQ.
J-Alves3456e032023-07-20 12:20:05 +01002700 * The function looks into the flag to bypass multiple borrower checks:
2701 * - If not set returns FFA_SUCCESS if all specified permissions are valid.
2702 * - If set returns FFA_SUCCESS if the descriptor contains the permissions
2703 * to the caller of FFA_MEM_RETRIEVE_REQ and they are valid. Other permissions
2704 * are ignored, if provided.
J-Alves96de29f2022-04-26 16:05:24 +01002705 */
2706static struct ffa_value ffa_memory_retrieve_validate_memory_access_list(
2707 struct ffa_memory_region *memory_region,
J-Alves19e20cf2023-08-02 12:48:55 +01002708 struct ffa_memory_region *retrieve_request, ffa_id_t to_vm_id,
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002709 ffa_memory_access_permissions_t *permissions,
2710 struct ffa_memory_access **receiver_ret, uint32_t func_id)
J-Alves96de29f2022-04-26 16:05:24 +01002711{
2712 uint32_t retrieve_receiver_index;
J-Alves3456e032023-07-20 12:20:05 +01002713 bool bypass_multi_receiver_check =
2714 (retrieve_request->flags &
2715 FFA_MEMORY_REGION_FLAG_BYPASS_BORROWERS_CHECK) != 0U;
J-Alvesdcad8992023-09-15 14:10:35 +01002716 const uint32_t region_receiver_count = memory_region->receiver_count;
2717 struct ffa_value ret;
J-Alves96de29f2022-04-26 16:05:24 +01002718
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002719 assert(receiver_ret != NULL);
J-Alves96de29f2022-04-26 16:05:24 +01002720 assert(permissions != NULL);
2721
Karl Meakin84710f32023-10-12 15:14:49 +01002722 *permissions = (ffa_memory_access_permissions_t){0};
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002723
J-Alves3456e032023-07-20 12:20:05 +01002724 if (!bypass_multi_receiver_check) {
J-Alvesdcad8992023-09-15 14:10:35 +01002725 if (retrieve_request->receiver_count != region_receiver_count) {
J-Alves3456e032023-07-20 12:20:05 +01002726 dlog_verbose(
2727 "Retrieve request should contain same list of "
2728 "borrowers, as specified by the lender.\n");
2729 return ffa_error(FFA_INVALID_PARAMETERS);
2730 }
2731 } else {
2732 if (retrieve_request->receiver_count != 1) {
2733 dlog_verbose(
2734 "Set bypass multiple borrower check, receiver "
2735 "list must be sized 1 (%x)\n",
2736 memory_region->receiver_count);
2737 return ffa_error(FFA_INVALID_PARAMETERS);
2738 }
J-Alves96de29f2022-04-26 16:05:24 +01002739 }
2740
2741 retrieve_receiver_index = retrieve_request->receiver_count;
2742
J-Alves96de29f2022-04-26 16:05:24 +01002743 for (uint32_t i = 0U; i < retrieve_request->receiver_count; i++) {
2744 ffa_memory_access_permissions_t sent_permissions;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002745 struct ffa_memory_access *retrieve_request_receiver =
2746 ffa_memory_region_get_receiver(retrieve_request, i);
2747 assert(retrieve_request_receiver != NULL);
J-Alves96de29f2022-04-26 16:05:24 +01002748 ffa_memory_access_permissions_t requested_permissions =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002749 retrieve_request_receiver->receiver_permissions
2750 .permissions;
J-Alves19e20cf2023-08-02 12:48:55 +01002751 ffa_id_t current_receiver_id =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002752 retrieve_request_receiver->receiver_permissions
2753 .receiver;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002754 struct ffa_memory_access *receiver;
2755 uint32_t mem_region_receiver_index;
2756 bool permissions_RO;
2757 bool clear_memory_flags;
J-Alves96de29f2022-04-26 16:05:24 +01002758 bool found_to_id = current_receiver_id == to_vm_id;
2759
J-Alves3456e032023-07-20 12:20:05 +01002760 if (bypass_multi_receiver_check && !found_to_id) {
2761 dlog_verbose(
2762 "Bypass multiple borrower check for id %x.\n",
2763 current_receiver_id);
2764 continue;
2765 }
2766
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002767 if (retrieve_request_receiver->composite_memory_region_offset !=
2768 0U) {
2769 dlog_verbose(
2770 "Retriever specified address ranges not "
2771 "supported (got offset %d).\n",
2772 retrieve_request_receiver
2773 ->composite_memory_region_offset);
2774 return ffa_error(FFA_INVALID_PARAMETERS);
2775 }
2776
J-Alves96de29f2022-04-26 16:05:24 +01002777 /*
2778 * Find the current receiver in the transaction descriptor from
2779 * sender.
2780 */
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002781 mem_region_receiver_index =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002782 ffa_memory_region_get_receiver_index(
2783 memory_region, current_receiver_id);
J-Alves96de29f2022-04-26 16:05:24 +01002784
2785 if (mem_region_receiver_index ==
2786 memory_region->receiver_count) {
2787 dlog_verbose("%s: receiver %x not found\n", __func__,
2788 current_receiver_id);
2789 return ffa_error(FFA_DENIED);
2790 }
2791
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002792 receiver = ffa_memory_region_get_receiver(
2793 memory_region, mem_region_receiver_index);
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002794 assert(receiver != NULL);
2795
2796 sent_permissions = receiver->receiver_permissions.permissions;
J-Alves96de29f2022-04-26 16:05:24 +01002797
2798 if (found_to_id) {
2799 retrieve_receiver_index = i;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002800
2801 *receiver_ret = receiver;
J-Alves96de29f2022-04-26 16:05:24 +01002802 }
2803
2804 /*
J-Alvesdcad8992023-09-15 14:10:35 +01002805 * Check if retrieve request memory access list is valid:
2806 * - The retrieve request complies with the specification.
2807 * - Permissions are within those specified by the sender.
J-Alves96de29f2022-04-26 16:05:24 +01002808 */
J-Alvesdcad8992023-09-15 14:10:35 +01002809 ret = ffa_memory_retrieve_is_memory_access_valid(
Karl Meakin84710f32023-10-12 15:14:49 +01002810 func_id, sent_permissions.data_access,
2811 requested_permissions.data_access,
2812 sent_permissions.instruction_access,
2813 requested_permissions.instruction_access,
J-Alvesdcad8992023-09-15 14:10:35 +01002814 found_to_id ? permissions : NULL,
2815 region_receiver_count > 1);
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002816
J-Alvesdcad8992023-09-15 14:10:35 +01002817 if (ret.func != FFA_SUCCESS_32) {
2818 return ret;
J-Alves96de29f2022-04-26 16:05:24 +01002819 }
2820
Karl Meakin84710f32023-10-12 15:14:49 +01002821 permissions_RO =
2822 (permissions->data_access == FFA_DATA_ACCESS_RO);
J-Alvese5262372024-03-27 11:02:03 +00002823 clear_memory_flags =
2824 (retrieve_request->flags &
2825 (FFA_MEMORY_REGION_FLAG_CLEAR |
2826 FFA_MEMORY_REGION_FLAG_CLEAR_RELINQUISH)) != 0U;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002827
J-Alves96de29f2022-04-26 16:05:24 +01002828 /*
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002829 * Can't request PM to clear memory if only provided
2830 * with RO permissions.
J-Alves96de29f2022-04-26 16:05:24 +01002831 */
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002832 if (found_to_id && permissions_RO && clear_memory_flags) {
J-Alves96de29f2022-04-26 16:05:24 +01002833 dlog_verbose(
2834 "Receiver has RO permissions can not request "
2835 "clear.\n");
2836 return ffa_error(FFA_DENIED);
2837 }
Daniel Boulbyde974ca2023-12-12 13:53:31 +00002838
2839 /*
2840 * Check the impdef in the retrieve_request matches the value in
2841 * the original memory send.
2842 */
2843 if (ffa_version_from_memory_access_desc_size(
2844 memory_region->memory_access_desc_size) >=
2845 MAKE_FFA_VERSION(1, 2) &&
2846 ffa_version_from_memory_access_desc_size(
2847 retrieve_request->memory_access_desc_size) >=
2848 MAKE_FFA_VERSION(1, 2)) {
Daniel Boulbyde974ca2023-12-12 13:53:31 +00002849 if (receiver->impdef.val[0] !=
2850 retrieve_request_receiver->impdef.val[0] ||
2851 receiver->impdef.val[1] !=
2852 retrieve_request_receiver->impdef.val[1]) {
2853 dlog_verbose(
2854 "Impdef value in memory send does not "
2855 "match retrieve request value "
2856 "send value %#x %#x retrieve request "
2857 "value %#x %#x\n",
2858 receiver->impdef.val[0],
2859 receiver->impdef.val[1],
2860 retrieve_request_receiver->impdef
2861 .val[0],
2862 retrieve_request_receiver->impdef
2863 .val[1]);
2864 return ffa_error(FFA_INVALID_PARAMETERS);
2865 }
2866 }
J-Alves96de29f2022-04-26 16:05:24 +01002867 }
2868
2869 if (retrieve_receiver_index == retrieve_request->receiver_count) {
2870 dlog_verbose(
2871 "Retrieve request does not contain caller's (%x) "
2872 "permissions\n",
2873 to_vm_id);
2874 return ffa_error(FFA_INVALID_PARAMETERS);
2875 }
2876
2877 return (struct ffa_value){.func = FFA_SUCCESS_32};
2878}
2879
J-Alvesa9cd7e32022-07-01 13:49:33 +01002880/*
2881 * According to section 16.4.3 of FF-A v1.1 EAC0 specification, the hypervisor
2882 * may issue an FFA_MEM_RETRIEVE_REQ to obtain the memory region description
2883 * of a pending memory sharing operation whose allocator is the SPM, for
2884 * validation purposes before forwarding an FFA_MEM_RECLAIM call. In doing so
2885 * the memory region descriptor of the retrieve request must be zeroed with the
2886 * exception of the sender ID and handle.
2887 */
J-Alves4f0d9c12024-01-17 17:23:11 +00002888bool is_ffa_hypervisor_retrieve_request(struct ffa_memory_region *request,
2889 struct vm_locked to_locked)
J-Alvesa9cd7e32022-07-01 13:49:33 +01002890{
2891 return to_locked.vm->id == HF_HYPERVISOR_VM_ID &&
Karl Meakin84710f32023-10-12 15:14:49 +01002892 request->attributes.shareability == 0U &&
2893 request->attributes.cacheability == 0U &&
2894 request->attributes.type == 0U &&
2895 request->attributes.security == 0U && request->flags == 0U &&
J-Alvesa9cd7e32022-07-01 13:49:33 +01002896 request->tag == 0U && request->receiver_count == 0U &&
2897 plat_ffa_memory_handle_allocated_by_current_world(
2898 request->handle);
2899}
2900
2901/*
2902 * Helper to reset count of fragments retrieved by the hypervisor.
2903 */
2904static void ffa_memory_retrieve_complete_from_hyp(
2905 struct ffa_memory_share_state *share_state)
2906{
2907 if (share_state->hypervisor_fragment_count ==
2908 share_state->fragment_count) {
2909 share_state->hypervisor_fragment_count = 0;
2910 }
2911}
2912
J-Alves089004f2022-07-13 14:25:44 +01002913/**
J-Alves4f0d9c12024-01-17 17:23:11 +00002914 * Prepares the return of the ffa_value for the memory retrieve response.
2915 */
2916static struct ffa_value ffa_memory_retrieve_resp(uint32_t total_length,
2917 uint32_t fragment_length)
2918{
2919 return (struct ffa_value){.func = FFA_MEM_RETRIEVE_RESP_32,
2920 .arg1 = total_length,
2921 .arg2 = fragment_length};
2922}
2923
2924/**
J-Alves089004f2022-07-13 14:25:44 +01002925 * Validate that the memory region descriptor provided by the borrower on
2926 * FFA_MEM_RETRIEVE_REQ, against saved memory region provided by lender at the
2927 * memory sharing call.
2928 */
2929static struct ffa_value ffa_memory_retrieve_validate(
J-Alves4f0d9c12024-01-17 17:23:11 +00002930 ffa_id_t to_id, struct ffa_memory_region *retrieve_request,
2931 uint32_t retrieve_request_length,
J-Alves089004f2022-07-13 14:25:44 +01002932 struct ffa_memory_region *memory_region, uint32_t *receiver_index,
2933 uint32_t share_func)
2934{
2935 ffa_memory_region_flags_t transaction_type =
2936 retrieve_request->flags &
2937 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK;
Olivier Deprez4342a3c2022-02-28 09:37:25 +01002938 enum ffa_memory_security security_state;
J-Alves4f0d9c12024-01-17 17:23:11 +00002939 const uint64_t memory_access_desc_size =
2940 retrieve_request->memory_access_desc_size;
2941 const uint32_t expected_retrieve_request_length =
2942 retrieve_request->receivers_offset +
2943 (uint32_t)(retrieve_request->receiver_count *
2944 memory_access_desc_size);
J-Alves089004f2022-07-13 14:25:44 +01002945
2946 assert(retrieve_request != NULL);
2947 assert(memory_region != NULL);
2948 assert(receiver_index != NULL);
J-Alves089004f2022-07-13 14:25:44 +01002949
J-Alves4f0d9c12024-01-17 17:23:11 +00002950 if (retrieve_request_length != expected_retrieve_request_length) {
2951 dlog_verbose(
2952 "Invalid length for FFA_MEM_RETRIEVE_REQ, expected %d "
2953 "but was %d.\n",
2954 expected_retrieve_request_length,
2955 retrieve_request_length);
2956 return ffa_error(FFA_INVALID_PARAMETERS);
2957 }
2958
2959 if (retrieve_request->sender != memory_region->sender) {
2960 dlog_verbose(
2961 "Memory with handle %#x not fully sent, can't "
2962 "retrieve.\n",
2963 memory_region->handle);
2964 return ffa_error(FFA_DENIED);
2965 }
2966
2967 /*
2968 * The SPMC can only process retrieve requests to memory share
2969 * operations with one borrower from the other world. It can't
2970 * determine the ID of the NWd VM that invoked the retrieve
2971 * request interface call. It relies on the hypervisor to
2972 * validate the caller's ID against that provided in the
2973 * `receivers` list of the retrieve response.
2974 * In case there is only one borrower from the NWd in the
2975 * transaction descriptor, record that in the `receiver_id` for
2976 * later use, and validate in the retrieve request message.
2977 * This limitation is due to the fact SPMC can't determine the
2978 * index in the memory share structures state to update.
2979 */
2980 if (to_id == HF_HYPERVISOR_VM_ID) {
2981 uint32_t other_world_count = 0;
2982
2983 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
2984 struct ffa_memory_access *receiver =
2985 ffa_memory_region_get_receiver(retrieve_request,
2986 0);
2987 assert(receiver != NULL);
2988
2989 to_id = receiver->receiver_permissions.receiver;
2990
2991 if (!vm_id_is_current_world(to_id)) {
2992 other_world_count++;
2993 }
2994 }
2995
2996 if (other_world_count > 1) {
2997 dlog_verbose(
2998 "Support one receiver from the other "
2999 "world.\n");
3000 return ffa_error(FFA_NOT_SUPPORTED);
3001 }
3002 }
J-Alves089004f2022-07-13 14:25:44 +01003003 /*
3004 * Check that the transaction type expected by the receiver is
3005 * correct, if it has been specified.
3006 */
3007 if (transaction_type !=
3008 FFA_MEMORY_REGION_TRANSACTION_TYPE_UNSPECIFIED &&
3009 transaction_type != (memory_region->flags &
3010 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK)) {
3011 dlog_verbose(
3012 "Incorrect transaction type %#x for "
3013 "FFA_MEM_RETRIEVE_REQ, expected %#x for handle %#x.\n",
3014 transaction_type,
3015 memory_region->flags &
3016 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK,
3017 retrieve_request->handle);
3018 return ffa_error(FFA_INVALID_PARAMETERS);
3019 }
3020
3021 if (retrieve_request->tag != memory_region->tag) {
3022 dlog_verbose(
3023 "Incorrect tag %d for FFA_MEM_RETRIEVE_REQ, expected "
3024 "%d for handle %#x.\n",
3025 retrieve_request->tag, memory_region->tag,
3026 retrieve_request->handle);
3027 return ffa_error(FFA_INVALID_PARAMETERS);
3028 }
3029
J-Alves4f0d9c12024-01-17 17:23:11 +00003030 *receiver_index =
3031 ffa_memory_region_get_receiver_index(memory_region, to_id);
J-Alves089004f2022-07-13 14:25:44 +01003032
3033 if (*receiver_index == memory_region->receiver_count) {
3034 dlog_verbose(
3035 "Incorrect receiver VM ID %d for "
3036 "FFA_MEM_RETRIEVE_REQ, for handle %#x.\n",
J-Alves4f0d9c12024-01-17 17:23:11 +00003037 to_id, memory_region->handle);
J-Alves089004f2022-07-13 14:25:44 +01003038 return ffa_error(FFA_INVALID_PARAMETERS);
3039 }
3040
3041 if ((retrieve_request->flags &
3042 FFA_MEMORY_REGION_ADDRESS_RANGE_HINT_VALID) != 0U) {
3043 dlog_verbose(
3044 "Retriever specified 'address range alignment 'hint' "
3045 "not supported.\n");
3046 return ffa_error(FFA_INVALID_PARAMETERS);
3047 }
3048 if ((retrieve_request->flags &
3049 FFA_MEMORY_REGION_ADDRESS_RANGE_HINT_MASK) != 0) {
3050 dlog_verbose(
3051 "Bits 8-5 must be zero in memory region's flags "
3052 "(address range alignment hint not supported).\n");
3053 return ffa_error(FFA_INVALID_PARAMETERS);
3054 }
3055
3056 if ((retrieve_request->flags & ~0x7FF) != 0U) {
3057 dlog_verbose(
3058 "Bits 31-10 must be zero in memory region's flags.\n");
3059 return ffa_error(FFA_INVALID_PARAMETERS);
3060 }
3061
3062 if (share_func == FFA_MEM_SHARE_32 &&
3063 (retrieve_request->flags &
3064 (FFA_MEMORY_REGION_FLAG_CLEAR |
3065 FFA_MEMORY_REGION_FLAG_CLEAR_RELINQUISH)) != 0U) {
3066 dlog_verbose(
3067 "Memory Share operation can't clean after relinquish "
3068 "memory region.\n");
3069 return ffa_error(FFA_INVALID_PARAMETERS);
3070 }
3071
3072 /*
3073 * If the borrower needs the memory to be cleared before mapping
3074 * to its address space, the sender should have set the flag
3075 * when calling FFA_MEM_LEND/FFA_MEM_DONATE, else return
3076 * FFA_DENIED.
3077 */
3078 if ((retrieve_request->flags & FFA_MEMORY_REGION_FLAG_CLEAR) != 0U &&
3079 (memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR) == 0U) {
3080 dlog_verbose(
3081 "Borrower needs memory cleared. Sender needs to set "
3082 "flag for clearing memory.\n");
3083 return ffa_error(FFA_DENIED);
3084 }
3085
Olivier Deprez4342a3c2022-02-28 09:37:25 +01003086 /* Memory region attributes NS-Bit MBZ for FFA_MEM_RETRIEVE_REQ. */
Karl Meakin84710f32023-10-12 15:14:49 +01003087 security_state = retrieve_request->attributes.security;
Olivier Deprez4342a3c2022-02-28 09:37:25 +01003088 if (security_state != FFA_MEMORY_SECURITY_UNSPECIFIED) {
3089 dlog_verbose(
3090 "Invalid security state for memory retrieve request "
3091 "operation.\n");
3092 return ffa_error(FFA_INVALID_PARAMETERS);
3093 }
3094
J-Alves089004f2022-07-13 14:25:44 +01003095 /*
3096 * If memory type is not specified, bypass validation of memory
3097 * attributes in the retrieve request. The retriever is expecting to
3098 * obtain this information from the SPMC.
3099 */
Karl Meakin84710f32023-10-12 15:14:49 +01003100 if (retrieve_request->attributes.type == FFA_MEMORY_NOT_SPECIFIED_MEM) {
J-Alves089004f2022-07-13 14:25:44 +01003101 return (struct ffa_value){.func = FFA_SUCCESS_32};
3102 }
3103
3104 /*
3105 * Ensure receiver's attributes are compatible with how
3106 * Hafnium maps memory: Normal Memory, Inner shareable,
3107 * Write-Back Read-Allocate Write-Allocate Cacheable.
3108 */
3109 return ffa_memory_attributes_validate(retrieve_request->attributes);
3110}
3111
J-Alves4f0d9c12024-01-17 17:23:11 +00003112static struct ffa_value ffa_partition_retrieve_request(
3113 struct share_states_locked share_states,
3114 struct ffa_memory_share_state *share_state, struct vm_locked to_locked,
3115 struct ffa_memory_region *retrieve_request,
3116 uint32_t retrieve_request_length, struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003117{
Karl Meakin84710f32023-10-12 15:14:49 +01003118 ffa_memory_access_permissions_t permissions = {0};
Olivier Deprez878bd5b2021-04-15 19:05:10 +02003119 uint32_t memory_to_mode;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003120 struct ffa_value ret;
Andrew Walbranca808b12020-05-15 17:22:28 +01003121 struct ffa_composite_memory_region *composite;
3122 uint32_t total_length;
3123 uint32_t fragment_length;
J-Alves19e20cf2023-08-02 12:48:55 +01003124 ffa_id_t receiver_id = to_locked.vm->id;
J-Alves4f0d9c12024-01-17 17:23:11 +00003125 bool is_retrieve_complete = false;
J-Alves4f0d9c12024-01-17 17:23:11 +00003126 const uint64_t memory_access_desc_size =
Daniel Boulbyde974ca2023-12-12 13:53:31 +00003127 retrieve_request->memory_access_desc_size;
J-Alves4f0d9c12024-01-17 17:23:11 +00003128 uint32_t receiver_index;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003129 struct ffa_memory_access *receiver;
J-Alves4f0d9c12024-01-17 17:23:11 +00003130 ffa_memory_handle_t handle = retrieve_request->handle;
Karl Meakin84710f32023-10-12 15:14:49 +01003131 ffa_memory_attributes_t attributes = {0};
J-Alves460d36c2023-10-12 17:02:15 +01003132 uint32_t retrieve_mode = 0;
J-Alves4f0d9c12024-01-17 17:23:11 +00003133 struct ffa_memory_region *memory_region = share_state->memory_region;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003134
J-Alves96de29f2022-04-26 16:05:24 +01003135 if (!share_state->sending_complete) {
3136 dlog_verbose(
3137 "Memory with handle %#x not fully sent, can't "
3138 "retrieve.\n",
3139 handle);
J-Alves4f0d9c12024-01-17 17:23:11 +00003140 return ffa_error(FFA_INVALID_PARAMETERS);
J-Alves96de29f2022-04-26 16:05:24 +01003141 }
3142
J-Alves4f0d9c12024-01-17 17:23:11 +00003143 /*
3144 * Validate retrieve request, according to what was sent by the
3145 * sender. Function will output the `receiver_index` from the
3146 * provided memory region.
3147 */
3148 ret = ffa_memory_retrieve_validate(
3149 receiver_id, retrieve_request, retrieve_request_length,
3150 memory_region, &receiver_index, share_state->share_func);
J-Alves089004f2022-07-13 14:25:44 +01003151
J-Alves4f0d9c12024-01-17 17:23:11 +00003152 if (ret.func != FFA_SUCCESS_32) {
3153 return ret;
J-Alves089004f2022-07-13 14:25:44 +01003154 }
J-Alves96de29f2022-04-26 16:05:24 +01003155
J-Alves4f0d9c12024-01-17 17:23:11 +00003156 /*
3157 * Validate the requested permissions against the sent
3158 * permissions.
3159 * Outputs the permissions to give to retriever at S2
3160 * PTs.
3161 */
3162 ret = ffa_memory_retrieve_validate_memory_access_list(
3163 memory_region, retrieve_request, receiver_id, &permissions,
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003164 &receiver, share_state->share_func);
J-Alves4f0d9c12024-01-17 17:23:11 +00003165 if (ret.func != FFA_SUCCESS_32) {
3166 return ret;
3167 }
3168
3169 memory_to_mode = ffa_memory_permissions_to_mode(
3170 permissions, share_state->sender_orig_mode);
3171
3172 ret = ffa_retrieve_check_update(
3173 to_locked, share_state->fragments,
3174 share_state->fragment_constituent_counts,
3175 share_state->fragment_count, memory_to_mode,
J-Alves460d36c2023-10-12 17:02:15 +01003176 share_state->share_func, false, page_pool, &retrieve_mode,
3177 share_state->memory_protected);
J-Alves4f0d9c12024-01-17 17:23:11 +00003178
3179 if (ret.func != FFA_SUCCESS_32) {
3180 return ret;
3181 }
3182
3183 share_state->retrieved_fragment_count[receiver_index] = 1;
3184
3185 is_retrieve_complete =
3186 share_state->retrieved_fragment_count[receiver_index] ==
3187 share_state->fragment_count;
3188
J-Alvesb5084cf2022-07-06 14:20:12 +01003189 /* VMs acquire the RX buffer from SPMC. */
3190 CHECK(plat_ffa_acquire_receiver_rx(to_locked, &ret));
3191
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003192 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01003193 * Copy response to RX buffer of caller and deliver the message.
3194 * This must be done before the share_state is (possibly) freed.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003195 */
Andrew Walbranca808b12020-05-15 17:22:28 +01003196 composite = ffa_memory_region_get_composite(memory_region, 0);
J-Alves4f0d9c12024-01-17 17:23:11 +00003197
Andrew Walbranca808b12020-05-15 17:22:28 +01003198 /*
J-Alves460d36c2023-10-12 17:02:15 +01003199 * Set the security state in the memory retrieve response attributes
3200 * if specified by the target mode.
3201 */
3202 attributes = plat_ffa_memory_security_mode(memory_region->attributes,
3203 retrieve_mode);
3204
3205 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01003206 * Constituents which we received in the first fragment should
3207 * always fit in the first fragment we are sending, because the
3208 * header is the same size in both cases and we have a fixed
3209 * message buffer size. So `ffa_retrieved_memory_region_init`
3210 * should never fail.
Andrew Walbranca808b12020-05-15 17:22:28 +01003211 */
Olivier Deprez878bd5b2021-04-15 19:05:10 +02003212
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003213 /* Provide the permissions that had been provided. */
3214 receiver->receiver_permissions.permissions = permissions;
3215
3216 /*
3217 * Prepare the memory region descriptor for the retrieve response.
3218 * Provide the pointer to the receiver tracked in the share state
3219 * strucutures.
3220 */
Andrew Walbranca808b12020-05-15 17:22:28 +01003221 CHECK(ffa_retrieved_memory_region_init(
J-Alves2d8457f2022-10-05 11:06:41 +01003222 to_locked.vm->mailbox.recv, to_locked.vm->ffa_version,
Olivier Deprez878bd5b2021-04-15 19:05:10 +02003223 HF_MAILBOX_SIZE, memory_region->sender, attributes,
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003224 memory_region->flags, handle, permissions, receiver, 1,
3225 memory_access_desc_size, composite->page_count,
3226 composite->constituent_count, share_state->fragments[0],
Andrew Walbranca808b12020-05-15 17:22:28 +01003227 share_state->fragment_constituent_counts[0], &total_length,
3228 &fragment_length));
J-Alvesb5084cf2022-07-06 14:20:12 +01003229
J-Alves4f0d9c12024-01-17 17:23:11 +00003230 if (is_retrieve_complete) {
Andrew Walbranca808b12020-05-15 17:22:28 +01003231 ffa_memory_retrieve_complete(share_states, share_state,
3232 page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003233 }
J-Alves4f0d9c12024-01-17 17:23:11 +00003234
3235 return ffa_memory_retrieve_resp(total_length, fragment_length);
3236}
3237
3238static struct ffa_value ffa_hypervisor_retrieve_request(
3239 struct ffa_memory_share_state *share_state, struct vm_locked to_locked,
3240 struct ffa_memory_region *retrieve_request)
3241{
3242 struct ffa_value ret;
3243 struct ffa_composite_memory_region *composite;
3244 uint32_t total_length;
3245 uint32_t fragment_length;
J-Alves4f0d9c12024-01-17 17:23:11 +00003246 ffa_memory_attributes_t attributes;
J-Alves7b6ab612024-01-24 09:54:54 +00003247 uint64_t memory_access_desc_size;
J-Alves4f0d9c12024-01-17 17:23:11 +00003248 struct ffa_memory_region *memory_region;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003249 struct ffa_memory_access *receiver;
J-Alves4f0d9c12024-01-17 17:23:11 +00003250 ffa_memory_handle_t handle = retrieve_request->handle;
3251
J-Alves4f0d9c12024-01-17 17:23:11 +00003252 memory_region = share_state->memory_region;
3253
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003254 assert(to_locked.vm->id == HF_HYPERVISOR_VM_ID);
3255
J-Alves7b6ab612024-01-24 09:54:54 +00003256 switch (to_locked.vm->ffa_version) {
3257 case MAKE_FFA_VERSION(1, 2):
3258 memory_access_desc_size = sizeof(struct ffa_memory_access);
3259 break;
3260 case MAKE_FFA_VERSION(1, 0):
3261 case MAKE_FFA_VERSION(1, 1):
3262 memory_access_desc_size = sizeof(struct ffa_memory_access_v1_0);
3263 break;
3264 default:
3265 panic("version not supported: %x\n", to_locked.vm->ffa_version);
3266 }
3267
J-Alves4f0d9c12024-01-17 17:23:11 +00003268 if (share_state->hypervisor_fragment_count != 0U) {
3269 dlog_verbose(
3270 "Memory with handle %#x already retrieved by "
3271 "the hypervisor.\n",
3272 handle);
3273 return ffa_error(FFA_DENIED);
3274 }
3275
3276 share_state->hypervisor_fragment_count = 1;
3277
3278 ffa_memory_retrieve_complete_from_hyp(share_state);
3279
3280 /* VMs acquire the RX buffer from SPMC. */
3281 CHECK(plat_ffa_acquire_receiver_rx(to_locked, &ret));
3282
3283 /*
3284 * Copy response to RX buffer of caller and deliver the message.
3285 * This must be done before the share_state is (possibly) freed.
3286 */
3287 composite = ffa_memory_region_get_composite(memory_region, 0);
3288
3289 /*
3290 * Constituents which we received in the first fragment should
3291 * always fit in the first fragment we are sending, because the
3292 * header is the same size in both cases and we have a fixed
3293 * message buffer size. So `ffa_retrieved_memory_region_init`
3294 * should never fail.
3295 */
3296
3297 /*
3298 * Set the security state in the memory retrieve response attributes
3299 * if specified by the target mode.
3300 */
3301 attributes = plat_ffa_memory_security_mode(
3302 memory_region->attributes, share_state->sender_orig_mode);
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003303
3304 receiver = ffa_memory_region_get_receiver(memory_region, 0);
3305
J-Alves4f0d9c12024-01-17 17:23:11 +00003306 CHECK(ffa_retrieved_memory_region_init(
3307 to_locked.vm->mailbox.recv, to_locked.vm->ffa_version,
3308 HF_MAILBOX_SIZE, memory_region->sender, attributes,
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003309 memory_region->flags, handle,
3310 receiver->receiver_permissions.permissions, receiver,
3311 memory_region->receiver_count, memory_access_desc_size,
J-Alves4f0d9c12024-01-17 17:23:11 +00003312 composite->page_count, composite->constituent_count,
3313 share_state->fragments[0],
3314 share_state->fragment_constituent_counts[0], &total_length,
3315 &fragment_length));
3316
3317 return ffa_memory_retrieve_resp(total_length, fragment_length);
3318}
3319
3320struct ffa_value ffa_memory_retrieve(struct vm_locked to_locked,
3321 struct ffa_memory_region *retrieve_request,
3322 uint32_t retrieve_request_length,
3323 struct mpool *page_pool)
3324{
3325 ffa_memory_handle_t handle = retrieve_request->handle;
3326 struct share_states_locked share_states;
3327 struct ffa_memory_share_state *share_state;
3328 struct ffa_value ret;
3329
3330 dump_share_states();
3331
3332 share_states = share_states_lock();
3333 share_state = get_share_state(share_states, handle);
3334 if (share_state == NULL) {
3335 dlog_verbose("Invalid handle %#x for FFA_MEM_RETRIEVE_REQ.\n",
3336 handle);
3337 ret = ffa_error(FFA_INVALID_PARAMETERS);
3338 goto out;
3339 }
3340
3341 if (is_ffa_hypervisor_retrieve_request(retrieve_request, to_locked)) {
3342 ret = ffa_hypervisor_retrieve_request(share_state, to_locked,
3343 retrieve_request);
3344 } else {
3345 ret = ffa_partition_retrieve_request(
3346 share_states, share_state, to_locked, retrieve_request,
3347 retrieve_request_length, page_pool);
3348 }
3349
3350 /* Track use of the RX buffer if the handling has succeeded. */
3351 if (ret.func == FFA_MEM_RETRIEVE_RESP_32) {
3352 to_locked.vm->mailbox.recv_func = FFA_MEM_RETRIEVE_RESP_32;
3353 to_locked.vm->mailbox.state = MAILBOX_STATE_FULL;
3354 }
3355
Andrew Walbranca808b12020-05-15 17:22:28 +01003356out:
3357 share_states_unlock(&share_states);
3358 dump_share_states();
3359 return ret;
3360}
3361
J-Alves5da37d92022-10-24 16:33:48 +01003362/**
3363 * Determine expected fragment offset according to the FF-A version of
3364 * the caller.
3365 */
3366static uint32_t ffa_memory_retrieve_expected_offset_per_ffa_version(
3367 struct ffa_memory_region *memory_region,
3368 uint32_t retrieved_constituents_count, uint32_t ffa_version)
3369{
3370 uint32_t expected_fragment_offset;
3371 uint32_t composite_constituents_offset;
3372
Kathleen Capellae4fe2962023-09-01 17:08:47 -04003373 if (ffa_version >= MAKE_FFA_VERSION(1, 1)) {
J-Alves5da37d92022-10-24 16:33:48 +01003374 /*
3375 * Hafnium operates memory regions in FF-A v1.1 format, so we
3376 * can retrieve the constituents offset from descriptor.
3377 */
3378 composite_constituents_offset =
3379 ffa_composite_constituent_offset(memory_region, 0);
3380 } else if (ffa_version == MAKE_FFA_VERSION(1, 0)) {
3381 /*
3382 * If retriever is FF-A v1.0, determine the composite offset
3383 * as it is expected to have been configured in the
3384 * retrieve response.
3385 */
3386 composite_constituents_offset =
3387 sizeof(struct ffa_memory_region_v1_0) +
3388 RECEIVERS_COUNT_IN_RETRIEVE_RESP *
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003389 sizeof(struct ffa_memory_access_v1_0) +
J-Alves5da37d92022-10-24 16:33:48 +01003390 sizeof(struct ffa_composite_memory_region);
3391 } else {
3392 panic("%s received an invalid FF-A version.\n", __func__);
3393 }
3394
3395 expected_fragment_offset =
3396 composite_constituents_offset +
3397 retrieved_constituents_count *
3398 sizeof(struct ffa_memory_region_constituent) -
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003399 (uint32_t)(memory_region->memory_access_desc_size *
3400 (memory_region->receiver_count - 1));
J-Alves5da37d92022-10-24 16:33:48 +01003401
3402 return expected_fragment_offset;
3403}
3404
Andrew Walbranca808b12020-05-15 17:22:28 +01003405struct ffa_value ffa_memory_retrieve_continue(struct vm_locked to_locked,
3406 ffa_memory_handle_t handle,
3407 uint32_t fragment_offset,
J-Alves19e20cf2023-08-02 12:48:55 +01003408 ffa_id_t sender_vm_id,
Andrew Walbranca808b12020-05-15 17:22:28 +01003409 struct mpool *page_pool)
3410{
3411 struct ffa_memory_region *memory_region;
3412 struct share_states_locked share_states;
3413 struct ffa_memory_share_state *share_state;
3414 struct ffa_value ret;
3415 uint32_t fragment_index;
3416 uint32_t retrieved_constituents_count;
3417 uint32_t i;
3418 uint32_t expected_fragment_offset;
3419 uint32_t remaining_constituent_count;
3420 uint32_t fragment_length;
J-Alvesc7484f12022-05-13 12:41:14 +01003421 uint32_t receiver_index;
J-Alves59ed0042022-07-28 18:26:41 +01003422 bool continue_ffa_hyp_mem_retrieve_req;
Andrew Walbranca808b12020-05-15 17:22:28 +01003423
3424 dump_share_states();
3425
3426 share_states = share_states_lock();
Karl Meakin4a2854a2023-06-30 16:26:52 +01003427 share_state = get_share_state(share_states, handle);
J-Alvesb56aac82023-11-10 09:44:43 +00003428 if (share_state == NULL) {
Andrew Walbranca808b12020-05-15 17:22:28 +01003429 dlog_verbose("Invalid handle %#x for FFA_MEM_FRAG_RX.\n",
3430 handle);
3431 ret = ffa_error(FFA_INVALID_PARAMETERS);
3432 goto out;
3433 }
3434
3435 memory_region = share_state->memory_region;
3436 CHECK(memory_region != NULL);
3437
Andrew Walbranca808b12020-05-15 17:22:28 +01003438 if (!share_state->sending_complete) {
3439 dlog_verbose(
3440 "Memory with handle %#x not fully sent, can't "
3441 "retrieve.\n",
3442 handle);
3443 ret = ffa_error(FFA_INVALID_PARAMETERS);
3444 goto out;
3445 }
3446
J-Alves59ed0042022-07-28 18:26:41 +01003447 /*
3448 * If retrieve request from the hypervisor has been initiated in the
3449 * given share_state, continue it, else assume it is a continuation of
3450 * retrieve request from a NWd VM.
3451 */
3452 continue_ffa_hyp_mem_retrieve_req =
3453 (to_locked.vm->id == HF_HYPERVISOR_VM_ID) &&
3454 (share_state->hypervisor_fragment_count != 0U) &&
J-Alves661e1b72023-08-02 13:39:40 +01003455 ffa_is_vm_id(sender_vm_id);
Andrew Walbranca808b12020-05-15 17:22:28 +01003456
J-Alves59ed0042022-07-28 18:26:41 +01003457 if (!continue_ffa_hyp_mem_retrieve_req) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003458 receiver_index = ffa_memory_region_get_receiver_index(
J-Alves59ed0042022-07-28 18:26:41 +01003459 memory_region, to_locked.vm->id);
3460
3461 if (receiver_index == memory_region->receiver_count) {
3462 dlog_verbose(
3463 "Caller of FFA_MEM_FRAG_RX (%x) is not a "
3464 "borrower to memory sharing transaction (%x)\n",
3465 to_locked.vm->id, handle);
3466 ret = ffa_error(FFA_INVALID_PARAMETERS);
3467 goto out;
3468 }
3469
3470 if (share_state->retrieved_fragment_count[receiver_index] ==
3471 0 ||
3472 share_state->retrieved_fragment_count[receiver_index] >=
3473 share_state->fragment_count) {
3474 dlog_verbose(
3475 "Retrieval of memory with handle %#x not yet "
3476 "started or already completed (%d/%d fragments "
3477 "retrieved).\n",
3478 handle,
3479 share_state->retrieved_fragment_count
3480 [receiver_index],
3481 share_state->fragment_count);
3482 ret = ffa_error(FFA_INVALID_PARAMETERS);
3483 goto out;
3484 }
3485
3486 fragment_index =
3487 share_state->retrieved_fragment_count[receiver_index];
3488 } else {
3489 if (share_state->hypervisor_fragment_count == 0 ||
3490 share_state->hypervisor_fragment_count >=
3491 share_state->fragment_count) {
3492 dlog_verbose(
3493 "Retrieve of memory with handle %x not "
3494 "started from hypervisor.\n",
3495 handle);
3496 ret = ffa_error(FFA_INVALID_PARAMETERS);
3497 goto out;
3498 }
3499
3500 if (memory_region->sender != sender_vm_id) {
3501 dlog_verbose(
3502 "Sender ID (%x) is not as expected for memory "
3503 "handle %x\n",
3504 sender_vm_id, handle);
3505 ret = ffa_error(FFA_INVALID_PARAMETERS);
3506 goto out;
3507 }
3508
3509 fragment_index = share_state->hypervisor_fragment_count;
3510
3511 receiver_index = 0;
3512 }
Andrew Walbranca808b12020-05-15 17:22:28 +01003513
3514 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01003515 * Check that the given fragment offset is correct by counting
3516 * how many constituents were in the fragments previously sent.
Andrew Walbranca808b12020-05-15 17:22:28 +01003517 */
3518 retrieved_constituents_count = 0;
3519 for (i = 0; i < fragment_index; ++i) {
3520 retrieved_constituents_count +=
3521 share_state->fragment_constituent_counts[i];
3522 }
J-Alvesc7484f12022-05-13 12:41:14 +01003523
3524 CHECK(memory_region->receiver_count > 0);
3525
Andrew Walbranca808b12020-05-15 17:22:28 +01003526 expected_fragment_offset =
J-Alves5da37d92022-10-24 16:33:48 +01003527 ffa_memory_retrieve_expected_offset_per_ffa_version(
3528 memory_region, retrieved_constituents_count,
3529 to_locked.vm->ffa_version);
3530
Andrew Walbranca808b12020-05-15 17:22:28 +01003531 if (fragment_offset != expected_fragment_offset) {
3532 dlog_verbose("Fragment offset was %d but expected %d.\n",
3533 fragment_offset, expected_fragment_offset);
3534 ret = ffa_error(FFA_INVALID_PARAMETERS);
3535 goto out;
3536 }
3537
J-Alves4f0d9c12024-01-17 17:23:11 +00003538 /*
3539 * When hafnium is the hypervisor, acquire the RX buffer of a VM, that
3540 * is currently ownder by the SPMC.
3541 */
3542 assert(plat_ffa_acquire_receiver_rx(to_locked, &ret));
J-Alves59ed0042022-07-28 18:26:41 +01003543
Andrew Walbranca808b12020-05-15 17:22:28 +01003544 remaining_constituent_count = ffa_memory_fragment_init(
3545 to_locked.vm->mailbox.recv, HF_MAILBOX_SIZE,
3546 share_state->fragments[fragment_index],
3547 share_state->fragment_constituent_counts[fragment_index],
3548 &fragment_length);
3549 CHECK(remaining_constituent_count == 0);
J-Alves674e4de2024-01-17 16:20:32 +00003550
Andrew Walbranca808b12020-05-15 17:22:28 +01003551 to_locked.vm->mailbox.recv_func = FFA_MEM_FRAG_TX_32;
J-Alvese8c8c2b2022-12-16 15:34:48 +00003552 to_locked.vm->mailbox.state = MAILBOX_STATE_FULL;
Andrew Walbranca808b12020-05-15 17:22:28 +01003553
J-Alves59ed0042022-07-28 18:26:41 +01003554 if (!continue_ffa_hyp_mem_retrieve_req) {
3555 share_state->retrieved_fragment_count[receiver_index]++;
3556 if (share_state->retrieved_fragment_count[receiver_index] ==
3557 share_state->fragment_count) {
3558 ffa_memory_retrieve_complete(share_states, share_state,
3559 page_pool);
3560 }
3561 } else {
3562 share_state->hypervisor_fragment_count++;
3563
3564 ffa_memory_retrieve_complete_from_hyp(share_state);
3565 }
Andrew Walbranca808b12020-05-15 17:22:28 +01003566 ret = (struct ffa_value){.func = FFA_MEM_FRAG_TX_32,
3567 .arg1 = (uint32_t)handle,
3568 .arg2 = (uint32_t)(handle >> 32),
3569 .arg3 = fragment_length};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003570
3571out:
3572 share_states_unlock(&share_states);
3573 dump_share_states();
3574 return ret;
3575}
3576
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003577struct ffa_value ffa_memory_relinquish(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003578 struct vm_locked from_locked,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003579 struct ffa_mem_relinquish *relinquish_request, struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003580{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003581 ffa_memory_handle_t handle = relinquish_request->handle;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003582 struct share_states_locked share_states;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003583 struct ffa_memory_share_state *share_state;
3584 struct ffa_memory_region *memory_region;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003585 bool clear;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003586 struct ffa_value ret;
J-Alves8eb19162022-04-28 10:56:48 +01003587 uint32_t receiver_index;
J-Alves3c5b2072022-11-21 12:45:40 +00003588 bool receivers_relinquished_memory;
Karl Meakin84710f32023-10-12 15:14:49 +01003589 ffa_memory_access_permissions_t receiver_permissions = {0};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003590
Andrew Walbrana65a1322020-04-06 19:32:32 +01003591 if (relinquish_request->endpoint_count != 1) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003592 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01003593 "Stream endpoints not supported (got %d "
J-Alves668a86e2023-05-10 11:53:25 +01003594 "endpoints on FFA_MEM_RELINQUISH, expected 1).\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003595 relinquish_request->endpoint_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003596 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003597 }
3598
Andrew Walbrana65a1322020-04-06 19:32:32 +01003599 if (relinquish_request->endpoints[0] != from_locked.vm->id) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003600 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01003601 "VM ID %d in relinquish message doesn't match "
J-Alves668a86e2023-05-10 11:53:25 +01003602 "calling VM ID %d.\n",
Andrew Walbrana65a1322020-04-06 19:32:32 +01003603 relinquish_request->endpoints[0], from_locked.vm->id);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003604 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003605 }
3606
3607 dump_share_states();
3608
3609 share_states = share_states_lock();
Karl Meakin4a2854a2023-06-30 16:26:52 +01003610 share_state = get_share_state(share_states, handle);
J-Alvesb56aac82023-11-10 09:44:43 +00003611 if (share_state == NULL) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003612 dlog_verbose("Invalid handle %#x for FFA_MEM_RELINQUISH.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003613 handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003614 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003615 goto out;
3616 }
3617
Andrew Walbranca808b12020-05-15 17:22:28 +01003618 if (!share_state->sending_complete) {
3619 dlog_verbose(
3620 "Memory with handle %#x not fully sent, can't "
3621 "relinquish.\n",
3622 handle);
3623 ret = ffa_error(FFA_INVALID_PARAMETERS);
3624 goto out;
3625 }
3626
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003627 memory_region = share_state->memory_region;
3628 CHECK(memory_region != NULL);
3629
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003630 receiver_index = ffa_memory_region_get_receiver_index(
3631 memory_region, from_locked.vm->id);
J-Alves8eb19162022-04-28 10:56:48 +01003632
3633 if (receiver_index == memory_region->receiver_count) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003634 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01003635 "VM ID %d tried to relinquish memory region "
J-Alves668a86e2023-05-10 11:53:25 +01003636 "with handle %#x and it is not a valid borrower.\n",
J-Alves8eb19162022-04-28 10:56:48 +01003637 from_locked.vm->id, handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003638 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003639 goto out;
3640 }
3641
J-Alves8eb19162022-04-28 10:56:48 +01003642 if (share_state->retrieved_fragment_count[receiver_index] !=
Andrew Walbranca808b12020-05-15 17:22:28 +01003643 share_state->fragment_count) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003644 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01003645 "Memory with handle %#x not yet fully "
3646 "retrieved, "
J-Alves8eb19162022-04-28 10:56:48 +01003647 "receiver %x can't relinquish.\n",
3648 handle, from_locked.vm->id);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003649 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003650 goto out;
3651 }
3652
J-Alves3c5b2072022-11-21 12:45:40 +00003653 /*
3654 * Either clear if requested in relinquish call, or in a retrieve
3655 * request from one of the borrowers.
3656 */
3657 receivers_relinquished_memory = true;
3658
3659 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
3660 struct ffa_memory_access *receiver =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003661 ffa_memory_region_get_receiver(memory_region, i);
3662 assert(receiver != NULL);
J-Alves3c5b2072022-11-21 12:45:40 +00003663 if (receiver->receiver_permissions.receiver ==
3664 from_locked.vm->id) {
J-Alves639ddfc2023-11-21 14:17:26 +00003665 receiver_permissions =
3666 receiver->receiver_permissions.permissions;
J-Alves3c5b2072022-11-21 12:45:40 +00003667 continue;
3668 }
3669
3670 if (share_state->retrieved_fragment_count[i] != 0U) {
3671 receivers_relinquished_memory = false;
3672 break;
3673 }
3674 }
3675
3676 clear = receivers_relinquished_memory &&
Daniel Boulby2e14ebe2024-01-15 16:21:44 +00003677 ((relinquish_request->flags & FFA_MEMORY_REGION_FLAG_CLEAR) !=
3678 0U);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003679
3680 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01003681 * Clear is not allowed for memory that was shared, as the
3682 * original sender still has access to the memory.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003683 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003684 if (clear && share_state->share_func == FFA_MEM_SHARE_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003685 dlog_verbose("Memory which was shared can't be cleared.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003686 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003687 goto out;
3688 }
3689
J-Alvesb886d492024-04-15 10:55:29 +01003690 if (clear && receiver_permissions.data_access == FFA_DATA_ACCESS_RO) {
J-Alves639ddfc2023-11-21 14:17:26 +00003691 dlog_verbose("%s: RO memory can't use clear memory flag.\n",
3692 __func__);
3693 ret = ffa_error(FFA_DENIED);
3694 goto out;
3695 }
3696
Andrew Walbranca808b12020-05-15 17:22:28 +01003697 ret = ffa_relinquish_check_update(
J-Alves26483382023-04-20 12:01:49 +01003698 from_locked, share_state->fragments,
Andrew Walbranca808b12020-05-15 17:22:28 +01003699 share_state->fragment_constituent_counts,
3700 share_state->fragment_count, page_pool, clear);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003701
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003702 if (ret.func == FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003703 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01003704 * Mark memory handle as not retrieved, so it can be
3705 * reclaimed (or retrieved again).
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003706 */
J-Alves8eb19162022-04-28 10:56:48 +01003707 share_state->retrieved_fragment_count[receiver_index] = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003708 }
3709
3710out:
3711 share_states_unlock(&share_states);
3712 dump_share_states();
3713 return ret;
3714}
3715
3716/**
J-Alvesa9cd7e32022-07-01 13:49:33 +01003717 * Validates that the reclaim transition is allowed for the given
3718 * handle, updates the page table of the reclaiming VM, and frees the
3719 * internal state associated with the handle.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003720 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003721struct ffa_value ffa_memory_reclaim(struct vm_locked to_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01003722 ffa_memory_handle_t handle,
3723 ffa_memory_region_flags_t flags,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003724 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003725{
3726 struct share_states_locked share_states;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003727 struct ffa_memory_share_state *share_state;
3728 struct ffa_memory_region *memory_region;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003729 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003730
3731 dump_share_states();
3732
3733 share_states = share_states_lock();
Karl Meakin52cdfe72023-06-30 14:49:10 +01003734
Karl Meakin4a2854a2023-06-30 16:26:52 +01003735 share_state = get_share_state(share_states, handle);
J-Alvesb56aac82023-11-10 09:44:43 +00003736 if (share_state == NULL) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003737 dlog_verbose("Invalid handle %#x for FFA_MEM_RECLAIM.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003738 handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003739 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003740 goto out;
3741 }
Karl Meakin4a2854a2023-06-30 16:26:52 +01003742 memory_region = share_state->memory_region;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003743
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003744 CHECK(memory_region != NULL);
3745
J-Alvesa9cd7e32022-07-01 13:49:33 +01003746 if (vm_id_is_current_world(to_locked.vm->id) &&
3747 to_locked.vm->id != memory_region->sender) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003748 dlog_verbose(
Olivier Deprezf92e5d42020-11-13 16:00:54 +01003749 "VM %#x attempted to reclaim memory handle %#x "
3750 "originally sent by VM %#x.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003751 to_locked.vm->id, handle, memory_region->sender);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003752 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003753 goto out;
3754 }
3755
Andrew Walbranca808b12020-05-15 17:22:28 +01003756 if (!share_state->sending_complete) {
3757 dlog_verbose(
3758 "Memory with handle %#x not fully sent, can't "
3759 "reclaim.\n",
3760 handle);
3761 ret = ffa_error(FFA_INVALID_PARAMETERS);
3762 goto out;
3763 }
3764
J-Alves752236c2022-04-28 11:07:47 +01003765 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
3766 if (share_state->retrieved_fragment_count[i] != 0) {
3767 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01003768 "Tried to reclaim memory handle %#x "
J-Alves3c5b2072022-11-21 12:45:40 +00003769 "that has not been relinquished by all "
J-Alvesa9cd7e32022-07-01 13:49:33 +01003770 "borrowers(%x).\n",
J-Alves752236c2022-04-28 11:07:47 +01003771 handle,
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003772 ffa_memory_region_get_receiver(memory_region, i)
3773 ->receiver_permissions.receiver);
J-Alves752236c2022-04-28 11:07:47 +01003774 ret = ffa_error(FFA_DENIED);
3775 goto out;
3776 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003777 }
3778
Andrew Walbranca808b12020-05-15 17:22:28 +01003779 ret = ffa_retrieve_check_update(
J-Alves26483382023-04-20 12:01:49 +01003780 to_locked, share_state->fragments,
Andrew Walbranca808b12020-05-15 17:22:28 +01003781 share_state->fragment_constituent_counts,
J-Alves2a0d2882020-10-29 14:49:50 +00003782 share_state->fragment_count, share_state->sender_orig_mode,
J-Alves460d36c2023-10-12 17:02:15 +01003783 FFA_MEM_RECLAIM_32, flags & FFA_MEM_RECLAIM_CLEAR, page_pool,
J-Alvesfd206052023-05-22 16:45:00 +01003784 NULL, share_state->memory_protected);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003785
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003786 if (ret.func == FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003787 share_state_free(share_states, share_state, page_pool);
J-Alves3c5b2072022-11-21 12:45:40 +00003788 dlog_verbose("Freed share state after successful reclaim.\n");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003789 }
3790
3791out:
3792 share_states_unlock(&share_states);
3793 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01003794}