blob: 5169c576de3c870e883fef59bfec0f8bdc9a303e [file] [log] [blame]
Jose Marinho75509b42019-04-09 09:34:59 +01001/*
2 * Copyright 2019 The Hafnium Authors.
3 *
Andrew Walbrane959ec12020-06-17 15:01:09 +01004 * Use of this source code is governed by a BSD-style
5 * license that can be found in the LICENSE file or at
6 * https://opensource.org/licenses/BSD-3-Clause.
Jose Marinho75509b42019-04-09 09:34:59 +01007 */
8
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01009#include "hf/ffa_memory.h"
Andrew Walbran475c1452020-02-07 13:22:22 +000010
J-Alves460d36c2023-10-12 17:02:15 +010011#include <stdint.h>
12
Federico Recanati4fd065d2021-12-13 20:06:23 +010013#include "hf/arch/mm.h"
Olivier Deprez112d2b52020-09-30 07:39:23 +020014#include "hf/arch/other_world.h"
Olivier Deprez55a189e2021-06-09 15:45:27 +020015#include "hf/arch/plat/ffa.h"
Andrew Walbran290b0c92020-02-03 16:37:14 +000016
J-Alves5952d942022-12-22 16:03:00 +000017#include "hf/addr.h"
Jose Marinho75509b42019-04-09 09:34:59 +010018#include "hf/api.h"
Daniel Boulbya2f8c662021-11-26 17:52:53 +000019#include "hf/assert.h"
Jose Marinho09b1db82019-08-08 09:16:59 +010020#include "hf/check.h"
Jose Marinho75509b42019-04-09 09:34:59 +010021#include "hf/dlog.h"
J-Alves3456e032023-07-20 12:20:05 +010022#include "hf/ffa.h"
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010023#include "hf/ffa_internal.h"
J-Alves66652252022-07-06 09:49:51 +010024#include "hf/ffa_memory_internal.h"
J-Alves3456e032023-07-20 12:20:05 +010025#include "hf/ffa_partition_manifest.h"
J-Alves5952d942022-12-22 16:03:00 +000026#include "hf/mm.h"
Andrew Walbran475c1452020-02-07 13:22:22 +000027#include "hf/mpool.h"
J-Alvescf6253e2024-01-03 13:48:48 +000028#include "hf/panic.h"
29#include "hf/plat/memory_protect.h"
Jose Marinho75509b42019-04-09 09:34:59 +010030#include "hf/std.h"
Andrew Scull3c257452019-11-26 13:32:50 +000031#include "hf/vm.h"
Daniel Boulby44e9b3b2024-01-17 12:21:44 +000032#include "hf/vm_ids.h"
Jose Marinho75509b42019-04-09 09:34:59 +010033
J-Alves2d8457f2022-10-05 11:06:41 +010034#include "vmapi/hf/ffa_v1_0.h"
35
J-Alves5da37d92022-10-24 16:33:48 +010036#define RECEIVERS_COUNT_IN_RETRIEVE_RESP 1
37
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000038/**
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010039 * All access to members of a `struct ffa_memory_share_state` must be guarded
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000040 * by this lock.
41 */
42static struct spinlock share_states_lock_instance = SPINLOCK_INIT;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010043static struct ffa_memory_share_state share_states[MAX_MEM_SHARES];
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000044
45/**
J-Alvesed508c82023-05-04 16:09:48 +010046 * Return the offset to the first constituent within the
47 * `ffa_composite_memory_region` for the given receiver from an
48 * `ffa_memory_region`. The caller must check that the receiver_index is within
49 * bounds, and that it has a composite memory region offset.
50 */
51static uint32_t ffa_composite_constituent_offset(
52 struct ffa_memory_region *memory_region, uint32_t receiver_index)
53{
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +000054 struct ffa_memory_access *receiver;
55 uint32_t composite_offset;
J-Alvesed508c82023-05-04 16:09:48 +010056
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +000057 CHECK(receiver_index < memory_region->receiver_count);
58
59 receiver =
60 ffa_memory_region_get_receiver(memory_region, receiver_index);
61 CHECK(receiver != NULL);
62
63 composite_offset = receiver->composite_memory_region_offset;
64
65 CHECK(composite_offset != 0);
66
67 return composite_offset + sizeof(struct ffa_composite_memory_region);
J-Alvesed508c82023-05-04 16:09:48 +010068}
69
70/**
J-Alves917d2f22020-10-30 18:39:30 +000071 * Extracts the index from a memory handle allocated by Hafnium's current world.
72 */
73uint64_t ffa_memory_handle_get_index(ffa_memory_handle_t handle)
74{
75 return handle & ~FFA_MEMORY_HANDLE_ALLOCATOR_MASK;
76}
77
78/**
Karl Meakin52cdfe72023-06-30 14:49:10 +010079 * Initialises the next available `struct ffa_memory_share_state`. If `handle`
80 * is `FFA_MEMORY_HANDLE_INVALID` then allocates an appropriate handle,
81 * otherwise uses the provided handle which is assumed to be globally unique.
Andrew Walbranca808b12020-05-15 17:22:28 +010082 *
Karl Meakin52cdfe72023-06-30 14:49:10 +010083 * Returns a pointer to the allocated `ffa_memory_share_state` on success or
84 * `NULL` if none are available.
Andrew Walbranca808b12020-05-15 17:22:28 +010085 */
Karl Meakin52cdfe72023-06-30 14:49:10 +010086struct ffa_memory_share_state *allocate_share_state(
87 struct share_states_locked share_states, uint32_t share_func,
88 struct ffa_memory_region *memory_region, uint32_t fragment_length,
89 ffa_memory_handle_t handle)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000090{
Daniel Boulbya2f8c662021-11-26 17:52:53 +000091 assert(share_states.share_states != NULL);
92 assert(memory_region != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000093
Karl Meakin52cdfe72023-06-30 14:49:10 +010094 for (uint64_t i = 0; i < MAX_MEM_SHARES; ++i) {
Andrew Walbranca808b12020-05-15 17:22:28 +010095 if (share_states.share_states[i].share_func == 0) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010096 struct ffa_memory_share_state *allocated_state =
Andrew Walbranca808b12020-05-15 17:22:28 +010097 &share_states.share_states[i];
98 struct ffa_composite_memory_region *composite =
99 ffa_memory_region_get_composite(memory_region,
100 0);
101
102 if (handle == FFA_MEMORY_HANDLE_INVALID) {
J-Alvesee68c542020-10-29 17:48:20 +0000103 memory_region->handle =
Olivier Deprez55a189e2021-06-09 15:45:27 +0200104 plat_ffa_memory_handle_make(i);
Andrew Walbranca808b12020-05-15 17:22:28 +0100105 } else {
J-Alvesee68c542020-10-29 17:48:20 +0000106 memory_region->handle = handle;
Andrew Walbranca808b12020-05-15 17:22:28 +0100107 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000108 allocated_state->share_func = share_func;
109 allocated_state->memory_region = memory_region;
Andrew Walbranca808b12020-05-15 17:22:28 +0100110 allocated_state->fragment_count = 1;
111 allocated_state->fragments[0] = composite->constituents;
112 allocated_state->fragment_constituent_counts[0] =
113 (fragment_length -
114 ffa_composite_constituent_offset(memory_region,
115 0)) /
116 sizeof(struct ffa_memory_region_constituent);
117 allocated_state->sending_complete = false;
Karl Meakin52cdfe72023-06-30 14:49:10 +0100118 for (uint32_t j = 0; j < MAX_MEM_SHARE_RECIPIENTS;
119 ++j) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100120 allocated_state->retrieved_fragment_count[j] =
121 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000122 }
Karl Meakin52cdfe72023-06-30 14:49:10 +0100123 return allocated_state;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000124 }
125 }
126
Karl Meakin52cdfe72023-06-30 14:49:10 +0100127 return NULL;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000128}
129
130/** Locks the share states lock. */
131struct share_states_locked share_states_lock(void)
132{
133 sl_lock(&share_states_lock_instance);
134
135 return (struct share_states_locked){.share_states = share_states};
136}
137
138/** Unlocks the share states lock. */
J-Alves66652252022-07-06 09:49:51 +0100139void share_states_unlock(struct share_states_locked *share_states)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000140{
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000141 assert(share_states->share_states != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000142 share_states->share_states = NULL;
143 sl_unlock(&share_states_lock_instance);
144}
145
146/**
Andrew Walbranca808b12020-05-15 17:22:28 +0100147 * If the given handle is a valid handle for an allocated share state then
Karl Meakin4a2854a2023-06-30 16:26:52 +0100148 * returns a pointer to the share state. Otherwise returns NULL.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000149 */
Karl Meakin4a2854a2023-06-30 16:26:52 +0100150struct ffa_memory_share_state *get_share_state(
151 struct share_states_locked share_states, ffa_memory_handle_t handle)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000152{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100153 struct ffa_memory_share_state *share_state;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000154
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000155 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100156
157 /*
158 * First look for a share_state allocated by us, in which case the
159 * handle is based on the index.
160 */
Olivier Deprez55a189e2021-06-09 15:45:27 +0200161 if (plat_ffa_memory_handle_allocated_by_current_world(handle)) {
Karl Meakin4a2854a2023-06-30 16:26:52 +0100162 uint64_t index = ffa_memory_handle_get_index(handle);
163
Andrew Walbranca808b12020-05-15 17:22:28 +0100164 if (index < MAX_MEM_SHARES) {
165 share_state = &share_states.share_states[index];
166 if (share_state->share_func != 0) {
Karl Meakin4a2854a2023-06-30 16:26:52 +0100167 return share_state;
Andrew Walbranca808b12020-05-15 17:22:28 +0100168 }
169 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000170 }
171
Andrew Walbranca808b12020-05-15 17:22:28 +0100172 /* Fall back to a linear scan. */
Karl Meakin4a2854a2023-06-30 16:26:52 +0100173 for (uint64_t index = 0; index < MAX_MEM_SHARES; ++index) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100174 share_state = &share_states.share_states[index];
J-Alvesee68c542020-10-29 17:48:20 +0000175 if (share_state->memory_region != NULL &&
176 share_state->memory_region->handle == handle &&
Andrew Walbranca808b12020-05-15 17:22:28 +0100177 share_state->share_func != 0) {
Karl Meakin4a2854a2023-06-30 16:26:52 +0100178 return share_state;
Andrew Walbranca808b12020-05-15 17:22:28 +0100179 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000180 }
181
Karl Meakin4a2854a2023-06-30 16:26:52 +0100182 return NULL;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000183}
184
185/** Marks a share state as unallocated. */
J-Alvesfdd29272022-07-19 13:16:31 +0100186void share_state_free(struct share_states_locked share_states,
187 struct ffa_memory_share_state *share_state,
188 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000189{
Andrew Walbranca808b12020-05-15 17:22:28 +0100190 uint32_t i;
191
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000192 assert(share_states.share_states != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000193 share_state->share_func = 0;
Andrew Walbranca808b12020-05-15 17:22:28 +0100194 share_state->sending_complete = false;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000195 mpool_free(page_pool, share_state->memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +0100196 /*
197 * First fragment is part of the same page as the `memory_region`, so it
198 * doesn't need to be freed separately.
199 */
200 share_state->fragments[0] = NULL;
201 share_state->fragment_constituent_counts[0] = 0;
202 for (i = 1; i < share_state->fragment_count; ++i) {
203 mpool_free(page_pool, share_state->fragments[i]);
204 share_state->fragments[i] = NULL;
205 share_state->fragment_constituent_counts[i] = 0;
206 }
207 share_state->fragment_count = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000208 share_state->memory_region = NULL;
J-Alvesa9cd7e32022-07-01 13:49:33 +0100209 share_state->hypervisor_fragment_count = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000210}
211
Andrew Walbranca808b12020-05-15 17:22:28 +0100212/** Checks whether the given share state has been fully sent. */
J-Alvesfdd29272022-07-19 13:16:31 +0100213bool share_state_sending_complete(struct share_states_locked share_states,
214 struct ffa_memory_share_state *share_state)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000215{
Andrew Walbranca808b12020-05-15 17:22:28 +0100216 struct ffa_composite_memory_region *composite;
217 uint32_t expected_constituent_count;
218 uint32_t fragment_constituent_count_total = 0;
219 uint32_t i;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000220
Andrew Walbranca808b12020-05-15 17:22:28 +0100221 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000222 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100223
224 /*
225 * Share state must already be valid, or it's not possible to get hold
226 * of it.
227 */
228 CHECK(share_state->memory_region != NULL &&
229 share_state->share_func != 0);
230
231 composite =
232 ffa_memory_region_get_composite(share_state->memory_region, 0);
233 expected_constituent_count = composite->constituent_count;
234 for (i = 0; i < share_state->fragment_count; ++i) {
235 fragment_constituent_count_total +=
236 share_state->fragment_constituent_counts[i];
237 }
238 dlog_verbose(
239 "Checking completion: constituent count %d/%d from %d "
240 "fragments.\n",
241 fragment_constituent_count_total, expected_constituent_count,
242 share_state->fragment_count);
243
244 return fragment_constituent_count_total == expected_constituent_count;
245}
246
247/**
248 * Calculates the offset of the next fragment expected for the given share
249 * state.
250 */
J-Alvesfdd29272022-07-19 13:16:31 +0100251uint32_t share_state_next_fragment_offset(
Andrew Walbranca808b12020-05-15 17:22:28 +0100252 struct share_states_locked share_states,
253 struct ffa_memory_share_state *share_state)
254{
255 uint32_t next_fragment_offset;
256 uint32_t i;
257
258 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000259 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100260
261 next_fragment_offset =
262 ffa_composite_constituent_offset(share_state->memory_region, 0);
263 for (i = 0; i < share_state->fragment_count; ++i) {
264 next_fragment_offset +=
265 share_state->fragment_constituent_counts[i] *
266 sizeof(struct ffa_memory_region_constituent);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000267 }
268
Andrew Walbranca808b12020-05-15 17:22:28 +0100269 return next_fragment_offset;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000270}
271
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100272static void dump_memory_region(struct ffa_memory_region *memory_region)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000273{
274 uint32_t i;
275
276 if (LOG_LEVEL < LOG_LEVEL_VERBOSE) {
277 return;
278 }
279
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000280 dlog("from VM %#x, attributes %#x, flags %#x, handle %#x "
281 "tag %u, memory access descriptor size %u, to %u "
Andrew Walbrana65a1322020-04-06 19:32:32 +0100282 "recipients [",
283 memory_region->sender, memory_region->attributes,
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000284 memory_region->flags, memory_region->handle, memory_region->tag,
285 memory_region->memory_access_desc_size,
Andrew Walbrana65a1322020-04-06 19:32:32 +0100286 memory_region->receiver_count);
287 for (i = 0; i < memory_region->receiver_count; ++i) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +0000288 struct ffa_memory_access *receiver =
289 ffa_memory_region_get_receiver(memory_region, i);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000290 if (i != 0) {
291 dlog(", ");
292 }
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +0000293 dlog("Receiver %#x: %#x (offset %u)",
294 receiver->receiver_permissions.receiver,
295 receiver->receiver_permissions.permissions,
296 receiver->composite_memory_region_offset);
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000297 /* The impdef field is only present from v1.2 and later */
298 if (ffa_version_from_memory_access_desc_size(
299 memory_region->memory_access_desc_size) >=
300 MAKE_FFA_VERSION(1, 2)) {
301 dlog(", impdef: %#x %#x", receiver->impdef.val[0],
302 receiver->impdef.val[1]);
303 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000304 }
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000305 dlog("] at offset %u", memory_region->receivers_offset);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000306}
307
J-Alves66652252022-07-06 09:49:51 +0100308void dump_share_states(void)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000309{
310 uint32_t i;
311
312 if (LOG_LEVEL < LOG_LEVEL_VERBOSE) {
313 return;
314 }
315
316 dlog("Current share states:\n");
317 sl_lock(&share_states_lock_instance);
318 for (i = 0; i < MAX_MEM_SHARES; ++i) {
319 if (share_states[i].share_func != 0) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000320 switch (share_states[i].share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100321 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000322 dlog("SHARE");
323 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100324 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000325 dlog("LEND");
326 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100327 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000328 dlog("DONATE");
329 break;
330 default:
331 dlog("invalid share_func %#x",
332 share_states[i].share_func);
333 }
Olivier Deprez935e1b12020-12-22 18:01:29 +0100334 dlog(" %#x (", share_states[i].memory_region->handle);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000335 dump_memory_region(share_states[i].memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +0100336 if (share_states[i].sending_complete) {
337 dlog("): fully sent");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000338 } else {
Andrew Walbranca808b12020-05-15 17:22:28 +0100339 dlog("): partially sent");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000340 }
J-Alves2a0d2882020-10-29 14:49:50 +0000341 dlog(" with %d fragments, %d retrieved, "
342 " sender's original mode: %#x\n",
Andrew Walbranca808b12020-05-15 17:22:28 +0100343 share_states[i].fragment_count,
J-Alves2a0d2882020-10-29 14:49:50 +0000344 share_states[i].retrieved_fragment_count[0],
345 share_states[i].sender_orig_mode);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000346 }
347 }
348 sl_unlock(&share_states_lock_instance);
349}
350
Andrew Walbran475c1452020-02-07 13:22:22 +0000351/* TODO: Add device attributes: GRE, cacheability, shareability. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100352static inline uint32_t ffa_memory_permissions_to_mode(
J-Alves7cd5eb32020-10-16 19:06:10 +0100353 ffa_memory_access_permissions_t permissions, uint32_t default_mode)
Andrew Walbran475c1452020-02-07 13:22:22 +0000354{
355 uint32_t mode = 0;
356
Karl Meakin84710f32023-10-12 15:14:49 +0100357 switch (permissions.data_access) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100358 case FFA_DATA_ACCESS_RO:
Andrew Walbran475c1452020-02-07 13:22:22 +0000359 mode = MM_MODE_R;
360 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100361 case FFA_DATA_ACCESS_RW:
Andrew Walbran475c1452020-02-07 13:22:22 +0000362 mode = MM_MODE_R | MM_MODE_W;
363 break;
J-Alves7cd5eb32020-10-16 19:06:10 +0100364 case FFA_DATA_ACCESS_NOT_SPECIFIED:
365 mode = (default_mode & (MM_MODE_R | MM_MODE_W));
366 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100367 case FFA_DATA_ACCESS_RESERVED:
368 panic("Tried to convert FFA_DATA_ACCESS_RESERVED.");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100369 }
370
Karl Meakin84710f32023-10-12 15:14:49 +0100371 switch (permissions.instruction_access) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100372 case FFA_INSTRUCTION_ACCESS_NX:
Andrew Walbran475c1452020-02-07 13:22:22 +0000373 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100374 case FFA_INSTRUCTION_ACCESS_X:
Andrew Walbrana65a1322020-04-06 19:32:32 +0100375 mode |= MM_MODE_X;
376 break;
J-Alves7cd5eb32020-10-16 19:06:10 +0100377 case FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED:
378 mode |= (default_mode & MM_MODE_X);
379 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100380 case FFA_INSTRUCTION_ACCESS_RESERVED:
381 panic("Tried to convert FFA_INSTRUCTION_ACCESS_RESVERVED.");
Andrew Walbran475c1452020-02-07 13:22:22 +0000382 }
383
Olivier Deprez878bd5b2021-04-15 19:05:10 +0200384 /* Set the security state bit if necessary. */
385 if ((default_mode & plat_ffa_other_world_mode()) != 0) {
386 mode |= plat_ffa_other_world_mode();
387 }
388
Andrew Walbran475c1452020-02-07 13:22:22 +0000389 return mode;
390}
391
Jose Marinho75509b42019-04-09 09:34:59 +0100392/**
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000393 * Get the current mode in the stage-2 page table of the given vm of all the
394 * pages in the given constituents, if they all have the same mode, or return
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100395 * an appropriate FF-A error if not.
Jose Marinho75509b42019-04-09 09:34:59 +0100396 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100397static struct ffa_value constituents_get_mode(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000398 struct vm_locked vm, uint32_t *orig_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100399 struct ffa_memory_region_constituent **fragments,
400 const uint32_t *fragment_constituent_counts, uint32_t fragment_count)
Jose Marinho75509b42019-04-09 09:34:59 +0100401{
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100402 uint32_t i;
Andrew Walbranca808b12020-05-15 17:22:28 +0100403 uint32_t j;
Jose Marinho75509b42019-04-09 09:34:59 +0100404
Andrew Walbranca808b12020-05-15 17:22:28 +0100405 if (fragment_count == 0 || fragment_constituent_counts[0] == 0) {
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100406 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000407 * Fail if there are no constituents. Otherwise we would get an
408 * uninitialised *orig_mode.
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100409 */
Karl Meakin5df422c2023-07-11 17:31:38 +0100410 dlog_verbose("%s: no constituents\n", __func__);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100411 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho75509b42019-04-09 09:34:59 +0100412 }
413
Andrew Walbranca808b12020-05-15 17:22:28 +0100414 for (i = 0; i < fragment_count; ++i) {
415 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
416 ipaddr_t begin = ipa_init(fragments[i][j].address);
417 size_t size = fragments[i][j].page_count * PAGE_SIZE;
418 ipaddr_t end = ipa_add(begin, size);
419 uint32_t current_mode;
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100420
Andrew Walbranca808b12020-05-15 17:22:28 +0100421 /* Fail if addresses are not page-aligned. */
422 if (!is_aligned(ipa_addr(begin), PAGE_SIZE) ||
423 !is_aligned(ipa_addr(end), PAGE_SIZE)) {
Karl Meakin5df422c2023-07-11 17:31:38 +0100424 dlog_verbose("%s: addresses not page-aligned\n",
425 __func__);
Andrew Walbranca808b12020-05-15 17:22:28 +0100426 return ffa_error(FFA_INVALID_PARAMETERS);
427 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100428
Andrew Walbranca808b12020-05-15 17:22:28 +0100429 /*
430 * Ensure that this constituent memory range is all
431 * mapped with the same mode.
432 */
Raghu Krishnamurthy785d52f2021-02-13 00:02:40 -0800433 if (!vm_mem_get_mode(vm, begin, end, &current_mode)) {
Karl Meakin5df422c2023-07-11 17:31:38 +0100434 dlog_verbose(
435 "%s: constituent memory range %#x..%#x "
436 "not mapped with the same mode\n",
437 __func__, begin, end);
Andrew Walbranca808b12020-05-15 17:22:28 +0100438 return ffa_error(FFA_DENIED);
439 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100440
Andrew Walbranca808b12020-05-15 17:22:28 +0100441 /*
442 * Ensure that all constituents are mapped with the same
443 * mode.
444 */
445 if (i == 0) {
446 *orig_mode = current_mode;
447 } else if (current_mode != *orig_mode) {
448 dlog_verbose(
Karl Meakin5df422c2023-07-11 17:31:38 +0100449 "%s: expected mode %#x but was %#x for "
450 "%d pages at %#x.\n",
451 __func__, *orig_mode, current_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100452 fragments[i][j].page_count,
453 ipa_addr(begin));
454 return ffa_error(FFA_DENIED);
455 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100456 }
Jose Marinho75509b42019-04-09 09:34:59 +0100457 }
458
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100459 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000460}
461
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100462uint32_t ffa_version_from_memory_access_desc_size(
463 uint32_t memory_access_desc_size)
464{
465 switch (memory_access_desc_size) {
466 /*
467 * v1.0 and v1.1 memory access descriptors are the same size however
468 * v1.1 is the first version to include the memory access descriptor
469 * size field so return v1.1.
470 */
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000471 case sizeof(struct ffa_memory_access_v1_0):
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100472 return MAKE_FFA_VERSION(1, 1);
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000473 case sizeof(struct ffa_memory_access):
474 return MAKE_FFA_VERSION(1, 2);
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100475 }
476 return 0;
477}
478
479/**
480 * Check if the receivers size and offset given is valid for the senders
481 * FF-A version.
482 */
483static bool receiver_size_and_offset_valid_for_version(
484 uint32_t receivers_size, uint32_t receivers_offset,
485 uint32_t ffa_version)
486{
487 /*
488 * Check that the version that the memory access descriptor size belongs
489 * to is compatible with the FF-A version we believe the sender to be.
490 */
491 uint32_t expected_ffa_version =
492 ffa_version_from_memory_access_desc_size(receivers_size);
493 if (!FFA_VERSIONS_ARE_COMPATIBLE(expected_ffa_version, ffa_version)) {
494 return false;
495 }
496
497 /*
498 * Check the receivers_offset matches the version we found from
499 * memory access descriptor size.
500 */
501 switch (expected_ffa_version) {
502 case MAKE_FFA_VERSION(1, 1):
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000503 case MAKE_FFA_VERSION(1, 2):
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100504 return receivers_offset == sizeof(struct ffa_memory_region);
505 default:
506 return false;
507 }
508}
509
510/**
511 * Check the values set for fields in the memory region are valid and safe.
512 * Offset values are within safe bounds, receiver count will not cause overflows
513 * and reserved fields are 0.
514 */
515bool ffa_memory_region_sanity_check(struct ffa_memory_region *memory_region,
516 uint32_t ffa_version,
517 uint32_t fragment_length,
518 bool send_transaction)
519{
520 uint32_t receiver_count;
521 struct ffa_memory_access *receiver;
522 uint32_t composite_offset_0;
Daniel Boulbyf06b5232024-02-22 16:26:43 +0000523 struct ffa_memory_region_v1_0 *memory_region_v1_0 =
524 (struct ffa_memory_region_v1_0 *)memory_region;
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100525
526 if (ffa_version == MAKE_FFA_VERSION(1, 0)) {
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100527 /* Check the reserved fields are 0. */
528 if (memory_region_v1_0->reserved_0 != 0 ||
529 memory_region_v1_0->reserved_1 != 0) {
530 dlog_verbose("Reserved fields must be 0.\n");
531 return false;
532 }
533
534 receiver_count = memory_region_v1_0->receiver_count;
535 } else {
536 uint32_t receivers_size =
537 memory_region->memory_access_desc_size;
538 uint32_t receivers_offset = memory_region->receivers_offset;
539
540 /* Check the reserved field is 0. */
541 if (memory_region->reserved[0] != 0 ||
542 memory_region->reserved[1] != 0 ||
543 memory_region->reserved[2] != 0) {
544 dlog_verbose("Reserved fields must be 0.\n");
545 return false;
546 }
547
548 /*
549 * Check memory_access_desc_size matches the size of the struct
550 * for the senders FF-A version.
551 */
552 if (!receiver_size_and_offset_valid_for_version(
553 receivers_size, receivers_offset, ffa_version)) {
554 dlog_verbose(
555 "Invalid memory access descriptor size %d, "
556 " or receiver offset %d, "
557 "for FF-A version %#x\n",
558 receivers_size, receivers_offset, ffa_version);
559 return false;
560 }
561
562 receiver_count = memory_region->receiver_count;
563 }
564
565 /* Check receiver count is not too large. */
Daniel Boulbyf06b5232024-02-22 16:26:43 +0000566 if (receiver_count > MAX_MEM_SHARE_RECIPIENTS || receiver_count < 1) {
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100567 dlog_verbose(
Daniel Boulbyf06b5232024-02-22 16:26:43 +0000568 "Receiver count must be 0 < receiver_count < %u "
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100569 "specified %u\n",
570 MAX_MEM_SHARE_RECIPIENTS, receiver_count);
571 return false;
572 }
573
574 /* Check values in the memory access descriptors. */
575 /*
576 * The composite offset values must be the same for all recievers so
577 * check the first one is valid and then they are all the same.
578 */
579 receiver = ffa_version == MAKE_FFA_VERSION(1, 0)
Daniel Boulbyf06b5232024-02-22 16:26:43 +0000580 ? (struct ffa_memory_access *)&memory_region_v1_0
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100581 ->receivers[0]
582 : ffa_memory_region_get_receiver(memory_region, 0);
583 assert(receiver != NULL);
584 composite_offset_0 = receiver->composite_memory_region_offset;
585
586 if (!send_transaction) {
587 if (composite_offset_0 != 0) {
588 dlog_verbose(
589 "Composite offset memory region descriptor "
590 "offset must be 0 for retrieve requests. "
591 "Currently %d",
592 composite_offset_0);
593 return false;
594 }
595 } else {
596 bool comp_offset_is_zero = composite_offset_0 == 0U;
597 bool comp_offset_lt_transaction_descriptor_size =
598 composite_offset_0 <
599 (sizeof(struct ffa_memory_region) +
600 (uint32_t)(memory_region->memory_access_desc_size *
601 memory_region->receiver_count));
602 bool comp_offset_with_comp_gt_fragment_length =
603 composite_offset_0 +
604 sizeof(struct ffa_composite_memory_region) >
605 fragment_length;
606 if (comp_offset_is_zero ||
607 comp_offset_lt_transaction_descriptor_size ||
608 comp_offset_with_comp_gt_fragment_length) {
609 dlog_verbose(
610 "Invalid composite memory region descriptor "
611 "offset for send transaction %u\n",
612 composite_offset_0);
613 return false;
614 }
615 }
616
617 for (int i = 0; i < memory_region->receiver_count; i++) {
618 uint32_t composite_offset;
619
620 if (ffa_version == MAKE_FFA_VERSION(1, 0)) {
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100621 struct ffa_memory_access_v1_0 *receiver_v1_0 =
622 &memory_region_v1_0->receivers[i];
623 /* Check reserved fields are 0 */
624 if (receiver_v1_0->reserved_0 != 0) {
625 dlog_verbose(
626 "Reserved field in the memory access "
627 " descriptor must be zero "
628 " Currently reciever %d has a reserved "
629 " field with a value of %d\n",
630 i, receiver_v1_0->reserved_0);
631 return false;
632 }
633 /*
634 * We can cast to the current version receiver as the
635 * remaining fields we are checking have the same
636 * offsets for all versions since memory access
637 * descriptors are forwards compatible.
638 */
639 receiver = (struct ffa_memory_access *)receiver_v1_0;
640 } else {
641 receiver = ffa_memory_region_get_receiver(memory_region,
642 i);
643 assert(receiver != NULL);
644
645 if (receiver->reserved_0 != 0) {
646 dlog_verbose(
647 "Reserved field in the memory access "
648 " descriptor must be zero "
649 " Currently reciever %d has a reserved "
650 " field with a value of %d\n",
651 i, receiver->reserved_0);
652 return false;
653 }
654 }
655
656 /* Check composite offset values are equal for all receivers. */
657 composite_offset = receiver->composite_memory_region_offset;
658 if (composite_offset != composite_offset_0) {
659 dlog_verbose(
660 "Composite offset %x differs from %x in index "
661 "%u\n",
662 composite_offset, composite_offset_0);
663 return false;
664 }
665 }
666 return true;
667}
668
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000669/**
J-Alves460d36c2023-10-12 17:02:15 +0100670 * If the receivers for the memory management operation are all from the
671 * secure world and this isn't a FFA_MEM_SHARE, then request memory security
672 * state update by returning MAP_ACTION_CHECK_PROTECT.
673 */
674static enum ffa_map_action ffa_mem_send_get_map_action(
675 bool all_receivers_from_current_world, ffa_id_t sender_id,
676 uint32_t mem_func_id)
677{
678 bool protect_memory =
679 (mem_func_id != FFA_MEM_SHARE_32 &&
680 all_receivers_from_current_world && ffa_is_vm_id(sender_id));
681
682 return protect_memory ? MAP_ACTION_CHECK_PROTECT : MAP_ACTION_CHECK;
683}
684
685/**
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000686 * Verify that all pages have the same mode, that the starting mode
687 * constitutes a valid state and obtain the next mode to apply
J-Alves460d36c2023-10-12 17:02:15 +0100688 * to the sending VM. It outputs the mapping action that needs to be
689 * invoked for the given memory range. On memory lend/donate there
690 * could be a need to protect the memory from the normal world.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000691 *
692 * Returns:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100693 * 1) FFA_DENIED if a state transition was not found;
694 * 2) FFA_DENIED if the pages being shared do not have the same mode within
Andrew Walbrana65a1322020-04-06 19:32:32 +0100695 * the <from> VM;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100696 * 3) FFA_INVALID_PARAMETERS if the beginning and end IPAs are not page
Andrew Walbrana65a1322020-04-06 19:32:32 +0100697 * aligned;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100698 * 4) FFA_INVALID_PARAMETERS if the requested share type was not handled.
699 * Or FFA_SUCCESS on success.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000700 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100701static struct ffa_value ffa_send_check_transition(
Andrew Walbrana65a1322020-04-06 19:32:32 +0100702 struct vm_locked from, uint32_t share_func,
Daniel Boulbya76fd912024-02-22 14:22:15 +0000703 struct ffa_memory_region *memory_region, uint32_t *orig_from_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100704 struct ffa_memory_region_constituent **fragments,
705 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
J-Alves460d36c2023-10-12 17:02:15 +0100706 uint32_t *from_mode, enum ffa_map_action *map_action)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000707{
708 const uint32_t state_mask =
709 MM_MODE_INVALID | MM_MODE_UNOWNED | MM_MODE_SHARED;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100710 struct ffa_value ret;
J-Alves460d36c2023-10-12 17:02:15 +0100711 bool all_receivers_from_current_world = true;
Daniel Boulbya76fd912024-02-22 14:22:15 +0000712 uint32_t receivers_count = memory_region->receiver_count;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000713
Andrew Walbranca808b12020-05-15 17:22:28 +0100714 ret = constituents_get_mode(from, orig_from_mode, fragments,
715 fragment_constituent_counts,
716 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100717 if (ret.func != FFA_SUCCESS_32) {
Olivier Depreze7eb1682022-03-16 17:09:03 +0100718 dlog_verbose("Inconsistent modes.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100719 return ret;
Andrew Scullb5f49e02019-10-02 13:20:47 +0100720 }
721
Daniel Boulby9764ff62024-01-30 17:47:39 +0000722 /*
723 * Device memory regions can only be lent from SP to SP and to a single
724 * borrower.
725 */
726 if ((*orig_from_mode & MM_MODE_D) != 0U &&
727 !(share_func == FFA_MEM_LEND_32 && !ffa_is_vm_id(from.vm->id) &&
728 receivers_count == 1)) {
729 dlog_verbose(
730 "Device memory can only be lent, from the secure world "
731 "and to a single borrower (mode is %#x).\n",
732 *orig_from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100733 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000734 }
735
736 /*
737 * Ensure the sender is the owner and has exclusive access to the
738 * memory.
739 */
740 if ((*orig_from_mode & state_mask) != 0) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100741 return ffa_error(FFA_DENIED);
Andrew Walbrana65a1322020-04-06 19:32:32 +0100742 }
743
Daniel Boulbya76fd912024-02-22 14:22:15 +0000744 assert(receivers_count > 0U);
J-Alves7cd5eb32020-10-16 19:06:10 +0100745
J-Alves363f5722022-04-25 17:37:37 +0100746 for (uint32_t i = 0U; i < receivers_count; i++) {
Daniel Boulbya76fd912024-02-22 14:22:15 +0000747 struct ffa_memory_access *receiver =
748 ffa_memory_region_get_receiver(memory_region, i);
749 assert(receiver != NULL);
J-Alves363f5722022-04-25 17:37:37 +0100750 ffa_memory_access_permissions_t permissions =
Daniel Boulbya76fd912024-02-22 14:22:15 +0000751 receiver->receiver_permissions.permissions;
J-Alves363f5722022-04-25 17:37:37 +0100752 uint32_t required_from_mode = ffa_memory_permissions_to_mode(
753 permissions, *orig_from_mode);
754
J-Alves788b4492023-04-18 14:01:23 +0100755 /*
756 * The assumption is that at this point, the operation from
757 * SP to a receiver VM, should have returned an FFA_ERROR
758 * already.
759 */
760 if (!ffa_is_vm_id(from.vm->id)) {
761 assert(!ffa_is_vm_id(
Daniel Boulbya76fd912024-02-22 14:22:15 +0000762 receiver->receiver_permissions.receiver));
J-Alves788b4492023-04-18 14:01:23 +0100763 }
764
J-Alves460d36c2023-10-12 17:02:15 +0100765 /* Track if all senders are from current world. */
766 all_receivers_from_current_world =
767 all_receivers_from_current_world &&
768 vm_id_is_current_world(
Daniel Boulbya76fd912024-02-22 14:22:15 +0000769 receiver->receiver_permissions.receiver);
J-Alves460d36c2023-10-12 17:02:15 +0100770
J-Alves363f5722022-04-25 17:37:37 +0100771 if ((*orig_from_mode & required_from_mode) !=
772 required_from_mode) {
773 dlog_verbose(
774 "Sender tried to send memory with permissions "
J-Alves788b4492023-04-18 14:01:23 +0100775 "which required mode %#x but only had %#x "
776 "itself.\n",
J-Alves363f5722022-04-25 17:37:37 +0100777 required_from_mode, *orig_from_mode);
778 return ffa_error(FFA_DENIED);
779 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000780 }
781
J-Alves460d36c2023-10-12 17:02:15 +0100782 *map_action = ffa_mem_send_get_map_action(
783 all_receivers_from_current_world, from.vm->id, share_func);
784
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000785 /* Find the appropriate new mode. */
786 *from_mode = ~state_mask & *orig_from_mode;
Andrew Walbrane7ad3c02019-12-24 17:03:04 +0000787 switch (share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100788 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000789 *from_mode |= MM_MODE_INVALID | MM_MODE_UNOWNED;
Jose Marinho75509b42019-04-09 09:34:59 +0100790 break;
791
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100792 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000793 *from_mode |= MM_MODE_INVALID;
Andrew Walbran648fc3e2019-10-22 16:23:05 +0100794 break;
795
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100796 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000797 *from_mode |= MM_MODE_SHARED;
Jose Marinho56c25732019-05-20 09:48:53 +0100798 break;
799
Jose Marinho75509b42019-04-09 09:34:59 +0100800 default:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100801 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho75509b42019-04-09 09:34:59 +0100802 }
803
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100804 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000805}
806
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100807static struct ffa_value ffa_relinquish_check_transition(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000808 struct vm_locked from, uint32_t *orig_from_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100809 struct ffa_memory_region_constituent **fragments,
810 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
811 uint32_t *from_mode)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000812{
813 const uint32_t state_mask =
814 MM_MODE_INVALID | MM_MODE_UNOWNED | MM_MODE_SHARED;
815 uint32_t orig_from_state;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100816 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000817
Andrew Walbranca808b12020-05-15 17:22:28 +0100818 ret = constituents_get_mode(from, orig_from_mode, fragments,
819 fragment_constituent_counts,
820 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100821 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbrana65a1322020-04-06 19:32:32 +0100822 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000823 }
824
825 /* Ensure the address range is normal memory and not a device. */
826 if (*orig_from_mode & MM_MODE_D) {
827 dlog_verbose("Can't relinquish device memory (mode is %#x).\n",
828 *orig_from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100829 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000830 }
831
832 /*
833 * Ensure the relinquishing VM is not the owner but has access to the
834 * memory.
835 */
836 orig_from_state = *orig_from_mode & state_mask;
837 if ((orig_from_state & ~MM_MODE_SHARED) != MM_MODE_UNOWNED) {
838 dlog_verbose(
839 "Tried to relinquish memory in state %#x (masked %#x "
Andrew Walbranca808b12020-05-15 17:22:28 +0100840 "but should be %#x).\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000841 *orig_from_mode, orig_from_state, MM_MODE_UNOWNED);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100842 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000843 }
844
845 /* Find the appropriate new mode. */
846 *from_mode = (~state_mask & *orig_from_mode) | MM_MODE_UNMAPPED_MASK;
847
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100848 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000849}
850
851/**
852 * Verify that all pages have the same mode, that the starting mode
853 * constitutes a valid state and obtain the next mode to apply
854 * to the retrieving VM.
855 *
856 * Returns:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100857 * 1) FFA_DENIED if a state transition was not found;
858 * 2) FFA_DENIED if the pages being shared do not have the same mode within
Andrew Walbrana65a1322020-04-06 19:32:32 +0100859 * the <to> VM;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100860 * 3) FFA_INVALID_PARAMETERS if the beginning and end IPAs are not page
Andrew Walbrana65a1322020-04-06 19:32:32 +0100861 * aligned;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100862 * 4) FFA_INVALID_PARAMETERS if the requested share type was not handled.
863 * Or FFA_SUCCESS on success.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000864 */
J-Alvesfc19b372022-07-06 12:17:35 +0100865struct ffa_value ffa_retrieve_check_transition(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000866 struct vm_locked to, uint32_t share_func,
Andrew Walbranca808b12020-05-15 17:22:28 +0100867 struct ffa_memory_region_constituent **fragments,
868 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
J-Alvesfd206052023-05-22 16:45:00 +0100869 uint32_t memory_to_attributes, uint32_t *to_mode, bool memory_protected,
870 enum ffa_map_action *map_action)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000871{
872 uint32_t orig_to_mode;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100873 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000874
Andrew Walbranca808b12020-05-15 17:22:28 +0100875 ret = constituents_get_mode(to, &orig_to_mode, fragments,
876 fragment_constituent_counts,
877 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100878 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100879 dlog_verbose("Inconsistent modes.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100880 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000881 }
882
J-Alves460d36c2023-10-12 17:02:15 +0100883 /* Find the appropriate new mode. */
884 *to_mode = memory_to_attributes;
885
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100886 if (share_func == FFA_MEM_RECLAIM_32) {
J-Alves9256f162021-12-09 13:18:43 +0000887 /*
888 * If the original ffa memory send call has been processed
889 * successfully, it is expected the orig_to_mode would overlay
890 * with `state_mask`, as a result of the function
891 * `ffa_send_check_transition`.
J-Alvesfd206052023-05-22 16:45:00 +0100892 *
893 * If Hafnium is the SPMC:
894 * - Caller of the reclaim interface is an SP, the memory shall
895 * have been protected throughout the flow.
896 * - Caller of the reclaim is from the NWd, the memory may have
897 * been protected at the time of lending/donating the memory.
898 * In such case, set action to unprotect memory in the
899 * handling of reclaim operation.
900 * - If Hafnium is the hypervisor memory shall never have been
901 * protected in memory lend/share/donate.
902 *
903 * More details in the doc comment of the function
904 * `ffa_region_group_identity_map`.
J-Alves9256f162021-12-09 13:18:43 +0000905 */
J-Alves59ed0042022-07-28 18:26:41 +0100906 if (vm_id_is_current_world(to.vm->id)) {
907 assert((orig_to_mode &
908 (MM_MODE_INVALID | MM_MODE_UNOWNED |
909 MM_MODE_SHARED)) != 0U);
J-Alvesfd206052023-05-22 16:45:00 +0100910 assert(!memory_protected);
911 } else if (to.vm->id == HF_OTHER_WORLD_ID &&
912 map_action != NULL && memory_protected) {
913 *map_action = MAP_ACTION_COMMIT_UNPROTECT;
J-Alves59ed0042022-07-28 18:26:41 +0100914 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000915 } else {
916 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +0100917 * If the retriever is from virtual FF-A instance:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000918 * Ensure the retriever has the expected state. We don't care
919 * about the MM_MODE_SHARED bit; either with or without it set
920 * are both valid representations of the !O-NA state.
921 */
J-Alvesa9cd7e32022-07-01 13:49:33 +0100922 if (vm_id_is_current_world(to.vm->id) &&
923 to.vm->id != HF_PRIMARY_VM_ID &&
924 (orig_to_mode & MM_MODE_UNMAPPED_MASK) !=
925 MM_MODE_UNMAPPED_MASK) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100926 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000927 }
J-Alves460d36c2023-10-12 17:02:15 +0100928
929 /*
930 * If memory has been protected before, clear the NS bit to
931 * allow the secure access from the SP.
932 */
933 if (memory_protected) {
934 *to_mode &= ~plat_ffa_other_world_mode();
935 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000936 }
937
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000938 switch (share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100939 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000940 *to_mode |= 0;
941 break;
942
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100943 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000944 *to_mode |= MM_MODE_UNOWNED;
945 break;
946
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100947 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000948 *to_mode |= MM_MODE_UNOWNED | MM_MODE_SHARED;
949 break;
950
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100951 case FFA_MEM_RECLAIM_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000952 *to_mode |= 0;
953 break;
954
955 default:
Andrew Walbranca808b12020-05-15 17:22:28 +0100956 dlog_error("Invalid share_func %#x.\n", share_func);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100957 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000958 }
959
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100960 return (struct ffa_value){.func = FFA_SUCCESS_32};
Jose Marinho75509b42019-04-09 09:34:59 +0100961}
Jose Marinho09b1db82019-08-08 09:16:59 +0100962
J-Alvescf6253e2024-01-03 13:48:48 +0000963/*
964 * Performs the operations related to the `action` MAP_ACTION_CHECK*.
965 * Returns:
966 * - FFA_SUCCESS_32: if all goes well.
967 * - FFA_ERROR_32: with FFA_NO_MEMORY, if there is no memory to manage
968 * the page table update. Or error code provided by the function
969 * `arch_memory_protect`.
970 */
971static struct ffa_value ffa_region_group_check_actions(
972 struct vm_locked vm_locked, paddr_t pa_begin, paddr_t pa_end,
973 struct mpool *ppool, uint32_t mode, enum ffa_map_action action,
974 bool *memory_protected)
975{
976 struct ffa_value ret;
977 bool is_memory_protected;
978
979 if (!vm_identity_prepare(vm_locked, pa_begin, pa_end, mode, ppool)) {
980 dlog_verbose(
981 "%s: memory can't be mapped to %x due to lack of "
982 "memory. Base: %lx end: %x\n",
983 __func__, vm_locked.vm->id, pa_addr(pa_begin),
984 pa_addr(pa_end));
985 return ffa_error(FFA_NO_MEMORY);
986 }
987
988 switch (action) {
989 case MAP_ACTION_CHECK:
990 /* No protect requested. */
991 is_memory_protected = false;
992 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
993 break;
994 case MAP_ACTION_CHECK_PROTECT: {
995 paddr_t last_protected_pa = pa_init(0);
996
997 ret = arch_memory_protect(pa_begin, pa_end, &last_protected_pa);
998
999 is_memory_protected = (ret.func == FFA_SUCCESS_32);
1000
1001 /*
1002 * - If protect memory has failed with FFA_DENIED, means some
1003 * range of memory was in the wrong state. In such case, SPM
1004 * reverts the state of the pages that were successfully
1005 * updated.
1006 * - If protect memory has failed with FFA_NOT_SUPPORTED, it
1007 * means the platform doesn't support the protection mechanism.
1008 * That said, it still permits the page table update to go
1009 * through. The variable
1010 * `is_memory_protected` will be equal to false.
1011 * - If protect memory has failed with FFA_INVALID_PARAMETERS,
1012 * break from switch and return the error.
1013 */
1014 if (ret.func == FFA_ERROR_32) {
1015 assert(!is_memory_protected);
1016 if (ffa_error_code(ret) == FFA_DENIED &&
1017 pa_addr(last_protected_pa) != (uintptr_t)0) {
1018 CHECK(arch_memory_unprotect(
1019 pa_begin,
1020 pa_add(last_protected_pa, PAGE_SIZE)));
1021 } else if (ffa_error_code(ret) == FFA_NOT_SUPPORTED) {
1022 ret = (struct ffa_value){
1023 .func = FFA_SUCCESS_32,
1024 };
1025 }
1026 }
1027 } break;
1028 default:
1029 panic("%s: invalid action to process %x\n", __func__, action);
1030 }
1031
1032 if (memory_protected != NULL) {
1033 *memory_protected = is_memory_protected;
1034 }
1035
1036 return ret;
1037}
1038
1039static void ffa_region_group_commit_actions(struct vm_locked vm_locked,
1040 paddr_t pa_begin, paddr_t pa_end,
1041 struct mpool *ppool, uint32_t mode,
1042 enum ffa_map_action action)
1043{
1044 switch (action) {
1045 case MAP_ACTION_COMMIT_UNPROTECT:
1046 /*
1047 * Checking that it should succeed because SPM should be
1048 * unprotecting memory that it had protected before.
1049 */
1050 CHECK(arch_memory_unprotect(pa_begin, pa_end));
1051 case MAP_ACTION_COMMIT:
1052 vm_identity_commit(vm_locked, pa_begin, pa_end, mode, ppool,
1053 NULL);
1054 break;
1055 default:
1056 panic("%s: invalid action to process %x\n", __func__, action);
1057 }
1058}
1059
Jose Marinho09b1db82019-08-08 09:16:59 +01001060/**
J-Alves063ad832023-10-03 18:05:40 +01001061 * Helper function to revert a failed "Protect" action from the SPMC:
1062 * - `fragment_count`: should specify the number of fragments to traverse from
1063 * `fragments`. This may not be the full amount of fragments that are part of
1064 * the share_state structure.
1065 * - `fragment_constituent_counts`: array holding the amount of constituents
1066 * per fragment.
1067 * - `end`: pointer to the constituent that failed the "protect" action. It
1068 * shall be part of the last fragment, and it shall make the loop below break.
1069 */
1070static void ffa_region_group_fragments_revert_protect(
1071 struct ffa_memory_region_constituent **fragments,
1072 const uint32_t *fragment_constituent_counts, uint32_t fragment_count,
1073 const struct ffa_memory_region_constituent *end)
1074{
1075 for (uint32_t i = 0; i < fragment_count; ++i) {
1076 for (uint32_t j = 0; j < fragment_constituent_counts[i]; ++j) {
1077 struct ffa_memory_region_constituent *constituent =
1078 &fragments[i][j];
1079 size_t size = constituent->page_count * PAGE_SIZE;
1080 paddr_t pa_begin =
1081 pa_from_ipa(ipa_init(constituent->address));
1082 paddr_t pa_end = pa_add(pa_begin, size);
1083
1084 dlog_verbose("%s: reverting fragment %x size %x\n",
1085 __func__, pa_addr(pa_begin), size);
1086
1087 if (constituent == end) {
1088 /*
1089 * The last constituent is expected to be in the
1090 * last fragment.
1091 */
1092 assert(i == fragment_count - 1);
1093 break;
1094 }
1095
1096 CHECK(arch_memory_unprotect(pa_begin, pa_end));
1097 }
1098 }
1099}
1100
1101/**
Jose Marinho09b1db82019-08-08 09:16:59 +01001102 * Updates a VM's page table such that the given set of physical address ranges
1103 * are mapped in the address space at the corresponding address ranges, in the
1104 * mode provided.
1105 *
J-Alves0a83dc22023-05-05 09:50:37 +01001106 * The enum ffa_map_action determines the action taken from a call to the
1107 * function below:
1108 * - If action is MAP_ACTION_CHECK, the page tables will be allocated from the
1109 * mpool but no mappings will actually be updated. This function must always
1110 * be called first with action set to MAP_ACTION_CHECK to check that it will
1111 * succeed before calling ffa_region_group_identity_map with whichever one of
1112 * the remaining actions, to avoid leaving the page table in a half-updated
1113 * state.
1114 * - The action MAP_ACTION_COMMIT allocates the page tables from the mpool, and
1115 * changes the memory mappings.
J-Alvescf6253e2024-01-03 13:48:48 +00001116 * - The action MAP_ACTION_CHECK_PROTECT extends the MAP_ACTION_CHECK with an
1117 * invocation to the monitor to update the security state of the memory,
1118 * to that of the SPMC.
1119 * - The action MAP_ACTION_COMMIT_UNPROTECT extends the MAP_ACTION_COMMIT
1120 * with a call into the monitor, to reset the security state of memory
1121 * that has priorly been mapped with the MAP_ACTION_CHECK_PROTECT action.
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001122 * vm_ptable_defrag should always be called after a series of page table
1123 * updates, whether they succeed or fail.
Jose Marinho09b1db82019-08-08 09:16:59 +01001124 *
J-Alvescf6253e2024-01-03 13:48:48 +00001125 * If all goes well, returns FFA_SUCCESS_32; or FFA_ERROR, with following
1126 * error codes:
1127 * - FFA_INVALID_PARAMETERS: invalid range of memory.
1128 * - FFA_DENIED:
1129 *
Jose Marinho09b1db82019-08-08 09:16:59 +01001130 * made to memory mappings.
1131 */
J-Alvescf6253e2024-01-03 13:48:48 +00001132struct ffa_value ffa_region_group_identity_map(
Andrew Walbranf4b51af2020-02-03 14:44:54 +00001133 struct vm_locked vm_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01001134 struct ffa_memory_region_constituent **fragments,
1135 const uint32_t *fragment_constituent_counts, uint32_t fragment_count,
J-Alvescf6253e2024-01-03 13:48:48 +00001136 uint32_t mode, struct mpool *ppool, enum ffa_map_action action,
1137 bool *memory_protected)
Jose Marinho09b1db82019-08-08 09:16:59 +01001138{
Andrew Walbranca808b12020-05-15 17:22:28 +01001139 uint32_t i;
1140 uint32_t j;
J-Alvescf6253e2024-01-03 13:48:48 +00001141 struct ffa_value ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01001142
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001143 if (vm_locked.vm->el0_partition) {
1144 mode |= MM_MODE_USER | MM_MODE_NG;
1145 }
1146
Andrew Walbranca808b12020-05-15 17:22:28 +01001147 /* Iterate over the memory region constituents within each fragment. */
1148 for (i = 0; i < fragment_count; ++i) {
1149 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
J-Alves063ad832023-10-03 18:05:40 +01001150 struct ffa_memory_region_constituent *constituent =
1151 &fragments[i][j];
1152 size_t size = constituent->page_count * PAGE_SIZE;
Andrew Walbranca808b12020-05-15 17:22:28 +01001153 paddr_t pa_begin =
J-Alves063ad832023-10-03 18:05:40 +01001154 pa_from_ipa(ipa_init(constituent->address));
Andrew Walbranca808b12020-05-15 17:22:28 +01001155 paddr_t pa_end = pa_add(pa_begin, size);
Jens Wiklander4f1880c2022-10-19 17:00:14 +02001156 uint32_t pa_bits =
1157 arch_mm_get_pa_bits(arch_mm_get_pa_range());
Federico Recanati4fd065d2021-12-13 20:06:23 +01001158
1159 /*
1160 * Ensure the requested region falls into system's PA
1161 * range.
1162 */
Jens Wiklander4f1880c2022-10-19 17:00:14 +02001163 if (((pa_addr(pa_begin) >> pa_bits) > 0) ||
1164 ((pa_addr(pa_end) >> pa_bits) > 0)) {
Federico Recanati4fd065d2021-12-13 20:06:23 +01001165 dlog_error("Region is outside of PA Range\n");
J-Alvescf6253e2024-01-03 13:48:48 +00001166 return ffa_error(FFA_INVALID_PARAMETERS);
Federico Recanati4fd065d2021-12-13 20:06:23 +01001167 }
Andrew Walbranca808b12020-05-15 17:22:28 +01001168
J-Alvescf6253e2024-01-03 13:48:48 +00001169 if (action <= MAP_ACTION_CHECK_PROTECT) {
1170 ret = ffa_region_group_check_actions(
1171 vm_locked, pa_begin, pa_end, ppool,
1172 mode, action, memory_protected);
J-Alves063ad832023-10-03 18:05:40 +01001173
1174 if (ret.func == FFA_ERROR_32 &&
1175 ffa_error_code(ret) == FFA_DENIED) {
1176 if (memory_protected != NULL) {
1177 assert(!*memory_protected);
1178 }
1179
1180 ffa_region_group_fragments_revert_protect(
1181 fragments,
1182 fragment_constituent_counts,
1183 i + 1, constituent);
1184 break;
1185 }
J-Alvescf6253e2024-01-03 13:48:48 +00001186 } else if (action >= MAP_ACTION_COMMIT &&
1187 action < MAP_ACTION_MAX) {
1188 ffa_region_group_commit_actions(
1189 vm_locked, pa_begin, pa_end, ppool,
1190 mode, action);
1191 ret = (struct ffa_value){
1192 .func = FFA_SUCCESS_32};
1193 } else {
1194 panic("%s: Unknown ffa_map_action.\n",
1195 __func__);
Andrew Walbranca808b12020-05-15 17:22:28 +01001196 }
Jose Marinho09b1db82019-08-08 09:16:59 +01001197 }
1198 }
1199
J-Alvescf6253e2024-01-03 13:48:48 +00001200 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01001201}
1202
1203/**
1204 * Clears a region of physical memory by overwriting it with zeros. The data is
1205 * flushed from the cache so the memory has been cleared across the system.
1206 */
J-Alves7db32002021-12-14 14:44:50 +00001207static bool clear_memory(paddr_t begin, paddr_t end, struct mpool *ppool,
1208 uint32_t extra_mode_attributes)
Jose Marinho09b1db82019-08-08 09:16:59 +01001209{
1210 /*
Fuad Tabbaed294af2019-12-20 10:43:01 +00001211 * TODO: change this to a CPU local single page window rather than a
Jose Marinho09b1db82019-08-08 09:16:59 +01001212 * global mapping of the whole range. Such an approach will limit
1213 * the changes to stage-1 tables and will allow only local
1214 * invalidation.
1215 */
1216 bool ret;
1217 struct mm_stage1_locked stage1_locked = mm_lock_stage1();
J-Alves7db32002021-12-14 14:44:50 +00001218 void *ptr = mm_identity_map(stage1_locked, begin, end,
1219 MM_MODE_W | (extra_mode_attributes &
1220 plat_ffa_other_world_mode()),
1221 ppool);
Jose Marinho09b1db82019-08-08 09:16:59 +01001222 size_t size = pa_difference(begin, end);
1223
1224 if (!ptr) {
Jose Marinho09b1db82019-08-08 09:16:59 +01001225 goto fail;
1226 }
1227
1228 memset_s(ptr, size, 0, size);
1229 arch_mm_flush_dcache(ptr, size);
1230 mm_unmap(stage1_locked, begin, end, ppool);
1231
1232 ret = true;
1233 goto out;
1234
1235fail:
1236 ret = false;
1237
1238out:
1239 mm_unlock_stage1(&stage1_locked);
1240
1241 return ret;
1242}
1243
1244/**
1245 * Clears a region of physical memory by overwriting it with zeros. The data is
1246 * flushed from the cache so the memory has been cleared across the system.
1247 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001248static bool ffa_clear_memory_constituents(
J-Alves7db32002021-12-14 14:44:50 +00001249 uint32_t security_state_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +01001250 struct ffa_memory_region_constituent **fragments,
1251 const uint32_t *fragment_constituent_counts, uint32_t fragment_count,
1252 struct mpool *page_pool)
Jose Marinho09b1db82019-08-08 09:16:59 +01001253{
1254 struct mpool local_page_pool;
Andrew Walbranca808b12020-05-15 17:22:28 +01001255 uint32_t i;
Jose Marinho09b1db82019-08-08 09:16:59 +01001256 bool ret = false;
1257
1258 /*
1259 * Create a local pool so any freed memory can't be used by another
1260 * thread. This is to ensure each constituent that is mapped can be
1261 * unmapped again afterwards.
1262 */
Andrew Walbran475c1452020-02-07 13:22:22 +00001263 mpool_init_with_fallback(&local_page_pool, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +01001264
Andrew Walbranca808b12020-05-15 17:22:28 +01001265 /* Iterate over the memory region constituents within each fragment. */
1266 for (i = 0; i < fragment_count; ++i) {
1267 uint32_t j;
Jose Marinho09b1db82019-08-08 09:16:59 +01001268
J-Alves8457f932023-10-11 16:41:45 +01001269 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001270 size_t size = fragments[i][j].page_count * PAGE_SIZE;
1271 paddr_t begin =
1272 pa_from_ipa(ipa_init(fragments[i][j].address));
1273 paddr_t end = pa_add(begin, size);
1274
J-Alves7db32002021-12-14 14:44:50 +00001275 if (!clear_memory(begin, end, &local_page_pool,
1276 security_state_mode)) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001277 /*
1278 * api_clear_memory will defrag on failure, so
1279 * no need to do it here.
1280 */
1281 goto out;
1282 }
Jose Marinho09b1db82019-08-08 09:16:59 +01001283 }
1284 }
1285
Jose Marinho09b1db82019-08-08 09:16:59 +01001286 ret = true;
1287
1288out:
1289 mpool_fini(&local_page_pool);
1290 return ret;
1291}
1292
J-Alves5952d942022-12-22 16:03:00 +00001293static bool is_memory_range_within(ipaddr_t begin, ipaddr_t end,
1294 ipaddr_t in_begin, ipaddr_t in_end)
1295{
1296 return (ipa_addr(begin) >= ipa_addr(in_begin) &&
1297 ipa_addr(begin) < ipa_addr(in_end)) ||
1298 (ipa_addr(end) <= ipa_addr(in_end) &&
1299 ipa_addr(end) > ipa_addr(in_begin));
1300}
1301
1302/**
1303 * Receives a memory range and looks for overlaps with the remainder
1304 * constituents of the memory share/lend/donate operation. Assumes they are
1305 * passed in order to avoid having to loop over all the elements at each call.
1306 * The function only compares the received memory ranges with those that follow
1307 * within the same fragment, and subsequent fragments from the same operation.
1308 */
1309static bool ffa_memory_check_overlap(
1310 struct ffa_memory_region_constituent **fragments,
1311 const uint32_t *fragment_constituent_counts,
1312 const uint32_t fragment_count, const uint32_t current_fragment,
1313 const uint32_t current_constituent)
1314{
1315 uint32_t i = current_fragment;
1316 uint32_t j = current_constituent;
1317 ipaddr_t current_begin = ipa_init(fragments[i][j].address);
1318 const uint32_t current_page_count = fragments[i][j].page_count;
1319 size_t current_size = current_page_count * PAGE_SIZE;
1320 ipaddr_t current_end = ipa_add(current_begin, current_size - 1);
1321
1322 if (current_size == 0 ||
1323 current_size > UINT64_MAX - ipa_addr(current_begin)) {
1324 dlog_verbose("Invalid page count. Addr: %x page_count: %x\n",
1325 current_begin, current_page_count);
1326 return false;
1327 }
1328
1329 for (; i < fragment_count; i++) {
1330 j = (i == current_fragment) ? j + 1 : 0;
1331
1332 for (; j < fragment_constituent_counts[i]; j++) {
1333 ipaddr_t begin = ipa_init(fragments[i][j].address);
1334 const uint32_t page_count = fragments[i][j].page_count;
1335 size_t size = page_count * PAGE_SIZE;
1336 ipaddr_t end = ipa_add(begin, size - 1);
1337
1338 if (size == 0 || size > UINT64_MAX - ipa_addr(begin)) {
1339 dlog_verbose(
1340 "Invalid page count. Addr: %x "
1341 "page_count: %x\n",
1342 begin, page_count);
1343 return false;
1344 }
1345
1346 /*
1347 * Check if current ranges is within begin and end, as
1348 * well as the reverse. This should help optimize the
1349 * loop, and reduce the number of iterations.
1350 */
1351 if (is_memory_range_within(begin, end, current_begin,
1352 current_end) ||
1353 is_memory_range_within(current_begin, current_end,
1354 begin, end)) {
1355 dlog_verbose(
1356 "Overlapping memory ranges: %#x - %#x "
1357 "with %#x - %#x\n",
1358 ipa_addr(begin), ipa_addr(end),
1359 ipa_addr(current_begin),
1360 ipa_addr(current_end));
1361 return true;
1362 }
1363 }
1364 }
1365
1366 return false;
1367}
1368
Jose Marinho09b1db82019-08-08 09:16:59 +01001369/**
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001370 * Validates and prepares memory to be sent from the calling VM to another.
Jose Marinho09b1db82019-08-08 09:16:59 +01001371 *
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001372 * This function requires the calling context to hold the <from> VM lock.
Jose Marinho09b1db82019-08-08 09:16:59 +01001373 *
1374 * Returns:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001375 * In case of error, one of the following values is returned:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001376 * 1) FFA_INVALID_PARAMETERS - The endpoint provided parameters were
Jose Marinho09b1db82019-08-08 09:16:59 +01001377 * erroneous;
Andrew Walbranf07f04d2020-05-01 18:09:00 +01001378 * 2) FFA_NO_MEMORY - Hafnium did not have sufficient memory to complete the
1379 * request.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001380 * 3) FFA_DENIED - The sender doesn't have sufficient access to send the
Andrew Walbrana65a1322020-04-06 19:32:32 +01001381 * memory with the given permissions.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001382 * Success is indicated by FFA_SUCCESS.
Jose Marinho09b1db82019-08-08 09:16:59 +01001383 */
Daniel Boulbya76fd912024-02-22 14:22:15 +00001384static struct ffa_value ffa_send_check_update(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001385 struct vm_locked from_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01001386 struct ffa_memory_region_constituent **fragments,
1387 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
J-Alves8f11cde2022-12-21 16:18:22 +00001388 uint32_t composite_total_page_count, uint32_t share_func,
Daniel Boulbya76fd912024-02-22 14:22:15 +00001389 struct ffa_memory_region *memory_region, struct mpool *page_pool,
1390 uint32_t *orig_from_mode_ret, bool *memory_protected)
Jose Marinho09b1db82019-08-08 09:16:59 +01001391{
Andrew Walbranca808b12020-05-15 17:22:28 +01001392 uint32_t i;
J-Alves8f11cde2022-12-21 16:18:22 +00001393 uint32_t j;
Jose Marinho09b1db82019-08-08 09:16:59 +01001394 uint32_t orig_from_mode;
J-Alves460d36c2023-10-12 17:02:15 +01001395 uint32_t clean_mode;
Jose Marinho09b1db82019-08-08 09:16:59 +01001396 uint32_t from_mode;
Jose Marinho09b1db82019-08-08 09:16:59 +01001397 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001398 struct ffa_value ret;
J-Alves8f11cde2022-12-21 16:18:22 +00001399 uint32_t constituents_total_page_count = 0;
J-Alves460d36c2023-10-12 17:02:15 +01001400 enum ffa_map_action map_action = MAP_ACTION_CHECK;
Daniel Boulbya76fd912024-02-22 14:22:15 +00001401 bool clear = memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR;
Jose Marinho09b1db82019-08-08 09:16:59 +01001402
1403 /*
Andrew Walbrana65a1322020-04-06 19:32:32 +01001404 * Make sure constituents are properly aligned to a 64-bit boundary. If
1405 * not we would get alignment faults trying to read (64-bit) values.
Jose Marinho09b1db82019-08-08 09:16:59 +01001406 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001407 for (i = 0; i < fragment_count; ++i) {
1408 if (!is_aligned(fragments[i], 8)) {
1409 dlog_verbose("Constituents not aligned.\n");
1410 return ffa_error(FFA_INVALID_PARAMETERS);
1411 }
J-Alves8f11cde2022-12-21 16:18:22 +00001412 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
1413 constituents_total_page_count +=
1414 fragments[i][j].page_count;
J-Alves5952d942022-12-22 16:03:00 +00001415 if (ffa_memory_check_overlap(
1416 fragments, fragment_constituent_counts,
1417 fragment_count, i, j)) {
1418 return ffa_error(FFA_INVALID_PARAMETERS);
1419 }
J-Alves8f11cde2022-12-21 16:18:22 +00001420 }
1421 }
1422
1423 if (constituents_total_page_count != composite_total_page_count) {
1424 dlog_verbose(
1425 "Composite page count differs from calculated page "
1426 "count from constituents.\n");
1427 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho09b1db82019-08-08 09:16:59 +01001428 }
1429
1430 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001431 * Check if the state transition is lawful for the sender, ensure that
1432 * all constituents of a memory region being shared are at the same
1433 * state.
Jose Marinho09b1db82019-08-08 09:16:59 +01001434 */
J-Alves460d36c2023-10-12 17:02:15 +01001435 ret = ffa_send_check_transition(
Daniel Boulbya76fd912024-02-22 14:22:15 +00001436 from_locked, share_func, memory_region, &orig_from_mode,
1437 fragments, fragment_constituent_counts, fragment_count,
1438 &from_mode, &map_action);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001439 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001440 dlog_verbose("Invalid transition for send.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +01001441 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01001442 }
1443
Andrew Walbran37c574e2020-06-03 11:45:46 +01001444 if (orig_from_mode_ret != NULL) {
1445 *orig_from_mode_ret = orig_from_mode;
1446 }
1447
Jose Marinho09b1db82019-08-08 09:16:59 +01001448 /*
1449 * Create a local pool so any freed memory can't be used by another
1450 * thread. This is to ensure the original mapping can be restored if the
1451 * clear fails.
1452 */
Andrew Walbran475c1452020-02-07 13:22:22 +00001453 mpool_init_with_fallback(&local_page_pool, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +01001454
1455 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001456 * First reserve all required memory for the new page table entries
1457 * without committing, to make sure the entire operation will succeed
1458 * without exhausting the page pool.
J-Alves460d36c2023-10-12 17:02:15 +01001459 * Provide the map_action as populated by 'ffa_send_check_transition'.
1460 * It may request memory to be protected.
Jose Marinho09b1db82019-08-08 09:16:59 +01001461 */
J-Alvescf6253e2024-01-03 13:48:48 +00001462 ret = ffa_region_group_identity_map(
1463 from_locked, fragments, fragment_constituent_counts,
J-Alves460d36c2023-10-12 17:02:15 +01001464 fragment_count, from_mode, page_pool, map_action,
1465 memory_protected);
J-Alvescf6253e2024-01-03 13:48:48 +00001466 if (ret.func == FFA_ERROR_32) {
Jose Marinho09b1db82019-08-08 09:16:59 +01001467 goto out;
1468 }
1469
1470 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001471 * Update the mapping for the sender. This won't allocate because the
1472 * transaction was already prepared above, but may free pages in the
1473 * case that a whole block is being unmapped that was previously
1474 * partially mapped.
Jose Marinho09b1db82019-08-08 09:16:59 +01001475 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001476 CHECK(ffa_region_group_identity_map(
J-Alvescf6253e2024-01-03 13:48:48 +00001477 from_locked, fragments, fragment_constituent_counts,
1478 fragment_count, from_mode, &local_page_pool,
1479 MAP_ACTION_COMMIT, NULL)
1480 .func == FFA_SUCCESS_32);
Jose Marinho09b1db82019-08-08 09:16:59 +01001481
J-Alves460d36c2023-10-12 17:02:15 +01001482 /*
1483 * If memory has been protected, it is now part of the secure PAS
1484 * (happens for lend/donate from NWd to SWd), and the `orig_from_mode`
1485 * should have the MM_MODE_NS set, as such mask it in `clean_mode` for
1486 * SPM's S1 translation.
1487 * In case memory hasn't been protected, and it is in the non-secure
1488 * PAS (e.g. memory share from NWd to SWd), as such the SPM needs to
1489 * perform a non-secure memory access. In such case `clean_mode` takes
1490 * the same mode as `orig_from_mode`.
1491 */
1492 clean_mode = (memory_protected != NULL && *memory_protected)
1493 ? orig_from_mode & ~plat_ffa_other_world_mode()
1494 : orig_from_mode;
1495
Jose Marinho09b1db82019-08-08 09:16:59 +01001496 /* Clear the memory so no VM or device can see the previous contents. */
J-Alves460d36c2023-10-12 17:02:15 +01001497 if (clear && !ffa_clear_memory_constituents(
1498 clean_mode, fragments, fragment_constituent_counts,
1499 fragment_count, page_pool)) {
1500 map_action = (memory_protected != NULL && *memory_protected)
1501 ? MAP_ACTION_COMMIT_UNPROTECT
1502 : MAP_ACTION_COMMIT;
1503
Jose Marinho09b1db82019-08-08 09:16:59 +01001504 /*
1505 * On failure, roll back by returning memory to the sender. This
1506 * may allocate pages which were previously freed into
1507 * `local_page_pool` by the call above, but will never allocate
1508 * more pages than that so can never fail.
1509 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001510 CHECK(ffa_region_group_identity_map(
J-Alvescf6253e2024-01-03 13:48:48 +00001511 from_locked, fragments,
1512 fragment_constituent_counts, fragment_count,
1513 orig_from_mode, &local_page_pool,
1514 MAP_ACTION_COMMIT, NULL)
1515 .func == FFA_SUCCESS_32);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001516 ret = ffa_error(FFA_NO_MEMORY);
Jose Marinho09b1db82019-08-08 09:16:59 +01001517 goto out;
1518 }
1519
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001520 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001521
1522out:
1523 mpool_fini(&local_page_pool);
1524
1525 /*
1526 * Tidy up the page table by reclaiming failed mappings (if there was an
1527 * error) or merging entries into blocks where possible (on success).
1528 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001529 vm_ptable_defrag(from_locked, page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001530
1531 return ret;
1532}
1533
1534/**
1535 * Validates and maps memory shared from one VM to another.
1536 *
1537 * This function requires the calling context to hold the <to> lock.
1538 *
1539 * Returns:
1540 * In case of error, one of the following values is returned:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001541 * 1) FFA_INVALID_PARAMETERS - The endpoint provided parameters were
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001542 * erroneous;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001543 * 2) FFA_NO_MEMORY - Hafnium did not have sufficient memory to complete
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001544 * the request.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001545 * Success is indicated by FFA_SUCCESS.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001546 */
J-Alvesb5084cf2022-07-06 14:20:12 +01001547struct ffa_value ffa_retrieve_check_update(
J-Alves26483382023-04-20 12:01:49 +01001548 struct vm_locked to_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01001549 struct ffa_memory_region_constituent **fragments,
1550 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
J-Alves26483382023-04-20 12:01:49 +01001551 uint32_t sender_orig_mode, uint32_t share_func, bool clear,
J-Alves460d36c2023-10-12 17:02:15 +01001552 struct mpool *page_pool, uint32_t *response_mode, bool memory_protected)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001553{
Andrew Walbranca808b12020-05-15 17:22:28 +01001554 uint32_t i;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001555 uint32_t to_mode;
1556 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001557 struct ffa_value ret;
J-Alvesfd206052023-05-22 16:45:00 +01001558 enum ffa_map_action map_action = MAP_ACTION_COMMIT;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001559
1560 /*
Andrew Walbranca808b12020-05-15 17:22:28 +01001561 * Make sure constituents are properly aligned to a 64-bit boundary. If
1562 * not we would get alignment faults trying to read (64-bit) values.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001563 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001564 for (i = 0; i < fragment_count; ++i) {
1565 if (!is_aligned(fragments[i], 8)) {
J-Alvesb5084cf2022-07-06 14:20:12 +01001566 dlog_verbose("Fragment not properly aligned.\n");
Andrew Walbranca808b12020-05-15 17:22:28 +01001567 return ffa_error(FFA_INVALID_PARAMETERS);
1568 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001569 }
1570
1571 /*
1572 * Check if the state transition is lawful for the recipient, and ensure
1573 * that all constituents of the memory region being retrieved are at the
1574 * same state.
1575 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001576 ret = ffa_retrieve_check_transition(
1577 to_locked, share_func, fragments, fragment_constituent_counts,
J-Alvesfd206052023-05-22 16:45:00 +01001578 fragment_count, sender_orig_mode, &to_mode, memory_protected,
1579 &map_action);
J-Alves460d36c2023-10-12 17:02:15 +01001580
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001581 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001582 dlog_verbose("Invalid transition for retrieve.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +01001583 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001584 }
1585
1586 /*
1587 * Create a local pool so any freed memory can't be used by another
1588 * thread. This is to ensure the original mapping can be restored if the
1589 * clear fails.
1590 */
1591 mpool_init_with_fallback(&local_page_pool, page_pool);
1592
1593 /*
1594 * First reserve all required memory for the new page table entries in
1595 * the recipient page tables without committing, to make sure the entire
1596 * operation will succeed without exhausting the page pool.
1597 */
J-Alvescf6253e2024-01-03 13:48:48 +00001598 ret = ffa_region_group_identity_map(
1599 to_locked, fragments, fragment_constituent_counts,
1600 fragment_count, to_mode, page_pool, MAP_ACTION_CHECK, NULL);
1601 if (ret.func == FFA_ERROR_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001602 /* TODO: partial defrag of failed range. */
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001603 goto out;
1604 }
1605
1606 /* Clear the memory so no VM or device can see the previous contents. */
J-Alves7db32002021-12-14 14:44:50 +00001607 if (clear &&
J-Alves26483382023-04-20 12:01:49 +01001608 !ffa_clear_memory_constituents(sender_orig_mode, fragments,
1609 fragment_constituent_counts,
1610 fragment_count, page_pool)) {
J-Alvesb5084cf2022-07-06 14:20:12 +01001611 dlog_verbose("Couldn't clear constituents.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001612 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001613 goto out;
1614 }
1615
Jose Marinho09b1db82019-08-08 09:16:59 +01001616 /*
1617 * Complete the transfer by mapping the memory into the recipient. This
1618 * won't allocate because the transaction was already prepared above, so
1619 * it doesn't need to use the `local_page_pool`.
1620 */
J-Alvesfd206052023-05-22 16:45:00 +01001621 CHECK(ffa_region_group_identity_map(
1622 to_locked, fragments, fragment_constituent_counts,
1623 fragment_count, to_mode, page_pool, map_action, NULL)
J-Alvescf6253e2024-01-03 13:48:48 +00001624 .func == FFA_SUCCESS_32);
Jose Marinho09b1db82019-08-08 09:16:59 +01001625
J-Alves460d36c2023-10-12 17:02:15 +01001626 /* Return the mode used in mapping the memory in retriever's PT. */
1627 if (response_mode != NULL) {
1628 *response_mode = to_mode;
1629 }
1630
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001631 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Jose Marinho09b1db82019-08-08 09:16:59 +01001632
1633out:
1634 mpool_fini(&local_page_pool);
1635
1636 /*
Andrew Walbranf07f04d2020-05-01 18:09:00 +01001637 * Tidy up the page table by reclaiming failed mappings (if there was an
1638 * error) or merging entries into blocks where possible (on success).
Jose Marinho09b1db82019-08-08 09:16:59 +01001639 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001640 vm_ptable_defrag(to_locked, page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001641
1642 return ret;
1643}
1644
Andrew Walbran996d1d12020-05-27 14:08:43 +01001645static struct ffa_value ffa_relinquish_check_update(
J-Alves26483382023-04-20 12:01:49 +01001646 struct vm_locked from_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01001647 struct ffa_memory_region_constituent **fragments,
1648 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
1649 struct mpool *page_pool, bool clear)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001650{
1651 uint32_t orig_from_mode;
1652 uint32_t from_mode;
1653 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001654 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001655
Andrew Walbranca808b12020-05-15 17:22:28 +01001656 ret = ffa_relinquish_check_transition(
1657 from_locked, &orig_from_mode, fragments,
1658 fragment_constituent_counts, fragment_count, &from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001659 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001660 dlog_verbose("Invalid transition for relinquish.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +01001661 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001662 }
1663
1664 /*
1665 * Create a local pool so any freed memory can't be used by another
1666 * thread. This is to ensure the original mapping can be restored if the
1667 * clear fails.
1668 */
1669 mpool_init_with_fallback(&local_page_pool, page_pool);
1670
1671 /*
1672 * First reserve all required memory for the new page table entries
1673 * without committing, to make sure the entire operation will succeed
1674 * without exhausting the page pool.
1675 */
J-Alvescf6253e2024-01-03 13:48:48 +00001676 ret = ffa_region_group_identity_map(
1677 from_locked, fragments, fragment_constituent_counts,
1678 fragment_count, from_mode, page_pool, MAP_ACTION_CHECK, NULL);
1679 if (ret.func == FFA_ERROR_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001680 goto out;
1681 }
1682
1683 /*
1684 * Update the mapping for the sender. This won't allocate because the
1685 * transaction was already prepared above, but may free pages in the
1686 * case that a whole block is being unmapped that was previously
1687 * partially mapped.
1688 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001689 CHECK(ffa_region_group_identity_map(
J-Alvescf6253e2024-01-03 13:48:48 +00001690 from_locked, fragments, fragment_constituent_counts,
1691 fragment_count, from_mode, &local_page_pool,
1692 MAP_ACTION_COMMIT, NULL)
1693 .func == FFA_SUCCESS_32);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001694
1695 /* Clear the memory so no VM or device can see the previous contents. */
J-Alves7db32002021-12-14 14:44:50 +00001696 if (clear &&
J-Alves26483382023-04-20 12:01:49 +01001697 !ffa_clear_memory_constituents(orig_from_mode, fragments,
1698 fragment_constituent_counts,
1699 fragment_count, page_pool)) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001700 /*
1701 * On failure, roll back by returning memory to the sender. This
1702 * may allocate pages which were previously freed into
1703 * `local_page_pool` by the call above, but will never allocate
1704 * more pages than that so can never fail.
1705 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001706 CHECK(ffa_region_group_identity_map(
J-Alvescf6253e2024-01-03 13:48:48 +00001707 from_locked, fragments,
1708 fragment_constituent_counts, fragment_count,
1709 orig_from_mode, &local_page_pool,
1710 MAP_ACTION_COMMIT, NULL)
1711 .func == FFA_SUCCESS_32);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001712
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001713 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001714 goto out;
1715 }
1716
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001717 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001718
1719out:
1720 mpool_fini(&local_page_pool);
1721
1722 /*
1723 * Tidy up the page table by reclaiming failed mappings (if there was an
1724 * error) or merging entries into blocks where possible (on success).
1725 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001726 vm_ptable_defrag(from_locked, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +01001727
1728 return ret;
1729}
1730
1731/**
Andrew Walbranca808b12020-05-15 17:22:28 +01001732 * Complete a memory sending operation by checking that it is valid, updating
1733 * the sender page table, and then either marking the share state as having
1734 * completed sending (on success) or freeing it (on failure).
1735 *
1736 * Returns FFA_SUCCESS with the handle encoded, or the relevant FFA_ERROR.
1737 */
J-Alvesfdd29272022-07-19 13:16:31 +01001738struct ffa_value ffa_memory_send_complete(
Andrew Walbranca808b12020-05-15 17:22:28 +01001739 struct vm_locked from_locked, struct share_states_locked share_states,
Andrew Walbran37c574e2020-06-03 11:45:46 +01001740 struct ffa_memory_share_state *share_state, struct mpool *page_pool,
1741 uint32_t *orig_from_mode_ret)
Andrew Walbranca808b12020-05-15 17:22:28 +01001742{
1743 struct ffa_memory_region *memory_region = share_state->memory_region;
J-Alves8f11cde2022-12-21 16:18:22 +00001744 struct ffa_composite_memory_region *composite;
Andrew Walbranca808b12020-05-15 17:22:28 +01001745 struct ffa_value ret;
1746
1747 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +00001748 assert(share_states.share_states != NULL);
J-Alves8f11cde2022-12-21 16:18:22 +00001749 assert(memory_region != NULL);
1750 composite = ffa_memory_region_get_composite(memory_region, 0);
1751 assert(composite != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +01001752
1753 /* Check that state is valid in sender page table and update. */
1754 ret = ffa_send_check_update(
1755 from_locked, share_state->fragments,
1756 share_state->fragment_constituent_counts,
J-Alves8f11cde2022-12-21 16:18:22 +00001757 share_state->fragment_count, composite->page_count,
Daniel Boulbya76fd912024-02-22 14:22:15 +00001758 share_state->share_func, memory_region, page_pool,
J-Alves460d36c2023-10-12 17:02:15 +01001759 orig_from_mode_ret, &share_state->memory_protected);
Andrew Walbranca808b12020-05-15 17:22:28 +01001760 if (ret.func != FFA_SUCCESS_32) {
1761 /*
1762 * Free share state, it failed to send so it can't be retrieved.
1763 */
Karl Meakin4cec5e82023-06-30 16:30:22 +01001764 dlog_verbose("%s: failed to send check update: %s(%s)\n",
1765 __func__, ffa_func_name(ret.func),
1766 ffa_error_name(ffa_error_code(ret)));
Andrew Walbranca808b12020-05-15 17:22:28 +01001767 share_state_free(share_states, share_state, page_pool);
1768 return ret;
1769 }
1770
1771 share_state->sending_complete = true;
Karl Meakin4cec5e82023-06-30 16:30:22 +01001772 dlog_verbose("%s: marked sending complete.\n", __func__);
Andrew Walbranca808b12020-05-15 17:22:28 +01001773
J-Alvesee68c542020-10-29 17:48:20 +00001774 return ffa_mem_success(share_state->memory_region->handle);
Andrew Walbranca808b12020-05-15 17:22:28 +01001775}
1776
1777/**
Daniel Boulby9764ff62024-01-30 17:47:39 +00001778 * Check that the memory attributes match Hafnium expectations.
1779 * Cacheability:
1780 * - Normal Memory as `FFA_MEMORY_CACHE_WRITE_BACK`.
1781 * - Device memory as `FFA_MEMORY_DEV_NGNRNE`.
1782 *
1783 * Shareability:
1784 * - Inner Shareable.
Federico Recanatia98603a2021-12-20 18:04:03 +01001785 */
1786static struct ffa_value ffa_memory_attributes_validate(
J-Alves7a99d0d2023-02-08 13:49:48 +00001787 ffa_memory_attributes_t attributes)
Federico Recanatia98603a2021-12-20 18:04:03 +01001788{
1789 enum ffa_memory_type memory_type;
1790 enum ffa_memory_cacheability cacheability;
1791 enum ffa_memory_shareability shareability;
1792
Karl Meakin84710f32023-10-12 15:14:49 +01001793 memory_type = attributes.type;
Daniel Boulby9764ff62024-01-30 17:47:39 +00001794 cacheability = attributes.cacheability;
1795 if (memory_type == FFA_MEMORY_NORMAL_MEM &&
1796 cacheability != FFA_MEMORY_CACHE_WRITE_BACK) {
1797 dlog_verbose(
1798 "Normal Memory: Invalid cacheability %s, "
1799 "expected %s.\n",
1800 ffa_memory_cacheability_name(cacheability),
1801 ffa_memory_cacheability_name(
1802 FFA_MEMORY_CACHE_WRITE_BACK));
Federico Recanati3d953f32022-02-17 09:31:29 +01001803 return ffa_error(FFA_DENIED);
Federico Recanatia98603a2021-12-20 18:04:03 +01001804 }
Daniel Boulby9764ff62024-01-30 17:47:39 +00001805 if (memory_type == FFA_MEMORY_DEVICE_MEM &&
1806 cacheability != FFA_MEMORY_DEV_NGNRNE) {
1807 dlog_verbose(
1808 "Device Memory: Invalid cacheability %s, "
1809 "expected %s.\n",
1810 ffa_device_memory_cacheability_name(cacheability),
1811 ffa_device_memory_cacheability_name(
1812 FFA_MEMORY_DEV_NGNRNE));
Federico Recanati3d953f32022-02-17 09:31:29 +01001813 return ffa_error(FFA_DENIED);
Federico Recanatia98603a2021-12-20 18:04:03 +01001814 }
1815
Karl Meakin84710f32023-10-12 15:14:49 +01001816 shareability = attributes.shareability;
Federico Recanatia98603a2021-12-20 18:04:03 +01001817 if (shareability != FFA_MEMORY_INNER_SHAREABLE) {
Karl Meakinf98b2aa2023-10-12 16:09:59 +01001818 dlog_verbose("Invalid shareability %s, expected %s.\n",
1819 ffa_memory_shareability_name(shareability),
1820 ffa_memory_shareability_name(
1821 FFA_MEMORY_INNER_SHAREABLE));
Federico Recanati3d953f32022-02-17 09:31:29 +01001822 return ffa_error(FFA_DENIED);
Federico Recanatia98603a2021-12-20 18:04:03 +01001823 }
1824
1825 return (struct ffa_value){.func = FFA_SUCCESS_32};
1826}
1827
1828/**
Andrew Walbrana65a1322020-04-06 19:32:32 +01001829 * Check that the given `memory_region` represents a valid memory send request
1830 * of the given `share_func` type, return the clear flag and permissions via the
1831 * respective output parameters, and update the permissions if necessary.
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001832 *
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001833 * Returns FFA_SUCCESS if the request was valid, or the relevant FFA_ERROR if
Andrew Walbrana65a1322020-04-06 19:32:32 +01001834 * not.
1835 */
J-Alves66652252022-07-06 09:49:51 +01001836struct ffa_value ffa_memory_send_validate(
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001837 struct vm_locked from_locked, struct ffa_memory_region *memory_region,
1838 uint32_t memory_share_length, uint32_t fragment_length,
J-Alves363f5722022-04-25 17:37:37 +01001839 uint32_t share_func)
Andrew Walbrana65a1322020-04-06 19:32:32 +01001840{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001841 struct ffa_composite_memory_region *composite;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001842 struct ffa_memory_access *receiver =
1843 ffa_memory_region_get_receiver(memory_region, 0);
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001844 uint64_t receivers_end;
1845 uint64_t min_length;
Federico Recanati872cd692022-01-05 13:10:10 +01001846 uint32_t composite_memory_region_offset;
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001847 uint32_t constituents_start;
Andrew Walbran130a8ae2020-05-15 16:27:15 +01001848 uint32_t constituents_length;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001849 enum ffa_data_access data_access;
1850 enum ffa_instruction_access instruction_access;
Olivier Deprez4342a3c2022-02-28 09:37:25 +01001851 enum ffa_memory_security security_state;
Karl Meakinf98b2aa2023-10-12 16:09:59 +01001852 enum ffa_memory_type type;
Federico Recanatia98603a2021-12-20 18:04:03 +01001853 struct ffa_value ret;
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001854 const size_t minimum_first_fragment_length =
Daniel Boulby41ef8ba2023-10-13 17:01:22 +01001855 memory_region->receivers_offset +
1856 memory_region->memory_access_desc_size +
1857 sizeof(struct ffa_composite_memory_region);
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001858
1859 if (fragment_length < minimum_first_fragment_length) {
1860 dlog_verbose("Fragment length %u too short (min %u).\n",
1861 (size_t)fragment_length,
1862 minimum_first_fragment_length);
1863 return ffa_error(FFA_INVALID_PARAMETERS);
1864 }
1865
Demi Marie Obenour73a1e942023-02-04 14:09:18 -05001866 static_assert(sizeof(struct ffa_memory_region_constituent) == 16,
1867 "struct ffa_memory_region_constituent must be 16 bytes");
1868 if (!is_aligned(fragment_length,
1869 sizeof(struct ffa_memory_region_constituent)) ||
1870 !is_aligned(memory_share_length,
1871 sizeof(struct ffa_memory_region_constituent))) {
1872 dlog_verbose(
1873 "Fragment length %u or total length %u"
1874 " is not 16-byte aligned.\n",
1875 fragment_length, memory_share_length);
1876 return ffa_error(FFA_INVALID_PARAMETERS);
1877 }
1878
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001879 if (fragment_length > memory_share_length) {
1880 dlog_verbose(
1881 "Fragment length %u greater than total length %u.\n",
1882 (size_t)fragment_length, (size_t)memory_share_length);
1883 return ffa_error(FFA_INVALID_PARAMETERS);
1884 }
Andrew Walbrana65a1322020-04-06 19:32:32 +01001885
J-Alves95df0ef2022-12-07 10:09:48 +00001886 /* The sender must match the caller. */
1887 if ((!vm_id_is_current_world(from_locked.vm->id) &&
1888 vm_id_is_current_world(memory_region->sender)) ||
1889 (vm_id_is_current_world(from_locked.vm->id) &&
1890 memory_region->sender != from_locked.vm->id)) {
1891 dlog_verbose("Invalid memory sender ID.\n");
1892 return ffa_error(FFA_DENIED);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001893 }
1894
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001895 if (memory_region->receiver_count <= 0) {
1896 dlog_verbose("No receivers!\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001897 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001898 }
1899
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001900 /*
1901 * Ensure that the composite header is within the memory bounds and
1902 * doesn't overlap the first part of the message. Cast to uint64_t
1903 * to prevent overflow.
1904 */
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001905 receivers_end = ((uint64_t)memory_region->memory_access_desc_size *
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001906 (uint64_t)memory_region->receiver_count) +
Daniel Boulby41ef8ba2023-10-13 17:01:22 +01001907 memory_region->receivers_offset;
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001908 min_length = receivers_end +
1909 sizeof(struct ffa_composite_memory_region) +
1910 sizeof(struct ffa_memory_region_constituent);
1911 if (min_length > memory_share_length) {
1912 dlog_verbose("Share too short: got %u but minimum is %u.\n",
1913 (size_t)memory_share_length, (size_t)min_length);
1914 return ffa_error(FFA_INVALID_PARAMETERS);
1915 }
1916
1917 composite_memory_region_offset =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001918 receiver->composite_memory_region_offset;
Andrew Walbrana65a1322020-04-06 19:32:32 +01001919
1920 /*
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001921 * Check that the composite memory region descriptor is after the access
1922 * descriptors, is at least 16-byte aligned, and fits in the first
1923 * fragment.
Andrew Walbrana65a1322020-04-06 19:32:32 +01001924 */
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001925 if ((composite_memory_region_offset < receivers_end) ||
1926 (composite_memory_region_offset % 16 != 0) ||
1927 (composite_memory_region_offset >
1928 fragment_length - sizeof(struct ffa_composite_memory_region))) {
1929 dlog_verbose(
1930 "Invalid composite memory region descriptor offset "
1931 "%u.\n",
1932 (size_t)composite_memory_region_offset);
1933 return ffa_error(FFA_INVALID_PARAMETERS);
1934 }
1935
1936 /*
1937 * Compute the start of the constituent regions. Already checked
1938 * to be not more than fragment_length and thus not more than
1939 * memory_share_length.
1940 */
1941 constituents_start = composite_memory_region_offset +
1942 sizeof(struct ffa_composite_memory_region);
1943 constituents_length = memory_share_length - constituents_start;
1944
1945 /*
1946 * Check that the number of constituents is consistent with the length
1947 * of the constituent region.
1948 */
1949 composite = ffa_memory_region_get_composite(memory_region, 0);
1950 if ((constituents_length %
1951 sizeof(struct ffa_memory_region_constituent) !=
1952 0) ||
1953 ((constituents_length /
1954 sizeof(struct ffa_memory_region_constituent)) !=
1955 composite->constituent_count)) {
1956 dlog_verbose("Invalid length %u or composite offset %u.\n",
1957 (size_t)memory_share_length,
1958 (size_t)composite_memory_region_offset);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001959 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001960 }
Andrew Walbranca808b12020-05-15 17:22:28 +01001961 if (fragment_length < memory_share_length &&
1962 fragment_length < HF_MAILBOX_SIZE) {
1963 dlog_warning(
1964 "Initial fragment length %d smaller than mailbox "
1965 "size.\n",
1966 fragment_length);
1967 }
Andrew Walbrana65a1322020-04-06 19:32:32 +01001968
Andrew Walbrana65a1322020-04-06 19:32:32 +01001969 /*
1970 * Clear is not allowed for memory sharing, as the sender still has
1971 * access to the memory.
1972 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001973 if ((memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR) &&
1974 share_func == FFA_MEM_SHARE_32) {
Andrew Walbrana65a1322020-04-06 19:32:32 +01001975 dlog_verbose("Memory can't be cleared while being shared.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001976 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001977 }
1978
1979 /* No other flags are allowed/supported here. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001980 if (memory_region->flags & ~FFA_MEMORY_REGION_FLAG_CLEAR) {
Andrew Walbrana65a1322020-04-06 19:32:32 +01001981 dlog_verbose("Invalid flags %#x.\n", memory_region->flags);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001982 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001983 }
1984
J-Alves363f5722022-04-25 17:37:37 +01001985 /* Check that the permissions are valid, for each specified receiver. */
1986 for (uint32_t i = 0U; i < memory_region->receiver_count; i++) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001987 struct ffa_memory_region_attributes receiver_permissions;
1988
1989 receiver = ffa_memory_region_get_receiver(memory_region, i);
1990 assert(receiver != NULL);
1991 receiver_permissions = receiver->receiver_permissions;
J-Alves363f5722022-04-25 17:37:37 +01001992 ffa_memory_access_permissions_t permissions =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001993 receiver_permissions.permissions;
1994 ffa_id_t receiver_id = receiver_permissions.receiver;
J-Alves363f5722022-04-25 17:37:37 +01001995
1996 if (memory_region->sender == receiver_id) {
1997 dlog_verbose("Can't share memory with itself.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001998 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001999 }
Federico Recanati85090c42021-12-15 13:17:54 +01002000
J-Alves363f5722022-04-25 17:37:37 +01002001 for (uint32_t j = i + 1; j < memory_region->receiver_count;
2002 j++) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002003 struct ffa_memory_access *other_receiver =
2004 ffa_memory_region_get_receiver(memory_region,
2005 j);
2006 assert(other_receiver != NULL);
2007
J-Alves363f5722022-04-25 17:37:37 +01002008 if (receiver_id ==
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002009 other_receiver->receiver_permissions.receiver) {
J-Alves363f5722022-04-25 17:37:37 +01002010 dlog_verbose(
2011 "Repeated receiver(%x) in memory send "
2012 "operation.\n",
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002013 other_receiver->receiver_permissions
2014 .receiver);
J-Alves363f5722022-04-25 17:37:37 +01002015 return ffa_error(FFA_INVALID_PARAMETERS);
2016 }
2017 }
2018
2019 if (composite_memory_region_offset !=
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002020 receiver->composite_memory_region_offset) {
J-Alves363f5722022-04-25 17:37:37 +01002021 dlog_verbose(
2022 "All ffa_memory_access should point to the "
2023 "same composite memory region offset.\n");
2024 return ffa_error(FFA_INVALID_PARAMETERS);
2025 }
2026
Karl Meakin84710f32023-10-12 15:14:49 +01002027 data_access = permissions.data_access;
2028 instruction_access = permissions.instruction_access;
J-Alves363f5722022-04-25 17:37:37 +01002029 if (data_access == FFA_DATA_ACCESS_RESERVED ||
2030 instruction_access == FFA_INSTRUCTION_ACCESS_RESERVED) {
2031 dlog_verbose(
2032 "Reserved value for receiver permissions "
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002033 "(data_access = %s, instruction_access = %s)\n",
2034 ffa_data_access_name(data_access),
2035 ffa_instruction_access_name(
2036 instruction_access));
J-Alves363f5722022-04-25 17:37:37 +01002037 return ffa_error(FFA_INVALID_PARAMETERS);
2038 }
2039 if (instruction_access !=
2040 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
2041 dlog_verbose(
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002042 "Invalid instruction access permissions %s "
2043 "for sending memory, expected %s.\n",
2044 ffa_instruction_access_name(instruction_access),
2045 ffa_instruction_access_name(
2046 FFA_INSTRUCTION_ACCESS_RESERVED));
J-Alves363f5722022-04-25 17:37:37 +01002047 return ffa_error(FFA_INVALID_PARAMETERS);
2048 }
2049 if (share_func == FFA_MEM_SHARE_32) {
2050 if (data_access == FFA_DATA_ACCESS_NOT_SPECIFIED) {
2051 dlog_verbose(
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002052 "Invalid data access permissions %s "
2053 "for sharing memory, expected %s.\n",
2054 ffa_data_access_name(data_access),
2055 ffa_data_access_name(
2056 FFA_DATA_ACCESS_NOT_SPECIFIED));
J-Alves363f5722022-04-25 17:37:37 +01002057 return ffa_error(FFA_INVALID_PARAMETERS);
2058 }
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002059 /*
2060 * According to section 10.10.3 of the FF-A v1.1 EAC0
2061 * spec, NX is required for share operations (but must
2062 * not be specified by the sender) so set it in the
2063 * copy that we store, ready to be returned to the
2064 * retriever.
2065 */
2066 if (vm_id_is_current_world(receiver_id)) {
Karl Meakin84710f32023-10-12 15:14:49 +01002067 permissions.instruction_access =
2068 FFA_INSTRUCTION_ACCESS_NX;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002069 receiver_permissions.permissions = permissions;
2070 }
J-Alves363f5722022-04-25 17:37:37 +01002071 }
2072 if (share_func == FFA_MEM_LEND_32 &&
2073 data_access == FFA_DATA_ACCESS_NOT_SPECIFIED) {
2074 dlog_verbose(
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002075 "Invalid data access permissions %s for "
2076 "lending memory, expected %s.\n",
2077 ffa_data_access_name(data_access),
2078 ffa_data_access_name(
2079 FFA_DATA_ACCESS_NOT_SPECIFIED));
J-Alves363f5722022-04-25 17:37:37 +01002080 return ffa_error(FFA_INVALID_PARAMETERS);
2081 }
2082
2083 if (share_func == FFA_MEM_DONATE_32 &&
2084 data_access != FFA_DATA_ACCESS_NOT_SPECIFIED) {
2085 dlog_verbose(
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002086 "Invalid data access permissions %s for "
2087 "donating memory, expected %s.\n",
2088 ffa_data_access_name(data_access),
2089 ffa_data_access_name(
2090 FFA_DATA_ACCESS_NOT_SPECIFIED));
J-Alves363f5722022-04-25 17:37:37 +01002091 return ffa_error(FFA_INVALID_PARAMETERS);
2092 }
Andrew Walbrana65a1322020-04-06 19:32:32 +01002093 }
2094
Olivier Deprez4342a3c2022-02-28 09:37:25 +01002095 /* Memory region attributes NS-Bit MBZ for FFA_MEM_SHARE/LEND/DONATE. */
Karl Meakin84710f32023-10-12 15:14:49 +01002096 security_state = memory_region->attributes.security;
Olivier Deprez4342a3c2022-02-28 09:37:25 +01002097 if (security_state != FFA_MEMORY_SECURITY_UNSPECIFIED) {
2098 dlog_verbose(
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002099 "Invalid security state %s for memory share operation, "
2100 "expected %s.\n",
2101 ffa_memory_security_name(security_state),
2102 ffa_memory_security_name(
2103 FFA_MEMORY_SECURITY_UNSPECIFIED));
Olivier Deprez4342a3c2022-02-28 09:37:25 +01002104 return ffa_error(FFA_INVALID_PARAMETERS);
2105 }
2106
Federico Recanatid937f5e2021-12-20 17:38:23 +01002107 /*
J-Alves807794e2022-06-16 13:42:47 +01002108 * If a memory donate or lend with single borrower, the memory type
2109 * shall not be specified by the sender.
Federico Recanatid937f5e2021-12-20 17:38:23 +01002110 */
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002111 type = memory_region->attributes.type;
J-Alves807794e2022-06-16 13:42:47 +01002112 if (share_func == FFA_MEM_DONATE_32 ||
2113 (share_func == FFA_MEM_LEND_32 &&
2114 memory_region->receiver_count == 1)) {
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002115 if (type != FFA_MEMORY_NOT_SPECIFIED_MEM) {
J-Alves807794e2022-06-16 13:42:47 +01002116 dlog_verbose(
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002117 "Invalid memory type %s for memory share "
2118 "operation, expected %s.\n",
2119 ffa_memory_type_name(type),
2120 ffa_memory_type_name(
2121 FFA_MEMORY_NOT_SPECIFIED_MEM));
J-Alves807794e2022-06-16 13:42:47 +01002122 return ffa_error(FFA_INVALID_PARAMETERS);
2123 }
2124 } else {
2125 /*
2126 * Check that sender's memory attributes match Hafnium
2127 * expectations: Normal Memory, Inner shareable, Write-Back
2128 * Read-Allocate Write-Allocate Cacheable.
2129 */
2130 ret = ffa_memory_attributes_validate(memory_region->attributes);
2131 if (ret.func != FFA_SUCCESS_32) {
2132 return ret;
2133 }
Federico Recanatid937f5e2021-12-20 17:38:23 +01002134 }
2135
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002136 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbrana65a1322020-04-06 19:32:32 +01002137}
2138
2139/**
Andrew Walbranca808b12020-05-15 17:22:28 +01002140 * Gets the share state for continuing an operation to donate, lend or share
2141 * memory, and checks that it is a valid request.
2142 *
2143 * Returns FFA_SUCCESS if the request was valid, or the relevant FFA_ERROR if
2144 * not.
2145 */
J-Alvesfdd29272022-07-19 13:16:31 +01002146struct ffa_value ffa_memory_send_continue_validate(
Andrew Walbranca808b12020-05-15 17:22:28 +01002147 struct share_states_locked share_states, ffa_memory_handle_t handle,
J-Alves19e20cf2023-08-02 12:48:55 +01002148 struct ffa_memory_share_state **share_state_ret, ffa_id_t from_vm_id,
Andrew Walbranca808b12020-05-15 17:22:28 +01002149 struct mpool *page_pool)
2150{
2151 struct ffa_memory_share_state *share_state;
2152 struct ffa_memory_region *memory_region;
2153
Daniel Boulbya2f8c662021-11-26 17:52:53 +00002154 assert(share_state_ret != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +01002155
2156 /*
2157 * Look up the share state by handle and make sure that the VM ID
2158 * matches.
2159 */
Karl Meakin4a2854a2023-06-30 16:26:52 +01002160 share_state = get_share_state(share_states, handle);
J-Alvesb56aac82023-11-10 09:44:43 +00002161 if (share_state == NULL) {
Andrew Walbranca808b12020-05-15 17:22:28 +01002162 dlog_verbose(
2163 "Invalid handle %#x for memory send continuation.\n",
2164 handle);
2165 return ffa_error(FFA_INVALID_PARAMETERS);
2166 }
2167 memory_region = share_state->memory_region;
2168
J-Alvesfdd29272022-07-19 13:16:31 +01002169 if (vm_id_is_current_world(from_vm_id) &&
2170 memory_region->sender != from_vm_id) {
Andrew Walbranca808b12020-05-15 17:22:28 +01002171 dlog_verbose("Invalid sender %d.\n", memory_region->sender);
2172 return ffa_error(FFA_INVALID_PARAMETERS);
2173 }
2174
2175 if (share_state->sending_complete) {
2176 dlog_verbose(
2177 "Sending of memory handle %#x is already complete.\n",
2178 handle);
2179 return ffa_error(FFA_INVALID_PARAMETERS);
2180 }
2181
2182 if (share_state->fragment_count == MAX_FRAGMENTS) {
2183 /*
2184 * Log a warning as this is a sign that MAX_FRAGMENTS should
2185 * probably be increased.
2186 */
2187 dlog_warning(
2188 "Too many fragments for memory share with handle %#x; "
2189 "only %d supported.\n",
2190 handle, MAX_FRAGMENTS);
2191 /* Free share state, as it's not possible to complete it. */
2192 share_state_free(share_states, share_state, page_pool);
2193 return ffa_error(FFA_NO_MEMORY);
2194 }
2195
2196 *share_state_ret = share_state;
2197
2198 return (struct ffa_value){.func = FFA_SUCCESS_32};
2199}
2200
2201/**
J-Alves95df0ef2022-12-07 10:09:48 +00002202 * Checks if there is at least one receiver from the other world.
2203 */
J-Alvesfdd29272022-07-19 13:16:31 +01002204bool memory_region_receivers_from_other_world(
J-Alves95df0ef2022-12-07 10:09:48 +00002205 struct ffa_memory_region *memory_region)
2206{
2207 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002208 struct ffa_memory_access *receiver =
2209 ffa_memory_region_get_receiver(memory_region, i);
2210 assert(receiver != NULL);
2211 ffa_id_t receiver_id = receiver->receiver_permissions.receiver;
2212
2213 if (!vm_id_is_current_world(receiver_id)) {
J-Alves95df0ef2022-12-07 10:09:48 +00002214 return true;
2215 }
2216 }
2217 return false;
2218}
2219
2220/**
J-Alves9da280b2022-12-21 14:55:39 +00002221 * Validates a call to donate, lend or share memory in which Hafnium is the
2222 * designated allocator of the memory handle. In practice, this also means
2223 * Hafnium is responsible for managing the state structures for the transaction.
2224 * If Hafnium is the SPMC, it should allocate the memory handle when either the
2225 * sender is an SP or there is at least one borrower that is an SP.
2226 * If Hafnium is the hypervisor, it should allocate the memory handle when
2227 * operation involves only NWd VMs.
2228 *
2229 * If validation goes well, Hafnium updates the stage-2 page tables of the
2230 * sender. Validation consists of checking if the message length and number of
2231 * memory region constituents match, and if the transition is valid for the
2232 * type of memory sending operation.
Andrew Walbran475c1452020-02-07 13:22:22 +00002233 *
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002234 * Assumes that the caller has already found and locked the sender VM and copied
2235 * the memory region descriptor from the sender's TX buffer to a freshly
2236 * allocated page from Hafnium's internal pool. The caller must have also
2237 * validated that the receiver VM ID is valid.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002238 *
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002239 * This function takes ownership of the `memory_region` passed in and will free
2240 * it when necessary; it must not be freed by the caller.
Jose Marinho09b1db82019-08-08 09:16:59 +01002241 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002242struct ffa_value ffa_memory_send(struct vm_locked from_locked,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002243 struct ffa_memory_region *memory_region,
Andrew Walbran130a8ae2020-05-15 16:27:15 +01002244 uint32_t memory_share_length,
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002245 uint32_t fragment_length, uint32_t share_func,
2246 struct mpool *page_pool)
Jose Marinho09b1db82019-08-08 09:16:59 +01002247{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002248 struct ffa_value ret;
Andrew Walbranca808b12020-05-15 17:22:28 +01002249 struct share_states_locked share_states;
2250 struct ffa_memory_share_state *share_state;
Jose Marinho09b1db82019-08-08 09:16:59 +01002251
2252 /*
Andrew Walbrana65a1322020-04-06 19:32:32 +01002253 * If there is an error validating the `memory_region` then we need to
2254 * free it because we own it but we won't be storing it in a share state
2255 * after all.
Jose Marinho09b1db82019-08-08 09:16:59 +01002256 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002257 ret = ffa_memory_send_validate(from_locked, memory_region,
2258 memory_share_length, fragment_length,
J-Alves363f5722022-04-25 17:37:37 +01002259 share_func);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002260 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002261 mpool_free(page_pool, memory_region);
Andrew Walbrana65a1322020-04-06 19:32:32 +01002262 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01002263 }
2264
Andrew Walbrana65a1322020-04-06 19:32:32 +01002265 /* Set flag for share function, ready to be retrieved later. */
2266 switch (share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002267 case FFA_MEM_SHARE_32:
Andrew Walbrana65a1322020-04-06 19:32:32 +01002268 memory_region->flags |=
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002269 FFA_MEMORY_REGION_TRANSACTION_TYPE_SHARE;
Andrew Walbrana65a1322020-04-06 19:32:32 +01002270 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002271 case FFA_MEM_LEND_32:
2272 memory_region->flags |= FFA_MEMORY_REGION_TRANSACTION_TYPE_LEND;
Andrew Walbrana65a1322020-04-06 19:32:32 +01002273 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002274 case FFA_MEM_DONATE_32:
Andrew Walbrana65a1322020-04-06 19:32:32 +01002275 memory_region->flags |=
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002276 FFA_MEMORY_REGION_TRANSACTION_TYPE_DONATE;
Andrew Walbrana65a1322020-04-06 19:32:32 +01002277 break;
Jose Marinho09b1db82019-08-08 09:16:59 +01002278 }
2279
Andrew Walbranca808b12020-05-15 17:22:28 +01002280 share_states = share_states_lock();
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002281 /*
2282 * Allocate a share state before updating the page table. Otherwise if
2283 * updating the page table succeeded but allocating the share state
2284 * failed then it would leave the memory in a state where nobody could
2285 * get it back.
2286 */
Karl Meakin52cdfe72023-06-30 14:49:10 +01002287 share_state = allocate_share_state(share_states, share_func,
2288 memory_region, fragment_length,
2289 FFA_MEMORY_HANDLE_INVALID);
J-Alvesb56aac82023-11-10 09:44:43 +00002290 if (share_state == NULL) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002291 dlog_verbose("Failed to allocate share state.\n");
2292 mpool_free(page_pool, memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +01002293 ret = ffa_error(FFA_NO_MEMORY);
2294 goto out;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002295 }
2296
Andrew Walbranca808b12020-05-15 17:22:28 +01002297 if (fragment_length == memory_share_length) {
2298 /* No more fragments to come, everything fit in one message. */
J-Alves2a0d2882020-10-29 14:49:50 +00002299 ret = ffa_memory_send_complete(
2300 from_locked, share_states, share_state, page_pool,
2301 &(share_state->sender_orig_mode));
Andrew Walbranca808b12020-05-15 17:22:28 +01002302 } else {
J-Alvesfdd29272022-07-19 13:16:31 +01002303 /*
2304 * Use sender ID from 'memory_region' assuming
2305 * that at this point it has been validated:
2306 * - MBZ at virtual FF-A instance.
2307 */
J-Alves19e20cf2023-08-02 12:48:55 +01002308 ffa_id_t sender_to_ret =
J-Alvesfdd29272022-07-19 13:16:31 +01002309 (from_locked.vm->id == HF_OTHER_WORLD_ID)
2310 ? memory_region->sender
2311 : 0;
Andrew Walbranca808b12020-05-15 17:22:28 +01002312 ret = (struct ffa_value){
2313 .func = FFA_MEM_FRAG_RX_32,
J-Alvesee68c542020-10-29 17:48:20 +00002314 .arg1 = (uint32_t)memory_region->handle,
2315 .arg2 = (uint32_t)(memory_region->handle >> 32),
J-Alvesfdd29272022-07-19 13:16:31 +01002316 .arg3 = fragment_length,
2317 .arg4 = (uint32_t)(sender_to_ret & 0xffff) << 16};
Andrew Walbranca808b12020-05-15 17:22:28 +01002318 }
2319
2320out:
2321 share_states_unlock(&share_states);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002322 dump_share_states();
Andrew Walbranca808b12020-05-15 17:22:28 +01002323 return ret;
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002324}
2325
2326/**
J-Alves8505a8a2022-06-15 18:10:18 +01002327 * Continues an operation to donate, lend or share memory to a VM from current
2328 * world. If this is the last fragment then checks that the transition is valid
2329 * for the type of memory sending operation and updates the stage-2 page tables
2330 * of the sender.
Andrew Walbranca808b12020-05-15 17:22:28 +01002331 *
2332 * Assumes that the caller has already found and locked the sender VM and copied
2333 * the memory region descriptor from the sender's TX buffer to a freshly
2334 * allocated page from Hafnium's internal pool.
2335 *
2336 * This function takes ownership of the `fragment` passed in; it must not be
2337 * freed by the caller.
2338 */
2339struct ffa_value ffa_memory_send_continue(struct vm_locked from_locked,
2340 void *fragment,
2341 uint32_t fragment_length,
2342 ffa_memory_handle_t handle,
2343 struct mpool *page_pool)
2344{
2345 struct share_states_locked share_states = share_states_lock();
2346 struct ffa_memory_share_state *share_state;
2347 struct ffa_value ret;
2348 struct ffa_memory_region *memory_region;
2349
Demi Marie Obenour73a1e942023-02-04 14:09:18 -05002350 CHECK(is_aligned(fragment,
2351 alignof(struct ffa_memory_region_constituent)));
2352 if (fragment_length % sizeof(struct ffa_memory_region_constituent) !=
2353 0) {
2354 dlog_verbose("Fragment length %u misaligned.\n",
2355 fragment_length);
2356 ret = ffa_error(FFA_INVALID_PARAMETERS);
2357 goto out_free_fragment;
2358 }
2359
Andrew Walbranca808b12020-05-15 17:22:28 +01002360 ret = ffa_memory_send_continue_validate(share_states, handle,
2361 &share_state,
2362 from_locked.vm->id, page_pool);
2363 if (ret.func != FFA_SUCCESS_32) {
2364 goto out_free_fragment;
2365 }
2366 memory_region = share_state->memory_region;
2367
J-Alves95df0ef2022-12-07 10:09:48 +00002368 if (memory_region_receivers_from_other_world(memory_region)) {
Andrew Walbranca808b12020-05-15 17:22:28 +01002369 dlog_error(
2370 "Got hypervisor-allocated handle for memory send to "
J-Alves8505a8a2022-06-15 18:10:18 +01002371 "other world. This should never happen, and indicates "
2372 "a bug in "
Andrew Walbranca808b12020-05-15 17:22:28 +01002373 "EL3 code.\n");
2374 ret = ffa_error(FFA_INVALID_PARAMETERS);
2375 goto out_free_fragment;
2376 }
2377
2378 /* Add this fragment. */
2379 share_state->fragments[share_state->fragment_count] = fragment;
2380 share_state->fragment_constituent_counts[share_state->fragment_count] =
2381 fragment_length / sizeof(struct ffa_memory_region_constituent);
2382 share_state->fragment_count++;
2383
2384 /* Check whether the memory send operation is now ready to complete. */
2385 if (share_state_sending_complete(share_states, share_state)) {
J-Alves2a0d2882020-10-29 14:49:50 +00002386 ret = ffa_memory_send_complete(
2387 from_locked, share_states, share_state, page_pool,
2388 &(share_state->sender_orig_mode));
Andrew Walbranca808b12020-05-15 17:22:28 +01002389 } else {
2390 ret = (struct ffa_value){
2391 .func = FFA_MEM_FRAG_RX_32,
2392 .arg1 = (uint32_t)handle,
2393 .arg2 = (uint32_t)(handle >> 32),
2394 .arg3 = share_state_next_fragment_offset(share_states,
2395 share_state)};
2396 }
2397 goto out;
2398
2399out_free_fragment:
2400 mpool_free(page_pool, fragment);
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002401
2402out:
Andrew Walbranca808b12020-05-15 17:22:28 +01002403 share_states_unlock(&share_states);
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002404 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002405}
2406
Andrew Walbranca808b12020-05-15 17:22:28 +01002407/** Clean up after the receiver has finished retrieving a memory region. */
2408static void ffa_memory_retrieve_complete(
2409 struct share_states_locked share_states,
2410 struct ffa_memory_share_state *share_state, struct mpool *page_pool)
2411{
2412 if (share_state->share_func == FFA_MEM_DONATE_32) {
2413 /*
2414 * Memory that has been donated can't be relinquished,
2415 * so no need to keep the share state around.
2416 */
2417 share_state_free(share_states, share_state, page_pool);
2418 dlog_verbose("Freed share state for donate.\n");
2419 }
2420}
2421
J-Alves2d8457f2022-10-05 11:06:41 +01002422/**
2423 * Initialises the given memory region descriptor to be used for an
2424 * `FFA_MEM_RETRIEVE_RESP`, including the given constituents for the first
2425 * fragment.
2426 * The memory region descriptor is initialized according to retriever's
2427 * FF-A version.
2428 *
2429 * Returns true on success, or false if the given constituents won't all fit in
2430 * the first fragment.
2431 */
2432static bool ffa_retrieved_memory_region_init(
2433 void *response, uint32_t ffa_version, size_t response_max_size,
J-Alves19e20cf2023-08-02 12:48:55 +01002434 ffa_id_t sender, ffa_memory_attributes_t attributes,
J-Alves2d8457f2022-10-05 11:06:41 +01002435 ffa_memory_region_flags_t flags, ffa_memory_handle_t handle,
Daniel Boulbyde974ca2023-12-12 13:53:31 +00002436 ffa_memory_access_permissions_t permissions,
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002437 struct ffa_memory_access *receivers, size_t receiver_count,
2438 uint32_t memory_access_desc_size, uint32_t page_count,
2439 uint32_t total_constituent_count,
J-Alves2d8457f2022-10-05 11:06:41 +01002440 const struct ffa_memory_region_constituent constituents[],
2441 uint32_t fragment_constituent_count, uint32_t *total_length,
2442 uint32_t *fragment_length)
2443{
2444 struct ffa_composite_memory_region *composite_memory_region;
J-Alves2d8457f2022-10-05 11:06:41 +01002445 uint32_t i;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002446 uint32_t composite_offset;
J-Alves2d8457f2022-10-05 11:06:41 +01002447 uint32_t constituents_offset;
J-Alves2d8457f2022-10-05 11:06:41 +01002448
2449 assert(response != NULL);
2450
2451 if (ffa_version == MAKE_FFA_VERSION(1, 0)) {
2452 struct ffa_memory_region_v1_0 *retrieve_response =
2453 (struct ffa_memory_region_v1_0 *)response;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002454 struct ffa_memory_access_v1_0 *receiver;
J-Alves2d8457f2022-10-05 11:06:41 +01002455
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002456 ffa_memory_region_init_header_v1_0(retrieve_response, sender,
2457 attributes, flags, handle, 0,
2458 receiver_count);
J-Alves2d8457f2022-10-05 11:06:41 +01002459
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002460 receiver = (struct ffa_memory_access_v1_0 *)
2461 retrieve_response->receivers;
J-Alves2d8457f2022-10-05 11:06:41 +01002462 receiver_count = retrieve_response->receiver_count;
2463
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002464 for (uint32_t i = 0; i < receiver_count; i++) {
2465 ffa_id_t receiver_id =
2466 receivers[i].receiver_permissions.receiver;
2467 ffa_memory_receiver_flags_t recv_flags =
2468 receivers[i].receiver_permissions.flags;
2469
2470 /*
2471 * Initialized here as in memory retrieve responses we
2472 * currently expect one borrower to be specified.
2473 */
2474 ffa_memory_access_init_v1_0(
Karl Meakin84710f32023-10-12 15:14:49 +01002475 receiver, receiver_id, permissions.data_access,
2476 permissions.instruction_access, recv_flags);
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002477 }
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002478
2479 composite_offset =
J-Alves2d8457f2022-10-05 11:06:41 +01002480 sizeof(struct ffa_memory_region_v1_0) +
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002481 receiver_count * sizeof(struct ffa_memory_access_v1_0);
2482 receiver->composite_memory_region_offset = composite_offset;
J-Alves2d8457f2022-10-05 11:06:41 +01002483
2484 composite_memory_region = ffa_memory_region_get_composite_v1_0(
2485 retrieve_response, 0);
2486 } else {
J-Alves2d8457f2022-10-05 11:06:41 +01002487 struct ffa_memory_region *retrieve_response =
2488 (struct ffa_memory_region *)response;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002489 struct ffa_memory_access *retrieve_response_receivers;
J-Alves2d8457f2022-10-05 11:06:41 +01002490
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002491 ffa_memory_region_init_header(
2492 retrieve_response, sender, attributes, flags, handle, 0,
2493 receiver_count, memory_access_desc_size);
J-Alves2d8457f2022-10-05 11:06:41 +01002494
2495 /*
2496 * Note that `sizeof(struct_ffa_memory_region)` and
2497 * `sizeof(struct ffa_memory_access)` must both be multiples of
2498 * 16 (as verified by the asserts in `ffa_memory.c`, so it is
2499 * guaranteed that the offset we calculate here is aligned to a
2500 * 64-bit boundary and so 64-bit values can be copied without
2501 * alignment faults.
2502 */
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002503 composite_offset =
Daniel Boulby41ef8ba2023-10-13 17:01:22 +01002504 retrieve_response->receivers_offset +
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002505 (uint32_t)(receiver_count *
2506 retrieve_response->memory_access_desc_size);
J-Alves2d8457f2022-10-05 11:06:41 +01002507
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002508 retrieve_response_receivers =
2509 ffa_memory_region_get_receiver(retrieve_response, 0);
2510 assert(retrieve_response_receivers != NULL);
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002511
2512 /*
2513 * Initialized here as in memory retrieve responses we currently
2514 * expect one borrower to be specified.
2515 */
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002516 memcpy_s(retrieve_response_receivers,
2517 sizeof(struct ffa_memory_access) * receiver_count,
2518 receivers,
2519 sizeof(struct ffa_memory_access) * receiver_count);
2520
2521 retrieve_response_receivers->composite_memory_region_offset =
2522 composite_offset;
2523
J-Alves2d8457f2022-10-05 11:06:41 +01002524 composite_memory_region =
2525 ffa_memory_region_get_composite(retrieve_response, 0);
2526 }
2527
J-Alves2d8457f2022-10-05 11:06:41 +01002528 assert(composite_memory_region != NULL);
2529
J-Alves2d8457f2022-10-05 11:06:41 +01002530 composite_memory_region->page_count = page_count;
2531 composite_memory_region->constituent_count = total_constituent_count;
2532 composite_memory_region->reserved_0 = 0;
2533
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002534 constituents_offset =
2535 composite_offset + sizeof(struct ffa_composite_memory_region);
J-Alves2d8457f2022-10-05 11:06:41 +01002536 if (constituents_offset +
2537 fragment_constituent_count *
2538 sizeof(struct ffa_memory_region_constituent) >
2539 response_max_size) {
2540 return false;
2541 }
2542
2543 for (i = 0; i < fragment_constituent_count; ++i) {
2544 composite_memory_region->constituents[i] = constituents[i];
2545 }
2546
2547 if (total_length != NULL) {
2548 *total_length =
2549 constituents_offset +
2550 composite_memory_region->constituent_count *
2551 sizeof(struct ffa_memory_region_constituent);
2552 }
2553 if (fragment_length != NULL) {
2554 *fragment_length =
2555 constituents_offset +
2556 fragment_constituent_count *
2557 sizeof(struct ffa_memory_region_constituent);
2558 }
2559
2560 return true;
2561}
2562
J-Alves96de29f2022-04-26 16:05:24 +01002563/**
2564 * Validates the retrieved permissions against those specified by the lender
2565 * of memory share operation. Optionally can help set the permissions to be used
2566 * for the S2 mapping, through the `permissions` argument.
J-Alvesdcad8992023-09-15 14:10:35 +01002567 * Returns FFA_SUCCESS if all the fields are valid. FFA_ERROR, with error code:
2568 * - FFA_INVALID_PARAMETERS -> if the fields have invalid values as per the
2569 * specification for each ABI.
2570 * - FFA_DENIED -> if the permissions specified by the retriever are not
2571 * less permissive than those provided by the sender.
J-Alves96de29f2022-04-26 16:05:24 +01002572 */
J-Alvesdcad8992023-09-15 14:10:35 +01002573static struct ffa_value ffa_memory_retrieve_is_memory_access_valid(
2574 uint32_t share_func, enum ffa_data_access sent_data_access,
J-Alves96de29f2022-04-26 16:05:24 +01002575 enum ffa_data_access requested_data_access,
2576 enum ffa_instruction_access sent_instruction_access,
2577 enum ffa_instruction_access requested_instruction_access,
J-Alvesdcad8992023-09-15 14:10:35 +01002578 ffa_memory_access_permissions_t *permissions, bool multiple_borrowers)
J-Alves96de29f2022-04-26 16:05:24 +01002579{
2580 switch (sent_data_access) {
2581 case FFA_DATA_ACCESS_NOT_SPECIFIED:
2582 case FFA_DATA_ACCESS_RW:
2583 if (requested_data_access == FFA_DATA_ACCESS_NOT_SPECIFIED ||
2584 requested_data_access == FFA_DATA_ACCESS_RW) {
2585 if (permissions != NULL) {
Karl Meakin84710f32023-10-12 15:14:49 +01002586 permissions->data_access = FFA_DATA_ACCESS_RW;
J-Alves96de29f2022-04-26 16:05:24 +01002587 }
2588 break;
2589 }
2590 /* Intentional fall-through. */
2591 case FFA_DATA_ACCESS_RO:
2592 if (requested_data_access == FFA_DATA_ACCESS_NOT_SPECIFIED ||
2593 requested_data_access == FFA_DATA_ACCESS_RO) {
2594 if (permissions != NULL) {
Karl Meakin84710f32023-10-12 15:14:49 +01002595 permissions->data_access = FFA_DATA_ACCESS_RO;
J-Alves96de29f2022-04-26 16:05:24 +01002596 }
2597 break;
2598 }
2599 dlog_verbose(
2600 "Invalid data access requested; sender specified "
2601 "permissions %#x but receiver requested %#x.\n",
2602 sent_data_access, requested_data_access);
J-Alvesdcad8992023-09-15 14:10:35 +01002603 return ffa_error(FFA_DENIED);
J-Alves96de29f2022-04-26 16:05:24 +01002604 case FFA_DATA_ACCESS_RESERVED:
2605 panic("Got unexpected FFA_DATA_ACCESS_RESERVED. Should be "
2606 "checked before this point.");
2607 }
2608
J-Alvesdcad8992023-09-15 14:10:35 +01002609 /*
2610 * For operations with a single borrower, If it is an FFA_MEMORY_LEND
2611 * or FFA_MEMORY_DONATE the retriever should have specifed the
2612 * instruction permissions it wishes to receive.
2613 */
2614 switch (share_func) {
2615 case FFA_MEM_SHARE_32:
2616 if (requested_instruction_access !=
2617 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
2618 dlog_verbose(
2619 "%s: for share instruction permissions must "
2620 "NOT be specified.\n",
2621 __func__);
2622 return ffa_error(FFA_INVALID_PARAMETERS);
2623 }
2624 break;
2625 case FFA_MEM_LEND_32:
2626 /*
2627 * For operations with multiple borrowers only permit XN
2628 * permissions, and both Sender and borrower should have used
2629 * FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED.
2630 */
2631 if (multiple_borrowers) {
2632 if (requested_instruction_access !=
2633 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
2634 dlog_verbose(
2635 "%s: lend/share/donate with multiple "
2636 "borrowers "
2637 "instruction permissions must NOT be "
2638 "specified.\n",
2639 __func__);
2640 return ffa_error(FFA_INVALID_PARAMETERS);
2641 }
2642 break;
2643 }
2644 /* Fall through if the operation targets a single borrower. */
2645 case FFA_MEM_DONATE_32:
2646 if (!multiple_borrowers &&
2647 requested_instruction_access ==
2648 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
2649 dlog_verbose(
2650 "%s: for lend/donate with single borrower "
2651 "instruction permissions must be speficified "
2652 "by borrower\n",
2653 __func__);
2654 return ffa_error(FFA_INVALID_PARAMETERS);
2655 }
2656 break;
2657 default:
2658 panic("%s: Wrong func id provided.\n", __func__);
2659 }
2660
J-Alves96de29f2022-04-26 16:05:24 +01002661 switch (sent_instruction_access) {
2662 case FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED:
2663 case FFA_INSTRUCTION_ACCESS_X:
J-Alvesdcad8992023-09-15 14:10:35 +01002664 if (requested_instruction_access == FFA_INSTRUCTION_ACCESS_X) {
J-Alves96de29f2022-04-26 16:05:24 +01002665 if (permissions != NULL) {
Karl Meakin84710f32023-10-12 15:14:49 +01002666 permissions->instruction_access =
2667 FFA_INSTRUCTION_ACCESS_X;
J-Alves96de29f2022-04-26 16:05:24 +01002668 }
2669 break;
2670 }
J-Alvesdcad8992023-09-15 14:10:35 +01002671 /*
2672 * Fall through if requested permissions are less
2673 * permissive than those provided by the sender.
2674 */
J-Alves96de29f2022-04-26 16:05:24 +01002675 case FFA_INSTRUCTION_ACCESS_NX:
2676 if (requested_instruction_access ==
2677 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED ||
2678 requested_instruction_access == FFA_INSTRUCTION_ACCESS_NX) {
2679 if (permissions != NULL) {
Karl Meakin84710f32023-10-12 15:14:49 +01002680 permissions->instruction_access =
2681 FFA_INSTRUCTION_ACCESS_NX;
J-Alves96de29f2022-04-26 16:05:24 +01002682 }
2683 break;
2684 }
2685 dlog_verbose(
2686 "Invalid instruction access requested; sender "
2687 "specified permissions %#x but receiver requested "
2688 "%#x.\n",
2689 sent_instruction_access, requested_instruction_access);
J-Alvesdcad8992023-09-15 14:10:35 +01002690 return ffa_error(FFA_DENIED);
J-Alves96de29f2022-04-26 16:05:24 +01002691 case FFA_INSTRUCTION_ACCESS_RESERVED:
2692 panic("Got unexpected FFA_INSTRUCTION_ACCESS_RESERVED. Should "
2693 "be checked before this point.");
2694 }
2695
J-Alvesdcad8992023-09-15 14:10:35 +01002696 return (struct ffa_value){.func = FFA_SUCCESS_32};
J-Alves96de29f2022-04-26 16:05:24 +01002697}
2698
2699/**
2700 * Validate the receivers' permissions in the retrieve request against those
2701 * specified by the lender.
2702 * In the `permissions` argument returns the permissions to set at S2 for the
2703 * caller to the FFA_MEMORY_RETRIEVE_REQ.
J-Alves3456e032023-07-20 12:20:05 +01002704 * The function looks into the flag to bypass multiple borrower checks:
2705 * - If not set returns FFA_SUCCESS if all specified permissions are valid.
2706 * - If set returns FFA_SUCCESS if the descriptor contains the permissions
2707 * to the caller of FFA_MEM_RETRIEVE_REQ and they are valid. Other permissions
2708 * are ignored, if provided.
J-Alves96de29f2022-04-26 16:05:24 +01002709 */
2710static struct ffa_value ffa_memory_retrieve_validate_memory_access_list(
2711 struct ffa_memory_region *memory_region,
J-Alves19e20cf2023-08-02 12:48:55 +01002712 struct ffa_memory_region *retrieve_request, ffa_id_t to_vm_id,
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002713 ffa_memory_access_permissions_t *permissions,
2714 struct ffa_memory_access **receiver_ret, uint32_t func_id)
J-Alves96de29f2022-04-26 16:05:24 +01002715{
2716 uint32_t retrieve_receiver_index;
J-Alves3456e032023-07-20 12:20:05 +01002717 bool bypass_multi_receiver_check =
2718 (retrieve_request->flags &
2719 FFA_MEMORY_REGION_FLAG_BYPASS_BORROWERS_CHECK) != 0U;
J-Alvesdcad8992023-09-15 14:10:35 +01002720 const uint32_t region_receiver_count = memory_region->receiver_count;
2721 struct ffa_value ret;
J-Alves96de29f2022-04-26 16:05:24 +01002722
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002723 assert(receiver_ret != NULL);
J-Alves96de29f2022-04-26 16:05:24 +01002724 assert(permissions != NULL);
2725
Karl Meakin84710f32023-10-12 15:14:49 +01002726 *permissions = (ffa_memory_access_permissions_t){0};
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002727
J-Alves3456e032023-07-20 12:20:05 +01002728 if (!bypass_multi_receiver_check) {
J-Alvesdcad8992023-09-15 14:10:35 +01002729 if (retrieve_request->receiver_count != region_receiver_count) {
J-Alves3456e032023-07-20 12:20:05 +01002730 dlog_verbose(
2731 "Retrieve request should contain same list of "
2732 "borrowers, as specified by the lender.\n");
2733 return ffa_error(FFA_INVALID_PARAMETERS);
2734 }
2735 } else {
2736 if (retrieve_request->receiver_count != 1) {
2737 dlog_verbose(
2738 "Set bypass multiple borrower check, receiver "
2739 "list must be sized 1 (%x)\n",
2740 memory_region->receiver_count);
2741 return ffa_error(FFA_INVALID_PARAMETERS);
2742 }
J-Alves96de29f2022-04-26 16:05:24 +01002743 }
2744
2745 retrieve_receiver_index = retrieve_request->receiver_count;
2746
J-Alves96de29f2022-04-26 16:05:24 +01002747 for (uint32_t i = 0U; i < retrieve_request->receiver_count; i++) {
2748 ffa_memory_access_permissions_t sent_permissions;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002749 struct ffa_memory_access *retrieve_request_receiver =
2750 ffa_memory_region_get_receiver(retrieve_request, i);
2751 assert(retrieve_request_receiver != NULL);
J-Alves96de29f2022-04-26 16:05:24 +01002752 ffa_memory_access_permissions_t requested_permissions =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002753 retrieve_request_receiver->receiver_permissions
2754 .permissions;
J-Alves19e20cf2023-08-02 12:48:55 +01002755 ffa_id_t current_receiver_id =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002756 retrieve_request_receiver->receiver_permissions
2757 .receiver;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002758 struct ffa_memory_access *receiver;
2759 uint32_t mem_region_receiver_index;
2760 bool permissions_RO;
2761 bool clear_memory_flags;
J-Alves96de29f2022-04-26 16:05:24 +01002762 bool found_to_id = current_receiver_id == to_vm_id;
2763
J-Alves3456e032023-07-20 12:20:05 +01002764 if (bypass_multi_receiver_check && !found_to_id) {
2765 dlog_verbose(
2766 "Bypass multiple borrower check for id %x.\n",
2767 current_receiver_id);
2768 continue;
2769 }
2770
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002771 if (retrieve_request_receiver->composite_memory_region_offset !=
2772 0U) {
2773 dlog_verbose(
2774 "Retriever specified address ranges not "
2775 "supported (got offset %d).\n",
2776 retrieve_request_receiver
2777 ->composite_memory_region_offset);
2778 return ffa_error(FFA_INVALID_PARAMETERS);
2779 }
2780
J-Alves96de29f2022-04-26 16:05:24 +01002781 /*
2782 * Find the current receiver in the transaction descriptor from
2783 * sender.
2784 */
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002785 mem_region_receiver_index =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002786 ffa_memory_region_get_receiver_index(
2787 memory_region, current_receiver_id);
J-Alves96de29f2022-04-26 16:05:24 +01002788
2789 if (mem_region_receiver_index ==
2790 memory_region->receiver_count) {
2791 dlog_verbose("%s: receiver %x not found\n", __func__,
2792 current_receiver_id);
2793 return ffa_error(FFA_DENIED);
2794 }
2795
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002796 receiver = ffa_memory_region_get_receiver(
2797 memory_region, mem_region_receiver_index);
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002798 assert(receiver != NULL);
2799
2800 sent_permissions = receiver->receiver_permissions.permissions;
J-Alves96de29f2022-04-26 16:05:24 +01002801
2802 if (found_to_id) {
2803 retrieve_receiver_index = i;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002804
2805 *receiver_ret = receiver;
J-Alves96de29f2022-04-26 16:05:24 +01002806 }
2807
2808 /*
J-Alvesdcad8992023-09-15 14:10:35 +01002809 * Check if retrieve request memory access list is valid:
2810 * - The retrieve request complies with the specification.
2811 * - Permissions are within those specified by the sender.
J-Alves96de29f2022-04-26 16:05:24 +01002812 */
J-Alvesdcad8992023-09-15 14:10:35 +01002813 ret = ffa_memory_retrieve_is_memory_access_valid(
Karl Meakin84710f32023-10-12 15:14:49 +01002814 func_id, sent_permissions.data_access,
2815 requested_permissions.data_access,
2816 sent_permissions.instruction_access,
2817 requested_permissions.instruction_access,
J-Alvesdcad8992023-09-15 14:10:35 +01002818 found_to_id ? permissions : NULL,
2819 region_receiver_count > 1);
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002820
J-Alvesdcad8992023-09-15 14:10:35 +01002821 if (ret.func != FFA_SUCCESS_32) {
2822 return ret;
J-Alves96de29f2022-04-26 16:05:24 +01002823 }
2824
Karl Meakin84710f32023-10-12 15:14:49 +01002825 permissions_RO =
2826 (permissions->data_access == FFA_DATA_ACCESS_RO);
J-Alvese5262372024-03-27 11:02:03 +00002827 clear_memory_flags =
2828 (retrieve_request->flags &
2829 (FFA_MEMORY_REGION_FLAG_CLEAR |
2830 FFA_MEMORY_REGION_FLAG_CLEAR_RELINQUISH)) != 0U;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002831
J-Alves96de29f2022-04-26 16:05:24 +01002832 /*
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002833 * Can't request PM to clear memory if only provided
2834 * with RO permissions.
J-Alves96de29f2022-04-26 16:05:24 +01002835 */
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002836 if (found_to_id && permissions_RO && clear_memory_flags) {
J-Alves96de29f2022-04-26 16:05:24 +01002837 dlog_verbose(
2838 "Receiver has RO permissions can not request "
2839 "clear.\n");
2840 return ffa_error(FFA_DENIED);
2841 }
Daniel Boulbyde974ca2023-12-12 13:53:31 +00002842
2843 /*
2844 * Check the impdef in the retrieve_request matches the value in
2845 * the original memory send.
2846 */
2847 if (ffa_version_from_memory_access_desc_size(
2848 memory_region->memory_access_desc_size) >=
2849 MAKE_FFA_VERSION(1, 2) &&
2850 ffa_version_from_memory_access_desc_size(
2851 retrieve_request->memory_access_desc_size) >=
2852 MAKE_FFA_VERSION(1, 2)) {
Daniel Boulbyde974ca2023-12-12 13:53:31 +00002853 if (receiver->impdef.val[0] !=
2854 retrieve_request_receiver->impdef.val[0] ||
2855 receiver->impdef.val[1] !=
2856 retrieve_request_receiver->impdef.val[1]) {
2857 dlog_verbose(
2858 "Impdef value in memory send does not "
2859 "match retrieve request value "
2860 "send value %#x %#x retrieve request "
2861 "value %#x %#x\n",
2862 receiver->impdef.val[0],
2863 receiver->impdef.val[1],
2864 retrieve_request_receiver->impdef
2865 .val[0],
2866 retrieve_request_receiver->impdef
2867 .val[1]);
2868 return ffa_error(FFA_INVALID_PARAMETERS);
2869 }
2870 }
J-Alves96de29f2022-04-26 16:05:24 +01002871 }
2872
2873 if (retrieve_receiver_index == retrieve_request->receiver_count) {
2874 dlog_verbose(
2875 "Retrieve request does not contain caller's (%x) "
2876 "permissions\n",
2877 to_vm_id);
2878 return ffa_error(FFA_INVALID_PARAMETERS);
2879 }
2880
2881 return (struct ffa_value){.func = FFA_SUCCESS_32};
2882}
2883
J-Alvesa9cd7e32022-07-01 13:49:33 +01002884/*
2885 * According to section 16.4.3 of FF-A v1.1 EAC0 specification, the hypervisor
2886 * may issue an FFA_MEM_RETRIEVE_REQ to obtain the memory region description
2887 * of a pending memory sharing operation whose allocator is the SPM, for
2888 * validation purposes before forwarding an FFA_MEM_RECLAIM call. In doing so
2889 * the memory region descriptor of the retrieve request must be zeroed with the
2890 * exception of the sender ID and handle.
2891 */
J-Alves4f0d9c12024-01-17 17:23:11 +00002892bool is_ffa_hypervisor_retrieve_request(struct ffa_memory_region *request,
2893 struct vm_locked to_locked)
J-Alvesa9cd7e32022-07-01 13:49:33 +01002894{
2895 return to_locked.vm->id == HF_HYPERVISOR_VM_ID &&
Karl Meakin84710f32023-10-12 15:14:49 +01002896 request->attributes.shareability == 0U &&
2897 request->attributes.cacheability == 0U &&
2898 request->attributes.type == 0U &&
2899 request->attributes.security == 0U && request->flags == 0U &&
J-Alvesa9cd7e32022-07-01 13:49:33 +01002900 request->tag == 0U && request->receiver_count == 0U &&
2901 plat_ffa_memory_handle_allocated_by_current_world(
2902 request->handle);
2903}
2904
2905/*
2906 * Helper to reset count of fragments retrieved by the hypervisor.
2907 */
2908static void ffa_memory_retrieve_complete_from_hyp(
2909 struct ffa_memory_share_state *share_state)
2910{
2911 if (share_state->hypervisor_fragment_count ==
2912 share_state->fragment_count) {
2913 share_state->hypervisor_fragment_count = 0;
2914 }
2915}
2916
J-Alves089004f2022-07-13 14:25:44 +01002917/**
J-Alves4f0d9c12024-01-17 17:23:11 +00002918 * Prepares the return of the ffa_value for the memory retrieve response.
2919 */
2920static struct ffa_value ffa_memory_retrieve_resp(uint32_t total_length,
2921 uint32_t fragment_length)
2922{
2923 return (struct ffa_value){.func = FFA_MEM_RETRIEVE_RESP_32,
2924 .arg1 = total_length,
2925 .arg2 = fragment_length};
2926}
2927
2928/**
J-Alves089004f2022-07-13 14:25:44 +01002929 * Validate that the memory region descriptor provided by the borrower on
2930 * FFA_MEM_RETRIEVE_REQ, against saved memory region provided by lender at the
2931 * memory sharing call.
2932 */
2933static struct ffa_value ffa_memory_retrieve_validate(
J-Alves4f0d9c12024-01-17 17:23:11 +00002934 ffa_id_t to_id, struct ffa_memory_region *retrieve_request,
2935 uint32_t retrieve_request_length,
J-Alves089004f2022-07-13 14:25:44 +01002936 struct ffa_memory_region *memory_region, uint32_t *receiver_index,
2937 uint32_t share_func)
2938{
2939 ffa_memory_region_flags_t transaction_type =
2940 retrieve_request->flags &
2941 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK;
Olivier Deprez4342a3c2022-02-28 09:37:25 +01002942 enum ffa_memory_security security_state;
J-Alves4f0d9c12024-01-17 17:23:11 +00002943 const uint64_t memory_access_desc_size =
2944 retrieve_request->memory_access_desc_size;
2945 const uint32_t expected_retrieve_request_length =
2946 retrieve_request->receivers_offset +
2947 (uint32_t)(retrieve_request->receiver_count *
2948 memory_access_desc_size);
J-Alves089004f2022-07-13 14:25:44 +01002949
2950 assert(retrieve_request != NULL);
2951 assert(memory_region != NULL);
2952 assert(receiver_index != NULL);
J-Alves089004f2022-07-13 14:25:44 +01002953
J-Alves4f0d9c12024-01-17 17:23:11 +00002954 if (retrieve_request_length != expected_retrieve_request_length) {
2955 dlog_verbose(
2956 "Invalid length for FFA_MEM_RETRIEVE_REQ, expected %d "
2957 "but was %d.\n",
2958 expected_retrieve_request_length,
2959 retrieve_request_length);
2960 return ffa_error(FFA_INVALID_PARAMETERS);
2961 }
2962
2963 if (retrieve_request->sender != memory_region->sender) {
2964 dlog_verbose(
2965 "Memory with handle %#x not fully sent, can't "
2966 "retrieve.\n",
2967 memory_region->handle);
2968 return ffa_error(FFA_DENIED);
2969 }
2970
2971 /*
2972 * The SPMC can only process retrieve requests to memory share
2973 * operations with one borrower from the other world. It can't
2974 * determine the ID of the NWd VM that invoked the retrieve
2975 * request interface call. It relies on the hypervisor to
2976 * validate the caller's ID against that provided in the
2977 * `receivers` list of the retrieve response.
2978 * In case there is only one borrower from the NWd in the
2979 * transaction descriptor, record that in the `receiver_id` for
2980 * later use, and validate in the retrieve request message.
2981 * This limitation is due to the fact SPMC can't determine the
2982 * index in the memory share structures state to update.
2983 */
2984 if (to_id == HF_HYPERVISOR_VM_ID) {
2985 uint32_t other_world_count = 0;
2986
2987 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
2988 struct ffa_memory_access *receiver =
2989 ffa_memory_region_get_receiver(retrieve_request,
2990 0);
2991 assert(receiver != NULL);
2992
2993 to_id = receiver->receiver_permissions.receiver;
2994
2995 if (!vm_id_is_current_world(to_id)) {
2996 other_world_count++;
2997 }
2998 }
2999
3000 if (other_world_count > 1) {
3001 dlog_verbose(
3002 "Support one receiver from the other "
3003 "world.\n");
3004 return ffa_error(FFA_NOT_SUPPORTED);
3005 }
3006 }
J-Alves089004f2022-07-13 14:25:44 +01003007 /*
3008 * Check that the transaction type expected by the receiver is
3009 * correct, if it has been specified.
3010 */
3011 if (transaction_type !=
3012 FFA_MEMORY_REGION_TRANSACTION_TYPE_UNSPECIFIED &&
3013 transaction_type != (memory_region->flags &
3014 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK)) {
3015 dlog_verbose(
3016 "Incorrect transaction type %#x for "
3017 "FFA_MEM_RETRIEVE_REQ, expected %#x for handle %#x.\n",
3018 transaction_type,
3019 memory_region->flags &
3020 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK,
3021 retrieve_request->handle);
3022 return ffa_error(FFA_INVALID_PARAMETERS);
3023 }
3024
3025 if (retrieve_request->tag != memory_region->tag) {
3026 dlog_verbose(
3027 "Incorrect tag %d for FFA_MEM_RETRIEVE_REQ, expected "
3028 "%d for handle %#x.\n",
3029 retrieve_request->tag, memory_region->tag,
3030 retrieve_request->handle);
3031 return ffa_error(FFA_INVALID_PARAMETERS);
3032 }
3033
J-Alves4f0d9c12024-01-17 17:23:11 +00003034 *receiver_index =
3035 ffa_memory_region_get_receiver_index(memory_region, to_id);
J-Alves089004f2022-07-13 14:25:44 +01003036
3037 if (*receiver_index == memory_region->receiver_count) {
3038 dlog_verbose(
3039 "Incorrect receiver VM ID %d for "
3040 "FFA_MEM_RETRIEVE_REQ, for handle %#x.\n",
J-Alves4f0d9c12024-01-17 17:23:11 +00003041 to_id, memory_region->handle);
J-Alves089004f2022-07-13 14:25:44 +01003042 return ffa_error(FFA_INVALID_PARAMETERS);
3043 }
3044
3045 if ((retrieve_request->flags &
3046 FFA_MEMORY_REGION_ADDRESS_RANGE_HINT_VALID) != 0U) {
3047 dlog_verbose(
3048 "Retriever specified 'address range alignment 'hint' "
3049 "not supported.\n");
3050 return ffa_error(FFA_INVALID_PARAMETERS);
3051 }
3052 if ((retrieve_request->flags &
3053 FFA_MEMORY_REGION_ADDRESS_RANGE_HINT_MASK) != 0) {
3054 dlog_verbose(
3055 "Bits 8-5 must be zero in memory region's flags "
3056 "(address range alignment hint not supported).\n");
3057 return ffa_error(FFA_INVALID_PARAMETERS);
3058 }
3059
3060 if ((retrieve_request->flags & ~0x7FF) != 0U) {
3061 dlog_verbose(
3062 "Bits 31-10 must be zero in memory region's flags.\n");
3063 return ffa_error(FFA_INVALID_PARAMETERS);
3064 }
3065
3066 if (share_func == FFA_MEM_SHARE_32 &&
3067 (retrieve_request->flags &
3068 (FFA_MEMORY_REGION_FLAG_CLEAR |
3069 FFA_MEMORY_REGION_FLAG_CLEAR_RELINQUISH)) != 0U) {
3070 dlog_verbose(
3071 "Memory Share operation can't clean after relinquish "
3072 "memory region.\n");
3073 return ffa_error(FFA_INVALID_PARAMETERS);
3074 }
3075
3076 /*
3077 * If the borrower needs the memory to be cleared before mapping
3078 * to its address space, the sender should have set the flag
3079 * when calling FFA_MEM_LEND/FFA_MEM_DONATE, else return
3080 * FFA_DENIED.
3081 */
3082 if ((retrieve_request->flags & FFA_MEMORY_REGION_FLAG_CLEAR) != 0U &&
3083 (memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR) == 0U) {
3084 dlog_verbose(
3085 "Borrower needs memory cleared. Sender needs to set "
3086 "flag for clearing memory.\n");
3087 return ffa_error(FFA_DENIED);
3088 }
3089
Olivier Deprez4342a3c2022-02-28 09:37:25 +01003090 /* Memory region attributes NS-Bit MBZ for FFA_MEM_RETRIEVE_REQ. */
Karl Meakin84710f32023-10-12 15:14:49 +01003091 security_state = retrieve_request->attributes.security;
Olivier Deprez4342a3c2022-02-28 09:37:25 +01003092 if (security_state != FFA_MEMORY_SECURITY_UNSPECIFIED) {
3093 dlog_verbose(
3094 "Invalid security state for memory retrieve request "
3095 "operation.\n");
3096 return ffa_error(FFA_INVALID_PARAMETERS);
3097 }
3098
J-Alves089004f2022-07-13 14:25:44 +01003099 /*
3100 * If memory type is not specified, bypass validation of memory
3101 * attributes in the retrieve request. The retriever is expecting to
3102 * obtain this information from the SPMC.
3103 */
Karl Meakin84710f32023-10-12 15:14:49 +01003104 if (retrieve_request->attributes.type == FFA_MEMORY_NOT_SPECIFIED_MEM) {
J-Alves089004f2022-07-13 14:25:44 +01003105 return (struct ffa_value){.func = FFA_SUCCESS_32};
3106 }
3107
3108 /*
3109 * Ensure receiver's attributes are compatible with how
3110 * Hafnium maps memory: Normal Memory, Inner shareable,
3111 * Write-Back Read-Allocate Write-Allocate Cacheable.
3112 */
3113 return ffa_memory_attributes_validate(retrieve_request->attributes);
3114}
3115
J-Alves4f0d9c12024-01-17 17:23:11 +00003116static struct ffa_value ffa_partition_retrieve_request(
3117 struct share_states_locked share_states,
3118 struct ffa_memory_share_state *share_state, struct vm_locked to_locked,
3119 struct ffa_memory_region *retrieve_request,
3120 uint32_t retrieve_request_length, struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003121{
Karl Meakin84710f32023-10-12 15:14:49 +01003122 ffa_memory_access_permissions_t permissions = {0};
Olivier Deprez878bd5b2021-04-15 19:05:10 +02003123 uint32_t memory_to_mode;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003124 struct ffa_value ret;
Andrew Walbranca808b12020-05-15 17:22:28 +01003125 struct ffa_composite_memory_region *composite;
3126 uint32_t total_length;
3127 uint32_t fragment_length;
J-Alves19e20cf2023-08-02 12:48:55 +01003128 ffa_id_t receiver_id = to_locked.vm->id;
J-Alves4f0d9c12024-01-17 17:23:11 +00003129 bool is_retrieve_complete = false;
J-Alves4f0d9c12024-01-17 17:23:11 +00003130 const uint64_t memory_access_desc_size =
Daniel Boulbyde974ca2023-12-12 13:53:31 +00003131 retrieve_request->memory_access_desc_size;
J-Alves4f0d9c12024-01-17 17:23:11 +00003132 uint32_t receiver_index;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003133 struct ffa_memory_access *receiver;
J-Alves4f0d9c12024-01-17 17:23:11 +00003134 ffa_memory_handle_t handle = retrieve_request->handle;
Karl Meakin84710f32023-10-12 15:14:49 +01003135 ffa_memory_attributes_t attributes = {0};
J-Alves460d36c2023-10-12 17:02:15 +01003136 uint32_t retrieve_mode = 0;
J-Alves4f0d9c12024-01-17 17:23:11 +00003137 struct ffa_memory_region *memory_region = share_state->memory_region;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003138
J-Alves96de29f2022-04-26 16:05:24 +01003139 if (!share_state->sending_complete) {
3140 dlog_verbose(
3141 "Memory with handle %#x not fully sent, can't "
3142 "retrieve.\n",
3143 handle);
J-Alves4f0d9c12024-01-17 17:23:11 +00003144 return ffa_error(FFA_INVALID_PARAMETERS);
J-Alves96de29f2022-04-26 16:05:24 +01003145 }
3146
J-Alves4f0d9c12024-01-17 17:23:11 +00003147 /*
3148 * Validate retrieve request, according to what was sent by the
3149 * sender. Function will output the `receiver_index` from the
3150 * provided memory region.
3151 */
3152 ret = ffa_memory_retrieve_validate(
3153 receiver_id, retrieve_request, retrieve_request_length,
3154 memory_region, &receiver_index, share_state->share_func);
J-Alves089004f2022-07-13 14:25:44 +01003155
J-Alves4f0d9c12024-01-17 17:23:11 +00003156 if (ret.func != FFA_SUCCESS_32) {
3157 return ret;
J-Alves089004f2022-07-13 14:25:44 +01003158 }
J-Alves96de29f2022-04-26 16:05:24 +01003159
J-Alves4f0d9c12024-01-17 17:23:11 +00003160 /*
3161 * Validate the requested permissions against the sent
3162 * permissions.
3163 * Outputs the permissions to give to retriever at S2
3164 * PTs.
3165 */
3166 ret = ffa_memory_retrieve_validate_memory_access_list(
3167 memory_region, retrieve_request, receiver_id, &permissions,
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003168 &receiver, share_state->share_func);
J-Alves4f0d9c12024-01-17 17:23:11 +00003169 if (ret.func != FFA_SUCCESS_32) {
3170 return ret;
3171 }
3172
3173 memory_to_mode = ffa_memory_permissions_to_mode(
3174 permissions, share_state->sender_orig_mode);
3175
3176 ret = ffa_retrieve_check_update(
3177 to_locked, share_state->fragments,
3178 share_state->fragment_constituent_counts,
3179 share_state->fragment_count, memory_to_mode,
J-Alves460d36c2023-10-12 17:02:15 +01003180 share_state->share_func, false, page_pool, &retrieve_mode,
3181 share_state->memory_protected);
J-Alves4f0d9c12024-01-17 17:23:11 +00003182
3183 if (ret.func != FFA_SUCCESS_32) {
3184 return ret;
3185 }
3186
3187 share_state->retrieved_fragment_count[receiver_index] = 1;
3188
3189 is_retrieve_complete =
3190 share_state->retrieved_fragment_count[receiver_index] ==
3191 share_state->fragment_count;
3192
J-Alvesb5084cf2022-07-06 14:20:12 +01003193 /* VMs acquire the RX buffer from SPMC. */
3194 CHECK(plat_ffa_acquire_receiver_rx(to_locked, &ret));
3195
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003196 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01003197 * Copy response to RX buffer of caller and deliver the message.
3198 * This must be done before the share_state is (possibly) freed.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003199 */
Andrew Walbranca808b12020-05-15 17:22:28 +01003200 composite = ffa_memory_region_get_composite(memory_region, 0);
J-Alves4f0d9c12024-01-17 17:23:11 +00003201
Andrew Walbranca808b12020-05-15 17:22:28 +01003202 /*
J-Alves460d36c2023-10-12 17:02:15 +01003203 * Set the security state in the memory retrieve response attributes
3204 * if specified by the target mode.
3205 */
3206 attributes = plat_ffa_memory_security_mode(memory_region->attributes,
3207 retrieve_mode);
3208
3209 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01003210 * Constituents which we received in the first fragment should
3211 * always fit in the first fragment we are sending, because the
3212 * header is the same size in both cases and we have a fixed
3213 * message buffer size. So `ffa_retrieved_memory_region_init`
3214 * should never fail.
Andrew Walbranca808b12020-05-15 17:22:28 +01003215 */
Olivier Deprez878bd5b2021-04-15 19:05:10 +02003216
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003217 /* Provide the permissions that had been provided. */
3218 receiver->receiver_permissions.permissions = permissions;
3219
3220 /*
3221 * Prepare the memory region descriptor for the retrieve response.
3222 * Provide the pointer to the receiver tracked in the share state
3223 * strucutures.
3224 */
Andrew Walbranca808b12020-05-15 17:22:28 +01003225 CHECK(ffa_retrieved_memory_region_init(
J-Alves2d8457f2022-10-05 11:06:41 +01003226 to_locked.vm->mailbox.recv, to_locked.vm->ffa_version,
Olivier Deprez878bd5b2021-04-15 19:05:10 +02003227 HF_MAILBOX_SIZE, memory_region->sender, attributes,
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003228 memory_region->flags, handle, permissions, receiver, 1,
3229 memory_access_desc_size, composite->page_count,
3230 composite->constituent_count, share_state->fragments[0],
Andrew Walbranca808b12020-05-15 17:22:28 +01003231 share_state->fragment_constituent_counts[0], &total_length,
3232 &fragment_length));
J-Alvesb5084cf2022-07-06 14:20:12 +01003233
J-Alves4f0d9c12024-01-17 17:23:11 +00003234 if (is_retrieve_complete) {
Andrew Walbranca808b12020-05-15 17:22:28 +01003235 ffa_memory_retrieve_complete(share_states, share_state,
3236 page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003237 }
J-Alves4f0d9c12024-01-17 17:23:11 +00003238
3239 return ffa_memory_retrieve_resp(total_length, fragment_length);
3240}
3241
3242static struct ffa_value ffa_hypervisor_retrieve_request(
3243 struct ffa_memory_share_state *share_state, struct vm_locked to_locked,
3244 struct ffa_memory_region *retrieve_request)
3245{
3246 struct ffa_value ret;
3247 struct ffa_composite_memory_region *composite;
3248 uint32_t total_length;
3249 uint32_t fragment_length;
J-Alves4f0d9c12024-01-17 17:23:11 +00003250 ffa_memory_attributes_t attributes;
J-Alves7b6ab612024-01-24 09:54:54 +00003251 uint64_t memory_access_desc_size;
J-Alves4f0d9c12024-01-17 17:23:11 +00003252 struct ffa_memory_region *memory_region;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003253 struct ffa_memory_access *receiver;
J-Alves4f0d9c12024-01-17 17:23:11 +00003254 ffa_memory_handle_t handle = retrieve_request->handle;
3255
J-Alves4f0d9c12024-01-17 17:23:11 +00003256 memory_region = share_state->memory_region;
3257
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003258 assert(to_locked.vm->id == HF_HYPERVISOR_VM_ID);
3259
J-Alves7b6ab612024-01-24 09:54:54 +00003260 switch (to_locked.vm->ffa_version) {
3261 case MAKE_FFA_VERSION(1, 2):
3262 memory_access_desc_size = sizeof(struct ffa_memory_access);
3263 break;
3264 case MAKE_FFA_VERSION(1, 0):
3265 case MAKE_FFA_VERSION(1, 1):
3266 memory_access_desc_size = sizeof(struct ffa_memory_access_v1_0);
3267 break;
3268 default:
3269 panic("version not supported: %x\n", to_locked.vm->ffa_version);
3270 }
3271
J-Alves4f0d9c12024-01-17 17:23:11 +00003272 if (share_state->hypervisor_fragment_count != 0U) {
3273 dlog_verbose(
3274 "Memory with handle %#x already retrieved by "
3275 "the hypervisor.\n",
3276 handle);
3277 return ffa_error(FFA_DENIED);
3278 }
3279
3280 share_state->hypervisor_fragment_count = 1;
3281
3282 ffa_memory_retrieve_complete_from_hyp(share_state);
3283
3284 /* VMs acquire the RX buffer from SPMC. */
3285 CHECK(plat_ffa_acquire_receiver_rx(to_locked, &ret));
3286
3287 /*
3288 * Copy response to RX buffer of caller and deliver the message.
3289 * This must be done before the share_state is (possibly) freed.
3290 */
3291 composite = ffa_memory_region_get_composite(memory_region, 0);
3292
3293 /*
3294 * Constituents which we received in the first fragment should
3295 * always fit in the first fragment we are sending, because the
3296 * header is the same size in both cases and we have a fixed
3297 * message buffer size. So `ffa_retrieved_memory_region_init`
3298 * should never fail.
3299 */
3300
3301 /*
3302 * Set the security state in the memory retrieve response attributes
3303 * if specified by the target mode.
3304 */
3305 attributes = plat_ffa_memory_security_mode(
3306 memory_region->attributes, share_state->sender_orig_mode);
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003307
3308 receiver = ffa_memory_region_get_receiver(memory_region, 0);
3309
J-Alves4f0d9c12024-01-17 17:23:11 +00003310 CHECK(ffa_retrieved_memory_region_init(
3311 to_locked.vm->mailbox.recv, to_locked.vm->ffa_version,
3312 HF_MAILBOX_SIZE, memory_region->sender, attributes,
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003313 memory_region->flags, handle,
3314 receiver->receiver_permissions.permissions, receiver,
3315 memory_region->receiver_count, memory_access_desc_size,
J-Alves4f0d9c12024-01-17 17:23:11 +00003316 composite->page_count, composite->constituent_count,
3317 share_state->fragments[0],
3318 share_state->fragment_constituent_counts[0], &total_length,
3319 &fragment_length));
3320
3321 return ffa_memory_retrieve_resp(total_length, fragment_length);
3322}
3323
3324struct ffa_value ffa_memory_retrieve(struct vm_locked to_locked,
3325 struct ffa_memory_region *retrieve_request,
3326 uint32_t retrieve_request_length,
3327 struct mpool *page_pool)
3328{
3329 ffa_memory_handle_t handle = retrieve_request->handle;
3330 struct share_states_locked share_states;
3331 struct ffa_memory_share_state *share_state;
3332 struct ffa_value ret;
3333
3334 dump_share_states();
3335
3336 share_states = share_states_lock();
3337 share_state = get_share_state(share_states, handle);
3338 if (share_state == NULL) {
3339 dlog_verbose("Invalid handle %#x for FFA_MEM_RETRIEVE_REQ.\n",
3340 handle);
3341 ret = ffa_error(FFA_INVALID_PARAMETERS);
3342 goto out;
3343 }
3344
3345 if (is_ffa_hypervisor_retrieve_request(retrieve_request, to_locked)) {
3346 ret = ffa_hypervisor_retrieve_request(share_state, to_locked,
3347 retrieve_request);
3348 } else {
3349 ret = ffa_partition_retrieve_request(
3350 share_states, share_state, to_locked, retrieve_request,
3351 retrieve_request_length, page_pool);
3352 }
3353
3354 /* Track use of the RX buffer if the handling has succeeded. */
3355 if (ret.func == FFA_MEM_RETRIEVE_RESP_32) {
3356 to_locked.vm->mailbox.recv_func = FFA_MEM_RETRIEVE_RESP_32;
3357 to_locked.vm->mailbox.state = MAILBOX_STATE_FULL;
3358 }
3359
Andrew Walbranca808b12020-05-15 17:22:28 +01003360out:
3361 share_states_unlock(&share_states);
3362 dump_share_states();
3363 return ret;
3364}
3365
J-Alves5da37d92022-10-24 16:33:48 +01003366/**
3367 * Determine expected fragment offset according to the FF-A version of
3368 * the caller.
3369 */
3370static uint32_t ffa_memory_retrieve_expected_offset_per_ffa_version(
3371 struct ffa_memory_region *memory_region,
3372 uint32_t retrieved_constituents_count, uint32_t ffa_version)
3373{
3374 uint32_t expected_fragment_offset;
3375 uint32_t composite_constituents_offset;
3376
Kathleen Capellae4fe2962023-09-01 17:08:47 -04003377 if (ffa_version >= MAKE_FFA_VERSION(1, 1)) {
J-Alves5da37d92022-10-24 16:33:48 +01003378 /*
3379 * Hafnium operates memory regions in FF-A v1.1 format, so we
3380 * can retrieve the constituents offset from descriptor.
3381 */
3382 composite_constituents_offset =
3383 ffa_composite_constituent_offset(memory_region, 0);
3384 } else if (ffa_version == MAKE_FFA_VERSION(1, 0)) {
3385 /*
3386 * If retriever is FF-A v1.0, determine the composite offset
3387 * as it is expected to have been configured in the
3388 * retrieve response.
3389 */
3390 composite_constituents_offset =
3391 sizeof(struct ffa_memory_region_v1_0) +
3392 RECEIVERS_COUNT_IN_RETRIEVE_RESP *
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003393 sizeof(struct ffa_memory_access_v1_0) +
J-Alves5da37d92022-10-24 16:33:48 +01003394 sizeof(struct ffa_composite_memory_region);
3395 } else {
3396 panic("%s received an invalid FF-A version.\n", __func__);
3397 }
3398
3399 expected_fragment_offset =
3400 composite_constituents_offset +
3401 retrieved_constituents_count *
3402 sizeof(struct ffa_memory_region_constituent) -
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003403 (uint32_t)(memory_region->memory_access_desc_size *
3404 (memory_region->receiver_count - 1));
J-Alves5da37d92022-10-24 16:33:48 +01003405
3406 return expected_fragment_offset;
3407}
3408
Andrew Walbranca808b12020-05-15 17:22:28 +01003409struct ffa_value ffa_memory_retrieve_continue(struct vm_locked to_locked,
3410 ffa_memory_handle_t handle,
3411 uint32_t fragment_offset,
J-Alves19e20cf2023-08-02 12:48:55 +01003412 ffa_id_t sender_vm_id,
Andrew Walbranca808b12020-05-15 17:22:28 +01003413 struct mpool *page_pool)
3414{
3415 struct ffa_memory_region *memory_region;
3416 struct share_states_locked share_states;
3417 struct ffa_memory_share_state *share_state;
3418 struct ffa_value ret;
3419 uint32_t fragment_index;
3420 uint32_t retrieved_constituents_count;
3421 uint32_t i;
3422 uint32_t expected_fragment_offset;
3423 uint32_t remaining_constituent_count;
3424 uint32_t fragment_length;
J-Alvesc7484f12022-05-13 12:41:14 +01003425 uint32_t receiver_index;
J-Alves59ed0042022-07-28 18:26:41 +01003426 bool continue_ffa_hyp_mem_retrieve_req;
Andrew Walbranca808b12020-05-15 17:22:28 +01003427
3428 dump_share_states();
3429
3430 share_states = share_states_lock();
Karl Meakin4a2854a2023-06-30 16:26:52 +01003431 share_state = get_share_state(share_states, handle);
J-Alvesb56aac82023-11-10 09:44:43 +00003432 if (share_state == NULL) {
Andrew Walbranca808b12020-05-15 17:22:28 +01003433 dlog_verbose("Invalid handle %#x for FFA_MEM_FRAG_RX.\n",
3434 handle);
3435 ret = ffa_error(FFA_INVALID_PARAMETERS);
3436 goto out;
3437 }
3438
3439 memory_region = share_state->memory_region;
3440 CHECK(memory_region != NULL);
3441
Andrew Walbranca808b12020-05-15 17:22:28 +01003442 if (!share_state->sending_complete) {
3443 dlog_verbose(
3444 "Memory with handle %#x not fully sent, can't "
3445 "retrieve.\n",
3446 handle);
3447 ret = ffa_error(FFA_INVALID_PARAMETERS);
3448 goto out;
3449 }
3450
J-Alves59ed0042022-07-28 18:26:41 +01003451 /*
3452 * If retrieve request from the hypervisor has been initiated in the
3453 * given share_state, continue it, else assume it is a continuation of
3454 * retrieve request from a NWd VM.
3455 */
3456 continue_ffa_hyp_mem_retrieve_req =
3457 (to_locked.vm->id == HF_HYPERVISOR_VM_ID) &&
3458 (share_state->hypervisor_fragment_count != 0U) &&
J-Alves661e1b72023-08-02 13:39:40 +01003459 ffa_is_vm_id(sender_vm_id);
Andrew Walbranca808b12020-05-15 17:22:28 +01003460
J-Alves59ed0042022-07-28 18:26:41 +01003461 if (!continue_ffa_hyp_mem_retrieve_req) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003462 receiver_index = ffa_memory_region_get_receiver_index(
J-Alves59ed0042022-07-28 18:26:41 +01003463 memory_region, to_locked.vm->id);
3464
3465 if (receiver_index == memory_region->receiver_count) {
3466 dlog_verbose(
3467 "Caller of FFA_MEM_FRAG_RX (%x) is not a "
3468 "borrower to memory sharing transaction (%x)\n",
3469 to_locked.vm->id, handle);
3470 ret = ffa_error(FFA_INVALID_PARAMETERS);
3471 goto out;
3472 }
3473
3474 if (share_state->retrieved_fragment_count[receiver_index] ==
3475 0 ||
3476 share_state->retrieved_fragment_count[receiver_index] >=
3477 share_state->fragment_count) {
3478 dlog_verbose(
3479 "Retrieval of memory with handle %#x not yet "
3480 "started or already completed (%d/%d fragments "
3481 "retrieved).\n",
3482 handle,
3483 share_state->retrieved_fragment_count
3484 [receiver_index],
3485 share_state->fragment_count);
3486 ret = ffa_error(FFA_INVALID_PARAMETERS);
3487 goto out;
3488 }
3489
3490 fragment_index =
3491 share_state->retrieved_fragment_count[receiver_index];
3492 } else {
3493 if (share_state->hypervisor_fragment_count == 0 ||
3494 share_state->hypervisor_fragment_count >=
3495 share_state->fragment_count) {
3496 dlog_verbose(
3497 "Retrieve of memory with handle %x not "
3498 "started from hypervisor.\n",
3499 handle);
3500 ret = ffa_error(FFA_INVALID_PARAMETERS);
3501 goto out;
3502 }
3503
3504 if (memory_region->sender != sender_vm_id) {
3505 dlog_verbose(
3506 "Sender ID (%x) is not as expected for memory "
3507 "handle %x\n",
3508 sender_vm_id, handle);
3509 ret = ffa_error(FFA_INVALID_PARAMETERS);
3510 goto out;
3511 }
3512
3513 fragment_index = share_state->hypervisor_fragment_count;
3514
3515 receiver_index = 0;
3516 }
Andrew Walbranca808b12020-05-15 17:22:28 +01003517
3518 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01003519 * Check that the given fragment offset is correct by counting
3520 * how many constituents were in the fragments previously sent.
Andrew Walbranca808b12020-05-15 17:22:28 +01003521 */
3522 retrieved_constituents_count = 0;
3523 for (i = 0; i < fragment_index; ++i) {
3524 retrieved_constituents_count +=
3525 share_state->fragment_constituent_counts[i];
3526 }
J-Alvesc7484f12022-05-13 12:41:14 +01003527
3528 CHECK(memory_region->receiver_count > 0);
3529
Andrew Walbranca808b12020-05-15 17:22:28 +01003530 expected_fragment_offset =
J-Alves5da37d92022-10-24 16:33:48 +01003531 ffa_memory_retrieve_expected_offset_per_ffa_version(
3532 memory_region, retrieved_constituents_count,
3533 to_locked.vm->ffa_version);
3534
Andrew Walbranca808b12020-05-15 17:22:28 +01003535 if (fragment_offset != expected_fragment_offset) {
3536 dlog_verbose("Fragment offset was %d but expected %d.\n",
3537 fragment_offset, expected_fragment_offset);
3538 ret = ffa_error(FFA_INVALID_PARAMETERS);
3539 goto out;
3540 }
3541
J-Alves4f0d9c12024-01-17 17:23:11 +00003542 /*
3543 * When hafnium is the hypervisor, acquire the RX buffer of a VM, that
3544 * is currently ownder by the SPMC.
3545 */
3546 assert(plat_ffa_acquire_receiver_rx(to_locked, &ret));
J-Alves59ed0042022-07-28 18:26:41 +01003547
Andrew Walbranca808b12020-05-15 17:22:28 +01003548 remaining_constituent_count = ffa_memory_fragment_init(
3549 to_locked.vm->mailbox.recv, HF_MAILBOX_SIZE,
3550 share_state->fragments[fragment_index],
3551 share_state->fragment_constituent_counts[fragment_index],
3552 &fragment_length);
3553 CHECK(remaining_constituent_count == 0);
J-Alves674e4de2024-01-17 16:20:32 +00003554
Andrew Walbranca808b12020-05-15 17:22:28 +01003555 to_locked.vm->mailbox.recv_func = FFA_MEM_FRAG_TX_32;
J-Alvese8c8c2b2022-12-16 15:34:48 +00003556 to_locked.vm->mailbox.state = MAILBOX_STATE_FULL;
Andrew Walbranca808b12020-05-15 17:22:28 +01003557
J-Alves59ed0042022-07-28 18:26:41 +01003558 if (!continue_ffa_hyp_mem_retrieve_req) {
3559 share_state->retrieved_fragment_count[receiver_index]++;
3560 if (share_state->retrieved_fragment_count[receiver_index] ==
3561 share_state->fragment_count) {
3562 ffa_memory_retrieve_complete(share_states, share_state,
3563 page_pool);
3564 }
3565 } else {
3566 share_state->hypervisor_fragment_count++;
3567
3568 ffa_memory_retrieve_complete_from_hyp(share_state);
3569 }
Andrew Walbranca808b12020-05-15 17:22:28 +01003570 ret = (struct ffa_value){.func = FFA_MEM_FRAG_TX_32,
3571 .arg1 = (uint32_t)handle,
3572 .arg2 = (uint32_t)(handle >> 32),
3573 .arg3 = fragment_length};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003574
3575out:
3576 share_states_unlock(&share_states);
3577 dump_share_states();
3578 return ret;
3579}
3580
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003581struct ffa_value ffa_memory_relinquish(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003582 struct vm_locked from_locked,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003583 struct ffa_mem_relinquish *relinquish_request, struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003584{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003585 ffa_memory_handle_t handle = relinquish_request->handle;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003586 struct share_states_locked share_states;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003587 struct ffa_memory_share_state *share_state;
3588 struct ffa_memory_region *memory_region;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003589 bool clear;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003590 struct ffa_value ret;
J-Alves8eb19162022-04-28 10:56:48 +01003591 uint32_t receiver_index;
J-Alves3c5b2072022-11-21 12:45:40 +00003592 bool receivers_relinquished_memory;
Karl Meakin84710f32023-10-12 15:14:49 +01003593 ffa_memory_access_permissions_t receiver_permissions = {0};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003594
Andrew Walbrana65a1322020-04-06 19:32:32 +01003595 if (relinquish_request->endpoint_count != 1) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003596 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01003597 "Stream endpoints not supported (got %d "
J-Alves668a86e2023-05-10 11:53:25 +01003598 "endpoints on FFA_MEM_RELINQUISH, expected 1).\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003599 relinquish_request->endpoint_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003600 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003601 }
3602
Andrew Walbrana65a1322020-04-06 19:32:32 +01003603 if (relinquish_request->endpoints[0] != from_locked.vm->id) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003604 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01003605 "VM ID %d in relinquish message doesn't match "
J-Alves668a86e2023-05-10 11:53:25 +01003606 "calling VM ID %d.\n",
Andrew Walbrana65a1322020-04-06 19:32:32 +01003607 relinquish_request->endpoints[0], from_locked.vm->id);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003608 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003609 }
3610
3611 dump_share_states();
3612
3613 share_states = share_states_lock();
Karl Meakin4a2854a2023-06-30 16:26:52 +01003614 share_state = get_share_state(share_states, handle);
J-Alvesb56aac82023-11-10 09:44:43 +00003615 if (share_state == NULL) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003616 dlog_verbose("Invalid handle %#x for FFA_MEM_RELINQUISH.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003617 handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003618 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003619 goto out;
3620 }
3621
Andrew Walbranca808b12020-05-15 17:22:28 +01003622 if (!share_state->sending_complete) {
3623 dlog_verbose(
3624 "Memory with handle %#x not fully sent, can't "
3625 "relinquish.\n",
3626 handle);
3627 ret = ffa_error(FFA_INVALID_PARAMETERS);
3628 goto out;
3629 }
3630
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003631 memory_region = share_state->memory_region;
3632 CHECK(memory_region != NULL);
3633
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003634 receiver_index = ffa_memory_region_get_receiver_index(
3635 memory_region, from_locked.vm->id);
J-Alves8eb19162022-04-28 10:56:48 +01003636
3637 if (receiver_index == memory_region->receiver_count) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003638 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01003639 "VM ID %d tried to relinquish memory region "
J-Alves668a86e2023-05-10 11:53:25 +01003640 "with handle %#x and it is not a valid borrower.\n",
J-Alves8eb19162022-04-28 10:56:48 +01003641 from_locked.vm->id, handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003642 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003643 goto out;
3644 }
3645
J-Alves8eb19162022-04-28 10:56:48 +01003646 if (share_state->retrieved_fragment_count[receiver_index] !=
Andrew Walbranca808b12020-05-15 17:22:28 +01003647 share_state->fragment_count) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003648 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01003649 "Memory with handle %#x not yet fully "
3650 "retrieved, "
J-Alves8eb19162022-04-28 10:56:48 +01003651 "receiver %x can't relinquish.\n",
3652 handle, from_locked.vm->id);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003653 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003654 goto out;
3655 }
3656
J-Alves3c5b2072022-11-21 12:45:40 +00003657 /*
3658 * Either clear if requested in relinquish call, or in a retrieve
3659 * request from one of the borrowers.
3660 */
3661 receivers_relinquished_memory = true;
3662
3663 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
3664 struct ffa_memory_access *receiver =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003665 ffa_memory_region_get_receiver(memory_region, i);
3666 assert(receiver != NULL);
J-Alves3c5b2072022-11-21 12:45:40 +00003667 if (receiver->receiver_permissions.receiver ==
3668 from_locked.vm->id) {
J-Alves639ddfc2023-11-21 14:17:26 +00003669 receiver_permissions =
3670 receiver->receiver_permissions.permissions;
J-Alves3c5b2072022-11-21 12:45:40 +00003671 continue;
3672 }
3673
3674 if (share_state->retrieved_fragment_count[i] != 0U) {
3675 receivers_relinquished_memory = false;
3676 break;
3677 }
3678 }
3679
3680 clear = receivers_relinquished_memory &&
Daniel Boulby2e14ebe2024-01-15 16:21:44 +00003681 ((relinquish_request->flags & FFA_MEMORY_REGION_FLAG_CLEAR) !=
3682 0U);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003683
3684 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01003685 * Clear is not allowed for memory that was shared, as the
3686 * original sender still has access to the memory.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003687 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003688 if (clear && share_state->share_func == FFA_MEM_SHARE_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003689 dlog_verbose("Memory which was shared can't be cleared.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003690 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003691 goto out;
3692 }
3693
Karl Meakin84710f32023-10-12 15:14:49 +01003694 if (clear && receiver_permissions.instruction_access == 0 &&
3695 receiver_permissions.data_access == FFA_DATA_ACCESS_RO) {
J-Alves639ddfc2023-11-21 14:17:26 +00003696 dlog_verbose("%s: RO memory can't use clear memory flag.\n",
3697 __func__);
3698 ret = ffa_error(FFA_DENIED);
3699 goto out;
3700 }
3701
Andrew Walbranca808b12020-05-15 17:22:28 +01003702 ret = ffa_relinquish_check_update(
J-Alves26483382023-04-20 12:01:49 +01003703 from_locked, share_state->fragments,
Andrew Walbranca808b12020-05-15 17:22:28 +01003704 share_state->fragment_constituent_counts,
3705 share_state->fragment_count, page_pool, clear);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003706
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003707 if (ret.func == FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003708 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01003709 * Mark memory handle as not retrieved, so it can be
3710 * reclaimed (or retrieved again).
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003711 */
J-Alves8eb19162022-04-28 10:56:48 +01003712 share_state->retrieved_fragment_count[receiver_index] = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003713 }
3714
3715out:
3716 share_states_unlock(&share_states);
3717 dump_share_states();
3718 return ret;
3719}
3720
3721/**
J-Alvesa9cd7e32022-07-01 13:49:33 +01003722 * Validates that the reclaim transition is allowed for the given
3723 * handle, updates the page table of the reclaiming VM, and frees the
3724 * internal state associated with the handle.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003725 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003726struct ffa_value ffa_memory_reclaim(struct vm_locked to_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01003727 ffa_memory_handle_t handle,
3728 ffa_memory_region_flags_t flags,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003729 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003730{
3731 struct share_states_locked share_states;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003732 struct ffa_memory_share_state *share_state;
3733 struct ffa_memory_region *memory_region;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003734 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003735
3736 dump_share_states();
3737
3738 share_states = share_states_lock();
Karl Meakin52cdfe72023-06-30 14:49:10 +01003739
Karl Meakin4a2854a2023-06-30 16:26:52 +01003740 share_state = get_share_state(share_states, handle);
J-Alvesb56aac82023-11-10 09:44:43 +00003741 if (share_state == NULL) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003742 dlog_verbose("Invalid handle %#x for FFA_MEM_RECLAIM.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003743 handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003744 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003745 goto out;
3746 }
Karl Meakin4a2854a2023-06-30 16:26:52 +01003747 memory_region = share_state->memory_region;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003748
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003749 CHECK(memory_region != NULL);
3750
J-Alvesa9cd7e32022-07-01 13:49:33 +01003751 if (vm_id_is_current_world(to_locked.vm->id) &&
3752 to_locked.vm->id != memory_region->sender) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003753 dlog_verbose(
Olivier Deprezf92e5d42020-11-13 16:00:54 +01003754 "VM %#x attempted to reclaim memory handle %#x "
3755 "originally sent by VM %#x.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003756 to_locked.vm->id, handle, memory_region->sender);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003757 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003758 goto out;
3759 }
3760
Andrew Walbranca808b12020-05-15 17:22:28 +01003761 if (!share_state->sending_complete) {
3762 dlog_verbose(
3763 "Memory with handle %#x not fully sent, can't "
3764 "reclaim.\n",
3765 handle);
3766 ret = ffa_error(FFA_INVALID_PARAMETERS);
3767 goto out;
3768 }
3769
J-Alves752236c2022-04-28 11:07:47 +01003770 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
3771 if (share_state->retrieved_fragment_count[i] != 0) {
3772 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01003773 "Tried to reclaim memory handle %#x "
J-Alves3c5b2072022-11-21 12:45:40 +00003774 "that has not been relinquished by all "
J-Alvesa9cd7e32022-07-01 13:49:33 +01003775 "borrowers(%x).\n",
J-Alves752236c2022-04-28 11:07:47 +01003776 handle,
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003777 ffa_memory_region_get_receiver(memory_region, i)
3778 ->receiver_permissions.receiver);
J-Alves752236c2022-04-28 11:07:47 +01003779 ret = ffa_error(FFA_DENIED);
3780 goto out;
3781 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003782 }
3783
Andrew Walbranca808b12020-05-15 17:22:28 +01003784 ret = ffa_retrieve_check_update(
J-Alves26483382023-04-20 12:01:49 +01003785 to_locked, share_state->fragments,
Andrew Walbranca808b12020-05-15 17:22:28 +01003786 share_state->fragment_constituent_counts,
J-Alves2a0d2882020-10-29 14:49:50 +00003787 share_state->fragment_count, share_state->sender_orig_mode,
J-Alves460d36c2023-10-12 17:02:15 +01003788 FFA_MEM_RECLAIM_32, flags & FFA_MEM_RECLAIM_CLEAR, page_pool,
J-Alvesfd206052023-05-22 16:45:00 +01003789 NULL, share_state->memory_protected);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003790
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003791 if (ret.func == FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003792 share_state_free(share_states, share_state, page_pool);
J-Alves3c5b2072022-11-21 12:45:40 +00003793 dlog_verbose("Freed share state after successful reclaim.\n");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003794 }
3795
3796out:
3797 share_states_unlock(&share_states);
3798 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01003799}