blob: 6e21aa15e40ff89cc5f730b5f17f379be17f6ff5 [file] [log] [blame]
Jose Marinho75509b42019-04-09 09:34:59 +01001/*
2 * Copyright 2019 The Hafnium Authors.
3 *
Andrew Walbrane959ec12020-06-17 15:01:09 +01004 * Use of this source code is governed by a BSD-style
5 * license that can be found in the LICENSE file or at
6 * https://opensource.org/licenses/BSD-3-Clause.
Jose Marinho75509b42019-04-09 09:34:59 +01007 */
8
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01009#include "hf/ffa_memory.h"
Andrew Walbran475c1452020-02-07 13:22:22 +000010
Federico Recanati4fd065d2021-12-13 20:06:23 +010011#include "hf/arch/mm.h"
Olivier Deprez112d2b52020-09-30 07:39:23 +020012#include "hf/arch/other_world.h"
Olivier Deprez55a189e2021-06-09 15:45:27 +020013#include "hf/arch/plat/ffa.h"
Andrew Walbran290b0c92020-02-03 16:37:14 +000014
J-Alves5952d942022-12-22 16:03:00 +000015#include "hf/addr.h"
Jose Marinho75509b42019-04-09 09:34:59 +010016#include "hf/api.h"
Daniel Boulbya2f8c662021-11-26 17:52:53 +000017#include "hf/assert.h"
Jose Marinho09b1db82019-08-08 09:16:59 +010018#include "hf/check.h"
Jose Marinho75509b42019-04-09 09:34:59 +010019#include "hf/dlog.h"
J-Alves3456e032023-07-20 12:20:05 +010020#include "hf/ffa.h"
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010021#include "hf/ffa_internal.h"
J-Alves66652252022-07-06 09:49:51 +010022#include "hf/ffa_memory_internal.h"
J-Alves3456e032023-07-20 12:20:05 +010023#include "hf/ffa_partition_manifest.h"
J-Alves5952d942022-12-22 16:03:00 +000024#include "hf/mm.h"
Andrew Walbran475c1452020-02-07 13:22:22 +000025#include "hf/mpool.h"
Jose Marinho75509b42019-04-09 09:34:59 +010026#include "hf/std.h"
Andrew Scull3c257452019-11-26 13:32:50 +000027#include "hf/vm.h"
Jose Marinho75509b42019-04-09 09:34:59 +010028
J-Alves2d8457f2022-10-05 11:06:41 +010029#include "vmapi/hf/ffa_v1_0.h"
30
J-Alves5da37d92022-10-24 16:33:48 +010031#define RECEIVERS_COUNT_IN_RETRIEVE_RESP 1
32
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000033/**
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010034 * All access to members of a `struct ffa_memory_share_state` must be guarded
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000035 * by this lock.
36 */
37static struct spinlock share_states_lock_instance = SPINLOCK_INIT;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010038static struct ffa_memory_share_state share_states[MAX_MEM_SHARES];
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000039
40/**
J-Alves917d2f22020-10-30 18:39:30 +000041 * Extracts the index from a memory handle allocated by Hafnium's current world.
42 */
43uint64_t ffa_memory_handle_get_index(ffa_memory_handle_t handle)
44{
45 return handle & ~FFA_MEMORY_HANDLE_ALLOCATOR_MASK;
46}
47
48/**
Karl Meakin52cdfe72023-06-30 14:49:10 +010049 * Initialises the next available `struct ffa_memory_share_state`. If `handle`
50 * is `FFA_MEMORY_HANDLE_INVALID` then allocates an appropriate handle,
51 * otherwise uses the provided handle which is assumed to be globally unique.
Andrew Walbranca808b12020-05-15 17:22:28 +010052 *
Karl Meakin52cdfe72023-06-30 14:49:10 +010053 * Returns a pointer to the allocated `ffa_memory_share_state` on success or
54 * `NULL` if none are available.
Andrew Walbranca808b12020-05-15 17:22:28 +010055 */
Karl Meakin52cdfe72023-06-30 14:49:10 +010056struct ffa_memory_share_state *allocate_share_state(
57 struct share_states_locked share_states, uint32_t share_func,
58 struct ffa_memory_region *memory_region, uint32_t fragment_length,
59 ffa_memory_handle_t handle)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000060{
Daniel Boulbya2f8c662021-11-26 17:52:53 +000061 assert(share_states.share_states != NULL);
62 assert(memory_region != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000063
Karl Meakin52cdfe72023-06-30 14:49:10 +010064 for (uint64_t i = 0; i < MAX_MEM_SHARES; ++i) {
Andrew Walbranca808b12020-05-15 17:22:28 +010065 if (share_states.share_states[i].share_func == 0) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010066 struct ffa_memory_share_state *allocated_state =
Andrew Walbranca808b12020-05-15 17:22:28 +010067 &share_states.share_states[i];
68 struct ffa_composite_memory_region *composite =
69 ffa_memory_region_get_composite(memory_region,
70 0);
71
72 if (handle == FFA_MEMORY_HANDLE_INVALID) {
J-Alvesee68c542020-10-29 17:48:20 +000073 memory_region->handle =
Olivier Deprez55a189e2021-06-09 15:45:27 +020074 plat_ffa_memory_handle_make(i);
Andrew Walbranca808b12020-05-15 17:22:28 +010075 } else {
J-Alvesee68c542020-10-29 17:48:20 +000076 memory_region->handle = handle;
Andrew Walbranca808b12020-05-15 17:22:28 +010077 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000078 allocated_state->share_func = share_func;
79 allocated_state->memory_region = memory_region;
Andrew Walbranca808b12020-05-15 17:22:28 +010080 allocated_state->fragment_count = 1;
81 allocated_state->fragments[0] = composite->constituents;
82 allocated_state->fragment_constituent_counts[0] =
83 (fragment_length -
84 ffa_composite_constituent_offset(memory_region,
85 0)) /
86 sizeof(struct ffa_memory_region_constituent);
87 allocated_state->sending_complete = false;
Karl Meakin52cdfe72023-06-30 14:49:10 +010088 for (uint32_t j = 0; j < MAX_MEM_SHARE_RECIPIENTS;
89 ++j) {
Andrew Walbranca808b12020-05-15 17:22:28 +010090 allocated_state->retrieved_fragment_count[j] =
91 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000092 }
Karl Meakin52cdfe72023-06-30 14:49:10 +010093 return allocated_state;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000094 }
95 }
96
Karl Meakin52cdfe72023-06-30 14:49:10 +010097 return NULL;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000098}
99
100/** Locks the share states lock. */
101struct share_states_locked share_states_lock(void)
102{
103 sl_lock(&share_states_lock_instance);
104
105 return (struct share_states_locked){.share_states = share_states};
106}
107
108/** Unlocks the share states lock. */
J-Alves66652252022-07-06 09:49:51 +0100109void share_states_unlock(struct share_states_locked *share_states)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000110{
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000111 assert(share_states->share_states != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000112 share_states->share_states = NULL;
113 sl_unlock(&share_states_lock_instance);
114}
115
116/**
Andrew Walbranca808b12020-05-15 17:22:28 +0100117 * If the given handle is a valid handle for an allocated share state then
Karl Meakin4a2854a2023-06-30 16:26:52 +0100118 * returns a pointer to the share state. Otherwise returns NULL.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000119 */
Karl Meakin4a2854a2023-06-30 16:26:52 +0100120struct ffa_memory_share_state *get_share_state(
121 struct share_states_locked share_states, ffa_memory_handle_t handle)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000122{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100123 struct ffa_memory_share_state *share_state;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000124
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000125 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100126
127 /*
128 * First look for a share_state allocated by us, in which case the
129 * handle is based on the index.
130 */
Olivier Deprez55a189e2021-06-09 15:45:27 +0200131 if (plat_ffa_memory_handle_allocated_by_current_world(handle)) {
Karl Meakin4a2854a2023-06-30 16:26:52 +0100132 uint64_t index = ffa_memory_handle_get_index(handle);
133
Andrew Walbranca808b12020-05-15 17:22:28 +0100134 if (index < MAX_MEM_SHARES) {
135 share_state = &share_states.share_states[index];
136 if (share_state->share_func != 0) {
Karl Meakin4a2854a2023-06-30 16:26:52 +0100137 return share_state;
Andrew Walbranca808b12020-05-15 17:22:28 +0100138 }
139 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000140 }
141
Andrew Walbranca808b12020-05-15 17:22:28 +0100142 /* Fall back to a linear scan. */
Karl Meakin4a2854a2023-06-30 16:26:52 +0100143 for (uint64_t index = 0; index < MAX_MEM_SHARES; ++index) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100144 share_state = &share_states.share_states[index];
J-Alvesee68c542020-10-29 17:48:20 +0000145 if (share_state->memory_region != NULL &&
146 share_state->memory_region->handle == handle &&
Andrew Walbranca808b12020-05-15 17:22:28 +0100147 share_state->share_func != 0) {
Karl Meakin4a2854a2023-06-30 16:26:52 +0100148 return share_state;
Andrew Walbranca808b12020-05-15 17:22:28 +0100149 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000150 }
151
Karl Meakin4a2854a2023-06-30 16:26:52 +0100152 return NULL;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000153}
154
155/** Marks a share state as unallocated. */
J-Alvesfdd29272022-07-19 13:16:31 +0100156void share_state_free(struct share_states_locked share_states,
157 struct ffa_memory_share_state *share_state,
158 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000159{
Andrew Walbranca808b12020-05-15 17:22:28 +0100160 uint32_t i;
161
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000162 assert(share_states.share_states != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000163 share_state->share_func = 0;
Andrew Walbranca808b12020-05-15 17:22:28 +0100164 share_state->sending_complete = false;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000165 mpool_free(page_pool, share_state->memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +0100166 /*
167 * First fragment is part of the same page as the `memory_region`, so it
168 * doesn't need to be freed separately.
169 */
170 share_state->fragments[0] = NULL;
171 share_state->fragment_constituent_counts[0] = 0;
172 for (i = 1; i < share_state->fragment_count; ++i) {
173 mpool_free(page_pool, share_state->fragments[i]);
174 share_state->fragments[i] = NULL;
175 share_state->fragment_constituent_counts[i] = 0;
176 }
177 share_state->fragment_count = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000178 share_state->memory_region = NULL;
J-Alvesa9cd7e32022-07-01 13:49:33 +0100179 share_state->hypervisor_fragment_count = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000180}
181
Andrew Walbranca808b12020-05-15 17:22:28 +0100182/** Checks whether the given share state has been fully sent. */
J-Alvesfdd29272022-07-19 13:16:31 +0100183bool share_state_sending_complete(struct share_states_locked share_states,
184 struct ffa_memory_share_state *share_state)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000185{
Andrew Walbranca808b12020-05-15 17:22:28 +0100186 struct ffa_composite_memory_region *composite;
187 uint32_t expected_constituent_count;
188 uint32_t fragment_constituent_count_total = 0;
189 uint32_t i;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000190
Andrew Walbranca808b12020-05-15 17:22:28 +0100191 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000192 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100193
194 /*
195 * Share state must already be valid, or it's not possible to get hold
196 * of it.
197 */
198 CHECK(share_state->memory_region != NULL &&
199 share_state->share_func != 0);
200
201 composite =
202 ffa_memory_region_get_composite(share_state->memory_region, 0);
203 expected_constituent_count = composite->constituent_count;
204 for (i = 0; i < share_state->fragment_count; ++i) {
205 fragment_constituent_count_total +=
206 share_state->fragment_constituent_counts[i];
207 }
208 dlog_verbose(
209 "Checking completion: constituent count %d/%d from %d "
210 "fragments.\n",
211 fragment_constituent_count_total, expected_constituent_count,
212 share_state->fragment_count);
213
214 return fragment_constituent_count_total == expected_constituent_count;
215}
216
217/**
218 * Calculates the offset of the next fragment expected for the given share
219 * state.
220 */
J-Alvesfdd29272022-07-19 13:16:31 +0100221uint32_t share_state_next_fragment_offset(
Andrew Walbranca808b12020-05-15 17:22:28 +0100222 struct share_states_locked share_states,
223 struct ffa_memory_share_state *share_state)
224{
225 uint32_t next_fragment_offset;
226 uint32_t i;
227
228 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000229 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100230
231 next_fragment_offset =
232 ffa_composite_constituent_offset(share_state->memory_region, 0);
233 for (i = 0; i < share_state->fragment_count; ++i) {
234 next_fragment_offset +=
235 share_state->fragment_constituent_counts[i] *
236 sizeof(struct ffa_memory_region_constituent);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000237 }
238
Andrew Walbranca808b12020-05-15 17:22:28 +0100239 return next_fragment_offset;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000240}
241
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100242static void dump_memory_region(struct ffa_memory_region *memory_region)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000243{
244 uint32_t i;
245
246 if (LOG_LEVEL < LOG_LEVEL_VERBOSE) {
247 return;
248 }
249
Olivier Deprez935e1b12020-12-22 18:01:29 +0100250 dlog("from VM %#x, attributes %#x, flags %#x, tag %u, to "
Olivier Deprezf92e5d42020-11-13 16:00:54 +0100251 "%u "
Andrew Walbrana65a1322020-04-06 19:32:32 +0100252 "recipients [",
253 memory_region->sender, memory_region->attributes,
Olivier Deprez935e1b12020-12-22 18:01:29 +0100254 memory_region->flags, memory_region->tag,
Andrew Walbrana65a1322020-04-06 19:32:32 +0100255 memory_region->receiver_count);
256 for (i = 0; i < memory_region->receiver_count; ++i) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000257 if (i != 0) {
258 dlog(", ");
259 }
Olivier Deprezf92e5d42020-11-13 16:00:54 +0100260 dlog("VM %#x: %#x (offset %u)",
Andrew Walbrana65a1322020-04-06 19:32:32 +0100261 memory_region->receivers[i].receiver_permissions.receiver,
262 memory_region->receivers[i]
263 .receiver_permissions.permissions,
264 memory_region->receivers[i]
265 .composite_memory_region_offset);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000266 }
267 dlog("]");
268}
269
J-Alves66652252022-07-06 09:49:51 +0100270void dump_share_states(void)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000271{
272 uint32_t i;
273
274 if (LOG_LEVEL < LOG_LEVEL_VERBOSE) {
275 return;
276 }
277
278 dlog("Current share states:\n");
279 sl_lock(&share_states_lock_instance);
280 for (i = 0; i < MAX_MEM_SHARES; ++i) {
281 if (share_states[i].share_func != 0) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000282 switch (share_states[i].share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100283 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000284 dlog("SHARE");
285 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100286 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000287 dlog("LEND");
288 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100289 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000290 dlog("DONATE");
291 break;
292 default:
293 dlog("invalid share_func %#x",
294 share_states[i].share_func);
295 }
Olivier Deprez935e1b12020-12-22 18:01:29 +0100296 dlog(" %#x (", share_states[i].memory_region->handle);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000297 dump_memory_region(share_states[i].memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +0100298 if (share_states[i].sending_complete) {
299 dlog("): fully sent");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000300 } else {
Andrew Walbranca808b12020-05-15 17:22:28 +0100301 dlog("): partially sent");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000302 }
J-Alves2a0d2882020-10-29 14:49:50 +0000303 dlog(" with %d fragments, %d retrieved, "
304 " sender's original mode: %#x\n",
Andrew Walbranca808b12020-05-15 17:22:28 +0100305 share_states[i].fragment_count,
J-Alves2a0d2882020-10-29 14:49:50 +0000306 share_states[i].retrieved_fragment_count[0],
307 share_states[i].sender_orig_mode);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000308 }
309 }
310 sl_unlock(&share_states_lock_instance);
311}
312
Andrew Walbran475c1452020-02-07 13:22:22 +0000313/* TODO: Add device attributes: GRE, cacheability, shareability. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100314static inline uint32_t ffa_memory_permissions_to_mode(
J-Alves7cd5eb32020-10-16 19:06:10 +0100315 ffa_memory_access_permissions_t permissions, uint32_t default_mode)
Andrew Walbran475c1452020-02-07 13:22:22 +0000316{
317 uint32_t mode = 0;
318
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100319 switch (ffa_get_data_access_attr(permissions)) {
320 case FFA_DATA_ACCESS_RO:
Andrew Walbran475c1452020-02-07 13:22:22 +0000321 mode = MM_MODE_R;
322 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100323 case FFA_DATA_ACCESS_RW:
Andrew Walbran475c1452020-02-07 13:22:22 +0000324 mode = MM_MODE_R | MM_MODE_W;
325 break;
J-Alves7cd5eb32020-10-16 19:06:10 +0100326 case FFA_DATA_ACCESS_NOT_SPECIFIED:
327 mode = (default_mode & (MM_MODE_R | MM_MODE_W));
328 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100329 case FFA_DATA_ACCESS_RESERVED:
330 panic("Tried to convert FFA_DATA_ACCESS_RESERVED.");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100331 }
332
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100333 switch (ffa_get_instruction_access_attr(permissions)) {
334 case FFA_INSTRUCTION_ACCESS_NX:
Andrew Walbran475c1452020-02-07 13:22:22 +0000335 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100336 case FFA_INSTRUCTION_ACCESS_X:
Andrew Walbrana65a1322020-04-06 19:32:32 +0100337 mode |= MM_MODE_X;
338 break;
J-Alves7cd5eb32020-10-16 19:06:10 +0100339 case FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED:
340 mode |= (default_mode & MM_MODE_X);
341 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100342 case FFA_INSTRUCTION_ACCESS_RESERVED:
343 panic("Tried to convert FFA_INSTRUCTION_ACCESS_RESVERVED.");
Andrew Walbran475c1452020-02-07 13:22:22 +0000344 }
345
Olivier Deprez878bd5b2021-04-15 19:05:10 +0200346 /* Set the security state bit if necessary. */
347 if ((default_mode & plat_ffa_other_world_mode()) != 0) {
348 mode |= plat_ffa_other_world_mode();
349 }
350
Andrew Walbran475c1452020-02-07 13:22:22 +0000351 return mode;
352}
353
Jose Marinho75509b42019-04-09 09:34:59 +0100354/**
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000355 * Get the current mode in the stage-2 page table of the given vm of all the
356 * pages in the given constituents, if they all have the same mode, or return
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100357 * an appropriate FF-A error if not.
Jose Marinho75509b42019-04-09 09:34:59 +0100358 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100359static struct ffa_value constituents_get_mode(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000360 struct vm_locked vm, uint32_t *orig_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100361 struct ffa_memory_region_constituent **fragments,
362 const uint32_t *fragment_constituent_counts, uint32_t fragment_count)
Jose Marinho75509b42019-04-09 09:34:59 +0100363{
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100364 uint32_t i;
Andrew Walbranca808b12020-05-15 17:22:28 +0100365 uint32_t j;
Jose Marinho75509b42019-04-09 09:34:59 +0100366
Andrew Walbranca808b12020-05-15 17:22:28 +0100367 if (fragment_count == 0 || fragment_constituent_counts[0] == 0) {
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100368 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000369 * Fail if there are no constituents. Otherwise we would get an
370 * uninitialised *orig_mode.
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100371 */
Karl Meakin5df422c2023-07-11 17:31:38 +0100372 dlog_verbose("%s: no constituents\n", __func__);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100373 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho75509b42019-04-09 09:34:59 +0100374 }
375
Andrew Walbranca808b12020-05-15 17:22:28 +0100376 for (i = 0; i < fragment_count; ++i) {
377 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
378 ipaddr_t begin = ipa_init(fragments[i][j].address);
379 size_t size = fragments[i][j].page_count * PAGE_SIZE;
380 ipaddr_t end = ipa_add(begin, size);
381 uint32_t current_mode;
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100382
Andrew Walbranca808b12020-05-15 17:22:28 +0100383 /* Fail if addresses are not page-aligned. */
384 if (!is_aligned(ipa_addr(begin), PAGE_SIZE) ||
385 !is_aligned(ipa_addr(end), PAGE_SIZE)) {
Karl Meakin5df422c2023-07-11 17:31:38 +0100386 dlog_verbose("%s: addresses not page-aligned\n",
387 __func__);
Andrew Walbranca808b12020-05-15 17:22:28 +0100388 return ffa_error(FFA_INVALID_PARAMETERS);
389 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100390
Andrew Walbranca808b12020-05-15 17:22:28 +0100391 /*
392 * Ensure that this constituent memory range is all
393 * mapped with the same mode.
394 */
Raghu Krishnamurthy785d52f2021-02-13 00:02:40 -0800395 if (!vm_mem_get_mode(vm, begin, end, &current_mode)) {
Karl Meakin5df422c2023-07-11 17:31:38 +0100396 dlog_verbose(
397 "%s: constituent memory range %#x..%#x "
398 "not mapped with the same mode\n",
399 __func__, begin, end);
Andrew Walbranca808b12020-05-15 17:22:28 +0100400 return ffa_error(FFA_DENIED);
401 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100402
Andrew Walbranca808b12020-05-15 17:22:28 +0100403 /*
404 * Ensure that all constituents are mapped with the same
405 * mode.
406 */
407 if (i == 0) {
408 *orig_mode = current_mode;
409 } else if (current_mode != *orig_mode) {
410 dlog_verbose(
Karl Meakin5df422c2023-07-11 17:31:38 +0100411 "%s: expected mode %#x but was %#x for "
412 "%d pages at %#x.\n",
413 __func__, *orig_mode, current_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100414 fragments[i][j].page_count,
415 ipa_addr(begin));
416 return ffa_error(FFA_DENIED);
417 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100418 }
Jose Marinho75509b42019-04-09 09:34:59 +0100419 }
420
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100421 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000422}
423
424/**
425 * Verify that all pages have the same mode, that the starting mode
426 * constitutes a valid state and obtain the next mode to apply
427 * to the sending VM.
428 *
429 * Returns:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100430 * 1) FFA_DENIED if a state transition was not found;
431 * 2) FFA_DENIED if the pages being shared do not have the same mode within
Andrew Walbrana65a1322020-04-06 19:32:32 +0100432 * the <from> VM;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100433 * 3) FFA_INVALID_PARAMETERS if the beginning and end IPAs are not page
Andrew Walbrana65a1322020-04-06 19:32:32 +0100434 * aligned;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100435 * 4) FFA_INVALID_PARAMETERS if the requested share type was not handled.
436 * Or FFA_SUCCESS on success.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000437 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100438static struct ffa_value ffa_send_check_transition(
Andrew Walbrana65a1322020-04-06 19:32:32 +0100439 struct vm_locked from, uint32_t share_func,
J-Alves363f5722022-04-25 17:37:37 +0100440 struct ffa_memory_access *receivers, uint32_t receivers_count,
441 uint32_t *orig_from_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100442 struct ffa_memory_region_constituent **fragments,
443 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
444 uint32_t *from_mode)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000445{
446 const uint32_t state_mask =
447 MM_MODE_INVALID | MM_MODE_UNOWNED | MM_MODE_SHARED;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100448 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000449
Andrew Walbranca808b12020-05-15 17:22:28 +0100450 ret = constituents_get_mode(from, orig_from_mode, fragments,
451 fragment_constituent_counts,
452 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100453 if (ret.func != FFA_SUCCESS_32) {
Olivier Depreze7eb1682022-03-16 17:09:03 +0100454 dlog_verbose("Inconsistent modes.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100455 return ret;
Andrew Scullb5f49e02019-10-02 13:20:47 +0100456 }
457
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000458 /* Ensure the address range is normal memory and not a device. */
J-Alves788b4492023-04-18 14:01:23 +0100459 if ((*orig_from_mode & MM_MODE_D) != 0U) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000460 dlog_verbose("Can't share device memory (mode is %#x).\n",
461 *orig_from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100462 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000463 }
464
465 /*
466 * Ensure the sender is the owner and has exclusive access to the
467 * memory.
468 */
469 if ((*orig_from_mode & state_mask) != 0) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100470 return ffa_error(FFA_DENIED);
Andrew Walbrana65a1322020-04-06 19:32:32 +0100471 }
472
J-Alves363f5722022-04-25 17:37:37 +0100473 assert(receivers != NULL && receivers_count > 0U);
J-Alves7cd5eb32020-10-16 19:06:10 +0100474
J-Alves363f5722022-04-25 17:37:37 +0100475 for (uint32_t i = 0U; i < receivers_count; i++) {
476 ffa_memory_access_permissions_t permissions =
477 receivers[i].receiver_permissions.permissions;
478 uint32_t required_from_mode = ffa_memory_permissions_to_mode(
479 permissions, *orig_from_mode);
480
J-Alves788b4492023-04-18 14:01:23 +0100481 /*
482 * The assumption is that at this point, the operation from
483 * SP to a receiver VM, should have returned an FFA_ERROR
484 * already.
485 */
486 if (!ffa_is_vm_id(from.vm->id)) {
487 assert(!ffa_is_vm_id(
488 receivers[i].receiver_permissions.receiver));
489 }
490
J-Alves363f5722022-04-25 17:37:37 +0100491 if ((*orig_from_mode & required_from_mode) !=
492 required_from_mode) {
493 dlog_verbose(
494 "Sender tried to send memory with permissions "
J-Alves788b4492023-04-18 14:01:23 +0100495 "which required mode %#x but only had %#x "
496 "itself.\n",
J-Alves363f5722022-04-25 17:37:37 +0100497 required_from_mode, *orig_from_mode);
498 return ffa_error(FFA_DENIED);
499 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000500 }
501
502 /* Find the appropriate new mode. */
503 *from_mode = ~state_mask & *orig_from_mode;
Andrew Walbrane7ad3c02019-12-24 17:03:04 +0000504 switch (share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100505 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000506 *from_mode |= MM_MODE_INVALID | MM_MODE_UNOWNED;
Jose Marinho75509b42019-04-09 09:34:59 +0100507 break;
508
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100509 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000510 *from_mode |= MM_MODE_INVALID;
Andrew Walbran648fc3e2019-10-22 16:23:05 +0100511 break;
512
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100513 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000514 *from_mode |= MM_MODE_SHARED;
Jose Marinho56c25732019-05-20 09:48:53 +0100515 break;
516
Jose Marinho75509b42019-04-09 09:34:59 +0100517 default:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100518 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho75509b42019-04-09 09:34:59 +0100519 }
520
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100521 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000522}
523
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100524static struct ffa_value ffa_relinquish_check_transition(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000525 struct vm_locked from, uint32_t *orig_from_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100526 struct ffa_memory_region_constituent **fragments,
527 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
528 uint32_t *from_mode)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000529{
530 const uint32_t state_mask =
531 MM_MODE_INVALID | MM_MODE_UNOWNED | MM_MODE_SHARED;
532 uint32_t orig_from_state;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100533 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000534
Andrew Walbranca808b12020-05-15 17:22:28 +0100535 ret = constituents_get_mode(from, orig_from_mode, fragments,
536 fragment_constituent_counts,
537 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100538 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbrana65a1322020-04-06 19:32:32 +0100539 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000540 }
541
542 /* Ensure the address range is normal memory and not a device. */
543 if (*orig_from_mode & MM_MODE_D) {
544 dlog_verbose("Can't relinquish device memory (mode is %#x).\n",
545 *orig_from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100546 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000547 }
548
549 /*
550 * Ensure the relinquishing VM is not the owner but has access to the
551 * memory.
552 */
553 orig_from_state = *orig_from_mode & state_mask;
554 if ((orig_from_state & ~MM_MODE_SHARED) != MM_MODE_UNOWNED) {
555 dlog_verbose(
556 "Tried to relinquish memory in state %#x (masked %#x "
Andrew Walbranca808b12020-05-15 17:22:28 +0100557 "but should be %#x).\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000558 *orig_from_mode, orig_from_state, MM_MODE_UNOWNED);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100559 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000560 }
561
562 /* Find the appropriate new mode. */
563 *from_mode = (~state_mask & *orig_from_mode) | MM_MODE_UNMAPPED_MASK;
564
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100565 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000566}
567
568/**
569 * Verify that all pages have the same mode, that the starting mode
570 * constitutes a valid state and obtain the next mode to apply
571 * to the retrieving VM.
572 *
573 * Returns:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100574 * 1) FFA_DENIED if a state transition was not found;
575 * 2) FFA_DENIED if the pages being shared do not have the same mode within
Andrew Walbrana65a1322020-04-06 19:32:32 +0100576 * the <to> VM;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100577 * 3) FFA_INVALID_PARAMETERS if the beginning and end IPAs are not page
Andrew Walbrana65a1322020-04-06 19:32:32 +0100578 * aligned;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100579 * 4) FFA_INVALID_PARAMETERS if the requested share type was not handled.
580 * Or FFA_SUCCESS on success.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000581 */
J-Alvesfc19b372022-07-06 12:17:35 +0100582struct ffa_value ffa_retrieve_check_transition(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000583 struct vm_locked to, uint32_t share_func,
Andrew Walbranca808b12020-05-15 17:22:28 +0100584 struct ffa_memory_region_constituent **fragments,
585 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
586 uint32_t memory_to_attributes, uint32_t *to_mode)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000587{
588 uint32_t orig_to_mode;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100589 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000590
Andrew Walbranca808b12020-05-15 17:22:28 +0100591 ret = constituents_get_mode(to, &orig_to_mode, fragments,
592 fragment_constituent_counts,
593 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100594 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100595 dlog_verbose("Inconsistent modes.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100596 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000597 }
598
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100599 if (share_func == FFA_MEM_RECLAIM_32) {
J-Alves9256f162021-12-09 13:18:43 +0000600 /*
601 * If the original ffa memory send call has been processed
602 * successfully, it is expected the orig_to_mode would overlay
603 * with `state_mask`, as a result of the function
604 * `ffa_send_check_transition`.
605 */
J-Alves59ed0042022-07-28 18:26:41 +0100606 if (vm_id_is_current_world(to.vm->id)) {
607 assert((orig_to_mode &
608 (MM_MODE_INVALID | MM_MODE_UNOWNED |
609 MM_MODE_SHARED)) != 0U);
610 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000611 } else {
612 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +0100613 * If the retriever is from virtual FF-A instance:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000614 * Ensure the retriever has the expected state. We don't care
615 * about the MM_MODE_SHARED bit; either with or without it set
616 * are both valid representations of the !O-NA state.
617 */
J-Alvesa9cd7e32022-07-01 13:49:33 +0100618 if (vm_id_is_current_world(to.vm->id) &&
619 to.vm->id != HF_PRIMARY_VM_ID &&
620 (orig_to_mode & MM_MODE_UNMAPPED_MASK) !=
621 MM_MODE_UNMAPPED_MASK) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100622 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000623 }
624 }
625
626 /* Find the appropriate new mode. */
627 *to_mode = memory_to_attributes;
628 switch (share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100629 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000630 *to_mode |= 0;
631 break;
632
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100633 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000634 *to_mode |= MM_MODE_UNOWNED;
635 break;
636
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100637 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000638 *to_mode |= MM_MODE_UNOWNED | MM_MODE_SHARED;
639 break;
640
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100641 case FFA_MEM_RECLAIM_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000642 *to_mode |= 0;
643 break;
644
645 default:
Andrew Walbranca808b12020-05-15 17:22:28 +0100646 dlog_error("Invalid share_func %#x.\n", share_func);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100647 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000648 }
649
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100650 return (struct ffa_value){.func = FFA_SUCCESS_32};
Jose Marinho75509b42019-04-09 09:34:59 +0100651}
Jose Marinho09b1db82019-08-08 09:16:59 +0100652
653/**
654 * Updates a VM's page table such that the given set of physical address ranges
655 * are mapped in the address space at the corresponding address ranges, in the
656 * mode provided.
657 *
658 * If commit is false, the page tables will be allocated from the mpool but no
659 * mappings will actually be updated. This function must always be called first
660 * with commit false to check that it will succeed before calling with commit
661 * true, to avoid leaving the page table in a half-updated state. To make a
662 * series of changes atomically you can call them all with commit false before
663 * calling them all with commit true.
664 *
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -0700665 * vm_ptable_defrag should always be called after a series of page table
666 * updates, whether they succeed or fail.
Jose Marinho09b1db82019-08-08 09:16:59 +0100667 *
668 * Returns true on success, or false if the update failed and no changes were
669 * made to memory mappings.
670 */
J-Alves66652252022-07-06 09:49:51 +0100671bool ffa_region_group_identity_map(
Andrew Walbranf4b51af2020-02-03 14:44:54 +0000672 struct vm_locked vm_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +0100673 struct ffa_memory_region_constituent **fragments,
674 const uint32_t *fragment_constituent_counts, uint32_t fragment_count,
Daniel Boulby4dd3f532021-09-21 09:57:08 +0100675 uint32_t mode, struct mpool *ppool, bool commit)
Jose Marinho09b1db82019-08-08 09:16:59 +0100676{
Andrew Walbranca808b12020-05-15 17:22:28 +0100677 uint32_t i;
678 uint32_t j;
Jose Marinho09b1db82019-08-08 09:16:59 +0100679
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -0700680 if (vm_locked.vm->el0_partition) {
681 mode |= MM_MODE_USER | MM_MODE_NG;
682 }
683
Andrew Walbranca808b12020-05-15 17:22:28 +0100684 /* Iterate over the memory region constituents within each fragment. */
685 for (i = 0; i < fragment_count; ++i) {
686 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
687 size_t size = fragments[i][j].page_count * PAGE_SIZE;
688 paddr_t pa_begin =
689 pa_from_ipa(ipa_init(fragments[i][j].address));
690 paddr_t pa_end = pa_add(pa_begin, size);
Jens Wiklander4f1880c2022-10-19 17:00:14 +0200691 uint32_t pa_bits =
692 arch_mm_get_pa_bits(arch_mm_get_pa_range());
Federico Recanati4fd065d2021-12-13 20:06:23 +0100693
694 /*
695 * Ensure the requested region falls into system's PA
696 * range.
697 */
Jens Wiklander4f1880c2022-10-19 17:00:14 +0200698 if (((pa_addr(pa_begin) >> pa_bits) > 0) ||
699 ((pa_addr(pa_end) >> pa_bits) > 0)) {
Federico Recanati4fd065d2021-12-13 20:06:23 +0100700 dlog_error("Region is outside of PA Range\n");
701 return false;
702 }
Andrew Walbranca808b12020-05-15 17:22:28 +0100703
704 if (commit) {
705 vm_identity_commit(vm_locked, pa_begin, pa_end,
706 mode, ppool, NULL);
707 } else if (!vm_identity_prepare(vm_locked, pa_begin,
708 pa_end, mode, ppool)) {
709 return false;
710 }
Jose Marinho09b1db82019-08-08 09:16:59 +0100711 }
712 }
713
714 return true;
715}
716
717/**
718 * Clears a region of physical memory by overwriting it with zeros. The data is
719 * flushed from the cache so the memory has been cleared across the system.
720 */
J-Alves7db32002021-12-14 14:44:50 +0000721static bool clear_memory(paddr_t begin, paddr_t end, struct mpool *ppool,
722 uint32_t extra_mode_attributes)
Jose Marinho09b1db82019-08-08 09:16:59 +0100723{
724 /*
Fuad Tabbaed294af2019-12-20 10:43:01 +0000725 * TODO: change this to a CPU local single page window rather than a
Jose Marinho09b1db82019-08-08 09:16:59 +0100726 * global mapping of the whole range. Such an approach will limit
727 * the changes to stage-1 tables and will allow only local
728 * invalidation.
729 */
730 bool ret;
731 struct mm_stage1_locked stage1_locked = mm_lock_stage1();
J-Alves7db32002021-12-14 14:44:50 +0000732 void *ptr = mm_identity_map(stage1_locked, begin, end,
733 MM_MODE_W | (extra_mode_attributes &
734 plat_ffa_other_world_mode()),
735 ppool);
Jose Marinho09b1db82019-08-08 09:16:59 +0100736 size_t size = pa_difference(begin, end);
737
738 if (!ptr) {
Jose Marinho09b1db82019-08-08 09:16:59 +0100739 goto fail;
740 }
741
742 memset_s(ptr, size, 0, size);
743 arch_mm_flush_dcache(ptr, size);
744 mm_unmap(stage1_locked, begin, end, ppool);
745
746 ret = true;
747 goto out;
748
749fail:
750 ret = false;
751
752out:
753 mm_unlock_stage1(&stage1_locked);
754
755 return ret;
756}
757
758/**
759 * Clears a region of physical memory by overwriting it with zeros. The data is
760 * flushed from the cache so the memory has been cleared across the system.
761 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100762static bool ffa_clear_memory_constituents(
J-Alves7db32002021-12-14 14:44:50 +0000763 uint32_t security_state_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100764 struct ffa_memory_region_constituent **fragments,
765 const uint32_t *fragment_constituent_counts, uint32_t fragment_count,
766 struct mpool *page_pool)
Jose Marinho09b1db82019-08-08 09:16:59 +0100767{
768 struct mpool local_page_pool;
Andrew Walbranca808b12020-05-15 17:22:28 +0100769 uint32_t i;
Jose Marinho09b1db82019-08-08 09:16:59 +0100770 bool ret = false;
771
772 /*
773 * Create a local pool so any freed memory can't be used by another
774 * thread. This is to ensure each constituent that is mapped can be
775 * unmapped again afterwards.
776 */
Andrew Walbran475c1452020-02-07 13:22:22 +0000777 mpool_init_with_fallback(&local_page_pool, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +0100778
Andrew Walbranca808b12020-05-15 17:22:28 +0100779 /* Iterate over the memory region constituents within each fragment. */
780 for (i = 0; i < fragment_count; ++i) {
781 uint32_t j;
Jose Marinho09b1db82019-08-08 09:16:59 +0100782
Andrew Walbranca808b12020-05-15 17:22:28 +0100783 for (j = 0; j < fragment_constituent_counts[j]; ++j) {
784 size_t size = fragments[i][j].page_count * PAGE_SIZE;
785 paddr_t begin =
786 pa_from_ipa(ipa_init(fragments[i][j].address));
787 paddr_t end = pa_add(begin, size);
788
J-Alves7db32002021-12-14 14:44:50 +0000789 if (!clear_memory(begin, end, &local_page_pool,
790 security_state_mode)) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100791 /*
792 * api_clear_memory will defrag on failure, so
793 * no need to do it here.
794 */
795 goto out;
796 }
Jose Marinho09b1db82019-08-08 09:16:59 +0100797 }
798 }
799
Jose Marinho09b1db82019-08-08 09:16:59 +0100800 ret = true;
801
802out:
803 mpool_fini(&local_page_pool);
804 return ret;
805}
806
J-Alves5952d942022-12-22 16:03:00 +0000807static bool is_memory_range_within(ipaddr_t begin, ipaddr_t end,
808 ipaddr_t in_begin, ipaddr_t in_end)
809{
810 return (ipa_addr(begin) >= ipa_addr(in_begin) &&
811 ipa_addr(begin) < ipa_addr(in_end)) ||
812 (ipa_addr(end) <= ipa_addr(in_end) &&
813 ipa_addr(end) > ipa_addr(in_begin));
814}
815
816/**
817 * Receives a memory range and looks for overlaps with the remainder
818 * constituents of the memory share/lend/donate operation. Assumes they are
819 * passed in order to avoid having to loop over all the elements at each call.
820 * The function only compares the received memory ranges with those that follow
821 * within the same fragment, and subsequent fragments from the same operation.
822 */
823static bool ffa_memory_check_overlap(
824 struct ffa_memory_region_constituent **fragments,
825 const uint32_t *fragment_constituent_counts,
826 const uint32_t fragment_count, const uint32_t current_fragment,
827 const uint32_t current_constituent)
828{
829 uint32_t i = current_fragment;
830 uint32_t j = current_constituent;
831 ipaddr_t current_begin = ipa_init(fragments[i][j].address);
832 const uint32_t current_page_count = fragments[i][j].page_count;
833 size_t current_size = current_page_count * PAGE_SIZE;
834 ipaddr_t current_end = ipa_add(current_begin, current_size - 1);
835
836 if (current_size == 0 ||
837 current_size > UINT64_MAX - ipa_addr(current_begin)) {
838 dlog_verbose("Invalid page count. Addr: %x page_count: %x\n",
839 current_begin, current_page_count);
840 return false;
841 }
842
843 for (; i < fragment_count; i++) {
844 j = (i == current_fragment) ? j + 1 : 0;
845
846 for (; j < fragment_constituent_counts[i]; j++) {
847 ipaddr_t begin = ipa_init(fragments[i][j].address);
848 const uint32_t page_count = fragments[i][j].page_count;
849 size_t size = page_count * PAGE_SIZE;
850 ipaddr_t end = ipa_add(begin, size - 1);
851
852 if (size == 0 || size > UINT64_MAX - ipa_addr(begin)) {
853 dlog_verbose(
854 "Invalid page count. Addr: %x "
855 "page_count: %x\n",
856 begin, page_count);
857 return false;
858 }
859
860 /*
861 * Check if current ranges is within begin and end, as
862 * well as the reverse. This should help optimize the
863 * loop, and reduce the number of iterations.
864 */
865 if (is_memory_range_within(begin, end, current_begin,
866 current_end) ||
867 is_memory_range_within(current_begin, current_end,
868 begin, end)) {
869 dlog_verbose(
870 "Overlapping memory ranges: %#x - %#x "
871 "with %#x - %#x\n",
872 ipa_addr(begin), ipa_addr(end),
873 ipa_addr(current_begin),
874 ipa_addr(current_end));
875 return true;
876 }
877 }
878 }
879
880 return false;
881}
882
Jose Marinho09b1db82019-08-08 09:16:59 +0100883/**
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000884 * Validates and prepares memory to be sent from the calling VM to another.
Jose Marinho09b1db82019-08-08 09:16:59 +0100885 *
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000886 * This function requires the calling context to hold the <from> VM lock.
Jose Marinho09b1db82019-08-08 09:16:59 +0100887 *
888 * Returns:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000889 * In case of error, one of the following values is returned:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100890 * 1) FFA_INVALID_PARAMETERS - The endpoint provided parameters were
Jose Marinho09b1db82019-08-08 09:16:59 +0100891 * erroneous;
Andrew Walbranf07f04d2020-05-01 18:09:00 +0100892 * 2) FFA_NO_MEMORY - Hafnium did not have sufficient memory to complete the
893 * request.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100894 * 3) FFA_DENIED - The sender doesn't have sufficient access to send the
Andrew Walbrana65a1322020-04-06 19:32:32 +0100895 * memory with the given permissions.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100896 * Success is indicated by FFA_SUCCESS.
Jose Marinho09b1db82019-08-08 09:16:59 +0100897 */
J-Alves66652252022-07-06 09:49:51 +0100898struct ffa_value ffa_send_check_update(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000899 struct vm_locked from_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +0100900 struct ffa_memory_region_constituent **fragments,
901 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
J-Alves8f11cde2022-12-21 16:18:22 +0000902 uint32_t composite_total_page_count, uint32_t share_func,
903 struct ffa_memory_access *receivers, uint32_t receivers_count,
904 struct mpool *page_pool, bool clear, uint32_t *orig_from_mode_ret)
Jose Marinho09b1db82019-08-08 09:16:59 +0100905{
Andrew Walbranca808b12020-05-15 17:22:28 +0100906 uint32_t i;
J-Alves8f11cde2022-12-21 16:18:22 +0000907 uint32_t j;
Jose Marinho09b1db82019-08-08 09:16:59 +0100908 uint32_t orig_from_mode;
909 uint32_t from_mode;
Jose Marinho09b1db82019-08-08 09:16:59 +0100910 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100911 struct ffa_value ret;
J-Alves8f11cde2022-12-21 16:18:22 +0000912 uint32_t constituents_total_page_count = 0;
Jose Marinho09b1db82019-08-08 09:16:59 +0100913
914 /*
Andrew Walbrana65a1322020-04-06 19:32:32 +0100915 * Make sure constituents are properly aligned to a 64-bit boundary. If
916 * not we would get alignment faults trying to read (64-bit) values.
Jose Marinho09b1db82019-08-08 09:16:59 +0100917 */
Andrew Walbranca808b12020-05-15 17:22:28 +0100918 for (i = 0; i < fragment_count; ++i) {
919 if (!is_aligned(fragments[i], 8)) {
920 dlog_verbose("Constituents not aligned.\n");
921 return ffa_error(FFA_INVALID_PARAMETERS);
922 }
J-Alves8f11cde2022-12-21 16:18:22 +0000923 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
924 constituents_total_page_count +=
925 fragments[i][j].page_count;
J-Alves5952d942022-12-22 16:03:00 +0000926 if (ffa_memory_check_overlap(
927 fragments, fragment_constituent_counts,
928 fragment_count, i, j)) {
929 return ffa_error(FFA_INVALID_PARAMETERS);
930 }
J-Alves8f11cde2022-12-21 16:18:22 +0000931 }
932 }
933
934 if (constituents_total_page_count != composite_total_page_count) {
935 dlog_verbose(
936 "Composite page count differs from calculated page "
937 "count from constituents.\n");
938 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho09b1db82019-08-08 09:16:59 +0100939 }
940
941 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000942 * Check if the state transition is lawful for the sender, ensure that
943 * all constituents of a memory region being shared are at the same
944 * state.
Jose Marinho09b1db82019-08-08 09:16:59 +0100945 */
J-Alves363f5722022-04-25 17:37:37 +0100946 ret = ffa_send_check_transition(from_locked, share_func, receivers,
947 receivers_count, &orig_from_mode,
948 fragments, fragment_constituent_counts,
Andrew Walbranca808b12020-05-15 17:22:28 +0100949 fragment_count, &from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100950 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100951 dlog_verbose("Invalid transition for send.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100952 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +0100953 }
954
Andrew Walbran37c574e2020-06-03 11:45:46 +0100955 if (orig_from_mode_ret != NULL) {
956 *orig_from_mode_ret = orig_from_mode;
957 }
958
Jose Marinho09b1db82019-08-08 09:16:59 +0100959 /*
960 * Create a local pool so any freed memory can't be used by another
961 * thread. This is to ensure the original mapping can be restored if the
962 * clear fails.
963 */
Andrew Walbran475c1452020-02-07 13:22:22 +0000964 mpool_init_with_fallback(&local_page_pool, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +0100965
966 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000967 * First reserve all required memory for the new page table entries
968 * without committing, to make sure the entire operation will succeed
969 * without exhausting the page pool.
Jose Marinho09b1db82019-08-08 09:16:59 +0100970 */
Andrew Walbranca808b12020-05-15 17:22:28 +0100971 if (!ffa_region_group_identity_map(
972 from_locked, fragments, fragment_constituent_counts,
973 fragment_count, from_mode, page_pool, false)) {
Jose Marinho09b1db82019-08-08 09:16:59 +0100974 /* TODO: partial defrag of failed range. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100975 ret = ffa_error(FFA_NO_MEMORY);
Jose Marinho09b1db82019-08-08 09:16:59 +0100976 goto out;
977 }
978
979 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000980 * Update the mapping for the sender. This won't allocate because the
981 * transaction was already prepared above, but may free pages in the
982 * case that a whole block is being unmapped that was previously
983 * partially mapped.
Jose Marinho09b1db82019-08-08 09:16:59 +0100984 */
Andrew Walbranca808b12020-05-15 17:22:28 +0100985 CHECK(ffa_region_group_identity_map(
986 from_locked, fragments, fragment_constituent_counts,
987 fragment_count, from_mode, &local_page_pool, true));
Jose Marinho09b1db82019-08-08 09:16:59 +0100988
989 /* Clear the memory so no VM or device can see the previous contents. */
J-Alves7db32002021-12-14 14:44:50 +0000990 if (clear &&
J-Alves26483382023-04-20 12:01:49 +0100991 !ffa_clear_memory_constituents(orig_from_mode, fragments,
992 fragment_constituent_counts,
993 fragment_count, page_pool)) {
Jose Marinho09b1db82019-08-08 09:16:59 +0100994 /*
995 * On failure, roll back by returning memory to the sender. This
996 * may allocate pages which were previously freed into
997 * `local_page_pool` by the call above, but will never allocate
998 * more pages than that so can never fail.
999 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001000 CHECK(ffa_region_group_identity_map(
Andrew Walbranca808b12020-05-15 17:22:28 +01001001 from_locked, fragments, fragment_constituent_counts,
1002 fragment_count, orig_from_mode, &local_page_pool,
1003 true));
Jose Marinho09b1db82019-08-08 09:16:59 +01001004
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001005 ret = ffa_error(FFA_NO_MEMORY);
Jose Marinho09b1db82019-08-08 09:16:59 +01001006 goto out;
1007 }
1008
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001009 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001010
1011out:
1012 mpool_fini(&local_page_pool);
1013
1014 /*
1015 * Tidy up the page table by reclaiming failed mappings (if there was an
1016 * error) or merging entries into blocks where possible (on success).
1017 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001018 vm_ptable_defrag(from_locked, page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001019
1020 return ret;
1021}
1022
1023/**
1024 * Validates and maps memory shared from one VM to another.
1025 *
1026 * This function requires the calling context to hold the <to> lock.
1027 *
1028 * Returns:
1029 * In case of error, one of the following values is returned:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001030 * 1) FFA_INVALID_PARAMETERS - The endpoint provided parameters were
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001031 * erroneous;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001032 * 2) FFA_NO_MEMORY - Hafnium did not have sufficient memory to complete
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001033 * the request.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001034 * Success is indicated by FFA_SUCCESS.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001035 */
J-Alvesb5084cf2022-07-06 14:20:12 +01001036struct ffa_value ffa_retrieve_check_update(
J-Alves26483382023-04-20 12:01:49 +01001037 struct vm_locked to_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01001038 struct ffa_memory_region_constituent **fragments,
1039 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
J-Alves26483382023-04-20 12:01:49 +01001040 uint32_t sender_orig_mode, uint32_t share_func, bool clear,
Andrew Walbranca808b12020-05-15 17:22:28 +01001041 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001042{
Andrew Walbranca808b12020-05-15 17:22:28 +01001043 uint32_t i;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001044 uint32_t to_mode;
1045 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001046 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001047
1048 /*
Andrew Walbranca808b12020-05-15 17:22:28 +01001049 * Make sure constituents are properly aligned to a 64-bit boundary. If
1050 * not we would get alignment faults trying to read (64-bit) values.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001051 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001052 for (i = 0; i < fragment_count; ++i) {
1053 if (!is_aligned(fragments[i], 8)) {
J-Alvesb5084cf2022-07-06 14:20:12 +01001054 dlog_verbose("Fragment not properly aligned.\n");
Andrew Walbranca808b12020-05-15 17:22:28 +01001055 return ffa_error(FFA_INVALID_PARAMETERS);
1056 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001057 }
1058
1059 /*
1060 * Check if the state transition is lawful for the recipient, and ensure
1061 * that all constituents of the memory region being retrieved are at the
1062 * same state.
1063 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001064 ret = ffa_retrieve_check_transition(
1065 to_locked, share_func, fragments, fragment_constituent_counts,
J-Alves26483382023-04-20 12:01:49 +01001066 fragment_count, sender_orig_mode, &to_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001067 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001068 dlog_verbose("Invalid transition for retrieve.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +01001069 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001070 }
1071
1072 /*
1073 * Create a local pool so any freed memory can't be used by another
1074 * thread. This is to ensure the original mapping can be restored if the
1075 * clear fails.
1076 */
1077 mpool_init_with_fallback(&local_page_pool, page_pool);
1078
1079 /*
1080 * First reserve all required memory for the new page table entries in
1081 * the recipient page tables without committing, to make sure the entire
1082 * operation will succeed without exhausting the page pool.
1083 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001084 if (!ffa_region_group_identity_map(
1085 to_locked, fragments, fragment_constituent_counts,
1086 fragment_count, to_mode, page_pool, false)) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001087 /* TODO: partial defrag of failed range. */
1088 dlog_verbose(
1089 "Insufficient memory to update recipient page "
1090 "table.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001091 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001092 goto out;
1093 }
1094
1095 /* Clear the memory so no VM or device can see the previous contents. */
J-Alves7db32002021-12-14 14:44:50 +00001096 if (clear &&
J-Alves26483382023-04-20 12:01:49 +01001097 !ffa_clear_memory_constituents(sender_orig_mode, fragments,
1098 fragment_constituent_counts,
1099 fragment_count, page_pool)) {
J-Alvesb5084cf2022-07-06 14:20:12 +01001100 dlog_verbose("Couldn't clear constituents.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001101 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001102 goto out;
1103 }
1104
Jose Marinho09b1db82019-08-08 09:16:59 +01001105 /*
1106 * Complete the transfer by mapping the memory into the recipient. This
1107 * won't allocate because the transaction was already prepared above, so
1108 * it doesn't need to use the `local_page_pool`.
1109 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001110 CHECK(ffa_region_group_identity_map(
1111 to_locked, fragments, fragment_constituent_counts,
1112 fragment_count, to_mode, page_pool, true));
Jose Marinho09b1db82019-08-08 09:16:59 +01001113
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001114 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Jose Marinho09b1db82019-08-08 09:16:59 +01001115
1116out:
1117 mpool_fini(&local_page_pool);
1118
1119 /*
Andrew Walbranf07f04d2020-05-01 18:09:00 +01001120 * Tidy up the page table by reclaiming failed mappings (if there was an
1121 * error) or merging entries into blocks where possible (on success).
Jose Marinho09b1db82019-08-08 09:16:59 +01001122 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001123 vm_ptable_defrag(to_locked, page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001124
1125 return ret;
1126}
1127
Andrew Walbran996d1d12020-05-27 14:08:43 +01001128static struct ffa_value ffa_relinquish_check_update(
J-Alves26483382023-04-20 12:01:49 +01001129 struct vm_locked from_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01001130 struct ffa_memory_region_constituent **fragments,
1131 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
1132 struct mpool *page_pool, bool clear)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001133{
1134 uint32_t orig_from_mode;
1135 uint32_t from_mode;
1136 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001137 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001138
Andrew Walbranca808b12020-05-15 17:22:28 +01001139 ret = ffa_relinquish_check_transition(
1140 from_locked, &orig_from_mode, fragments,
1141 fragment_constituent_counts, fragment_count, &from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001142 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001143 dlog_verbose("Invalid transition for relinquish.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +01001144 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001145 }
1146
1147 /*
1148 * Create a local pool so any freed memory can't be used by another
1149 * thread. This is to ensure the original mapping can be restored if the
1150 * clear fails.
1151 */
1152 mpool_init_with_fallback(&local_page_pool, page_pool);
1153
1154 /*
1155 * First reserve all required memory for the new page table entries
1156 * without committing, to make sure the entire operation will succeed
1157 * without exhausting the page pool.
1158 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001159 if (!ffa_region_group_identity_map(
1160 from_locked, fragments, fragment_constituent_counts,
1161 fragment_count, from_mode, page_pool, false)) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001162 /* TODO: partial defrag of failed range. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001163 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001164 goto out;
1165 }
1166
1167 /*
1168 * Update the mapping for the sender. This won't allocate because the
1169 * transaction was already prepared above, but may free pages in the
1170 * case that a whole block is being unmapped that was previously
1171 * partially mapped.
1172 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001173 CHECK(ffa_region_group_identity_map(
1174 from_locked, fragments, fragment_constituent_counts,
1175 fragment_count, from_mode, &local_page_pool, true));
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001176
1177 /* Clear the memory so no VM or device can see the previous contents. */
J-Alves7db32002021-12-14 14:44:50 +00001178 if (clear &&
J-Alves26483382023-04-20 12:01:49 +01001179 !ffa_clear_memory_constituents(orig_from_mode, fragments,
1180 fragment_constituent_counts,
1181 fragment_count, page_pool)) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001182 /*
1183 * On failure, roll back by returning memory to the sender. This
1184 * may allocate pages which were previously freed into
1185 * `local_page_pool` by the call above, but will never allocate
1186 * more pages than that so can never fail.
1187 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001188 CHECK(ffa_region_group_identity_map(
Andrew Walbranca808b12020-05-15 17:22:28 +01001189 from_locked, fragments, fragment_constituent_counts,
1190 fragment_count, orig_from_mode, &local_page_pool,
1191 true));
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001192
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001193 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001194 goto out;
1195 }
1196
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001197 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001198
1199out:
1200 mpool_fini(&local_page_pool);
1201
1202 /*
1203 * Tidy up the page table by reclaiming failed mappings (if there was an
1204 * error) or merging entries into blocks where possible (on success).
1205 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001206 vm_ptable_defrag(from_locked, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +01001207
1208 return ret;
1209}
1210
1211/**
Andrew Walbranca808b12020-05-15 17:22:28 +01001212 * Complete a memory sending operation by checking that it is valid, updating
1213 * the sender page table, and then either marking the share state as having
1214 * completed sending (on success) or freeing it (on failure).
1215 *
1216 * Returns FFA_SUCCESS with the handle encoded, or the relevant FFA_ERROR.
1217 */
J-Alvesfdd29272022-07-19 13:16:31 +01001218struct ffa_value ffa_memory_send_complete(
Andrew Walbranca808b12020-05-15 17:22:28 +01001219 struct vm_locked from_locked, struct share_states_locked share_states,
Andrew Walbran37c574e2020-06-03 11:45:46 +01001220 struct ffa_memory_share_state *share_state, struct mpool *page_pool,
1221 uint32_t *orig_from_mode_ret)
Andrew Walbranca808b12020-05-15 17:22:28 +01001222{
1223 struct ffa_memory_region *memory_region = share_state->memory_region;
J-Alves8f11cde2022-12-21 16:18:22 +00001224 struct ffa_composite_memory_region *composite;
Andrew Walbranca808b12020-05-15 17:22:28 +01001225 struct ffa_value ret;
1226
1227 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +00001228 assert(share_states.share_states != NULL);
J-Alves8f11cde2022-12-21 16:18:22 +00001229 assert(memory_region != NULL);
1230 composite = ffa_memory_region_get_composite(memory_region, 0);
1231 assert(composite != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +01001232
1233 /* Check that state is valid in sender page table and update. */
1234 ret = ffa_send_check_update(
1235 from_locked, share_state->fragments,
1236 share_state->fragment_constituent_counts,
J-Alves8f11cde2022-12-21 16:18:22 +00001237 share_state->fragment_count, composite->page_count,
1238 share_state->share_func, memory_region->receivers,
1239 memory_region->receiver_count, page_pool,
1240 memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR,
Andrew Walbran37c574e2020-06-03 11:45:46 +01001241 orig_from_mode_ret);
Andrew Walbranca808b12020-05-15 17:22:28 +01001242 if (ret.func != FFA_SUCCESS_32) {
1243 /*
1244 * Free share state, it failed to send so it can't be retrieved.
1245 */
Karl Meakin4cec5e82023-06-30 16:30:22 +01001246 dlog_verbose("%s: failed to send check update: %s(%s)\n",
1247 __func__, ffa_func_name(ret.func),
1248 ffa_error_name(ffa_error_code(ret)));
Andrew Walbranca808b12020-05-15 17:22:28 +01001249 share_state_free(share_states, share_state, page_pool);
1250 return ret;
1251 }
1252
1253 share_state->sending_complete = true;
Karl Meakin4cec5e82023-06-30 16:30:22 +01001254 dlog_verbose("%s: marked sending complete.\n", __func__);
Andrew Walbranca808b12020-05-15 17:22:28 +01001255
J-Alvesee68c542020-10-29 17:48:20 +00001256 return ffa_mem_success(share_state->memory_region->handle);
Andrew Walbranca808b12020-05-15 17:22:28 +01001257}
1258
1259/**
Federico Recanatia98603a2021-12-20 18:04:03 +01001260 * Check that the memory attributes match Hafnium expectations:
1261 * Normal Memory, Inner shareable, Write-Back Read-Allocate
1262 * Write-Allocate Cacheable.
1263 */
1264static struct ffa_value ffa_memory_attributes_validate(
J-Alves7a99d0d2023-02-08 13:49:48 +00001265 ffa_memory_attributes_t attributes)
Federico Recanatia98603a2021-12-20 18:04:03 +01001266{
1267 enum ffa_memory_type memory_type;
1268 enum ffa_memory_cacheability cacheability;
1269 enum ffa_memory_shareability shareability;
1270
1271 memory_type = ffa_get_memory_type_attr(attributes);
1272 if (memory_type != FFA_MEMORY_NORMAL_MEM) {
1273 dlog_verbose("Invalid memory type %#x, expected %#x.\n",
1274 memory_type, FFA_MEMORY_NORMAL_MEM);
Federico Recanati3d953f32022-02-17 09:31:29 +01001275 return ffa_error(FFA_DENIED);
Federico Recanatia98603a2021-12-20 18:04:03 +01001276 }
1277
1278 cacheability = ffa_get_memory_cacheability_attr(attributes);
1279 if (cacheability != FFA_MEMORY_CACHE_WRITE_BACK) {
1280 dlog_verbose("Invalid cacheability %#x, expected %#x.\n",
1281 cacheability, FFA_MEMORY_CACHE_WRITE_BACK);
Federico Recanati3d953f32022-02-17 09:31:29 +01001282 return ffa_error(FFA_DENIED);
Federico Recanatia98603a2021-12-20 18:04:03 +01001283 }
1284
1285 shareability = ffa_get_memory_shareability_attr(attributes);
1286 if (shareability != FFA_MEMORY_INNER_SHAREABLE) {
1287 dlog_verbose("Invalid shareability %#x, expected #%x.\n",
1288 shareability, FFA_MEMORY_INNER_SHAREABLE);
Federico Recanati3d953f32022-02-17 09:31:29 +01001289 return ffa_error(FFA_DENIED);
Federico Recanatia98603a2021-12-20 18:04:03 +01001290 }
1291
1292 return (struct ffa_value){.func = FFA_SUCCESS_32};
1293}
1294
1295/**
Andrew Walbrana65a1322020-04-06 19:32:32 +01001296 * Check that the given `memory_region` represents a valid memory send request
1297 * of the given `share_func` type, return the clear flag and permissions via the
1298 * respective output parameters, and update the permissions if necessary.
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001299 *
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001300 * Returns FFA_SUCCESS if the request was valid, or the relevant FFA_ERROR if
Andrew Walbrana65a1322020-04-06 19:32:32 +01001301 * not.
1302 */
J-Alves66652252022-07-06 09:49:51 +01001303struct ffa_value ffa_memory_send_validate(
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001304 struct vm_locked from_locked, struct ffa_memory_region *memory_region,
1305 uint32_t memory_share_length, uint32_t fragment_length,
J-Alves363f5722022-04-25 17:37:37 +01001306 uint32_t share_func)
Andrew Walbrana65a1322020-04-06 19:32:32 +01001307{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001308 struct ffa_composite_memory_region *composite;
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001309 uint64_t receivers_end;
1310 uint64_t min_length;
Federico Recanati872cd692022-01-05 13:10:10 +01001311 uint32_t composite_memory_region_offset;
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001312 uint32_t constituents_start;
Andrew Walbran130a8ae2020-05-15 16:27:15 +01001313 uint32_t constituents_length;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001314 enum ffa_data_access data_access;
1315 enum ffa_instruction_access instruction_access;
Olivier Deprez4342a3c2022-02-28 09:37:25 +01001316 enum ffa_memory_security security_state;
Federico Recanatia98603a2021-12-20 18:04:03 +01001317 struct ffa_value ret;
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001318 const size_t minimum_first_fragment_length =
1319 (sizeof(struct ffa_memory_region) +
1320 sizeof(struct ffa_memory_access) +
1321 sizeof(struct ffa_composite_memory_region));
1322
1323 if (fragment_length < minimum_first_fragment_length) {
1324 dlog_verbose("Fragment length %u too short (min %u).\n",
1325 (size_t)fragment_length,
1326 minimum_first_fragment_length);
1327 return ffa_error(FFA_INVALID_PARAMETERS);
1328 }
1329
Demi Marie Obenour73a1e942023-02-04 14:09:18 -05001330 static_assert(sizeof(struct ffa_memory_region_constituent) == 16,
1331 "struct ffa_memory_region_constituent must be 16 bytes");
1332 if (!is_aligned(fragment_length,
1333 sizeof(struct ffa_memory_region_constituent)) ||
1334 !is_aligned(memory_share_length,
1335 sizeof(struct ffa_memory_region_constituent))) {
1336 dlog_verbose(
1337 "Fragment length %u or total length %u"
1338 " is not 16-byte aligned.\n",
1339 fragment_length, memory_share_length);
1340 return ffa_error(FFA_INVALID_PARAMETERS);
1341 }
1342
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001343 if (fragment_length > memory_share_length) {
1344 dlog_verbose(
1345 "Fragment length %u greater than total length %u.\n",
1346 (size_t)fragment_length, (size_t)memory_share_length);
1347 return ffa_error(FFA_INVALID_PARAMETERS);
1348 }
Andrew Walbrana65a1322020-04-06 19:32:32 +01001349
J-Alves0b6653d2022-04-22 13:17:38 +01001350 assert(memory_region->receivers_offset ==
1351 offsetof(struct ffa_memory_region, receivers));
1352 assert(memory_region->memory_access_desc_size ==
1353 sizeof(struct ffa_memory_access));
1354
J-Alves95df0ef2022-12-07 10:09:48 +00001355 /* The sender must match the caller. */
1356 if ((!vm_id_is_current_world(from_locked.vm->id) &&
1357 vm_id_is_current_world(memory_region->sender)) ||
1358 (vm_id_is_current_world(from_locked.vm->id) &&
1359 memory_region->sender != from_locked.vm->id)) {
1360 dlog_verbose("Invalid memory sender ID.\n");
1361 return ffa_error(FFA_DENIED);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001362 }
1363
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001364 if (memory_region->receiver_count <= 0) {
1365 dlog_verbose("No receivers!\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001366 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001367 }
1368
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001369 /*
1370 * Ensure that the composite header is within the memory bounds and
1371 * doesn't overlap the first part of the message. Cast to uint64_t
1372 * to prevent overflow.
1373 */
1374 receivers_end = ((uint64_t)sizeof(struct ffa_memory_access) *
1375 (uint64_t)memory_region->receiver_count) +
1376 sizeof(struct ffa_memory_region);
1377 min_length = receivers_end +
1378 sizeof(struct ffa_composite_memory_region) +
1379 sizeof(struct ffa_memory_region_constituent);
1380 if (min_length > memory_share_length) {
1381 dlog_verbose("Share too short: got %u but minimum is %u.\n",
1382 (size_t)memory_share_length, (size_t)min_length);
1383 return ffa_error(FFA_INVALID_PARAMETERS);
1384 }
1385
1386 composite_memory_region_offset =
1387 memory_region->receivers[0].composite_memory_region_offset;
Andrew Walbrana65a1322020-04-06 19:32:32 +01001388
1389 /*
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001390 * Check that the composite memory region descriptor is after the access
1391 * descriptors, is at least 16-byte aligned, and fits in the first
1392 * fragment.
Andrew Walbrana65a1322020-04-06 19:32:32 +01001393 */
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001394 if ((composite_memory_region_offset < receivers_end) ||
1395 (composite_memory_region_offset % 16 != 0) ||
1396 (composite_memory_region_offset >
1397 fragment_length - sizeof(struct ffa_composite_memory_region))) {
1398 dlog_verbose(
1399 "Invalid composite memory region descriptor offset "
1400 "%u.\n",
1401 (size_t)composite_memory_region_offset);
1402 return ffa_error(FFA_INVALID_PARAMETERS);
1403 }
1404
1405 /*
1406 * Compute the start of the constituent regions. Already checked
1407 * to be not more than fragment_length and thus not more than
1408 * memory_share_length.
1409 */
1410 constituents_start = composite_memory_region_offset +
1411 sizeof(struct ffa_composite_memory_region);
1412 constituents_length = memory_share_length - constituents_start;
1413
1414 /*
1415 * Check that the number of constituents is consistent with the length
1416 * of the constituent region.
1417 */
1418 composite = ffa_memory_region_get_composite(memory_region, 0);
1419 if ((constituents_length %
1420 sizeof(struct ffa_memory_region_constituent) !=
1421 0) ||
1422 ((constituents_length /
1423 sizeof(struct ffa_memory_region_constituent)) !=
1424 composite->constituent_count)) {
1425 dlog_verbose("Invalid length %u or composite offset %u.\n",
1426 (size_t)memory_share_length,
1427 (size_t)composite_memory_region_offset);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001428 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001429 }
Andrew Walbranca808b12020-05-15 17:22:28 +01001430 if (fragment_length < memory_share_length &&
1431 fragment_length < HF_MAILBOX_SIZE) {
1432 dlog_warning(
1433 "Initial fragment length %d smaller than mailbox "
1434 "size.\n",
1435 fragment_length);
1436 }
Andrew Walbrana65a1322020-04-06 19:32:32 +01001437
Andrew Walbrana65a1322020-04-06 19:32:32 +01001438 /*
1439 * Clear is not allowed for memory sharing, as the sender still has
1440 * access to the memory.
1441 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001442 if ((memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR) &&
1443 share_func == FFA_MEM_SHARE_32) {
Andrew Walbrana65a1322020-04-06 19:32:32 +01001444 dlog_verbose("Memory can't be cleared while being shared.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001445 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001446 }
1447
1448 /* No other flags are allowed/supported here. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001449 if (memory_region->flags & ~FFA_MEMORY_REGION_FLAG_CLEAR) {
Andrew Walbrana65a1322020-04-06 19:32:32 +01001450 dlog_verbose("Invalid flags %#x.\n", memory_region->flags);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001451 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001452 }
1453
J-Alves363f5722022-04-25 17:37:37 +01001454 /* Check that the permissions are valid, for each specified receiver. */
1455 for (uint32_t i = 0U; i < memory_region->receiver_count; i++) {
1456 ffa_memory_access_permissions_t permissions =
1457 memory_region->receivers[i]
1458 .receiver_permissions.permissions;
J-Alves19e20cf2023-08-02 12:48:55 +01001459 ffa_id_t receiver_id = memory_region->receivers[i]
1460 .receiver_permissions.receiver;
J-Alves363f5722022-04-25 17:37:37 +01001461
1462 if (memory_region->sender == receiver_id) {
1463 dlog_verbose("Can't share memory with itself.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001464 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001465 }
Federico Recanati85090c42021-12-15 13:17:54 +01001466
J-Alves363f5722022-04-25 17:37:37 +01001467 for (uint32_t j = i + 1; j < memory_region->receiver_count;
1468 j++) {
1469 if (receiver_id ==
1470 memory_region->receivers[j]
1471 .receiver_permissions.receiver) {
1472 dlog_verbose(
1473 "Repeated receiver(%x) in memory send "
1474 "operation.\n",
1475 memory_region->receivers[j]
1476 .receiver_permissions.receiver);
1477 return ffa_error(FFA_INVALID_PARAMETERS);
1478 }
1479 }
1480
1481 if (composite_memory_region_offset !=
1482 memory_region->receivers[i]
1483 .composite_memory_region_offset) {
1484 dlog_verbose(
1485 "All ffa_memory_access should point to the "
1486 "same composite memory region offset.\n");
1487 return ffa_error(FFA_INVALID_PARAMETERS);
1488 }
1489
1490 data_access = ffa_get_data_access_attr(permissions);
1491 instruction_access =
1492 ffa_get_instruction_access_attr(permissions);
1493 if (data_access == FFA_DATA_ACCESS_RESERVED ||
1494 instruction_access == FFA_INSTRUCTION_ACCESS_RESERVED) {
1495 dlog_verbose(
1496 "Reserved value for receiver permissions "
1497 "%#x.\n",
1498 permissions);
1499 return ffa_error(FFA_INVALID_PARAMETERS);
1500 }
1501 if (instruction_access !=
1502 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
1503 dlog_verbose(
1504 "Invalid instruction access permissions %#x "
1505 "for sending memory.\n",
1506 permissions);
1507 return ffa_error(FFA_INVALID_PARAMETERS);
1508 }
1509 if (share_func == FFA_MEM_SHARE_32) {
1510 if (data_access == FFA_DATA_ACCESS_NOT_SPECIFIED) {
1511 dlog_verbose(
1512 "Invalid data access permissions %#x "
1513 "for sharing memory.\n",
1514 permissions);
1515 return ffa_error(FFA_INVALID_PARAMETERS);
1516 }
J-Alves363f5722022-04-25 17:37:37 +01001517 }
1518 if (share_func == FFA_MEM_LEND_32 &&
1519 data_access == FFA_DATA_ACCESS_NOT_SPECIFIED) {
1520 dlog_verbose(
1521 "Invalid data access permissions %#x for "
1522 "lending memory.\n",
1523 permissions);
1524 return ffa_error(FFA_INVALID_PARAMETERS);
1525 }
1526
1527 if (share_func == FFA_MEM_DONATE_32 &&
1528 data_access != FFA_DATA_ACCESS_NOT_SPECIFIED) {
1529 dlog_verbose(
1530 "Invalid data access permissions %#x for "
1531 "donating memory.\n",
1532 permissions);
1533 return ffa_error(FFA_INVALID_PARAMETERS);
1534 }
Andrew Walbrana65a1322020-04-06 19:32:32 +01001535 }
1536
Olivier Deprez4342a3c2022-02-28 09:37:25 +01001537 /* Memory region attributes NS-Bit MBZ for FFA_MEM_SHARE/LEND/DONATE. */
1538 security_state =
1539 ffa_get_memory_security_attr(memory_region->attributes);
1540 if (security_state != FFA_MEMORY_SECURITY_UNSPECIFIED) {
1541 dlog_verbose(
1542 "Invalid security state for memory share operation.\n");
1543 return ffa_error(FFA_INVALID_PARAMETERS);
1544 }
1545
Federico Recanatid937f5e2021-12-20 17:38:23 +01001546 /*
J-Alves807794e2022-06-16 13:42:47 +01001547 * If a memory donate or lend with single borrower, the memory type
1548 * shall not be specified by the sender.
Federico Recanatid937f5e2021-12-20 17:38:23 +01001549 */
J-Alves807794e2022-06-16 13:42:47 +01001550 if (share_func == FFA_MEM_DONATE_32 ||
1551 (share_func == FFA_MEM_LEND_32 &&
1552 memory_region->receiver_count == 1)) {
1553 if (ffa_get_memory_type_attr(memory_region->attributes) !=
1554 FFA_MEMORY_NOT_SPECIFIED_MEM) {
1555 dlog_verbose(
1556 "Memory type shall not be specified by "
1557 "sender.\n");
1558 return ffa_error(FFA_INVALID_PARAMETERS);
1559 }
1560 } else {
1561 /*
1562 * Check that sender's memory attributes match Hafnium
1563 * expectations: Normal Memory, Inner shareable, Write-Back
1564 * Read-Allocate Write-Allocate Cacheable.
1565 */
1566 ret = ffa_memory_attributes_validate(memory_region->attributes);
1567 if (ret.func != FFA_SUCCESS_32) {
1568 return ret;
1569 }
Federico Recanatid937f5e2021-12-20 17:38:23 +01001570 }
1571
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001572 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbrana65a1322020-04-06 19:32:32 +01001573}
1574
1575/**
Andrew Walbranca808b12020-05-15 17:22:28 +01001576 * Gets the share state for continuing an operation to donate, lend or share
1577 * memory, and checks that it is a valid request.
1578 *
1579 * Returns FFA_SUCCESS if the request was valid, or the relevant FFA_ERROR if
1580 * not.
1581 */
J-Alvesfdd29272022-07-19 13:16:31 +01001582struct ffa_value ffa_memory_send_continue_validate(
Andrew Walbranca808b12020-05-15 17:22:28 +01001583 struct share_states_locked share_states, ffa_memory_handle_t handle,
J-Alves19e20cf2023-08-02 12:48:55 +01001584 struct ffa_memory_share_state **share_state_ret, ffa_id_t from_vm_id,
Andrew Walbranca808b12020-05-15 17:22:28 +01001585 struct mpool *page_pool)
1586{
1587 struct ffa_memory_share_state *share_state;
1588 struct ffa_memory_region *memory_region;
1589
Daniel Boulbya2f8c662021-11-26 17:52:53 +00001590 assert(share_state_ret != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +01001591
1592 /*
1593 * Look up the share state by handle and make sure that the VM ID
1594 * matches.
1595 */
Karl Meakin4a2854a2023-06-30 16:26:52 +01001596 share_state = get_share_state(share_states, handle);
1597 if (!share_state) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001598 dlog_verbose(
1599 "Invalid handle %#x for memory send continuation.\n",
1600 handle);
1601 return ffa_error(FFA_INVALID_PARAMETERS);
1602 }
1603 memory_region = share_state->memory_region;
1604
J-Alvesfdd29272022-07-19 13:16:31 +01001605 if (vm_id_is_current_world(from_vm_id) &&
1606 memory_region->sender != from_vm_id) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001607 dlog_verbose("Invalid sender %d.\n", memory_region->sender);
1608 return ffa_error(FFA_INVALID_PARAMETERS);
1609 }
1610
1611 if (share_state->sending_complete) {
1612 dlog_verbose(
1613 "Sending of memory handle %#x is already complete.\n",
1614 handle);
1615 return ffa_error(FFA_INVALID_PARAMETERS);
1616 }
1617
1618 if (share_state->fragment_count == MAX_FRAGMENTS) {
1619 /*
1620 * Log a warning as this is a sign that MAX_FRAGMENTS should
1621 * probably be increased.
1622 */
1623 dlog_warning(
1624 "Too many fragments for memory share with handle %#x; "
1625 "only %d supported.\n",
1626 handle, MAX_FRAGMENTS);
1627 /* Free share state, as it's not possible to complete it. */
1628 share_state_free(share_states, share_state, page_pool);
1629 return ffa_error(FFA_NO_MEMORY);
1630 }
1631
1632 *share_state_ret = share_state;
1633
1634 return (struct ffa_value){.func = FFA_SUCCESS_32};
1635}
1636
1637/**
J-Alves95df0ef2022-12-07 10:09:48 +00001638 * Checks if there is at least one receiver from the other world.
1639 */
J-Alvesfdd29272022-07-19 13:16:31 +01001640bool memory_region_receivers_from_other_world(
J-Alves95df0ef2022-12-07 10:09:48 +00001641 struct ffa_memory_region *memory_region)
1642{
1643 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
J-Alves19e20cf2023-08-02 12:48:55 +01001644 ffa_id_t receiver = memory_region->receivers[i]
1645 .receiver_permissions.receiver;
J-Alves95df0ef2022-12-07 10:09:48 +00001646 if (!vm_id_is_current_world(receiver)) {
1647 return true;
1648 }
1649 }
1650 return false;
1651}
1652
1653/**
J-Alves9da280b2022-12-21 14:55:39 +00001654 * Validates a call to donate, lend or share memory in which Hafnium is the
1655 * designated allocator of the memory handle. In practice, this also means
1656 * Hafnium is responsible for managing the state structures for the transaction.
1657 * If Hafnium is the SPMC, it should allocate the memory handle when either the
1658 * sender is an SP or there is at least one borrower that is an SP.
1659 * If Hafnium is the hypervisor, it should allocate the memory handle when
1660 * operation involves only NWd VMs.
1661 *
1662 * If validation goes well, Hafnium updates the stage-2 page tables of the
1663 * sender. Validation consists of checking if the message length and number of
1664 * memory region constituents match, and if the transition is valid for the
1665 * type of memory sending operation.
Andrew Walbran475c1452020-02-07 13:22:22 +00001666 *
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001667 * Assumes that the caller has already found and locked the sender VM and copied
1668 * the memory region descriptor from the sender's TX buffer to a freshly
1669 * allocated page from Hafnium's internal pool. The caller must have also
1670 * validated that the receiver VM ID is valid.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001671 *
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001672 * This function takes ownership of the `memory_region` passed in and will free
1673 * it when necessary; it must not be freed by the caller.
Jose Marinho09b1db82019-08-08 09:16:59 +01001674 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001675struct ffa_value ffa_memory_send(struct vm_locked from_locked,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001676 struct ffa_memory_region *memory_region,
Andrew Walbran130a8ae2020-05-15 16:27:15 +01001677 uint32_t memory_share_length,
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001678 uint32_t fragment_length, uint32_t share_func,
1679 struct mpool *page_pool)
Jose Marinho09b1db82019-08-08 09:16:59 +01001680{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001681 struct ffa_value ret;
Andrew Walbranca808b12020-05-15 17:22:28 +01001682 struct share_states_locked share_states;
1683 struct ffa_memory_share_state *share_state;
Jose Marinho09b1db82019-08-08 09:16:59 +01001684
1685 /*
Andrew Walbrana65a1322020-04-06 19:32:32 +01001686 * If there is an error validating the `memory_region` then we need to
1687 * free it because we own it but we won't be storing it in a share state
1688 * after all.
Jose Marinho09b1db82019-08-08 09:16:59 +01001689 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001690 ret = ffa_memory_send_validate(from_locked, memory_region,
1691 memory_share_length, fragment_length,
J-Alves363f5722022-04-25 17:37:37 +01001692 share_func);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001693 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001694 mpool_free(page_pool, memory_region);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001695 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01001696 }
1697
Andrew Walbrana65a1322020-04-06 19:32:32 +01001698 /* Set flag for share function, ready to be retrieved later. */
1699 switch (share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001700 case FFA_MEM_SHARE_32:
Andrew Walbrana65a1322020-04-06 19:32:32 +01001701 memory_region->flags |=
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001702 FFA_MEMORY_REGION_TRANSACTION_TYPE_SHARE;
Andrew Walbrana65a1322020-04-06 19:32:32 +01001703 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001704 case FFA_MEM_LEND_32:
1705 memory_region->flags |= FFA_MEMORY_REGION_TRANSACTION_TYPE_LEND;
Andrew Walbrana65a1322020-04-06 19:32:32 +01001706 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001707 case FFA_MEM_DONATE_32:
Andrew Walbrana65a1322020-04-06 19:32:32 +01001708 memory_region->flags |=
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001709 FFA_MEMORY_REGION_TRANSACTION_TYPE_DONATE;
Andrew Walbrana65a1322020-04-06 19:32:32 +01001710 break;
Jose Marinho09b1db82019-08-08 09:16:59 +01001711 }
1712
Andrew Walbranca808b12020-05-15 17:22:28 +01001713 share_states = share_states_lock();
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001714 /*
1715 * Allocate a share state before updating the page table. Otherwise if
1716 * updating the page table succeeded but allocating the share state
1717 * failed then it would leave the memory in a state where nobody could
1718 * get it back.
1719 */
Karl Meakin52cdfe72023-06-30 14:49:10 +01001720 share_state = allocate_share_state(share_states, share_func,
1721 memory_region, fragment_length,
1722 FFA_MEMORY_HANDLE_INVALID);
1723 if (!share_state) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001724 dlog_verbose("Failed to allocate share state.\n");
1725 mpool_free(page_pool, memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +01001726 ret = ffa_error(FFA_NO_MEMORY);
1727 goto out;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001728 }
1729
Andrew Walbranca808b12020-05-15 17:22:28 +01001730 if (fragment_length == memory_share_length) {
1731 /* No more fragments to come, everything fit in one message. */
J-Alves2a0d2882020-10-29 14:49:50 +00001732 ret = ffa_memory_send_complete(
1733 from_locked, share_states, share_state, page_pool,
1734 &(share_state->sender_orig_mode));
Andrew Walbranca808b12020-05-15 17:22:28 +01001735 } else {
J-Alvesfdd29272022-07-19 13:16:31 +01001736 /*
1737 * Use sender ID from 'memory_region' assuming
1738 * that at this point it has been validated:
1739 * - MBZ at virtual FF-A instance.
1740 */
J-Alves19e20cf2023-08-02 12:48:55 +01001741 ffa_id_t sender_to_ret =
J-Alvesfdd29272022-07-19 13:16:31 +01001742 (from_locked.vm->id == HF_OTHER_WORLD_ID)
1743 ? memory_region->sender
1744 : 0;
Andrew Walbranca808b12020-05-15 17:22:28 +01001745 ret = (struct ffa_value){
1746 .func = FFA_MEM_FRAG_RX_32,
J-Alvesee68c542020-10-29 17:48:20 +00001747 .arg1 = (uint32_t)memory_region->handle,
1748 .arg2 = (uint32_t)(memory_region->handle >> 32),
J-Alvesfdd29272022-07-19 13:16:31 +01001749 .arg3 = fragment_length,
1750 .arg4 = (uint32_t)(sender_to_ret & 0xffff) << 16};
Andrew Walbranca808b12020-05-15 17:22:28 +01001751 }
1752
1753out:
1754 share_states_unlock(&share_states);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001755 dump_share_states();
Andrew Walbranca808b12020-05-15 17:22:28 +01001756 return ret;
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001757}
1758
1759/**
J-Alves8505a8a2022-06-15 18:10:18 +01001760 * Continues an operation to donate, lend or share memory to a VM from current
1761 * world. If this is the last fragment then checks that the transition is valid
1762 * for the type of memory sending operation and updates the stage-2 page tables
1763 * of the sender.
Andrew Walbranca808b12020-05-15 17:22:28 +01001764 *
1765 * Assumes that the caller has already found and locked the sender VM and copied
1766 * the memory region descriptor from the sender's TX buffer to a freshly
1767 * allocated page from Hafnium's internal pool.
1768 *
1769 * This function takes ownership of the `fragment` passed in; it must not be
1770 * freed by the caller.
1771 */
1772struct ffa_value ffa_memory_send_continue(struct vm_locked from_locked,
1773 void *fragment,
1774 uint32_t fragment_length,
1775 ffa_memory_handle_t handle,
1776 struct mpool *page_pool)
1777{
1778 struct share_states_locked share_states = share_states_lock();
1779 struct ffa_memory_share_state *share_state;
1780 struct ffa_value ret;
1781 struct ffa_memory_region *memory_region;
1782
Demi Marie Obenour73a1e942023-02-04 14:09:18 -05001783 CHECK(is_aligned(fragment,
1784 alignof(struct ffa_memory_region_constituent)));
1785 if (fragment_length % sizeof(struct ffa_memory_region_constituent) !=
1786 0) {
1787 dlog_verbose("Fragment length %u misaligned.\n",
1788 fragment_length);
1789 ret = ffa_error(FFA_INVALID_PARAMETERS);
1790 goto out_free_fragment;
1791 }
1792
Andrew Walbranca808b12020-05-15 17:22:28 +01001793 ret = ffa_memory_send_continue_validate(share_states, handle,
1794 &share_state,
1795 from_locked.vm->id, page_pool);
1796 if (ret.func != FFA_SUCCESS_32) {
1797 goto out_free_fragment;
1798 }
1799 memory_region = share_state->memory_region;
1800
J-Alves95df0ef2022-12-07 10:09:48 +00001801 if (memory_region_receivers_from_other_world(memory_region)) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001802 dlog_error(
1803 "Got hypervisor-allocated handle for memory send to "
J-Alves8505a8a2022-06-15 18:10:18 +01001804 "other world. This should never happen, and indicates "
1805 "a bug in "
Andrew Walbranca808b12020-05-15 17:22:28 +01001806 "EL3 code.\n");
1807 ret = ffa_error(FFA_INVALID_PARAMETERS);
1808 goto out_free_fragment;
1809 }
1810
1811 /* Add this fragment. */
1812 share_state->fragments[share_state->fragment_count] = fragment;
1813 share_state->fragment_constituent_counts[share_state->fragment_count] =
1814 fragment_length / sizeof(struct ffa_memory_region_constituent);
1815 share_state->fragment_count++;
1816
1817 /* Check whether the memory send operation is now ready to complete. */
1818 if (share_state_sending_complete(share_states, share_state)) {
J-Alves2a0d2882020-10-29 14:49:50 +00001819 ret = ffa_memory_send_complete(
1820 from_locked, share_states, share_state, page_pool,
1821 &(share_state->sender_orig_mode));
Andrew Walbranca808b12020-05-15 17:22:28 +01001822 } else {
1823 ret = (struct ffa_value){
1824 .func = FFA_MEM_FRAG_RX_32,
1825 .arg1 = (uint32_t)handle,
1826 .arg2 = (uint32_t)(handle >> 32),
1827 .arg3 = share_state_next_fragment_offset(share_states,
1828 share_state)};
1829 }
1830 goto out;
1831
1832out_free_fragment:
1833 mpool_free(page_pool, fragment);
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001834
1835out:
Andrew Walbranca808b12020-05-15 17:22:28 +01001836 share_states_unlock(&share_states);
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001837 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001838}
1839
Andrew Walbranca808b12020-05-15 17:22:28 +01001840/** Clean up after the receiver has finished retrieving a memory region. */
1841static void ffa_memory_retrieve_complete(
1842 struct share_states_locked share_states,
1843 struct ffa_memory_share_state *share_state, struct mpool *page_pool)
1844{
1845 if (share_state->share_func == FFA_MEM_DONATE_32) {
1846 /*
1847 * Memory that has been donated can't be relinquished,
1848 * so no need to keep the share state around.
1849 */
1850 share_state_free(share_states, share_state, page_pool);
1851 dlog_verbose("Freed share state for donate.\n");
1852 }
1853}
1854
J-Alves2d8457f2022-10-05 11:06:41 +01001855/**
1856 * Initialises the given memory region descriptor to be used for an
1857 * `FFA_MEM_RETRIEVE_RESP`, including the given constituents for the first
1858 * fragment.
1859 * The memory region descriptor is initialized according to retriever's
1860 * FF-A version.
1861 *
1862 * Returns true on success, or false if the given constituents won't all fit in
1863 * the first fragment.
1864 */
1865static bool ffa_retrieved_memory_region_init(
1866 void *response, uint32_t ffa_version, size_t response_max_size,
J-Alves19e20cf2023-08-02 12:48:55 +01001867 ffa_id_t sender, ffa_memory_attributes_t attributes,
J-Alves2d8457f2022-10-05 11:06:41 +01001868 ffa_memory_region_flags_t flags, ffa_memory_handle_t handle,
J-Alves19e20cf2023-08-02 12:48:55 +01001869 ffa_id_t receiver_id, ffa_memory_access_permissions_t permissions,
J-Alves2d8457f2022-10-05 11:06:41 +01001870 uint32_t page_count, uint32_t total_constituent_count,
1871 const struct ffa_memory_region_constituent constituents[],
1872 uint32_t fragment_constituent_count, uint32_t *total_length,
1873 uint32_t *fragment_length)
1874{
1875 struct ffa_composite_memory_region *composite_memory_region;
1876 struct ffa_memory_access *receiver;
1877 uint32_t i;
1878 uint32_t constituents_offset;
1879 uint32_t receiver_count;
1880
1881 assert(response != NULL);
1882
1883 if (ffa_version == MAKE_FFA_VERSION(1, 0)) {
1884 struct ffa_memory_region_v1_0 *retrieve_response =
1885 (struct ffa_memory_region_v1_0 *)response;
1886
J-Alves5da37d92022-10-24 16:33:48 +01001887 ffa_memory_region_init_header_v1_0(
1888 retrieve_response, sender, attributes, flags, handle, 0,
1889 RECEIVERS_COUNT_IN_RETRIEVE_RESP);
J-Alves2d8457f2022-10-05 11:06:41 +01001890
1891 receiver = &retrieve_response->receivers[0];
1892 receiver_count = retrieve_response->receiver_count;
1893
1894 receiver->composite_memory_region_offset =
1895 sizeof(struct ffa_memory_region_v1_0) +
1896 receiver_count * sizeof(struct ffa_memory_access);
1897
1898 composite_memory_region = ffa_memory_region_get_composite_v1_0(
1899 retrieve_response, 0);
1900 } else {
1901 /* Default to FF-A v1.1 version. */
1902 struct ffa_memory_region *retrieve_response =
1903 (struct ffa_memory_region *)response;
1904
1905 ffa_memory_region_init_header(retrieve_response, sender,
1906 attributes, flags, handle, 0, 1);
1907
1908 receiver = &retrieve_response->receivers[0];
1909 receiver_count = retrieve_response->receiver_count;
1910
1911 /*
1912 * Note that `sizeof(struct_ffa_memory_region)` and
1913 * `sizeof(struct ffa_memory_access)` must both be multiples of
1914 * 16 (as verified by the asserts in `ffa_memory.c`, so it is
1915 * guaranteed that the offset we calculate here is aligned to a
1916 * 64-bit boundary and so 64-bit values can be copied without
1917 * alignment faults.
1918 */
1919 receiver->composite_memory_region_offset =
1920 sizeof(struct ffa_memory_region) +
1921 receiver_count * sizeof(struct ffa_memory_access);
1922
1923 composite_memory_region =
1924 ffa_memory_region_get_composite(retrieve_response, 0);
1925 }
1926
1927 assert(receiver != NULL);
1928 assert(composite_memory_region != NULL);
1929
1930 /*
1931 * Initialized here as in memory retrieve responses we currently expect
1932 * one borrower to be specified.
1933 */
1934 ffa_memory_access_init_permissions(receiver, receiver_id, 0, 0, flags);
1935 receiver->receiver_permissions.permissions = permissions;
1936
1937 composite_memory_region->page_count = page_count;
1938 composite_memory_region->constituent_count = total_constituent_count;
1939 composite_memory_region->reserved_0 = 0;
1940
1941 constituents_offset = receiver->composite_memory_region_offset +
1942 sizeof(struct ffa_composite_memory_region);
1943 if (constituents_offset +
1944 fragment_constituent_count *
1945 sizeof(struct ffa_memory_region_constituent) >
1946 response_max_size) {
1947 return false;
1948 }
1949
1950 for (i = 0; i < fragment_constituent_count; ++i) {
1951 composite_memory_region->constituents[i] = constituents[i];
1952 }
1953
1954 if (total_length != NULL) {
1955 *total_length =
1956 constituents_offset +
1957 composite_memory_region->constituent_count *
1958 sizeof(struct ffa_memory_region_constituent);
1959 }
1960 if (fragment_length != NULL) {
1961 *fragment_length =
1962 constituents_offset +
1963 fragment_constituent_count *
1964 sizeof(struct ffa_memory_region_constituent);
1965 }
1966
1967 return true;
1968}
1969
J-Alves96de29f2022-04-26 16:05:24 +01001970/*
1971 * Gets the receiver's access permissions from 'struct ffa_memory_region' and
1972 * returns its index in the receiver's array. If receiver's ID doesn't exist
1973 * in the array, return the region's 'receiver_count'.
1974 */
J-Alvesb5084cf2022-07-06 14:20:12 +01001975uint32_t ffa_memory_region_get_receiver(struct ffa_memory_region *memory_region,
J-Alves19e20cf2023-08-02 12:48:55 +01001976 ffa_id_t receiver)
J-Alves96de29f2022-04-26 16:05:24 +01001977{
1978 struct ffa_memory_access *receivers;
1979 uint32_t i;
1980
1981 assert(memory_region != NULL);
1982
1983 receivers = memory_region->receivers;
1984
1985 for (i = 0U; i < memory_region->receiver_count; i++) {
1986 if (receivers[i].receiver_permissions.receiver == receiver) {
1987 break;
1988 }
1989 }
1990
1991 return i;
1992}
1993
1994/**
1995 * Validates the retrieved permissions against those specified by the lender
1996 * of memory share operation. Optionally can help set the permissions to be used
1997 * for the S2 mapping, through the `permissions` argument.
J-Alvesdcad8992023-09-15 14:10:35 +01001998 * Returns FFA_SUCCESS if all the fields are valid. FFA_ERROR, with error code:
1999 * - FFA_INVALID_PARAMETERS -> if the fields have invalid values as per the
2000 * specification for each ABI.
2001 * - FFA_DENIED -> if the permissions specified by the retriever are not
2002 * less permissive than those provided by the sender.
J-Alves96de29f2022-04-26 16:05:24 +01002003 */
J-Alvesdcad8992023-09-15 14:10:35 +01002004static struct ffa_value ffa_memory_retrieve_is_memory_access_valid(
2005 uint32_t share_func, enum ffa_data_access sent_data_access,
J-Alves96de29f2022-04-26 16:05:24 +01002006 enum ffa_data_access requested_data_access,
2007 enum ffa_instruction_access sent_instruction_access,
2008 enum ffa_instruction_access requested_instruction_access,
J-Alvesdcad8992023-09-15 14:10:35 +01002009 ffa_memory_access_permissions_t *permissions, bool multiple_borrowers)
J-Alves96de29f2022-04-26 16:05:24 +01002010{
2011 switch (sent_data_access) {
2012 case FFA_DATA_ACCESS_NOT_SPECIFIED:
2013 case FFA_DATA_ACCESS_RW:
2014 if (requested_data_access == FFA_DATA_ACCESS_NOT_SPECIFIED ||
2015 requested_data_access == FFA_DATA_ACCESS_RW) {
2016 if (permissions != NULL) {
2017 ffa_set_data_access_attr(permissions,
2018 FFA_DATA_ACCESS_RW);
2019 }
2020 break;
2021 }
2022 /* Intentional fall-through. */
2023 case FFA_DATA_ACCESS_RO:
2024 if (requested_data_access == FFA_DATA_ACCESS_NOT_SPECIFIED ||
2025 requested_data_access == FFA_DATA_ACCESS_RO) {
2026 if (permissions != NULL) {
2027 ffa_set_data_access_attr(permissions,
2028 FFA_DATA_ACCESS_RO);
2029 }
2030 break;
2031 }
2032 dlog_verbose(
2033 "Invalid data access requested; sender specified "
2034 "permissions %#x but receiver requested %#x.\n",
2035 sent_data_access, requested_data_access);
J-Alvesdcad8992023-09-15 14:10:35 +01002036 return ffa_error(FFA_DENIED);
J-Alves96de29f2022-04-26 16:05:24 +01002037 case FFA_DATA_ACCESS_RESERVED:
2038 panic("Got unexpected FFA_DATA_ACCESS_RESERVED. Should be "
2039 "checked before this point.");
2040 }
2041
J-Alvesdcad8992023-09-15 14:10:35 +01002042 /*
2043 * For operations with a single borrower, If it is an FFA_MEMORY_LEND
2044 * or FFA_MEMORY_DONATE the retriever should have specifed the
2045 * instruction permissions it wishes to receive.
2046 */
2047 switch (share_func) {
2048 case FFA_MEM_SHARE_32:
2049 if (requested_instruction_access !=
2050 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
2051 dlog_verbose(
2052 "%s: for share instruction permissions must "
2053 "NOT be specified.\n",
2054 __func__);
2055 return ffa_error(FFA_INVALID_PARAMETERS);
2056 }
2057 break;
2058 case FFA_MEM_LEND_32:
2059 /*
2060 * For operations with multiple borrowers only permit XN
2061 * permissions, and both Sender and borrower should have used
2062 * FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED.
2063 */
2064 if (multiple_borrowers) {
2065 if (requested_instruction_access !=
2066 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
2067 dlog_verbose(
2068 "%s: lend/share/donate with multiple "
2069 "borrowers "
2070 "instruction permissions must NOT be "
2071 "specified.\n",
2072 __func__);
2073 return ffa_error(FFA_INVALID_PARAMETERS);
2074 }
2075 break;
2076 }
2077 /* Fall through if the operation targets a single borrower. */
2078 case FFA_MEM_DONATE_32:
2079 if (!multiple_borrowers &&
2080 requested_instruction_access ==
2081 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
2082 dlog_verbose(
2083 "%s: for lend/donate with single borrower "
2084 "instruction permissions must be speficified "
2085 "by borrower\n",
2086 __func__);
2087 return ffa_error(FFA_INVALID_PARAMETERS);
2088 }
2089 break;
2090 default:
2091 panic("%s: Wrong func id provided.\n", __func__);
2092 }
2093
J-Alves96de29f2022-04-26 16:05:24 +01002094 switch (sent_instruction_access) {
2095 case FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED:
2096 case FFA_INSTRUCTION_ACCESS_X:
J-Alvesdcad8992023-09-15 14:10:35 +01002097 if (requested_instruction_access == FFA_INSTRUCTION_ACCESS_X) {
J-Alves96de29f2022-04-26 16:05:24 +01002098 if (permissions != NULL) {
2099 ffa_set_instruction_access_attr(
2100 permissions, FFA_INSTRUCTION_ACCESS_X);
2101 }
2102 break;
2103 }
J-Alvesdcad8992023-09-15 14:10:35 +01002104 /*
2105 * Fall through if requested permissions are less
2106 * permissive than those provided by the sender.
2107 */
J-Alves96de29f2022-04-26 16:05:24 +01002108 case FFA_INSTRUCTION_ACCESS_NX:
2109 if (requested_instruction_access ==
2110 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED ||
2111 requested_instruction_access == FFA_INSTRUCTION_ACCESS_NX) {
2112 if (permissions != NULL) {
2113 ffa_set_instruction_access_attr(
2114 permissions, FFA_INSTRUCTION_ACCESS_NX);
2115 }
2116 break;
2117 }
2118 dlog_verbose(
2119 "Invalid instruction access requested; sender "
2120 "specified permissions %#x but receiver requested "
2121 "%#x.\n",
2122 sent_instruction_access, requested_instruction_access);
J-Alvesdcad8992023-09-15 14:10:35 +01002123 return ffa_error(FFA_DENIED);
J-Alves96de29f2022-04-26 16:05:24 +01002124 case FFA_INSTRUCTION_ACCESS_RESERVED:
2125 panic("Got unexpected FFA_INSTRUCTION_ACCESS_RESERVED. Should "
2126 "be checked before this point.");
2127 }
2128
J-Alvesdcad8992023-09-15 14:10:35 +01002129 return (struct ffa_value){.func = FFA_SUCCESS_32};
J-Alves96de29f2022-04-26 16:05:24 +01002130}
2131
2132/**
2133 * Validate the receivers' permissions in the retrieve request against those
2134 * specified by the lender.
2135 * In the `permissions` argument returns the permissions to set at S2 for the
2136 * caller to the FFA_MEMORY_RETRIEVE_REQ.
J-Alves3456e032023-07-20 12:20:05 +01002137 * The function looks into the flag to bypass multiple borrower checks:
2138 * - If not set returns FFA_SUCCESS if all specified permissions are valid.
2139 * - If set returns FFA_SUCCESS if the descriptor contains the permissions
2140 * to the caller of FFA_MEM_RETRIEVE_REQ and they are valid. Other permissions
2141 * are ignored, if provided.
J-Alves96de29f2022-04-26 16:05:24 +01002142 */
2143static struct ffa_value ffa_memory_retrieve_validate_memory_access_list(
2144 struct ffa_memory_region *memory_region,
J-Alves19e20cf2023-08-02 12:48:55 +01002145 struct ffa_memory_region *retrieve_request, ffa_id_t to_vm_id,
J-Alvesdcad8992023-09-15 14:10:35 +01002146 ffa_memory_access_permissions_t *permissions, uint32_t func_id)
J-Alves96de29f2022-04-26 16:05:24 +01002147{
2148 uint32_t retrieve_receiver_index;
J-Alves3456e032023-07-20 12:20:05 +01002149 bool bypass_multi_receiver_check =
2150 (retrieve_request->flags &
2151 FFA_MEMORY_REGION_FLAG_BYPASS_BORROWERS_CHECK) != 0U;
J-Alvesdcad8992023-09-15 14:10:35 +01002152 const uint32_t region_receiver_count = memory_region->receiver_count;
2153 struct ffa_value ret;
J-Alves96de29f2022-04-26 16:05:24 +01002154
2155 assert(permissions != NULL);
2156
J-Alves3456e032023-07-20 12:20:05 +01002157 if (!bypass_multi_receiver_check) {
J-Alvesdcad8992023-09-15 14:10:35 +01002158 if (retrieve_request->receiver_count != region_receiver_count) {
J-Alves3456e032023-07-20 12:20:05 +01002159 dlog_verbose(
2160 "Retrieve request should contain same list of "
2161 "borrowers, as specified by the lender.\n");
2162 return ffa_error(FFA_INVALID_PARAMETERS);
2163 }
2164 } else {
2165 if (retrieve_request->receiver_count != 1) {
2166 dlog_verbose(
2167 "Set bypass multiple borrower check, receiver "
2168 "list must be sized 1 (%x)\n",
2169 memory_region->receiver_count);
2170 return ffa_error(FFA_INVALID_PARAMETERS);
2171 }
J-Alves96de29f2022-04-26 16:05:24 +01002172 }
2173
2174 retrieve_receiver_index = retrieve_request->receiver_count;
2175
2176 /* Should be populated with the permissions of the retriever. */
2177 *permissions = 0;
2178
2179 for (uint32_t i = 0U; i < retrieve_request->receiver_count; i++) {
2180 ffa_memory_access_permissions_t sent_permissions;
2181 struct ffa_memory_access *current_receiver =
2182 &retrieve_request->receivers[i];
2183 ffa_memory_access_permissions_t requested_permissions =
2184 current_receiver->receiver_permissions.permissions;
J-Alves19e20cf2023-08-02 12:48:55 +01002185 ffa_id_t current_receiver_id =
J-Alves96de29f2022-04-26 16:05:24 +01002186 current_receiver->receiver_permissions.receiver;
2187 bool found_to_id = current_receiver_id == to_vm_id;
2188
J-Alves3456e032023-07-20 12:20:05 +01002189 if (bypass_multi_receiver_check && !found_to_id) {
2190 dlog_verbose(
2191 "Bypass multiple borrower check for id %x.\n",
2192 current_receiver_id);
2193 continue;
2194 }
2195
J-Alves96de29f2022-04-26 16:05:24 +01002196 /*
2197 * Find the current receiver in the transaction descriptor from
2198 * sender.
2199 */
2200 uint32_t mem_region_receiver_index =
2201 ffa_memory_region_get_receiver(memory_region,
2202 current_receiver_id);
2203
2204 if (mem_region_receiver_index ==
2205 memory_region->receiver_count) {
2206 dlog_verbose("%s: receiver %x not found\n", __func__,
2207 current_receiver_id);
2208 return ffa_error(FFA_DENIED);
2209 }
2210
2211 sent_permissions =
2212 memory_region->receivers[mem_region_receiver_index]
2213 .receiver_permissions.permissions;
2214
2215 if (found_to_id) {
2216 retrieve_receiver_index = i;
2217 }
2218
2219 /*
2220 * Since we are traversing the list of receivers, save the index
2221 * of the caller. As it needs to be there.
2222 */
2223
2224 if (current_receiver->composite_memory_region_offset != 0U) {
2225 dlog_verbose(
2226 "Retriever specified address ranges not "
2227 "supported (got offset %d).\n",
2228 current_receiver
2229 ->composite_memory_region_offset);
2230 return ffa_error(FFA_INVALID_PARAMETERS);
2231 }
2232
2233 /*
J-Alvesdcad8992023-09-15 14:10:35 +01002234 * Check if retrieve request memory access list is valid:
2235 * - The retrieve request complies with the specification.
2236 * - Permissions are within those specified by the sender.
J-Alves96de29f2022-04-26 16:05:24 +01002237 */
J-Alvesdcad8992023-09-15 14:10:35 +01002238 ret = ffa_memory_retrieve_is_memory_access_valid(
2239 func_id, ffa_get_data_access_attr(sent_permissions),
2240 ffa_get_data_access_attr(requested_permissions),
2241 ffa_get_instruction_access_attr(sent_permissions),
2242 ffa_get_instruction_access_attr(requested_permissions),
2243 found_to_id ? permissions : NULL,
2244 region_receiver_count > 1);
2245 if (ret.func != FFA_SUCCESS_32) {
2246 return ret;
J-Alves96de29f2022-04-26 16:05:24 +01002247 }
2248
2249 /*
2250 * Can't request PM to clear memory if only provided with RO
2251 * permissions.
2252 */
2253 if (found_to_id &&
2254 (ffa_get_data_access_attr(*permissions) ==
2255 FFA_DATA_ACCESS_RO) &&
2256 (retrieve_request->flags & FFA_MEMORY_REGION_FLAG_CLEAR) !=
2257 0U) {
2258 dlog_verbose(
2259 "Receiver has RO permissions can not request "
2260 "clear.\n");
2261 return ffa_error(FFA_DENIED);
2262 }
2263 }
2264
2265 if (retrieve_receiver_index == retrieve_request->receiver_count) {
2266 dlog_verbose(
2267 "Retrieve request does not contain caller's (%x) "
2268 "permissions\n",
2269 to_vm_id);
2270 return ffa_error(FFA_INVALID_PARAMETERS);
2271 }
2272
2273 return (struct ffa_value){.func = FFA_SUCCESS_32};
2274}
2275
J-Alvesa9cd7e32022-07-01 13:49:33 +01002276/*
2277 * According to section 16.4.3 of FF-A v1.1 EAC0 specification, the hypervisor
2278 * may issue an FFA_MEM_RETRIEVE_REQ to obtain the memory region description
2279 * of a pending memory sharing operation whose allocator is the SPM, for
2280 * validation purposes before forwarding an FFA_MEM_RECLAIM call. In doing so
2281 * the memory region descriptor of the retrieve request must be zeroed with the
2282 * exception of the sender ID and handle.
2283 */
2284bool is_ffa_memory_retrieve_borrower_request(struct ffa_memory_region *request,
2285 struct vm_locked to_locked)
2286{
2287 return to_locked.vm->id == HF_HYPERVISOR_VM_ID &&
2288 request->attributes == 0U && request->flags == 0U &&
2289 request->tag == 0U && request->receiver_count == 0U &&
2290 plat_ffa_memory_handle_allocated_by_current_world(
2291 request->handle);
2292}
2293
2294/*
2295 * Helper to reset count of fragments retrieved by the hypervisor.
2296 */
2297static void ffa_memory_retrieve_complete_from_hyp(
2298 struct ffa_memory_share_state *share_state)
2299{
2300 if (share_state->hypervisor_fragment_count ==
2301 share_state->fragment_count) {
2302 share_state->hypervisor_fragment_count = 0;
2303 }
2304}
2305
J-Alves089004f2022-07-13 14:25:44 +01002306/**
2307 * Validate that the memory region descriptor provided by the borrower on
2308 * FFA_MEM_RETRIEVE_REQ, against saved memory region provided by lender at the
2309 * memory sharing call.
2310 */
2311static struct ffa_value ffa_memory_retrieve_validate(
J-Alves19e20cf2023-08-02 12:48:55 +01002312 ffa_id_t receiver_id, struct ffa_memory_region *retrieve_request,
J-Alves089004f2022-07-13 14:25:44 +01002313 struct ffa_memory_region *memory_region, uint32_t *receiver_index,
2314 uint32_t share_func)
2315{
2316 ffa_memory_region_flags_t transaction_type =
2317 retrieve_request->flags &
2318 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK;
Olivier Deprez4342a3c2022-02-28 09:37:25 +01002319 enum ffa_memory_security security_state;
J-Alves089004f2022-07-13 14:25:44 +01002320
2321 assert(retrieve_request != NULL);
2322 assert(memory_region != NULL);
2323 assert(receiver_index != NULL);
2324 assert(retrieve_request->sender == memory_region->sender);
2325
2326 /*
2327 * Check that the transaction type expected by the receiver is
2328 * correct, if it has been specified.
2329 */
2330 if (transaction_type !=
2331 FFA_MEMORY_REGION_TRANSACTION_TYPE_UNSPECIFIED &&
2332 transaction_type != (memory_region->flags &
2333 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK)) {
2334 dlog_verbose(
2335 "Incorrect transaction type %#x for "
2336 "FFA_MEM_RETRIEVE_REQ, expected %#x for handle %#x.\n",
2337 transaction_type,
2338 memory_region->flags &
2339 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK,
2340 retrieve_request->handle);
2341 return ffa_error(FFA_INVALID_PARAMETERS);
2342 }
2343
2344 if (retrieve_request->tag != memory_region->tag) {
2345 dlog_verbose(
2346 "Incorrect tag %d for FFA_MEM_RETRIEVE_REQ, expected "
2347 "%d for handle %#x.\n",
2348 retrieve_request->tag, memory_region->tag,
2349 retrieve_request->handle);
2350 return ffa_error(FFA_INVALID_PARAMETERS);
2351 }
2352
2353 *receiver_index =
2354 ffa_memory_region_get_receiver(memory_region, receiver_id);
2355
2356 if (*receiver_index == memory_region->receiver_count) {
2357 dlog_verbose(
2358 "Incorrect receiver VM ID %d for "
2359 "FFA_MEM_RETRIEVE_REQ, for handle %#x.\n",
J-Alves59ed0042022-07-28 18:26:41 +01002360 receiver_id, memory_region->handle);
J-Alves089004f2022-07-13 14:25:44 +01002361 return ffa_error(FFA_INVALID_PARAMETERS);
2362 }
2363
2364 if ((retrieve_request->flags &
2365 FFA_MEMORY_REGION_ADDRESS_RANGE_HINT_VALID) != 0U) {
2366 dlog_verbose(
2367 "Retriever specified 'address range alignment 'hint' "
2368 "not supported.\n");
2369 return ffa_error(FFA_INVALID_PARAMETERS);
2370 }
2371 if ((retrieve_request->flags &
2372 FFA_MEMORY_REGION_ADDRESS_RANGE_HINT_MASK) != 0) {
2373 dlog_verbose(
2374 "Bits 8-5 must be zero in memory region's flags "
2375 "(address range alignment hint not supported).\n");
2376 return ffa_error(FFA_INVALID_PARAMETERS);
2377 }
2378
2379 if ((retrieve_request->flags & ~0x7FF) != 0U) {
2380 dlog_verbose(
2381 "Bits 31-10 must be zero in memory region's flags.\n");
2382 return ffa_error(FFA_INVALID_PARAMETERS);
2383 }
2384
2385 if (share_func == FFA_MEM_SHARE_32 &&
2386 (retrieve_request->flags &
2387 (FFA_MEMORY_REGION_FLAG_CLEAR |
2388 FFA_MEMORY_REGION_FLAG_CLEAR_RELINQUISH)) != 0U) {
2389 dlog_verbose(
2390 "Memory Share operation can't clean after relinquish "
2391 "memory region.\n");
2392 return ffa_error(FFA_INVALID_PARAMETERS);
2393 }
2394
2395 /*
2396 * If the borrower needs the memory to be cleared before mapping
2397 * to its address space, the sender should have set the flag
2398 * when calling FFA_MEM_LEND/FFA_MEM_DONATE, else return
2399 * FFA_DENIED.
2400 */
2401 if ((retrieve_request->flags & FFA_MEMORY_REGION_FLAG_CLEAR) != 0U &&
2402 (memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR) == 0U) {
2403 dlog_verbose(
2404 "Borrower needs memory cleared. Sender needs to set "
2405 "flag for clearing memory.\n");
2406 return ffa_error(FFA_DENIED);
2407 }
2408
Olivier Deprez4342a3c2022-02-28 09:37:25 +01002409 /* Memory region attributes NS-Bit MBZ for FFA_MEM_RETRIEVE_REQ. */
2410 security_state =
2411 ffa_get_memory_security_attr(retrieve_request->attributes);
2412 if (security_state != FFA_MEMORY_SECURITY_UNSPECIFIED) {
2413 dlog_verbose(
2414 "Invalid security state for memory retrieve request "
2415 "operation.\n");
2416 return ffa_error(FFA_INVALID_PARAMETERS);
2417 }
2418
J-Alves089004f2022-07-13 14:25:44 +01002419 /*
2420 * If memory type is not specified, bypass validation of memory
2421 * attributes in the retrieve request. The retriever is expecting to
2422 * obtain this information from the SPMC.
2423 */
2424 if (ffa_get_memory_type_attr(retrieve_request->attributes) ==
2425 FFA_MEMORY_NOT_SPECIFIED_MEM) {
2426 return (struct ffa_value){.func = FFA_SUCCESS_32};
2427 }
2428
2429 /*
2430 * Ensure receiver's attributes are compatible with how
2431 * Hafnium maps memory: Normal Memory, Inner shareable,
2432 * Write-Back Read-Allocate Write-Allocate Cacheable.
2433 */
2434 return ffa_memory_attributes_validate(retrieve_request->attributes);
2435}
2436
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002437struct ffa_value ffa_memory_retrieve(struct vm_locked to_locked,
2438 struct ffa_memory_region *retrieve_request,
Andrew Walbran130a8ae2020-05-15 16:27:15 +01002439 uint32_t retrieve_request_length,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002440 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002441{
Andrew Walbran130a8ae2020-05-15 16:27:15 +01002442 uint32_t expected_retrieve_request_length =
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002443 sizeof(struct ffa_memory_region) +
Andrew Walbrana65a1322020-04-06 19:32:32 +01002444 retrieve_request->receiver_count *
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002445 sizeof(struct ffa_memory_access);
2446 ffa_memory_handle_t handle = retrieve_request->handle;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002447 struct ffa_memory_region *memory_region;
J-Alvesa9cd7e32022-07-01 13:49:33 +01002448 ffa_memory_access_permissions_t permissions = 0;
Olivier Deprez878bd5b2021-04-15 19:05:10 +02002449 uint32_t memory_to_mode;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002450 struct share_states_locked share_states;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002451 struct ffa_memory_share_state *share_state;
2452 struct ffa_value ret;
Andrew Walbranca808b12020-05-15 17:22:28 +01002453 struct ffa_composite_memory_region *composite;
2454 uint32_t total_length;
2455 uint32_t fragment_length;
J-Alves19e20cf2023-08-02 12:48:55 +01002456 ffa_id_t receiver_id = to_locked.vm->id;
J-Alvesa9cd7e32022-07-01 13:49:33 +01002457 bool is_send_complete = false;
Olivier Deprez878bd5b2021-04-15 19:05:10 +02002458 ffa_memory_attributes_t attributes;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002459
2460 dump_share_states();
2461
Andrew Walbran130a8ae2020-05-15 16:27:15 +01002462 if (retrieve_request_length != expected_retrieve_request_length) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002463 dlog_verbose(
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002464 "Invalid length for FFA_MEM_RETRIEVE_REQ, expected %d "
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002465 "but was %d.\n",
Andrew Walbran130a8ae2020-05-15 16:27:15 +01002466 expected_retrieve_request_length,
2467 retrieve_request_length);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002468 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002469 }
2470
2471 share_states = share_states_lock();
Karl Meakin4a2854a2023-06-30 16:26:52 +01002472 share_state = get_share_state(share_states, handle);
2473 if (!share_state) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002474 dlog_verbose("Invalid handle %#x for FFA_MEM_RETRIEVE_REQ.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002475 handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002476 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002477 goto out;
2478 }
2479
J-Alves96de29f2022-04-26 16:05:24 +01002480 if (!share_state->sending_complete) {
2481 dlog_verbose(
2482 "Memory with handle %#x not fully sent, can't "
2483 "retrieve.\n",
2484 handle);
2485 ret = ffa_error(FFA_INVALID_PARAMETERS);
2486 goto out;
2487 }
2488
Andrew Walbrana65a1322020-04-06 19:32:32 +01002489 memory_region = share_state->memory_region;
J-Alves089004f2022-07-13 14:25:44 +01002490
Andrew Walbrana65a1322020-04-06 19:32:32 +01002491 CHECK(memory_region != NULL);
2492
J-Alves089004f2022-07-13 14:25:44 +01002493 if (retrieve_request->sender != memory_region->sender) {
2494 dlog_verbose(
2495 "Memory with handle %#x not fully sent, can't "
2496 "retrieve.\n",
2497 handle);
2498 ret = ffa_error(FFA_INVALID_PARAMETERS);
2499 goto out;
2500 }
J-Alves96de29f2022-04-26 16:05:24 +01002501
J-Alvesa9cd7e32022-07-01 13:49:33 +01002502 if (!is_ffa_memory_retrieve_borrower_request(retrieve_request,
2503 to_locked)) {
2504 uint32_t receiver_index;
J-Alvesa9cd7e32022-07-01 13:49:33 +01002505
J-Alvesb5084cf2022-07-06 14:20:12 +01002506 /*
2507 * The SPMC can only process retrieve requests to memory share
2508 * operations with one borrower from the other world. It can't
2509 * determine the ID of the NWd VM that invoked the retrieve
2510 * request interface call. It relies on the hypervisor to
2511 * validate the caller's ID against that provided in the
2512 * `receivers` list of the retrieve response.
2513 * In case there is only one borrower from the NWd in the
2514 * transaction descriptor, record that in the `receiver_id` for
2515 * later use, and validate in the retrieve request message.
J-Alves3fa82aa2023-09-20 18:19:21 +01002516 * This limitation is due to the fact SPMC can't determine the
2517 * index in the memory share structures state to update.
J-Alvesb5084cf2022-07-06 14:20:12 +01002518 */
2519 if (to_locked.vm->id == HF_HYPERVISOR_VM_ID) {
2520 uint32_t other_world_count = 0;
2521
2522 for (uint32_t i = 0; i < memory_region->receiver_count;
2523 i++) {
2524 receiver_id =
2525 retrieve_request->receivers[0]
2526 .receiver_permissions.receiver;
2527 if (!vm_id_is_current_world(receiver_id)) {
2528 other_world_count++;
2529 }
2530 }
2531 if (other_world_count > 1) {
2532 dlog_verbose(
2533 "Support one receiver from the other "
2534 "world.\n");
2535 return ffa_error(FFA_NOT_SUPPORTED);
2536 }
2537 }
2538
2539 /*
2540 * Validate retrieve request, according to what was sent by the
2541 * sender. Function will output the `receiver_index` from the
J-Alves3fa82aa2023-09-20 18:19:21 +01002542 * provided memory region.
J-Alvesb5084cf2022-07-06 14:20:12 +01002543 */
J-Alves089004f2022-07-13 14:25:44 +01002544 ret = ffa_memory_retrieve_validate(
2545 receiver_id, retrieve_request, memory_region,
2546 &receiver_index, share_state->share_func);
2547 if (ret.func != FFA_SUCCESS_32) {
J-Alvesa9cd7e32022-07-01 13:49:33 +01002548 goto out;
2549 }
2550
2551 if (share_state->retrieved_fragment_count[receiver_index] !=
2552 0U) {
2553 dlog_verbose(
2554 "Memory with handle %#x already retrieved.\n",
2555 handle);
2556 ret = ffa_error(FFA_DENIED);
2557 goto out;
2558 }
2559
J-Alves3fa82aa2023-09-20 18:19:21 +01002560 /*
2561 * Validate the requested permissions against the sent
2562 * permissions.
2563 * Outputs the permissions to give to retriever at S2
2564 * PTs.
2565 */
J-Alvesa9cd7e32022-07-01 13:49:33 +01002566 ret = ffa_memory_retrieve_validate_memory_access_list(
2567 memory_region, retrieve_request, receiver_id,
J-Alvesdcad8992023-09-15 14:10:35 +01002568 &permissions, share_state->share_func);
J-Alves614d9f42022-06-28 14:03:10 +01002569 if (ret.func != FFA_SUCCESS_32) {
2570 goto out;
2571 }
Federico Recanatia98603a2021-12-20 18:04:03 +01002572
Olivier Deprez878bd5b2021-04-15 19:05:10 +02002573 memory_to_mode = ffa_memory_permissions_to_mode(
J-Alvesa9cd7e32022-07-01 13:49:33 +01002574 permissions, share_state->sender_orig_mode);
J-Alves40e260e2022-09-22 17:52:43 +01002575
J-Alvesa9cd7e32022-07-01 13:49:33 +01002576 ret = ffa_retrieve_check_update(
J-Alves26483382023-04-20 12:01:49 +01002577 to_locked, share_state->fragments,
J-Alvesa9cd7e32022-07-01 13:49:33 +01002578 share_state->fragment_constituent_counts,
Olivier Deprez878bd5b2021-04-15 19:05:10 +02002579 share_state->fragment_count, memory_to_mode,
J-Alvesa9cd7e32022-07-01 13:49:33 +01002580 share_state->share_func, false, page_pool);
2581
2582 if (ret.func != FFA_SUCCESS_32) {
2583 goto out;
2584 }
2585
2586 share_state->retrieved_fragment_count[receiver_index] = 1;
2587 is_send_complete =
2588 share_state->retrieved_fragment_count[receiver_index] ==
2589 share_state->fragment_count;
J-Alves3c5b2072022-11-21 12:45:40 +00002590
2591 share_state->clear_after_relinquish =
2592 (retrieve_request->flags &
2593 FFA_MEMORY_REGION_FLAG_CLEAR_RELINQUISH) != 0U;
2594
J-Alvesa9cd7e32022-07-01 13:49:33 +01002595 } else {
2596 if (share_state->hypervisor_fragment_count != 0U) {
2597 dlog_verbose(
J-Alvesb5084cf2022-07-06 14:20:12 +01002598 "Memory with handle %#x already retrieved by "
J-Alvesa9cd7e32022-07-01 13:49:33 +01002599 "the hypervisor.\n",
2600 handle);
2601 ret = ffa_error(FFA_DENIED);
2602 goto out;
2603 }
2604
2605 share_state->hypervisor_fragment_count = 1;
2606
2607 ffa_memory_retrieve_complete_from_hyp(share_state);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002608 }
2609
J-Alvesb5084cf2022-07-06 14:20:12 +01002610 /* VMs acquire the RX buffer from SPMC. */
2611 CHECK(plat_ffa_acquire_receiver_rx(to_locked, &ret));
2612
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002613 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01002614 * Copy response to RX buffer of caller and deliver the message.
2615 * This must be done before the share_state is (possibly) freed.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002616 */
Andrew Walbrana65a1322020-04-06 19:32:32 +01002617 /* TODO: combine attributes from sender and request. */
Andrew Walbranca808b12020-05-15 17:22:28 +01002618 composite = ffa_memory_region_get_composite(memory_region, 0);
2619 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01002620 * Constituents which we received in the first fragment should
2621 * always fit in the first fragment we are sending, because the
2622 * header is the same size in both cases and we have a fixed
2623 * message buffer size. So `ffa_retrieved_memory_region_init`
2624 * should never fail.
Andrew Walbranca808b12020-05-15 17:22:28 +01002625 */
Olivier Deprez878bd5b2021-04-15 19:05:10 +02002626
2627 /*
2628 * Set the security state in the memory retrieve response attributes
2629 * if specified by the target mode.
2630 */
2631 attributes = plat_ffa_memory_security_mode(
2632 memory_region->attributes, share_state->sender_orig_mode);
2633
Andrew Walbranca808b12020-05-15 17:22:28 +01002634 CHECK(ffa_retrieved_memory_region_init(
J-Alves2d8457f2022-10-05 11:06:41 +01002635 to_locked.vm->mailbox.recv, to_locked.vm->ffa_version,
Olivier Deprez878bd5b2021-04-15 19:05:10 +02002636 HF_MAILBOX_SIZE, memory_region->sender, attributes,
2637 memory_region->flags, handle, receiver_id, permissions,
2638 composite->page_count, composite->constituent_count,
2639 share_state->fragments[0],
Andrew Walbranca808b12020-05-15 17:22:28 +01002640 share_state->fragment_constituent_counts[0], &total_length,
2641 &fragment_length));
J-Alvesb5084cf2022-07-06 14:20:12 +01002642
Andrew Walbranca808b12020-05-15 17:22:28 +01002643 to_locked.vm->mailbox.recv_size = fragment_length;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002644 to_locked.vm->mailbox.recv_sender = HF_HYPERVISOR_VM_ID;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002645 to_locked.vm->mailbox.recv_func = FFA_MEM_RETRIEVE_RESP_32;
J-Alvese8c8c2b2022-12-16 15:34:48 +00002646 to_locked.vm->mailbox.state = MAILBOX_STATE_FULL;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002647
J-Alvesa9cd7e32022-07-01 13:49:33 +01002648 if (is_send_complete) {
Andrew Walbranca808b12020-05-15 17:22:28 +01002649 ffa_memory_retrieve_complete(share_states, share_state,
2650 page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002651 }
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002652 ret = (struct ffa_value){.func = FFA_MEM_RETRIEVE_RESP_32,
Andrew Walbranca808b12020-05-15 17:22:28 +01002653 .arg1 = total_length,
2654 .arg2 = fragment_length};
Andrew Walbranca808b12020-05-15 17:22:28 +01002655out:
2656 share_states_unlock(&share_states);
2657 dump_share_states();
2658 return ret;
2659}
2660
J-Alves5da37d92022-10-24 16:33:48 +01002661/**
2662 * Determine expected fragment offset according to the FF-A version of
2663 * the caller.
2664 */
2665static uint32_t ffa_memory_retrieve_expected_offset_per_ffa_version(
2666 struct ffa_memory_region *memory_region,
2667 uint32_t retrieved_constituents_count, uint32_t ffa_version)
2668{
2669 uint32_t expected_fragment_offset;
2670 uint32_t composite_constituents_offset;
2671
2672 if (ffa_version == MAKE_FFA_VERSION(1, 1)) {
2673 /*
2674 * Hafnium operates memory regions in FF-A v1.1 format, so we
2675 * can retrieve the constituents offset from descriptor.
2676 */
2677 composite_constituents_offset =
2678 ffa_composite_constituent_offset(memory_region, 0);
2679 } else if (ffa_version == MAKE_FFA_VERSION(1, 0)) {
2680 /*
2681 * If retriever is FF-A v1.0, determine the composite offset
2682 * as it is expected to have been configured in the
2683 * retrieve response.
2684 */
2685 composite_constituents_offset =
2686 sizeof(struct ffa_memory_region_v1_0) +
2687 RECEIVERS_COUNT_IN_RETRIEVE_RESP *
2688 sizeof(struct ffa_memory_access) +
2689 sizeof(struct ffa_composite_memory_region);
2690 } else {
2691 panic("%s received an invalid FF-A version.\n", __func__);
2692 }
2693
2694 expected_fragment_offset =
2695 composite_constituents_offset +
2696 retrieved_constituents_count *
2697 sizeof(struct ffa_memory_region_constituent) -
2698 sizeof(struct ffa_memory_access) *
2699 (memory_region->receiver_count - 1);
2700
2701 return expected_fragment_offset;
2702}
2703
Andrew Walbranca808b12020-05-15 17:22:28 +01002704struct ffa_value ffa_memory_retrieve_continue(struct vm_locked to_locked,
2705 ffa_memory_handle_t handle,
2706 uint32_t fragment_offset,
J-Alves19e20cf2023-08-02 12:48:55 +01002707 ffa_id_t sender_vm_id,
Andrew Walbranca808b12020-05-15 17:22:28 +01002708 struct mpool *page_pool)
2709{
2710 struct ffa_memory_region *memory_region;
2711 struct share_states_locked share_states;
2712 struct ffa_memory_share_state *share_state;
2713 struct ffa_value ret;
2714 uint32_t fragment_index;
2715 uint32_t retrieved_constituents_count;
2716 uint32_t i;
2717 uint32_t expected_fragment_offset;
2718 uint32_t remaining_constituent_count;
2719 uint32_t fragment_length;
J-Alvesc7484f12022-05-13 12:41:14 +01002720 uint32_t receiver_index;
J-Alves59ed0042022-07-28 18:26:41 +01002721 bool continue_ffa_hyp_mem_retrieve_req;
Andrew Walbranca808b12020-05-15 17:22:28 +01002722
2723 dump_share_states();
2724
2725 share_states = share_states_lock();
Karl Meakin4a2854a2023-06-30 16:26:52 +01002726 share_state = get_share_state(share_states, handle);
2727 if (!share_state) {
Andrew Walbranca808b12020-05-15 17:22:28 +01002728 dlog_verbose("Invalid handle %#x for FFA_MEM_FRAG_RX.\n",
2729 handle);
2730 ret = ffa_error(FFA_INVALID_PARAMETERS);
2731 goto out;
2732 }
2733
2734 memory_region = share_state->memory_region;
2735 CHECK(memory_region != NULL);
2736
Andrew Walbranca808b12020-05-15 17:22:28 +01002737 if (!share_state->sending_complete) {
2738 dlog_verbose(
2739 "Memory with handle %#x not fully sent, can't "
2740 "retrieve.\n",
2741 handle);
2742 ret = ffa_error(FFA_INVALID_PARAMETERS);
2743 goto out;
2744 }
2745
J-Alves59ed0042022-07-28 18:26:41 +01002746 /*
2747 * If retrieve request from the hypervisor has been initiated in the
2748 * given share_state, continue it, else assume it is a continuation of
2749 * retrieve request from a NWd VM.
2750 */
2751 continue_ffa_hyp_mem_retrieve_req =
2752 (to_locked.vm->id == HF_HYPERVISOR_VM_ID) &&
2753 (share_state->hypervisor_fragment_count != 0U) &&
J-Alves661e1b72023-08-02 13:39:40 +01002754 ffa_is_vm_id(sender_vm_id);
Andrew Walbranca808b12020-05-15 17:22:28 +01002755
J-Alves59ed0042022-07-28 18:26:41 +01002756 if (!continue_ffa_hyp_mem_retrieve_req) {
2757 receiver_index = ffa_memory_region_get_receiver(
2758 memory_region, to_locked.vm->id);
2759
2760 if (receiver_index == memory_region->receiver_count) {
2761 dlog_verbose(
2762 "Caller of FFA_MEM_FRAG_RX (%x) is not a "
2763 "borrower to memory sharing transaction (%x)\n",
2764 to_locked.vm->id, handle);
2765 ret = ffa_error(FFA_INVALID_PARAMETERS);
2766 goto out;
2767 }
2768
2769 if (share_state->retrieved_fragment_count[receiver_index] ==
2770 0 ||
2771 share_state->retrieved_fragment_count[receiver_index] >=
2772 share_state->fragment_count) {
2773 dlog_verbose(
2774 "Retrieval of memory with handle %#x not yet "
2775 "started or already completed (%d/%d fragments "
2776 "retrieved).\n",
2777 handle,
2778 share_state->retrieved_fragment_count
2779 [receiver_index],
2780 share_state->fragment_count);
2781 ret = ffa_error(FFA_INVALID_PARAMETERS);
2782 goto out;
2783 }
2784
2785 fragment_index =
2786 share_state->retrieved_fragment_count[receiver_index];
2787 } else {
2788 if (share_state->hypervisor_fragment_count == 0 ||
2789 share_state->hypervisor_fragment_count >=
2790 share_state->fragment_count) {
2791 dlog_verbose(
2792 "Retrieve of memory with handle %x not "
2793 "started from hypervisor.\n",
2794 handle);
2795 ret = ffa_error(FFA_INVALID_PARAMETERS);
2796 goto out;
2797 }
2798
2799 if (memory_region->sender != sender_vm_id) {
2800 dlog_verbose(
2801 "Sender ID (%x) is not as expected for memory "
2802 "handle %x\n",
2803 sender_vm_id, handle);
2804 ret = ffa_error(FFA_INVALID_PARAMETERS);
2805 goto out;
2806 }
2807
2808 fragment_index = share_state->hypervisor_fragment_count;
2809
2810 receiver_index = 0;
2811 }
Andrew Walbranca808b12020-05-15 17:22:28 +01002812
2813 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01002814 * Check that the given fragment offset is correct by counting
2815 * how many constituents were in the fragments previously sent.
Andrew Walbranca808b12020-05-15 17:22:28 +01002816 */
2817 retrieved_constituents_count = 0;
2818 for (i = 0; i < fragment_index; ++i) {
2819 retrieved_constituents_count +=
2820 share_state->fragment_constituent_counts[i];
2821 }
J-Alvesc7484f12022-05-13 12:41:14 +01002822
2823 CHECK(memory_region->receiver_count > 0);
2824
Andrew Walbranca808b12020-05-15 17:22:28 +01002825 expected_fragment_offset =
J-Alves5da37d92022-10-24 16:33:48 +01002826 ffa_memory_retrieve_expected_offset_per_ffa_version(
2827 memory_region, retrieved_constituents_count,
2828 to_locked.vm->ffa_version);
2829
Andrew Walbranca808b12020-05-15 17:22:28 +01002830 if (fragment_offset != expected_fragment_offset) {
2831 dlog_verbose("Fragment offset was %d but expected %d.\n",
2832 fragment_offset, expected_fragment_offset);
2833 ret = ffa_error(FFA_INVALID_PARAMETERS);
2834 goto out;
2835 }
2836
J-Alves59ed0042022-07-28 18:26:41 +01002837 /* VMs acquire the RX buffer from SPMC. */
2838 CHECK(plat_ffa_acquire_receiver_rx(to_locked, &ret));
2839
Andrew Walbranca808b12020-05-15 17:22:28 +01002840 remaining_constituent_count = ffa_memory_fragment_init(
2841 to_locked.vm->mailbox.recv, HF_MAILBOX_SIZE,
2842 share_state->fragments[fragment_index],
2843 share_state->fragment_constituent_counts[fragment_index],
2844 &fragment_length);
2845 CHECK(remaining_constituent_count == 0);
2846 to_locked.vm->mailbox.recv_size = fragment_length;
2847 to_locked.vm->mailbox.recv_sender = HF_HYPERVISOR_VM_ID;
2848 to_locked.vm->mailbox.recv_func = FFA_MEM_FRAG_TX_32;
J-Alvese8c8c2b2022-12-16 15:34:48 +00002849 to_locked.vm->mailbox.state = MAILBOX_STATE_FULL;
Andrew Walbranca808b12020-05-15 17:22:28 +01002850
J-Alves59ed0042022-07-28 18:26:41 +01002851 if (!continue_ffa_hyp_mem_retrieve_req) {
2852 share_state->retrieved_fragment_count[receiver_index]++;
2853 if (share_state->retrieved_fragment_count[receiver_index] ==
2854 share_state->fragment_count) {
2855 ffa_memory_retrieve_complete(share_states, share_state,
2856 page_pool);
2857 }
2858 } else {
2859 share_state->hypervisor_fragment_count++;
2860
2861 ffa_memory_retrieve_complete_from_hyp(share_state);
2862 }
Andrew Walbranca808b12020-05-15 17:22:28 +01002863 ret = (struct ffa_value){.func = FFA_MEM_FRAG_TX_32,
2864 .arg1 = (uint32_t)handle,
2865 .arg2 = (uint32_t)(handle >> 32),
2866 .arg3 = fragment_length};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002867
2868out:
2869 share_states_unlock(&share_states);
2870 dump_share_states();
2871 return ret;
2872}
2873
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002874struct ffa_value ffa_memory_relinquish(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002875 struct vm_locked from_locked,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002876 struct ffa_mem_relinquish *relinquish_request, struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002877{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002878 ffa_memory_handle_t handle = relinquish_request->handle;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002879 struct share_states_locked share_states;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002880 struct ffa_memory_share_state *share_state;
2881 struct ffa_memory_region *memory_region;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002882 bool clear;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002883 struct ffa_value ret;
J-Alves8eb19162022-04-28 10:56:48 +01002884 uint32_t receiver_index;
J-Alves3c5b2072022-11-21 12:45:40 +00002885 bool receivers_relinquished_memory;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002886
Andrew Walbrana65a1322020-04-06 19:32:32 +01002887 if (relinquish_request->endpoint_count != 1) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002888 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01002889 "Stream endpoints not supported (got %d "
J-Alves668a86e2023-05-10 11:53:25 +01002890 "endpoints on FFA_MEM_RELINQUISH, expected 1).\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002891 relinquish_request->endpoint_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002892 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002893 }
2894
Andrew Walbrana65a1322020-04-06 19:32:32 +01002895 if (relinquish_request->endpoints[0] != from_locked.vm->id) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002896 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01002897 "VM ID %d in relinquish message doesn't match "
J-Alves668a86e2023-05-10 11:53:25 +01002898 "calling VM ID %d.\n",
Andrew Walbrana65a1322020-04-06 19:32:32 +01002899 relinquish_request->endpoints[0], from_locked.vm->id);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002900 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002901 }
2902
2903 dump_share_states();
2904
2905 share_states = share_states_lock();
Karl Meakin4a2854a2023-06-30 16:26:52 +01002906 share_state = get_share_state(share_states, handle);
2907 if (!share_state) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002908 dlog_verbose("Invalid handle %#x for FFA_MEM_RELINQUISH.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002909 handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002910 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002911 goto out;
2912 }
2913
Andrew Walbranca808b12020-05-15 17:22:28 +01002914 if (!share_state->sending_complete) {
2915 dlog_verbose(
2916 "Memory with handle %#x not fully sent, can't "
2917 "relinquish.\n",
2918 handle);
2919 ret = ffa_error(FFA_INVALID_PARAMETERS);
2920 goto out;
2921 }
2922
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002923 memory_region = share_state->memory_region;
2924 CHECK(memory_region != NULL);
2925
J-Alves8eb19162022-04-28 10:56:48 +01002926 receiver_index = ffa_memory_region_get_receiver(memory_region,
2927 from_locked.vm->id);
2928
2929 if (receiver_index == memory_region->receiver_count) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002930 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01002931 "VM ID %d tried to relinquish memory region "
J-Alves668a86e2023-05-10 11:53:25 +01002932 "with handle %#x and it is not a valid borrower.\n",
J-Alves8eb19162022-04-28 10:56:48 +01002933 from_locked.vm->id, handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002934 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002935 goto out;
2936 }
2937
J-Alves8eb19162022-04-28 10:56:48 +01002938 if (share_state->retrieved_fragment_count[receiver_index] !=
Andrew Walbranca808b12020-05-15 17:22:28 +01002939 share_state->fragment_count) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002940 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01002941 "Memory with handle %#x not yet fully "
2942 "retrieved, "
J-Alves8eb19162022-04-28 10:56:48 +01002943 "receiver %x can't relinquish.\n",
2944 handle, from_locked.vm->id);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002945 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002946 goto out;
2947 }
2948
J-Alves3c5b2072022-11-21 12:45:40 +00002949 /*
2950 * Either clear if requested in relinquish call, or in a retrieve
2951 * request from one of the borrowers.
2952 */
2953 receivers_relinquished_memory = true;
2954
2955 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
2956 struct ffa_memory_access *receiver =
2957 &memory_region->receivers[i];
2958
2959 if (receiver->receiver_permissions.receiver ==
2960 from_locked.vm->id) {
2961 continue;
2962 }
2963
2964 if (share_state->retrieved_fragment_count[i] != 0U) {
2965 receivers_relinquished_memory = false;
2966 break;
2967 }
2968 }
2969
2970 clear = receivers_relinquished_memory &&
2971 (share_state->clear_after_relinquish ||
2972 (relinquish_request->flags & FFA_MEMORY_REGION_FLAG_CLEAR) !=
2973 0U);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002974
2975 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01002976 * Clear is not allowed for memory that was shared, as the
2977 * original sender still has access to the memory.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002978 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002979 if (clear && share_state->share_func == FFA_MEM_SHARE_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002980 dlog_verbose("Memory which was shared can't be cleared.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002981 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002982 goto out;
2983 }
2984
Andrew Walbranca808b12020-05-15 17:22:28 +01002985 ret = ffa_relinquish_check_update(
J-Alves26483382023-04-20 12:01:49 +01002986 from_locked, share_state->fragments,
Andrew Walbranca808b12020-05-15 17:22:28 +01002987 share_state->fragment_constituent_counts,
2988 share_state->fragment_count, page_pool, clear);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002989
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002990 if (ret.func == FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002991 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01002992 * Mark memory handle as not retrieved, so it can be
2993 * reclaimed (or retrieved again).
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002994 */
J-Alves8eb19162022-04-28 10:56:48 +01002995 share_state->retrieved_fragment_count[receiver_index] = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002996 }
2997
2998out:
2999 share_states_unlock(&share_states);
3000 dump_share_states();
3001 return ret;
3002}
3003
3004/**
J-Alvesa9cd7e32022-07-01 13:49:33 +01003005 * Validates that the reclaim transition is allowed for the given
3006 * handle, updates the page table of the reclaiming VM, and frees the
3007 * internal state associated with the handle.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003008 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003009struct ffa_value ffa_memory_reclaim(struct vm_locked to_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01003010 ffa_memory_handle_t handle,
3011 ffa_memory_region_flags_t flags,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003012 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003013{
3014 struct share_states_locked share_states;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003015 struct ffa_memory_share_state *share_state;
3016 struct ffa_memory_region *memory_region;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003017 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003018
3019 dump_share_states();
3020
3021 share_states = share_states_lock();
Karl Meakin52cdfe72023-06-30 14:49:10 +01003022
Karl Meakin4a2854a2023-06-30 16:26:52 +01003023 share_state = get_share_state(share_states, handle);
3024 if (!share_state) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003025 dlog_verbose("Invalid handle %#x for FFA_MEM_RECLAIM.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003026 handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003027 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003028 goto out;
3029 }
Karl Meakin4a2854a2023-06-30 16:26:52 +01003030 memory_region = share_state->memory_region;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003031
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003032 CHECK(memory_region != NULL);
3033
J-Alvesa9cd7e32022-07-01 13:49:33 +01003034 if (vm_id_is_current_world(to_locked.vm->id) &&
3035 to_locked.vm->id != memory_region->sender) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003036 dlog_verbose(
Olivier Deprezf92e5d42020-11-13 16:00:54 +01003037 "VM %#x attempted to reclaim memory handle %#x "
3038 "originally sent by VM %#x.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003039 to_locked.vm->id, handle, memory_region->sender);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003040 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003041 goto out;
3042 }
3043
Andrew Walbranca808b12020-05-15 17:22:28 +01003044 if (!share_state->sending_complete) {
3045 dlog_verbose(
3046 "Memory with handle %#x not fully sent, can't "
3047 "reclaim.\n",
3048 handle);
3049 ret = ffa_error(FFA_INVALID_PARAMETERS);
3050 goto out;
3051 }
3052
J-Alves752236c2022-04-28 11:07:47 +01003053 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
3054 if (share_state->retrieved_fragment_count[i] != 0) {
3055 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01003056 "Tried to reclaim memory handle %#x "
J-Alves3c5b2072022-11-21 12:45:40 +00003057 "that has not been relinquished by all "
J-Alvesa9cd7e32022-07-01 13:49:33 +01003058 "borrowers(%x).\n",
J-Alves752236c2022-04-28 11:07:47 +01003059 handle,
3060 memory_region->receivers[i]
3061 .receiver_permissions.receiver);
3062 ret = ffa_error(FFA_DENIED);
3063 goto out;
3064 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003065 }
3066
Andrew Walbranca808b12020-05-15 17:22:28 +01003067 ret = ffa_retrieve_check_update(
J-Alves26483382023-04-20 12:01:49 +01003068 to_locked, share_state->fragments,
Andrew Walbranca808b12020-05-15 17:22:28 +01003069 share_state->fragment_constituent_counts,
J-Alves2a0d2882020-10-29 14:49:50 +00003070 share_state->fragment_count, share_state->sender_orig_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +01003071 FFA_MEM_RECLAIM_32, flags & FFA_MEM_RECLAIM_CLEAR, page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003072
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003073 if (ret.func == FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003074 share_state_free(share_states, share_state, page_pool);
J-Alves3c5b2072022-11-21 12:45:40 +00003075 dlog_verbose("Freed share state after successful reclaim.\n");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003076 }
3077
3078out:
3079 share_states_unlock(&share_states);
3080 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01003081}