blob: 5c8b5bef7371c795fffc6582b52bd9f020dab160 [file] [log] [blame]
Jose Marinho75509b42019-04-09 09:34:59 +01001/*
2 * Copyright 2019 The Hafnium Authors.
3 *
Andrew Walbrane959ec12020-06-17 15:01:09 +01004 * Use of this source code is governed by a BSD-style
5 * license that can be found in the LICENSE file or at
6 * https://opensource.org/licenses/BSD-3-Clause.
Jose Marinho75509b42019-04-09 09:34:59 +01007 */
8
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01009#include "hf/ffa_memory.h"
Andrew Walbran475c1452020-02-07 13:22:22 +000010
Federico Recanati4fd065d2021-12-13 20:06:23 +010011#include "hf/arch/mm.h"
Olivier Deprez112d2b52020-09-30 07:39:23 +020012#include "hf/arch/other_world.h"
Olivier Deprez55a189e2021-06-09 15:45:27 +020013#include "hf/arch/plat/ffa.h"
Andrew Walbran290b0c92020-02-03 16:37:14 +000014
J-Alves5952d942022-12-22 16:03:00 +000015#include "hf/addr.h"
Jose Marinho75509b42019-04-09 09:34:59 +010016#include "hf/api.h"
Daniel Boulbya2f8c662021-11-26 17:52:53 +000017#include "hf/assert.h"
Jose Marinho09b1db82019-08-08 09:16:59 +010018#include "hf/check.h"
Jose Marinho75509b42019-04-09 09:34:59 +010019#include "hf/dlog.h"
J-Alves3456e032023-07-20 12:20:05 +010020#include "hf/ffa.h"
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010021#include "hf/ffa_internal.h"
J-Alves66652252022-07-06 09:49:51 +010022#include "hf/ffa_memory_internal.h"
J-Alves3456e032023-07-20 12:20:05 +010023#include "hf/ffa_partition_manifest.h"
J-Alves5952d942022-12-22 16:03:00 +000024#include "hf/mm.h"
Andrew Walbran475c1452020-02-07 13:22:22 +000025#include "hf/mpool.h"
Jose Marinho75509b42019-04-09 09:34:59 +010026#include "hf/std.h"
Andrew Scull3c257452019-11-26 13:32:50 +000027#include "hf/vm.h"
Jose Marinho75509b42019-04-09 09:34:59 +010028
J-Alves2d8457f2022-10-05 11:06:41 +010029#include "vmapi/hf/ffa_v1_0.h"
30
J-Alves5da37d92022-10-24 16:33:48 +010031#define RECEIVERS_COUNT_IN_RETRIEVE_RESP 1
32
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000033/**
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010034 * All access to members of a `struct ffa_memory_share_state` must be guarded
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000035 * by this lock.
36 */
37static struct spinlock share_states_lock_instance = SPINLOCK_INIT;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010038static struct ffa_memory_share_state share_states[MAX_MEM_SHARES];
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000039
40/**
J-Alves917d2f22020-10-30 18:39:30 +000041 * Extracts the index from a memory handle allocated by Hafnium's current world.
42 */
43uint64_t ffa_memory_handle_get_index(ffa_memory_handle_t handle)
44{
45 return handle & ~FFA_MEMORY_HANDLE_ALLOCATOR_MASK;
46}
47
48/**
Karl Meakin52cdfe72023-06-30 14:49:10 +010049 * Initialises the next available `struct ffa_memory_share_state`. If `handle`
50 * is `FFA_MEMORY_HANDLE_INVALID` then allocates an appropriate handle,
51 * otherwise uses the provided handle which is assumed to be globally unique.
Andrew Walbranca808b12020-05-15 17:22:28 +010052 *
Karl Meakin52cdfe72023-06-30 14:49:10 +010053 * Returns a pointer to the allocated `ffa_memory_share_state` on success or
54 * `NULL` if none are available.
Andrew Walbranca808b12020-05-15 17:22:28 +010055 */
Karl Meakin52cdfe72023-06-30 14:49:10 +010056struct ffa_memory_share_state *allocate_share_state(
57 struct share_states_locked share_states, uint32_t share_func,
58 struct ffa_memory_region *memory_region, uint32_t fragment_length,
59 ffa_memory_handle_t handle)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000060{
Daniel Boulbya2f8c662021-11-26 17:52:53 +000061 assert(share_states.share_states != NULL);
62 assert(memory_region != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000063
Karl Meakin52cdfe72023-06-30 14:49:10 +010064 for (uint64_t i = 0; i < MAX_MEM_SHARES; ++i) {
Andrew Walbranca808b12020-05-15 17:22:28 +010065 if (share_states.share_states[i].share_func == 0) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010066 struct ffa_memory_share_state *allocated_state =
Andrew Walbranca808b12020-05-15 17:22:28 +010067 &share_states.share_states[i];
68 struct ffa_composite_memory_region *composite =
69 ffa_memory_region_get_composite(memory_region,
70 0);
71
72 if (handle == FFA_MEMORY_HANDLE_INVALID) {
J-Alvesee68c542020-10-29 17:48:20 +000073 memory_region->handle =
Olivier Deprez55a189e2021-06-09 15:45:27 +020074 plat_ffa_memory_handle_make(i);
Andrew Walbranca808b12020-05-15 17:22:28 +010075 } else {
J-Alvesee68c542020-10-29 17:48:20 +000076 memory_region->handle = handle;
Andrew Walbranca808b12020-05-15 17:22:28 +010077 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000078 allocated_state->share_func = share_func;
79 allocated_state->memory_region = memory_region;
Andrew Walbranca808b12020-05-15 17:22:28 +010080 allocated_state->fragment_count = 1;
81 allocated_state->fragments[0] = composite->constituents;
82 allocated_state->fragment_constituent_counts[0] =
83 (fragment_length -
84 ffa_composite_constituent_offset(memory_region,
85 0)) /
86 sizeof(struct ffa_memory_region_constituent);
87 allocated_state->sending_complete = false;
Karl Meakin52cdfe72023-06-30 14:49:10 +010088 for (uint32_t j = 0; j < MAX_MEM_SHARE_RECIPIENTS;
89 ++j) {
Andrew Walbranca808b12020-05-15 17:22:28 +010090 allocated_state->retrieved_fragment_count[j] =
91 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000092 }
Karl Meakin52cdfe72023-06-30 14:49:10 +010093 return allocated_state;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000094 }
95 }
96
Karl Meakin52cdfe72023-06-30 14:49:10 +010097 return NULL;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000098}
99
100/** Locks the share states lock. */
101struct share_states_locked share_states_lock(void)
102{
103 sl_lock(&share_states_lock_instance);
104
105 return (struct share_states_locked){.share_states = share_states};
106}
107
108/** Unlocks the share states lock. */
J-Alves66652252022-07-06 09:49:51 +0100109void share_states_unlock(struct share_states_locked *share_states)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000110{
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000111 assert(share_states->share_states != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000112 share_states->share_states = NULL;
113 sl_unlock(&share_states_lock_instance);
114}
115
116/**
Andrew Walbranca808b12020-05-15 17:22:28 +0100117 * If the given handle is a valid handle for an allocated share state then
Karl Meakin4a2854a2023-06-30 16:26:52 +0100118 * returns a pointer to the share state. Otherwise returns NULL.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000119 */
Karl Meakin4a2854a2023-06-30 16:26:52 +0100120struct ffa_memory_share_state *get_share_state(
121 struct share_states_locked share_states, ffa_memory_handle_t handle)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000122{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100123 struct ffa_memory_share_state *share_state;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000124
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000125 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100126
127 /*
128 * First look for a share_state allocated by us, in which case the
129 * handle is based on the index.
130 */
Olivier Deprez55a189e2021-06-09 15:45:27 +0200131 if (plat_ffa_memory_handle_allocated_by_current_world(handle)) {
Karl Meakin4a2854a2023-06-30 16:26:52 +0100132 uint64_t index = ffa_memory_handle_get_index(handle);
133
Andrew Walbranca808b12020-05-15 17:22:28 +0100134 if (index < MAX_MEM_SHARES) {
135 share_state = &share_states.share_states[index];
136 if (share_state->share_func != 0) {
Karl Meakin4a2854a2023-06-30 16:26:52 +0100137 return share_state;
Andrew Walbranca808b12020-05-15 17:22:28 +0100138 }
139 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000140 }
141
Andrew Walbranca808b12020-05-15 17:22:28 +0100142 /* Fall back to a linear scan. */
Karl Meakin4a2854a2023-06-30 16:26:52 +0100143 for (uint64_t index = 0; index < MAX_MEM_SHARES; ++index) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100144 share_state = &share_states.share_states[index];
J-Alvesee68c542020-10-29 17:48:20 +0000145 if (share_state->memory_region != NULL &&
146 share_state->memory_region->handle == handle &&
Andrew Walbranca808b12020-05-15 17:22:28 +0100147 share_state->share_func != 0) {
Karl Meakin4a2854a2023-06-30 16:26:52 +0100148 return share_state;
Andrew Walbranca808b12020-05-15 17:22:28 +0100149 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000150 }
151
Karl Meakin4a2854a2023-06-30 16:26:52 +0100152 return NULL;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000153}
154
155/** Marks a share state as unallocated. */
J-Alvesfdd29272022-07-19 13:16:31 +0100156void share_state_free(struct share_states_locked share_states,
157 struct ffa_memory_share_state *share_state,
158 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000159{
Andrew Walbranca808b12020-05-15 17:22:28 +0100160 uint32_t i;
161
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000162 assert(share_states.share_states != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000163 share_state->share_func = 0;
Andrew Walbranca808b12020-05-15 17:22:28 +0100164 share_state->sending_complete = false;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000165 mpool_free(page_pool, share_state->memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +0100166 /*
167 * First fragment is part of the same page as the `memory_region`, so it
168 * doesn't need to be freed separately.
169 */
170 share_state->fragments[0] = NULL;
171 share_state->fragment_constituent_counts[0] = 0;
172 for (i = 1; i < share_state->fragment_count; ++i) {
173 mpool_free(page_pool, share_state->fragments[i]);
174 share_state->fragments[i] = NULL;
175 share_state->fragment_constituent_counts[i] = 0;
176 }
177 share_state->fragment_count = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000178 share_state->memory_region = NULL;
J-Alvesa9cd7e32022-07-01 13:49:33 +0100179 share_state->hypervisor_fragment_count = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000180}
181
Andrew Walbranca808b12020-05-15 17:22:28 +0100182/** Checks whether the given share state has been fully sent. */
J-Alvesfdd29272022-07-19 13:16:31 +0100183bool share_state_sending_complete(struct share_states_locked share_states,
184 struct ffa_memory_share_state *share_state)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000185{
Andrew Walbranca808b12020-05-15 17:22:28 +0100186 struct ffa_composite_memory_region *composite;
187 uint32_t expected_constituent_count;
188 uint32_t fragment_constituent_count_total = 0;
189 uint32_t i;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000190
Andrew Walbranca808b12020-05-15 17:22:28 +0100191 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000192 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100193
194 /*
195 * Share state must already be valid, or it's not possible to get hold
196 * of it.
197 */
198 CHECK(share_state->memory_region != NULL &&
199 share_state->share_func != 0);
200
201 composite =
202 ffa_memory_region_get_composite(share_state->memory_region, 0);
203 expected_constituent_count = composite->constituent_count;
204 for (i = 0; i < share_state->fragment_count; ++i) {
205 fragment_constituent_count_total +=
206 share_state->fragment_constituent_counts[i];
207 }
208 dlog_verbose(
209 "Checking completion: constituent count %d/%d from %d "
210 "fragments.\n",
211 fragment_constituent_count_total, expected_constituent_count,
212 share_state->fragment_count);
213
214 return fragment_constituent_count_total == expected_constituent_count;
215}
216
217/**
218 * Calculates the offset of the next fragment expected for the given share
219 * state.
220 */
J-Alvesfdd29272022-07-19 13:16:31 +0100221uint32_t share_state_next_fragment_offset(
Andrew Walbranca808b12020-05-15 17:22:28 +0100222 struct share_states_locked share_states,
223 struct ffa_memory_share_state *share_state)
224{
225 uint32_t next_fragment_offset;
226 uint32_t i;
227
228 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000229 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100230
231 next_fragment_offset =
232 ffa_composite_constituent_offset(share_state->memory_region, 0);
233 for (i = 0; i < share_state->fragment_count; ++i) {
234 next_fragment_offset +=
235 share_state->fragment_constituent_counts[i] *
236 sizeof(struct ffa_memory_region_constituent);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000237 }
238
Andrew Walbranca808b12020-05-15 17:22:28 +0100239 return next_fragment_offset;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000240}
241
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100242static void dump_memory_region(struct ffa_memory_region *memory_region)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000243{
244 uint32_t i;
245
246 if (LOG_LEVEL < LOG_LEVEL_VERBOSE) {
247 return;
248 }
249
Olivier Deprez935e1b12020-12-22 18:01:29 +0100250 dlog("from VM %#x, attributes %#x, flags %#x, tag %u, to "
Olivier Deprezf92e5d42020-11-13 16:00:54 +0100251 "%u "
Andrew Walbrana65a1322020-04-06 19:32:32 +0100252 "recipients [",
253 memory_region->sender, memory_region->attributes,
Olivier Deprez935e1b12020-12-22 18:01:29 +0100254 memory_region->flags, memory_region->tag,
Andrew Walbrana65a1322020-04-06 19:32:32 +0100255 memory_region->receiver_count);
256 for (i = 0; i < memory_region->receiver_count; ++i) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000257 if (i != 0) {
258 dlog(", ");
259 }
Olivier Deprezf92e5d42020-11-13 16:00:54 +0100260 dlog("VM %#x: %#x (offset %u)",
Andrew Walbrana65a1322020-04-06 19:32:32 +0100261 memory_region->receivers[i].receiver_permissions.receiver,
262 memory_region->receivers[i]
263 .receiver_permissions.permissions,
264 memory_region->receivers[i]
265 .composite_memory_region_offset);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000266 }
267 dlog("]");
268}
269
J-Alves66652252022-07-06 09:49:51 +0100270void dump_share_states(void)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000271{
272 uint32_t i;
273
274 if (LOG_LEVEL < LOG_LEVEL_VERBOSE) {
275 return;
276 }
277
278 dlog("Current share states:\n");
279 sl_lock(&share_states_lock_instance);
280 for (i = 0; i < MAX_MEM_SHARES; ++i) {
281 if (share_states[i].share_func != 0) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000282 switch (share_states[i].share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100283 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000284 dlog("SHARE");
285 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100286 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000287 dlog("LEND");
288 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100289 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000290 dlog("DONATE");
291 break;
292 default:
293 dlog("invalid share_func %#x",
294 share_states[i].share_func);
295 }
Olivier Deprez935e1b12020-12-22 18:01:29 +0100296 dlog(" %#x (", share_states[i].memory_region->handle);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000297 dump_memory_region(share_states[i].memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +0100298 if (share_states[i].sending_complete) {
299 dlog("): fully sent");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000300 } else {
Andrew Walbranca808b12020-05-15 17:22:28 +0100301 dlog("): partially sent");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000302 }
J-Alves2a0d2882020-10-29 14:49:50 +0000303 dlog(" with %d fragments, %d retrieved, "
304 " sender's original mode: %#x\n",
Andrew Walbranca808b12020-05-15 17:22:28 +0100305 share_states[i].fragment_count,
J-Alves2a0d2882020-10-29 14:49:50 +0000306 share_states[i].retrieved_fragment_count[0],
307 share_states[i].sender_orig_mode);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000308 }
309 }
310 sl_unlock(&share_states_lock_instance);
311}
312
Andrew Walbran475c1452020-02-07 13:22:22 +0000313/* TODO: Add device attributes: GRE, cacheability, shareability. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100314static inline uint32_t ffa_memory_permissions_to_mode(
J-Alves7cd5eb32020-10-16 19:06:10 +0100315 ffa_memory_access_permissions_t permissions, uint32_t default_mode)
Andrew Walbran475c1452020-02-07 13:22:22 +0000316{
317 uint32_t mode = 0;
318
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100319 switch (ffa_get_data_access_attr(permissions)) {
320 case FFA_DATA_ACCESS_RO:
Andrew Walbran475c1452020-02-07 13:22:22 +0000321 mode = MM_MODE_R;
322 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100323 case FFA_DATA_ACCESS_RW:
Andrew Walbran475c1452020-02-07 13:22:22 +0000324 mode = MM_MODE_R | MM_MODE_W;
325 break;
J-Alves7cd5eb32020-10-16 19:06:10 +0100326 case FFA_DATA_ACCESS_NOT_SPECIFIED:
327 mode = (default_mode & (MM_MODE_R | MM_MODE_W));
328 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100329 case FFA_DATA_ACCESS_RESERVED:
330 panic("Tried to convert FFA_DATA_ACCESS_RESERVED.");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100331 }
332
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100333 switch (ffa_get_instruction_access_attr(permissions)) {
334 case FFA_INSTRUCTION_ACCESS_NX:
Andrew Walbran475c1452020-02-07 13:22:22 +0000335 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100336 case FFA_INSTRUCTION_ACCESS_X:
Andrew Walbrana65a1322020-04-06 19:32:32 +0100337 mode |= MM_MODE_X;
338 break;
J-Alves7cd5eb32020-10-16 19:06:10 +0100339 case FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED:
340 mode |= (default_mode & MM_MODE_X);
341 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100342 case FFA_INSTRUCTION_ACCESS_RESERVED:
343 panic("Tried to convert FFA_INSTRUCTION_ACCESS_RESVERVED.");
Andrew Walbran475c1452020-02-07 13:22:22 +0000344 }
345
Olivier Deprez878bd5b2021-04-15 19:05:10 +0200346 /* Set the security state bit if necessary. */
347 if ((default_mode & plat_ffa_other_world_mode()) != 0) {
348 mode |= plat_ffa_other_world_mode();
349 }
350
Andrew Walbran475c1452020-02-07 13:22:22 +0000351 return mode;
352}
353
Jose Marinho75509b42019-04-09 09:34:59 +0100354/**
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000355 * Get the current mode in the stage-2 page table of the given vm of all the
356 * pages in the given constituents, if they all have the same mode, or return
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100357 * an appropriate FF-A error if not.
Jose Marinho75509b42019-04-09 09:34:59 +0100358 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100359static struct ffa_value constituents_get_mode(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000360 struct vm_locked vm, uint32_t *orig_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100361 struct ffa_memory_region_constituent **fragments,
362 const uint32_t *fragment_constituent_counts, uint32_t fragment_count)
Jose Marinho75509b42019-04-09 09:34:59 +0100363{
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100364 uint32_t i;
Andrew Walbranca808b12020-05-15 17:22:28 +0100365 uint32_t j;
Jose Marinho75509b42019-04-09 09:34:59 +0100366
Andrew Walbranca808b12020-05-15 17:22:28 +0100367 if (fragment_count == 0 || fragment_constituent_counts[0] == 0) {
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100368 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000369 * Fail if there are no constituents. Otherwise we would get an
370 * uninitialised *orig_mode.
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100371 */
Karl Meakin5df422c2023-07-11 17:31:38 +0100372 dlog_verbose("%s: no constituents\n", __func__);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100373 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho75509b42019-04-09 09:34:59 +0100374 }
375
Andrew Walbranca808b12020-05-15 17:22:28 +0100376 for (i = 0; i < fragment_count; ++i) {
377 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
378 ipaddr_t begin = ipa_init(fragments[i][j].address);
379 size_t size = fragments[i][j].page_count * PAGE_SIZE;
380 ipaddr_t end = ipa_add(begin, size);
381 uint32_t current_mode;
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100382
Andrew Walbranca808b12020-05-15 17:22:28 +0100383 /* Fail if addresses are not page-aligned. */
384 if (!is_aligned(ipa_addr(begin), PAGE_SIZE) ||
385 !is_aligned(ipa_addr(end), PAGE_SIZE)) {
Karl Meakin5df422c2023-07-11 17:31:38 +0100386 dlog_verbose("%s: addresses not page-aligned\n",
387 __func__);
Andrew Walbranca808b12020-05-15 17:22:28 +0100388 return ffa_error(FFA_INVALID_PARAMETERS);
389 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100390
Andrew Walbranca808b12020-05-15 17:22:28 +0100391 /*
392 * Ensure that this constituent memory range is all
393 * mapped with the same mode.
394 */
Raghu Krishnamurthy785d52f2021-02-13 00:02:40 -0800395 if (!vm_mem_get_mode(vm, begin, end, &current_mode)) {
Karl Meakin5df422c2023-07-11 17:31:38 +0100396 dlog_verbose(
397 "%s: constituent memory range %#x..%#x "
398 "not mapped with the same mode\n",
399 __func__, begin, end);
Andrew Walbranca808b12020-05-15 17:22:28 +0100400 return ffa_error(FFA_DENIED);
401 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100402
Andrew Walbranca808b12020-05-15 17:22:28 +0100403 /*
404 * Ensure that all constituents are mapped with the same
405 * mode.
406 */
407 if (i == 0) {
408 *orig_mode = current_mode;
409 } else if (current_mode != *orig_mode) {
410 dlog_verbose(
Karl Meakin5df422c2023-07-11 17:31:38 +0100411 "%s: expected mode %#x but was %#x for "
412 "%d pages at %#x.\n",
413 __func__, *orig_mode, current_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100414 fragments[i][j].page_count,
415 ipa_addr(begin));
416 return ffa_error(FFA_DENIED);
417 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100418 }
Jose Marinho75509b42019-04-09 09:34:59 +0100419 }
420
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100421 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000422}
423
424/**
425 * Verify that all pages have the same mode, that the starting mode
426 * constitutes a valid state and obtain the next mode to apply
427 * to the sending VM.
428 *
429 * Returns:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100430 * 1) FFA_DENIED if a state transition was not found;
431 * 2) FFA_DENIED if the pages being shared do not have the same mode within
Andrew Walbrana65a1322020-04-06 19:32:32 +0100432 * the <from> VM;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100433 * 3) FFA_INVALID_PARAMETERS if the beginning and end IPAs are not page
Andrew Walbrana65a1322020-04-06 19:32:32 +0100434 * aligned;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100435 * 4) FFA_INVALID_PARAMETERS if the requested share type was not handled.
436 * Or FFA_SUCCESS on success.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000437 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100438static struct ffa_value ffa_send_check_transition(
Andrew Walbrana65a1322020-04-06 19:32:32 +0100439 struct vm_locked from, uint32_t share_func,
J-Alves363f5722022-04-25 17:37:37 +0100440 struct ffa_memory_access *receivers, uint32_t receivers_count,
441 uint32_t *orig_from_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100442 struct ffa_memory_region_constituent **fragments,
443 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
444 uint32_t *from_mode)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000445{
446 const uint32_t state_mask =
447 MM_MODE_INVALID | MM_MODE_UNOWNED | MM_MODE_SHARED;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100448 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000449
Andrew Walbranca808b12020-05-15 17:22:28 +0100450 ret = constituents_get_mode(from, orig_from_mode, fragments,
451 fragment_constituent_counts,
452 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100453 if (ret.func != FFA_SUCCESS_32) {
Olivier Depreze7eb1682022-03-16 17:09:03 +0100454 dlog_verbose("Inconsistent modes.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100455 return ret;
Andrew Scullb5f49e02019-10-02 13:20:47 +0100456 }
457
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000458 /* Ensure the address range is normal memory and not a device. */
459 if (*orig_from_mode & MM_MODE_D) {
460 dlog_verbose("Can't share device memory (mode is %#x).\n",
461 *orig_from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100462 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000463 }
464
465 /*
466 * Ensure the sender is the owner and has exclusive access to the
467 * memory.
468 */
469 if ((*orig_from_mode & state_mask) != 0) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100470 return ffa_error(FFA_DENIED);
Andrew Walbrana65a1322020-04-06 19:32:32 +0100471 }
472
J-Alves363f5722022-04-25 17:37:37 +0100473 assert(receivers != NULL && receivers_count > 0U);
J-Alves7cd5eb32020-10-16 19:06:10 +0100474
J-Alves363f5722022-04-25 17:37:37 +0100475 for (uint32_t i = 0U; i < receivers_count; i++) {
476 ffa_memory_access_permissions_t permissions =
477 receivers[i].receiver_permissions.permissions;
478 uint32_t required_from_mode = ffa_memory_permissions_to_mode(
479 permissions, *orig_from_mode);
480
481 if ((*orig_from_mode & required_from_mode) !=
482 required_from_mode) {
483 dlog_verbose(
484 "Sender tried to send memory with permissions "
485 "which "
486 "required mode %#x but only had %#x itself.\n",
487 required_from_mode, *orig_from_mode);
488 return ffa_error(FFA_DENIED);
489 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000490 }
491
492 /* Find the appropriate new mode. */
493 *from_mode = ~state_mask & *orig_from_mode;
Andrew Walbrane7ad3c02019-12-24 17:03:04 +0000494 switch (share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100495 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000496 *from_mode |= MM_MODE_INVALID | MM_MODE_UNOWNED;
Jose Marinho75509b42019-04-09 09:34:59 +0100497 break;
498
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100499 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000500 *from_mode |= MM_MODE_INVALID;
Andrew Walbran648fc3e2019-10-22 16:23:05 +0100501 break;
502
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100503 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000504 *from_mode |= MM_MODE_SHARED;
Jose Marinho56c25732019-05-20 09:48:53 +0100505 break;
506
Jose Marinho75509b42019-04-09 09:34:59 +0100507 default:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100508 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho75509b42019-04-09 09:34:59 +0100509 }
510
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100511 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000512}
513
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100514static struct ffa_value ffa_relinquish_check_transition(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000515 struct vm_locked from, uint32_t *orig_from_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100516 struct ffa_memory_region_constituent **fragments,
517 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
518 uint32_t *from_mode)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000519{
520 const uint32_t state_mask =
521 MM_MODE_INVALID | MM_MODE_UNOWNED | MM_MODE_SHARED;
522 uint32_t orig_from_state;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100523 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000524
Andrew Walbranca808b12020-05-15 17:22:28 +0100525 ret = constituents_get_mode(from, orig_from_mode, fragments,
526 fragment_constituent_counts,
527 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100528 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbrana65a1322020-04-06 19:32:32 +0100529 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000530 }
531
532 /* Ensure the address range is normal memory and not a device. */
533 if (*orig_from_mode & MM_MODE_D) {
534 dlog_verbose("Can't relinquish device memory (mode is %#x).\n",
535 *orig_from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100536 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000537 }
538
539 /*
540 * Ensure the relinquishing VM is not the owner but has access to the
541 * memory.
542 */
543 orig_from_state = *orig_from_mode & state_mask;
544 if ((orig_from_state & ~MM_MODE_SHARED) != MM_MODE_UNOWNED) {
545 dlog_verbose(
546 "Tried to relinquish memory in state %#x (masked %#x "
Andrew Walbranca808b12020-05-15 17:22:28 +0100547 "but should be %#x).\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000548 *orig_from_mode, orig_from_state, MM_MODE_UNOWNED);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100549 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000550 }
551
552 /* Find the appropriate new mode. */
553 *from_mode = (~state_mask & *orig_from_mode) | MM_MODE_UNMAPPED_MASK;
554
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100555 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000556}
557
558/**
559 * Verify that all pages have the same mode, that the starting mode
560 * constitutes a valid state and obtain the next mode to apply
561 * to the retrieving VM.
562 *
563 * Returns:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100564 * 1) FFA_DENIED if a state transition was not found;
565 * 2) FFA_DENIED if the pages being shared do not have the same mode within
Andrew Walbrana65a1322020-04-06 19:32:32 +0100566 * the <to> VM;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100567 * 3) FFA_INVALID_PARAMETERS if the beginning and end IPAs are not page
Andrew Walbrana65a1322020-04-06 19:32:32 +0100568 * aligned;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100569 * 4) FFA_INVALID_PARAMETERS if the requested share type was not handled.
570 * Or FFA_SUCCESS on success.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000571 */
J-Alvesfc19b372022-07-06 12:17:35 +0100572struct ffa_value ffa_retrieve_check_transition(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000573 struct vm_locked to, uint32_t share_func,
Andrew Walbranca808b12020-05-15 17:22:28 +0100574 struct ffa_memory_region_constituent **fragments,
575 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
576 uint32_t memory_to_attributes, uint32_t *to_mode)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000577{
578 uint32_t orig_to_mode;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100579 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000580
Andrew Walbranca808b12020-05-15 17:22:28 +0100581 ret = constituents_get_mode(to, &orig_to_mode, fragments,
582 fragment_constituent_counts,
583 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100584 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100585 dlog_verbose("Inconsistent modes.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100586 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000587 }
588
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100589 if (share_func == FFA_MEM_RECLAIM_32) {
J-Alves9256f162021-12-09 13:18:43 +0000590 /*
591 * If the original ffa memory send call has been processed
592 * successfully, it is expected the orig_to_mode would overlay
593 * with `state_mask`, as a result of the function
594 * `ffa_send_check_transition`.
595 */
J-Alves59ed0042022-07-28 18:26:41 +0100596 if (vm_id_is_current_world(to.vm->id)) {
597 assert((orig_to_mode &
598 (MM_MODE_INVALID | MM_MODE_UNOWNED |
599 MM_MODE_SHARED)) != 0U);
600 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000601 } else {
602 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +0100603 * If the retriever is from virtual FF-A instance:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000604 * Ensure the retriever has the expected state. We don't care
605 * about the MM_MODE_SHARED bit; either with or without it set
606 * are both valid representations of the !O-NA state.
607 */
J-Alvesa9cd7e32022-07-01 13:49:33 +0100608 if (vm_id_is_current_world(to.vm->id) &&
609 to.vm->id != HF_PRIMARY_VM_ID &&
610 (orig_to_mode & MM_MODE_UNMAPPED_MASK) !=
611 MM_MODE_UNMAPPED_MASK) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100612 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000613 }
614 }
615
616 /* Find the appropriate new mode. */
617 *to_mode = memory_to_attributes;
618 switch (share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100619 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000620 *to_mode |= 0;
621 break;
622
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100623 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000624 *to_mode |= MM_MODE_UNOWNED;
625 break;
626
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100627 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000628 *to_mode |= MM_MODE_UNOWNED | MM_MODE_SHARED;
629 break;
630
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100631 case FFA_MEM_RECLAIM_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000632 *to_mode |= 0;
633 break;
634
635 default:
Andrew Walbranca808b12020-05-15 17:22:28 +0100636 dlog_error("Invalid share_func %#x.\n", share_func);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100637 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000638 }
639
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100640 return (struct ffa_value){.func = FFA_SUCCESS_32};
Jose Marinho75509b42019-04-09 09:34:59 +0100641}
Jose Marinho09b1db82019-08-08 09:16:59 +0100642
643/**
644 * Updates a VM's page table such that the given set of physical address ranges
645 * are mapped in the address space at the corresponding address ranges, in the
646 * mode provided.
647 *
648 * If commit is false, the page tables will be allocated from the mpool but no
649 * mappings will actually be updated. This function must always be called first
650 * with commit false to check that it will succeed before calling with commit
651 * true, to avoid leaving the page table in a half-updated state. To make a
652 * series of changes atomically you can call them all with commit false before
653 * calling them all with commit true.
654 *
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -0700655 * vm_ptable_defrag should always be called after a series of page table
656 * updates, whether they succeed or fail.
Jose Marinho09b1db82019-08-08 09:16:59 +0100657 *
658 * Returns true on success, or false if the update failed and no changes were
659 * made to memory mappings.
660 */
J-Alves66652252022-07-06 09:49:51 +0100661bool ffa_region_group_identity_map(
Andrew Walbranf4b51af2020-02-03 14:44:54 +0000662 struct vm_locked vm_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +0100663 struct ffa_memory_region_constituent **fragments,
664 const uint32_t *fragment_constituent_counts, uint32_t fragment_count,
Daniel Boulby4dd3f532021-09-21 09:57:08 +0100665 uint32_t mode, struct mpool *ppool, bool commit)
Jose Marinho09b1db82019-08-08 09:16:59 +0100666{
Andrew Walbranca808b12020-05-15 17:22:28 +0100667 uint32_t i;
668 uint32_t j;
Jose Marinho09b1db82019-08-08 09:16:59 +0100669
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -0700670 if (vm_locked.vm->el0_partition) {
671 mode |= MM_MODE_USER | MM_MODE_NG;
672 }
673
Andrew Walbranca808b12020-05-15 17:22:28 +0100674 /* Iterate over the memory region constituents within each fragment. */
675 for (i = 0; i < fragment_count; ++i) {
676 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
677 size_t size = fragments[i][j].page_count * PAGE_SIZE;
678 paddr_t pa_begin =
679 pa_from_ipa(ipa_init(fragments[i][j].address));
680 paddr_t pa_end = pa_add(pa_begin, size);
Jens Wiklander4f1880c2022-10-19 17:00:14 +0200681 uint32_t pa_bits =
682 arch_mm_get_pa_bits(arch_mm_get_pa_range());
Federico Recanati4fd065d2021-12-13 20:06:23 +0100683
684 /*
685 * Ensure the requested region falls into system's PA
686 * range.
687 */
Jens Wiklander4f1880c2022-10-19 17:00:14 +0200688 if (((pa_addr(pa_begin) >> pa_bits) > 0) ||
689 ((pa_addr(pa_end) >> pa_bits) > 0)) {
Federico Recanati4fd065d2021-12-13 20:06:23 +0100690 dlog_error("Region is outside of PA Range\n");
691 return false;
692 }
Andrew Walbranca808b12020-05-15 17:22:28 +0100693
694 if (commit) {
695 vm_identity_commit(vm_locked, pa_begin, pa_end,
696 mode, ppool, NULL);
697 } else if (!vm_identity_prepare(vm_locked, pa_begin,
698 pa_end, mode, ppool)) {
699 return false;
700 }
Jose Marinho09b1db82019-08-08 09:16:59 +0100701 }
702 }
703
704 return true;
705}
706
707/**
708 * Clears a region of physical memory by overwriting it with zeros. The data is
709 * flushed from the cache so the memory has been cleared across the system.
710 */
J-Alves7db32002021-12-14 14:44:50 +0000711static bool clear_memory(paddr_t begin, paddr_t end, struct mpool *ppool,
712 uint32_t extra_mode_attributes)
Jose Marinho09b1db82019-08-08 09:16:59 +0100713{
714 /*
Fuad Tabbaed294af2019-12-20 10:43:01 +0000715 * TODO: change this to a CPU local single page window rather than a
Jose Marinho09b1db82019-08-08 09:16:59 +0100716 * global mapping of the whole range. Such an approach will limit
717 * the changes to stage-1 tables and will allow only local
718 * invalidation.
719 */
720 bool ret;
721 struct mm_stage1_locked stage1_locked = mm_lock_stage1();
J-Alves7db32002021-12-14 14:44:50 +0000722 void *ptr = mm_identity_map(stage1_locked, begin, end,
723 MM_MODE_W | (extra_mode_attributes &
724 plat_ffa_other_world_mode()),
725 ppool);
Jose Marinho09b1db82019-08-08 09:16:59 +0100726 size_t size = pa_difference(begin, end);
727
728 if (!ptr) {
Jose Marinho09b1db82019-08-08 09:16:59 +0100729 goto fail;
730 }
731
732 memset_s(ptr, size, 0, size);
733 arch_mm_flush_dcache(ptr, size);
734 mm_unmap(stage1_locked, begin, end, ppool);
735
736 ret = true;
737 goto out;
738
739fail:
740 ret = false;
741
742out:
743 mm_unlock_stage1(&stage1_locked);
744
745 return ret;
746}
747
748/**
749 * Clears a region of physical memory by overwriting it with zeros. The data is
750 * flushed from the cache so the memory has been cleared across the system.
751 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100752static bool ffa_clear_memory_constituents(
J-Alves7db32002021-12-14 14:44:50 +0000753 uint32_t security_state_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100754 struct ffa_memory_region_constituent **fragments,
755 const uint32_t *fragment_constituent_counts, uint32_t fragment_count,
756 struct mpool *page_pool)
Jose Marinho09b1db82019-08-08 09:16:59 +0100757{
758 struct mpool local_page_pool;
Andrew Walbranca808b12020-05-15 17:22:28 +0100759 uint32_t i;
Jose Marinho09b1db82019-08-08 09:16:59 +0100760 bool ret = false;
761
762 /*
763 * Create a local pool so any freed memory can't be used by another
764 * thread. This is to ensure each constituent that is mapped can be
765 * unmapped again afterwards.
766 */
Andrew Walbran475c1452020-02-07 13:22:22 +0000767 mpool_init_with_fallback(&local_page_pool, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +0100768
Andrew Walbranca808b12020-05-15 17:22:28 +0100769 /* Iterate over the memory region constituents within each fragment. */
770 for (i = 0; i < fragment_count; ++i) {
771 uint32_t j;
Jose Marinho09b1db82019-08-08 09:16:59 +0100772
Andrew Walbranca808b12020-05-15 17:22:28 +0100773 for (j = 0; j < fragment_constituent_counts[j]; ++j) {
774 size_t size = fragments[i][j].page_count * PAGE_SIZE;
775 paddr_t begin =
776 pa_from_ipa(ipa_init(fragments[i][j].address));
777 paddr_t end = pa_add(begin, size);
778
J-Alves7db32002021-12-14 14:44:50 +0000779 if (!clear_memory(begin, end, &local_page_pool,
780 security_state_mode)) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100781 /*
782 * api_clear_memory will defrag on failure, so
783 * no need to do it here.
784 */
785 goto out;
786 }
Jose Marinho09b1db82019-08-08 09:16:59 +0100787 }
788 }
789
Jose Marinho09b1db82019-08-08 09:16:59 +0100790 ret = true;
791
792out:
793 mpool_fini(&local_page_pool);
794 return ret;
795}
796
J-Alves5952d942022-12-22 16:03:00 +0000797static bool is_memory_range_within(ipaddr_t begin, ipaddr_t end,
798 ipaddr_t in_begin, ipaddr_t in_end)
799{
800 return (ipa_addr(begin) >= ipa_addr(in_begin) &&
801 ipa_addr(begin) < ipa_addr(in_end)) ||
802 (ipa_addr(end) <= ipa_addr(in_end) &&
803 ipa_addr(end) > ipa_addr(in_begin));
804}
805
806/**
807 * Receives a memory range and looks for overlaps with the remainder
808 * constituents of the memory share/lend/donate operation. Assumes they are
809 * passed in order to avoid having to loop over all the elements at each call.
810 * The function only compares the received memory ranges with those that follow
811 * within the same fragment, and subsequent fragments from the same operation.
812 */
813static bool ffa_memory_check_overlap(
814 struct ffa_memory_region_constituent **fragments,
815 const uint32_t *fragment_constituent_counts,
816 const uint32_t fragment_count, const uint32_t current_fragment,
817 const uint32_t current_constituent)
818{
819 uint32_t i = current_fragment;
820 uint32_t j = current_constituent;
821 ipaddr_t current_begin = ipa_init(fragments[i][j].address);
822 const uint32_t current_page_count = fragments[i][j].page_count;
823 size_t current_size = current_page_count * PAGE_SIZE;
824 ipaddr_t current_end = ipa_add(current_begin, current_size - 1);
825
826 if (current_size == 0 ||
827 current_size > UINT64_MAX - ipa_addr(current_begin)) {
828 dlog_verbose("Invalid page count. Addr: %x page_count: %x\n",
829 current_begin, current_page_count);
830 return false;
831 }
832
833 for (; i < fragment_count; i++) {
834 j = (i == current_fragment) ? j + 1 : 0;
835
836 for (; j < fragment_constituent_counts[i]; j++) {
837 ipaddr_t begin = ipa_init(fragments[i][j].address);
838 const uint32_t page_count = fragments[i][j].page_count;
839 size_t size = page_count * PAGE_SIZE;
840 ipaddr_t end = ipa_add(begin, size - 1);
841
842 if (size == 0 || size > UINT64_MAX - ipa_addr(begin)) {
843 dlog_verbose(
844 "Invalid page count. Addr: %x "
845 "page_count: %x\n",
846 begin, page_count);
847 return false;
848 }
849
850 /*
851 * Check if current ranges is within begin and end, as
852 * well as the reverse. This should help optimize the
853 * loop, and reduce the number of iterations.
854 */
855 if (is_memory_range_within(begin, end, current_begin,
856 current_end) ||
857 is_memory_range_within(current_begin, current_end,
858 begin, end)) {
859 dlog_verbose(
860 "Overlapping memory ranges: %#x - %#x "
861 "with %#x - %#x\n",
862 ipa_addr(begin), ipa_addr(end),
863 ipa_addr(current_begin),
864 ipa_addr(current_end));
865 return true;
866 }
867 }
868 }
869
870 return false;
871}
872
Jose Marinho09b1db82019-08-08 09:16:59 +0100873/**
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000874 * Validates and prepares memory to be sent from the calling VM to another.
Jose Marinho09b1db82019-08-08 09:16:59 +0100875 *
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000876 * This function requires the calling context to hold the <from> VM lock.
Jose Marinho09b1db82019-08-08 09:16:59 +0100877 *
878 * Returns:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000879 * In case of error, one of the following values is returned:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100880 * 1) FFA_INVALID_PARAMETERS - The endpoint provided parameters were
Jose Marinho09b1db82019-08-08 09:16:59 +0100881 * erroneous;
Andrew Walbranf07f04d2020-05-01 18:09:00 +0100882 * 2) FFA_NO_MEMORY - Hafnium did not have sufficient memory to complete the
883 * request.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100884 * 3) FFA_DENIED - The sender doesn't have sufficient access to send the
Andrew Walbrana65a1322020-04-06 19:32:32 +0100885 * memory with the given permissions.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100886 * Success is indicated by FFA_SUCCESS.
Jose Marinho09b1db82019-08-08 09:16:59 +0100887 */
J-Alves66652252022-07-06 09:49:51 +0100888struct ffa_value ffa_send_check_update(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000889 struct vm_locked from_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +0100890 struct ffa_memory_region_constituent **fragments,
891 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
J-Alves8f11cde2022-12-21 16:18:22 +0000892 uint32_t composite_total_page_count, uint32_t share_func,
893 struct ffa_memory_access *receivers, uint32_t receivers_count,
894 struct mpool *page_pool, bool clear, uint32_t *orig_from_mode_ret)
Jose Marinho09b1db82019-08-08 09:16:59 +0100895{
Andrew Walbranca808b12020-05-15 17:22:28 +0100896 uint32_t i;
J-Alves8f11cde2022-12-21 16:18:22 +0000897 uint32_t j;
Jose Marinho09b1db82019-08-08 09:16:59 +0100898 uint32_t orig_from_mode;
899 uint32_t from_mode;
Jose Marinho09b1db82019-08-08 09:16:59 +0100900 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100901 struct ffa_value ret;
J-Alves8f11cde2022-12-21 16:18:22 +0000902 uint32_t constituents_total_page_count = 0;
Jose Marinho09b1db82019-08-08 09:16:59 +0100903
904 /*
Andrew Walbrana65a1322020-04-06 19:32:32 +0100905 * Make sure constituents are properly aligned to a 64-bit boundary. If
906 * not we would get alignment faults trying to read (64-bit) values.
Jose Marinho09b1db82019-08-08 09:16:59 +0100907 */
Andrew Walbranca808b12020-05-15 17:22:28 +0100908 for (i = 0; i < fragment_count; ++i) {
909 if (!is_aligned(fragments[i], 8)) {
910 dlog_verbose("Constituents not aligned.\n");
911 return ffa_error(FFA_INVALID_PARAMETERS);
912 }
J-Alves8f11cde2022-12-21 16:18:22 +0000913 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
914 constituents_total_page_count +=
915 fragments[i][j].page_count;
J-Alves5952d942022-12-22 16:03:00 +0000916 if (ffa_memory_check_overlap(
917 fragments, fragment_constituent_counts,
918 fragment_count, i, j)) {
919 return ffa_error(FFA_INVALID_PARAMETERS);
920 }
J-Alves8f11cde2022-12-21 16:18:22 +0000921 }
922 }
923
924 if (constituents_total_page_count != composite_total_page_count) {
925 dlog_verbose(
926 "Composite page count differs from calculated page "
927 "count from constituents.\n");
928 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho09b1db82019-08-08 09:16:59 +0100929 }
930
931 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000932 * Check if the state transition is lawful for the sender, ensure that
933 * all constituents of a memory region being shared are at the same
934 * state.
Jose Marinho09b1db82019-08-08 09:16:59 +0100935 */
J-Alves363f5722022-04-25 17:37:37 +0100936 ret = ffa_send_check_transition(from_locked, share_func, receivers,
937 receivers_count, &orig_from_mode,
938 fragments, fragment_constituent_counts,
Andrew Walbranca808b12020-05-15 17:22:28 +0100939 fragment_count, &from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100940 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100941 dlog_verbose("Invalid transition for send.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100942 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +0100943 }
944
Andrew Walbran37c574e2020-06-03 11:45:46 +0100945 if (orig_from_mode_ret != NULL) {
946 *orig_from_mode_ret = orig_from_mode;
947 }
948
Jose Marinho09b1db82019-08-08 09:16:59 +0100949 /*
950 * Create a local pool so any freed memory can't be used by another
951 * thread. This is to ensure the original mapping can be restored if the
952 * clear fails.
953 */
Andrew Walbran475c1452020-02-07 13:22:22 +0000954 mpool_init_with_fallback(&local_page_pool, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +0100955
956 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000957 * First reserve all required memory for the new page table entries
958 * without committing, to make sure the entire operation will succeed
959 * without exhausting the page pool.
Jose Marinho09b1db82019-08-08 09:16:59 +0100960 */
Andrew Walbranca808b12020-05-15 17:22:28 +0100961 if (!ffa_region_group_identity_map(
962 from_locked, fragments, fragment_constituent_counts,
963 fragment_count, from_mode, page_pool, false)) {
Jose Marinho09b1db82019-08-08 09:16:59 +0100964 /* TODO: partial defrag of failed range. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100965 ret = ffa_error(FFA_NO_MEMORY);
Jose Marinho09b1db82019-08-08 09:16:59 +0100966 goto out;
967 }
968
969 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000970 * Update the mapping for the sender. This won't allocate because the
971 * transaction was already prepared above, but may free pages in the
972 * case that a whole block is being unmapped that was previously
973 * partially mapped.
Jose Marinho09b1db82019-08-08 09:16:59 +0100974 */
Andrew Walbranca808b12020-05-15 17:22:28 +0100975 CHECK(ffa_region_group_identity_map(
976 from_locked, fragments, fragment_constituent_counts,
977 fragment_count, from_mode, &local_page_pool, true));
Jose Marinho09b1db82019-08-08 09:16:59 +0100978
979 /* Clear the memory so no VM or device can see the previous contents. */
J-Alves7db32002021-12-14 14:44:50 +0000980 if (clear &&
981 !ffa_clear_memory_constituents(
982 plat_ffa_owner_world_mode(from_locked.vm->id), fragments,
983 fragment_constituent_counts, fragment_count, page_pool)) {
Jose Marinho09b1db82019-08-08 09:16:59 +0100984 /*
985 * On failure, roll back by returning memory to the sender. This
986 * may allocate pages which were previously freed into
987 * `local_page_pool` by the call above, but will never allocate
988 * more pages than that so can never fail.
989 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100990 CHECK(ffa_region_group_identity_map(
Andrew Walbranca808b12020-05-15 17:22:28 +0100991 from_locked, fragments, fragment_constituent_counts,
992 fragment_count, orig_from_mode, &local_page_pool,
993 true));
Jose Marinho09b1db82019-08-08 09:16:59 +0100994
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100995 ret = ffa_error(FFA_NO_MEMORY);
Jose Marinho09b1db82019-08-08 09:16:59 +0100996 goto out;
997 }
998
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100999 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001000
1001out:
1002 mpool_fini(&local_page_pool);
1003
1004 /*
1005 * Tidy up the page table by reclaiming failed mappings (if there was an
1006 * error) or merging entries into blocks where possible (on success).
1007 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001008 vm_ptable_defrag(from_locked, page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001009
1010 return ret;
1011}
1012
1013/**
1014 * Validates and maps memory shared from one VM to another.
1015 *
1016 * This function requires the calling context to hold the <to> lock.
1017 *
1018 * Returns:
1019 * In case of error, one of the following values is returned:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001020 * 1) FFA_INVALID_PARAMETERS - The endpoint provided parameters were
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001021 * erroneous;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001022 * 2) FFA_NO_MEMORY - Hafnium did not have sufficient memory to complete
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001023 * the request.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001024 * Success is indicated by FFA_SUCCESS.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001025 */
J-Alvesb5084cf2022-07-06 14:20:12 +01001026struct ffa_value ffa_retrieve_check_update(
J-Alves19e20cf2023-08-02 12:48:55 +01001027 struct vm_locked to_locked, ffa_id_t from_id,
Andrew Walbranca808b12020-05-15 17:22:28 +01001028 struct ffa_memory_region_constituent **fragments,
1029 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
1030 uint32_t memory_to_attributes, uint32_t share_func, bool clear,
1031 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001032{
Andrew Walbranca808b12020-05-15 17:22:28 +01001033 uint32_t i;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001034 uint32_t to_mode;
1035 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001036 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001037
1038 /*
Andrew Walbranca808b12020-05-15 17:22:28 +01001039 * Make sure constituents are properly aligned to a 64-bit boundary. If
1040 * not we would get alignment faults trying to read (64-bit) values.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001041 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001042 for (i = 0; i < fragment_count; ++i) {
1043 if (!is_aligned(fragments[i], 8)) {
J-Alvesb5084cf2022-07-06 14:20:12 +01001044 dlog_verbose("Fragment not properly aligned.\n");
Andrew Walbranca808b12020-05-15 17:22:28 +01001045 return ffa_error(FFA_INVALID_PARAMETERS);
1046 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001047 }
1048
1049 /*
1050 * Check if the state transition is lawful for the recipient, and ensure
1051 * that all constituents of the memory region being retrieved are at the
1052 * same state.
1053 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001054 ret = ffa_retrieve_check_transition(
1055 to_locked, share_func, fragments, fragment_constituent_counts,
1056 fragment_count, memory_to_attributes, &to_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001057 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001058 dlog_verbose("Invalid transition for retrieve.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +01001059 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001060 }
1061
1062 /*
1063 * Create a local pool so any freed memory can't be used by another
1064 * thread. This is to ensure the original mapping can be restored if the
1065 * clear fails.
1066 */
1067 mpool_init_with_fallback(&local_page_pool, page_pool);
1068
1069 /*
1070 * First reserve all required memory for the new page table entries in
1071 * the recipient page tables without committing, to make sure the entire
1072 * operation will succeed without exhausting the page pool.
1073 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001074 if (!ffa_region_group_identity_map(
1075 to_locked, fragments, fragment_constituent_counts,
1076 fragment_count, to_mode, page_pool, false)) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001077 /* TODO: partial defrag of failed range. */
1078 dlog_verbose(
1079 "Insufficient memory to update recipient page "
1080 "table.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001081 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001082 goto out;
1083 }
1084
1085 /* Clear the memory so no VM or device can see the previous contents. */
J-Alves7db32002021-12-14 14:44:50 +00001086 if (clear &&
1087 !ffa_clear_memory_constituents(
1088 plat_ffa_owner_world_mode(from_id), fragments,
1089 fragment_constituent_counts, fragment_count, page_pool)) {
J-Alvesb5084cf2022-07-06 14:20:12 +01001090 dlog_verbose("Couldn't clear constituents.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001091 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001092 goto out;
1093 }
1094
Jose Marinho09b1db82019-08-08 09:16:59 +01001095 /*
1096 * Complete the transfer by mapping the memory into the recipient. This
1097 * won't allocate because the transaction was already prepared above, so
1098 * it doesn't need to use the `local_page_pool`.
1099 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001100 CHECK(ffa_region_group_identity_map(
1101 to_locked, fragments, fragment_constituent_counts,
1102 fragment_count, to_mode, page_pool, true));
Jose Marinho09b1db82019-08-08 09:16:59 +01001103
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001104 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Jose Marinho09b1db82019-08-08 09:16:59 +01001105
1106out:
1107 mpool_fini(&local_page_pool);
1108
1109 /*
Andrew Walbranf07f04d2020-05-01 18:09:00 +01001110 * Tidy up the page table by reclaiming failed mappings (if there was an
1111 * error) or merging entries into blocks where possible (on success).
Jose Marinho09b1db82019-08-08 09:16:59 +01001112 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001113 vm_ptable_defrag(to_locked, page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001114
1115 return ret;
1116}
1117
Andrew Walbran996d1d12020-05-27 14:08:43 +01001118static struct ffa_value ffa_relinquish_check_update(
J-Alves19e20cf2023-08-02 12:48:55 +01001119 struct vm_locked from_locked, ffa_id_t owner_id,
Andrew Walbranca808b12020-05-15 17:22:28 +01001120 struct ffa_memory_region_constituent **fragments,
1121 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
1122 struct mpool *page_pool, bool clear)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001123{
1124 uint32_t orig_from_mode;
1125 uint32_t from_mode;
1126 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001127 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001128
Andrew Walbranca808b12020-05-15 17:22:28 +01001129 ret = ffa_relinquish_check_transition(
1130 from_locked, &orig_from_mode, fragments,
1131 fragment_constituent_counts, fragment_count, &from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001132 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001133 dlog_verbose("Invalid transition for relinquish.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +01001134 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001135 }
1136
1137 /*
1138 * Create a local pool so any freed memory can't be used by another
1139 * thread. This is to ensure the original mapping can be restored if the
1140 * clear fails.
1141 */
1142 mpool_init_with_fallback(&local_page_pool, page_pool);
1143
1144 /*
1145 * First reserve all required memory for the new page table entries
1146 * without committing, to make sure the entire operation will succeed
1147 * without exhausting the page pool.
1148 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001149 if (!ffa_region_group_identity_map(
1150 from_locked, fragments, fragment_constituent_counts,
1151 fragment_count, from_mode, page_pool, false)) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001152 /* TODO: partial defrag of failed range. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001153 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001154 goto out;
1155 }
1156
1157 /*
1158 * Update the mapping for the sender. This won't allocate because the
1159 * transaction was already prepared above, but may free pages in the
1160 * case that a whole block is being unmapped that was previously
1161 * partially mapped.
1162 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001163 CHECK(ffa_region_group_identity_map(
1164 from_locked, fragments, fragment_constituent_counts,
1165 fragment_count, from_mode, &local_page_pool, true));
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001166
1167 /* Clear the memory so no VM or device can see the previous contents. */
J-Alves7db32002021-12-14 14:44:50 +00001168 if (clear &&
1169 !ffa_clear_memory_constituents(
J-Alves3c5b2072022-11-21 12:45:40 +00001170 plat_ffa_owner_world_mode(owner_id), fragments,
J-Alves7db32002021-12-14 14:44:50 +00001171 fragment_constituent_counts, fragment_count, page_pool)) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001172 /*
1173 * On failure, roll back by returning memory to the sender. This
1174 * may allocate pages which were previously freed into
1175 * `local_page_pool` by the call above, but will never allocate
1176 * more pages than that so can never fail.
1177 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001178 CHECK(ffa_region_group_identity_map(
Andrew Walbranca808b12020-05-15 17:22:28 +01001179 from_locked, fragments, fragment_constituent_counts,
1180 fragment_count, orig_from_mode, &local_page_pool,
1181 true));
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001182
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001183 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001184 goto out;
1185 }
1186
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001187 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001188
1189out:
1190 mpool_fini(&local_page_pool);
1191
1192 /*
1193 * Tidy up the page table by reclaiming failed mappings (if there was an
1194 * error) or merging entries into blocks where possible (on success).
1195 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001196 vm_ptable_defrag(from_locked, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +01001197
1198 return ret;
1199}
1200
1201/**
Andrew Walbranca808b12020-05-15 17:22:28 +01001202 * Complete a memory sending operation by checking that it is valid, updating
1203 * the sender page table, and then either marking the share state as having
1204 * completed sending (on success) or freeing it (on failure).
1205 *
1206 * Returns FFA_SUCCESS with the handle encoded, or the relevant FFA_ERROR.
1207 */
J-Alvesfdd29272022-07-19 13:16:31 +01001208struct ffa_value ffa_memory_send_complete(
Andrew Walbranca808b12020-05-15 17:22:28 +01001209 struct vm_locked from_locked, struct share_states_locked share_states,
Andrew Walbran37c574e2020-06-03 11:45:46 +01001210 struct ffa_memory_share_state *share_state, struct mpool *page_pool,
1211 uint32_t *orig_from_mode_ret)
Andrew Walbranca808b12020-05-15 17:22:28 +01001212{
1213 struct ffa_memory_region *memory_region = share_state->memory_region;
J-Alves8f11cde2022-12-21 16:18:22 +00001214 struct ffa_composite_memory_region *composite;
Andrew Walbranca808b12020-05-15 17:22:28 +01001215 struct ffa_value ret;
1216
1217 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +00001218 assert(share_states.share_states != NULL);
J-Alves8f11cde2022-12-21 16:18:22 +00001219 assert(memory_region != NULL);
1220 composite = ffa_memory_region_get_composite(memory_region, 0);
1221 assert(composite != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +01001222
1223 /* Check that state is valid in sender page table and update. */
1224 ret = ffa_send_check_update(
1225 from_locked, share_state->fragments,
1226 share_state->fragment_constituent_counts,
J-Alves8f11cde2022-12-21 16:18:22 +00001227 share_state->fragment_count, composite->page_count,
1228 share_state->share_func, memory_region->receivers,
1229 memory_region->receiver_count, page_pool,
1230 memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR,
Andrew Walbran37c574e2020-06-03 11:45:46 +01001231 orig_from_mode_ret);
Andrew Walbranca808b12020-05-15 17:22:28 +01001232 if (ret.func != FFA_SUCCESS_32) {
1233 /*
1234 * Free share state, it failed to send so it can't be retrieved.
1235 */
Karl Meakin4cec5e82023-06-30 16:30:22 +01001236 dlog_verbose("%s: failed to send check update: %s(%s)\n",
1237 __func__, ffa_func_name(ret.func),
1238 ffa_error_name(ffa_error_code(ret)));
Andrew Walbranca808b12020-05-15 17:22:28 +01001239 share_state_free(share_states, share_state, page_pool);
1240 return ret;
1241 }
1242
1243 share_state->sending_complete = true;
Karl Meakin4cec5e82023-06-30 16:30:22 +01001244 dlog_verbose("%s: marked sending complete.\n", __func__);
Andrew Walbranca808b12020-05-15 17:22:28 +01001245
J-Alvesee68c542020-10-29 17:48:20 +00001246 return ffa_mem_success(share_state->memory_region->handle);
Andrew Walbranca808b12020-05-15 17:22:28 +01001247}
1248
1249/**
Federico Recanatia98603a2021-12-20 18:04:03 +01001250 * Check that the memory attributes match Hafnium expectations:
1251 * Normal Memory, Inner shareable, Write-Back Read-Allocate
1252 * Write-Allocate Cacheable.
1253 */
1254static struct ffa_value ffa_memory_attributes_validate(
J-Alves7a99d0d2023-02-08 13:49:48 +00001255 ffa_memory_attributes_t attributes)
Federico Recanatia98603a2021-12-20 18:04:03 +01001256{
1257 enum ffa_memory_type memory_type;
1258 enum ffa_memory_cacheability cacheability;
1259 enum ffa_memory_shareability shareability;
1260
1261 memory_type = ffa_get_memory_type_attr(attributes);
1262 if (memory_type != FFA_MEMORY_NORMAL_MEM) {
1263 dlog_verbose("Invalid memory type %#x, expected %#x.\n",
1264 memory_type, FFA_MEMORY_NORMAL_MEM);
Federico Recanati3d953f32022-02-17 09:31:29 +01001265 return ffa_error(FFA_DENIED);
Federico Recanatia98603a2021-12-20 18:04:03 +01001266 }
1267
1268 cacheability = ffa_get_memory_cacheability_attr(attributes);
1269 if (cacheability != FFA_MEMORY_CACHE_WRITE_BACK) {
1270 dlog_verbose("Invalid cacheability %#x, expected %#x.\n",
1271 cacheability, FFA_MEMORY_CACHE_WRITE_BACK);
Federico Recanati3d953f32022-02-17 09:31:29 +01001272 return ffa_error(FFA_DENIED);
Federico Recanatia98603a2021-12-20 18:04:03 +01001273 }
1274
1275 shareability = ffa_get_memory_shareability_attr(attributes);
1276 if (shareability != FFA_MEMORY_INNER_SHAREABLE) {
1277 dlog_verbose("Invalid shareability %#x, expected #%x.\n",
1278 shareability, FFA_MEMORY_INNER_SHAREABLE);
Federico Recanati3d953f32022-02-17 09:31:29 +01001279 return ffa_error(FFA_DENIED);
Federico Recanatia98603a2021-12-20 18:04:03 +01001280 }
1281
1282 return (struct ffa_value){.func = FFA_SUCCESS_32};
1283}
1284
1285/**
Andrew Walbrana65a1322020-04-06 19:32:32 +01001286 * Check that the given `memory_region` represents a valid memory send request
1287 * of the given `share_func` type, return the clear flag and permissions via the
1288 * respective output parameters, and update the permissions if necessary.
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001289 *
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001290 * Returns FFA_SUCCESS if the request was valid, or the relevant FFA_ERROR if
Andrew Walbrana65a1322020-04-06 19:32:32 +01001291 * not.
1292 */
J-Alves66652252022-07-06 09:49:51 +01001293struct ffa_value ffa_memory_send_validate(
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001294 struct vm_locked from_locked, struct ffa_memory_region *memory_region,
1295 uint32_t memory_share_length, uint32_t fragment_length,
J-Alves363f5722022-04-25 17:37:37 +01001296 uint32_t share_func)
Andrew Walbrana65a1322020-04-06 19:32:32 +01001297{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001298 struct ffa_composite_memory_region *composite;
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001299 uint64_t receivers_end;
1300 uint64_t min_length;
Federico Recanati872cd692022-01-05 13:10:10 +01001301 uint32_t composite_memory_region_offset;
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001302 uint32_t constituents_start;
Andrew Walbran130a8ae2020-05-15 16:27:15 +01001303 uint32_t constituents_length;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001304 enum ffa_data_access data_access;
1305 enum ffa_instruction_access instruction_access;
Olivier Deprez4342a3c2022-02-28 09:37:25 +01001306 enum ffa_memory_security security_state;
Federico Recanatia98603a2021-12-20 18:04:03 +01001307 struct ffa_value ret;
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001308 const size_t minimum_first_fragment_length =
1309 (sizeof(struct ffa_memory_region) +
1310 sizeof(struct ffa_memory_access) +
1311 sizeof(struct ffa_composite_memory_region));
1312
1313 if (fragment_length < minimum_first_fragment_length) {
1314 dlog_verbose("Fragment length %u too short (min %u).\n",
1315 (size_t)fragment_length,
1316 minimum_first_fragment_length);
1317 return ffa_error(FFA_INVALID_PARAMETERS);
1318 }
1319
Demi Marie Obenour73a1e942023-02-04 14:09:18 -05001320 static_assert(sizeof(struct ffa_memory_region_constituent) == 16,
1321 "struct ffa_memory_region_constituent must be 16 bytes");
1322 if (!is_aligned(fragment_length,
1323 sizeof(struct ffa_memory_region_constituent)) ||
1324 !is_aligned(memory_share_length,
1325 sizeof(struct ffa_memory_region_constituent))) {
1326 dlog_verbose(
1327 "Fragment length %u or total length %u"
1328 " is not 16-byte aligned.\n",
1329 fragment_length, memory_share_length);
1330 return ffa_error(FFA_INVALID_PARAMETERS);
1331 }
1332
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001333 if (fragment_length > memory_share_length) {
1334 dlog_verbose(
1335 "Fragment length %u greater than total length %u.\n",
1336 (size_t)fragment_length, (size_t)memory_share_length);
1337 return ffa_error(FFA_INVALID_PARAMETERS);
1338 }
Andrew Walbrana65a1322020-04-06 19:32:32 +01001339
J-Alves0b6653d2022-04-22 13:17:38 +01001340 assert(memory_region->receivers_offset ==
1341 offsetof(struct ffa_memory_region, receivers));
1342 assert(memory_region->memory_access_desc_size ==
1343 sizeof(struct ffa_memory_access));
1344
J-Alves95df0ef2022-12-07 10:09:48 +00001345 /* The sender must match the caller. */
1346 if ((!vm_id_is_current_world(from_locked.vm->id) &&
1347 vm_id_is_current_world(memory_region->sender)) ||
1348 (vm_id_is_current_world(from_locked.vm->id) &&
1349 memory_region->sender != from_locked.vm->id)) {
1350 dlog_verbose("Invalid memory sender ID.\n");
1351 return ffa_error(FFA_DENIED);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001352 }
1353
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001354 if (memory_region->receiver_count <= 0) {
1355 dlog_verbose("No receivers!\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001356 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001357 }
1358
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001359 /*
1360 * Ensure that the composite header is within the memory bounds and
1361 * doesn't overlap the first part of the message. Cast to uint64_t
1362 * to prevent overflow.
1363 */
1364 receivers_end = ((uint64_t)sizeof(struct ffa_memory_access) *
1365 (uint64_t)memory_region->receiver_count) +
1366 sizeof(struct ffa_memory_region);
1367 min_length = receivers_end +
1368 sizeof(struct ffa_composite_memory_region) +
1369 sizeof(struct ffa_memory_region_constituent);
1370 if (min_length > memory_share_length) {
1371 dlog_verbose("Share too short: got %u but minimum is %u.\n",
1372 (size_t)memory_share_length, (size_t)min_length);
1373 return ffa_error(FFA_INVALID_PARAMETERS);
1374 }
1375
1376 composite_memory_region_offset =
1377 memory_region->receivers[0].composite_memory_region_offset;
Andrew Walbrana65a1322020-04-06 19:32:32 +01001378
1379 /*
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001380 * Check that the composite memory region descriptor is after the access
1381 * descriptors, is at least 16-byte aligned, and fits in the first
1382 * fragment.
Andrew Walbrana65a1322020-04-06 19:32:32 +01001383 */
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001384 if ((composite_memory_region_offset < receivers_end) ||
1385 (composite_memory_region_offset % 16 != 0) ||
1386 (composite_memory_region_offset >
1387 fragment_length - sizeof(struct ffa_composite_memory_region))) {
1388 dlog_verbose(
1389 "Invalid composite memory region descriptor offset "
1390 "%u.\n",
1391 (size_t)composite_memory_region_offset);
1392 return ffa_error(FFA_INVALID_PARAMETERS);
1393 }
1394
1395 /*
1396 * Compute the start of the constituent regions. Already checked
1397 * to be not more than fragment_length and thus not more than
1398 * memory_share_length.
1399 */
1400 constituents_start = composite_memory_region_offset +
1401 sizeof(struct ffa_composite_memory_region);
1402 constituents_length = memory_share_length - constituents_start;
1403
1404 /*
1405 * Check that the number of constituents is consistent with the length
1406 * of the constituent region.
1407 */
1408 composite = ffa_memory_region_get_composite(memory_region, 0);
1409 if ((constituents_length %
1410 sizeof(struct ffa_memory_region_constituent) !=
1411 0) ||
1412 ((constituents_length /
1413 sizeof(struct ffa_memory_region_constituent)) !=
1414 composite->constituent_count)) {
1415 dlog_verbose("Invalid length %u or composite offset %u.\n",
1416 (size_t)memory_share_length,
1417 (size_t)composite_memory_region_offset);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001418 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001419 }
Andrew Walbranca808b12020-05-15 17:22:28 +01001420 if (fragment_length < memory_share_length &&
1421 fragment_length < HF_MAILBOX_SIZE) {
1422 dlog_warning(
1423 "Initial fragment length %d smaller than mailbox "
1424 "size.\n",
1425 fragment_length);
1426 }
Andrew Walbrana65a1322020-04-06 19:32:32 +01001427
Andrew Walbrana65a1322020-04-06 19:32:32 +01001428 /*
1429 * Clear is not allowed for memory sharing, as the sender still has
1430 * access to the memory.
1431 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001432 if ((memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR) &&
1433 share_func == FFA_MEM_SHARE_32) {
Andrew Walbrana65a1322020-04-06 19:32:32 +01001434 dlog_verbose("Memory can't be cleared while being shared.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001435 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001436 }
1437
1438 /* No other flags are allowed/supported here. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001439 if (memory_region->flags & ~FFA_MEMORY_REGION_FLAG_CLEAR) {
Andrew Walbrana65a1322020-04-06 19:32:32 +01001440 dlog_verbose("Invalid flags %#x.\n", memory_region->flags);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001441 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001442 }
1443
J-Alves363f5722022-04-25 17:37:37 +01001444 /* Check that the permissions are valid, for each specified receiver. */
1445 for (uint32_t i = 0U; i < memory_region->receiver_count; i++) {
1446 ffa_memory_access_permissions_t permissions =
1447 memory_region->receivers[i]
1448 .receiver_permissions.permissions;
J-Alves19e20cf2023-08-02 12:48:55 +01001449 ffa_id_t receiver_id = memory_region->receivers[i]
1450 .receiver_permissions.receiver;
J-Alves363f5722022-04-25 17:37:37 +01001451
1452 if (memory_region->sender == receiver_id) {
1453 dlog_verbose("Can't share memory with itself.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001454 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001455 }
Federico Recanati85090c42021-12-15 13:17:54 +01001456
J-Alves363f5722022-04-25 17:37:37 +01001457 for (uint32_t j = i + 1; j < memory_region->receiver_count;
1458 j++) {
1459 if (receiver_id ==
1460 memory_region->receivers[j]
1461 .receiver_permissions.receiver) {
1462 dlog_verbose(
1463 "Repeated receiver(%x) in memory send "
1464 "operation.\n",
1465 memory_region->receivers[j]
1466 .receiver_permissions.receiver);
1467 return ffa_error(FFA_INVALID_PARAMETERS);
1468 }
1469 }
1470
1471 if (composite_memory_region_offset !=
1472 memory_region->receivers[i]
1473 .composite_memory_region_offset) {
1474 dlog_verbose(
1475 "All ffa_memory_access should point to the "
1476 "same composite memory region offset.\n");
1477 return ffa_error(FFA_INVALID_PARAMETERS);
1478 }
1479
1480 data_access = ffa_get_data_access_attr(permissions);
1481 instruction_access =
1482 ffa_get_instruction_access_attr(permissions);
1483 if (data_access == FFA_DATA_ACCESS_RESERVED ||
1484 instruction_access == FFA_INSTRUCTION_ACCESS_RESERVED) {
1485 dlog_verbose(
1486 "Reserved value for receiver permissions "
1487 "%#x.\n",
1488 permissions);
1489 return ffa_error(FFA_INVALID_PARAMETERS);
1490 }
1491 if (instruction_access !=
1492 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
1493 dlog_verbose(
1494 "Invalid instruction access permissions %#x "
1495 "for sending memory.\n",
1496 permissions);
1497 return ffa_error(FFA_INVALID_PARAMETERS);
1498 }
1499 if (share_func == FFA_MEM_SHARE_32) {
1500 if (data_access == FFA_DATA_ACCESS_NOT_SPECIFIED) {
1501 dlog_verbose(
1502 "Invalid data access permissions %#x "
1503 "for sharing memory.\n",
1504 permissions);
1505 return ffa_error(FFA_INVALID_PARAMETERS);
1506 }
J-Alves363f5722022-04-25 17:37:37 +01001507 }
1508 if (share_func == FFA_MEM_LEND_32 &&
1509 data_access == FFA_DATA_ACCESS_NOT_SPECIFIED) {
1510 dlog_verbose(
1511 "Invalid data access permissions %#x for "
1512 "lending memory.\n",
1513 permissions);
1514 return ffa_error(FFA_INVALID_PARAMETERS);
1515 }
1516
1517 if (share_func == FFA_MEM_DONATE_32 &&
1518 data_access != FFA_DATA_ACCESS_NOT_SPECIFIED) {
1519 dlog_verbose(
1520 "Invalid data access permissions %#x for "
1521 "donating memory.\n",
1522 permissions);
1523 return ffa_error(FFA_INVALID_PARAMETERS);
1524 }
Andrew Walbrana65a1322020-04-06 19:32:32 +01001525 }
1526
Olivier Deprez4342a3c2022-02-28 09:37:25 +01001527 /* Memory region attributes NS-Bit MBZ for FFA_MEM_SHARE/LEND/DONATE. */
1528 security_state =
1529 ffa_get_memory_security_attr(memory_region->attributes);
1530 if (security_state != FFA_MEMORY_SECURITY_UNSPECIFIED) {
1531 dlog_verbose(
1532 "Invalid security state for memory share operation.\n");
1533 return ffa_error(FFA_INVALID_PARAMETERS);
1534 }
1535
Federico Recanatid937f5e2021-12-20 17:38:23 +01001536 /*
J-Alves807794e2022-06-16 13:42:47 +01001537 * If a memory donate or lend with single borrower, the memory type
1538 * shall not be specified by the sender.
Federico Recanatid937f5e2021-12-20 17:38:23 +01001539 */
J-Alves807794e2022-06-16 13:42:47 +01001540 if (share_func == FFA_MEM_DONATE_32 ||
1541 (share_func == FFA_MEM_LEND_32 &&
1542 memory_region->receiver_count == 1)) {
1543 if (ffa_get_memory_type_attr(memory_region->attributes) !=
1544 FFA_MEMORY_NOT_SPECIFIED_MEM) {
1545 dlog_verbose(
1546 "Memory type shall not be specified by "
1547 "sender.\n");
1548 return ffa_error(FFA_INVALID_PARAMETERS);
1549 }
1550 } else {
1551 /*
1552 * Check that sender's memory attributes match Hafnium
1553 * expectations: Normal Memory, Inner shareable, Write-Back
1554 * Read-Allocate Write-Allocate Cacheable.
1555 */
1556 ret = ffa_memory_attributes_validate(memory_region->attributes);
1557 if (ret.func != FFA_SUCCESS_32) {
1558 return ret;
1559 }
Federico Recanatid937f5e2021-12-20 17:38:23 +01001560 }
1561
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001562 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbrana65a1322020-04-06 19:32:32 +01001563}
1564
1565/**
Andrew Walbranca808b12020-05-15 17:22:28 +01001566 * Gets the share state for continuing an operation to donate, lend or share
1567 * memory, and checks that it is a valid request.
1568 *
1569 * Returns FFA_SUCCESS if the request was valid, or the relevant FFA_ERROR if
1570 * not.
1571 */
J-Alvesfdd29272022-07-19 13:16:31 +01001572struct ffa_value ffa_memory_send_continue_validate(
Andrew Walbranca808b12020-05-15 17:22:28 +01001573 struct share_states_locked share_states, ffa_memory_handle_t handle,
J-Alves19e20cf2023-08-02 12:48:55 +01001574 struct ffa_memory_share_state **share_state_ret, ffa_id_t from_vm_id,
Andrew Walbranca808b12020-05-15 17:22:28 +01001575 struct mpool *page_pool)
1576{
1577 struct ffa_memory_share_state *share_state;
1578 struct ffa_memory_region *memory_region;
1579
Daniel Boulbya2f8c662021-11-26 17:52:53 +00001580 assert(share_state_ret != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +01001581
1582 /*
1583 * Look up the share state by handle and make sure that the VM ID
1584 * matches.
1585 */
Karl Meakin4a2854a2023-06-30 16:26:52 +01001586 share_state = get_share_state(share_states, handle);
1587 if (!share_state) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001588 dlog_verbose(
1589 "Invalid handle %#x for memory send continuation.\n",
1590 handle);
1591 return ffa_error(FFA_INVALID_PARAMETERS);
1592 }
1593 memory_region = share_state->memory_region;
1594
J-Alvesfdd29272022-07-19 13:16:31 +01001595 if (vm_id_is_current_world(from_vm_id) &&
1596 memory_region->sender != from_vm_id) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001597 dlog_verbose("Invalid sender %d.\n", memory_region->sender);
1598 return ffa_error(FFA_INVALID_PARAMETERS);
1599 }
1600
1601 if (share_state->sending_complete) {
1602 dlog_verbose(
1603 "Sending of memory handle %#x is already complete.\n",
1604 handle);
1605 return ffa_error(FFA_INVALID_PARAMETERS);
1606 }
1607
1608 if (share_state->fragment_count == MAX_FRAGMENTS) {
1609 /*
1610 * Log a warning as this is a sign that MAX_FRAGMENTS should
1611 * probably be increased.
1612 */
1613 dlog_warning(
1614 "Too many fragments for memory share with handle %#x; "
1615 "only %d supported.\n",
1616 handle, MAX_FRAGMENTS);
1617 /* Free share state, as it's not possible to complete it. */
1618 share_state_free(share_states, share_state, page_pool);
1619 return ffa_error(FFA_NO_MEMORY);
1620 }
1621
1622 *share_state_ret = share_state;
1623
1624 return (struct ffa_value){.func = FFA_SUCCESS_32};
1625}
1626
1627/**
J-Alves95df0ef2022-12-07 10:09:48 +00001628 * Checks if there is at least one receiver from the other world.
1629 */
J-Alvesfdd29272022-07-19 13:16:31 +01001630bool memory_region_receivers_from_other_world(
J-Alves95df0ef2022-12-07 10:09:48 +00001631 struct ffa_memory_region *memory_region)
1632{
1633 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
J-Alves19e20cf2023-08-02 12:48:55 +01001634 ffa_id_t receiver = memory_region->receivers[i]
1635 .receiver_permissions.receiver;
J-Alves95df0ef2022-12-07 10:09:48 +00001636 if (!vm_id_is_current_world(receiver)) {
1637 return true;
1638 }
1639 }
1640 return false;
1641}
1642
1643/**
J-Alves9da280b2022-12-21 14:55:39 +00001644 * Validates a call to donate, lend or share memory in which Hafnium is the
1645 * designated allocator of the memory handle. In practice, this also means
1646 * Hafnium is responsible for managing the state structures for the transaction.
1647 * If Hafnium is the SPMC, it should allocate the memory handle when either the
1648 * sender is an SP or there is at least one borrower that is an SP.
1649 * If Hafnium is the hypervisor, it should allocate the memory handle when
1650 * operation involves only NWd VMs.
1651 *
1652 * If validation goes well, Hafnium updates the stage-2 page tables of the
1653 * sender. Validation consists of checking if the message length and number of
1654 * memory region constituents match, and if the transition is valid for the
1655 * type of memory sending operation.
Andrew Walbran475c1452020-02-07 13:22:22 +00001656 *
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001657 * Assumes that the caller has already found and locked the sender VM and copied
1658 * the memory region descriptor from the sender's TX buffer to a freshly
1659 * allocated page from Hafnium's internal pool. The caller must have also
1660 * validated that the receiver VM ID is valid.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001661 *
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001662 * This function takes ownership of the `memory_region` passed in and will free
1663 * it when necessary; it must not be freed by the caller.
Jose Marinho09b1db82019-08-08 09:16:59 +01001664 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001665struct ffa_value ffa_memory_send(struct vm_locked from_locked,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001666 struct ffa_memory_region *memory_region,
Andrew Walbran130a8ae2020-05-15 16:27:15 +01001667 uint32_t memory_share_length,
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001668 uint32_t fragment_length, uint32_t share_func,
1669 struct mpool *page_pool)
Jose Marinho09b1db82019-08-08 09:16:59 +01001670{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001671 struct ffa_value ret;
Andrew Walbranca808b12020-05-15 17:22:28 +01001672 struct share_states_locked share_states;
1673 struct ffa_memory_share_state *share_state;
Jose Marinho09b1db82019-08-08 09:16:59 +01001674
1675 /*
Andrew Walbrana65a1322020-04-06 19:32:32 +01001676 * If there is an error validating the `memory_region` then we need to
1677 * free it because we own it but we won't be storing it in a share state
1678 * after all.
Jose Marinho09b1db82019-08-08 09:16:59 +01001679 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001680 ret = ffa_memory_send_validate(from_locked, memory_region,
1681 memory_share_length, fragment_length,
J-Alves363f5722022-04-25 17:37:37 +01001682 share_func);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001683 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001684 mpool_free(page_pool, memory_region);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001685 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01001686 }
1687
Andrew Walbrana65a1322020-04-06 19:32:32 +01001688 /* Set flag for share function, ready to be retrieved later. */
1689 switch (share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001690 case FFA_MEM_SHARE_32:
Andrew Walbrana65a1322020-04-06 19:32:32 +01001691 memory_region->flags |=
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001692 FFA_MEMORY_REGION_TRANSACTION_TYPE_SHARE;
Andrew Walbrana65a1322020-04-06 19:32:32 +01001693 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001694 case FFA_MEM_LEND_32:
1695 memory_region->flags |= FFA_MEMORY_REGION_TRANSACTION_TYPE_LEND;
Andrew Walbrana65a1322020-04-06 19:32:32 +01001696 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001697 case FFA_MEM_DONATE_32:
Andrew Walbrana65a1322020-04-06 19:32:32 +01001698 memory_region->flags |=
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001699 FFA_MEMORY_REGION_TRANSACTION_TYPE_DONATE;
Andrew Walbrana65a1322020-04-06 19:32:32 +01001700 break;
Jose Marinho09b1db82019-08-08 09:16:59 +01001701 }
1702
Andrew Walbranca808b12020-05-15 17:22:28 +01001703 share_states = share_states_lock();
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001704 /*
1705 * Allocate a share state before updating the page table. Otherwise if
1706 * updating the page table succeeded but allocating the share state
1707 * failed then it would leave the memory in a state where nobody could
1708 * get it back.
1709 */
Karl Meakin52cdfe72023-06-30 14:49:10 +01001710 share_state = allocate_share_state(share_states, share_func,
1711 memory_region, fragment_length,
1712 FFA_MEMORY_HANDLE_INVALID);
1713 if (!share_state) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001714 dlog_verbose("Failed to allocate share state.\n");
1715 mpool_free(page_pool, memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +01001716 ret = ffa_error(FFA_NO_MEMORY);
1717 goto out;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001718 }
1719
Andrew Walbranca808b12020-05-15 17:22:28 +01001720 if (fragment_length == memory_share_length) {
1721 /* No more fragments to come, everything fit in one message. */
J-Alves2a0d2882020-10-29 14:49:50 +00001722 ret = ffa_memory_send_complete(
1723 from_locked, share_states, share_state, page_pool,
1724 &(share_state->sender_orig_mode));
Andrew Walbranca808b12020-05-15 17:22:28 +01001725 } else {
J-Alvesfdd29272022-07-19 13:16:31 +01001726 /*
1727 * Use sender ID from 'memory_region' assuming
1728 * that at this point it has been validated:
1729 * - MBZ at virtual FF-A instance.
1730 */
J-Alves19e20cf2023-08-02 12:48:55 +01001731 ffa_id_t sender_to_ret =
J-Alvesfdd29272022-07-19 13:16:31 +01001732 (from_locked.vm->id == HF_OTHER_WORLD_ID)
1733 ? memory_region->sender
1734 : 0;
Andrew Walbranca808b12020-05-15 17:22:28 +01001735 ret = (struct ffa_value){
1736 .func = FFA_MEM_FRAG_RX_32,
J-Alvesee68c542020-10-29 17:48:20 +00001737 .arg1 = (uint32_t)memory_region->handle,
1738 .arg2 = (uint32_t)(memory_region->handle >> 32),
J-Alvesfdd29272022-07-19 13:16:31 +01001739 .arg3 = fragment_length,
1740 .arg4 = (uint32_t)(sender_to_ret & 0xffff) << 16};
Andrew Walbranca808b12020-05-15 17:22:28 +01001741 }
1742
1743out:
1744 share_states_unlock(&share_states);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001745 dump_share_states();
Andrew Walbranca808b12020-05-15 17:22:28 +01001746 return ret;
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001747}
1748
1749/**
J-Alves8505a8a2022-06-15 18:10:18 +01001750 * Continues an operation to donate, lend or share memory to a VM from current
1751 * world. If this is the last fragment then checks that the transition is valid
1752 * for the type of memory sending operation and updates the stage-2 page tables
1753 * of the sender.
Andrew Walbranca808b12020-05-15 17:22:28 +01001754 *
1755 * Assumes that the caller has already found and locked the sender VM and copied
1756 * the memory region descriptor from the sender's TX buffer to a freshly
1757 * allocated page from Hafnium's internal pool.
1758 *
1759 * This function takes ownership of the `fragment` passed in; it must not be
1760 * freed by the caller.
1761 */
1762struct ffa_value ffa_memory_send_continue(struct vm_locked from_locked,
1763 void *fragment,
1764 uint32_t fragment_length,
1765 ffa_memory_handle_t handle,
1766 struct mpool *page_pool)
1767{
1768 struct share_states_locked share_states = share_states_lock();
1769 struct ffa_memory_share_state *share_state;
1770 struct ffa_value ret;
1771 struct ffa_memory_region *memory_region;
1772
Demi Marie Obenour73a1e942023-02-04 14:09:18 -05001773 CHECK(is_aligned(fragment,
1774 alignof(struct ffa_memory_region_constituent)));
1775 if (fragment_length % sizeof(struct ffa_memory_region_constituent) !=
1776 0) {
1777 dlog_verbose("Fragment length %u misaligned.\n",
1778 fragment_length);
1779 ret = ffa_error(FFA_INVALID_PARAMETERS);
1780 goto out_free_fragment;
1781 }
1782
Andrew Walbranca808b12020-05-15 17:22:28 +01001783 ret = ffa_memory_send_continue_validate(share_states, handle,
1784 &share_state,
1785 from_locked.vm->id, page_pool);
1786 if (ret.func != FFA_SUCCESS_32) {
1787 goto out_free_fragment;
1788 }
1789 memory_region = share_state->memory_region;
1790
J-Alves95df0ef2022-12-07 10:09:48 +00001791 if (memory_region_receivers_from_other_world(memory_region)) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001792 dlog_error(
1793 "Got hypervisor-allocated handle for memory send to "
J-Alves8505a8a2022-06-15 18:10:18 +01001794 "other world. This should never happen, and indicates "
1795 "a bug in "
Andrew Walbranca808b12020-05-15 17:22:28 +01001796 "EL3 code.\n");
1797 ret = ffa_error(FFA_INVALID_PARAMETERS);
1798 goto out_free_fragment;
1799 }
1800
1801 /* Add this fragment. */
1802 share_state->fragments[share_state->fragment_count] = fragment;
1803 share_state->fragment_constituent_counts[share_state->fragment_count] =
1804 fragment_length / sizeof(struct ffa_memory_region_constituent);
1805 share_state->fragment_count++;
1806
1807 /* Check whether the memory send operation is now ready to complete. */
1808 if (share_state_sending_complete(share_states, share_state)) {
J-Alves2a0d2882020-10-29 14:49:50 +00001809 ret = ffa_memory_send_complete(
1810 from_locked, share_states, share_state, page_pool,
1811 &(share_state->sender_orig_mode));
Andrew Walbranca808b12020-05-15 17:22:28 +01001812 } else {
1813 ret = (struct ffa_value){
1814 .func = FFA_MEM_FRAG_RX_32,
1815 .arg1 = (uint32_t)handle,
1816 .arg2 = (uint32_t)(handle >> 32),
1817 .arg3 = share_state_next_fragment_offset(share_states,
1818 share_state)};
1819 }
1820 goto out;
1821
1822out_free_fragment:
1823 mpool_free(page_pool, fragment);
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001824
1825out:
Andrew Walbranca808b12020-05-15 17:22:28 +01001826 share_states_unlock(&share_states);
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001827 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001828}
1829
Andrew Walbranca808b12020-05-15 17:22:28 +01001830/** Clean up after the receiver has finished retrieving a memory region. */
1831static void ffa_memory_retrieve_complete(
1832 struct share_states_locked share_states,
1833 struct ffa_memory_share_state *share_state, struct mpool *page_pool)
1834{
1835 if (share_state->share_func == FFA_MEM_DONATE_32) {
1836 /*
1837 * Memory that has been donated can't be relinquished,
1838 * so no need to keep the share state around.
1839 */
1840 share_state_free(share_states, share_state, page_pool);
1841 dlog_verbose("Freed share state for donate.\n");
1842 }
1843}
1844
J-Alves2d8457f2022-10-05 11:06:41 +01001845/**
1846 * Initialises the given memory region descriptor to be used for an
1847 * `FFA_MEM_RETRIEVE_RESP`, including the given constituents for the first
1848 * fragment.
1849 * The memory region descriptor is initialized according to retriever's
1850 * FF-A version.
1851 *
1852 * Returns true on success, or false if the given constituents won't all fit in
1853 * the first fragment.
1854 */
1855static bool ffa_retrieved_memory_region_init(
1856 void *response, uint32_t ffa_version, size_t response_max_size,
J-Alves19e20cf2023-08-02 12:48:55 +01001857 ffa_id_t sender, ffa_memory_attributes_t attributes,
J-Alves2d8457f2022-10-05 11:06:41 +01001858 ffa_memory_region_flags_t flags, ffa_memory_handle_t handle,
J-Alves19e20cf2023-08-02 12:48:55 +01001859 ffa_id_t receiver_id, ffa_memory_access_permissions_t permissions,
J-Alves2d8457f2022-10-05 11:06:41 +01001860 uint32_t page_count, uint32_t total_constituent_count,
1861 const struct ffa_memory_region_constituent constituents[],
1862 uint32_t fragment_constituent_count, uint32_t *total_length,
1863 uint32_t *fragment_length)
1864{
1865 struct ffa_composite_memory_region *composite_memory_region;
1866 struct ffa_memory_access *receiver;
1867 uint32_t i;
1868 uint32_t constituents_offset;
1869 uint32_t receiver_count;
1870
1871 assert(response != NULL);
1872
1873 if (ffa_version == MAKE_FFA_VERSION(1, 0)) {
1874 struct ffa_memory_region_v1_0 *retrieve_response =
1875 (struct ffa_memory_region_v1_0 *)response;
1876
J-Alves5da37d92022-10-24 16:33:48 +01001877 ffa_memory_region_init_header_v1_0(
1878 retrieve_response, sender, attributes, flags, handle, 0,
1879 RECEIVERS_COUNT_IN_RETRIEVE_RESP);
J-Alves2d8457f2022-10-05 11:06:41 +01001880
1881 receiver = &retrieve_response->receivers[0];
1882 receiver_count = retrieve_response->receiver_count;
1883
1884 receiver->composite_memory_region_offset =
1885 sizeof(struct ffa_memory_region_v1_0) +
1886 receiver_count * sizeof(struct ffa_memory_access);
1887
1888 composite_memory_region = ffa_memory_region_get_composite_v1_0(
1889 retrieve_response, 0);
1890 } else {
1891 /* Default to FF-A v1.1 version. */
1892 struct ffa_memory_region *retrieve_response =
1893 (struct ffa_memory_region *)response;
1894
1895 ffa_memory_region_init_header(retrieve_response, sender,
1896 attributes, flags, handle, 0, 1);
1897
1898 receiver = &retrieve_response->receivers[0];
1899 receiver_count = retrieve_response->receiver_count;
1900
1901 /*
1902 * Note that `sizeof(struct_ffa_memory_region)` and
1903 * `sizeof(struct ffa_memory_access)` must both be multiples of
1904 * 16 (as verified by the asserts in `ffa_memory.c`, so it is
1905 * guaranteed that the offset we calculate here is aligned to a
1906 * 64-bit boundary and so 64-bit values can be copied without
1907 * alignment faults.
1908 */
1909 receiver->composite_memory_region_offset =
1910 sizeof(struct ffa_memory_region) +
1911 receiver_count * sizeof(struct ffa_memory_access);
1912
1913 composite_memory_region =
1914 ffa_memory_region_get_composite(retrieve_response, 0);
1915 }
1916
1917 assert(receiver != NULL);
1918 assert(composite_memory_region != NULL);
1919
1920 /*
1921 * Initialized here as in memory retrieve responses we currently expect
1922 * one borrower to be specified.
1923 */
1924 ffa_memory_access_init_permissions(receiver, receiver_id, 0, 0, flags);
1925 receiver->receiver_permissions.permissions = permissions;
1926
1927 composite_memory_region->page_count = page_count;
1928 composite_memory_region->constituent_count = total_constituent_count;
1929 composite_memory_region->reserved_0 = 0;
1930
1931 constituents_offset = receiver->composite_memory_region_offset +
1932 sizeof(struct ffa_composite_memory_region);
1933 if (constituents_offset +
1934 fragment_constituent_count *
1935 sizeof(struct ffa_memory_region_constituent) >
1936 response_max_size) {
1937 return false;
1938 }
1939
1940 for (i = 0; i < fragment_constituent_count; ++i) {
1941 composite_memory_region->constituents[i] = constituents[i];
1942 }
1943
1944 if (total_length != NULL) {
1945 *total_length =
1946 constituents_offset +
1947 composite_memory_region->constituent_count *
1948 sizeof(struct ffa_memory_region_constituent);
1949 }
1950 if (fragment_length != NULL) {
1951 *fragment_length =
1952 constituents_offset +
1953 fragment_constituent_count *
1954 sizeof(struct ffa_memory_region_constituent);
1955 }
1956
1957 return true;
1958}
1959
J-Alves96de29f2022-04-26 16:05:24 +01001960/*
1961 * Gets the receiver's access permissions from 'struct ffa_memory_region' and
1962 * returns its index in the receiver's array. If receiver's ID doesn't exist
1963 * in the array, return the region's 'receiver_count'.
1964 */
J-Alvesb5084cf2022-07-06 14:20:12 +01001965uint32_t ffa_memory_region_get_receiver(struct ffa_memory_region *memory_region,
J-Alves19e20cf2023-08-02 12:48:55 +01001966 ffa_id_t receiver)
J-Alves96de29f2022-04-26 16:05:24 +01001967{
1968 struct ffa_memory_access *receivers;
1969 uint32_t i;
1970
1971 assert(memory_region != NULL);
1972
1973 receivers = memory_region->receivers;
1974
1975 for (i = 0U; i < memory_region->receiver_count; i++) {
1976 if (receivers[i].receiver_permissions.receiver == receiver) {
1977 break;
1978 }
1979 }
1980
1981 return i;
1982}
1983
1984/**
1985 * Validates the retrieved permissions against those specified by the lender
1986 * of memory share operation. Optionally can help set the permissions to be used
1987 * for the S2 mapping, through the `permissions` argument.
J-Alvesdcad8992023-09-15 14:10:35 +01001988 * Returns FFA_SUCCESS if all the fields are valid. FFA_ERROR, with error code:
1989 * - FFA_INVALID_PARAMETERS -> if the fields have invalid values as per the
1990 * specification for each ABI.
1991 * - FFA_DENIED -> if the permissions specified by the retriever are not
1992 * less permissive than those provided by the sender.
J-Alves96de29f2022-04-26 16:05:24 +01001993 */
J-Alvesdcad8992023-09-15 14:10:35 +01001994static struct ffa_value ffa_memory_retrieve_is_memory_access_valid(
1995 uint32_t share_func, enum ffa_data_access sent_data_access,
J-Alves96de29f2022-04-26 16:05:24 +01001996 enum ffa_data_access requested_data_access,
1997 enum ffa_instruction_access sent_instruction_access,
1998 enum ffa_instruction_access requested_instruction_access,
J-Alvesdcad8992023-09-15 14:10:35 +01001999 ffa_memory_access_permissions_t *permissions, bool multiple_borrowers)
J-Alves96de29f2022-04-26 16:05:24 +01002000{
2001 switch (sent_data_access) {
2002 case FFA_DATA_ACCESS_NOT_SPECIFIED:
2003 case FFA_DATA_ACCESS_RW:
2004 if (requested_data_access == FFA_DATA_ACCESS_NOT_SPECIFIED ||
2005 requested_data_access == FFA_DATA_ACCESS_RW) {
2006 if (permissions != NULL) {
2007 ffa_set_data_access_attr(permissions,
2008 FFA_DATA_ACCESS_RW);
2009 }
2010 break;
2011 }
2012 /* Intentional fall-through. */
2013 case FFA_DATA_ACCESS_RO:
2014 if (requested_data_access == FFA_DATA_ACCESS_NOT_SPECIFIED ||
2015 requested_data_access == FFA_DATA_ACCESS_RO) {
2016 if (permissions != NULL) {
2017 ffa_set_data_access_attr(permissions,
2018 FFA_DATA_ACCESS_RO);
2019 }
2020 break;
2021 }
2022 dlog_verbose(
2023 "Invalid data access requested; sender specified "
2024 "permissions %#x but receiver requested %#x.\n",
2025 sent_data_access, requested_data_access);
J-Alvesdcad8992023-09-15 14:10:35 +01002026 return ffa_error(FFA_DENIED);
J-Alves96de29f2022-04-26 16:05:24 +01002027 case FFA_DATA_ACCESS_RESERVED:
2028 panic("Got unexpected FFA_DATA_ACCESS_RESERVED. Should be "
2029 "checked before this point.");
2030 }
2031
J-Alvesdcad8992023-09-15 14:10:35 +01002032 /*
2033 * For operations with a single borrower, If it is an FFA_MEMORY_LEND
2034 * or FFA_MEMORY_DONATE the retriever should have specifed the
2035 * instruction permissions it wishes to receive.
2036 */
2037 switch (share_func) {
2038 case FFA_MEM_SHARE_32:
2039 if (requested_instruction_access !=
2040 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
2041 dlog_verbose(
2042 "%s: for share instruction permissions must "
2043 "NOT be specified.\n",
2044 __func__);
2045 return ffa_error(FFA_INVALID_PARAMETERS);
2046 }
2047 break;
2048 case FFA_MEM_LEND_32:
2049 /*
2050 * For operations with multiple borrowers only permit XN
2051 * permissions, and both Sender and borrower should have used
2052 * FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED.
2053 */
2054 if (multiple_borrowers) {
2055 if (requested_instruction_access !=
2056 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
2057 dlog_verbose(
2058 "%s: lend/share/donate with multiple "
2059 "borrowers "
2060 "instruction permissions must NOT be "
2061 "specified.\n",
2062 __func__);
2063 return ffa_error(FFA_INVALID_PARAMETERS);
2064 }
2065 break;
2066 }
2067 /* Fall through if the operation targets a single borrower. */
2068 case FFA_MEM_DONATE_32:
2069 if (!multiple_borrowers &&
2070 requested_instruction_access ==
2071 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
2072 dlog_verbose(
2073 "%s: for lend/donate with single borrower "
2074 "instruction permissions must be speficified "
2075 "by borrower\n",
2076 __func__);
2077 return ffa_error(FFA_INVALID_PARAMETERS);
2078 }
2079 break;
2080 default:
2081 panic("%s: Wrong func id provided.\n", __func__);
2082 }
2083
J-Alves96de29f2022-04-26 16:05:24 +01002084 switch (sent_instruction_access) {
2085 case FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED:
2086 case FFA_INSTRUCTION_ACCESS_X:
J-Alvesdcad8992023-09-15 14:10:35 +01002087 if (requested_instruction_access == FFA_INSTRUCTION_ACCESS_X) {
J-Alves96de29f2022-04-26 16:05:24 +01002088 if (permissions != NULL) {
2089 ffa_set_instruction_access_attr(
2090 permissions, FFA_INSTRUCTION_ACCESS_X);
2091 }
2092 break;
2093 }
J-Alvesdcad8992023-09-15 14:10:35 +01002094 /*
2095 * Fall through if requested permissions are less
2096 * permissive than those provided by the sender.
2097 */
J-Alves96de29f2022-04-26 16:05:24 +01002098 case FFA_INSTRUCTION_ACCESS_NX:
2099 if (requested_instruction_access ==
2100 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED ||
2101 requested_instruction_access == FFA_INSTRUCTION_ACCESS_NX) {
2102 if (permissions != NULL) {
2103 ffa_set_instruction_access_attr(
2104 permissions, FFA_INSTRUCTION_ACCESS_NX);
2105 }
2106 break;
2107 }
2108 dlog_verbose(
2109 "Invalid instruction access requested; sender "
2110 "specified permissions %#x but receiver requested "
2111 "%#x.\n",
2112 sent_instruction_access, requested_instruction_access);
J-Alvesdcad8992023-09-15 14:10:35 +01002113 return ffa_error(FFA_DENIED);
J-Alves96de29f2022-04-26 16:05:24 +01002114 case FFA_INSTRUCTION_ACCESS_RESERVED:
2115 panic("Got unexpected FFA_INSTRUCTION_ACCESS_RESERVED. Should "
2116 "be checked before this point.");
2117 }
2118
J-Alvesdcad8992023-09-15 14:10:35 +01002119 return (struct ffa_value){.func = FFA_SUCCESS_32};
J-Alves96de29f2022-04-26 16:05:24 +01002120}
2121
2122/**
2123 * Validate the receivers' permissions in the retrieve request against those
2124 * specified by the lender.
2125 * In the `permissions` argument returns the permissions to set at S2 for the
2126 * caller to the FFA_MEMORY_RETRIEVE_REQ.
J-Alves3456e032023-07-20 12:20:05 +01002127 * The function looks into the flag to bypass multiple borrower checks:
2128 * - If not set returns FFA_SUCCESS if all specified permissions are valid.
2129 * - If set returns FFA_SUCCESS if the descriptor contains the permissions
2130 * to the caller of FFA_MEM_RETRIEVE_REQ and they are valid. Other permissions
2131 * are ignored, if provided.
J-Alves96de29f2022-04-26 16:05:24 +01002132 */
2133static struct ffa_value ffa_memory_retrieve_validate_memory_access_list(
2134 struct ffa_memory_region *memory_region,
J-Alves19e20cf2023-08-02 12:48:55 +01002135 struct ffa_memory_region *retrieve_request, ffa_id_t to_vm_id,
J-Alvesdcad8992023-09-15 14:10:35 +01002136 ffa_memory_access_permissions_t *permissions, uint32_t func_id)
J-Alves96de29f2022-04-26 16:05:24 +01002137{
2138 uint32_t retrieve_receiver_index;
J-Alves3456e032023-07-20 12:20:05 +01002139 bool bypass_multi_receiver_check =
2140 (retrieve_request->flags &
2141 FFA_MEMORY_REGION_FLAG_BYPASS_BORROWERS_CHECK) != 0U;
J-Alvesdcad8992023-09-15 14:10:35 +01002142 const uint32_t region_receiver_count = memory_region->receiver_count;
2143 struct ffa_value ret;
J-Alves96de29f2022-04-26 16:05:24 +01002144
2145 assert(permissions != NULL);
2146
J-Alves3456e032023-07-20 12:20:05 +01002147 if (!bypass_multi_receiver_check) {
J-Alvesdcad8992023-09-15 14:10:35 +01002148 if (retrieve_request->receiver_count != region_receiver_count) {
J-Alves3456e032023-07-20 12:20:05 +01002149 dlog_verbose(
2150 "Retrieve request should contain same list of "
2151 "borrowers, as specified by the lender.\n");
2152 return ffa_error(FFA_INVALID_PARAMETERS);
2153 }
2154 } else {
2155 if (retrieve_request->receiver_count != 1) {
2156 dlog_verbose(
2157 "Set bypass multiple borrower check, receiver "
2158 "list must be sized 1 (%x)\n",
2159 memory_region->receiver_count);
2160 return ffa_error(FFA_INVALID_PARAMETERS);
2161 }
J-Alves96de29f2022-04-26 16:05:24 +01002162 }
2163
2164 retrieve_receiver_index = retrieve_request->receiver_count;
2165
2166 /* Should be populated with the permissions of the retriever. */
2167 *permissions = 0;
2168
2169 for (uint32_t i = 0U; i < retrieve_request->receiver_count; i++) {
2170 ffa_memory_access_permissions_t sent_permissions;
2171 struct ffa_memory_access *current_receiver =
2172 &retrieve_request->receivers[i];
2173 ffa_memory_access_permissions_t requested_permissions =
2174 current_receiver->receiver_permissions.permissions;
J-Alves19e20cf2023-08-02 12:48:55 +01002175 ffa_id_t current_receiver_id =
J-Alves96de29f2022-04-26 16:05:24 +01002176 current_receiver->receiver_permissions.receiver;
2177 bool found_to_id = current_receiver_id == to_vm_id;
2178
J-Alves3456e032023-07-20 12:20:05 +01002179 if (bypass_multi_receiver_check && !found_to_id) {
2180 dlog_verbose(
2181 "Bypass multiple borrower check for id %x.\n",
2182 current_receiver_id);
2183 continue;
2184 }
2185
J-Alves96de29f2022-04-26 16:05:24 +01002186 /*
2187 * Find the current receiver in the transaction descriptor from
2188 * sender.
2189 */
2190 uint32_t mem_region_receiver_index =
2191 ffa_memory_region_get_receiver(memory_region,
2192 current_receiver_id);
2193
2194 if (mem_region_receiver_index ==
2195 memory_region->receiver_count) {
2196 dlog_verbose("%s: receiver %x not found\n", __func__,
2197 current_receiver_id);
2198 return ffa_error(FFA_DENIED);
2199 }
2200
2201 sent_permissions =
2202 memory_region->receivers[mem_region_receiver_index]
2203 .receiver_permissions.permissions;
2204
2205 if (found_to_id) {
2206 retrieve_receiver_index = i;
2207 }
2208
2209 /*
2210 * Since we are traversing the list of receivers, save the index
2211 * of the caller. As it needs to be there.
2212 */
2213
2214 if (current_receiver->composite_memory_region_offset != 0U) {
2215 dlog_verbose(
2216 "Retriever specified address ranges not "
2217 "supported (got offset %d).\n",
2218 current_receiver
2219 ->composite_memory_region_offset);
2220 return ffa_error(FFA_INVALID_PARAMETERS);
2221 }
2222
2223 /*
J-Alvesdcad8992023-09-15 14:10:35 +01002224 * Check if retrieve request memory access list is valid:
2225 * - The retrieve request complies with the specification.
2226 * - Permissions are within those specified by the sender.
J-Alves96de29f2022-04-26 16:05:24 +01002227 */
J-Alvesdcad8992023-09-15 14:10:35 +01002228 ret = ffa_memory_retrieve_is_memory_access_valid(
2229 func_id, ffa_get_data_access_attr(sent_permissions),
2230 ffa_get_data_access_attr(requested_permissions),
2231 ffa_get_instruction_access_attr(sent_permissions),
2232 ffa_get_instruction_access_attr(requested_permissions),
2233 found_to_id ? permissions : NULL,
2234 region_receiver_count > 1);
2235 if (ret.func != FFA_SUCCESS_32) {
2236 return ret;
J-Alves96de29f2022-04-26 16:05:24 +01002237 }
2238
2239 /*
2240 * Can't request PM to clear memory if only provided with RO
2241 * permissions.
2242 */
2243 if (found_to_id &&
2244 (ffa_get_data_access_attr(*permissions) ==
2245 FFA_DATA_ACCESS_RO) &&
2246 (retrieve_request->flags & FFA_MEMORY_REGION_FLAG_CLEAR) !=
2247 0U) {
2248 dlog_verbose(
2249 "Receiver has RO permissions can not request "
2250 "clear.\n");
2251 return ffa_error(FFA_DENIED);
2252 }
2253 }
2254
2255 if (retrieve_receiver_index == retrieve_request->receiver_count) {
2256 dlog_verbose(
2257 "Retrieve request does not contain caller's (%x) "
2258 "permissions\n",
2259 to_vm_id);
2260 return ffa_error(FFA_INVALID_PARAMETERS);
2261 }
2262
2263 return (struct ffa_value){.func = FFA_SUCCESS_32};
2264}
2265
J-Alvesa9cd7e32022-07-01 13:49:33 +01002266/*
2267 * According to section 16.4.3 of FF-A v1.1 EAC0 specification, the hypervisor
2268 * may issue an FFA_MEM_RETRIEVE_REQ to obtain the memory region description
2269 * of a pending memory sharing operation whose allocator is the SPM, for
2270 * validation purposes before forwarding an FFA_MEM_RECLAIM call. In doing so
2271 * the memory region descriptor of the retrieve request must be zeroed with the
2272 * exception of the sender ID and handle.
2273 */
2274bool is_ffa_memory_retrieve_borrower_request(struct ffa_memory_region *request,
2275 struct vm_locked to_locked)
2276{
2277 return to_locked.vm->id == HF_HYPERVISOR_VM_ID &&
2278 request->attributes == 0U && request->flags == 0U &&
2279 request->tag == 0U && request->receiver_count == 0U &&
2280 plat_ffa_memory_handle_allocated_by_current_world(
2281 request->handle);
2282}
2283
2284/*
2285 * Helper to reset count of fragments retrieved by the hypervisor.
2286 */
2287static void ffa_memory_retrieve_complete_from_hyp(
2288 struct ffa_memory_share_state *share_state)
2289{
2290 if (share_state->hypervisor_fragment_count ==
2291 share_state->fragment_count) {
2292 share_state->hypervisor_fragment_count = 0;
2293 }
2294}
2295
J-Alves089004f2022-07-13 14:25:44 +01002296/**
2297 * Validate that the memory region descriptor provided by the borrower on
2298 * FFA_MEM_RETRIEVE_REQ, against saved memory region provided by lender at the
2299 * memory sharing call.
2300 */
2301static struct ffa_value ffa_memory_retrieve_validate(
J-Alves19e20cf2023-08-02 12:48:55 +01002302 ffa_id_t receiver_id, struct ffa_memory_region *retrieve_request,
J-Alves089004f2022-07-13 14:25:44 +01002303 struct ffa_memory_region *memory_region, uint32_t *receiver_index,
2304 uint32_t share_func)
2305{
2306 ffa_memory_region_flags_t transaction_type =
2307 retrieve_request->flags &
2308 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK;
Olivier Deprez4342a3c2022-02-28 09:37:25 +01002309 enum ffa_memory_security security_state;
J-Alves089004f2022-07-13 14:25:44 +01002310
2311 assert(retrieve_request != NULL);
2312 assert(memory_region != NULL);
2313 assert(receiver_index != NULL);
2314 assert(retrieve_request->sender == memory_region->sender);
2315
2316 /*
2317 * Check that the transaction type expected by the receiver is
2318 * correct, if it has been specified.
2319 */
2320 if (transaction_type !=
2321 FFA_MEMORY_REGION_TRANSACTION_TYPE_UNSPECIFIED &&
2322 transaction_type != (memory_region->flags &
2323 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK)) {
2324 dlog_verbose(
2325 "Incorrect transaction type %#x for "
2326 "FFA_MEM_RETRIEVE_REQ, expected %#x for handle %#x.\n",
2327 transaction_type,
2328 memory_region->flags &
2329 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK,
2330 retrieve_request->handle);
2331 return ffa_error(FFA_INVALID_PARAMETERS);
2332 }
2333
2334 if (retrieve_request->tag != memory_region->tag) {
2335 dlog_verbose(
2336 "Incorrect tag %d for FFA_MEM_RETRIEVE_REQ, expected "
2337 "%d for handle %#x.\n",
2338 retrieve_request->tag, memory_region->tag,
2339 retrieve_request->handle);
2340 return ffa_error(FFA_INVALID_PARAMETERS);
2341 }
2342
2343 *receiver_index =
2344 ffa_memory_region_get_receiver(memory_region, receiver_id);
2345
2346 if (*receiver_index == memory_region->receiver_count) {
2347 dlog_verbose(
2348 "Incorrect receiver VM ID %d for "
2349 "FFA_MEM_RETRIEVE_REQ, for handle %#x.\n",
J-Alves59ed0042022-07-28 18:26:41 +01002350 receiver_id, memory_region->handle);
J-Alves089004f2022-07-13 14:25:44 +01002351 return ffa_error(FFA_INVALID_PARAMETERS);
2352 }
2353
2354 if ((retrieve_request->flags &
2355 FFA_MEMORY_REGION_ADDRESS_RANGE_HINT_VALID) != 0U) {
2356 dlog_verbose(
2357 "Retriever specified 'address range alignment 'hint' "
2358 "not supported.\n");
2359 return ffa_error(FFA_INVALID_PARAMETERS);
2360 }
2361 if ((retrieve_request->flags &
2362 FFA_MEMORY_REGION_ADDRESS_RANGE_HINT_MASK) != 0) {
2363 dlog_verbose(
2364 "Bits 8-5 must be zero in memory region's flags "
2365 "(address range alignment hint not supported).\n");
2366 return ffa_error(FFA_INVALID_PARAMETERS);
2367 }
2368
2369 if ((retrieve_request->flags & ~0x7FF) != 0U) {
2370 dlog_verbose(
2371 "Bits 31-10 must be zero in memory region's flags.\n");
2372 return ffa_error(FFA_INVALID_PARAMETERS);
2373 }
2374
2375 if (share_func == FFA_MEM_SHARE_32 &&
2376 (retrieve_request->flags &
2377 (FFA_MEMORY_REGION_FLAG_CLEAR |
2378 FFA_MEMORY_REGION_FLAG_CLEAR_RELINQUISH)) != 0U) {
2379 dlog_verbose(
2380 "Memory Share operation can't clean after relinquish "
2381 "memory region.\n");
2382 return ffa_error(FFA_INVALID_PARAMETERS);
2383 }
2384
2385 /*
2386 * If the borrower needs the memory to be cleared before mapping
2387 * to its address space, the sender should have set the flag
2388 * when calling FFA_MEM_LEND/FFA_MEM_DONATE, else return
2389 * FFA_DENIED.
2390 */
2391 if ((retrieve_request->flags & FFA_MEMORY_REGION_FLAG_CLEAR) != 0U &&
2392 (memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR) == 0U) {
2393 dlog_verbose(
2394 "Borrower needs memory cleared. Sender needs to set "
2395 "flag for clearing memory.\n");
2396 return ffa_error(FFA_DENIED);
2397 }
2398
Olivier Deprez4342a3c2022-02-28 09:37:25 +01002399 /* Memory region attributes NS-Bit MBZ for FFA_MEM_RETRIEVE_REQ. */
2400 security_state =
2401 ffa_get_memory_security_attr(retrieve_request->attributes);
2402 if (security_state != FFA_MEMORY_SECURITY_UNSPECIFIED) {
2403 dlog_verbose(
2404 "Invalid security state for memory retrieve request "
2405 "operation.\n");
2406 return ffa_error(FFA_INVALID_PARAMETERS);
2407 }
2408
J-Alves089004f2022-07-13 14:25:44 +01002409 /*
2410 * If memory type is not specified, bypass validation of memory
2411 * attributes in the retrieve request. The retriever is expecting to
2412 * obtain this information from the SPMC.
2413 */
2414 if (ffa_get_memory_type_attr(retrieve_request->attributes) ==
2415 FFA_MEMORY_NOT_SPECIFIED_MEM) {
2416 return (struct ffa_value){.func = FFA_SUCCESS_32};
2417 }
2418
2419 /*
2420 * Ensure receiver's attributes are compatible with how
2421 * Hafnium maps memory: Normal Memory, Inner shareable,
2422 * Write-Back Read-Allocate Write-Allocate Cacheable.
2423 */
2424 return ffa_memory_attributes_validate(retrieve_request->attributes);
2425}
2426
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002427struct ffa_value ffa_memory_retrieve(struct vm_locked to_locked,
2428 struct ffa_memory_region *retrieve_request,
Andrew Walbran130a8ae2020-05-15 16:27:15 +01002429 uint32_t retrieve_request_length,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002430 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002431{
Andrew Walbran130a8ae2020-05-15 16:27:15 +01002432 uint32_t expected_retrieve_request_length =
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002433 sizeof(struct ffa_memory_region) +
Andrew Walbrana65a1322020-04-06 19:32:32 +01002434 retrieve_request->receiver_count *
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002435 sizeof(struct ffa_memory_access);
2436 ffa_memory_handle_t handle = retrieve_request->handle;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002437 struct ffa_memory_region *memory_region;
J-Alvesa9cd7e32022-07-01 13:49:33 +01002438 ffa_memory_access_permissions_t permissions = 0;
Olivier Deprez878bd5b2021-04-15 19:05:10 +02002439 uint32_t memory_to_mode;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002440 struct share_states_locked share_states;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002441 struct ffa_memory_share_state *share_state;
2442 struct ffa_value ret;
Andrew Walbranca808b12020-05-15 17:22:28 +01002443 struct ffa_composite_memory_region *composite;
2444 uint32_t total_length;
2445 uint32_t fragment_length;
J-Alves19e20cf2023-08-02 12:48:55 +01002446 ffa_id_t receiver_id = to_locked.vm->id;
J-Alvesa9cd7e32022-07-01 13:49:33 +01002447 bool is_send_complete = false;
Olivier Deprez878bd5b2021-04-15 19:05:10 +02002448 ffa_memory_attributes_t attributes;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002449
2450 dump_share_states();
2451
Andrew Walbran130a8ae2020-05-15 16:27:15 +01002452 if (retrieve_request_length != expected_retrieve_request_length) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002453 dlog_verbose(
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002454 "Invalid length for FFA_MEM_RETRIEVE_REQ, expected %d "
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002455 "but was %d.\n",
Andrew Walbran130a8ae2020-05-15 16:27:15 +01002456 expected_retrieve_request_length,
2457 retrieve_request_length);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002458 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002459 }
2460
2461 share_states = share_states_lock();
Karl Meakin4a2854a2023-06-30 16:26:52 +01002462 share_state = get_share_state(share_states, handle);
2463 if (!share_state) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002464 dlog_verbose("Invalid handle %#x for FFA_MEM_RETRIEVE_REQ.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002465 handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002466 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002467 goto out;
2468 }
2469
J-Alves96de29f2022-04-26 16:05:24 +01002470 if (!share_state->sending_complete) {
2471 dlog_verbose(
2472 "Memory with handle %#x not fully sent, can't "
2473 "retrieve.\n",
2474 handle);
2475 ret = ffa_error(FFA_INVALID_PARAMETERS);
2476 goto out;
2477 }
2478
Andrew Walbrana65a1322020-04-06 19:32:32 +01002479 memory_region = share_state->memory_region;
J-Alves089004f2022-07-13 14:25:44 +01002480
Andrew Walbrana65a1322020-04-06 19:32:32 +01002481 CHECK(memory_region != NULL);
2482
J-Alves089004f2022-07-13 14:25:44 +01002483 if (retrieve_request->sender != memory_region->sender) {
2484 dlog_verbose(
2485 "Memory with handle %#x not fully sent, can't "
2486 "retrieve.\n",
2487 handle);
2488 ret = ffa_error(FFA_INVALID_PARAMETERS);
2489 goto out;
2490 }
J-Alves96de29f2022-04-26 16:05:24 +01002491
J-Alvesa9cd7e32022-07-01 13:49:33 +01002492 if (!is_ffa_memory_retrieve_borrower_request(retrieve_request,
2493 to_locked)) {
2494 uint32_t receiver_index;
J-Alvesa9cd7e32022-07-01 13:49:33 +01002495
J-Alvesb5084cf2022-07-06 14:20:12 +01002496 /*
2497 * The SPMC can only process retrieve requests to memory share
2498 * operations with one borrower from the other world. It can't
2499 * determine the ID of the NWd VM that invoked the retrieve
2500 * request interface call. It relies on the hypervisor to
2501 * validate the caller's ID against that provided in the
2502 * `receivers` list of the retrieve response.
2503 * In case there is only one borrower from the NWd in the
2504 * transaction descriptor, record that in the `receiver_id` for
2505 * later use, and validate in the retrieve request message.
J-Alves3fa82aa2023-09-20 18:19:21 +01002506 * This limitation is due to the fact SPMC can't determine the
2507 * index in the memory share structures state to update.
J-Alvesb5084cf2022-07-06 14:20:12 +01002508 */
2509 if (to_locked.vm->id == HF_HYPERVISOR_VM_ID) {
2510 uint32_t other_world_count = 0;
2511
2512 for (uint32_t i = 0; i < memory_region->receiver_count;
2513 i++) {
2514 receiver_id =
2515 retrieve_request->receivers[0]
2516 .receiver_permissions.receiver;
2517 if (!vm_id_is_current_world(receiver_id)) {
2518 other_world_count++;
2519 }
2520 }
2521 if (other_world_count > 1) {
2522 dlog_verbose(
2523 "Support one receiver from the other "
2524 "world.\n");
2525 return ffa_error(FFA_NOT_SUPPORTED);
2526 }
2527 }
2528
2529 /*
2530 * Validate retrieve request, according to what was sent by the
2531 * sender. Function will output the `receiver_index` from the
J-Alves3fa82aa2023-09-20 18:19:21 +01002532 * provided memory region.
J-Alvesb5084cf2022-07-06 14:20:12 +01002533 */
J-Alves089004f2022-07-13 14:25:44 +01002534 ret = ffa_memory_retrieve_validate(
2535 receiver_id, retrieve_request, memory_region,
2536 &receiver_index, share_state->share_func);
2537 if (ret.func != FFA_SUCCESS_32) {
J-Alvesa9cd7e32022-07-01 13:49:33 +01002538 goto out;
2539 }
2540
2541 if (share_state->retrieved_fragment_count[receiver_index] !=
2542 0U) {
2543 dlog_verbose(
2544 "Memory with handle %#x already retrieved.\n",
2545 handle);
2546 ret = ffa_error(FFA_DENIED);
2547 goto out;
2548 }
2549
J-Alves3fa82aa2023-09-20 18:19:21 +01002550 /*
2551 * Validate the requested permissions against the sent
2552 * permissions.
2553 * Outputs the permissions to give to retriever at S2
2554 * PTs.
2555 */
J-Alvesa9cd7e32022-07-01 13:49:33 +01002556 ret = ffa_memory_retrieve_validate_memory_access_list(
2557 memory_region, retrieve_request, receiver_id,
J-Alvesdcad8992023-09-15 14:10:35 +01002558 &permissions, share_state->share_func);
J-Alves614d9f42022-06-28 14:03:10 +01002559 if (ret.func != FFA_SUCCESS_32) {
2560 goto out;
2561 }
Federico Recanatia98603a2021-12-20 18:04:03 +01002562
Olivier Deprez878bd5b2021-04-15 19:05:10 +02002563 memory_to_mode = ffa_memory_permissions_to_mode(
J-Alvesa9cd7e32022-07-01 13:49:33 +01002564 permissions, share_state->sender_orig_mode);
J-Alves40e260e2022-09-22 17:52:43 +01002565
J-Alvesa9cd7e32022-07-01 13:49:33 +01002566 ret = ffa_retrieve_check_update(
2567 to_locked, memory_region->sender,
2568 share_state->fragments,
2569 share_state->fragment_constituent_counts,
Olivier Deprez878bd5b2021-04-15 19:05:10 +02002570 share_state->fragment_count, memory_to_mode,
J-Alvesa9cd7e32022-07-01 13:49:33 +01002571 share_state->share_func, false, page_pool);
2572
2573 if (ret.func != FFA_SUCCESS_32) {
2574 goto out;
2575 }
2576
2577 share_state->retrieved_fragment_count[receiver_index] = 1;
2578 is_send_complete =
2579 share_state->retrieved_fragment_count[receiver_index] ==
2580 share_state->fragment_count;
J-Alves3c5b2072022-11-21 12:45:40 +00002581
2582 share_state->clear_after_relinquish =
2583 (retrieve_request->flags &
2584 FFA_MEMORY_REGION_FLAG_CLEAR_RELINQUISH) != 0U;
2585
J-Alvesa9cd7e32022-07-01 13:49:33 +01002586 } else {
2587 if (share_state->hypervisor_fragment_count != 0U) {
2588 dlog_verbose(
J-Alvesb5084cf2022-07-06 14:20:12 +01002589 "Memory with handle %#x already retrieved by "
J-Alvesa9cd7e32022-07-01 13:49:33 +01002590 "the hypervisor.\n",
2591 handle);
2592 ret = ffa_error(FFA_DENIED);
2593 goto out;
2594 }
2595
2596 share_state->hypervisor_fragment_count = 1;
2597
2598 ffa_memory_retrieve_complete_from_hyp(share_state);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002599 }
2600
J-Alvesb5084cf2022-07-06 14:20:12 +01002601 /* VMs acquire the RX buffer from SPMC. */
2602 CHECK(plat_ffa_acquire_receiver_rx(to_locked, &ret));
2603
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002604 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01002605 * Copy response to RX buffer of caller and deliver the message.
2606 * This must be done before the share_state is (possibly) freed.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002607 */
Andrew Walbrana65a1322020-04-06 19:32:32 +01002608 /* TODO: combine attributes from sender and request. */
Andrew Walbranca808b12020-05-15 17:22:28 +01002609 composite = ffa_memory_region_get_composite(memory_region, 0);
2610 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01002611 * Constituents which we received in the first fragment should
2612 * always fit in the first fragment we are sending, because the
2613 * header is the same size in both cases and we have a fixed
2614 * message buffer size. So `ffa_retrieved_memory_region_init`
2615 * should never fail.
Andrew Walbranca808b12020-05-15 17:22:28 +01002616 */
Olivier Deprez878bd5b2021-04-15 19:05:10 +02002617
2618 /*
2619 * Set the security state in the memory retrieve response attributes
2620 * if specified by the target mode.
2621 */
2622 attributes = plat_ffa_memory_security_mode(
2623 memory_region->attributes, share_state->sender_orig_mode);
2624
Andrew Walbranca808b12020-05-15 17:22:28 +01002625 CHECK(ffa_retrieved_memory_region_init(
J-Alves2d8457f2022-10-05 11:06:41 +01002626 to_locked.vm->mailbox.recv, to_locked.vm->ffa_version,
Olivier Deprez878bd5b2021-04-15 19:05:10 +02002627 HF_MAILBOX_SIZE, memory_region->sender, attributes,
2628 memory_region->flags, handle, receiver_id, permissions,
2629 composite->page_count, composite->constituent_count,
2630 share_state->fragments[0],
Andrew Walbranca808b12020-05-15 17:22:28 +01002631 share_state->fragment_constituent_counts[0], &total_length,
2632 &fragment_length));
J-Alvesb5084cf2022-07-06 14:20:12 +01002633
Andrew Walbranca808b12020-05-15 17:22:28 +01002634 to_locked.vm->mailbox.recv_size = fragment_length;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002635 to_locked.vm->mailbox.recv_sender = HF_HYPERVISOR_VM_ID;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002636 to_locked.vm->mailbox.recv_func = FFA_MEM_RETRIEVE_RESP_32;
J-Alvese8c8c2b2022-12-16 15:34:48 +00002637 to_locked.vm->mailbox.state = MAILBOX_STATE_FULL;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002638
J-Alvesa9cd7e32022-07-01 13:49:33 +01002639 if (is_send_complete) {
Andrew Walbranca808b12020-05-15 17:22:28 +01002640 ffa_memory_retrieve_complete(share_states, share_state,
2641 page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002642 }
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002643 ret = (struct ffa_value){.func = FFA_MEM_RETRIEVE_RESP_32,
Andrew Walbranca808b12020-05-15 17:22:28 +01002644 .arg1 = total_length,
2645 .arg2 = fragment_length};
Andrew Walbranca808b12020-05-15 17:22:28 +01002646out:
2647 share_states_unlock(&share_states);
2648 dump_share_states();
2649 return ret;
2650}
2651
J-Alves5da37d92022-10-24 16:33:48 +01002652/**
2653 * Determine expected fragment offset according to the FF-A version of
2654 * the caller.
2655 */
2656static uint32_t ffa_memory_retrieve_expected_offset_per_ffa_version(
2657 struct ffa_memory_region *memory_region,
2658 uint32_t retrieved_constituents_count, uint32_t ffa_version)
2659{
2660 uint32_t expected_fragment_offset;
2661 uint32_t composite_constituents_offset;
2662
2663 if (ffa_version == MAKE_FFA_VERSION(1, 1)) {
2664 /*
2665 * Hafnium operates memory regions in FF-A v1.1 format, so we
2666 * can retrieve the constituents offset from descriptor.
2667 */
2668 composite_constituents_offset =
2669 ffa_composite_constituent_offset(memory_region, 0);
2670 } else if (ffa_version == MAKE_FFA_VERSION(1, 0)) {
2671 /*
2672 * If retriever is FF-A v1.0, determine the composite offset
2673 * as it is expected to have been configured in the
2674 * retrieve response.
2675 */
2676 composite_constituents_offset =
2677 sizeof(struct ffa_memory_region_v1_0) +
2678 RECEIVERS_COUNT_IN_RETRIEVE_RESP *
2679 sizeof(struct ffa_memory_access) +
2680 sizeof(struct ffa_composite_memory_region);
2681 } else {
2682 panic("%s received an invalid FF-A version.\n", __func__);
2683 }
2684
2685 expected_fragment_offset =
2686 composite_constituents_offset +
2687 retrieved_constituents_count *
2688 sizeof(struct ffa_memory_region_constituent) -
2689 sizeof(struct ffa_memory_access) *
2690 (memory_region->receiver_count - 1);
2691
2692 return expected_fragment_offset;
2693}
2694
Andrew Walbranca808b12020-05-15 17:22:28 +01002695struct ffa_value ffa_memory_retrieve_continue(struct vm_locked to_locked,
2696 ffa_memory_handle_t handle,
2697 uint32_t fragment_offset,
J-Alves19e20cf2023-08-02 12:48:55 +01002698 ffa_id_t sender_vm_id,
Andrew Walbranca808b12020-05-15 17:22:28 +01002699 struct mpool *page_pool)
2700{
2701 struct ffa_memory_region *memory_region;
2702 struct share_states_locked share_states;
2703 struct ffa_memory_share_state *share_state;
2704 struct ffa_value ret;
2705 uint32_t fragment_index;
2706 uint32_t retrieved_constituents_count;
2707 uint32_t i;
2708 uint32_t expected_fragment_offset;
2709 uint32_t remaining_constituent_count;
2710 uint32_t fragment_length;
J-Alvesc7484f12022-05-13 12:41:14 +01002711 uint32_t receiver_index;
J-Alves59ed0042022-07-28 18:26:41 +01002712 bool continue_ffa_hyp_mem_retrieve_req;
Andrew Walbranca808b12020-05-15 17:22:28 +01002713
2714 dump_share_states();
2715
2716 share_states = share_states_lock();
Karl Meakin4a2854a2023-06-30 16:26:52 +01002717 share_state = get_share_state(share_states, handle);
2718 if (!share_state) {
Andrew Walbranca808b12020-05-15 17:22:28 +01002719 dlog_verbose("Invalid handle %#x for FFA_MEM_FRAG_RX.\n",
2720 handle);
2721 ret = ffa_error(FFA_INVALID_PARAMETERS);
2722 goto out;
2723 }
2724
2725 memory_region = share_state->memory_region;
2726 CHECK(memory_region != NULL);
2727
Andrew Walbranca808b12020-05-15 17:22:28 +01002728 if (!share_state->sending_complete) {
2729 dlog_verbose(
2730 "Memory with handle %#x not fully sent, can't "
2731 "retrieve.\n",
2732 handle);
2733 ret = ffa_error(FFA_INVALID_PARAMETERS);
2734 goto out;
2735 }
2736
J-Alves59ed0042022-07-28 18:26:41 +01002737 /*
2738 * If retrieve request from the hypervisor has been initiated in the
2739 * given share_state, continue it, else assume it is a continuation of
2740 * retrieve request from a NWd VM.
2741 */
2742 continue_ffa_hyp_mem_retrieve_req =
2743 (to_locked.vm->id == HF_HYPERVISOR_VM_ID) &&
2744 (share_state->hypervisor_fragment_count != 0U) &&
J-Alves661e1b72023-08-02 13:39:40 +01002745 ffa_is_vm_id(sender_vm_id);
Andrew Walbranca808b12020-05-15 17:22:28 +01002746
J-Alves59ed0042022-07-28 18:26:41 +01002747 if (!continue_ffa_hyp_mem_retrieve_req) {
2748 receiver_index = ffa_memory_region_get_receiver(
2749 memory_region, to_locked.vm->id);
2750
2751 if (receiver_index == memory_region->receiver_count) {
2752 dlog_verbose(
2753 "Caller of FFA_MEM_FRAG_RX (%x) is not a "
2754 "borrower to memory sharing transaction (%x)\n",
2755 to_locked.vm->id, handle);
2756 ret = ffa_error(FFA_INVALID_PARAMETERS);
2757 goto out;
2758 }
2759
2760 if (share_state->retrieved_fragment_count[receiver_index] ==
2761 0 ||
2762 share_state->retrieved_fragment_count[receiver_index] >=
2763 share_state->fragment_count) {
2764 dlog_verbose(
2765 "Retrieval of memory with handle %#x not yet "
2766 "started or already completed (%d/%d fragments "
2767 "retrieved).\n",
2768 handle,
2769 share_state->retrieved_fragment_count
2770 [receiver_index],
2771 share_state->fragment_count);
2772 ret = ffa_error(FFA_INVALID_PARAMETERS);
2773 goto out;
2774 }
2775
2776 fragment_index =
2777 share_state->retrieved_fragment_count[receiver_index];
2778 } else {
2779 if (share_state->hypervisor_fragment_count == 0 ||
2780 share_state->hypervisor_fragment_count >=
2781 share_state->fragment_count) {
2782 dlog_verbose(
2783 "Retrieve of memory with handle %x not "
2784 "started from hypervisor.\n",
2785 handle);
2786 ret = ffa_error(FFA_INVALID_PARAMETERS);
2787 goto out;
2788 }
2789
2790 if (memory_region->sender != sender_vm_id) {
2791 dlog_verbose(
2792 "Sender ID (%x) is not as expected for memory "
2793 "handle %x\n",
2794 sender_vm_id, handle);
2795 ret = ffa_error(FFA_INVALID_PARAMETERS);
2796 goto out;
2797 }
2798
2799 fragment_index = share_state->hypervisor_fragment_count;
2800
2801 receiver_index = 0;
2802 }
Andrew Walbranca808b12020-05-15 17:22:28 +01002803
2804 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01002805 * Check that the given fragment offset is correct by counting
2806 * how many constituents were in the fragments previously sent.
Andrew Walbranca808b12020-05-15 17:22:28 +01002807 */
2808 retrieved_constituents_count = 0;
2809 for (i = 0; i < fragment_index; ++i) {
2810 retrieved_constituents_count +=
2811 share_state->fragment_constituent_counts[i];
2812 }
J-Alvesc7484f12022-05-13 12:41:14 +01002813
2814 CHECK(memory_region->receiver_count > 0);
2815
Andrew Walbranca808b12020-05-15 17:22:28 +01002816 expected_fragment_offset =
J-Alves5da37d92022-10-24 16:33:48 +01002817 ffa_memory_retrieve_expected_offset_per_ffa_version(
2818 memory_region, retrieved_constituents_count,
2819 to_locked.vm->ffa_version);
2820
Andrew Walbranca808b12020-05-15 17:22:28 +01002821 if (fragment_offset != expected_fragment_offset) {
2822 dlog_verbose("Fragment offset was %d but expected %d.\n",
2823 fragment_offset, expected_fragment_offset);
2824 ret = ffa_error(FFA_INVALID_PARAMETERS);
2825 goto out;
2826 }
2827
J-Alves59ed0042022-07-28 18:26:41 +01002828 /* VMs acquire the RX buffer from SPMC. */
2829 CHECK(plat_ffa_acquire_receiver_rx(to_locked, &ret));
2830
Andrew Walbranca808b12020-05-15 17:22:28 +01002831 remaining_constituent_count = ffa_memory_fragment_init(
2832 to_locked.vm->mailbox.recv, HF_MAILBOX_SIZE,
2833 share_state->fragments[fragment_index],
2834 share_state->fragment_constituent_counts[fragment_index],
2835 &fragment_length);
2836 CHECK(remaining_constituent_count == 0);
2837 to_locked.vm->mailbox.recv_size = fragment_length;
2838 to_locked.vm->mailbox.recv_sender = HF_HYPERVISOR_VM_ID;
2839 to_locked.vm->mailbox.recv_func = FFA_MEM_FRAG_TX_32;
J-Alvese8c8c2b2022-12-16 15:34:48 +00002840 to_locked.vm->mailbox.state = MAILBOX_STATE_FULL;
Andrew Walbranca808b12020-05-15 17:22:28 +01002841
J-Alves59ed0042022-07-28 18:26:41 +01002842 if (!continue_ffa_hyp_mem_retrieve_req) {
2843 share_state->retrieved_fragment_count[receiver_index]++;
2844 if (share_state->retrieved_fragment_count[receiver_index] ==
2845 share_state->fragment_count) {
2846 ffa_memory_retrieve_complete(share_states, share_state,
2847 page_pool);
2848 }
2849 } else {
2850 share_state->hypervisor_fragment_count++;
2851
2852 ffa_memory_retrieve_complete_from_hyp(share_state);
2853 }
Andrew Walbranca808b12020-05-15 17:22:28 +01002854 ret = (struct ffa_value){.func = FFA_MEM_FRAG_TX_32,
2855 .arg1 = (uint32_t)handle,
2856 .arg2 = (uint32_t)(handle >> 32),
2857 .arg3 = fragment_length};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002858
2859out:
2860 share_states_unlock(&share_states);
2861 dump_share_states();
2862 return ret;
2863}
2864
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002865struct ffa_value ffa_memory_relinquish(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002866 struct vm_locked from_locked,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002867 struct ffa_mem_relinquish *relinquish_request, struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002868{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002869 ffa_memory_handle_t handle = relinquish_request->handle;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002870 struct share_states_locked share_states;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002871 struct ffa_memory_share_state *share_state;
2872 struct ffa_memory_region *memory_region;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002873 bool clear;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002874 struct ffa_value ret;
J-Alves8eb19162022-04-28 10:56:48 +01002875 uint32_t receiver_index;
J-Alves3c5b2072022-11-21 12:45:40 +00002876 bool receivers_relinquished_memory;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002877
Andrew Walbrana65a1322020-04-06 19:32:32 +01002878 if (relinquish_request->endpoint_count != 1) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002879 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01002880 "Stream endpoints not supported (got %d "
J-Alves668a86e2023-05-10 11:53:25 +01002881 "endpoints on FFA_MEM_RELINQUISH, expected 1).\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002882 relinquish_request->endpoint_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002883 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002884 }
2885
Andrew Walbrana65a1322020-04-06 19:32:32 +01002886 if (relinquish_request->endpoints[0] != from_locked.vm->id) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002887 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01002888 "VM ID %d in relinquish message doesn't match "
J-Alves668a86e2023-05-10 11:53:25 +01002889 "calling VM ID %d.\n",
Andrew Walbrana65a1322020-04-06 19:32:32 +01002890 relinquish_request->endpoints[0], from_locked.vm->id);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002891 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002892 }
2893
2894 dump_share_states();
2895
2896 share_states = share_states_lock();
Karl Meakin4a2854a2023-06-30 16:26:52 +01002897 share_state = get_share_state(share_states, handle);
2898 if (!share_state) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002899 dlog_verbose("Invalid handle %#x for FFA_MEM_RELINQUISH.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002900 handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002901 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002902 goto out;
2903 }
2904
Andrew Walbranca808b12020-05-15 17:22:28 +01002905 if (!share_state->sending_complete) {
2906 dlog_verbose(
2907 "Memory with handle %#x not fully sent, can't "
2908 "relinquish.\n",
2909 handle);
2910 ret = ffa_error(FFA_INVALID_PARAMETERS);
2911 goto out;
2912 }
2913
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002914 memory_region = share_state->memory_region;
2915 CHECK(memory_region != NULL);
2916
J-Alves8eb19162022-04-28 10:56:48 +01002917 receiver_index = ffa_memory_region_get_receiver(memory_region,
2918 from_locked.vm->id);
2919
2920 if (receiver_index == memory_region->receiver_count) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002921 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01002922 "VM ID %d tried to relinquish memory region "
J-Alves668a86e2023-05-10 11:53:25 +01002923 "with handle %#x and it is not a valid borrower.\n",
J-Alves8eb19162022-04-28 10:56:48 +01002924 from_locked.vm->id, handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002925 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002926 goto out;
2927 }
2928
J-Alves8eb19162022-04-28 10:56:48 +01002929 if (share_state->retrieved_fragment_count[receiver_index] !=
Andrew Walbranca808b12020-05-15 17:22:28 +01002930 share_state->fragment_count) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002931 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01002932 "Memory with handle %#x not yet fully "
2933 "retrieved, "
J-Alves8eb19162022-04-28 10:56:48 +01002934 "receiver %x can't relinquish.\n",
2935 handle, from_locked.vm->id);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002936 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002937 goto out;
2938 }
2939
J-Alves3c5b2072022-11-21 12:45:40 +00002940 /*
2941 * Either clear if requested in relinquish call, or in a retrieve
2942 * request from one of the borrowers.
2943 */
2944 receivers_relinquished_memory = true;
2945
2946 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
2947 struct ffa_memory_access *receiver =
2948 &memory_region->receivers[i];
2949
2950 if (receiver->receiver_permissions.receiver ==
2951 from_locked.vm->id) {
2952 continue;
2953 }
2954
2955 if (share_state->retrieved_fragment_count[i] != 0U) {
2956 receivers_relinquished_memory = false;
2957 break;
2958 }
2959 }
2960
2961 clear = receivers_relinquished_memory &&
2962 (share_state->clear_after_relinquish ||
2963 (relinquish_request->flags & FFA_MEMORY_REGION_FLAG_CLEAR) !=
2964 0U);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002965
2966 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01002967 * Clear is not allowed for memory that was shared, as the
2968 * original sender still has access to the memory.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002969 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002970 if (clear && share_state->share_func == FFA_MEM_SHARE_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002971 dlog_verbose("Memory which was shared can't be cleared.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002972 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002973 goto out;
2974 }
2975
Andrew Walbranca808b12020-05-15 17:22:28 +01002976 ret = ffa_relinquish_check_update(
J-Alves3c5b2072022-11-21 12:45:40 +00002977 from_locked, memory_region->sender, share_state->fragments,
Andrew Walbranca808b12020-05-15 17:22:28 +01002978 share_state->fragment_constituent_counts,
2979 share_state->fragment_count, page_pool, clear);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002980
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002981 if (ret.func == FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002982 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01002983 * Mark memory handle as not retrieved, so it can be
2984 * reclaimed (or retrieved again).
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002985 */
J-Alves8eb19162022-04-28 10:56:48 +01002986 share_state->retrieved_fragment_count[receiver_index] = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002987 }
2988
2989out:
2990 share_states_unlock(&share_states);
2991 dump_share_states();
2992 return ret;
2993}
2994
2995/**
J-Alvesa9cd7e32022-07-01 13:49:33 +01002996 * Validates that the reclaim transition is allowed for the given
2997 * handle, updates the page table of the reclaiming VM, and frees the
2998 * internal state associated with the handle.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002999 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003000struct ffa_value ffa_memory_reclaim(struct vm_locked to_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01003001 ffa_memory_handle_t handle,
3002 ffa_memory_region_flags_t flags,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003003 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003004{
3005 struct share_states_locked share_states;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003006 struct ffa_memory_share_state *share_state;
3007 struct ffa_memory_region *memory_region;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003008 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003009
3010 dump_share_states();
3011
3012 share_states = share_states_lock();
Karl Meakin52cdfe72023-06-30 14:49:10 +01003013
Karl Meakin4a2854a2023-06-30 16:26:52 +01003014 share_state = get_share_state(share_states, handle);
3015 if (!share_state) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003016 dlog_verbose("Invalid handle %#x for FFA_MEM_RECLAIM.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003017 handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003018 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003019 goto out;
3020 }
Karl Meakin4a2854a2023-06-30 16:26:52 +01003021 memory_region = share_state->memory_region;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003022
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003023 CHECK(memory_region != NULL);
3024
J-Alvesa9cd7e32022-07-01 13:49:33 +01003025 if (vm_id_is_current_world(to_locked.vm->id) &&
3026 to_locked.vm->id != memory_region->sender) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003027 dlog_verbose(
Olivier Deprezf92e5d42020-11-13 16:00:54 +01003028 "VM %#x attempted to reclaim memory handle %#x "
3029 "originally sent by VM %#x.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003030 to_locked.vm->id, handle, memory_region->sender);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003031 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003032 goto out;
3033 }
3034
Andrew Walbranca808b12020-05-15 17:22:28 +01003035 if (!share_state->sending_complete) {
3036 dlog_verbose(
3037 "Memory with handle %#x not fully sent, can't "
3038 "reclaim.\n",
3039 handle);
3040 ret = ffa_error(FFA_INVALID_PARAMETERS);
3041 goto out;
3042 }
3043
J-Alves752236c2022-04-28 11:07:47 +01003044 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
3045 if (share_state->retrieved_fragment_count[i] != 0) {
3046 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01003047 "Tried to reclaim memory handle %#x "
J-Alves3c5b2072022-11-21 12:45:40 +00003048 "that has not been relinquished by all "
J-Alvesa9cd7e32022-07-01 13:49:33 +01003049 "borrowers(%x).\n",
J-Alves752236c2022-04-28 11:07:47 +01003050 handle,
3051 memory_region->receivers[i]
3052 .receiver_permissions.receiver);
3053 ret = ffa_error(FFA_DENIED);
3054 goto out;
3055 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003056 }
3057
Andrew Walbranca808b12020-05-15 17:22:28 +01003058 ret = ffa_retrieve_check_update(
J-Alves7db32002021-12-14 14:44:50 +00003059 to_locked, memory_region->sender, share_state->fragments,
Andrew Walbranca808b12020-05-15 17:22:28 +01003060 share_state->fragment_constituent_counts,
J-Alves2a0d2882020-10-29 14:49:50 +00003061 share_state->fragment_count, share_state->sender_orig_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +01003062 FFA_MEM_RECLAIM_32, flags & FFA_MEM_RECLAIM_CLEAR, page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003063
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003064 if (ret.func == FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003065 share_state_free(share_states, share_state, page_pool);
J-Alves3c5b2072022-11-21 12:45:40 +00003066 dlog_verbose("Freed share state after successful reclaim.\n");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003067 }
3068
3069out:
3070 share_states_unlock(&share_states);
3071 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01003072}