blob: 238dabbe63f44bf708e0737a512e53522cfe8c44 [file] [log] [blame]
Jose Marinho75509b42019-04-09 09:34:59 +01001/*
2 * Copyright 2019 The Hafnium Authors.
3 *
Andrew Walbrane959ec12020-06-17 15:01:09 +01004 * Use of this source code is governed by a BSD-style
5 * license that can be found in the LICENSE file or at
6 * https://opensource.org/licenses/BSD-3-Clause.
Jose Marinho75509b42019-04-09 09:34:59 +01007 */
8
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01009#include "hf/ffa_memory.h"
Andrew Walbran475c1452020-02-07 13:22:22 +000010
Federico Recanati4fd065d2021-12-13 20:06:23 +010011#include "hf/arch/mm.h"
Olivier Deprez112d2b52020-09-30 07:39:23 +020012#include "hf/arch/other_world.h"
Olivier Deprez55a189e2021-06-09 15:45:27 +020013#include "hf/arch/plat/ffa.h"
Andrew Walbran290b0c92020-02-03 16:37:14 +000014
J-Alves5952d942022-12-22 16:03:00 +000015#include "hf/addr.h"
Jose Marinho75509b42019-04-09 09:34:59 +010016#include "hf/api.h"
Daniel Boulbya2f8c662021-11-26 17:52:53 +000017#include "hf/assert.h"
Jose Marinho09b1db82019-08-08 09:16:59 +010018#include "hf/check.h"
Jose Marinho75509b42019-04-09 09:34:59 +010019#include "hf/dlog.h"
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010020#include "hf/ffa_internal.h"
J-Alves66652252022-07-06 09:49:51 +010021#include "hf/ffa_memory_internal.h"
J-Alves5952d942022-12-22 16:03:00 +000022#include "hf/mm.h"
Andrew Walbran475c1452020-02-07 13:22:22 +000023#include "hf/mpool.h"
Jose Marinho75509b42019-04-09 09:34:59 +010024#include "hf/std.h"
Andrew Scull3c257452019-11-26 13:32:50 +000025#include "hf/vm.h"
Jose Marinho75509b42019-04-09 09:34:59 +010026
J-Alves2d8457f2022-10-05 11:06:41 +010027#include "vmapi/hf/ffa_v1_0.h"
28
J-Alves5da37d92022-10-24 16:33:48 +010029#define RECEIVERS_COUNT_IN_RETRIEVE_RESP 1
30
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000031/**
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010032 * All access to members of a `struct ffa_memory_share_state` must be guarded
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000033 * by this lock.
34 */
35static struct spinlock share_states_lock_instance = SPINLOCK_INIT;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010036static struct ffa_memory_share_state share_states[MAX_MEM_SHARES];
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000037
38/**
J-Alves917d2f22020-10-30 18:39:30 +000039 * Extracts the index from a memory handle allocated by Hafnium's current world.
40 */
41uint64_t ffa_memory_handle_get_index(ffa_memory_handle_t handle)
42{
43 return handle & ~FFA_MEMORY_HANDLE_ALLOCATOR_MASK;
44}
45
46/**
Karl Meakin52cdfe72023-06-30 14:49:10 +010047 * Initialises the next available `struct ffa_memory_share_state`. If `handle`
48 * is `FFA_MEMORY_HANDLE_INVALID` then allocates an appropriate handle,
49 * otherwise uses the provided handle which is assumed to be globally unique.
Andrew Walbranca808b12020-05-15 17:22:28 +010050 *
Karl Meakin52cdfe72023-06-30 14:49:10 +010051 * Returns a pointer to the allocated `ffa_memory_share_state` on success or
52 * `NULL` if none are available.
Andrew Walbranca808b12020-05-15 17:22:28 +010053 */
Karl Meakin52cdfe72023-06-30 14:49:10 +010054struct ffa_memory_share_state *allocate_share_state(
55 struct share_states_locked share_states, uint32_t share_func,
56 struct ffa_memory_region *memory_region, uint32_t fragment_length,
57 ffa_memory_handle_t handle)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000058{
Daniel Boulbya2f8c662021-11-26 17:52:53 +000059 assert(share_states.share_states != NULL);
60 assert(memory_region != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000061
Karl Meakin52cdfe72023-06-30 14:49:10 +010062 for (uint64_t i = 0; i < MAX_MEM_SHARES; ++i) {
Andrew Walbranca808b12020-05-15 17:22:28 +010063 if (share_states.share_states[i].share_func == 0) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010064 struct ffa_memory_share_state *allocated_state =
Andrew Walbranca808b12020-05-15 17:22:28 +010065 &share_states.share_states[i];
66 struct ffa_composite_memory_region *composite =
67 ffa_memory_region_get_composite(memory_region,
68 0);
69
70 if (handle == FFA_MEMORY_HANDLE_INVALID) {
J-Alvesee68c542020-10-29 17:48:20 +000071 memory_region->handle =
Olivier Deprez55a189e2021-06-09 15:45:27 +020072 plat_ffa_memory_handle_make(i);
Andrew Walbranca808b12020-05-15 17:22:28 +010073 } else {
J-Alvesee68c542020-10-29 17:48:20 +000074 memory_region->handle = handle;
Andrew Walbranca808b12020-05-15 17:22:28 +010075 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000076 allocated_state->share_func = share_func;
77 allocated_state->memory_region = memory_region;
Andrew Walbranca808b12020-05-15 17:22:28 +010078 allocated_state->fragment_count = 1;
79 allocated_state->fragments[0] = composite->constituents;
80 allocated_state->fragment_constituent_counts[0] =
81 (fragment_length -
82 ffa_composite_constituent_offset(memory_region,
83 0)) /
84 sizeof(struct ffa_memory_region_constituent);
85 allocated_state->sending_complete = false;
Karl Meakin52cdfe72023-06-30 14:49:10 +010086 for (uint32_t j = 0; j < MAX_MEM_SHARE_RECIPIENTS;
87 ++j) {
Andrew Walbranca808b12020-05-15 17:22:28 +010088 allocated_state->retrieved_fragment_count[j] =
89 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000090 }
Karl Meakin52cdfe72023-06-30 14:49:10 +010091 return allocated_state;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000092 }
93 }
94
Karl Meakin52cdfe72023-06-30 14:49:10 +010095 return NULL;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000096}
97
98/** Locks the share states lock. */
99struct share_states_locked share_states_lock(void)
100{
101 sl_lock(&share_states_lock_instance);
102
103 return (struct share_states_locked){.share_states = share_states};
104}
105
106/** Unlocks the share states lock. */
J-Alves66652252022-07-06 09:49:51 +0100107void share_states_unlock(struct share_states_locked *share_states)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000108{
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000109 assert(share_states->share_states != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000110 share_states->share_states = NULL;
111 sl_unlock(&share_states_lock_instance);
112}
113
114/**
Andrew Walbranca808b12020-05-15 17:22:28 +0100115 * If the given handle is a valid handle for an allocated share state then
Karl Meakin4a2854a2023-06-30 16:26:52 +0100116 * returns a pointer to the share state. Otherwise returns NULL.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000117 */
Karl Meakin4a2854a2023-06-30 16:26:52 +0100118struct ffa_memory_share_state *get_share_state(
119 struct share_states_locked share_states, ffa_memory_handle_t handle)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000120{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100121 struct ffa_memory_share_state *share_state;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000122
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000123 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100124
125 /*
126 * First look for a share_state allocated by us, in which case the
127 * handle is based on the index.
128 */
Olivier Deprez55a189e2021-06-09 15:45:27 +0200129 if (plat_ffa_memory_handle_allocated_by_current_world(handle)) {
Karl Meakin4a2854a2023-06-30 16:26:52 +0100130 uint64_t index = ffa_memory_handle_get_index(handle);
131
Andrew Walbranca808b12020-05-15 17:22:28 +0100132 if (index < MAX_MEM_SHARES) {
133 share_state = &share_states.share_states[index];
134 if (share_state->share_func != 0) {
Karl Meakin4a2854a2023-06-30 16:26:52 +0100135 return share_state;
Andrew Walbranca808b12020-05-15 17:22:28 +0100136 }
137 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000138 }
139
Andrew Walbranca808b12020-05-15 17:22:28 +0100140 /* Fall back to a linear scan. */
Karl Meakin4a2854a2023-06-30 16:26:52 +0100141 for (uint64_t index = 0; index < MAX_MEM_SHARES; ++index) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100142 share_state = &share_states.share_states[index];
J-Alvesee68c542020-10-29 17:48:20 +0000143 if (share_state->memory_region != NULL &&
144 share_state->memory_region->handle == handle &&
Andrew Walbranca808b12020-05-15 17:22:28 +0100145 share_state->share_func != 0) {
Karl Meakin4a2854a2023-06-30 16:26:52 +0100146 return share_state;
Andrew Walbranca808b12020-05-15 17:22:28 +0100147 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000148 }
149
Karl Meakin4a2854a2023-06-30 16:26:52 +0100150 return NULL;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000151}
152
153/** Marks a share state as unallocated. */
J-Alvesfdd29272022-07-19 13:16:31 +0100154void share_state_free(struct share_states_locked share_states,
155 struct ffa_memory_share_state *share_state,
156 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000157{
Andrew Walbranca808b12020-05-15 17:22:28 +0100158 uint32_t i;
159
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000160 assert(share_states.share_states != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000161 share_state->share_func = 0;
Andrew Walbranca808b12020-05-15 17:22:28 +0100162 share_state->sending_complete = false;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000163 mpool_free(page_pool, share_state->memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +0100164 /*
165 * First fragment is part of the same page as the `memory_region`, so it
166 * doesn't need to be freed separately.
167 */
168 share_state->fragments[0] = NULL;
169 share_state->fragment_constituent_counts[0] = 0;
170 for (i = 1; i < share_state->fragment_count; ++i) {
171 mpool_free(page_pool, share_state->fragments[i]);
172 share_state->fragments[i] = NULL;
173 share_state->fragment_constituent_counts[i] = 0;
174 }
175 share_state->fragment_count = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000176 share_state->memory_region = NULL;
J-Alvesa9cd7e32022-07-01 13:49:33 +0100177 share_state->hypervisor_fragment_count = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000178}
179
Andrew Walbranca808b12020-05-15 17:22:28 +0100180/** Checks whether the given share state has been fully sent. */
J-Alvesfdd29272022-07-19 13:16:31 +0100181bool share_state_sending_complete(struct share_states_locked share_states,
182 struct ffa_memory_share_state *share_state)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000183{
Andrew Walbranca808b12020-05-15 17:22:28 +0100184 struct ffa_composite_memory_region *composite;
185 uint32_t expected_constituent_count;
186 uint32_t fragment_constituent_count_total = 0;
187 uint32_t i;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000188
Andrew Walbranca808b12020-05-15 17:22:28 +0100189 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000190 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100191
192 /*
193 * Share state must already be valid, or it's not possible to get hold
194 * of it.
195 */
196 CHECK(share_state->memory_region != NULL &&
197 share_state->share_func != 0);
198
199 composite =
200 ffa_memory_region_get_composite(share_state->memory_region, 0);
201 expected_constituent_count = composite->constituent_count;
202 for (i = 0; i < share_state->fragment_count; ++i) {
203 fragment_constituent_count_total +=
204 share_state->fragment_constituent_counts[i];
205 }
206 dlog_verbose(
207 "Checking completion: constituent count %d/%d from %d "
208 "fragments.\n",
209 fragment_constituent_count_total, expected_constituent_count,
210 share_state->fragment_count);
211
212 return fragment_constituent_count_total == expected_constituent_count;
213}
214
215/**
216 * Calculates the offset of the next fragment expected for the given share
217 * state.
218 */
J-Alvesfdd29272022-07-19 13:16:31 +0100219uint32_t share_state_next_fragment_offset(
Andrew Walbranca808b12020-05-15 17:22:28 +0100220 struct share_states_locked share_states,
221 struct ffa_memory_share_state *share_state)
222{
223 uint32_t next_fragment_offset;
224 uint32_t i;
225
226 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000227 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100228
229 next_fragment_offset =
230 ffa_composite_constituent_offset(share_state->memory_region, 0);
231 for (i = 0; i < share_state->fragment_count; ++i) {
232 next_fragment_offset +=
233 share_state->fragment_constituent_counts[i] *
234 sizeof(struct ffa_memory_region_constituent);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000235 }
236
Andrew Walbranca808b12020-05-15 17:22:28 +0100237 return next_fragment_offset;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000238}
239
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100240static void dump_memory_region(struct ffa_memory_region *memory_region)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000241{
242 uint32_t i;
243
244 if (LOG_LEVEL < LOG_LEVEL_VERBOSE) {
245 return;
246 }
247
Olivier Deprez935e1b12020-12-22 18:01:29 +0100248 dlog("from VM %#x, attributes %#x, flags %#x, tag %u, to "
Olivier Deprezf92e5d42020-11-13 16:00:54 +0100249 "%u "
Andrew Walbrana65a1322020-04-06 19:32:32 +0100250 "recipients [",
251 memory_region->sender, memory_region->attributes,
Olivier Deprez935e1b12020-12-22 18:01:29 +0100252 memory_region->flags, memory_region->tag,
Andrew Walbrana65a1322020-04-06 19:32:32 +0100253 memory_region->receiver_count);
254 for (i = 0; i < memory_region->receiver_count; ++i) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000255 if (i != 0) {
256 dlog(", ");
257 }
Olivier Deprezf92e5d42020-11-13 16:00:54 +0100258 dlog("VM %#x: %#x (offset %u)",
Andrew Walbrana65a1322020-04-06 19:32:32 +0100259 memory_region->receivers[i].receiver_permissions.receiver,
260 memory_region->receivers[i]
261 .receiver_permissions.permissions,
262 memory_region->receivers[i]
263 .composite_memory_region_offset);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000264 }
265 dlog("]");
266}
267
J-Alves66652252022-07-06 09:49:51 +0100268void dump_share_states(void)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000269{
270 uint32_t i;
271
272 if (LOG_LEVEL < LOG_LEVEL_VERBOSE) {
273 return;
274 }
275
276 dlog("Current share states:\n");
277 sl_lock(&share_states_lock_instance);
278 for (i = 0; i < MAX_MEM_SHARES; ++i) {
279 if (share_states[i].share_func != 0) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000280 switch (share_states[i].share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100281 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000282 dlog("SHARE");
283 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100284 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000285 dlog("LEND");
286 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100287 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000288 dlog("DONATE");
289 break;
290 default:
291 dlog("invalid share_func %#x",
292 share_states[i].share_func);
293 }
Olivier Deprez935e1b12020-12-22 18:01:29 +0100294 dlog(" %#x (", share_states[i].memory_region->handle);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000295 dump_memory_region(share_states[i].memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +0100296 if (share_states[i].sending_complete) {
297 dlog("): fully sent");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000298 } else {
Andrew Walbranca808b12020-05-15 17:22:28 +0100299 dlog("): partially sent");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000300 }
J-Alves2a0d2882020-10-29 14:49:50 +0000301 dlog(" with %d fragments, %d retrieved, "
302 " sender's original mode: %#x\n",
Andrew Walbranca808b12020-05-15 17:22:28 +0100303 share_states[i].fragment_count,
J-Alves2a0d2882020-10-29 14:49:50 +0000304 share_states[i].retrieved_fragment_count[0],
305 share_states[i].sender_orig_mode);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000306 }
307 }
308 sl_unlock(&share_states_lock_instance);
309}
310
Andrew Walbran475c1452020-02-07 13:22:22 +0000311/* TODO: Add device attributes: GRE, cacheability, shareability. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100312static inline uint32_t ffa_memory_permissions_to_mode(
J-Alves7cd5eb32020-10-16 19:06:10 +0100313 ffa_memory_access_permissions_t permissions, uint32_t default_mode)
Andrew Walbran475c1452020-02-07 13:22:22 +0000314{
315 uint32_t mode = 0;
316
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100317 switch (ffa_get_data_access_attr(permissions)) {
318 case FFA_DATA_ACCESS_RO:
Andrew Walbran475c1452020-02-07 13:22:22 +0000319 mode = MM_MODE_R;
320 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100321 case FFA_DATA_ACCESS_RW:
Andrew Walbran475c1452020-02-07 13:22:22 +0000322 mode = MM_MODE_R | MM_MODE_W;
323 break;
J-Alves7cd5eb32020-10-16 19:06:10 +0100324 case FFA_DATA_ACCESS_NOT_SPECIFIED:
325 mode = (default_mode & (MM_MODE_R | MM_MODE_W));
326 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100327 case FFA_DATA_ACCESS_RESERVED:
328 panic("Tried to convert FFA_DATA_ACCESS_RESERVED.");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100329 }
330
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100331 switch (ffa_get_instruction_access_attr(permissions)) {
332 case FFA_INSTRUCTION_ACCESS_NX:
Andrew Walbran475c1452020-02-07 13:22:22 +0000333 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100334 case FFA_INSTRUCTION_ACCESS_X:
Andrew Walbrana65a1322020-04-06 19:32:32 +0100335 mode |= MM_MODE_X;
336 break;
J-Alves7cd5eb32020-10-16 19:06:10 +0100337 case FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED:
338 mode |= (default_mode & MM_MODE_X);
339 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100340 case FFA_INSTRUCTION_ACCESS_RESERVED:
341 panic("Tried to convert FFA_INSTRUCTION_ACCESS_RESVERVED.");
Andrew Walbran475c1452020-02-07 13:22:22 +0000342 }
343
Olivier Deprez878bd5b2021-04-15 19:05:10 +0200344 /* Set the security state bit if necessary. */
345 if ((default_mode & plat_ffa_other_world_mode()) != 0) {
346 mode |= plat_ffa_other_world_mode();
347 }
348
Andrew Walbran475c1452020-02-07 13:22:22 +0000349 return mode;
350}
351
Jose Marinho75509b42019-04-09 09:34:59 +0100352/**
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000353 * Get the current mode in the stage-2 page table of the given vm of all the
354 * pages in the given constituents, if they all have the same mode, or return
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100355 * an appropriate FF-A error if not.
Jose Marinho75509b42019-04-09 09:34:59 +0100356 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100357static struct ffa_value constituents_get_mode(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000358 struct vm_locked vm, uint32_t *orig_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100359 struct ffa_memory_region_constituent **fragments,
360 const uint32_t *fragment_constituent_counts, uint32_t fragment_count)
Jose Marinho75509b42019-04-09 09:34:59 +0100361{
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100362 uint32_t i;
Andrew Walbranca808b12020-05-15 17:22:28 +0100363 uint32_t j;
Jose Marinho75509b42019-04-09 09:34:59 +0100364
Andrew Walbranca808b12020-05-15 17:22:28 +0100365 if (fragment_count == 0 || fragment_constituent_counts[0] == 0) {
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100366 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000367 * Fail if there are no constituents. Otherwise we would get an
368 * uninitialised *orig_mode.
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100369 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100370 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho75509b42019-04-09 09:34:59 +0100371 }
372
Andrew Walbranca808b12020-05-15 17:22:28 +0100373 for (i = 0; i < fragment_count; ++i) {
374 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
375 ipaddr_t begin = ipa_init(fragments[i][j].address);
376 size_t size = fragments[i][j].page_count * PAGE_SIZE;
377 ipaddr_t end = ipa_add(begin, size);
378 uint32_t current_mode;
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100379
Andrew Walbranca808b12020-05-15 17:22:28 +0100380 /* Fail if addresses are not page-aligned. */
381 if (!is_aligned(ipa_addr(begin), PAGE_SIZE) ||
382 !is_aligned(ipa_addr(end), PAGE_SIZE)) {
383 return ffa_error(FFA_INVALID_PARAMETERS);
384 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100385
Andrew Walbranca808b12020-05-15 17:22:28 +0100386 /*
387 * Ensure that this constituent memory range is all
388 * mapped with the same mode.
389 */
Raghu Krishnamurthy785d52f2021-02-13 00:02:40 -0800390 if (!vm_mem_get_mode(vm, begin, end, &current_mode)) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100391 return ffa_error(FFA_DENIED);
392 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100393
Andrew Walbranca808b12020-05-15 17:22:28 +0100394 /*
395 * Ensure that all constituents are mapped with the same
396 * mode.
397 */
398 if (i == 0) {
399 *orig_mode = current_mode;
400 } else if (current_mode != *orig_mode) {
401 dlog_verbose(
402 "Expected mode %#x but was %#x for %d "
403 "pages at %#x.\n",
404 *orig_mode, current_mode,
405 fragments[i][j].page_count,
406 ipa_addr(begin));
407 return ffa_error(FFA_DENIED);
408 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100409 }
Jose Marinho75509b42019-04-09 09:34:59 +0100410 }
411
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100412 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000413}
414
415/**
416 * Verify that all pages have the same mode, that the starting mode
417 * constitutes a valid state and obtain the next mode to apply
418 * to the sending VM.
419 *
420 * Returns:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100421 * 1) FFA_DENIED if a state transition was not found;
422 * 2) FFA_DENIED if the pages being shared do not have the same mode within
Andrew Walbrana65a1322020-04-06 19:32:32 +0100423 * the <from> VM;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100424 * 3) FFA_INVALID_PARAMETERS if the beginning and end IPAs are not page
Andrew Walbrana65a1322020-04-06 19:32:32 +0100425 * aligned;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100426 * 4) FFA_INVALID_PARAMETERS if the requested share type was not handled.
427 * Or FFA_SUCCESS on success.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000428 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100429static struct ffa_value ffa_send_check_transition(
Andrew Walbrana65a1322020-04-06 19:32:32 +0100430 struct vm_locked from, uint32_t share_func,
J-Alves363f5722022-04-25 17:37:37 +0100431 struct ffa_memory_access *receivers, uint32_t receivers_count,
432 uint32_t *orig_from_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100433 struct ffa_memory_region_constituent **fragments,
434 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
435 uint32_t *from_mode)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000436{
437 const uint32_t state_mask =
438 MM_MODE_INVALID | MM_MODE_UNOWNED | MM_MODE_SHARED;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100439 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000440
Andrew Walbranca808b12020-05-15 17:22:28 +0100441 ret = constituents_get_mode(from, orig_from_mode, fragments,
442 fragment_constituent_counts,
443 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100444 if (ret.func != FFA_SUCCESS_32) {
Olivier Depreze7eb1682022-03-16 17:09:03 +0100445 dlog_verbose("Inconsistent modes.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100446 return ret;
Andrew Scullb5f49e02019-10-02 13:20:47 +0100447 }
448
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000449 /* Ensure the address range is normal memory and not a device. */
450 if (*orig_from_mode & MM_MODE_D) {
451 dlog_verbose("Can't share device memory (mode is %#x).\n",
452 *orig_from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100453 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000454 }
455
456 /*
457 * Ensure the sender is the owner and has exclusive access to the
458 * memory.
459 */
460 if ((*orig_from_mode & state_mask) != 0) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100461 return ffa_error(FFA_DENIED);
Andrew Walbrana65a1322020-04-06 19:32:32 +0100462 }
463
J-Alves363f5722022-04-25 17:37:37 +0100464 assert(receivers != NULL && receivers_count > 0U);
J-Alves7cd5eb32020-10-16 19:06:10 +0100465
J-Alves363f5722022-04-25 17:37:37 +0100466 for (uint32_t i = 0U; i < receivers_count; i++) {
467 ffa_memory_access_permissions_t permissions =
468 receivers[i].receiver_permissions.permissions;
469 uint32_t required_from_mode = ffa_memory_permissions_to_mode(
470 permissions, *orig_from_mode);
471
472 if ((*orig_from_mode & required_from_mode) !=
473 required_from_mode) {
474 dlog_verbose(
475 "Sender tried to send memory with permissions "
476 "which "
477 "required mode %#x but only had %#x itself.\n",
478 required_from_mode, *orig_from_mode);
479 return ffa_error(FFA_DENIED);
480 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000481 }
482
483 /* Find the appropriate new mode. */
484 *from_mode = ~state_mask & *orig_from_mode;
Andrew Walbrane7ad3c02019-12-24 17:03:04 +0000485 switch (share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100486 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000487 *from_mode |= MM_MODE_INVALID | MM_MODE_UNOWNED;
Jose Marinho75509b42019-04-09 09:34:59 +0100488 break;
489
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100490 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000491 *from_mode |= MM_MODE_INVALID;
Andrew Walbran648fc3e2019-10-22 16:23:05 +0100492 break;
493
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100494 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000495 *from_mode |= MM_MODE_SHARED;
Jose Marinho56c25732019-05-20 09:48:53 +0100496 break;
497
Jose Marinho75509b42019-04-09 09:34:59 +0100498 default:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100499 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho75509b42019-04-09 09:34:59 +0100500 }
501
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100502 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000503}
504
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100505static struct ffa_value ffa_relinquish_check_transition(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000506 struct vm_locked from, uint32_t *orig_from_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100507 struct ffa_memory_region_constituent **fragments,
508 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
509 uint32_t *from_mode)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000510{
511 const uint32_t state_mask =
512 MM_MODE_INVALID | MM_MODE_UNOWNED | MM_MODE_SHARED;
513 uint32_t orig_from_state;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100514 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000515
Andrew Walbranca808b12020-05-15 17:22:28 +0100516 ret = constituents_get_mode(from, orig_from_mode, fragments,
517 fragment_constituent_counts,
518 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100519 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbrana65a1322020-04-06 19:32:32 +0100520 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000521 }
522
523 /* Ensure the address range is normal memory and not a device. */
524 if (*orig_from_mode & MM_MODE_D) {
525 dlog_verbose("Can't relinquish device memory (mode is %#x).\n",
526 *orig_from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100527 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000528 }
529
530 /*
531 * Ensure the relinquishing VM is not the owner but has access to the
532 * memory.
533 */
534 orig_from_state = *orig_from_mode & state_mask;
535 if ((orig_from_state & ~MM_MODE_SHARED) != MM_MODE_UNOWNED) {
536 dlog_verbose(
537 "Tried to relinquish memory in state %#x (masked %#x "
Andrew Walbranca808b12020-05-15 17:22:28 +0100538 "but should be %#x).\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000539 *orig_from_mode, orig_from_state, MM_MODE_UNOWNED);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100540 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000541 }
542
543 /* Find the appropriate new mode. */
544 *from_mode = (~state_mask & *orig_from_mode) | MM_MODE_UNMAPPED_MASK;
545
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100546 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000547}
548
549/**
550 * Verify that all pages have the same mode, that the starting mode
551 * constitutes a valid state and obtain the next mode to apply
552 * to the retrieving VM.
553 *
554 * Returns:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100555 * 1) FFA_DENIED if a state transition was not found;
556 * 2) FFA_DENIED if the pages being shared do not have the same mode within
Andrew Walbrana65a1322020-04-06 19:32:32 +0100557 * the <to> VM;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100558 * 3) FFA_INVALID_PARAMETERS if the beginning and end IPAs are not page
Andrew Walbrana65a1322020-04-06 19:32:32 +0100559 * aligned;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100560 * 4) FFA_INVALID_PARAMETERS if the requested share type was not handled.
561 * Or FFA_SUCCESS on success.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000562 */
J-Alvesfc19b372022-07-06 12:17:35 +0100563struct ffa_value ffa_retrieve_check_transition(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000564 struct vm_locked to, uint32_t share_func,
Andrew Walbranca808b12020-05-15 17:22:28 +0100565 struct ffa_memory_region_constituent **fragments,
566 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
567 uint32_t memory_to_attributes, uint32_t *to_mode)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000568{
569 uint32_t orig_to_mode;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100570 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000571
Andrew Walbranca808b12020-05-15 17:22:28 +0100572 ret = constituents_get_mode(to, &orig_to_mode, fragments,
573 fragment_constituent_counts,
574 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100575 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100576 dlog_verbose("Inconsistent modes.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100577 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000578 }
579
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100580 if (share_func == FFA_MEM_RECLAIM_32) {
J-Alves9256f162021-12-09 13:18:43 +0000581 /*
582 * If the original ffa memory send call has been processed
583 * successfully, it is expected the orig_to_mode would overlay
584 * with `state_mask`, as a result of the function
585 * `ffa_send_check_transition`.
586 */
J-Alves59ed0042022-07-28 18:26:41 +0100587 if (vm_id_is_current_world(to.vm->id)) {
588 assert((orig_to_mode &
589 (MM_MODE_INVALID | MM_MODE_UNOWNED |
590 MM_MODE_SHARED)) != 0U);
591 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000592 } else {
593 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +0100594 * If the retriever is from virtual FF-A instance:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000595 * Ensure the retriever has the expected state. We don't care
596 * about the MM_MODE_SHARED bit; either with or without it set
597 * are both valid representations of the !O-NA state.
598 */
J-Alvesa9cd7e32022-07-01 13:49:33 +0100599 if (vm_id_is_current_world(to.vm->id) &&
600 to.vm->id != HF_PRIMARY_VM_ID &&
601 (orig_to_mode & MM_MODE_UNMAPPED_MASK) !=
602 MM_MODE_UNMAPPED_MASK) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100603 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000604 }
605 }
606
607 /* Find the appropriate new mode. */
608 *to_mode = memory_to_attributes;
609 switch (share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100610 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000611 *to_mode |= 0;
612 break;
613
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100614 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000615 *to_mode |= MM_MODE_UNOWNED;
616 break;
617
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100618 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000619 *to_mode |= MM_MODE_UNOWNED | MM_MODE_SHARED;
620 break;
621
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100622 case FFA_MEM_RECLAIM_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000623 *to_mode |= 0;
624 break;
625
626 default:
Andrew Walbranca808b12020-05-15 17:22:28 +0100627 dlog_error("Invalid share_func %#x.\n", share_func);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100628 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000629 }
630
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100631 return (struct ffa_value){.func = FFA_SUCCESS_32};
Jose Marinho75509b42019-04-09 09:34:59 +0100632}
Jose Marinho09b1db82019-08-08 09:16:59 +0100633
634/**
635 * Updates a VM's page table such that the given set of physical address ranges
636 * are mapped in the address space at the corresponding address ranges, in the
637 * mode provided.
638 *
639 * If commit is false, the page tables will be allocated from the mpool but no
640 * mappings will actually be updated. This function must always be called first
641 * with commit false to check that it will succeed before calling with commit
642 * true, to avoid leaving the page table in a half-updated state. To make a
643 * series of changes atomically you can call them all with commit false before
644 * calling them all with commit true.
645 *
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -0700646 * vm_ptable_defrag should always be called after a series of page table
647 * updates, whether they succeed or fail.
Jose Marinho09b1db82019-08-08 09:16:59 +0100648 *
649 * Returns true on success, or false if the update failed and no changes were
650 * made to memory mappings.
651 */
J-Alves66652252022-07-06 09:49:51 +0100652bool ffa_region_group_identity_map(
Andrew Walbranf4b51af2020-02-03 14:44:54 +0000653 struct vm_locked vm_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +0100654 struct ffa_memory_region_constituent **fragments,
655 const uint32_t *fragment_constituent_counts, uint32_t fragment_count,
Daniel Boulby4dd3f532021-09-21 09:57:08 +0100656 uint32_t mode, struct mpool *ppool, bool commit)
Jose Marinho09b1db82019-08-08 09:16:59 +0100657{
Andrew Walbranca808b12020-05-15 17:22:28 +0100658 uint32_t i;
659 uint32_t j;
Jose Marinho09b1db82019-08-08 09:16:59 +0100660
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -0700661 if (vm_locked.vm->el0_partition) {
662 mode |= MM_MODE_USER | MM_MODE_NG;
663 }
664
Andrew Walbranca808b12020-05-15 17:22:28 +0100665 /* Iterate over the memory region constituents within each fragment. */
666 for (i = 0; i < fragment_count; ++i) {
667 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
668 size_t size = fragments[i][j].page_count * PAGE_SIZE;
669 paddr_t pa_begin =
670 pa_from_ipa(ipa_init(fragments[i][j].address));
671 paddr_t pa_end = pa_add(pa_begin, size);
Jens Wiklander4f1880c2022-10-19 17:00:14 +0200672 uint32_t pa_bits =
673 arch_mm_get_pa_bits(arch_mm_get_pa_range());
Federico Recanati4fd065d2021-12-13 20:06:23 +0100674
675 /*
676 * Ensure the requested region falls into system's PA
677 * range.
678 */
Jens Wiklander4f1880c2022-10-19 17:00:14 +0200679 if (((pa_addr(pa_begin) >> pa_bits) > 0) ||
680 ((pa_addr(pa_end) >> pa_bits) > 0)) {
Federico Recanati4fd065d2021-12-13 20:06:23 +0100681 dlog_error("Region is outside of PA Range\n");
682 return false;
683 }
Andrew Walbranca808b12020-05-15 17:22:28 +0100684
685 if (commit) {
686 vm_identity_commit(vm_locked, pa_begin, pa_end,
687 mode, ppool, NULL);
688 } else if (!vm_identity_prepare(vm_locked, pa_begin,
689 pa_end, mode, ppool)) {
690 return false;
691 }
Jose Marinho09b1db82019-08-08 09:16:59 +0100692 }
693 }
694
695 return true;
696}
697
698/**
699 * Clears a region of physical memory by overwriting it with zeros. The data is
700 * flushed from the cache so the memory has been cleared across the system.
701 */
J-Alves7db32002021-12-14 14:44:50 +0000702static bool clear_memory(paddr_t begin, paddr_t end, struct mpool *ppool,
703 uint32_t extra_mode_attributes)
Jose Marinho09b1db82019-08-08 09:16:59 +0100704{
705 /*
Fuad Tabbaed294af2019-12-20 10:43:01 +0000706 * TODO: change this to a CPU local single page window rather than a
Jose Marinho09b1db82019-08-08 09:16:59 +0100707 * global mapping of the whole range. Such an approach will limit
708 * the changes to stage-1 tables and will allow only local
709 * invalidation.
710 */
711 bool ret;
712 struct mm_stage1_locked stage1_locked = mm_lock_stage1();
J-Alves7db32002021-12-14 14:44:50 +0000713 void *ptr = mm_identity_map(stage1_locked, begin, end,
714 MM_MODE_W | (extra_mode_attributes &
715 plat_ffa_other_world_mode()),
716 ppool);
Jose Marinho09b1db82019-08-08 09:16:59 +0100717 size_t size = pa_difference(begin, end);
718
719 if (!ptr) {
Jose Marinho09b1db82019-08-08 09:16:59 +0100720 goto fail;
721 }
722
723 memset_s(ptr, size, 0, size);
724 arch_mm_flush_dcache(ptr, size);
725 mm_unmap(stage1_locked, begin, end, ppool);
726
727 ret = true;
728 goto out;
729
730fail:
731 ret = false;
732
733out:
734 mm_unlock_stage1(&stage1_locked);
735
736 return ret;
737}
738
739/**
740 * Clears a region of physical memory by overwriting it with zeros. The data is
741 * flushed from the cache so the memory has been cleared across the system.
742 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100743static bool ffa_clear_memory_constituents(
J-Alves7db32002021-12-14 14:44:50 +0000744 uint32_t security_state_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100745 struct ffa_memory_region_constituent **fragments,
746 const uint32_t *fragment_constituent_counts, uint32_t fragment_count,
747 struct mpool *page_pool)
Jose Marinho09b1db82019-08-08 09:16:59 +0100748{
749 struct mpool local_page_pool;
Andrew Walbranca808b12020-05-15 17:22:28 +0100750 uint32_t i;
Jose Marinho09b1db82019-08-08 09:16:59 +0100751 bool ret = false;
752
753 /*
754 * Create a local pool so any freed memory can't be used by another
755 * thread. This is to ensure each constituent that is mapped can be
756 * unmapped again afterwards.
757 */
Andrew Walbran475c1452020-02-07 13:22:22 +0000758 mpool_init_with_fallback(&local_page_pool, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +0100759
Andrew Walbranca808b12020-05-15 17:22:28 +0100760 /* Iterate over the memory region constituents within each fragment. */
761 for (i = 0; i < fragment_count; ++i) {
762 uint32_t j;
Jose Marinho09b1db82019-08-08 09:16:59 +0100763
Andrew Walbranca808b12020-05-15 17:22:28 +0100764 for (j = 0; j < fragment_constituent_counts[j]; ++j) {
765 size_t size = fragments[i][j].page_count * PAGE_SIZE;
766 paddr_t begin =
767 pa_from_ipa(ipa_init(fragments[i][j].address));
768 paddr_t end = pa_add(begin, size);
769
J-Alves7db32002021-12-14 14:44:50 +0000770 if (!clear_memory(begin, end, &local_page_pool,
771 security_state_mode)) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100772 /*
773 * api_clear_memory will defrag on failure, so
774 * no need to do it here.
775 */
776 goto out;
777 }
Jose Marinho09b1db82019-08-08 09:16:59 +0100778 }
779 }
780
Jose Marinho09b1db82019-08-08 09:16:59 +0100781 ret = true;
782
783out:
784 mpool_fini(&local_page_pool);
785 return ret;
786}
787
J-Alves5952d942022-12-22 16:03:00 +0000788static bool is_memory_range_within(ipaddr_t begin, ipaddr_t end,
789 ipaddr_t in_begin, ipaddr_t in_end)
790{
791 return (ipa_addr(begin) >= ipa_addr(in_begin) &&
792 ipa_addr(begin) < ipa_addr(in_end)) ||
793 (ipa_addr(end) <= ipa_addr(in_end) &&
794 ipa_addr(end) > ipa_addr(in_begin));
795}
796
797/**
798 * Receives a memory range and looks for overlaps with the remainder
799 * constituents of the memory share/lend/donate operation. Assumes they are
800 * passed in order to avoid having to loop over all the elements at each call.
801 * The function only compares the received memory ranges with those that follow
802 * within the same fragment, and subsequent fragments from the same operation.
803 */
804static bool ffa_memory_check_overlap(
805 struct ffa_memory_region_constituent **fragments,
806 const uint32_t *fragment_constituent_counts,
807 const uint32_t fragment_count, const uint32_t current_fragment,
808 const uint32_t current_constituent)
809{
810 uint32_t i = current_fragment;
811 uint32_t j = current_constituent;
812 ipaddr_t current_begin = ipa_init(fragments[i][j].address);
813 const uint32_t current_page_count = fragments[i][j].page_count;
814 size_t current_size = current_page_count * PAGE_SIZE;
815 ipaddr_t current_end = ipa_add(current_begin, current_size - 1);
816
817 if (current_size == 0 ||
818 current_size > UINT64_MAX - ipa_addr(current_begin)) {
819 dlog_verbose("Invalid page count. Addr: %x page_count: %x\n",
820 current_begin, current_page_count);
821 return false;
822 }
823
824 for (; i < fragment_count; i++) {
825 j = (i == current_fragment) ? j + 1 : 0;
826
827 for (; j < fragment_constituent_counts[i]; j++) {
828 ipaddr_t begin = ipa_init(fragments[i][j].address);
829 const uint32_t page_count = fragments[i][j].page_count;
830 size_t size = page_count * PAGE_SIZE;
831 ipaddr_t end = ipa_add(begin, size - 1);
832
833 if (size == 0 || size > UINT64_MAX - ipa_addr(begin)) {
834 dlog_verbose(
835 "Invalid page count. Addr: %x "
836 "page_count: %x\n",
837 begin, page_count);
838 return false;
839 }
840
841 /*
842 * Check if current ranges is within begin and end, as
843 * well as the reverse. This should help optimize the
844 * loop, and reduce the number of iterations.
845 */
846 if (is_memory_range_within(begin, end, current_begin,
847 current_end) ||
848 is_memory_range_within(current_begin, current_end,
849 begin, end)) {
850 dlog_verbose(
851 "Overlapping memory ranges: %#x - %#x "
852 "with %#x - %#x\n",
853 ipa_addr(begin), ipa_addr(end),
854 ipa_addr(current_begin),
855 ipa_addr(current_end));
856 return true;
857 }
858 }
859 }
860
861 return false;
862}
863
Jose Marinho09b1db82019-08-08 09:16:59 +0100864/**
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000865 * Validates and prepares memory to be sent from the calling VM to another.
Jose Marinho09b1db82019-08-08 09:16:59 +0100866 *
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000867 * This function requires the calling context to hold the <from> VM lock.
Jose Marinho09b1db82019-08-08 09:16:59 +0100868 *
869 * Returns:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000870 * In case of error, one of the following values is returned:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100871 * 1) FFA_INVALID_PARAMETERS - The endpoint provided parameters were
Jose Marinho09b1db82019-08-08 09:16:59 +0100872 * erroneous;
Andrew Walbranf07f04d2020-05-01 18:09:00 +0100873 * 2) FFA_NO_MEMORY - Hafnium did not have sufficient memory to complete the
874 * request.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100875 * 3) FFA_DENIED - The sender doesn't have sufficient access to send the
Andrew Walbrana65a1322020-04-06 19:32:32 +0100876 * memory with the given permissions.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100877 * Success is indicated by FFA_SUCCESS.
Jose Marinho09b1db82019-08-08 09:16:59 +0100878 */
J-Alves66652252022-07-06 09:49:51 +0100879struct ffa_value ffa_send_check_update(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000880 struct vm_locked from_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +0100881 struct ffa_memory_region_constituent **fragments,
882 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
J-Alves8f11cde2022-12-21 16:18:22 +0000883 uint32_t composite_total_page_count, uint32_t share_func,
884 struct ffa_memory_access *receivers, uint32_t receivers_count,
885 struct mpool *page_pool, bool clear, uint32_t *orig_from_mode_ret)
Jose Marinho09b1db82019-08-08 09:16:59 +0100886{
Andrew Walbranca808b12020-05-15 17:22:28 +0100887 uint32_t i;
J-Alves8f11cde2022-12-21 16:18:22 +0000888 uint32_t j;
Jose Marinho09b1db82019-08-08 09:16:59 +0100889 uint32_t orig_from_mode;
890 uint32_t from_mode;
Jose Marinho09b1db82019-08-08 09:16:59 +0100891 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100892 struct ffa_value ret;
J-Alves8f11cde2022-12-21 16:18:22 +0000893 uint32_t constituents_total_page_count = 0;
Jose Marinho09b1db82019-08-08 09:16:59 +0100894
895 /*
Andrew Walbrana65a1322020-04-06 19:32:32 +0100896 * Make sure constituents are properly aligned to a 64-bit boundary. If
897 * not we would get alignment faults trying to read (64-bit) values.
Jose Marinho09b1db82019-08-08 09:16:59 +0100898 */
Andrew Walbranca808b12020-05-15 17:22:28 +0100899 for (i = 0; i < fragment_count; ++i) {
900 if (!is_aligned(fragments[i], 8)) {
901 dlog_verbose("Constituents not aligned.\n");
902 return ffa_error(FFA_INVALID_PARAMETERS);
903 }
J-Alves8f11cde2022-12-21 16:18:22 +0000904 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
905 constituents_total_page_count +=
906 fragments[i][j].page_count;
J-Alves5952d942022-12-22 16:03:00 +0000907 if (ffa_memory_check_overlap(
908 fragments, fragment_constituent_counts,
909 fragment_count, i, j)) {
910 return ffa_error(FFA_INVALID_PARAMETERS);
911 }
J-Alves8f11cde2022-12-21 16:18:22 +0000912 }
913 }
914
915 if (constituents_total_page_count != composite_total_page_count) {
916 dlog_verbose(
917 "Composite page count differs from calculated page "
918 "count from constituents.\n");
919 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho09b1db82019-08-08 09:16:59 +0100920 }
921
922 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000923 * Check if the state transition is lawful for the sender, ensure that
924 * all constituents of a memory region being shared are at the same
925 * state.
Jose Marinho09b1db82019-08-08 09:16:59 +0100926 */
J-Alves363f5722022-04-25 17:37:37 +0100927 ret = ffa_send_check_transition(from_locked, share_func, receivers,
928 receivers_count, &orig_from_mode,
929 fragments, fragment_constituent_counts,
Andrew Walbranca808b12020-05-15 17:22:28 +0100930 fragment_count, &from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100931 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100932 dlog_verbose("Invalid transition for send.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100933 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +0100934 }
935
Andrew Walbran37c574e2020-06-03 11:45:46 +0100936 if (orig_from_mode_ret != NULL) {
937 *orig_from_mode_ret = orig_from_mode;
938 }
939
Jose Marinho09b1db82019-08-08 09:16:59 +0100940 /*
941 * Create a local pool so any freed memory can't be used by another
942 * thread. This is to ensure the original mapping can be restored if the
943 * clear fails.
944 */
Andrew Walbran475c1452020-02-07 13:22:22 +0000945 mpool_init_with_fallback(&local_page_pool, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +0100946
947 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000948 * First reserve all required memory for the new page table entries
949 * without committing, to make sure the entire operation will succeed
950 * without exhausting the page pool.
Jose Marinho09b1db82019-08-08 09:16:59 +0100951 */
Andrew Walbranca808b12020-05-15 17:22:28 +0100952 if (!ffa_region_group_identity_map(
953 from_locked, fragments, fragment_constituent_counts,
954 fragment_count, from_mode, page_pool, false)) {
Jose Marinho09b1db82019-08-08 09:16:59 +0100955 /* TODO: partial defrag of failed range. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100956 ret = ffa_error(FFA_NO_MEMORY);
Jose Marinho09b1db82019-08-08 09:16:59 +0100957 goto out;
958 }
959
960 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000961 * Update the mapping for the sender. This won't allocate because the
962 * transaction was already prepared above, but may free pages in the
963 * case that a whole block is being unmapped that was previously
964 * partially mapped.
Jose Marinho09b1db82019-08-08 09:16:59 +0100965 */
Andrew Walbranca808b12020-05-15 17:22:28 +0100966 CHECK(ffa_region_group_identity_map(
967 from_locked, fragments, fragment_constituent_counts,
968 fragment_count, from_mode, &local_page_pool, true));
Jose Marinho09b1db82019-08-08 09:16:59 +0100969
970 /* Clear the memory so no VM or device can see the previous contents. */
J-Alves7db32002021-12-14 14:44:50 +0000971 if (clear &&
972 !ffa_clear_memory_constituents(
973 plat_ffa_owner_world_mode(from_locked.vm->id), fragments,
974 fragment_constituent_counts, fragment_count, page_pool)) {
Jose Marinho09b1db82019-08-08 09:16:59 +0100975 /*
976 * On failure, roll back by returning memory to the sender. This
977 * may allocate pages which were previously freed into
978 * `local_page_pool` by the call above, but will never allocate
979 * more pages than that so can never fail.
980 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100981 CHECK(ffa_region_group_identity_map(
Andrew Walbranca808b12020-05-15 17:22:28 +0100982 from_locked, fragments, fragment_constituent_counts,
983 fragment_count, orig_from_mode, &local_page_pool,
984 true));
Jose Marinho09b1db82019-08-08 09:16:59 +0100985
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100986 ret = ffa_error(FFA_NO_MEMORY);
Jose Marinho09b1db82019-08-08 09:16:59 +0100987 goto out;
988 }
989
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100990 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000991
992out:
993 mpool_fini(&local_page_pool);
994
995 /*
996 * Tidy up the page table by reclaiming failed mappings (if there was an
997 * error) or merging entries into blocks where possible (on success).
998 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -0700999 vm_ptable_defrag(from_locked, page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001000
1001 return ret;
1002}
1003
1004/**
1005 * Validates and maps memory shared from one VM to another.
1006 *
1007 * This function requires the calling context to hold the <to> lock.
1008 *
1009 * Returns:
1010 * In case of error, one of the following values is returned:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001011 * 1) FFA_INVALID_PARAMETERS - The endpoint provided parameters were
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001012 * erroneous;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001013 * 2) FFA_NO_MEMORY - Hafnium did not have sufficient memory to complete
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001014 * the request.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001015 * Success is indicated by FFA_SUCCESS.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001016 */
J-Alvesb5084cf2022-07-06 14:20:12 +01001017struct ffa_value ffa_retrieve_check_update(
J-Alves7db32002021-12-14 14:44:50 +00001018 struct vm_locked to_locked, ffa_vm_id_t from_id,
Andrew Walbranca808b12020-05-15 17:22:28 +01001019 struct ffa_memory_region_constituent **fragments,
1020 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
1021 uint32_t memory_to_attributes, uint32_t share_func, bool clear,
1022 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001023{
Andrew Walbranca808b12020-05-15 17:22:28 +01001024 uint32_t i;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001025 uint32_t to_mode;
1026 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001027 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001028
1029 /*
Andrew Walbranca808b12020-05-15 17:22:28 +01001030 * Make sure constituents are properly aligned to a 64-bit boundary. If
1031 * not we would get alignment faults trying to read (64-bit) values.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001032 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001033 for (i = 0; i < fragment_count; ++i) {
1034 if (!is_aligned(fragments[i], 8)) {
J-Alvesb5084cf2022-07-06 14:20:12 +01001035 dlog_verbose("Fragment not properly aligned.\n");
Andrew Walbranca808b12020-05-15 17:22:28 +01001036 return ffa_error(FFA_INVALID_PARAMETERS);
1037 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001038 }
1039
1040 /*
1041 * Check if the state transition is lawful for the recipient, and ensure
1042 * that all constituents of the memory region being retrieved are at the
1043 * same state.
1044 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001045 ret = ffa_retrieve_check_transition(
1046 to_locked, share_func, fragments, fragment_constituent_counts,
1047 fragment_count, memory_to_attributes, &to_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001048 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001049 dlog_verbose("Invalid transition for retrieve.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +01001050 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001051 }
1052
1053 /*
1054 * Create a local pool so any freed memory can't be used by another
1055 * thread. This is to ensure the original mapping can be restored if the
1056 * clear fails.
1057 */
1058 mpool_init_with_fallback(&local_page_pool, page_pool);
1059
1060 /*
1061 * First reserve all required memory for the new page table entries in
1062 * the recipient page tables without committing, to make sure the entire
1063 * operation will succeed without exhausting the page pool.
1064 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001065 if (!ffa_region_group_identity_map(
1066 to_locked, fragments, fragment_constituent_counts,
1067 fragment_count, to_mode, page_pool, false)) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001068 /* TODO: partial defrag of failed range. */
1069 dlog_verbose(
1070 "Insufficient memory to update recipient page "
1071 "table.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001072 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001073 goto out;
1074 }
1075
1076 /* Clear the memory so no VM or device can see the previous contents. */
J-Alves7db32002021-12-14 14:44:50 +00001077 if (clear &&
1078 !ffa_clear_memory_constituents(
1079 plat_ffa_owner_world_mode(from_id), fragments,
1080 fragment_constituent_counts, fragment_count, page_pool)) {
J-Alvesb5084cf2022-07-06 14:20:12 +01001081 dlog_verbose("Couldn't clear constituents.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001082 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001083 goto out;
1084 }
1085
Jose Marinho09b1db82019-08-08 09:16:59 +01001086 /*
1087 * Complete the transfer by mapping the memory into the recipient. This
1088 * won't allocate because the transaction was already prepared above, so
1089 * it doesn't need to use the `local_page_pool`.
1090 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001091 CHECK(ffa_region_group_identity_map(
1092 to_locked, fragments, fragment_constituent_counts,
1093 fragment_count, to_mode, page_pool, true));
Jose Marinho09b1db82019-08-08 09:16:59 +01001094
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001095 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Jose Marinho09b1db82019-08-08 09:16:59 +01001096
1097out:
1098 mpool_fini(&local_page_pool);
1099
1100 /*
Andrew Walbranf07f04d2020-05-01 18:09:00 +01001101 * Tidy up the page table by reclaiming failed mappings (if there was an
1102 * error) or merging entries into blocks where possible (on success).
Jose Marinho09b1db82019-08-08 09:16:59 +01001103 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001104 vm_ptable_defrag(to_locked, page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001105
1106 return ret;
1107}
1108
Andrew Walbran996d1d12020-05-27 14:08:43 +01001109static struct ffa_value ffa_relinquish_check_update(
J-Alves3c5b2072022-11-21 12:45:40 +00001110 struct vm_locked from_locked, ffa_vm_id_t owner_id,
Andrew Walbranca808b12020-05-15 17:22:28 +01001111 struct ffa_memory_region_constituent **fragments,
1112 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
1113 struct mpool *page_pool, bool clear)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001114{
1115 uint32_t orig_from_mode;
1116 uint32_t from_mode;
1117 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001118 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001119
Andrew Walbranca808b12020-05-15 17:22:28 +01001120 ret = ffa_relinquish_check_transition(
1121 from_locked, &orig_from_mode, fragments,
1122 fragment_constituent_counts, fragment_count, &from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001123 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001124 dlog_verbose("Invalid transition for relinquish.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +01001125 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001126 }
1127
1128 /*
1129 * Create a local pool so any freed memory can't be used by another
1130 * thread. This is to ensure the original mapping can be restored if the
1131 * clear fails.
1132 */
1133 mpool_init_with_fallback(&local_page_pool, page_pool);
1134
1135 /*
1136 * First reserve all required memory for the new page table entries
1137 * without committing, to make sure the entire operation will succeed
1138 * without exhausting the page pool.
1139 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001140 if (!ffa_region_group_identity_map(
1141 from_locked, fragments, fragment_constituent_counts,
1142 fragment_count, from_mode, page_pool, false)) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001143 /* TODO: partial defrag of failed range. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001144 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001145 goto out;
1146 }
1147
1148 /*
1149 * Update the mapping for the sender. This won't allocate because the
1150 * transaction was already prepared above, but may free pages in the
1151 * case that a whole block is being unmapped that was previously
1152 * partially mapped.
1153 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001154 CHECK(ffa_region_group_identity_map(
1155 from_locked, fragments, fragment_constituent_counts,
1156 fragment_count, from_mode, &local_page_pool, true));
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001157
1158 /* Clear the memory so no VM or device can see the previous contents. */
J-Alves7db32002021-12-14 14:44:50 +00001159 if (clear &&
1160 !ffa_clear_memory_constituents(
J-Alves3c5b2072022-11-21 12:45:40 +00001161 plat_ffa_owner_world_mode(owner_id), fragments,
J-Alves7db32002021-12-14 14:44:50 +00001162 fragment_constituent_counts, fragment_count, page_pool)) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001163 /*
1164 * On failure, roll back by returning memory to the sender. This
1165 * may allocate pages which were previously freed into
1166 * `local_page_pool` by the call above, but will never allocate
1167 * more pages than that so can never fail.
1168 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001169 CHECK(ffa_region_group_identity_map(
Andrew Walbranca808b12020-05-15 17:22:28 +01001170 from_locked, fragments, fragment_constituent_counts,
1171 fragment_count, orig_from_mode, &local_page_pool,
1172 true));
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001173
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001174 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001175 goto out;
1176 }
1177
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001178 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001179
1180out:
1181 mpool_fini(&local_page_pool);
1182
1183 /*
1184 * Tidy up the page table by reclaiming failed mappings (if there was an
1185 * error) or merging entries into blocks where possible (on success).
1186 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001187 vm_ptable_defrag(from_locked, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +01001188
1189 return ret;
1190}
1191
1192/**
Andrew Walbranca808b12020-05-15 17:22:28 +01001193 * Complete a memory sending operation by checking that it is valid, updating
1194 * the sender page table, and then either marking the share state as having
1195 * completed sending (on success) or freeing it (on failure).
1196 *
1197 * Returns FFA_SUCCESS with the handle encoded, or the relevant FFA_ERROR.
1198 */
J-Alvesfdd29272022-07-19 13:16:31 +01001199struct ffa_value ffa_memory_send_complete(
Andrew Walbranca808b12020-05-15 17:22:28 +01001200 struct vm_locked from_locked, struct share_states_locked share_states,
Andrew Walbran37c574e2020-06-03 11:45:46 +01001201 struct ffa_memory_share_state *share_state, struct mpool *page_pool,
1202 uint32_t *orig_from_mode_ret)
Andrew Walbranca808b12020-05-15 17:22:28 +01001203{
1204 struct ffa_memory_region *memory_region = share_state->memory_region;
J-Alves8f11cde2022-12-21 16:18:22 +00001205 struct ffa_composite_memory_region *composite;
Andrew Walbranca808b12020-05-15 17:22:28 +01001206 struct ffa_value ret;
1207
1208 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +00001209 assert(share_states.share_states != NULL);
J-Alves8f11cde2022-12-21 16:18:22 +00001210 assert(memory_region != NULL);
1211 composite = ffa_memory_region_get_composite(memory_region, 0);
1212 assert(composite != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +01001213
1214 /* Check that state is valid in sender page table and update. */
1215 ret = ffa_send_check_update(
1216 from_locked, share_state->fragments,
1217 share_state->fragment_constituent_counts,
J-Alves8f11cde2022-12-21 16:18:22 +00001218 share_state->fragment_count, composite->page_count,
1219 share_state->share_func, memory_region->receivers,
1220 memory_region->receiver_count, page_pool,
1221 memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR,
Andrew Walbran37c574e2020-06-03 11:45:46 +01001222 orig_from_mode_ret);
Andrew Walbranca808b12020-05-15 17:22:28 +01001223 if (ret.func != FFA_SUCCESS_32) {
1224 /*
1225 * Free share state, it failed to send so it can't be retrieved.
1226 */
1227 dlog_verbose("Complete failed, freeing share state.\n");
1228 share_state_free(share_states, share_state, page_pool);
1229 return ret;
1230 }
1231
1232 share_state->sending_complete = true;
1233 dlog_verbose("Marked sending complete.\n");
1234
J-Alvesee68c542020-10-29 17:48:20 +00001235 return ffa_mem_success(share_state->memory_region->handle);
Andrew Walbranca808b12020-05-15 17:22:28 +01001236}
1237
1238/**
Federico Recanatia98603a2021-12-20 18:04:03 +01001239 * Check that the memory attributes match Hafnium expectations:
1240 * Normal Memory, Inner shareable, Write-Back Read-Allocate
1241 * Write-Allocate Cacheable.
1242 */
1243static struct ffa_value ffa_memory_attributes_validate(
J-Alves7a99d0d2023-02-08 13:49:48 +00001244 ffa_memory_attributes_t attributes)
Federico Recanatia98603a2021-12-20 18:04:03 +01001245{
1246 enum ffa_memory_type memory_type;
1247 enum ffa_memory_cacheability cacheability;
1248 enum ffa_memory_shareability shareability;
1249
1250 memory_type = ffa_get_memory_type_attr(attributes);
1251 if (memory_type != FFA_MEMORY_NORMAL_MEM) {
1252 dlog_verbose("Invalid memory type %#x, expected %#x.\n",
1253 memory_type, FFA_MEMORY_NORMAL_MEM);
Federico Recanati3d953f32022-02-17 09:31:29 +01001254 return ffa_error(FFA_DENIED);
Federico Recanatia98603a2021-12-20 18:04:03 +01001255 }
1256
1257 cacheability = ffa_get_memory_cacheability_attr(attributes);
1258 if (cacheability != FFA_MEMORY_CACHE_WRITE_BACK) {
1259 dlog_verbose("Invalid cacheability %#x, expected %#x.\n",
1260 cacheability, FFA_MEMORY_CACHE_WRITE_BACK);
Federico Recanati3d953f32022-02-17 09:31:29 +01001261 return ffa_error(FFA_DENIED);
Federico Recanatia98603a2021-12-20 18:04:03 +01001262 }
1263
1264 shareability = ffa_get_memory_shareability_attr(attributes);
1265 if (shareability != FFA_MEMORY_INNER_SHAREABLE) {
1266 dlog_verbose("Invalid shareability %#x, expected #%x.\n",
1267 shareability, FFA_MEMORY_INNER_SHAREABLE);
Federico Recanati3d953f32022-02-17 09:31:29 +01001268 return ffa_error(FFA_DENIED);
Federico Recanatia98603a2021-12-20 18:04:03 +01001269 }
1270
1271 return (struct ffa_value){.func = FFA_SUCCESS_32};
1272}
1273
1274/**
Andrew Walbrana65a1322020-04-06 19:32:32 +01001275 * Check that the given `memory_region` represents a valid memory send request
1276 * of the given `share_func` type, return the clear flag and permissions via the
1277 * respective output parameters, and update the permissions if necessary.
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001278 *
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001279 * Returns FFA_SUCCESS if the request was valid, or the relevant FFA_ERROR if
Andrew Walbrana65a1322020-04-06 19:32:32 +01001280 * not.
1281 */
J-Alves66652252022-07-06 09:49:51 +01001282struct ffa_value ffa_memory_send_validate(
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001283 struct vm_locked from_locked, struct ffa_memory_region *memory_region,
1284 uint32_t memory_share_length, uint32_t fragment_length,
J-Alves363f5722022-04-25 17:37:37 +01001285 uint32_t share_func)
Andrew Walbrana65a1322020-04-06 19:32:32 +01001286{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001287 struct ffa_composite_memory_region *composite;
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001288 uint64_t receivers_end;
1289 uint64_t min_length;
Federico Recanati872cd692022-01-05 13:10:10 +01001290 uint32_t composite_memory_region_offset;
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001291 uint32_t constituents_start;
Andrew Walbran130a8ae2020-05-15 16:27:15 +01001292 uint32_t constituents_length;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001293 enum ffa_data_access data_access;
1294 enum ffa_instruction_access instruction_access;
Olivier Deprez4342a3c2022-02-28 09:37:25 +01001295 enum ffa_memory_security security_state;
Federico Recanatia98603a2021-12-20 18:04:03 +01001296 struct ffa_value ret;
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001297 const size_t minimum_first_fragment_length =
1298 (sizeof(struct ffa_memory_region) +
1299 sizeof(struct ffa_memory_access) +
1300 sizeof(struct ffa_composite_memory_region));
1301
1302 if (fragment_length < minimum_first_fragment_length) {
1303 dlog_verbose("Fragment length %u too short (min %u).\n",
1304 (size_t)fragment_length,
1305 minimum_first_fragment_length);
1306 return ffa_error(FFA_INVALID_PARAMETERS);
1307 }
1308
Demi Marie Obenour73a1e942023-02-04 14:09:18 -05001309 static_assert(sizeof(struct ffa_memory_region_constituent) == 16,
1310 "struct ffa_memory_region_constituent must be 16 bytes");
1311 if (!is_aligned(fragment_length,
1312 sizeof(struct ffa_memory_region_constituent)) ||
1313 !is_aligned(memory_share_length,
1314 sizeof(struct ffa_memory_region_constituent))) {
1315 dlog_verbose(
1316 "Fragment length %u or total length %u"
1317 " is not 16-byte aligned.\n",
1318 fragment_length, memory_share_length);
1319 return ffa_error(FFA_INVALID_PARAMETERS);
1320 }
1321
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001322 if (fragment_length > memory_share_length) {
1323 dlog_verbose(
1324 "Fragment length %u greater than total length %u.\n",
1325 (size_t)fragment_length, (size_t)memory_share_length);
1326 return ffa_error(FFA_INVALID_PARAMETERS);
1327 }
Andrew Walbrana65a1322020-04-06 19:32:32 +01001328
J-Alves0b6653d2022-04-22 13:17:38 +01001329 assert(memory_region->receivers_offset ==
1330 offsetof(struct ffa_memory_region, receivers));
1331 assert(memory_region->memory_access_desc_size ==
1332 sizeof(struct ffa_memory_access));
1333
J-Alves95df0ef2022-12-07 10:09:48 +00001334 /* The sender must match the caller. */
1335 if ((!vm_id_is_current_world(from_locked.vm->id) &&
1336 vm_id_is_current_world(memory_region->sender)) ||
1337 (vm_id_is_current_world(from_locked.vm->id) &&
1338 memory_region->sender != from_locked.vm->id)) {
1339 dlog_verbose("Invalid memory sender ID.\n");
1340 return ffa_error(FFA_DENIED);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001341 }
1342
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001343 if (memory_region->receiver_count <= 0) {
1344 dlog_verbose("No receivers!\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001345 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001346 }
1347
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001348 /*
1349 * Ensure that the composite header is within the memory bounds and
1350 * doesn't overlap the first part of the message. Cast to uint64_t
1351 * to prevent overflow.
1352 */
1353 receivers_end = ((uint64_t)sizeof(struct ffa_memory_access) *
1354 (uint64_t)memory_region->receiver_count) +
1355 sizeof(struct ffa_memory_region);
1356 min_length = receivers_end +
1357 sizeof(struct ffa_composite_memory_region) +
1358 sizeof(struct ffa_memory_region_constituent);
1359 if (min_length > memory_share_length) {
1360 dlog_verbose("Share too short: got %u but minimum is %u.\n",
1361 (size_t)memory_share_length, (size_t)min_length);
1362 return ffa_error(FFA_INVALID_PARAMETERS);
1363 }
1364
1365 composite_memory_region_offset =
1366 memory_region->receivers[0].composite_memory_region_offset;
Andrew Walbrana65a1322020-04-06 19:32:32 +01001367
1368 /*
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001369 * Check that the composite memory region descriptor is after the access
1370 * descriptors, is at least 16-byte aligned, and fits in the first
1371 * fragment.
Andrew Walbrana65a1322020-04-06 19:32:32 +01001372 */
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001373 if ((composite_memory_region_offset < receivers_end) ||
1374 (composite_memory_region_offset % 16 != 0) ||
1375 (composite_memory_region_offset >
1376 fragment_length - sizeof(struct ffa_composite_memory_region))) {
1377 dlog_verbose(
1378 "Invalid composite memory region descriptor offset "
1379 "%u.\n",
1380 (size_t)composite_memory_region_offset);
1381 return ffa_error(FFA_INVALID_PARAMETERS);
1382 }
1383
1384 /*
1385 * Compute the start of the constituent regions. Already checked
1386 * to be not more than fragment_length and thus not more than
1387 * memory_share_length.
1388 */
1389 constituents_start = composite_memory_region_offset +
1390 sizeof(struct ffa_composite_memory_region);
1391 constituents_length = memory_share_length - constituents_start;
1392
1393 /*
1394 * Check that the number of constituents is consistent with the length
1395 * of the constituent region.
1396 */
1397 composite = ffa_memory_region_get_composite(memory_region, 0);
1398 if ((constituents_length %
1399 sizeof(struct ffa_memory_region_constituent) !=
1400 0) ||
1401 ((constituents_length /
1402 sizeof(struct ffa_memory_region_constituent)) !=
1403 composite->constituent_count)) {
1404 dlog_verbose("Invalid length %u or composite offset %u.\n",
1405 (size_t)memory_share_length,
1406 (size_t)composite_memory_region_offset);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001407 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001408 }
Andrew Walbranca808b12020-05-15 17:22:28 +01001409 if (fragment_length < memory_share_length &&
1410 fragment_length < HF_MAILBOX_SIZE) {
1411 dlog_warning(
1412 "Initial fragment length %d smaller than mailbox "
1413 "size.\n",
1414 fragment_length);
1415 }
Andrew Walbrana65a1322020-04-06 19:32:32 +01001416
Andrew Walbrana65a1322020-04-06 19:32:32 +01001417 /*
1418 * Clear is not allowed for memory sharing, as the sender still has
1419 * access to the memory.
1420 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001421 if ((memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR) &&
1422 share_func == FFA_MEM_SHARE_32) {
Andrew Walbrana65a1322020-04-06 19:32:32 +01001423 dlog_verbose("Memory can't be cleared while being shared.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001424 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001425 }
1426
1427 /* No other flags are allowed/supported here. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001428 if (memory_region->flags & ~FFA_MEMORY_REGION_FLAG_CLEAR) {
Andrew Walbrana65a1322020-04-06 19:32:32 +01001429 dlog_verbose("Invalid flags %#x.\n", memory_region->flags);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001430 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001431 }
1432
J-Alves363f5722022-04-25 17:37:37 +01001433 /* Check that the permissions are valid, for each specified receiver. */
1434 for (uint32_t i = 0U; i < memory_region->receiver_count; i++) {
1435 ffa_memory_access_permissions_t permissions =
1436 memory_region->receivers[i]
1437 .receiver_permissions.permissions;
1438 ffa_vm_id_t receiver_id =
1439 memory_region->receivers[i]
1440 .receiver_permissions.receiver;
1441
1442 if (memory_region->sender == receiver_id) {
1443 dlog_verbose("Can't share memory with itself.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001444 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001445 }
Federico Recanati85090c42021-12-15 13:17:54 +01001446
J-Alves363f5722022-04-25 17:37:37 +01001447 for (uint32_t j = i + 1; j < memory_region->receiver_count;
1448 j++) {
1449 if (receiver_id ==
1450 memory_region->receivers[j]
1451 .receiver_permissions.receiver) {
1452 dlog_verbose(
1453 "Repeated receiver(%x) in memory send "
1454 "operation.\n",
1455 memory_region->receivers[j]
1456 .receiver_permissions.receiver);
1457 return ffa_error(FFA_INVALID_PARAMETERS);
1458 }
1459 }
1460
1461 if (composite_memory_region_offset !=
1462 memory_region->receivers[i]
1463 .composite_memory_region_offset) {
1464 dlog_verbose(
1465 "All ffa_memory_access should point to the "
1466 "same composite memory region offset.\n");
1467 return ffa_error(FFA_INVALID_PARAMETERS);
1468 }
1469
1470 data_access = ffa_get_data_access_attr(permissions);
1471 instruction_access =
1472 ffa_get_instruction_access_attr(permissions);
1473 if (data_access == FFA_DATA_ACCESS_RESERVED ||
1474 instruction_access == FFA_INSTRUCTION_ACCESS_RESERVED) {
1475 dlog_verbose(
1476 "Reserved value for receiver permissions "
1477 "%#x.\n",
1478 permissions);
1479 return ffa_error(FFA_INVALID_PARAMETERS);
1480 }
1481 if (instruction_access !=
1482 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
1483 dlog_verbose(
1484 "Invalid instruction access permissions %#x "
1485 "for sending memory.\n",
1486 permissions);
1487 return ffa_error(FFA_INVALID_PARAMETERS);
1488 }
1489 if (share_func == FFA_MEM_SHARE_32) {
1490 if (data_access == FFA_DATA_ACCESS_NOT_SPECIFIED) {
1491 dlog_verbose(
1492 "Invalid data access permissions %#x "
1493 "for sharing memory.\n",
1494 permissions);
1495 return ffa_error(FFA_INVALID_PARAMETERS);
1496 }
1497 /*
1498 * According to section 10.10.3 of the FF-A v1.1 EAC0
1499 * spec, NX is required for share operations (but must
1500 * not be specified by the sender) so set it in the
1501 * copy that we store, ready to be returned to the
1502 * retriever.
1503 */
J-Alvesb19731a2022-06-20 17:30:33 +01001504 if (vm_id_is_current_world(receiver_id)) {
1505 ffa_set_instruction_access_attr(
1506 &permissions,
1507 FFA_INSTRUCTION_ACCESS_NX);
1508 memory_region->receivers[i]
1509 .receiver_permissions.permissions =
1510 permissions;
1511 }
J-Alves363f5722022-04-25 17:37:37 +01001512 }
1513 if (share_func == FFA_MEM_LEND_32 &&
1514 data_access == FFA_DATA_ACCESS_NOT_SPECIFIED) {
1515 dlog_verbose(
1516 "Invalid data access permissions %#x for "
1517 "lending memory.\n",
1518 permissions);
1519 return ffa_error(FFA_INVALID_PARAMETERS);
1520 }
1521
1522 if (share_func == FFA_MEM_DONATE_32 &&
1523 data_access != FFA_DATA_ACCESS_NOT_SPECIFIED) {
1524 dlog_verbose(
1525 "Invalid data access permissions %#x for "
1526 "donating memory.\n",
1527 permissions);
1528 return ffa_error(FFA_INVALID_PARAMETERS);
1529 }
Andrew Walbrana65a1322020-04-06 19:32:32 +01001530 }
1531
Olivier Deprez4342a3c2022-02-28 09:37:25 +01001532 /* Memory region attributes NS-Bit MBZ for FFA_MEM_SHARE/LEND/DONATE. */
1533 security_state =
1534 ffa_get_memory_security_attr(memory_region->attributes);
1535 if (security_state != FFA_MEMORY_SECURITY_UNSPECIFIED) {
1536 dlog_verbose(
1537 "Invalid security state for memory share operation.\n");
1538 return ffa_error(FFA_INVALID_PARAMETERS);
1539 }
1540
Federico Recanatid937f5e2021-12-20 17:38:23 +01001541 /*
J-Alves807794e2022-06-16 13:42:47 +01001542 * If a memory donate or lend with single borrower, the memory type
1543 * shall not be specified by the sender.
Federico Recanatid937f5e2021-12-20 17:38:23 +01001544 */
J-Alves807794e2022-06-16 13:42:47 +01001545 if (share_func == FFA_MEM_DONATE_32 ||
1546 (share_func == FFA_MEM_LEND_32 &&
1547 memory_region->receiver_count == 1)) {
1548 if (ffa_get_memory_type_attr(memory_region->attributes) !=
1549 FFA_MEMORY_NOT_SPECIFIED_MEM) {
1550 dlog_verbose(
1551 "Memory type shall not be specified by "
1552 "sender.\n");
1553 return ffa_error(FFA_INVALID_PARAMETERS);
1554 }
1555 } else {
1556 /*
1557 * Check that sender's memory attributes match Hafnium
1558 * expectations: Normal Memory, Inner shareable, Write-Back
1559 * Read-Allocate Write-Allocate Cacheable.
1560 */
1561 ret = ffa_memory_attributes_validate(memory_region->attributes);
1562 if (ret.func != FFA_SUCCESS_32) {
1563 return ret;
1564 }
Federico Recanatid937f5e2021-12-20 17:38:23 +01001565 }
1566
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001567 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbrana65a1322020-04-06 19:32:32 +01001568}
1569
1570/**
Andrew Walbranca808b12020-05-15 17:22:28 +01001571 * Gets the share state for continuing an operation to donate, lend or share
1572 * memory, and checks that it is a valid request.
1573 *
1574 * Returns FFA_SUCCESS if the request was valid, or the relevant FFA_ERROR if
1575 * not.
1576 */
J-Alvesfdd29272022-07-19 13:16:31 +01001577struct ffa_value ffa_memory_send_continue_validate(
Andrew Walbranca808b12020-05-15 17:22:28 +01001578 struct share_states_locked share_states, ffa_memory_handle_t handle,
1579 struct ffa_memory_share_state **share_state_ret, ffa_vm_id_t from_vm_id,
1580 struct mpool *page_pool)
1581{
1582 struct ffa_memory_share_state *share_state;
1583 struct ffa_memory_region *memory_region;
1584
Daniel Boulbya2f8c662021-11-26 17:52:53 +00001585 assert(share_state_ret != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +01001586
1587 /*
1588 * Look up the share state by handle and make sure that the VM ID
1589 * matches.
1590 */
Karl Meakin4a2854a2023-06-30 16:26:52 +01001591 share_state = get_share_state(share_states, handle);
1592 if (!share_state) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001593 dlog_verbose(
1594 "Invalid handle %#x for memory send continuation.\n",
1595 handle);
1596 return ffa_error(FFA_INVALID_PARAMETERS);
1597 }
1598 memory_region = share_state->memory_region;
1599
J-Alvesfdd29272022-07-19 13:16:31 +01001600 if (vm_id_is_current_world(from_vm_id) &&
1601 memory_region->sender != from_vm_id) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001602 dlog_verbose("Invalid sender %d.\n", memory_region->sender);
1603 return ffa_error(FFA_INVALID_PARAMETERS);
1604 }
1605
1606 if (share_state->sending_complete) {
1607 dlog_verbose(
1608 "Sending of memory handle %#x is already complete.\n",
1609 handle);
1610 return ffa_error(FFA_INVALID_PARAMETERS);
1611 }
1612
1613 if (share_state->fragment_count == MAX_FRAGMENTS) {
1614 /*
1615 * Log a warning as this is a sign that MAX_FRAGMENTS should
1616 * probably be increased.
1617 */
1618 dlog_warning(
1619 "Too many fragments for memory share with handle %#x; "
1620 "only %d supported.\n",
1621 handle, MAX_FRAGMENTS);
1622 /* Free share state, as it's not possible to complete it. */
1623 share_state_free(share_states, share_state, page_pool);
1624 return ffa_error(FFA_NO_MEMORY);
1625 }
1626
1627 *share_state_ret = share_state;
1628
1629 return (struct ffa_value){.func = FFA_SUCCESS_32};
1630}
1631
1632/**
J-Alves95df0ef2022-12-07 10:09:48 +00001633 * Checks if there is at least one receiver from the other world.
1634 */
J-Alvesfdd29272022-07-19 13:16:31 +01001635bool memory_region_receivers_from_other_world(
J-Alves95df0ef2022-12-07 10:09:48 +00001636 struct ffa_memory_region *memory_region)
1637{
1638 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
1639 ffa_vm_id_t receiver = memory_region->receivers[i]
1640 .receiver_permissions.receiver;
1641 if (!vm_id_is_current_world(receiver)) {
1642 return true;
1643 }
1644 }
1645 return false;
1646}
1647
1648/**
J-Alves9da280b2022-12-21 14:55:39 +00001649 * Validates a call to donate, lend or share memory in which Hafnium is the
1650 * designated allocator of the memory handle. In practice, this also means
1651 * Hafnium is responsible for managing the state structures for the transaction.
1652 * If Hafnium is the SPMC, it should allocate the memory handle when either the
1653 * sender is an SP or there is at least one borrower that is an SP.
1654 * If Hafnium is the hypervisor, it should allocate the memory handle when
1655 * operation involves only NWd VMs.
1656 *
1657 * If validation goes well, Hafnium updates the stage-2 page tables of the
1658 * sender. Validation consists of checking if the message length and number of
1659 * memory region constituents match, and if the transition is valid for the
1660 * type of memory sending operation.
Andrew Walbran475c1452020-02-07 13:22:22 +00001661 *
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001662 * Assumes that the caller has already found and locked the sender VM and copied
1663 * the memory region descriptor from the sender's TX buffer to a freshly
1664 * allocated page from Hafnium's internal pool. The caller must have also
1665 * validated that the receiver VM ID is valid.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001666 *
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001667 * This function takes ownership of the `memory_region` passed in and will free
1668 * it when necessary; it must not be freed by the caller.
Jose Marinho09b1db82019-08-08 09:16:59 +01001669 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001670struct ffa_value ffa_memory_send(struct vm_locked from_locked,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001671 struct ffa_memory_region *memory_region,
Andrew Walbran130a8ae2020-05-15 16:27:15 +01001672 uint32_t memory_share_length,
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001673 uint32_t fragment_length, uint32_t share_func,
1674 struct mpool *page_pool)
Jose Marinho09b1db82019-08-08 09:16:59 +01001675{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001676 struct ffa_value ret;
Andrew Walbranca808b12020-05-15 17:22:28 +01001677 struct share_states_locked share_states;
1678 struct ffa_memory_share_state *share_state;
Jose Marinho09b1db82019-08-08 09:16:59 +01001679
1680 /*
Andrew Walbrana65a1322020-04-06 19:32:32 +01001681 * If there is an error validating the `memory_region` then we need to
1682 * free it because we own it but we won't be storing it in a share state
1683 * after all.
Jose Marinho09b1db82019-08-08 09:16:59 +01001684 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001685 ret = ffa_memory_send_validate(from_locked, memory_region,
1686 memory_share_length, fragment_length,
J-Alves363f5722022-04-25 17:37:37 +01001687 share_func);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001688 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001689 mpool_free(page_pool, memory_region);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001690 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01001691 }
1692
Andrew Walbrana65a1322020-04-06 19:32:32 +01001693 /* Set flag for share function, ready to be retrieved later. */
1694 switch (share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001695 case FFA_MEM_SHARE_32:
Andrew Walbrana65a1322020-04-06 19:32:32 +01001696 memory_region->flags |=
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001697 FFA_MEMORY_REGION_TRANSACTION_TYPE_SHARE;
Andrew Walbrana65a1322020-04-06 19:32:32 +01001698 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001699 case FFA_MEM_LEND_32:
1700 memory_region->flags |= FFA_MEMORY_REGION_TRANSACTION_TYPE_LEND;
Andrew Walbrana65a1322020-04-06 19:32:32 +01001701 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001702 case FFA_MEM_DONATE_32:
Andrew Walbrana65a1322020-04-06 19:32:32 +01001703 memory_region->flags |=
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001704 FFA_MEMORY_REGION_TRANSACTION_TYPE_DONATE;
Andrew Walbrana65a1322020-04-06 19:32:32 +01001705 break;
Jose Marinho09b1db82019-08-08 09:16:59 +01001706 }
1707
Andrew Walbranca808b12020-05-15 17:22:28 +01001708 share_states = share_states_lock();
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001709 /*
1710 * Allocate a share state before updating the page table. Otherwise if
1711 * updating the page table succeeded but allocating the share state
1712 * failed then it would leave the memory in a state where nobody could
1713 * get it back.
1714 */
Karl Meakin52cdfe72023-06-30 14:49:10 +01001715 share_state = allocate_share_state(share_states, share_func,
1716 memory_region, fragment_length,
1717 FFA_MEMORY_HANDLE_INVALID);
1718 if (!share_state) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001719 dlog_verbose("Failed to allocate share state.\n");
1720 mpool_free(page_pool, memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +01001721 ret = ffa_error(FFA_NO_MEMORY);
1722 goto out;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001723 }
1724
Andrew Walbranca808b12020-05-15 17:22:28 +01001725 if (fragment_length == memory_share_length) {
1726 /* No more fragments to come, everything fit in one message. */
J-Alves2a0d2882020-10-29 14:49:50 +00001727 ret = ffa_memory_send_complete(
1728 from_locked, share_states, share_state, page_pool,
1729 &(share_state->sender_orig_mode));
Andrew Walbranca808b12020-05-15 17:22:28 +01001730 } else {
J-Alvesfdd29272022-07-19 13:16:31 +01001731 /*
1732 * Use sender ID from 'memory_region' assuming
1733 * that at this point it has been validated:
1734 * - MBZ at virtual FF-A instance.
1735 */
1736 ffa_vm_id_t sender_to_ret =
1737 (from_locked.vm->id == HF_OTHER_WORLD_ID)
1738 ? memory_region->sender
1739 : 0;
Andrew Walbranca808b12020-05-15 17:22:28 +01001740 ret = (struct ffa_value){
1741 .func = FFA_MEM_FRAG_RX_32,
J-Alvesee68c542020-10-29 17:48:20 +00001742 .arg1 = (uint32_t)memory_region->handle,
1743 .arg2 = (uint32_t)(memory_region->handle >> 32),
J-Alvesfdd29272022-07-19 13:16:31 +01001744 .arg3 = fragment_length,
1745 .arg4 = (uint32_t)(sender_to_ret & 0xffff) << 16};
Andrew Walbranca808b12020-05-15 17:22:28 +01001746 }
1747
1748out:
1749 share_states_unlock(&share_states);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001750 dump_share_states();
Andrew Walbranca808b12020-05-15 17:22:28 +01001751 return ret;
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001752}
1753
1754/**
J-Alves8505a8a2022-06-15 18:10:18 +01001755 * Continues an operation to donate, lend or share memory to a VM from current
1756 * world. If this is the last fragment then checks that the transition is valid
1757 * for the type of memory sending operation and updates the stage-2 page tables
1758 * of the sender.
Andrew Walbranca808b12020-05-15 17:22:28 +01001759 *
1760 * Assumes that the caller has already found and locked the sender VM and copied
1761 * the memory region descriptor from the sender's TX buffer to a freshly
1762 * allocated page from Hafnium's internal pool.
1763 *
1764 * This function takes ownership of the `fragment` passed in; it must not be
1765 * freed by the caller.
1766 */
1767struct ffa_value ffa_memory_send_continue(struct vm_locked from_locked,
1768 void *fragment,
1769 uint32_t fragment_length,
1770 ffa_memory_handle_t handle,
1771 struct mpool *page_pool)
1772{
1773 struct share_states_locked share_states = share_states_lock();
1774 struct ffa_memory_share_state *share_state;
1775 struct ffa_value ret;
1776 struct ffa_memory_region *memory_region;
1777
Demi Marie Obenour73a1e942023-02-04 14:09:18 -05001778 CHECK(is_aligned(fragment,
1779 alignof(struct ffa_memory_region_constituent)));
1780 if (fragment_length % sizeof(struct ffa_memory_region_constituent) !=
1781 0) {
1782 dlog_verbose("Fragment length %u misaligned.\n",
1783 fragment_length);
1784 ret = ffa_error(FFA_INVALID_PARAMETERS);
1785 goto out_free_fragment;
1786 }
1787
Andrew Walbranca808b12020-05-15 17:22:28 +01001788 ret = ffa_memory_send_continue_validate(share_states, handle,
1789 &share_state,
1790 from_locked.vm->id, page_pool);
1791 if (ret.func != FFA_SUCCESS_32) {
1792 goto out_free_fragment;
1793 }
1794 memory_region = share_state->memory_region;
1795
J-Alves95df0ef2022-12-07 10:09:48 +00001796 if (memory_region_receivers_from_other_world(memory_region)) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001797 dlog_error(
1798 "Got hypervisor-allocated handle for memory send to "
J-Alves8505a8a2022-06-15 18:10:18 +01001799 "other world. This should never happen, and indicates "
1800 "a bug in "
Andrew Walbranca808b12020-05-15 17:22:28 +01001801 "EL3 code.\n");
1802 ret = ffa_error(FFA_INVALID_PARAMETERS);
1803 goto out_free_fragment;
1804 }
1805
1806 /* Add this fragment. */
1807 share_state->fragments[share_state->fragment_count] = fragment;
1808 share_state->fragment_constituent_counts[share_state->fragment_count] =
1809 fragment_length / sizeof(struct ffa_memory_region_constituent);
1810 share_state->fragment_count++;
1811
1812 /* Check whether the memory send operation is now ready to complete. */
1813 if (share_state_sending_complete(share_states, share_state)) {
J-Alves2a0d2882020-10-29 14:49:50 +00001814 ret = ffa_memory_send_complete(
1815 from_locked, share_states, share_state, page_pool,
1816 &(share_state->sender_orig_mode));
Andrew Walbranca808b12020-05-15 17:22:28 +01001817 } else {
1818 ret = (struct ffa_value){
1819 .func = FFA_MEM_FRAG_RX_32,
1820 .arg1 = (uint32_t)handle,
1821 .arg2 = (uint32_t)(handle >> 32),
1822 .arg3 = share_state_next_fragment_offset(share_states,
1823 share_state)};
1824 }
1825 goto out;
1826
1827out_free_fragment:
1828 mpool_free(page_pool, fragment);
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001829
1830out:
Andrew Walbranca808b12020-05-15 17:22:28 +01001831 share_states_unlock(&share_states);
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001832 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001833}
1834
Andrew Walbranca808b12020-05-15 17:22:28 +01001835/** Clean up after the receiver has finished retrieving a memory region. */
1836static void ffa_memory_retrieve_complete(
1837 struct share_states_locked share_states,
1838 struct ffa_memory_share_state *share_state, struct mpool *page_pool)
1839{
1840 if (share_state->share_func == FFA_MEM_DONATE_32) {
1841 /*
1842 * Memory that has been donated can't be relinquished,
1843 * so no need to keep the share state around.
1844 */
1845 share_state_free(share_states, share_state, page_pool);
1846 dlog_verbose("Freed share state for donate.\n");
1847 }
1848}
1849
J-Alves2d8457f2022-10-05 11:06:41 +01001850/**
1851 * Initialises the given memory region descriptor to be used for an
1852 * `FFA_MEM_RETRIEVE_RESP`, including the given constituents for the first
1853 * fragment.
1854 * The memory region descriptor is initialized according to retriever's
1855 * FF-A version.
1856 *
1857 * Returns true on success, or false if the given constituents won't all fit in
1858 * the first fragment.
1859 */
1860static bool ffa_retrieved_memory_region_init(
1861 void *response, uint32_t ffa_version, size_t response_max_size,
1862 ffa_vm_id_t sender, ffa_memory_attributes_t attributes,
1863 ffa_memory_region_flags_t flags, ffa_memory_handle_t handle,
1864 ffa_vm_id_t receiver_id, ffa_memory_access_permissions_t permissions,
1865 uint32_t page_count, uint32_t total_constituent_count,
1866 const struct ffa_memory_region_constituent constituents[],
1867 uint32_t fragment_constituent_count, uint32_t *total_length,
1868 uint32_t *fragment_length)
1869{
1870 struct ffa_composite_memory_region *composite_memory_region;
1871 struct ffa_memory_access *receiver;
1872 uint32_t i;
1873 uint32_t constituents_offset;
1874 uint32_t receiver_count;
1875
1876 assert(response != NULL);
1877
1878 if (ffa_version == MAKE_FFA_VERSION(1, 0)) {
1879 struct ffa_memory_region_v1_0 *retrieve_response =
1880 (struct ffa_memory_region_v1_0 *)response;
1881
J-Alves5da37d92022-10-24 16:33:48 +01001882 ffa_memory_region_init_header_v1_0(
1883 retrieve_response, sender, attributes, flags, handle, 0,
1884 RECEIVERS_COUNT_IN_RETRIEVE_RESP);
J-Alves2d8457f2022-10-05 11:06:41 +01001885
1886 receiver = &retrieve_response->receivers[0];
1887 receiver_count = retrieve_response->receiver_count;
1888
1889 receiver->composite_memory_region_offset =
1890 sizeof(struct ffa_memory_region_v1_0) +
1891 receiver_count * sizeof(struct ffa_memory_access);
1892
1893 composite_memory_region = ffa_memory_region_get_composite_v1_0(
1894 retrieve_response, 0);
1895 } else {
1896 /* Default to FF-A v1.1 version. */
1897 struct ffa_memory_region *retrieve_response =
1898 (struct ffa_memory_region *)response;
1899
1900 ffa_memory_region_init_header(retrieve_response, sender,
1901 attributes, flags, handle, 0, 1);
1902
1903 receiver = &retrieve_response->receivers[0];
1904 receiver_count = retrieve_response->receiver_count;
1905
1906 /*
1907 * Note that `sizeof(struct_ffa_memory_region)` and
1908 * `sizeof(struct ffa_memory_access)` must both be multiples of
1909 * 16 (as verified by the asserts in `ffa_memory.c`, so it is
1910 * guaranteed that the offset we calculate here is aligned to a
1911 * 64-bit boundary and so 64-bit values can be copied without
1912 * alignment faults.
1913 */
1914 receiver->composite_memory_region_offset =
1915 sizeof(struct ffa_memory_region) +
1916 receiver_count * sizeof(struct ffa_memory_access);
1917
1918 composite_memory_region =
1919 ffa_memory_region_get_composite(retrieve_response, 0);
1920 }
1921
1922 assert(receiver != NULL);
1923 assert(composite_memory_region != NULL);
1924
1925 /*
1926 * Initialized here as in memory retrieve responses we currently expect
1927 * one borrower to be specified.
1928 */
1929 ffa_memory_access_init_permissions(receiver, receiver_id, 0, 0, flags);
1930 receiver->receiver_permissions.permissions = permissions;
1931
1932 composite_memory_region->page_count = page_count;
1933 composite_memory_region->constituent_count = total_constituent_count;
1934 composite_memory_region->reserved_0 = 0;
1935
1936 constituents_offset = receiver->composite_memory_region_offset +
1937 sizeof(struct ffa_composite_memory_region);
1938 if (constituents_offset +
1939 fragment_constituent_count *
1940 sizeof(struct ffa_memory_region_constituent) >
1941 response_max_size) {
1942 return false;
1943 }
1944
1945 for (i = 0; i < fragment_constituent_count; ++i) {
1946 composite_memory_region->constituents[i] = constituents[i];
1947 }
1948
1949 if (total_length != NULL) {
1950 *total_length =
1951 constituents_offset +
1952 composite_memory_region->constituent_count *
1953 sizeof(struct ffa_memory_region_constituent);
1954 }
1955 if (fragment_length != NULL) {
1956 *fragment_length =
1957 constituents_offset +
1958 fragment_constituent_count *
1959 sizeof(struct ffa_memory_region_constituent);
1960 }
1961
1962 return true;
1963}
1964
J-Alves96de29f2022-04-26 16:05:24 +01001965/*
1966 * Gets the receiver's access permissions from 'struct ffa_memory_region' and
1967 * returns its index in the receiver's array. If receiver's ID doesn't exist
1968 * in the array, return the region's 'receiver_count'.
1969 */
J-Alvesb5084cf2022-07-06 14:20:12 +01001970uint32_t ffa_memory_region_get_receiver(struct ffa_memory_region *memory_region,
1971 ffa_vm_id_t receiver)
J-Alves96de29f2022-04-26 16:05:24 +01001972{
1973 struct ffa_memory_access *receivers;
1974 uint32_t i;
1975
1976 assert(memory_region != NULL);
1977
1978 receivers = memory_region->receivers;
1979
1980 for (i = 0U; i < memory_region->receiver_count; i++) {
1981 if (receivers[i].receiver_permissions.receiver == receiver) {
1982 break;
1983 }
1984 }
1985
1986 return i;
1987}
1988
1989/**
1990 * Validates the retrieved permissions against those specified by the lender
1991 * of memory share operation. Optionally can help set the permissions to be used
1992 * for the S2 mapping, through the `permissions` argument.
1993 * Returns true if permissions are valid, false otherwise.
1994 */
1995static bool ffa_memory_retrieve_is_memory_access_valid(
1996 enum ffa_data_access sent_data_access,
1997 enum ffa_data_access requested_data_access,
1998 enum ffa_instruction_access sent_instruction_access,
1999 enum ffa_instruction_access requested_instruction_access,
2000 ffa_memory_access_permissions_t *permissions)
2001{
2002 switch (sent_data_access) {
2003 case FFA_DATA_ACCESS_NOT_SPECIFIED:
2004 case FFA_DATA_ACCESS_RW:
2005 if (requested_data_access == FFA_DATA_ACCESS_NOT_SPECIFIED ||
2006 requested_data_access == FFA_DATA_ACCESS_RW) {
2007 if (permissions != NULL) {
2008 ffa_set_data_access_attr(permissions,
2009 FFA_DATA_ACCESS_RW);
2010 }
2011 break;
2012 }
2013 /* Intentional fall-through. */
2014 case FFA_DATA_ACCESS_RO:
2015 if (requested_data_access == FFA_DATA_ACCESS_NOT_SPECIFIED ||
2016 requested_data_access == FFA_DATA_ACCESS_RO) {
2017 if (permissions != NULL) {
2018 ffa_set_data_access_attr(permissions,
2019 FFA_DATA_ACCESS_RO);
2020 }
2021 break;
2022 }
2023 dlog_verbose(
2024 "Invalid data access requested; sender specified "
2025 "permissions %#x but receiver requested %#x.\n",
2026 sent_data_access, requested_data_access);
2027 return false;
2028 case FFA_DATA_ACCESS_RESERVED:
2029 panic("Got unexpected FFA_DATA_ACCESS_RESERVED. Should be "
2030 "checked before this point.");
2031 }
2032
2033 switch (sent_instruction_access) {
2034 case FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED:
2035 case FFA_INSTRUCTION_ACCESS_X:
2036 if (requested_instruction_access ==
2037 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED ||
2038 requested_instruction_access == FFA_INSTRUCTION_ACCESS_X) {
2039 if (permissions != NULL) {
2040 ffa_set_instruction_access_attr(
2041 permissions, FFA_INSTRUCTION_ACCESS_X);
2042 }
2043 break;
2044 }
2045 case FFA_INSTRUCTION_ACCESS_NX:
2046 if (requested_instruction_access ==
2047 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED ||
2048 requested_instruction_access == FFA_INSTRUCTION_ACCESS_NX) {
2049 if (permissions != NULL) {
2050 ffa_set_instruction_access_attr(
2051 permissions, FFA_INSTRUCTION_ACCESS_NX);
2052 }
2053 break;
2054 }
2055 dlog_verbose(
2056 "Invalid instruction access requested; sender "
2057 "specified permissions %#x but receiver requested "
2058 "%#x.\n",
2059 sent_instruction_access, requested_instruction_access);
2060 return false;
2061 case FFA_INSTRUCTION_ACCESS_RESERVED:
2062 panic("Got unexpected FFA_INSTRUCTION_ACCESS_RESERVED. Should "
2063 "be checked before this point.");
2064 }
2065
2066 return true;
2067}
2068
2069/**
2070 * Validate the receivers' permissions in the retrieve request against those
2071 * specified by the lender.
2072 * In the `permissions` argument returns the permissions to set at S2 for the
2073 * caller to the FFA_MEMORY_RETRIEVE_REQ.
2074 * Returns FFA_SUCCESS if all specified permissions are valid.
2075 */
2076static struct ffa_value ffa_memory_retrieve_validate_memory_access_list(
2077 struct ffa_memory_region *memory_region,
2078 struct ffa_memory_region *retrieve_request, ffa_vm_id_t to_vm_id,
2079 ffa_memory_access_permissions_t *permissions)
2080{
2081 uint32_t retrieve_receiver_index;
2082
2083 assert(permissions != NULL);
2084
2085 if (retrieve_request->receiver_count != memory_region->receiver_count) {
2086 dlog_verbose(
2087 "Retrieve request should contain same list of "
2088 "borrowers, as specified by the lender.\n");
2089 return ffa_error(FFA_INVALID_PARAMETERS);
2090 }
2091
2092 retrieve_receiver_index = retrieve_request->receiver_count;
2093
2094 /* Should be populated with the permissions of the retriever. */
2095 *permissions = 0;
2096
2097 for (uint32_t i = 0U; i < retrieve_request->receiver_count; i++) {
2098 ffa_memory_access_permissions_t sent_permissions;
2099 struct ffa_memory_access *current_receiver =
2100 &retrieve_request->receivers[i];
2101 ffa_memory_access_permissions_t requested_permissions =
2102 current_receiver->receiver_permissions.permissions;
2103 ffa_vm_id_t current_receiver_id =
2104 current_receiver->receiver_permissions.receiver;
2105 bool found_to_id = current_receiver_id == to_vm_id;
2106
2107 /*
2108 * Find the current receiver in the transaction descriptor from
2109 * sender.
2110 */
2111 uint32_t mem_region_receiver_index =
2112 ffa_memory_region_get_receiver(memory_region,
2113 current_receiver_id);
2114
2115 if (mem_region_receiver_index ==
2116 memory_region->receiver_count) {
2117 dlog_verbose("%s: receiver %x not found\n", __func__,
2118 current_receiver_id);
2119 return ffa_error(FFA_DENIED);
2120 }
2121
2122 sent_permissions =
2123 memory_region->receivers[mem_region_receiver_index]
2124 .receiver_permissions.permissions;
2125
2126 if (found_to_id) {
2127 retrieve_receiver_index = i;
2128 }
2129
2130 /*
2131 * Since we are traversing the list of receivers, save the index
2132 * of the caller. As it needs to be there.
2133 */
2134
2135 if (current_receiver->composite_memory_region_offset != 0U) {
2136 dlog_verbose(
2137 "Retriever specified address ranges not "
2138 "supported (got offset %d).\n",
2139 current_receiver
2140 ->composite_memory_region_offset);
2141 return ffa_error(FFA_INVALID_PARAMETERS);
2142 }
2143
2144 /*
2145 * Check permissions from sender against permissions requested
2146 * by receiver.
2147 */
2148 if (!ffa_memory_retrieve_is_memory_access_valid(
2149 ffa_get_data_access_attr(sent_permissions),
2150 ffa_get_data_access_attr(requested_permissions),
2151 ffa_get_instruction_access_attr(sent_permissions),
2152 ffa_get_instruction_access_attr(
2153 requested_permissions),
2154 found_to_id ? permissions : NULL)) {
2155 return ffa_error(FFA_DENIED);
2156 }
2157
2158 /*
2159 * Can't request PM to clear memory if only provided with RO
2160 * permissions.
2161 */
2162 if (found_to_id &&
2163 (ffa_get_data_access_attr(*permissions) ==
2164 FFA_DATA_ACCESS_RO) &&
2165 (retrieve_request->flags & FFA_MEMORY_REGION_FLAG_CLEAR) !=
2166 0U) {
2167 dlog_verbose(
2168 "Receiver has RO permissions can not request "
2169 "clear.\n");
2170 return ffa_error(FFA_DENIED);
2171 }
2172 }
2173
2174 if (retrieve_receiver_index == retrieve_request->receiver_count) {
2175 dlog_verbose(
2176 "Retrieve request does not contain caller's (%x) "
2177 "permissions\n",
2178 to_vm_id);
2179 return ffa_error(FFA_INVALID_PARAMETERS);
2180 }
2181
2182 return (struct ffa_value){.func = FFA_SUCCESS_32};
2183}
2184
J-Alvesa9cd7e32022-07-01 13:49:33 +01002185/*
2186 * According to section 16.4.3 of FF-A v1.1 EAC0 specification, the hypervisor
2187 * may issue an FFA_MEM_RETRIEVE_REQ to obtain the memory region description
2188 * of a pending memory sharing operation whose allocator is the SPM, for
2189 * validation purposes before forwarding an FFA_MEM_RECLAIM call. In doing so
2190 * the memory region descriptor of the retrieve request must be zeroed with the
2191 * exception of the sender ID and handle.
2192 */
2193bool is_ffa_memory_retrieve_borrower_request(struct ffa_memory_region *request,
2194 struct vm_locked to_locked)
2195{
2196 return to_locked.vm->id == HF_HYPERVISOR_VM_ID &&
2197 request->attributes == 0U && request->flags == 0U &&
2198 request->tag == 0U && request->receiver_count == 0U &&
2199 plat_ffa_memory_handle_allocated_by_current_world(
2200 request->handle);
2201}
2202
2203/*
2204 * Helper to reset count of fragments retrieved by the hypervisor.
2205 */
2206static void ffa_memory_retrieve_complete_from_hyp(
2207 struct ffa_memory_share_state *share_state)
2208{
2209 if (share_state->hypervisor_fragment_count ==
2210 share_state->fragment_count) {
2211 share_state->hypervisor_fragment_count = 0;
2212 }
2213}
2214
J-Alves089004f2022-07-13 14:25:44 +01002215/**
2216 * Validate that the memory region descriptor provided by the borrower on
2217 * FFA_MEM_RETRIEVE_REQ, against saved memory region provided by lender at the
2218 * memory sharing call.
2219 */
2220static struct ffa_value ffa_memory_retrieve_validate(
2221 ffa_vm_id_t receiver_id, struct ffa_memory_region *retrieve_request,
2222 struct ffa_memory_region *memory_region, uint32_t *receiver_index,
2223 uint32_t share_func)
2224{
2225 ffa_memory_region_flags_t transaction_type =
2226 retrieve_request->flags &
2227 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK;
Olivier Deprez4342a3c2022-02-28 09:37:25 +01002228 enum ffa_memory_security security_state;
J-Alves089004f2022-07-13 14:25:44 +01002229
2230 assert(retrieve_request != NULL);
2231 assert(memory_region != NULL);
2232 assert(receiver_index != NULL);
2233 assert(retrieve_request->sender == memory_region->sender);
2234
2235 /*
2236 * Check that the transaction type expected by the receiver is
2237 * correct, if it has been specified.
2238 */
2239 if (transaction_type !=
2240 FFA_MEMORY_REGION_TRANSACTION_TYPE_UNSPECIFIED &&
2241 transaction_type != (memory_region->flags &
2242 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK)) {
2243 dlog_verbose(
2244 "Incorrect transaction type %#x for "
2245 "FFA_MEM_RETRIEVE_REQ, expected %#x for handle %#x.\n",
2246 transaction_type,
2247 memory_region->flags &
2248 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK,
2249 retrieve_request->handle);
2250 return ffa_error(FFA_INVALID_PARAMETERS);
2251 }
2252
2253 if (retrieve_request->tag != memory_region->tag) {
2254 dlog_verbose(
2255 "Incorrect tag %d for FFA_MEM_RETRIEVE_REQ, expected "
2256 "%d for handle %#x.\n",
2257 retrieve_request->tag, memory_region->tag,
2258 retrieve_request->handle);
2259 return ffa_error(FFA_INVALID_PARAMETERS);
2260 }
2261
2262 *receiver_index =
2263 ffa_memory_region_get_receiver(memory_region, receiver_id);
2264
2265 if (*receiver_index == memory_region->receiver_count) {
2266 dlog_verbose(
2267 "Incorrect receiver VM ID %d for "
2268 "FFA_MEM_RETRIEVE_REQ, for handle %#x.\n",
J-Alves59ed0042022-07-28 18:26:41 +01002269 receiver_id, memory_region->handle);
J-Alves089004f2022-07-13 14:25:44 +01002270 return ffa_error(FFA_INVALID_PARAMETERS);
2271 }
2272
2273 if ((retrieve_request->flags &
2274 FFA_MEMORY_REGION_ADDRESS_RANGE_HINT_VALID) != 0U) {
2275 dlog_verbose(
2276 "Retriever specified 'address range alignment 'hint' "
2277 "not supported.\n");
2278 return ffa_error(FFA_INVALID_PARAMETERS);
2279 }
2280 if ((retrieve_request->flags &
2281 FFA_MEMORY_REGION_ADDRESS_RANGE_HINT_MASK) != 0) {
2282 dlog_verbose(
2283 "Bits 8-5 must be zero in memory region's flags "
2284 "(address range alignment hint not supported).\n");
2285 return ffa_error(FFA_INVALID_PARAMETERS);
2286 }
2287
2288 if ((retrieve_request->flags & ~0x7FF) != 0U) {
2289 dlog_verbose(
2290 "Bits 31-10 must be zero in memory region's flags.\n");
2291 return ffa_error(FFA_INVALID_PARAMETERS);
2292 }
2293
2294 if (share_func == FFA_MEM_SHARE_32 &&
2295 (retrieve_request->flags &
2296 (FFA_MEMORY_REGION_FLAG_CLEAR |
2297 FFA_MEMORY_REGION_FLAG_CLEAR_RELINQUISH)) != 0U) {
2298 dlog_verbose(
2299 "Memory Share operation can't clean after relinquish "
2300 "memory region.\n");
2301 return ffa_error(FFA_INVALID_PARAMETERS);
2302 }
2303
2304 /*
2305 * If the borrower needs the memory to be cleared before mapping
2306 * to its address space, the sender should have set the flag
2307 * when calling FFA_MEM_LEND/FFA_MEM_DONATE, else return
2308 * FFA_DENIED.
2309 */
2310 if ((retrieve_request->flags & FFA_MEMORY_REGION_FLAG_CLEAR) != 0U &&
2311 (memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR) == 0U) {
2312 dlog_verbose(
2313 "Borrower needs memory cleared. Sender needs to set "
2314 "flag for clearing memory.\n");
2315 return ffa_error(FFA_DENIED);
2316 }
2317
Olivier Deprez4342a3c2022-02-28 09:37:25 +01002318 /* Memory region attributes NS-Bit MBZ for FFA_MEM_RETRIEVE_REQ. */
2319 security_state =
2320 ffa_get_memory_security_attr(retrieve_request->attributes);
2321 if (security_state != FFA_MEMORY_SECURITY_UNSPECIFIED) {
2322 dlog_verbose(
2323 "Invalid security state for memory retrieve request "
2324 "operation.\n");
2325 return ffa_error(FFA_INVALID_PARAMETERS);
2326 }
2327
J-Alves089004f2022-07-13 14:25:44 +01002328 /*
2329 * If memory type is not specified, bypass validation of memory
2330 * attributes in the retrieve request. The retriever is expecting to
2331 * obtain this information from the SPMC.
2332 */
2333 if (ffa_get_memory_type_attr(retrieve_request->attributes) ==
2334 FFA_MEMORY_NOT_SPECIFIED_MEM) {
2335 return (struct ffa_value){.func = FFA_SUCCESS_32};
2336 }
2337
2338 /*
2339 * Ensure receiver's attributes are compatible with how
2340 * Hafnium maps memory: Normal Memory, Inner shareable,
2341 * Write-Back Read-Allocate Write-Allocate Cacheable.
2342 */
2343 return ffa_memory_attributes_validate(retrieve_request->attributes);
2344}
2345
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002346struct ffa_value ffa_memory_retrieve(struct vm_locked to_locked,
2347 struct ffa_memory_region *retrieve_request,
Andrew Walbran130a8ae2020-05-15 16:27:15 +01002348 uint32_t retrieve_request_length,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002349 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002350{
Andrew Walbran130a8ae2020-05-15 16:27:15 +01002351 uint32_t expected_retrieve_request_length =
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002352 sizeof(struct ffa_memory_region) +
Andrew Walbrana65a1322020-04-06 19:32:32 +01002353 retrieve_request->receiver_count *
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002354 sizeof(struct ffa_memory_access);
2355 ffa_memory_handle_t handle = retrieve_request->handle;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002356 struct ffa_memory_region *memory_region;
J-Alvesa9cd7e32022-07-01 13:49:33 +01002357 ffa_memory_access_permissions_t permissions = 0;
Olivier Deprez878bd5b2021-04-15 19:05:10 +02002358 uint32_t memory_to_mode;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002359 struct share_states_locked share_states;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002360 struct ffa_memory_share_state *share_state;
2361 struct ffa_value ret;
Andrew Walbranca808b12020-05-15 17:22:28 +01002362 struct ffa_composite_memory_region *composite;
2363 uint32_t total_length;
2364 uint32_t fragment_length;
J-Alves089004f2022-07-13 14:25:44 +01002365 ffa_vm_id_t receiver_id = to_locked.vm->id;
J-Alvesa9cd7e32022-07-01 13:49:33 +01002366 bool is_send_complete = false;
Olivier Deprez878bd5b2021-04-15 19:05:10 +02002367 ffa_memory_attributes_t attributes;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002368
2369 dump_share_states();
2370
Andrew Walbran130a8ae2020-05-15 16:27:15 +01002371 if (retrieve_request_length != expected_retrieve_request_length) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002372 dlog_verbose(
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002373 "Invalid length for FFA_MEM_RETRIEVE_REQ, expected %d "
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002374 "but was %d.\n",
Andrew Walbran130a8ae2020-05-15 16:27:15 +01002375 expected_retrieve_request_length,
2376 retrieve_request_length);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002377 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002378 }
2379
2380 share_states = share_states_lock();
Karl Meakin4a2854a2023-06-30 16:26:52 +01002381 share_state = get_share_state(share_states, handle);
2382 if (!share_state) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002383 dlog_verbose("Invalid handle %#x for FFA_MEM_RETRIEVE_REQ.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002384 handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002385 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002386 goto out;
2387 }
2388
J-Alves96de29f2022-04-26 16:05:24 +01002389 if (!share_state->sending_complete) {
2390 dlog_verbose(
2391 "Memory with handle %#x not fully sent, can't "
2392 "retrieve.\n",
2393 handle);
2394 ret = ffa_error(FFA_INVALID_PARAMETERS);
2395 goto out;
2396 }
2397
Andrew Walbrana65a1322020-04-06 19:32:32 +01002398 memory_region = share_state->memory_region;
J-Alves089004f2022-07-13 14:25:44 +01002399
Andrew Walbrana65a1322020-04-06 19:32:32 +01002400 CHECK(memory_region != NULL);
2401
J-Alves089004f2022-07-13 14:25:44 +01002402 if (retrieve_request->sender != memory_region->sender) {
2403 dlog_verbose(
2404 "Memory with handle %#x not fully sent, can't "
2405 "retrieve.\n",
2406 handle);
2407 ret = ffa_error(FFA_INVALID_PARAMETERS);
2408 goto out;
2409 }
J-Alves96de29f2022-04-26 16:05:24 +01002410
J-Alvesa9cd7e32022-07-01 13:49:33 +01002411 if (!is_ffa_memory_retrieve_borrower_request(retrieve_request,
2412 to_locked)) {
2413 uint32_t receiver_index;
J-Alvesa9cd7e32022-07-01 13:49:33 +01002414
J-Alvesb5084cf2022-07-06 14:20:12 +01002415 /*
2416 * The SPMC can only process retrieve requests to memory share
2417 * operations with one borrower from the other world. It can't
2418 * determine the ID of the NWd VM that invoked the retrieve
2419 * request interface call. It relies on the hypervisor to
2420 * validate the caller's ID against that provided in the
2421 * `receivers` list of the retrieve response.
2422 * In case there is only one borrower from the NWd in the
2423 * transaction descriptor, record that in the `receiver_id` for
2424 * later use, and validate in the retrieve request message.
2425 */
2426 if (to_locked.vm->id == HF_HYPERVISOR_VM_ID) {
2427 uint32_t other_world_count = 0;
2428
2429 for (uint32_t i = 0; i < memory_region->receiver_count;
2430 i++) {
2431 receiver_id =
2432 retrieve_request->receivers[0]
2433 .receiver_permissions.receiver;
2434 if (!vm_id_is_current_world(receiver_id)) {
2435 other_world_count++;
2436 }
2437 }
2438 if (other_world_count > 1) {
2439 dlog_verbose(
2440 "Support one receiver from the other "
2441 "world.\n");
2442 return ffa_error(FFA_NOT_SUPPORTED);
2443 }
2444 }
2445
2446 /*
2447 * Validate retrieve request, according to what was sent by the
2448 * sender. Function will output the `receiver_index` from the
2449 * provided memory region, and will output `permissions` from
2450 * the validated requested permissions.
2451 */
J-Alves089004f2022-07-13 14:25:44 +01002452 ret = ffa_memory_retrieve_validate(
2453 receiver_id, retrieve_request, memory_region,
2454 &receiver_index, share_state->share_func);
2455 if (ret.func != FFA_SUCCESS_32) {
J-Alvesa9cd7e32022-07-01 13:49:33 +01002456 goto out;
2457 }
2458
2459 if (share_state->retrieved_fragment_count[receiver_index] !=
2460 0U) {
2461 dlog_verbose(
2462 "Memory with handle %#x already retrieved.\n",
2463 handle);
2464 ret = ffa_error(FFA_DENIED);
2465 goto out;
2466 }
2467
J-Alvesa9cd7e32022-07-01 13:49:33 +01002468 ret = ffa_memory_retrieve_validate_memory_access_list(
2469 memory_region, retrieve_request, receiver_id,
2470 &permissions);
J-Alves614d9f42022-06-28 14:03:10 +01002471 if (ret.func != FFA_SUCCESS_32) {
2472 goto out;
2473 }
Federico Recanatia98603a2021-12-20 18:04:03 +01002474
Olivier Deprez878bd5b2021-04-15 19:05:10 +02002475 memory_to_mode = ffa_memory_permissions_to_mode(
J-Alvesa9cd7e32022-07-01 13:49:33 +01002476 permissions, share_state->sender_orig_mode);
J-Alves40e260e2022-09-22 17:52:43 +01002477
J-Alvesa9cd7e32022-07-01 13:49:33 +01002478 ret = ffa_retrieve_check_update(
2479 to_locked, memory_region->sender,
2480 share_state->fragments,
2481 share_state->fragment_constituent_counts,
Olivier Deprez878bd5b2021-04-15 19:05:10 +02002482 share_state->fragment_count, memory_to_mode,
J-Alvesa9cd7e32022-07-01 13:49:33 +01002483 share_state->share_func, false, page_pool);
2484
2485 if (ret.func != FFA_SUCCESS_32) {
2486 goto out;
2487 }
2488
2489 share_state->retrieved_fragment_count[receiver_index] = 1;
2490 is_send_complete =
2491 share_state->retrieved_fragment_count[receiver_index] ==
2492 share_state->fragment_count;
J-Alves3c5b2072022-11-21 12:45:40 +00002493
2494 share_state->clear_after_relinquish =
2495 (retrieve_request->flags &
2496 FFA_MEMORY_REGION_FLAG_CLEAR_RELINQUISH) != 0U;
2497
J-Alvesa9cd7e32022-07-01 13:49:33 +01002498 } else {
2499 if (share_state->hypervisor_fragment_count != 0U) {
2500 dlog_verbose(
J-Alvesb5084cf2022-07-06 14:20:12 +01002501 "Memory with handle %#x already retrieved by "
J-Alvesa9cd7e32022-07-01 13:49:33 +01002502 "the hypervisor.\n",
2503 handle);
2504 ret = ffa_error(FFA_DENIED);
2505 goto out;
2506 }
2507
2508 share_state->hypervisor_fragment_count = 1;
2509
2510 ffa_memory_retrieve_complete_from_hyp(share_state);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002511 }
2512
J-Alvesb5084cf2022-07-06 14:20:12 +01002513 /* VMs acquire the RX buffer from SPMC. */
2514 CHECK(plat_ffa_acquire_receiver_rx(to_locked, &ret));
2515
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002516 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01002517 * Copy response to RX buffer of caller and deliver the message.
2518 * This must be done before the share_state is (possibly) freed.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002519 */
Andrew Walbrana65a1322020-04-06 19:32:32 +01002520 /* TODO: combine attributes from sender and request. */
Andrew Walbranca808b12020-05-15 17:22:28 +01002521 composite = ffa_memory_region_get_composite(memory_region, 0);
2522 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01002523 * Constituents which we received in the first fragment should
2524 * always fit in the first fragment we are sending, because the
2525 * header is the same size in both cases and we have a fixed
2526 * message buffer size. So `ffa_retrieved_memory_region_init`
2527 * should never fail.
Andrew Walbranca808b12020-05-15 17:22:28 +01002528 */
Olivier Deprez878bd5b2021-04-15 19:05:10 +02002529
2530 /*
2531 * Set the security state in the memory retrieve response attributes
2532 * if specified by the target mode.
2533 */
2534 attributes = plat_ffa_memory_security_mode(
2535 memory_region->attributes, share_state->sender_orig_mode);
2536
Andrew Walbranca808b12020-05-15 17:22:28 +01002537 CHECK(ffa_retrieved_memory_region_init(
J-Alves2d8457f2022-10-05 11:06:41 +01002538 to_locked.vm->mailbox.recv, to_locked.vm->ffa_version,
Olivier Deprez878bd5b2021-04-15 19:05:10 +02002539 HF_MAILBOX_SIZE, memory_region->sender, attributes,
2540 memory_region->flags, handle, receiver_id, permissions,
2541 composite->page_count, composite->constituent_count,
2542 share_state->fragments[0],
Andrew Walbranca808b12020-05-15 17:22:28 +01002543 share_state->fragment_constituent_counts[0], &total_length,
2544 &fragment_length));
J-Alvesb5084cf2022-07-06 14:20:12 +01002545
Andrew Walbranca808b12020-05-15 17:22:28 +01002546 to_locked.vm->mailbox.recv_size = fragment_length;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002547 to_locked.vm->mailbox.recv_sender = HF_HYPERVISOR_VM_ID;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002548 to_locked.vm->mailbox.recv_func = FFA_MEM_RETRIEVE_RESP_32;
J-Alvese8c8c2b2022-12-16 15:34:48 +00002549 to_locked.vm->mailbox.state = MAILBOX_STATE_FULL;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002550
J-Alvesa9cd7e32022-07-01 13:49:33 +01002551 if (is_send_complete) {
Andrew Walbranca808b12020-05-15 17:22:28 +01002552 ffa_memory_retrieve_complete(share_states, share_state,
2553 page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002554 }
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002555 ret = (struct ffa_value){.func = FFA_MEM_RETRIEVE_RESP_32,
Andrew Walbranca808b12020-05-15 17:22:28 +01002556 .arg1 = total_length,
2557 .arg2 = fragment_length};
Andrew Walbranca808b12020-05-15 17:22:28 +01002558out:
2559 share_states_unlock(&share_states);
2560 dump_share_states();
2561 return ret;
2562}
2563
J-Alves5da37d92022-10-24 16:33:48 +01002564/**
2565 * Determine expected fragment offset according to the FF-A version of
2566 * the caller.
2567 */
2568static uint32_t ffa_memory_retrieve_expected_offset_per_ffa_version(
2569 struct ffa_memory_region *memory_region,
2570 uint32_t retrieved_constituents_count, uint32_t ffa_version)
2571{
2572 uint32_t expected_fragment_offset;
2573 uint32_t composite_constituents_offset;
2574
2575 if (ffa_version == MAKE_FFA_VERSION(1, 1)) {
2576 /*
2577 * Hafnium operates memory regions in FF-A v1.1 format, so we
2578 * can retrieve the constituents offset from descriptor.
2579 */
2580 composite_constituents_offset =
2581 ffa_composite_constituent_offset(memory_region, 0);
2582 } else if (ffa_version == MAKE_FFA_VERSION(1, 0)) {
2583 /*
2584 * If retriever is FF-A v1.0, determine the composite offset
2585 * as it is expected to have been configured in the
2586 * retrieve response.
2587 */
2588 composite_constituents_offset =
2589 sizeof(struct ffa_memory_region_v1_0) +
2590 RECEIVERS_COUNT_IN_RETRIEVE_RESP *
2591 sizeof(struct ffa_memory_access) +
2592 sizeof(struct ffa_composite_memory_region);
2593 } else {
2594 panic("%s received an invalid FF-A version.\n", __func__);
2595 }
2596
2597 expected_fragment_offset =
2598 composite_constituents_offset +
2599 retrieved_constituents_count *
2600 sizeof(struct ffa_memory_region_constituent) -
2601 sizeof(struct ffa_memory_access) *
2602 (memory_region->receiver_count - 1);
2603
2604 return expected_fragment_offset;
2605}
2606
Andrew Walbranca808b12020-05-15 17:22:28 +01002607struct ffa_value ffa_memory_retrieve_continue(struct vm_locked to_locked,
2608 ffa_memory_handle_t handle,
2609 uint32_t fragment_offset,
J-Alves59ed0042022-07-28 18:26:41 +01002610 ffa_vm_id_t sender_vm_id,
Andrew Walbranca808b12020-05-15 17:22:28 +01002611 struct mpool *page_pool)
2612{
2613 struct ffa_memory_region *memory_region;
2614 struct share_states_locked share_states;
2615 struct ffa_memory_share_state *share_state;
2616 struct ffa_value ret;
2617 uint32_t fragment_index;
2618 uint32_t retrieved_constituents_count;
2619 uint32_t i;
2620 uint32_t expected_fragment_offset;
2621 uint32_t remaining_constituent_count;
2622 uint32_t fragment_length;
J-Alvesc7484f12022-05-13 12:41:14 +01002623 uint32_t receiver_index;
J-Alves59ed0042022-07-28 18:26:41 +01002624 bool continue_ffa_hyp_mem_retrieve_req;
Andrew Walbranca808b12020-05-15 17:22:28 +01002625
2626 dump_share_states();
2627
2628 share_states = share_states_lock();
Karl Meakin4a2854a2023-06-30 16:26:52 +01002629 share_state = get_share_state(share_states, handle);
2630 if (!share_state) {
Andrew Walbranca808b12020-05-15 17:22:28 +01002631 dlog_verbose("Invalid handle %#x for FFA_MEM_FRAG_RX.\n",
2632 handle);
2633 ret = ffa_error(FFA_INVALID_PARAMETERS);
2634 goto out;
2635 }
2636
2637 memory_region = share_state->memory_region;
2638 CHECK(memory_region != NULL);
2639
Andrew Walbranca808b12020-05-15 17:22:28 +01002640 if (!share_state->sending_complete) {
2641 dlog_verbose(
2642 "Memory with handle %#x not fully sent, can't "
2643 "retrieve.\n",
2644 handle);
2645 ret = ffa_error(FFA_INVALID_PARAMETERS);
2646 goto out;
2647 }
2648
J-Alves59ed0042022-07-28 18:26:41 +01002649 /*
2650 * If retrieve request from the hypervisor has been initiated in the
2651 * given share_state, continue it, else assume it is a continuation of
2652 * retrieve request from a NWd VM.
2653 */
2654 continue_ffa_hyp_mem_retrieve_req =
2655 (to_locked.vm->id == HF_HYPERVISOR_VM_ID) &&
2656 (share_state->hypervisor_fragment_count != 0U) &&
2657 plat_ffa_is_vm_id(sender_vm_id);
Andrew Walbranca808b12020-05-15 17:22:28 +01002658
J-Alves59ed0042022-07-28 18:26:41 +01002659 if (!continue_ffa_hyp_mem_retrieve_req) {
2660 receiver_index = ffa_memory_region_get_receiver(
2661 memory_region, to_locked.vm->id);
2662
2663 if (receiver_index == memory_region->receiver_count) {
2664 dlog_verbose(
2665 "Caller of FFA_MEM_FRAG_RX (%x) is not a "
2666 "borrower to memory sharing transaction (%x)\n",
2667 to_locked.vm->id, handle);
2668 ret = ffa_error(FFA_INVALID_PARAMETERS);
2669 goto out;
2670 }
2671
2672 if (share_state->retrieved_fragment_count[receiver_index] ==
2673 0 ||
2674 share_state->retrieved_fragment_count[receiver_index] >=
2675 share_state->fragment_count) {
2676 dlog_verbose(
2677 "Retrieval of memory with handle %#x not yet "
2678 "started or already completed (%d/%d fragments "
2679 "retrieved).\n",
2680 handle,
2681 share_state->retrieved_fragment_count
2682 [receiver_index],
2683 share_state->fragment_count);
2684 ret = ffa_error(FFA_INVALID_PARAMETERS);
2685 goto out;
2686 }
2687
2688 fragment_index =
2689 share_state->retrieved_fragment_count[receiver_index];
2690 } else {
2691 if (share_state->hypervisor_fragment_count == 0 ||
2692 share_state->hypervisor_fragment_count >=
2693 share_state->fragment_count) {
2694 dlog_verbose(
2695 "Retrieve of memory with handle %x not "
2696 "started from hypervisor.\n",
2697 handle);
2698 ret = ffa_error(FFA_INVALID_PARAMETERS);
2699 goto out;
2700 }
2701
2702 if (memory_region->sender != sender_vm_id) {
2703 dlog_verbose(
2704 "Sender ID (%x) is not as expected for memory "
2705 "handle %x\n",
2706 sender_vm_id, handle);
2707 ret = ffa_error(FFA_INVALID_PARAMETERS);
2708 goto out;
2709 }
2710
2711 fragment_index = share_state->hypervisor_fragment_count;
2712
2713 receiver_index = 0;
2714 }
Andrew Walbranca808b12020-05-15 17:22:28 +01002715
2716 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01002717 * Check that the given fragment offset is correct by counting
2718 * how many constituents were in the fragments previously sent.
Andrew Walbranca808b12020-05-15 17:22:28 +01002719 */
2720 retrieved_constituents_count = 0;
2721 for (i = 0; i < fragment_index; ++i) {
2722 retrieved_constituents_count +=
2723 share_state->fragment_constituent_counts[i];
2724 }
J-Alvesc7484f12022-05-13 12:41:14 +01002725
2726 CHECK(memory_region->receiver_count > 0);
2727
Andrew Walbranca808b12020-05-15 17:22:28 +01002728 expected_fragment_offset =
J-Alves5da37d92022-10-24 16:33:48 +01002729 ffa_memory_retrieve_expected_offset_per_ffa_version(
2730 memory_region, retrieved_constituents_count,
2731 to_locked.vm->ffa_version);
2732
Andrew Walbranca808b12020-05-15 17:22:28 +01002733 if (fragment_offset != expected_fragment_offset) {
2734 dlog_verbose("Fragment offset was %d but expected %d.\n",
2735 fragment_offset, expected_fragment_offset);
2736 ret = ffa_error(FFA_INVALID_PARAMETERS);
2737 goto out;
2738 }
2739
J-Alves59ed0042022-07-28 18:26:41 +01002740 /* VMs acquire the RX buffer from SPMC. */
2741 CHECK(plat_ffa_acquire_receiver_rx(to_locked, &ret));
2742
Andrew Walbranca808b12020-05-15 17:22:28 +01002743 remaining_constituent_count = ffa_memory_fragment_init(
2744 to_locked.vm->mailbox.recv, HF_MAILBOX_SIZE,
2745 share_state->fragments[fragment_index],
2746 share_state->fragment_constituent_counts[fragment_index],
2747 &fragment_length);
2748 CHECK(remaining_constituent_count == 0);
2749 to_locked.vm->mailbox.recv_size = fragment_length;
2750 to_locked.vm->mailbox.recv_sender = HF_HYPERVISOR_VM_ID;
2751 to_locked.vm->mailbox.recv_func = FFA_MEM_FRAG_TX_32;
J-Alvese8c8c2b2022-12-16 15:34:48 +00002752 to_locked.vm->mailbox.state = MAILBOX_STATE_FULL;
Andrew Walbranca808b12020-05-15 17:22:28 +01002753
J-Alves59ed0042022-07-28 18:26:41 +01002754 if (!continue_ffa_hyp_mem_retrieve_req) {
2755 share_state->retrieved_fragment_count[receiver_index]++;
2756 if (share_state->retrieved_fragment_count[receiver_index] ==
2757 share_state->fragment_count) {
2758 ffa_memory_retrieve_complete(share_states, share_state,
2759 page_pool);
2760 }
2761 } else {
2762 share_state->hypervisor_fragment_count++;
2763
2764 ffa_memory_retrieve_complete_from_hyp(share_state);
2765 }
Andrew Walbranca808b12020-05-15 17:22:28 +01002766 ret = (struct ffa_value){.func = FFA_MEM_FRAG_TX_32,
2767 .arg1 = (uint32_t)handle,
2768 .arg2 = (uint32_t)(handle >> 32),
2769 .arg3 = fragment_length};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002770
2771out:
2772 share_states_unlock(&share_states);
2773 dump_share_states();
2774 return ret;
2775}
2776
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002777struct ffa_value ffa_memory_relinquish(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002778 struct vm_locked from_locked,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002779 struct ffa_mem_relinquish *relinquish_request, struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002780{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002781 ffa_memory_handle_t handle = relinquish_request->handle;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002782 struct share_states_locked share_states;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002783 struct ffa_memory_share_state *share_state;
2784 struct ffa_memory_region *memory_region;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002785 bool clear;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002786 struct ffa_value ret;
J-Alves8eb19162022-04-28 10:56:48 +01002787 uint32_t receiver_index;
J-Alves3c5b2072022-11-21 12:45:40 +00002788 bool receivers_relinquished_memory;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002789
Andrew Walbrana65a1322020-04-06 19:32:32 +01002790 if (relinquish_request->endpoint_count != 1) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002791 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01002792 "Stream endpoints not supported (got %d "
J-Alves668a86e2023-05-10 11:53:25 +01002793 "endpoints on FFA_MEM_RELINQUISH, expected 1).\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002794 relinquish_request->endpoint_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002795 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002796 }
2797
Andrew Walbrana65a1322020-04-06 19:32:32 +01002798 if (relinquish_request->endpoints[0] != from_locked.vm->id) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002799 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01002800 "VM ID %d in relinquish message doesn't match "
J-Alves668a86e2023-05-10 11:53:25 +01002801 "calling VM ID %d.\n",
Andrew Walbrana65a1322020-04-06 19:32:32 +01002802 relinquish_request->endpoints[0], from_locked.vm->id);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002803 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002804 }
2805
2806 dump_share_states();
2807
2808 share_states = share_states_lock();
Karl Meakin4a2854a2023-06-30 16:26:52 +01002809 share_state = get_share_state(share_states, handle);
2810 if (!share_state) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002811 dlog_verbose("Invalid handle %#x for FFA_MEM_RELINQUISH.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002812 handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002813 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002814 goto out;
2815 }
2816
Andrew Walbranca808b12020-05-15 17:22:28 +01002817 if (!share_state->sending_complete) {
2818 dlog_verbose(
2819 "Memory with handle %#x not fully sent, can't "
2820 "relinquish.\n",
2821 handle);
2822 ret = ffa_error(FFA_INVALID_PARAMETERS);
2823 goto out;
2824 }
2825
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002826 memory_region = share_state->memory_region;
2827 CHECK(memory_region != NULL);
2828
J-Alves8eb19162022-04-28 10:56:48 +01002829 receiver_index = ffa_memory_region_get_receiver(memory_region,
2830 from_locked.vm->id);
2831
2832 if (receiver_index == memory_region->receiver_count) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002833 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01002834 "VM ID %d tried to relinquish memory region "
J-Alves668a86e2023-05-10 11:53:25 +01002835 "with handle %#x and it is not a valid borrower.\n",
J-Alves8eb19162022-04-28 10:56:48 +01002836 from_locked.vm->id, handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002837 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002838 goto out;
2839 }
2840
J-Alves8eb19162022-04-28 10:56:48 +01002841 if (share_state->retrieved_fragment_count[receiver_index] !=
Andrew Walbranca808b12020-05-15 17:22:28 +01002842 share_state->fragment_count) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002843 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01002844 "Memory with handle %#x not yet fully "
2845 "retrieved, "
J-Alves8eb19162022-04-28 10:56:48 +01002846 "receiver %x can't relinquish.\n",
2847 handle, from_locked.vm->id);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002848 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002849 goto out;
2850 }
2851
J-Alves3c5b2072022-11-21 12:45:40 +00002852 /*
2853 * Either clear if requested in relinquish call, or in a retrieve
2854 * request from one of the borrowers.
2855 */
2856 receivers_relinquished_memory = true;
2857
2858 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
2859 struct ffa_memory_access *receiver =
2860 &memory_region->receivers[i];
2861
2862 if (receiver->receiver_permissions.receiver ==
2863 from_locked.vm->id) {
2864 continue;
2865 }
2866
2867 if (share_state->retrieved_fragment_count[i] != 0U) {
2868 receivers_relinquished_memory = false;
2869 break;
2870 }
2871 }
2872
2873 clear = receivers_relinquished_memory &&
2874 (share_state->clear_after_relinquish ||
2875 (relinquish_request->flags & FFA_MEMORY_REGION_FLAG_CLEAR) !=
2876 0U);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002877
2878 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01002879 * Clear is not allowed for memory that was shared, as the
2880 * original sender still has access to the memory.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002881 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002882 if (clear && share_state->share_func == FFA_MEM_SHARE_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002883 dlog_verbose("Memory which was shared can't be cleared.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002884 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002885 goto out;
2886 }
2887
Andrew Walbranca808b12020-05-15 17:22:28 +01002888 ret = ffa_relinquish_check_update(
J-Alves3c5b2072022-11-21 12:45:40 +00002889 from_locked, memory_region->sender, share_state->fragments,
Andrew Walbranca808b12020-05-15 17:22:28 +01002890 share_state->fragment_constituent_counts,
2891 share_state->fragment_count, page_pool, clear);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002892
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002893 if (ret.func == FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002894 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01002895 * Mark memory handle as not retrieved, so it can be
2896 * reclaimed (or retrieved again).
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002897 */
J-Alves8eb19162022-04-28 10:56:48 +01002898 share_state->retrieved_fragment_count[receiver_index] = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002899 }
2900
2901out:
2902 share_states_unlock(&share_states);
2903 dump_share_states();
2904 return ret;
2905}
2906
2907/**
J-Alvesa9cd7e32022-07-01 13:49:33 +01002908 * Validates that the reclaim transition is allowed for the given
2909 * handle, updates the page table of the reclaiming VM, and frees the
2910 * internal state associated with the handle.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002911 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002912struct ffa_value ffa_memory_reclaim(struct vm_locked to_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01002913 ffa_memory_handle_t handle,
2914 ffa_memory_region_flags_t flags,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002915 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002916{
2917 struct share_states_locked share_states;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002918 struct ffa_memory_share_state *share_state;
2919 struct ffa_memory_region *memory_region;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002920 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002921
2922 dump_share_states();
2923
2924 share_states = share_states_lock();
Karl Meakin52cdfe72023-06-30 14:49:10 +01002925
Karl Meakin4a2854a2023-06-30 16:26:52 +01002926 share_state = get_share_state(share_states, handle);
2927 if (!share_state) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002928 dlog_verbose("Invalid handle %#x for FFA_MEM_RECLAIM.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002929 handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002930 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002931 goto out;
2932 }
Karl Meakin4a2854a2023-06-30 16:26:52 +01002933 memory_region = share_state->memory_region;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002934
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002935 CHECK(memory_region != NULL);
2936
J-Alvesa9cd7e32022-07-01 13:49:33 +01002937 if (vm_id_is_current_world(to_locked.vm->id) &&
2938 to_locked.vm->id != memory_region->sender) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002939 dlog_verbose(
Olivier Deprezf92e5d42020-11-13 16:00:54 +01002940 "VM %#x attempted to reclaim memory handle %#x "
2941 "originally sent by VM %#x.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002942 to_locked.vm->id, handle, memory_region->sender);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002943 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002944 goto out;
2945 }
2946
Andrew Walbranca808b12020-05-15 17:22:28 +01002947 if (!share_state->sending_complete) {
2948 dlog_verbose(
2949 "Memory with handle %#x not fully sent, can't "
2950 "reclaim.\n",
2951 handle);
2952 ret = ffa_error(FFA_INVALID_PARAMETERS);
2953 goto out;
2954 }
2955
J-Alves752236c2022-04-28 11:07:47 +01002956 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
2957 if (share_state->retrieved_fragment_count[i] != 0) {
2958 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01002959 "Tried to reclaim memory handle %#x "
J-Alves3c5b2072022-11-21 12:45:40 +00002960 "that has not been relinquished by all "
J-Alvesa9cd7e32022-07-01 13:49:33 +01002961 "borrowers(%x).\n",
J-Alves752236c2022-04-28 11:07:47 +01002962 handle,
2963 memory_region->receivers[i]
2964 .receiver_permissions.receiver);
2965 ret = ffa_error(FFA_DENIED);
2966 goto out;
2967 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002968 }
2969
Andrew Walbranca808b12020-05-15 17:22:28 +01002970 ret = ffa_retrieve_check_update(
J-Alves7db32002021-12-14 14:44:50 +00002971 to_locked, memory_region->sender, share_state->fragments,
Andrew Walbranca808b12020-05-15 17:22:28 +01002972 share_state->fragment_constituent_counts,
J-Alves2a0d2882020-10-29 14:49:50 +00002973 share_state->fragment_count, share_state->sender_orig_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +01002974 FFA_MEM_RECLAIM_32, flags & FFA_MEM_RECLAIM_CLEAR, page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002975
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002976 if (ret.func == FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002977 share_state_free(share_states, share_state, page_pool);
J-Alves3c5b2072022-11-21 12:45:40 +00002978 dlog_verbose("Freed share state after successful reclaim.\n");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002979 }
2980
2981out:
2982 share_states_unlock(&share_states);
2983 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01002984}