blob: 504308cad7a8d529aa05de68237ddc6ee0b377c9 [file] [log] [blame]
Jose Marinho75509b42019-04-09 09:34:59 +01001/*
2 * Copyright 2019 The Hafnium Authors.
3 *
Andrew Walbrane959ec12020-06-17 15:01:09 +01004 * Use of this source code is governed by a BSD-style
5 * license that can be found in the LICENSE file or at
6 * https://opensource.org/licenses/BSD-3-Clause.
Jose Marinho75509b42019-04-09 09:34:59 +01007 */
8
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01009#include "hf/ffa_memory.h"
Andrew Walbran475c1452020-02-07 13:22:22 +000010
Federico Recanati4fd065d2021-12-13 20:06:23 +010011#include "hf/arch/mm.h"
Olivier Deprez112d2b52020-09-30 07:39:23 +020012#include "hf/arch/other_world.h"
Olivier Deprez55a189e2021-06-09 15:45:27 +020013#include "hf/arch/plat/ffa.h"
Andrew Walbran290b0c92020-02-03 16:37:14 +000014
J-Alves5952d942022-12-22 16:03:00 +000015#include "hf/addr.h"
Jose Marinho75509b42019-04-09 09:34:59 +010016#include "hf/api.h"
Daniel Boulbya2f8c662021-11-26 17:52:53 +000017#include "hf/assert.h"
Jose Marinho09b1db82019-08-08 09:16:59 +010018#include "hf/check.h"
Jose Marinho75509b42019-04-09 09:34:59 +010019#include "hf/dlog.h"
J-Alves3456e032023-07-20 12:20:05 +010020#include "hf/ffa.h"
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010021#include "hf/ffa_internal.h"
J-Alves66652252022-07-06 09:49:51 +010022#include "hf/ffa_memory_internal.h"
J-Alves3456e032023-07-20 12:20:05 +010023#include "hf/ffa_partition_manifest.h"
J-Alves5952d942022-12-22 16:03:00 +000024#include "hf/mm.h"
Andrew Walbran475c1452020-02-07 13:22:22 +000025#include "hf/mpool.h"
Jose Marinho75509b42019-04-09 09:34:59 +010026#include "hf/std.h"
Andrew Scull3c257452019-11-26 13:32:50 +000027#include "hf/vm.h"
Daniel Boulby44e9b3b2024-01-17 12:21:44 +000028#include "hf/vm_ids.h"
Jose Marinho75509b42019-04-09 09:34:59 +010029
J-Alves2d8457f2022-10-05 11:06:41 +010030#include "vmapi/hf/ffa_v1_0.h"
31
J-Alves5da37d92022-10-24 16:33:48 +010032#define RECEIVERS_COUNT_IN_RETRIEVE_RESP 1
33
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000034/**
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010035 * All access to members of a `struct ffa_memory_share_state` must be guarded
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000036 * by this lock.
37 */
38static struct spinlock share_states_lock_instance = SPINLOCK_INIT;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010039static struct ffa_memory_share_state share_states[MAX_MEM_SHARES];
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000040
41/**
J-Alvesed508c82023-05-04 16:09:48 +010042 * Return the offset to the first constituent within the
43 * `ffa_composite_memory_region` for the given receiver from an
44 * `ffa_memory_region`. The caller must check that the receiver_index is within
45 * bounds, and that it has a composite memory region offset.
46 */
47static uint32_t ffa_composite_constituent_offset(
48 struct ffa_memory_region *memory_region, uint32_t receiver_index)
49{
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +000050 struct ffa_memory_access *receiver;
51 uint32_t composite_offset;
J-Alvesed508c82023-05-04 16:09:48 +010052
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +000053 CHECK(receiver_index < memory_region->receiver_count);
54
55 receiver =
56 ffa_memory_region_get_receiver(memory_region, receiver_index);
57 CHECK(receiver != NULL);
58
59 composite_offset = receiver->composite_memory_region_offset;
60
61 CHECK(composite_offset != 0);
62
63 return composite_offset + sizeof(struct ffa_composite_memory_region);
J-Alvesed508c82023-05-04 16:09:48 +010064}
65
66/**
J-Alves917d2f22020-10-30 18:39:30 +000067 * Extracts the index from a memory handle allocated by Hafnium's current world.
68 */
69uint64_t ffa_memory_handle_get_index(ffa_memory_handle_t handle)
70{
71 return handle & ~FFA_MEMORY_HANDLE_ALLOCATOR_MASK;
72}
73
74/**
Karl Meakin52cdfe72023-06-30 14:49:10 +010075 * Initialises the next available `struct ffa_memory_share_state`. If `handle`
76 * is `FFA_MEMORY_HANDLE_INVALID` then allocates an appropriate handle,
77 * otherwise uses the provided handle which is assumed to be globally unique.
Andrew Walbranca808b12020-05-15 17:22:28 +010078 *
Karl Meakin52cdfe72023-06-30 14:49:10 +010079 * Returns a pointer to the allocated `ffa_memory_share_state` on success or
80 * `NULL` if none are available.
Andrew Walbranca808b12020-05-15 17:22:28 +010081 */
Karl Meakin52cdfe72023-06-30 14:49:10 +010082struct ffa_memory_share_state *allocate_share_state(
83 struct share_states_locked share_states, uint32_t share_func,
84 struct ffa_memory_region *memory_region, uint32_t fragment_length,
85 ffa_memory_handle_t handle)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000086{
Daniel Boulbya2f8c662021-11-26 17:52:53 +000087 assert(share_states.share_states != NULL);
88 assert(memory_region != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000089
Karl Meakin52cdfe72023-06-30 14:49:10 +010090 for (uint64_t i = 0; i < MAX_MEM_SHARES; ++i) {
Andrew Walbranca808b12020-05-15 17:22:28 +010091 if (share_states.share_states[i].share_func == 0) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010092 struct ffa_memory_share_state *allocated_state =
Andrew Walbranca808b12020-05-15 17:22:28 +010093 &share_states.share_states[i];
94 struct ffa_composite_memory_region *composite =
95 ffa_memory_region_get_composite(memory_region,
96 0);
97
98 if (handle == FFA_MEMORY_HANDLE_INVALID) {
J-Alvesee68c542020-10-29 17:48:20 +000099 memory_region->handle =
Olivier Deprez55a189e2021-06-09 15:45:27 +0200100 plat_ffa_memory_handle_make(i);
Andrew Walbranca808b12020-05-15 17:22:28 +0100101 } else {
J-Alvesee68c542020-10-29 17:48:20 +0000102 memory_region->handle = handle;
Andrew Walbranca808b12020-05-15 17:22:28 +0100103 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000104 allocated_state->share_func = share_func;
105 allocated_state->memory_region = memory_region;
Andrew Walbranca808b12020-05-15 17:22:28 +0100106 allocated_state->fragment_count = 1;
107 allocated_state->fragments[0] = composite->constituents;
108 allocated_state->fragment_constituent_counts[0] =
109 (fragment_length -
110 ffa_composite_constituent_offset(memory_region,
111 0)) /
112 sizeof(struct ffa_memory_region_constituent);
113 allocated_state->sending_complete = false;
Karl Meakin52cdfe72023-06-30 14:49:10 +0100114 for (uint32_t j = 0; j < MAX_MEM_SHARE_RECIPIENTS;
115 ++j) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100116 allocated_state->retrieved_fragment_count[j] =
117 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000118 }
Karl Meakin52cdfe72023-06-30 14:49:10 +0100119 return allocated_state;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000120 }
121 }
122
Karl Meakin52cdfe72023-06-30 14:49:10 +0100123 return NULL;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000124}
125
126/** Locks the share states lock. */
127struct share_states_locked share_states_lock(void)
128{
129 sl_lock(&share_states_lock_instance);
130
131 return (struct share_states_locked){.share_states = share_states};
132}
133
134/** Unlocks the share states lock. */
J-Alves66652252022-07-06 09:49:51 +0100135void share_states_unlock(struct share_states_locked *share_states)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000136{
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000137 assert(share_states->share_states != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000138 share_states->share_states = NULL;
139 sl_unlock(&share_states_lock_instance);
140}
141
142/**
Andrew Walbranca808b12020-05-15 17:22:28 +0100143 * If the given handle is a valid handle for an allocated share state then
Karl Meakin4a2854a2023-06-30 16:26:52 +0100144 * returns a pointer to the share state. Otherwise returns NULL.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000145 */
Karl Meakin4a2854a2023-06-30 16:26:52 +0100146struct ffa_memory_share_state *get_share_state(
147 struct share_states_locked share_states, ffa_memory_handle_t handle)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000148{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100149 struct ffa_memory_share_state *share_state;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000150
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000151 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100152
153 /*
154 * First look for a share_state allocated by us, in which case the
155 * handle is based on the index.
156 */
Olivier Deprez55a189e2021-06-09 15:45:27 +0200157 if (plat_ffa_memory_handle_allocated_by_current_world(handle)) {
Karl Meakin4a2854a2023-06-30 16:26:52 +0100158 uint64_t index = ffa_memory_handle_get_index(handle);
159
Andrew Walbranca808b12020-05-15 17:22:28 +0100160 if (index < MAX_MEM_SHARES) {
161 share_state = &share_states.share_states[index];
162 if (share_state->share_func != 0) {
Karl Meakin4a2854a2023-06-30 16:26:52 +0100163 return share_state;
Andrew Walbranca808b12020-05-15 17:22:28 +0100164 }
165 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000166 }
167
Andrew Walbranca808b12020-05-15 17:22:28 +0100168 /* Fall back to a linear scan. */
Karl Meakin4a2854a2023-06-30 16:26:52 +0100169 for (uint64_t index = 0; index < MAX_MEM_SHARES; ++index) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100170 share_state = &share_states.share_states[index];
J-Alvesee68c542020-10-29 17:48:20 +0000171 if (share_state->memory_region != NULL &&
172 share_state->memory_region->handle == handle &&
Andrew Walbranca808b12020-05-15 17:22:28 +0100173 share_state->share_func != 0) {
Karl Meakin4a2854a2023-06-30 16:26:52 +0100174 return share_state;
Andrew Walbranca808b12020-05-15 17:22:28 +0100175 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000176 }
177
Karl Meakin4a2854a2023-06-30 16:26:52 +0100178 return NULL;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000179}
180
181/** Marks a share state as unallocated. */
J-Alvesfdd29272022-07-19 13:16:31 +0100182void share_state_free(struct share_states_locked share_states,
183 struct ffa_memory_share_state *share_state,
184 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000185{
Andrew Walbranca808b12020-05-15 17:22:28 +0100186 uint32_t i;
187
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000188 assert(share_states.share_states != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000189 share_state->share_func = 0;
Andrew Walbranca808b12020-05-15 17:22:28 +0100190 share_state->sending_complete = false;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000191 mpool_free(page_pool, share_state->memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +0100192 /*
193 * First fragment is part of the same page as the `memory_region`, so it
194 * doesn't need to be freed separately.
195 */
196 share_state->fragments[0] = NULL;
197 share_state->fragment_constituent_counts[0] = 0;
198 for (i = 1; i < share_state->fragment_count; ++i) {
199 mpool_free(page_pool, share_state->fragments[i]);
200 share_state->fragments[i] = NULL;
201 share_state->fragment_constituent_counts[i] = 0;
202 }
203 share_state->fragment_count = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000204 share_state->memory_region = NULL;
J-Alvesa9cd7e32022-07-01 13:49:33 +0100205 share_state->hypervisor_fragment_count = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000206}
207
Andrew Walbranca808b12020-05-15 17:22:28 +0100208/** Checks whether the given share state has been fully sent. */
J-Alvesfdd29272022-07-19 13:16:31 +0100209bool share_state_sending_complete(struct share_states_locked share_states,
210 struct ffa_memory_share_state *share_state)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000211{
Andrew Walbranca808b12020-05-15 17:22:28 +0100212 struct ffa_composite_memory_region *composite;
213 uint32_t expected_constituent_count;
214 uint32_t fragment_constituent_count_total = 0;
215 uint32_t i;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000216
Andrew Walbranca808b12020-05-15 17:22:28 +0100217 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000218 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100219
220 /*
221 * Share state must already be valid, or it's not possible to get hold
222 * of it.
223 */
224 CHECK(share_state->memory_region != NULL &&
225 share_state->share_func != 0);
226
227 composite =
228 ffa_memory_region_get_composite(share_state->memory_region, 0);
229 expected_constituent_count = composite->constituent_count;
230 for (i = 0; i < share_state->fragment_count; ++i) {
231 fragment_constituent_count_total +=
232 share_state->fragment_constituent_counts[i];
233 }
234 dlog_verbose(
235 "Checking completion: constituent count %d/%d from %d "
236 "fragments.\n",
237 fragment_constituent_count_total, expected_constituent_count,
238 share_state->fragment_count);
239
240 return fragment_constituent_count_total == expected_constituent_count;
241}
242
243/**
244 * Calculates the offset of the next fragment expected for the given share
245 * state.
246 */
J-Alvesfdd29272022-07-19 13:16:31 +0100247uint32_t share_state_next_fragment_offset(
Andrew Walbranca808b12020-05-15 17:22:28 +0100248 struct share_states_locked share_states,
249 struct ffa_memory_share_state *share_state)
250{
251 uint32_t next_fragment_offset;
252 uint32_t i;
253
254 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000255 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100256
257 next_fragment_offset =
258 ffa_composite_constituent_offset(share_state->memory_region, 0);
259 for (i = 0; i < share_state->fragment_count; ++i) {
260 next_fragment_offset +=
261 share_state->fragment_constituent_counts[i] *
262 sizeof(struct ffa_memory_region_constituent);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000263 }
264
Andrew Walbranca808b12020-05-15 17:22:28 +0100265 return next_fragment_offset;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000266}
267
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100268static void dump_memory_region(struct ffa_memory_region *memory_region)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000269{
270 uint32_t i;
271
272 if (LOG_LEVEL < LOG_LEVEL_VERBOSE) {
273 return;
274 }
275
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000276 dlog("from VM %#x, attributes %#x, flags %#x, handle %#x "
277 "tag %u, memory access descriptor size %u, to %u "
Andrew Walbrana65a1322020-04-06 19:32:32 +0100278 "recipients [",
279 memory_region->sender, memory_region->attributes,
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000280 memory_region->flags, memory_region->handle, memory_region->tag,
281 memory_region->memory_access_desc_size,
Andrew Walbrana65a1322020-04-06 19:32:32 +0100282 memory_region->receiver_count);
283 for (i = 0; i < memory_region->receiver_count; ++i) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +0000284 struct ffa_memory_access *receiver =
285 ffa_memory_region_get_receiver(memory_region, i);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000286 if (i != 0) {
287 dlog(", ");
288 }
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +0000289 dlog("Receiver %#x: %#x (offset %u)",
290 receiver->receiver_permissions.receiver,
291 receiver->receiver_permissions.permissions,
292 receiver->composite_memory_region_offset);
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000293 /* The impdef field is only present from v1.2 and later */
294 if (ffa_version_from_memory_access_desc_size(
295 memory_region->memory_access_desc_size) >=
296 MAKE_FFA_VERSION(1, 2)) {
297 dlog(", impdef: %#x %#x", receiver->impdef.val[0],
298 receiver->impdef.val[1]);
299 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000300 }
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000301 dlog("] at offset %u", memory_region->receivers_offset);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000302}
303
J-Alves66652252022-07-06 09:49:51 +0100304void dump_share_states(void)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000305{
306 uint32_t i;
307
308 if (LOG_LEVEL < LOG_LEVEL_VERBOSE) {
309 return;
310 }
311
312 dlog("Current share states:\n");
313 sl_lock(&share_states_lock_instance);
314 for (i = 0; i < MAX_MEM_SHARES; ++i) {
315 if (share_states[i].share_func != 0) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000316 switch (share_states[i].share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100317 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000318 dlog("SHARE");
319 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100320 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000321 dlog("LEND");
322 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100323 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000324 dlog("DONATE");
325 break;
326 default:
327 dlog("invalid share_func %#x",
328 share_states[i].share_func);
329 }
Olivier Deprez935e1b12020-12-22 18:01:29 +0100330 dlog(" %#x (", share_states[i].memory_region->handle);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000331 dump_memory_region(share_states[i].memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +0100332 if (share_states[i].sending_complete) {
333 dlog("): fully sent");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000334 } else {
Andrew Walbranca808b12020-05-15 17:22:28 +0100335 dlog("): partially sent");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000336 }
J-Alves2a0d2882020-10-29 14:49:50 +0000337 dlog(" with %d fragments, %d retrieved, "
338 " sender's original mode: %#x\n",
Andrew Walbranca808b12020-05-15 17:22:28 +0100339 share_states[i].fragment_count,
J-Alves2a0d2882020-10-29 14:49:50 +0000340 share_states[i].retrieved_fragment_count[0],
341 share_states[i].sender_orig_mode);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000342 }
343 }
344 sl_unlock(&share_states_lock_instance);
345}
346
Andrew Walbran475c1452020-02-07 13:22:22 +0000347/* TODO: Add device attributes: GRE, cacheability, shareability. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100348static inline uint32_t ffa_memory_permissions_to_mode(
J-Alves7cd5eb32020-10-16 19:06:10 +0100349 ffa_memory_access_permissions_t permissions, uint32_t default_mode)
Andrew Walbran475c1452020-02-07 13:22:22 +0000350{
351 uint32_t mode = 0;
352
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100353 switch (ffa_get_data_access_attr(permissions)) {
354 case FFA_DATA_ACCESS_RO:
Andrew Walbran475c1452020-02-07 13:22:22 +0000355 mode = MM_MODE_R;
356 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100357 case FFA_DATA_ACCESS_RW:
Andrew Walbran475c1452020-02-07 13:22:22 +0000358 mode = MM_MODE_R | MM_MODE_W;
359 break;
J-Alves7cd5eb32020-10-16 19:06:10 +0100360 case FFA_DATA_ACCESS_NOT_SPECIFIED:
361 mode = (default_mode & (MM_MODE_R | MM_MODE_W));
362 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100363 case FFA_DATA_ACCESS_RESERVED:
364 panic("Tried to convert FFA_DATA_ACCESS_RESERVED.");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100365 }
366
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100367 switch (ffa_get_instruction_access_attr(permissions)) {
368 case FFA_INSTRUCTION_ACCESS_NX:
Andrew Walbran475c1452020-02-07 13:22:22 +0000369 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100370 case FFA_INSTRUCTION_ACCESS_X:
Andrew Walbrana65a1322020-04-06 19:32:32 +0100371 mode |= MM_MODE_X;
372 break;
J-Alves7cd5eb32020-10-16 19:06:10 +0100373 case FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED:
374 mode |= (default_mode & MM_MODE_X);
375 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100376 case FFA_INSTRUCTION_ACCESS_RESERVED:
377 panic("Tried to convert FFA_INSTRUCTION_ACCESS_RESVERVED.");
Andrew Walbran475c1452020-02-07 13:22:22 +0000378 }
379
Olivier Deprez878bd5b2021-04-15 19:05:10 +0200380 /* Set the security state bit if necessary. */
381 if ((default_mode & plat_ffa_other_world_mode()) != 0) {
382 mode |= plat_ffa_other_world_mode();
383 }
384
Andrew Walbran475c1452020-02-07 13:22:22 +0000385 return mode;
386}
387
Jose Marinho75509b42019-04-09 09:34:59 +0100388/**
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000389 * Get the current mode in the stage-2 page table of the given vm of all the
390 * pages in the given constituents, if they all have the same mode, or return
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100391 * an appropriate FF-A error if not.
Jose Marinho75509b42019-04-09 09:34:59 +0100392 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100393static struct ffa_value constituents_get_mode(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000394 struct vm_locked vm, uint32_t *orig_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100395 struct ffa_memory_region_constituent **fragments,
396 const uint32_t *fragment_constituent_counts, uint32_t fragment_count)
Jose Marinho75509b42019-04-09 09:34:59 +0100397{
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100398 uint32_t i;
Andrew Walbranca808b12020-05-15 17:22:28 +0100399 uint32_t j;
Jose Marinho75509b42019-04-09 09:34:59 +0100400
Andrew Walbranca808b12020-05-15 17:22:28 +0100401 if (fragment_count == 0 || fragment_constituent_counts[0] == 0) {
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100402 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000403 * Fail if there are no constituents. Otherwise we would get an
404 * uninitialised *orig_mode.
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100405 */
Karl Meakin5df422c2023-07-11 17:31:38 +0100406 dlog_verbose("%s: no constituents\n", __func__);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100407 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho75509b42019-04-09 09:34:59 +0100408 }
409
Andrew Walbranca808b12020-05-15 17:22:28 +0100410 for (i = 0; i < fragment_count; ++i) {
411 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
412 ipaddr_t begin = ipa_init(fragments[i][j].address);
413 size_t size = fragments[i][j].page_count * PAGE_SIZE;
414 ipaddr_t end = ipa_add(begin, size);
415 uint32_t current_mode;
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100416
Andrew Walbranca808b12020-05-15 17:22:28 +0100417 /* Fail if addresses are not page-aligned. */
418 if (!is_aligned(ipa_addr(begin), PAGE_SIZE) ||
419 !is_aligned(ipa_addr(end), PAGE_SIZE)) {
Karl Meakin5df422c2023-07-11 17:31:38 +0100420 dlog_verbose("%s: addresses not page-aligned\n",
421 __func__);
Andrew Walbranca808b12020-05-15 17:22:28 +0100422 return ffa_error(FFA_INVALID_PARAMETERS);
423 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100424
Andrew Walbranca808b12020-05-15 17:22:28 +0100425 /*
426 * Ensure that this constituent memory range is all
427 * mapped with the same mode.
428 */
Raghu Krishnamurthy785d52f2021-02-13 00:02:40 -0800429 if (!vm_mem_get_mode(vm, begin, end, &current_mode)) {
Karl Meakin5df422c2023-07-11 17:31:38 +0100430 dlog_verbose(
431 "%s: constituent memory range %#x..%#x "
432 "not mapped with the same mode\n",
433 __func__, begin, end);
Andrew Walbranca808b12020-05-15 17:22:28 +0100434 return ffa_error(FFA_DENIED);
435 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100436
Andrew Walbranca808b12020-05-15 17:22:28 +0100437 /*
438 * Ensure that all constituents are mapped with the same
439 * mode.
440 */
441 if (i == 0) {
442 *orig_mode = current_mode;
443 } else if (current_mode != *orig_mode) {
444 dlog_verbose(
Karl Meakin5df422c2023-07-11 17:31:38 +0100445 "%s: expected mode %#x but was %#x for "
446 "%d pages at %#x.\n",
447 __func__, *orig_mode, current_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100448 fragments[i][j].page_count,
449 ipa_addr(begin));
450 return ffa_error(FFA_DENIED);
451 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100452 }
Jose Marinho75509b42019-04-09 09:34:59 +0100453 }
454
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100455 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000456}
457
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100458uint32_t ffa_version_from_memory_access_desc_size(
459 uint32_t memory_access_desc_size)
460{
461 switch (memory_access_desc_size) {
462 /*
463 * v1.0 and v1.1 memory access descriptors are the same size however
464 * v1.1 is the first version to include the memory access descriptor
465 * size field so return v1.1.
466 */
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000467 case sizeof(struct ffa_memory_access_v1_0):
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100468 return MAKE_FFA_VERSION(1, 1);
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000469 case sizeof(struct ffa_memory_access):
470 return MAKE_FFA_VERSION(1, 2);
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100471 }
472 return 0;
473}
474
475/**
476 * Check if the receivers size and offset given is valid for the senders
477 * FF-A version.
478 */
479static bool receiver_size_and_offset_valid_for_version(
480 uint32_t receivers_size, uint32_t receivers_offset,
481 uint32_t ffa_version)
482{
483 /*
484 * Check that the version that the memory access descriptor size belongs
485 * to is compatible with the FF-A version we believe the sender to be.
486 */
487 uint32_t expected_ffa_version =
488 ffa_version_from_memory_access_desc_size(receivers_size);
489 if (!FFA_VERSIONS_ARE_COMPATIBLE(expected_ffa_version, ffa_version)) {
490 return false;
491 }
492
493 /*
494 * Check the receivers_offset matches the version we found from
495 * memory access descriptor size.
496 */
497 switch (expected_ffa_version) {
498 case MAKE_FFA_VERSION(1, 1):
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000499 case MAKE_FFA_VERSION(1, 2):
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100500 return receivers_offset == sizeof(struct ffa_memory_region);
501 default:
502 return false;
503 }
504}
505
506/**
507 * Check the values set for fields in the memory region are valid and safe.
508 * Offset values are within safe bounds, receiver count will not cause overflows
509 * and reserved fields are 0.
510 */
511bool ffa_memory_region_sanity_check(struct ffa_memory_region *memory_region,
512 uint32_t ffa_version,
513 uint32_t fragment_length,
514 bool send_transaction)
515{
516 uint32_t receiver_count;
517 struct ffa_memory_access *receiver;
518 uint32_t composite_offset_0;
519
520 if (ffa_version == MAKE_FFA_VERSION(1, 0)) {
521 struct ffa_memory_region_v1_0 *memory_region_v1_0 =
522 (struct ffa_memory_region_v1_0 *)memory_region;
523 /* Check the reserved fields are 0. */
524 if (memory_region_v1_0->reserved_0 != 0 ||
525 memory_region_v1_0->reserved_1 != 0) {
526 dlog_verbose("Reserved fields must be 0.\n");
527 return false;
528 }
529
530 receiver_count = memory_region_v1_0->receiver_count;
531 } else {
532 uint32_t receivers_size =
533 memory_region->memory_access_desc_size;
534 uint32_t receivers_offset = memory_region->receivers_offset;
535
536 /* Check the reserved field is 0. */
537 if (memory_region->reserved[0] != 0 ||
538 memory_region->reserved[1] != 0 ||
539 memory_region->reserved[2] != 0) {
540 dlog_verbose("Reserved fields must be 0.\n");
541 return false;
542 }
543
544 /*
545 * Check memory_access_desc_size matches the size of the struct
546 * for the senders FF-A version.
547 */
548 if (!receiver_size_and_offset_valid_for_version(
549 receivers_size, receivers_offset, ffa_version)) {
550 dlog_verbose(
551 "Invalid memory access descriptor size %d, "
552 " or receiver offset %d, "
553 "for FF-A version %#x\n",
554 receivers_size, receivers_offset, ffa_version);
555 return false;
556 }
557
558 receiver_count = memory_region->receiver_count;
559 }
560
561 /* Check receiver count is not too large. */
562 if (receiver_count > MAX_MEM_SHARE_RECIPIENTS) {
563 dlog_verbose(
564 "Max number of recipients supported is %u "
565 "specified %u\n",
566 MAX_MEM_SHARE_RECIPIENTS, receiver_count);
567 return false;
568 }
569
570 /* Check values in the memory access descriptors. */
571 /*
572 * The composite offset values must be the same for all recievers so
573 * check the first one is valid and then they are all the same.
574 */
575 receiver = ffa_version == MAKE_FFA_VERSION(1, 0)
576 ? (struct ffa_memory_access *)&(
577 (struct ffa_memory_region_v1_0 *)
578 memory_region)
579 ->receivers[0]
580 : ffa_memory_region_get_receiver(memory_region, 0);
581 assert(receiver != NULL);
582 composite_offset_0 = receiver->composite_memory_region_offset;
583
584 if (!send_transaction) {
585 if (composite_offset_0 != 0) {
586 dlog_verbose(
587 "Composite offset memory region descriptor "
588 "offset must be 0 for retrieve requests. "
589 "Currently %d",
590 composite_offset_0);
591 return false;
592 }
593 } else {
594 bool comp_offset_is_zero = composite_offset_0 == 0U;
595 bool comp_offset_lt_transaction_descriptor_size =
596 composite_offset_0 <
597 (sizeof(struct ffa_memory_region) +
598 (uint32_t)(memory_region->memory_access_desc_size *
599 memory_region->receiver_count));
600 bool comp_offset_with_comp_gt_fragment_length =
601 composite_offset_0 +
602 sizeof(struct ffa_composite_memory_region) >
603 fragment_length;
604 if (comp_offset_is_zero ||
605 comp_offset_lt_transaction_descriptor_size ||
606 comp_offset_with_comp_gt_fragment_length) {
607 dlog_verbose(
608 "Invalid composite memory region descriptor "
609 "offset for send transaction %u\n",
610 composite_offset_0);
611 return false;
612 }
613 }
614
615 for (int i = 0; i < memory_region->receiver_count; i++) {
616 uint32_t composite_offset;
617
618 if (ffa_version == MAKE_FFA_VERSION(1, 0)) {
619 struct ffa_memory_region_v1_0 *memory_region_v1_0 =
620 (struct ffa_memory_region_v1_0 *)memory_region;
621
622 struct ffa_memory_access_v1_0 *receiver_v1_0 =
623 &memory_region_v1_0->receivers[i];
624 /* Check reserved fields are 0 */
625 if (receiver_v1_0->reserved_0 != 0) {
626 dlog_verbose(
627 "Reserved field in the memory access "
628 " descriptor must be zero "
629 " Currently reciever %d has a reserved "
630 " field with a value of %d\n",
631 i, receiver_v1_0->reserved_0);
632 return false;
633 }
634 /*
635 * We can cast to the current version receiver as the
636 * remaining fields we are checking have the same
637 * offsets for all versions since memory access
638 * descriptors are forwards compatible.
639 */
640 receiver = (struct ffa_memory_access *)receiver_v1_0;
641 } else {
642 receiver = ffa_memory_region_get_receiver(memory_region,
643 i);
644 assert(receiver != NULL);
645
646 if (receiver->reserved_0 != 0) {
647 dlog_verbose(
648 "Reserved field in the memory access "
649 " descriptor must be zero "
650 " Currently reciever %d has a reserved "
651 " field with a value of %d\n",
652 i, receiver->reserved_0);
653 return false;
654 }
655 }
656
657 /* Check composite offset values are equal for all receivers. */
658 composite_offset = receiver->composite_memory_region_offset;
659 if (composite_offset != composite_offset_0) {
660 dlog_verbose(
661 "Composite offset %x differs from %x in index "
662 "%u\n",
663 composite_offset, composite_offset_0);
664 return false;
665 }
666 }
667 return true;
668}
669
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000670/**
671 * Verify that all pages have the same mode, that the starting mode
672 * constitutes a valid state and obtain the next mode to apply
673 * to the sending VM.
674 *
675 * Returns:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100676 * 1) FFA_DENIED if a state transition was not found;
677 * 2) FFA_DENIED if the pages being shared do not have the same mode within
Andrew Walbrana65a1322020-04-06 19:32:32 +0100678 * the <from> VM;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100679 * 3) FFA_INVALID_PARAMETERS if the beginning and end IPAs are not page
Andrew Walbrana65a1322020-04-06 19:32:32 +0100680 * aligned;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100681 * 4) FFA_INVALID_PARAMETERS if the requested share type was not handled.
682 * Or FFA_SUCCESS on success.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000683 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100684static struct ffa_value ffa_send_check_transition(
Andrew Walbrana65a1322020-04-06 19:32:32 +0100685 struct vm_locked from, uint32_t share_func,
J-Alves363f5722022-04-25 17:37:37 +0100686 struct ffa_memory_access *receivers, uint32_t receivers_count,
687 uint32_t *orig_from_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100688 struct ffa_memory_region_constituent **fragments,
689 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
690 uint32_t *from_mode)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000691{
692 const uint32_t state_mask =
693 MM_MODE_INVALID | MM_MODE_UNOWNED | MM_MODE_SHARED;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100694 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000695
Andrew Walbranca808b12020-05-15 17:22:28 +0100696 ret = constituents_get_mode(from, orig_from_mode, fragments,
697 fragment_constituent_counts,
698 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100699 if (ret.func != FFA_SUCCESS_32) {
Olivier Depreze7eb1682022-03-16 17:09:03 +0100700 dlog_verbose("Inconsistent modes.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100701 return ret;
Andrew Scullb5f49e02019-10-02 13:20:47 +0100702 }
703
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000704 /* Ensure the address range is normal memory and not a device. */
J-Alves788b4492023-04-18 14:01:23 +0100705 if ((*orig_from_mode & MM_MODE_D) != 0U) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000706 dlog_verbose("Can't share device memory (mode is %#x).\n",
707 *orig_from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100708 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000709 }
710
711 /*
712 * Ensure the sender is the owner and has exclusive access to the
713 * memory.
714 */
715 if ((*orig_from_mode & state_mask) != 0) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100716 return ffa_error(FFA_DENIED);
Andrew Walbrana65a1322020-04-06 19:32:32 +0100717 }
718
J-Alves363f5722022-04-25 17:37:37 +0100719 assert(receivers != NULL && receivers_count > 0U);
J-Alves7cd5eb32020-10-16 19:06:10 +0100720
J-Alves363f5722022-04-25 17:37:37 +0100721 for (uint32_t i = 0U; i < receivers_count; i++) {
722 ffa_memory_access_permissions_t permissions =
723 receivers[i].receiver_permissions.permissions;
724 uint32_t required_from_mode = ffa_memory_permissions_to_mode(
725 permissions, *orig_from_mode);
726
J-Alves788b4492023-04-18 14:01:23 +0100727 /*
728 * The assumption is that at this point, the operation from
729 * SP to a receiver VM, should have returned an FFA_ERROR
730 * already.
731 */
732 if (!ffa_is_vm_id(from.vm->id)) {
733 assert(!ffa_is_vm_id(
734 receivers[i].receiver_permissions.receiver));
735 }
736
J-Alves363f5722022-04-25 17:37:37 +0100737 if ((*orig_from_mode & required_from_mode) !=
738 required_from_mode) {
739 dlog_verbose(
740 "Sender tried to send memory with permissions "
J-Alves788b4492023-04-18 14:01:23 +0100741 "which required mode %#x but only had %#x "
742 "itself.\n",
J-Alves363f5722022-04-25 17:37:37 +0100743 required_from_mode, *orig_from_mode);
744 return ffa_error(FFA_DENIED);
745 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000746 }
747
748 /* Find the appropriate new mode. */
749 *from_mode = ~state_mask & *orig_from_mode;
Andrew Walbrane7ad3c02019-12-24 17:03:04 +0000750 switch (share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100751 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000752 *from_mode |= MM_MODE_INVALID | MM_MODE_UNOWNED;
Jose Marinho75509b42019-04-09 09:34:59 +0100753 break;
754
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100755 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000756 *from_mode |= MM_MODE_INVALID;
Andrew Walbran648fc3e2019-10-22 16:23:05 +0100757 break;
758
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100759 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000760 *from_mode |= MM_MODE_SHARED;
Jose Marinho56c25732019-05-20 09:48:53 +0100761 break;
762
Jose Marinho75509b42019-04-09 09:34:59 +0100763 default:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100764 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho75509b42019-04-09 09:34:59 +0100765 }
766
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100767 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000768}
769
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100770static struct ffa_value ffa_relinquish_check_transition(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000771 struct vm_locked from, uint32_t *orig_from_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100772 struct ffa_memory_region_constituent **fragments,
773 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
774 uint32_t *from_mode)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000775{
776 const uint32_t state_mask =
777 MM_MODE_INVALID | MM_MODE_UNOWNED | MM_MODE_SHARED;
778 uint32_t orig_from_state;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100779 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000780
Andrew Walbranca808b12020-05-15 17:22:28 +0100781 ret = constituents_get_mode(from, orig_from_mode, fragments,
782 fragment_constituent_counts,
783 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100784 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbrana65a1322020-04-06 19:32:32 +0100785 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000786 }
787
788 /* Ensure the address range is normal memory and not a device. */
789 if (*orig_from_mode & MM_MODE_D) {
790 dlog_verbose("Can't relinquish device memory (mode is %#x).\n",
791 *orig_from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100792 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000793 }
794
795 /*
796 * Ensure the relinquishing VM is not the owner but has access to the
797 * memory.
798 */
799 orig_from_state = *orig_from_mode & state_mask;
800 if ((orig_from_state & ~MM_MODE_SHARED) != MM_MODE_UNOWNED) {
801 dlog_verbose(
802 "Tried to relinquish memory in state %#x (masked %#x "
Andrew Walbranca808b12020-05-15 17:22:28 +0100803 "but should be %#x).\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000804 *orig_from_mode, orig_from_state, MM_MODE_UNOWNED);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100805 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000806 }
807
808 /* Find the appropriate new mode. */
809 *from_mode = (~state_mask & *orig_from_mode) | MM_MODE_UNMAPPED_MASK;
810
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100811 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000812}
813
814/**
815 * Verify that all pages have the same mode, that the starting mode
816 * constitutes a valid state and obtain the next mode to apply
817 * to the retrieving VM.
818 *
819 * Returns:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100820 * 1) FFA_DENIED if a state transition was not found;
821 * 2) FFA_DENIED if the pages being shared do not have the same mode within
Andrew Walbrana65a1322020-04-06 19:32:32 +0100822 * the <to> VM;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100823 * 3) FFA_INVALID_PARAMETERS if the beginning and end IPAs are not page
Andrew Walbrana65a1322020-04-06 19:32:32 +0100824 * aligned;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100825 * 4) FFA_INVALID_PARAMETERS if the requested share type was not handled.
826 * Or FFA_SUCCESS on success.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000827 */
J-Alvesfc19b372022-07-06 12:17:35 +0100828struct ffa_value ffa_retrieve_check_transition(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000829 struct vm_locked to, uint32_t share_func,
Andrew Walbranca808b12020-05-15 17:22:28 +0100830 struct ffa_memory_region_constituent **fragments,
831 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
832 uint32_t memory_to_attributes, uint32_t *to_mode)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000833{
834 uint32_t orig_to_mode;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100835 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000836
Andrew Walbranca808b12020-05-15 17:22:28 +0100837 ret = constituents_get_mode(to, &orig_to_mode, fragments,
838 fragment_constituent_counts,
839 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100840 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100841 dlog_verbose("Inconsistent modes.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100842 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000843 }
844
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100845 if (share_func == FFA_MEM_RECLAIM_32) {
J-Alves9256f162021-12-09 13:18:43 +0000846 /*
847 * If the original ffa memory send call has been processed
848 * successfully, it is expected the orig_to_mode would overlay
849 * with `state_mask`, as a result of the function
850 * `ffa_send_check_transition`.
851 */
J-Alves59ed0042022-07-28 18:26:41 +0100852 if (vm_id_is_current_world(to.vm->id)) {
853 assert((orig_to_mode &
854 (MM_MODE_INVALID | MM_MODE_UNOWNED |
855 MM_MODE_SHARED)) != 0U);
856 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000857 } else {
858 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +0100859 * If the retriever is from virtual FF-A instance:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000860 * Ensure the retriever has the expected state. We don't care
861 * about the MM_MODE_SHARED bit; either with or without it set
862 * are both valid representations of the !O-NA state.
863 */
J-Alvesa9cd7e32022-07-01 13:49:33 +0100864 if (vm_id_is_current_world(to.vm->id) &&
865 to.vm->id != HF_PRIMARY_VM_ID &&
866 (orig_to_mode & MM_MODE_UNMAPPED_MASK) !=
867 MM_MODE_UNMAPPED_MASK) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100868 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000869 }
870 }
871
872 /* Find the appropriate new mode. */
873 *to_mode = memory_to_attributes;
874 switch (share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100875 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000876 *to_mode |= 0;
877 break;
878
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100879 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000880 *to_mode |= MM_MODE_UNOWNED;
881 break;
882
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100883 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000884 *to_mode |= MM_MODE_UNOWNED | MM_MODE_SHARED;
885 break;
886
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100887 case FFA_MEM_RECLAIM_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000888 *to_mode |= 0;
889 break;
890
891 default:
Andrew Walbranca808b12020-05-15 17:22:28 +0100892 dlog_error("Invalid share_func %#x.\n", share_func);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100893 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000894 }
895
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100896 return (struct ffa_value){.func = FFA_SUCCESS_32};
Jose Marinho75509b42019-04-09 09:34:59 +0100897}
Jose Marinho09b1db82019-08-08 09:16:59 +0100898
899/**
900 * Updates a VM's page table such that the given set of physical address ranges
901 * are mapped in the address space at the corresponding address ranges, in the
902 * mode provided.
903 *
904 * If commit is false, the page tables will be allocated from the mpool but no
905 * mappings will actually be updated. This function must always be called first
906 * with commit false to check that it will succeed before calling with commit
907 * true, to avoid leaving the page table in a half-updated state. To make a
908 * series of changes atomically you can call them all with commit false before
909 * calling them all with commit true.
910 *
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -0700911 * vm_ptable_defrag should always be called after a series of page table
912 * updates, whether they succeed or fail.
Jose Marinho09b1db82019-08-08 09:16:59 +0100913 *
914 * Returns true on success, or false if the update failed and no changes were
915 * made to memory mappings.
916 */
J-Alves66652252022-07-06 09:49:51 +0100917bool ffa_region_group_identity_map(
Andrew Walbranf4b51af2020-02-03 14:44:54 +0000918 struct vm_locked vm_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +0100919 struct ffa_memory_region_constituent **fragments,
920 const uint32_t *fragment_constituent_counts, uint32_t fragment_count,
Daniel Boulby4dd3f532021-09-21 09:57:08 +0100921 uint32_t mode, struct mpool *ppool, bool commit)
Jose Marinho09b1db82019-08-08 09:16:59 +0100922{
Andrew Walbranca808b12020-05-15 17:22:28 +0100923 uint32_t i;
924 uint32_t j;
Jose Marinho09b1db82019-08-08 09:16:59 +0100925
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -0700926 if (vm_locked.vm->el0_partition) {
927 mode |= MM_MODE_USER | MM_MODE_NG;
928 }
929
Andrew Walbranca808b12020-05-15 17:22:28 +0100930 /* Iterate over the memory region constituents within each fragment. */
931 for (i = 0; i < fragment_count; ++i) {
932 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
933 size_t size = fragments[i][j].page_count * PAGE_SIZE;
934 paddr_t pa_begin =
935 pa_from_ipa(ipa_init(fragments[i][j].address));
936 paddr_t pa_end = pa_add(pa_begin, size);
Jens Wiklander4f1880c2022-10-19 17:00:14 +0200937 uint32_t pa_bits =
938 arch_mm_get_pa_bits(arch_mm_get_pa_range());
Federico Recanati4fd065d2021-12-13 20:06:23 +0100939
940 /*
941 * Ensure the requested region falls into system's PA
942 * range.
943 */
Jens Wiklander4f1880c2022-10-19 17:00:14 +0200944 if (((pa_addr(pa_begin) >> pa_bits) > 0) ||
945 ((pa_addr(pa_end) >> pa_bits) > 0)) {
Federico Recanati4fd065d2021-12-13 20:06:23 +0100946 dlog_error("Region is outside of PA Range\n");
947 return false;
948 }
Andrew Walbranca808b12020-05-15 17:22:28 +0100949
950 if (commit) {
951 vm_identity_commit(vm_locked, pa_begin, pa_end,
952 mode, ppool, NULL);
953 } else if (!vm_identity_prepare(vm_locked, pa_begin,
954 pa_end, mode, ppool)) {
955 return false;
956 }
Jose Marinho09b1db82019-08-08 09:16:59 +0100957 }
958 }
959
960 return true;
961}
962
963/**
964 * Clears a region of physical memory by overwriting it with zeros. The data is
965 * flushed from the cache so the memory has been cleared across the system.
966 */
J-Alves7db32002021-12-14 14:44:50 +0000967static bool clear_memory(paddr_t begin, paddr_t end, struct mpool *ppool,
968 uint32_t extra_mode_attributes)
Jose Marinho09b1db82019-08-08 09:16:59 +0100969{
970 /*
Fuad Tabbaed294af2019-12-20 10:43:01 +0000971 * TODO: change this to a CPU local single page window rather than a
Jose Marinho09b1db82019-08-08 09:16:59 +0100972 * global mapping of the whole range. Such an approach will limit
973 * the changes to stage-1 tables and will allow only local
974 * invalidation.
975 */
976 bool ret;
977 struct mm_stage1_locked stage1_locked = mm_lock_stage1();
J-Alves7db32002021-12-14 14:44:50 +0000978 void *ptr = mm_identity_map(stage1_locked, begin, end,
979 MM_MODE_W | (extra_mode_attributes &
980 plat_ffa_other_world_mode()),
981 ppool);
Jose Marinho09b1db82019-08-08 09:16:59 +0100982 size_t size = pa_difference(begin, end);
983
984 if (!ptr) {
Jose Marinho09b1db82019-08-08 09:16:59 +0100985 goto fail;
986 }
987
988 memset_s(ptr, size, 0, size);
989 arch_mm_flush_dcache(ptr, size);
990 mm_unmap(stage1_locked, begin, end, ppool);
991
992 ret = true;
993 goto out;
994
995fail:
996 ret = false;
997
998out:
999 mm_unlock_stage1(&stage1_locked);
1000
1001 return ret;
1002}
1003
1004/**
1005 * Clears a region of physical memory by overwriting it with zeros. The data is
1006 * flushed from the cache so the memory has been cleared across the system.
1007 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001008static bool ffa_clear_memory_constituents(
J-Alves7db32002021-12-14 14:44:50 +00001009 uint32_t security_state_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +01001010 struct ffa_memory_region_constituent **fragments,
1011 const uint32_t *fragment_constituent_counts, uint32_t fragment_count,
1012 struct mpool *page_pool)
Jose Marinho09b1db82019-08-08 09:16:59 +01001013{
1014 struct mpool local_page_pool;
Andrew Walbranca808b12020-05-15 17:22:28 +01001015 uint32_t i;
Jose Marinho09b1db82019-08-08 09:16:59 +01001016 bool ret = false;
1017
1018 /*
1019 * Create a local pool so any freed memory can't be used by another
1020 * thread. This is to ensure each constituent that is mapped can be
1021 * unmapped again afterwards.
1022 */
Andrew Walbran475c1452020-02-07 13:22:22 +00001023 mpool_init_with_fallback(&local_page_pool, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +01001024
Andrew Walbranca808b12020-05-15 17:22:28 +01001025 /* Iterate over the memory region constituents within each fragment. */
1026 for (i = 0; i < fragment_count; ++i) {
1027 uint32_t j;
Jose Marinho09b1db82019-08-08 09:16:59 +01001028
J-Alves8457f932023-10-11 16:41:45 +01001029 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001030 size_t size = fragments[i][j].page_count * PAGE_SIZE;
1031 paddr_t begin =
1032 pa_from_ipa(ipa_init(fragments[i][j].address));
1033 paddr_t end = pa_add(begin, size);
1034
J-Alves7db32002021-12-14 14:44:50 +00001035 if (!clear_memory(begin, end, &local_page_pool,
1036 security_state_mode)) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001037 /*
1038 * api_clear_memory will defrag on failure, so
1039 * no need to do it here.
1040 */
1041 goto out;
1042 }
Jose Marinho09b1db82019-08-08 09:16:59 +01001043 }
1044 }
1045
Jose Marinho09b1db82019-08-08 09:16:59 +01001046 ret = true;
1047
1048out:
1049 mpool_fini(&local_page_pool);
1050 return ret;
1051}
1052
J-Alves5952d942022-12-22 16:03:00 +00001053static bool is_memory_range_within(ipaddr_t begin, ipaddr_t end,
1054 ipaddr_t in_begin, ipaddr_t in_end)
1055{
1056 return (ipa_addr(begin) >= ipa_addr(in_begin) &&
1057 ipa_addr(begin) < ipa_addr(in_end)) ||
1058 (ipa_addr(end) <= ipa_addr(in_end) &&
1059 ipa_addr(end) > ipa_addr(in_begin));
1060}
1061
1062/**
1063 * Receives a memory range and looks for overlaps with the remainder
1064 * constituents of the memory share/lend/donate operation. Assumes they are
1065 * passed in order to avoid having to loop over all the elements at each call.
1066 * The function only compares the received memory ranges with those that follow
1067 * within the same fragment, and subsequent fragments from the same operation.
1068 */
1069static bool ffa_memory_check_overlap(
1070 struct ffa_memory_region_constituent **fragments,
1071 const uint32_t *fragment_constituent_counts,
1072 const uint32_t fragment_count, const uint32_t current_fragment,
1073 const uint32_t current_constituent)
1074{
1075 uint32_t i = current_fragment;
1076 uint32_t j = current_constituent;
1077 ipaddr_t current_begin = ipa_init(fragments[i][j].address);
1078 const uint32_t current_page_count = fragments[i][j].page_count;
1079 size_t current_size = current_page_count * PAGE_SIZE;
1080 ipaddr_t current_end = ipa_add(current_begin, current_size - 1);
1081
1082 if (current_size == 0 ||
1083 current_size > UINT64_MAX - ipa_addr(current_begin)) {
1084 dlog_verbose("Invalid page count. Addr: %x page_count: %x\n",
1085 current_begin, current_page_count);
1086 return false;
1087 }
1088
1089 for (; i < fragment_count; i++) {
1090 j = (i == current_fragment) ? j + 1 : 0;
1091
1092 for (; j < fragment_constituent_counts[i]; j++) {
1093 ipaddr_t begin = ipa_init(fragments[i][j].address);
1094 const uint32_t page_count = fragments[i][j].page_count;
1095 size_t size = page_count * PAGE_SIZE;
1096 ipaddr_t end = ipa_add(begin, size - 1);
1097
1098 if (size == 0 || size > UINT64_MAX - ipa_addr(begin)) {
1099 dlog_verbose(
1100 "Invalid page count. Addr: %x "
1101 "page_count: %x\n",
1102 begin, page_count);
1103 return false;
1104 }
1105
1106 /*
1107 * Check if current ranges is within begin and end, as
1108 * well as the reverse. This should help optimize the
1109 * loop, and reduce the number of iterations.
1110 */
1111 if (is_memory_range_within(begin, end, current_begin,
1112 current_end) ||
1113 is_memory_range_within(current_begin, current_end,
1114 begin, end)) {
1115 dlog_verbose(
1116 "Overlapping memory ranges: %#x - %#x "
1117 "with %#x - %#x\n",
1118 ipa_addr(begin), ipa_addr(end),
1119 ipa_addr(current_begin),
1120 ipa_addr(current_end));
1121 return true;
1122 }
1123 }
1124 }
1125
1126 return false;
1127}
1128
Jose Marinho09b1db82019-08-08 09:16:59 +01001129/**
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001130 * Validates and prepares memory to be sent from the calling VM to another.
Jose Marinho09b1db82019-08-08 09:16:59 +01001131 *
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001132 * This function requires the calling context to hold the <from> VM lock.
Jose Marinho09b1db82019-08-08 09:16:59 +01001133 *
1134 * Returns:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001135 * In case of error, one of the following values is returned:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001136 * 1) FFA_INVALID_PARAMETERS - The endpoint provided parameters were
Jose Marinho09b1db82019-08-08 09:16:59 +01001137 * erroneous;
Andrew Walbranf07f04d2020-05-01 18:09:00 +01001138 * 2) FFA_NO_MEMORY - Hafnium did not have sufficient memory to complete the
1139 * request.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001140 * 3) FFA_DENIED - The sender doesn't have sufficient access to send the
Andrew Walbrana65a1322020-04-06 19:32:32 +01001141 * memory with the given permissions.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001142 * Success is indicated by FFA_SUCCESS.
Jose Marinho09b1db82019-08-08 09:16:59 +01001143 */
J-Alves66652252022-07-06 09:49:51 +01001144struct ffa_value ffa_send_check_update(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001145 struct vm_locked from_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01001146 struct ffa_memory_region_constituent **fragments,
1147 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
J-Alves8f11cde2022-12-21 16:18:22 +00001148 uint32_t composite_total_page_count, uint32_t share_func,
1149 struct ffa_memory_access *receivers, uint32_t receivers_count,
1150 struct mpool *page_pool, bool clear, uint32_t *orig_from_mode_ret)
Jose Marinho09b1db82019-08-08 09:16:59 +01001151{
Andrew Walbranca808b12020-05-15 17:22:28 +01001152 uint32_t i;
J-Alves8f11cde2022-12-21 16:18:22 +00001153 uint32_t j;
Jose Marinho09b1db82019-08-08 09:16:59 +01001154 uint32_t orig_from_mode;
1155 uint32_t from_mode;
Jose Marinho09b1db82019-08-08 09:16:59 +01001156 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001157 struct ffa_value ret;
J-Alves8f11cde2022-12-21 16:18:22 +00001158 uint32_t constituents_total_page_count = 0;
Jose Marinho09b1db82019-08-08 09:16:59 +01001159
1160 /*
Andrew Walbrana65a1322020-04-06 19:32:32 +01001161 * Make sure constituents are properly aligned to a 64-bit boundary. If
1162 * not we would get alignment faults trying to read (64-bit) values.
Jose Marinho09b1db82019-08-08 09:16:59 +01001163 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001164 for (i = 0; i < fragment_count; ++i) {
1165 if (!is_aligned(fragments[i], 8)) {
1166 dlog_verbose("Constituents not aligned.\n");
1167 return ffa_error(FFA_INVALID_PARAMETERS);
1168 }
J-Alves8f11cde2022-12-21 16:18:22 +00001169 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
1170 constituents_total_page_count +=
1171 fragments[i][j].page_count;
J-Alves5952d942022-12-22 16:03:00 +00001172 if (ffa_memory_check_overlap(
1173 fragments, fragment_constituent_counts,
1174 fragment_count, i, j)) {
1175 return ffa_error(FFA_INVALID_PARAMETERS);
1176 }
J-Alves8f11cde2022-12-21 16:18:22 +00001177 }
1178 }
1179
1180 if (constituents_total_page_count != composite_total_page_count) {
1181 dlog_verbose(
1182 "Composite page count differs from calculated page "
1183 "count from constituents.\n");
1184 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho09b1db82019-08-08 09:16:59 +01001185 }
1186
1187 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001188 * Check if the state transition is lawful for the sender, ensure that
1189 * all constituents of a memory region being shared are at the same
1190 * state.
Jose Marinho09b1db82019-08-08 09:16:59 +01001191 */
J-Alves363f5722022-04-25 17:37:37 +01001192 ret = ffa_send_check_transition(from_locked, share_func, receivers,
1193 receivers_count, &orig_from_mode,
1194 fragments, fragment_constituent_counts,
Andrew Walbranca808b12020-05-15 17:22:28 +01001195 fragment_count, &from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001196 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001197 dlog_verbose("Invalid transition for send.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +01001198 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01001199 }
1200
Andrew Walbran37c574e2020-06-03 11:45:46 +01001201 if (orig_from_mode_ret != NULL) {
1202 *orig_from_mode_ret = orig_from_mode;
1203 }
1204
Jose Marinho09b1db82019-08-08 09:16:59 +01001205 /*
1206 * Create a local pool so any freed memory can't be used by another
1207 * thread. This is to ensure the original mapping can be restored if the
1208 * clear fails.
1209 */
Andrew Walbran475c1452020-02-07 13:22:22 +00001210 mpool_init_with_fallback(&local_page_pool, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +01001211
1212 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001213 * First reserve all required memory for the new page table entries
1214 * without committing, to make sure the entire operation will succeed
1215 * without exhausting the page pool.
Jose Marinho09b1db82019-08-08 09:16:59 +01001216 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001217 if (!ffa_region_group_identity_map(
1218 from_locked, fragments, fragment_constituent_counts,
1219 fragment_count, from_mode, page_pool, false)) {
Jose Marinho09b1db82019-08-08 09:16:59 +01001220 /* TODO: partial defrag of failed range. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001221 ret = ffa_error(FFA_NO_MEMORY);
Jose Marinho09b1db82019-08-08 09:16:59 +01001222 goto out;
1223 }
1224
1225 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001226 * Update the mapping for the sender. This won't allocate because the
1227 * transaction was already prepared above, but may free pages in the
1228 * case that a whole block is being unmapped that was previously
1229 * partially mapped.
Jose Marinho09b1db82019-08-08 09:16:59 +01001230 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001231 CHECK(ffa_region_group_identity_map(
1232 from_locked, fragments, fragment_constituent_counts,
1233 fragment_count, from_mode, &local_page_pool, true));
Jose Marinho09b1db82019-08-08 09:16:59 +01001234
1235 /* Clear the memory so no VM or device can see the previous contents. */
J-Alves7db32002021-12-14 14:44:50 +00001236 if (clear &&
J-Alves26483382023-04-20 12:01:49 +01001237 !ffa_clear_memory_constituents(orig_from_mode, fragments,
1238 fragment_constituent_counts,
1239 fragment_count, page_pool)) {
Jose Marinho09b1db82019-08-08 09:16:59 +01001240 /*
1241 * On failure, roll back by returning memory to the sender. This
1242 * may allocate pages which were previously freed into
1243 * `local_page_pool` by the call above, but will never allocate
1244 * more pages than that so can never fail.
1245 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001246 CHECK(ffa_region_group_identity_map(
Andrew Walbranca808b12020-05-15 17:22:28 +01001247 from_locked, fragments, fragment_constituent_counts,
1248 fragment_count, orig_from_mode, &local_page_pool,
1249 true));
Jose Marinho09b1db82019-08-08 09:16:59 +01001250
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001251 ret = ffa_error(FFA_NO_MEMORY);
Jose Marinho09b1db82019-08-08 09:16:59 +01001252 goto out;
1253 }
1254
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001255 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001256
1257out:
1258 mpool_fini(&local_page_pool);
1259
1260 /*
1261 * Tidy up the page table by reclaiming failed mappings (if there was an
1262 * error) or merging entries into blocks where possible (on success).
1263 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001264 vm_ptable_defrag(from_locked, page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001265
1266 return ret;
1267}
1268
1269/**
1270 * Validates and maps memory shared from one VM to another.
1271 *
1272 * This function requires the calling context to hold the <to> lock.
1273 *
1274 * Returns:
1275 * In case of error, one of the following values is returned:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001276 * 1) FFA_INVALID_PARAMETERS - The endpoint provided parameters were
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001277 * erroneous;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001278 * 2) FFA_NO_MEMORY - Hafnium did not have sufficient memory to complete
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001279 * the request.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001280 * Success is indicated by FFA_SUCCESS.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001281 */
J-Alvesb5084cf2022-07-06 14:20:12 +01001282struct ffa_value ffa_retrieve_check_update(
J-Alves26483382023-04-20 12:01:49 +01001283 struct vm_locked to_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01001284 struct ffa_memory_region_constituent **fragments,
1285 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
J-Alves26483382023-04-20 12:01:49 +01001286 uint32_t sender_orig_mode, uint32_t share_func, bool clear,
Andrew Walbranca808b12020-05-15 17:22:28 +01001287 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001288{
Andrew Walbranca808b12020-05-15 17:22:28 +01001289 uint32_t i;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001290 uint32_t to_mode;
1291 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001292 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001293
1294 /*
Andrew Walbranca808b12020-05-15 17:22:28 +01001295 * Make sure constituents are properly aligned to a 64-bit boundary. If
1296 * not we would get alignment faults trying to read (64-bit) values.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001297 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001298 for (i = 0; i < fragment_count; ++i) {
1299 if (!is_aligned(fragments[i], 8)) {
J-Alvesb5084cf2022-07-06 14:20:12 +01001300 dlog_verbose("Fragment not properly aligned.\n");
Andrew Walbranca808b12020-05-15 17:22:28 +01001301 return ffa_error(FFA_INVALID_PARAMETERS);
1302 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001303 }
1304
1305 /*
1306 * Check if the state transition is lawful for the recipient, and ensure
1307 * that all constituents of the memory region being retrieved are at the
1308 * same state.
1309 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001310 ret = ffa_retrieve_check_transition(
1311 to_locked, share_func, fragments, fragment_constituent_counts,
J-Alves26483382023-04-20 12:01:49 +01001312 fragment_count, sender_orig_mode, &to_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001313 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001314 dlog_verbose("Invalid transition for retrieve.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +01001315 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001316 }
1317
1318 /*
1319 * Create a local pool so any freed memory can't be used by another
1320 * thread. This is to ensure the original mapping can be restored if the
1321 * clear fails.
1322 */
1323 mpool_init_with_fallback(&local_page_pool, page_pool);
1324
1325 /*
1326 * First reserve all required memory for the new page table entries in
1327 * the recipient page tables without committing, to make sure the entire
1328 * operation will succeed without exhausting the page pool.
1329 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001330 if (!ffa_region_group_identity_map(
1331 to_locked, fragments, fragment_constituent_counts,
1332 fragment_count, to_mode, page_pool, false)) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001333 /* TODO: partial defrag of failed range. */
1334 dlog_verbose(
1335 "Insufficient memory to update recipient page "
1336 "table.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001337 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001338 goto out;
1339 }
1340
1341 /* Clear the memory so no VM or device can see the previous contents. */
J-Alves7db32002021-12-14 14:44:50 +00001342 if (clear &&
J-Alves26483382023-04-20 12:01:49 +01001343 !ffa_clear_memory_constituents(sender_orig_mode, fragments,
1344 fragment_constituent_counts,
1345 fragment_count, page_pool)) {
J-Alvesb5084cf2022-07-06 14:20:12 +01001346 dlog_verbose("Couldn't clear constituents.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001347 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001348 goto out;
1349 }
1350
Jose Marinho09b1db82019-08-08 09:16:59 +01001351 /*
1352 * Complete the transfer by mapping the memory into the recipient. This
1353 * won't allocate because the transaction was already prepared above, so
1354 * it doesn't need to use the `local_page_pool`.
1355 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001356 CHECK(ffa_region_group_identity_map(
1357 to_locked, fragments, fragment_constituent_counts,
1358 fragment_count, to_mode, page_pool, true));
Jose Marinho09b1db82019-08-08 09:16:59 +01001359
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001360 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Jose Marinho09b1db82019-08-08 09:16:59 +01001361
1362out:
1363 mpool_fini(&local_page_pool);
1364
1365 /*
Andrew Walbranf07f04d2020-05-01 18:09:00 +01001366 * Tidy up the page table by reclaiming failed mappings (if there was an
1367 * error) or merging entries into blocks where possible (on success).
Jose Marinho09b1db82019-08-08 09:16:59 +01001368 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001369 vm_ptable_defrag(to_locked, page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001370
1371 return ret;
1372}
1373
Andrew Walbran996d1d12020-05-27 14:08:43 +01001374static struct ffa_value ffa_relinquish_check_update(
J-Alves26483382023-04-20 12:01:49 +01001375 struct vm_locked from_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01001376 struct ffa_memory_region_constituent **fragments,
1377 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
1378 struct mpool *page_pool, bool clear)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001379{
1380 uint32_t orig_from_mode;
1381 uint32_t from_mode;
1382 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001383 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001384
Andrew Walbranca808b12020-05-15 17:22:28 +01001385 ret = ffa_relinquish_check_transition(
1386 from_locked, &orig_from_mode, fragments,
1387 fragment_constituent_counts, fragment_count, &from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001388 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001389 dlog_verbose("Invalid transition for relinquish.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +01001390 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001391 }
1392
1393 /*
1394 * Create a local pool so any freed memory can't be used by another
1395 * thread. This is to ensure the original mapping can be restored if the
1396 * clear fails.
1397 */
1398 mpool_init_with_fallback(&local_page_pool, page_pool);
1399
1400 /*
1401 * First reserve all required memory for the new page table entries
1402 * without committing, to make sure the entire operation will succeed
1403 * without exhausting the page pool.
1404 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001405 if (!ffa_region_group_identity_map(
1406 from_locked, fragments, fragment_constituent_counts,
1407 fragment_count, from_mode, page_pool, false)) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001408 /* TODO: partial defrag of failed range. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001409 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001410 goto out;
1411 }
1412
1413 /*
1414 * Update the mapping for the sender. This won't allocate because the
1415 * transaction was already prepared above, but may free pages in the
1416 * case that a whole block is being unmapped that was previously
1417 * partially mapped.
1418 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001419 CHECK(ffa_region_group_identity_map(
1420 from_locked, fragments, fragment_constituent_counts,
1421 fragment_count, from_mode, &local_page_pool, true));
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001422
1423 /* Clear the memory so no VM or device can see the previous contents. */
J-Alves7db32002021-12-14 14:44:50 +00001424 if (clear &&
J-Alves26483382023-04-20 12:01:49 +01001425 !ffa_clear_memory_constituents(orig_from_mode, fragments,
1426 fragment_constituent_counts,
1427 fragment_count, page_pool)) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001428 /*
1429 * On failure, roll back by returning memory to the sender. This
1430 * may allocate pages which were previously freed into
1431 * `local_page_pool` by the call above, but will never allocate
1432 * more pages than that so can never fail.
1433 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001434 CHECK(ffa_region_group_identity_map(
Andrew Walbranca808b12020-05-15 17:22:28 +01001435 from_locked, fragments, fragment_constituent_counts,
1436 fragment_count, orig_from_mode, &local_page_pool,
1437 true));
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001438
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001439 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001440 goto out;
1441 }
1442
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001443 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001444
1445out:
1446 mpool_fini(&local_page_pool);
1447
1448 /*
1449 * Tidy up the page table by reclaiming failed mappings (if there was an
1450 * error) or merging entries into blocks where possible (on success).
1451 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001452 vm_ptable_defrag(from_locked, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +01001453
1454 return ret;
1455}
1456
1457/**
Andrew Walbranca808b12020-05-15 17:22:28 +01001458 * Complete a memory sending operation by checking that it is valid, updating
1459 * the sender page table, and then either marking the share state as having
1460 * completed sending (on success) or freeing it (on failure).
1461 *
1462 * Returns FFA_SUCCESS with the handle encoded, or the relevant FFA_ERROR.
1463 */
J-Alvesfdd29272022-07-19 13:16:31 +01001464struct ffa_value ffa_memory_send_complete(
Andrew Walbranca808b12020-05-15 17:22:28 +01001465 struct vm_locked from_locked, struct share_states_locked share_states,
Andrew Walbran37c574e2020-06-03 11:45:46 +01001466 struct ffa_memory_share_state *share_state, struct mpool *page_pool,
1467 uint32_t *orig_from_mode_ret)
Andrew Walbranca808b12020-05-15 17:22:28 +01001468{
1469 struct ffa_memory_region *memory_region = share_state->memory_region;
J-Alves8f11cde2022-12-21 16:18:22 +00001470 struct ffa_composite_memory_region *composite;
Daniel Boulby41ef8ba2023-10-13 17:01:22 +01001471 struct ffa_memory_access *receiver;
Andrew Walbranca808b12020-05-15 17:22:28 +01001472 struct ffa_value ret;
1473
1474 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +00001475 assert(share_states.share_states != NULL);
J-Alves8f11cde2022-12-21 16:18:22 +00001476 assert(memory_region != NULL);
1477 composite = ffa_memory_region_get_composite(memory_region, 0);
1478 assert(composite != NULL);
Daniel Boulby41ef8ba2023-10-13 17:01:22 +01001479 receiver = ffa_memory_region_get_receiver(memory_region, 0);
1480 assert(receiver != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +01001481
1482 /* Check that state is valid in sender page table and update. */
1483 ret = ffa_send_check_update(
1484 from_locked, share_state->fragments,
1485 share_state->fragment_constituent_counts,
J-Alves8f11cde2022-12-21 16:18:22 +00001486 share_state->fragment_count, composite->page_count,
Daniel Boulby41ef8ba2023-10-13 17:01:22 +01001487 share_state->share_func, receiver,
J-Alves8f11cde2022-12-21 16:18:22 +00001488 memory_region->receiver_count, page_pool,
1489 memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR,
Andrew Walbran37c574e2020-06-03 11:45:46 +01001490 orig_from_mode_ret);
Andrew Walbranca808b12020-05-15 17:22:28 +01001491 if (ret.func != FFA_SUCCESS_32) {
1492 /*
1493 * Free share state, it failed to send so it can't be retrieved.
1494 */
Karl Meakin4cec5e82023-06-30 16:30:22 +01001495 dlog_verbose("%s: failed to send check update: %s(%s)\n",
1496 __func__, ffa_func_name(ret.func),
1497 ffa_error_name(ffa_error_code(ret)));
Andrew Walbranca808b12020-05-15 17:22:28 +01001498 share_state_free(share_states, share_state, page_pool);
1499 return ret;
1500 }
1501
1502 share_state->sending_complete = true;
Karl Meakin4cec5e82023-06-30 16:30:22 +01001503 dlog_verbose("%s: marked sending complete.\n", __func__);
Andrew Walbranca808b12020-05-15 17:22:28 +01001504
J-Alvesee68c542020-10-29 17:48:20 +00001505 return ffa_mem_success(share_state->memory_region->handle);
Andrew Walbranca808b12020-05-15 17:22:28 +01001506}
1507
1508/**
Federico Recanatia98603a2021-12-20 18:04:03 +01001509 * Check that the memory attributes match Hafnium expectations:
1510 * Normal Memory, Inner shareable, Write-Back Read-Allocate
1511 * Write-Allocate Cacheable.
1512 */
1513static struct ffa_value ffa_memory_attributes_validate(
J-Alves7a99d0d2023-02-08 13:49:48 +00001514 ffa_memory_attributes_t attributes)
Federico Recanatia98603a2021-12-20 18:04:03 +01001515{
1516 enum ffa_memory_type memory_type;
1517 enum ffa_memory_cacheability cacheability;
1518 enum ffa_memory_shareability shareability;
1519
1520 memory_type = ffa_get_memory_type_attr(attributes);
1521 if (memory_type != FFA_MEMORY_NORMAL_MEM) {
1522 dlog_verbose("Invalid memory type %#x, expected %#x.\n",
1523 memory_type, FFA_MEMORY_NORMAL_MEM);
Federico Recanati3d953f32022-02-17 09:31:29 +01001524 return ffa_error(FFA_DENIED);
Federico Recanatia98603a2021-12-20 18:04:03 +01001525 }
1526
1527 cacheability = ffa_get_memory_cacheability_attr(attributes);
1528 if (cacheability != FFA_MEMORY_CACHE_WRITE_BACK) {
1529 dlog_verbose("Invalid cacheability %#x, expected %#x.\n",
1530 cacheability, FFA_MEMORY_CACHE_WRITE_BACK);
Federico Recanati3d953f32022-02-17 09:31:29 +01001531 return ffa_error(FFA_DENIED);
Federico Recanatia98603a2021-12-20 18:04:03 +01001532 }
1533
1534 shareability = ffa_get_memory_shareability_attr(attributes);
1535 if (shareability != FFA_MEMORY_INNER_SHAREABLE) {
1536 dlog_verbose("Invalid shareability %#x, expected #%x.\n",
1537 shareability, FFA_MEMORY_INNER_SHAREABLE);
Federico Recanati3d953f32022-02-17 09:31:29 +01001538 return ffa_error(FFA_DENIED);
Federico Recanatia98603a2021-12-20 18:04:03 +01001539 }
1540
1541 return (struct ffa_value){.func = FFA_SUCCESS_32};
1542}
1543
1544/**
Andrew Walbrana65a1322020-04-06 19:32:32 +01001545 * Check that the given `memory_region` represents a valid memory send request
1546 * of the given `share_func` type, return the clear flag and permissions via the
1547 * respective output parameters, and update the permissions if necessary.
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001548 *
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001549 * Returns FFA_SUCCESS if the request was valid, or the relevant FFA_ERROR if
Andrew Walbrana65a1322020-04-06 19:32:32 +01001550 * not.
1551 */
J-Alves66652252022-07-06 09:49:51 +01001552struct ffa_value ffa_memory_send_validate(
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001553 struct vm_locked from_locked, struct ffa_memory_region *memory_region,
1554 uint32_t memory_share_length, uint32_t fragment_length,
J-Alves363f5722022-04-25 17:37:37 +01001555 uint32_t share_func)
Andrew Walbrana65a1322020-04-06 19:32:32 +01001556{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001557 struct ffa_composite_memory_region *composite;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001558 struct ffa_memory_access *receiver =
1559 ffa_memory_region_get_receiver(memory_region, 0);
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001560 uint64_t receivers_end;
1561 uint64_t min_length;
Federico Recanati872cd692022-01-05 13:10:10 +01001562 uint32_t composite_memory_region_offset;
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001563 uint32_t constituents_start;
Andrew Walbran130a8ae2020-05-15 16:27:15 +01001564 uint32_t constituents_length;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001565 enum ffa_data_access data_access;
1566 enum ffa_instruction_access instruction_access;
Olivier Deprez4342a3c2022-02-28 09:37:25 +01001567 enum ffa_memory_security security_state;
Federico Recanatia98603a2021-12-20 18:04:03 +01001568 struct ffa_value ret;
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001569 const size_t minimum_first_fragment_length =
Daniel Boulby41ef8ba2023-10-13 17:01:22 +01001570 memory_region->receivers_offset +
1571 memory_region->memory_access_desc_size +
1572 sizeof(struct ffa_composite_memory_region);
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001573
1574 if (fragment_length < minimum_first_fragment_length) {
1575 dlog_verbose("Fragment length %u too short (min %u).\n",
1576 (size_t)fragment_length,
1577 minimum_first_fragment_length);
1578 return ffa_error(FFA_INVALID_PARAMETERS);
1579 }
1580
Demi Marie Obenour73a1e942023-02-04 14:09:18 -05001581 static_assert(sizeof(struct ffa_memory_region_constituent) == 16,
1582 "struct ffa_memory_region_constituent must be 16 bytes");
1583 if (!is_aligned(fragment_length,
1584 sizeof(struct ffa_memory_region_constituent)) ||
1585 !is_aligned(memory_share_length,
1586 sizeof(struct ffa_memory_region_constituent))) {
1587 dlog_verbose(
1588 "Fragment length %u or total length %u"
1589 " is not 16-byte aligned.\n",
1590 fragment_length, memory_share_length);
1591 return ffa_error(FFA_INVALID_PARAMETERS);
1592 }
1593
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001594 if (fragment_length > memory_share_length) {
1595 dlog_verbose(
1596 "Fragment length %u greater than total length %u.\n",
1597 (size_t)fragment_length, (size_t)memory_share_length);
1598 return ffa_error(FFA_INVALID_PARAMETERS);
1599 }
Andrew Walbrana65a1322020-04-06 19:32:32 +01001600
J-Alves95df0ef2022-12-07 10:09:48 +00001601 /* The sender must match the caller. */
1602 if ((!vm_id_is_current_world(from_locked.vm->id) &&
1603 vm_id_is_current_world(memory_region->sender)) ||
1604 (vm_id_is_current_world(from_locked.vm->id) &&
1605 memory_region->sender != from_locked.vm->id)) {
1606 dlog_verbose("Invalid memory sender ID.\n");
1607 return ffa_error(FFA_DENIED);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001608 }
1609
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001610 if (memory_region->receiver_count <= 0) {
1611 dlog_verbose("No receivers!\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001612 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001613 }
1614
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001615 /*
1616 * Ensure that the composite header is within the memory bounds and
1617 * doesn't overlap the first part of the message. Cast to uint64_t
1618 * to prevent overflow.
1619 */
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001620 receivers_end = ((uint64_t)memory_region->memory_access_desc_size *
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001621 (uint64_t)memory_region->receiver_count) +
Daniel Boulby41ef8ba2023-10-13 17:01:22 +01001622 memory_region->receivers_offset;
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001623 min_length = receivers_end +
1624 sizeof(struct ffa_composite_memory_region) +
1625 sizeof(struct ffa_memory_region_constituent);
1626 if (min_length > memory_share_length) {
1627 dlog_verbose("Share too short: got %u but minimum is %u.\n",
1628 (size_t)memory_share_length, (size_t)min_length);
1629 return ffa_error(FFA_INVALID_PARAMETERS);
1630 }
1631
1632 composite_memory_region_offset =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001633 receiver->composite_memory_region_offset;
Andrew Walbrana65a1322020-04-06 19:32:32 +01001634
1635 /*
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001636 * Check that the composite memory region descriptor is after the access
1637 * descriptors, is at least 16-byte aligned, and fits in the first
1638 * fragment.
Andrew Walbrana65a1322020-04-06 19:32:32 +01001639 */
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001640 if ((composite_memory_region_offset < receivers_end) ||
1641 (composite_memory_region_offset % 16 != 0) ||
1642 (composite_memory_region_offset >
1643 fragment_length - sizeof(struct ffa_composite_memory_region))) {
1644 dlog_verbose(
1645 "Invalid composite memory region descriptor offset "
1646 "%u.\n",
1647 (size_t)composite_memory_region_offset);
1648 return ffa_error(FFA_INVALID_PARAMETERS);
1649 }
1650
1651 /*
1652 * Compute the start of the constituent regions. Already checked
1653 * to be not more than fragment_length and thus not more than
1654 * memory_share_length.
1655 */
1656 constituents_start = composite_memory_region_offset +
1657 sizeof(struct ffa_composite_memory_region);
1658 constituents_length = memory_share_length - constituents_start;
1659
1660 /*
1661 * Check that the number of constituents is consistent with the length
1662 * of the constituent region.
1663 */
1664 composite = ffa_memory_region_get_composite(memory_region, 0);
1665 if ((constituents_length %
1666 sizeof(struct ffa_memory_region_constituent) !=
1667 0) ||
1668 ((constituents_length /
1669 sizeof(struct ffa_memory_region_constituent)) !=
1670 composite->constituent_count)) {
1671 dlog_verbose("Invalid length %u or composite offset %u.\n",
1672 (size_t)memory_share_length,
1673 (size_t)composite_memory_region_offset);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001674 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001675 }
Andrew Walbranca808b12020-05-15 17:22:28 +01001676 if (fragment_length < memory_share_length &&
1677 fragment_length < HF_MAILBOX_SIZE) {
1678 dlog_warning(
1679 "Initial fragment length %d smaller than mailbox "
1680 "size.\n",
1681 fragment_length);
1682 }
Andrew Walbrana65a1322020-04-06 19:32:32 +01001683
Andrew Walbrana65a1322020-04-06 19:32:32 +01001684 /*
1685 * Clear is not allowed for memory sharing, as the sender still has
1686 * access to the memory.
1687 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001688 if ((memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR) &&
1689 share_func == FFA_MEM_SHARE_32) {
Andrew Walbrana65a1322020-04-06 19:32:32 +01001690 dlog_verbose("Memory can't be cleared while being shared.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001691 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001692 }
1693
1694 /* No other flags are allowed/supported here. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001695 if (memory_region->flags & ~FFA_MEMORY_REGION_FLAG_CLEAR) {
Andrew Walbrana65a1322020-04-06 19:32:32 +01001696 dlog_verbose("Invalid flags %#x.\n", memory_region->flags);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001697 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001698 }
1699
J-Alves363f5722022-04-25 17:37:37 +01001700 /* Check that the permissions are valid, for each specified receiver. */
1701 for (uint32_t i = 0U; i < memory_region->receiver_count; i++) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001702 struct ffa_memory_region_attributes receiver_permissions;
1703
1704 receiver = ffa_memory_region_get_receiver(memory_region, i);
1705 assert(receiver != NULL);
1706 receiver_permissions = receiver->receiver_permissions;
J-Alves363f5722022-04-25 17:37:37 +01001707 ffa_memory_access_permissions_t permissions =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001708 receiver_permissions.permissions;
1709 ffa_id_t receiver_id = receiver_permissions.receiver;
J-Alves363f5722022-04-25 17:37:37 +01001710
1711 if (memory_region->sender == receiver_id) {
1712 dlog_verbose("Can't share memory with itself.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001713 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001714 }
Federico Recanati85090c42021-12-15 13:17:54 +01001715
J-Alves363f5722022-04-25 17:37:37 +01001716 for (uint32_t j = i + 1; j < memory_region->receiver_count;
1717 j++) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001718 struct ffa_memory_access *other_receiver =
1719 ffa_memory_region_get_receiver(memory_region,
1720 j);
1721 assert(other_receiver != NULL);
1722
J-Alves363f5722022-04-25 17:37:37 +01001723 if (receiver_id ==
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001724 other_receiver->receiver_permissions.receiver) {
J-Alves363f5722022-04-25 17:37:37 +01001725 dlog_verbose(
1726 "Repeated receiver(%x) in memory send "
1727 "operation.\n",
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001728 other_receiver->receiver_permissions
1729 .receiver);
J-Alves363f5722022-04-25 17:37:37 +01001730 return ffa_error(FFA_INVALID_PARAMETERS);
1731 }
1732 }
1733
1734 if (composite_memory_region_offset !=
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001735 receiver->composite_memory_region_offset) {
J-Alves363f5722022-04-25 17:37:37 +01001736 dlog_verbose(
1737 "All ffa_memory_access should point to the "
1738 "same composite memory region offset.\n");
1739 return ffa_error(FFA_INVALID_PARAMETERS);
1740 }
1741
1742 data_access = ffa_get_data_access_attr(permissions);
1743 instruction_access =
1744 ffa_get_instruction_access_attr(permissions);
1745 if (data_access == FFA_DATA_ACCESS_RESERVED ||
1746 instruction_access == FFA_INSTRUCTION_ACCESS_RESERVED) {
1747 dlog_verbose(
1748 "Reserved value for receiver permissions "
1749 "%#x.\n",
1750 permissions);
1751 return ffa_error(FFA_INVALID_PARAMETERS);
1752 }
1753 if (instruction_access !=
1754 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
1755 dlog_verbose(
1756 "Invalid instruction access permissions %#x "
1757 "for sending memory.\n",
1758 permissions);
1759 return ffa_error(FFA_INVALID_PARAMETERS);
1760 }
1761 if (share_func == FFA_MEM_SHARE_32) {
1762 if (data_access == FFA_DATA_ACCESS_NOT_SPECIFIED) {
1763 dlog_verbose(
1764 "Invalid data access permissions %#x "
1765 "for sharing memory.\n",
1766 permissions);
1767 return ffa_error(FFA_INVALID_PARAMETERS);
1768 }
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001769 /*
1770 * According to section 10.10.3 of the FF-A v1.1 EAC0
1771 * spec, NX is required for share operations (but must
1772 * not be specified by the sender) so set it in the
1773 * copy that we store, ready to be returned to the
1774 * retriever.
1775 */
1776 if (vm_id_is_current_world(receiver_id)) {
1777 ffa_set_instruction_access_attr(
1778 &permissions,
1779 FFA_INSTRUCTION_ACCESS_NX);
1780 receiver_permissions.permissions = permissions;
1781 }
J-Alves363f5722022-04-25 17:37:37 +01001782 }
1783 if (share_func == FFA_MEM_LEND_32 &&
1784 data_access == FFA_DATA_ACCESS_NOT_SPECIFIED) {
1785 dlog_verbose(
1786 "Invalid data access permissions %#x for "
1787 "lending memory.\n",
1788 permissions);
1789 return ffa_error(FFA_INVALID_PARAMETERS);
1790 }
1791
1792 if (share_func == FFA_MEM_DONATE_32 &&
1793 data_access != FFA_DATA_ACCESS_NOT_SPECIFIED) {
1794 dlog_verbose(
1795 "Invalid data access permissions %#x for "
1796 "donating memory.\n",
1797 permissions);
1798 return ffa_error(FFA_INVALID_PARAMETERS);
1799 }
Andrew Walbrana65a1322020-04-06 19:32:32 +01001800 }
1801
Olivier Deprez4342a3c2022-02-28 09:37:25 +01001802 /* Memory region attributes NS-Bit MBZ for FFA_MEM_SHARE/LEND/DONATE. */
1803 security_state =
1804 ffa_get_memory_security_attr(memory_region->attributes);
1805 if (security_state != FFA_MEMORY_SECURITY_UNSPECIFIED) {
1806 dlog_verbose(
1807 "Invalid security state for memory share operation.\n");
1808 return ffa_error(FFA_INVALID_PARAMETERS);
1809 }
1810
Federico Recanatid937f5e2021-12-20 17:38:23 +01001811 /*
J-Alves807794e2022-06-16 13:42:47 +01001812 * If a memory donate or lend with single borrower, the memory type
1813 * shall not be specified by the sender.
Federico Recanatid937f5e2021-12-20 17:38:23 +01001814 */
J-Alves807794e2022-06-16 13:42:47 +01001815 if (share_func == FFA_MEM_DONATE_32 ||
1816 (share_func == FFA_MEM_LEND_32 &&
1817 memory_region->receiver_count == 1)) {
1818 if (ffa_get_memory_type_attr(memory_region->attributes) !=
1819 FFA_MEMORY_NOT_SPECIFIED_MEM) {
1820 dlog_verbose(
1821 "Memory type shall not be specified by "
1822 "sender.\n");
1823 return ffa_error(FFA_INVALID_PARAMETERS);
1824 }
1825 } else {
1826 /*
1827 * Check that sender's memory attributes match Hafnium
1828 * expectations: Normal Memory, Inner shareable, Write-Back
1829 * Read-Allocate Write-Allocate Cacheable.
1830 */
1831 ret = ffa_memory_attributes_validate(memory_region->attributes);
1832 if (ret.func != FFA_SUCCESS_32) {
1833 return ret;
1834 }
Federico Recanatid937f5e2021-12-20 17:38:23 +01001835 }
1836
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001837 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbrana65a1322020-04-06 19:32:32 +01001838}
1839
1840/**
Andrew Walbranca808b12020-05-15 17:22:28 +01001841 * Gets the share state for continuing an operation to donate, lend or share
1842 * memory, and checks that it is a valid request.
1843 *
1844 * Returns FFA_SUCCESS if the request was valid, or the relevant FFA_ERROR if
1845 * not.
1846 */
J-Alvesfdd29272022-07-19 13:16:31 +01001847struct ffa_value ffa_memory_send_continue_validate(
Andrew Walbranca808b12020-05-15 17:22:28 +01001848 struct share_states_locked share_states, ffa_memory_handle_t handle,
J-Alves19e20cf2023-08-02 12:48:55 +01001849 struct ffa_memory_share_state **share_state_ret, ffa_id_t from_vm_id,
Andrew Walbranca808b12020-05-15 17:22:28 +01001850 struct mpool *page_pool)
1851{
1852 struct ffa_memory_share_state *share_state;
1853 struct ffa_memory_region *memory_region;
1854
Daniel Boulbya2f8c662021-11-26 17:52:53 +00001855 assert(share_state_ret != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +01001856
1857 /*
1858 * Look up the share state by handle and make sure that the VM ID
1859 * matches.
1860 */
Karl Meakin4a2854a2023-06-30 16:26:52 +01001861 share_state = get_share_state(share_states, handle);
J-Alvesb56aac82023-11-10 09:44:43 +00001862 if (share_state == NULL) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001863 dlog_verbose(
1864 "Invalid handle %#x for memory send continuation.\n",
1865 handle);
1866 return ffa_error(FFA_INVALID_PARAMETERS);
1867 }
1868 memory_region = share_state->memory_region;
1869
J-Alvesfdd29272022-07-19 13:16:31 +01001870 if (vm_id_is_current_world(from_vm_id) &&
1871 memory_region->sender != from_vm_id) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001872 dlog_verbose("Invalid sender %d.\n", memory_region->sender);
1873 return ffa_error(FFA_INVALID_PARAMETERS);
1874 }
1875
1876 if (share_state->sending_complete) {
1877 dlog_verbose(
1878 "Sending of memory handle %#x is already complete.\n",
1879 handle);
1880 return ffa_error(FFA_INVALID_PARAMETERS);
1881 }
1882
1883 if (share_state->fragment_count == MAX_FRAGMENTS) {
1884 /*
1885 * Log a warning as this is a sign that MAX_FRAGMENTS should
1886 * probably be increased.
1887 */
1888 dlog_warning(
1889 "Too many fragments for memory share with handle %#x; "
1890 "only %d supported.\n",
1891 handle, MAX_FRAGMENTS);
1892 /* Free share state, as it's not possible to complete it. */
1893 share_state_free(share_states, share_state, page_pool);
1894 return ffa_error(FFA_NO_MEMORY);
1895 }
1896
1897 *share_state_ret = share_state;
1898
1899 return (struct ffa_value){.func = FFA_SUCCESS_32};
1900}
1901
1902/**
J-Alves95df0ef2022-12-07 10:09:48 +00001903 * Checks if there is at least one receiver from the other world.
1904 */
J-Alvesfdd29272022-07-19 13:16:31 +01001905bool memory_region_receivers_from_other_world(
J-Alves95df0ef2022-12-07 10:09:48 +00001906 struct ffa_memory_region *memory_region)
1907{
1908 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001909 struct ffa_memory_access *receiver =
1910 ffa_memory_region_get_receiver(memory_region, i);
1911 assert(receiver != NULL);
1912 ffa_id_t receiver_id = receiver->receiver_permissions.receiver;
1913
1914 if (!vm_id_is_current_world(receiver_id)) {
J-Alves95df0ef2022-12-07 10:09:48 +00001915 return true;
1916 }
1917 }
1918 return false;
1919}
1920
1921/**
J-Alves9da280b2022-12-21 14:55:39 +00001922 * Validates a call to donate, lend or share memory in which Hafnium is the
1923 * designated allocator of the memory handle. In practice, this also means
1924 * Hafnium is responsible for managing the state structures for the transaction.
1925 * If Hafnium is the SPMC, it should allocate the memory handle when either the
1926 * sender is an SP or there is at least one borrower that is an SP.
1927 * If Hafnium is the hypervisor, it should allocate the memory handle when
1928 * operation involves only NWd VMs.
1929 *
1930 * If validation goes well, Hafnium updates the stage-2 page tables of the
1931 * sender. Validation consists of checking if the message length and number of
1932 * memory region constituents match, and if the transition is valid for the
1933 * type of memory sending operation.
Andrew Walbran475c1452020-02-07 13:22:22 +00001934 *
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001935 * Assumes that the caller has already found and locked the sender VM and copied
1936 * the memory region descriptor from the sender's TX buffer to a freshly
1937 * allocated page from Hafnium's internal pool. The caller must have also
1938 * validated that the receiver VM ID is valid.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001939 *
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001940 * This function takes ownership of the `memory_region` passed in and will free
1941 * it when necessary; it must not be freed by the caller.
Jose Marinho09b1db82019-08-08 09:16:59 +01001942 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001943struct ffa_value ffa_memory_send(struct vm_locked from_locked,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001944 struct ffa_memory_region *memory_region,
Andrew Walbran130a8ae2020-05-15 16:27:15 +01001945 uint32_t memory_share_length,
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001946 uint32_t fragment_length, uint32_t share_func,
1947 struct mpool *page_pool)
Jose Marinho09b1db82019-08-08 09:16:59 +01001948{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001949 struct ffa_value ret;
Andrew Walbranca808b12020-05-15 17:22:28 +01001950 struct share_states_locked share_states;
1951 struct ffa_memory_share_state *share_state;
Jose Marinho09b1db82019-08-08 09:16:59 +01001952
1953 /*
Andrew Walbrana65a1322020-04-06 19:32:32 +01001954 * If there is an error validating the `memory_region` then we need to
1955 * free it because we own it but we won't be storing it in a share state
1956 * after all.
Jose Marinho09b1db82019-08-08 09:16:59 +01001957 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001958 ret = ffa_memory_send_validate(from_locked, memory_region,
1959 memory_share_length, fragment_length,
J-Alves363f5722022-04-25 17:37:37 +01001960 share_func);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001961 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001962 mpool_free(page_pool, memory_region);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001963 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01001964 }
1965
Andrew Walbrana65a1322020-04-06 19:32:32 +01001966 /* Set flag for share function, ready to be retrieved later. */
1967 switch (share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001968 case FFA_MEM_SHARE_32:
Andrew Walbrana65a1322020-04-06 19:32:32 +01001969 memory_region->flags |=
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001970 FFA_MEMORY_REGION_TRANSACTION_TYPE_SHARE;
Andrew Walbrana65a1322020-04-06 19:32:32 +01001971 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001972 case FFA_MEM_LEND_32:
1973 memory_region->flags |= FFA_MEMORY_REGION_TRANSACTION_TYPE_LEND;
Andrew Walbrana65a1322020-04-06 19:32:32 +01001974 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001975 case FFA_MEM_DONATE_32:
Andrew Walbrana65a1322020-04-06 19:32:32 +01001976 memory_region->flags |=
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001977 FFA_MEMORY_REGION_TRANSACTION_TYPE_DONATE;
Andrew Walbrana65a1322020-04-06 19:32:32 +01001978 break;
Jose Marinho09b1db82019-08-08 09:16:59 +01001979 }
1980
Andrew Walbranca808b12020-05-15 17:22:28 +01001981 share_states = share_states_lock();
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001982 /*
1983 * Allocate a share state before updating the page table. Otherwise if
1984 * updating the page table succeeded but allocating the share state
1985 * failed then it would leave the memory in a state where nobody could
1986 * get it back.
1987 */
Karl Meakin52cdfe72023-06-30 14:49:10 +01001988 share_state = allocate_share_state(share_states, share_func,
1989 memory_region, fragment_length,
1990 FFA_MEMORY_HANDLE_INVALID);
J-Alvesb56aac82023-11-10 09:44:43 +00001991 if (share_state == NULL) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001992 dlog_verbose("Failed to allocate share state.\n");
1993 mpool_free(page_pool, memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +01001994 ret = ffa_error(FFA_NO_MEMORY);
1995 goto out;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001996 }
1997
Andrew Walbranca808b12020-05-15 17:22:28 +01001998 if (fragment_length == memory_share_length) {
1999 /* No more fragments to come, everything fit in one message. */
J-Alves2a0d2882020-10-29 14:49:50 +00002000 ret = ffa_memory_send_complete(
2001 from_locked, share_states, share_state, page_pool,
2002 &(share_state->sender_orig_mode));
Andrew Walbranca808b12020-05-15 17:22:28 +01002003 } else {
J-Alvesfdd29272022-07-19 13:16:31 +01002004 /*
2005 * Use sender ID from 'memory_region' assuming
2006 * that at this point it has been validated:
2007 * - MBZ at virtual FF-A instance.
2008 */
J-Alves19e20cf2023-08-02 12:48:55 +01002009 ffa_id_t sender_to_ret =
J-Alvesfdd29272022-07-19 13:16:31 +01002010 (from_locked.vm->id == HF_OTHER_WORLD_ID)
2011 ? memory_region->sender
2012 : 0;
Andrew Walbranca808b12020-05-15 17:22:28 +01002013 ret = (struct ffa_value){
2014 .func = FFA_MEM_FRAG_RX_32,
J-Alvesee68c542020-10-29 17:48:20 +00002015 .arg1 = (uint32_t)memory_region->handle,
2016 .arg2 = (uint32_t)(memory_region->handle >> 32),
J-Alvesfdd29272022-07-19 13:16:31 +01002017 .arg3 = fragment_length,
2018 .arg4 = (uint32_t)(sender_to_ret & 0xffff) << 16};
Andrew Walbranca808b12020-05-15 17:22:28 +01002019 }
2020
2021out:
2022 share_states_unlock(&share_states);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002023 dump_share_states();
Andrew Walbranca808b12020-05-15 17:22:28 +01002024 return ret;
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002025}
2026
2027/**
J-Alves8505a8a2022-06-15 18:10:18 +01002028 * Continues an operation to donate, lend or share memory to a VM from current
2029 * world. If this is the last fragment then checks that the transition is valid
2030 * for the type of memory sending operation and updates the stage-2 page tables
2031 * of the sender.
Andrew Walbranca808b12020-05-15 17:22:28 +01002032 *
2033 * Assumes that the caller has already found and locked the sender VM and copied
2034 * the memory region descriptor from the sender's TX buffer to a freshly
2035 * allocated page from Hafnium's internal pool.
2036 *
2037 * This function takes ownership of the `fragment` passed in; it must not be
2038 * freed by the caller.
2039 */
2040struct ffa_value ffa_memory_send_continue(struct vm_locked from_locked,
2041 void *fragment,
2042 uint32_t fragment_length,
2043 ffa_memory_handle_t handle,
2044 struct mpool *page_pool)
2045{
2046 struct share_states_locked share_states = share_states_lock();
2047 struct ffa_memory_share_state *share_state;
2048 struct ffa_value ret;
2049 struct ffa_memory_region *memory_region;
2050
Demi Marie Obenour73a1e942023-02-04 14:09:18 -05002051 CHECK(is_aligned(fragment,
2052 alignof(struct ffa_memory_region_constituent)));
2053 if (fragment_length % sizeof(struct ffa_memory_region_constituent) !=
2054 0) {
2055 dlog_verbose("Fragment length %u misaligned.\n",
2056 fragment_length);
2057 ret = ffa_error(FFA_INVALID_PARAMETERS);
2058 goto out_free_fragment;
2059 }
2060
Andrew Walbranca808b12020-05-15 17:22:28 +01002061 ret = ffa_memory_send_continue_validate(share_states, handle,
2062 &share_state,
2063 from_locked.vm->id, page_pool);
2064 if (ret.func != FFA_SUCCESS_32) {
2065 goto out_free_fragment;
2066 }
2067 memory_region = share_state->memory_region;
2068
J-Alves95df0ef2022-12-07 10:09:48 +00002069 if (memory_region_receivers_from_other_world(memory_region)) {
Andrew Walbranca808b12020-05-15 17:22:28 +01002070 dlog_error(
2071 "Got hypervisor-allocated handle for memory send to "
J-Alves8505a8a2022-06-15 18:10:18 +01002072 "other world. This should never happen, and indicates "
2073 "a bug in "
Andrew Walbranca808b12020-05-15 17:22:28 +01002074 "EL3 code.\n");
2075 ret = ffa_error(FFA_INVALID_PARAMETERS);
2076 goto out_free_fragment;
2077 }
2078
2079 /* Add this fragment. */
2080 share_state->fragments[share_state->fragment_count] = fragment;
2081 share_state->fragment_constituent_counts[share_state->fragment_count] =
2082 fragment_length / sizeof(struct ffa_memory_region_constituent);
2083 share_state->fragment_count++;
2084
2085 /* Check whether the memory send operation is now ready to complete. */
2086 if (share_state_sending_complete(share_states, share_state)) {
J-Alves2a0d2882020-10-29 14:49:50 +00002087 ret = ffa_memory_send_complete(
2088 from_locked, share_states, share_state, page_pool,
2089 &(share_state->sender_orig_mode));
Andrew Walbranca808b12020-05-15 17:22:28 +01002090 } else {
2091 ret = (struct ffa_value){
2092 .func = FFA_MEM_FRAG_RX_32,
2093 .arg1 = (uint32_t)handle,
2094 .arg2 = (uint32_t)(handle >> 32),
2095 .arg3 = share_state_next_fragment_offset(share_states,
2096 share_state)};
2097 }
2098 goto out;
2099
2100out_free_fragment:
2101 mpool_free(page_pool, fragment);
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002102
2103out:
Andrew Walbranca808b12020-05-15 17:22:28 +01002104 share_states_unlock(&share_states);
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002105 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002106}
2107
Andrew Walbranca808b12020-05-15 17:22:28 +01002108/** Clean up after the receiver has finished retrieving a memory region. */
2109static void ffa_memory_retrieve_complete(
2110 struct share_states_locked share_states,
2111 struct ffa_memory_share_state *share_state, struct mpool *page_pool)
2112{
2113 if (share_state->share_func == FFA_MEM_DONATE_32) {
2114 /*
2115 * Memory that has been donated can't be relinquished,
2116 * so no need to keep the share state around.
2117 */
2118 share_state_free(share_states, share_state, page_pool);
2119 dlog_verbose("Freed share state for donate.\n");
2120 }
2121}
2122
J-Alves2d8457f2022-10-05 11:06:41 +01002123/**
2124 * Initialises the given memory region descriptor to be used for an
2125 * `FFA_MEM_RETRIEVE_RESP`, including the given constituents for the first
2126 * fragment.
2127 * The memory region descriptor is initialized according to retriever's
2128 * FF-A version.
2129 *
2130 * Returns true on success, or false if the given constituents won't all fit in
2131 * the first fragment.
2132 */
2133static bool ffa_retrieved_memory_region_init(
2134 void *response, uint32_t ffa_version, size_t response_max_size,
J-Alves19e20cf2023-08-02 12:48:55 +01002135 ffa_id_t sender, ffa_memory_attributes_t attributes,
J-Alves2d8457f2022-10-05 11:06:41 +01002136 ffa_memory_region_flags_t flags, ffa_memory_handle_t handle,
Daniel Boulbyde974ca2023-12-12 13:53:31 +00002137 ffa_memory_access_permissions_t permissions,
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002138 struct ffa_memory_access *receivers, size_t receiver_count,
2139 uint32_t memory_access_desc_size, uint32_t page_count,
2140 uint32_t total_constituent_count,
J-Alves2d8457f2022-10-05 11:06:41 +01002141 const struct ffa_memory_region_constituent constituents[],
2142 uint32_t fragment_constituent_count, uint32_t *total_length,
2143 uint32_t *fragment_length)
2144{
2145 struct ffa_composite_memory_region *composite_memory_region;
J-Alves2d8457f2022-10-05 11:06:41 +01002146 uint32_t i;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002147 uint32_t composite_offset;
J-Alves2d8457f2022-10-05 11:06:41 +01002148 uint32_t constituents_offset;
J-Alves2d8457f2022-10-05 11:06:41 +01002149
2150 assert(response != NULL);
2151
2152 if (ffa_version == MAKE_FFA_VERSION(1, 0)) {
2153 struct ffa_memory_region_v1_0 *retrieve_response =
2154 (struct ffa_memory_region_v1_0 *)response;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002155 struct ffa_memory_access_v1_0 *receiver;
J-Alves2d8457f2022-10-05 11:06:41 +01002156
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002157 ffa_memory_region_init_header_v1_0(retrieve_response, sender,
2158 attributes, flags, handle, 0,
2159 receiver_count);
J-Alves2d8457f2022-10-05 11:06:41 +01002160
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002161 receiver = (struct ffa_memory_access_v1_0 *)
2162 retrieve_response->receivers;
J-Alves2d8457f2022-10-05 11:06:41 +01002163 receiver_count = retrieve_response->receiver_count;
2164
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002165 for (uint32_t i = 0; i < receiver_count; i++) {
2166 ffa_id_t receiver_id =
2167 receivers[i].receiver_permissions.receiver;
2168 ffa_memory_receiver_flags_t recv_flags =
2169 receivers[i].receiver_permissions.flags;
2170
2171 /*
2172 * Initialized here as in memory retrieve responses we
2173 * currently expect one borrower to be specified.
2174 */
2175 ffa_memory_access_init_v1_0(
2176 receiver, receiver_id,
2177 ffa_get_data_access_attr(permissions),
2178 ffa_get_instruction_access_attr(permissions),
2179 recv_flags);
2180 }
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002181
2182 composite_offset =
J-Alves2d8457f2022-10-05 11:06:41 +01002183 sizeof(struct ffa_memory_region_v1_0) +
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002184 receiver_count * sizeof(struct ffa_memory_access_v1_0);
2185 receiver->composite_memory_region_offset = composite_offset;
J-Alves2d8457f2022-10-05 11:06:41 +01002186
2187 composite_memory_region = ffa_memory_region_get_composite_v1_0(
2188 retrieve_response, 0);
2189 } else {
J-Alves2d8457f2022-10-05 11:06:41 +01002190 struct ffa_memory_region *retrieve_response =
2191 (struct ffa_memory_region *)response;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002192 struct ffa_memory_access *retrieve_response_receivers;
J-Alves2d8457f2022-10-05 11:06:41 +01002193
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002194 ffa_memory_region_init_header(
2195 retrieve_response, sender, attributes, flags, handle, 0,
2196 receiver_count, memory_access_desc_size);
J-Alves2d8457f2022-10-05 11:06:41 +01002197
2198 /*
2199 * Note that `sizeof(struct_ffa_memory_region)` and
2200 * `sizeof(struct ffa_memory_access)` must both be multiples of
2201 * 16 (as verified by the asserts in `ffa_memory.c`, so it is
2202 * guaranteed that the offset we calculate here is aligned to a
2203 * 64-bit boundary and so 64-bit values can be copied without
2204 * alignment faults.
2205 */
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002206 composite_offset =
Daniel Boulby41ef8ba2023-10-13 17:01:22 +01002207 retrieve_response->receivers_offset +
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002208 (uint32_t)(receiver_count *
2209 retrieve_response->memory_access_desc_size);
J-Alves2d8457f2022-10-05 11:06:41 +01002210
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002211 retrieve_response_receivers =
2212 ffa_memory_region_get_receiver(retrieve_response, 0);
2213 assert(retrieve_response_receivers != NULL);
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002214
2215 /*
2216 * Initialized here as in memory retrieve responses we currently
2217 * expect one borrower to be specified.
2218 */
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002219 memcpy_s(retrieve_response_receivers,
2220 sizeof(struct ffa_memory_access) * receiver_count,
2221 receivers,
2222 sizeof(struct ffa_memory_access) * receiver_count);
2223
2224 retrieve_response_receivers->composite_memory_region_offset =
2225 composite_offset;
2226
J-Alves2d8457f2022-10-05 11:06:41 +01002227 composite_memory_region =
2228 ffa_memory_region_get_composite(retrieve_response, 0);
2229 }
2230
J-Alves2d8457f2022-10-05 11:06:41 +01002231 assert(composite_memory_region != NULL);
2232
J-Alves2d8457f2022-10-05 11:06:41 +01002233 composite_memory_region->page_count = page_count;
2234 composite_memory_region->constituent_count = total_constituent_count;
2235 composite_memory_region->reserved_0 = 0;
2236
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002237 constituents_offset =
2238 composite_offset + sizeof(struct ffa_composite_memory_region);
J-Alves2d8457f2022-10-05 11:06:41 +01002239 if (constituents_offset +
2240 fragment_constituent_count *
2241 sizeof(struct ffa_memory_region_constituent) >
2242 response_max_size) {
2243 return false;
2244 }
2245
2246 for (i = 0; i < fragment_constituent_count; ++i) {
2247 composite_memory_region->constituents[i] = constituents[i];
2248 }
2249
2250 if (total_length != NULL) {
2251 *total_length =
2252 constituents_offset +
2253 composite_memory_region->constituent_count *
2254 sizeof(struct ffa_memory_region_constituent);
2255 }
2256 if (fragment_length != NULL) {
2257 *fragment_length =
2258 constituents_offset +
2259 fragment_constituent_count *
2260 sizeof(struct ffa_memory_region_constituent);
2261 }
2262
2263 return true;
2264}
2265
J-Alves96de29f2022-04-26 16:05:24 +01002266/**
2267 * Validates the retrieved permissions against those specified by the lender
2268 * of memory share operation. Optionally can help set the permissions to be used
2269 * for the S2 mapping, through the `permissions` argument.
J-Alvesdcad8992023-09-15 14:10:35 +01002270 * Returns FFA_SUCCESS if all the fields are valid. FFA_ERROR, with error code:
2271 * - FFA_INVALID_PARAMETERS -> if the fields have invalid values as per the
2272 * specification for each ABI.
2273 * - FFA_DENIED -> if the permissions specified by the retriever are not
2274 * less permissive than those provided by the sender.
J-Alves96de29f2022-04-26 16:05:24 +01002275 */
J-Alvesdcad8992023-09-15 14:10:35 +01002276static struct ffa_value ffa_memory_retrieve_is_memory_access_valid(
2277 uint32_t share_func, enum ffa_data_access sent_data_access,
J-Alves96de29f2022-04-26 16:05:24 +01002278 enum ffa_data_access requested_data_access,
2279 enum ffa_instruction_access sent_instruction_access,
2280 enum ffa_instruction_access requested_instruction_access,
J-Alvesdcad8992023-09-15 14:10:35 +01002281 ffa_memory_access_permissions_t *permissions, bool multiple_borrowers)
J-Alves96de29f2022-04-26 16:05:24 +01002282{
2283 switch (sent_data_access) {
2284 case FFA_DATA_ACCESS_NOT_SPECIFIED:
2285 case FFA_DATA_ACCESS_RW:
2286 if (requested_data_access == FFA_DATA_ACCESS_NOT_SPECIFIED ||
2287 requested_data_access == FFA_DATA_ACCESS_RW) {
2288 if (permissions != NULL) {
2289 ffa_set_data_access_attr(permissions,
2290 FFA_DATA_ACCESS_RW);
2291 }
2292 break;
2293 }
2294 /* Intentional fall-through. */
2295 case FFA_DATA_ACCESS_RO:
2296 if (requested_data_access == FFA_DATA_ACCESS_NOT_SPECIFIED ||
2297 requested_data_access == FFA_DATA_ACCESS_RO) {
2298 if (permissions != NULL) {
2299 ffa_set_data_access_attr(permissions,
2300 FFA_DATA_ACCESS_RO);
2301 }
2302 break;
2303 }
2304 dlog_verbose(
2305 "Invalid data access requested; sender specified "
2306 "permissions %#x but receiver requested %#x.\n",
2307 sent_data_access, requested_data_access);
J-Alvesdcad8992023-09-15 14:10:35 +01002308 return ffa_error(FFA_DENIED);
J-Alves96de29f2022-04-26 16:05:24 +01002309 case FFA_DATA_ACCESS_RESERVED:
2310 panic("Got unexpected FFA_DATA_ACCESS_RESERVED. Should be "
2311 "checked before this point.");
2312 }
2313
J-Alvesdcad8992023-09-15 14:10:35 +01002314 /*
2315 * For operations with a single borrower, If it is an FFA_MEMORY_LEND
2316 * or FFA_MEMORY_DONATE the retriever should have specifed the
2317 * instruction permissions it wishes to receive.
2318 */
2319 switch (share_func) {
2320 case FFA_MEM_SHARE_32:
2321 if (requested_instruction_access !=
2322 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
2323 dlog_verbose(
2324 "%s: for share instruction permissions must "
2325 "NOT be specified.\n",
2326 __func__);
2327 return ffa_error(FFA_INVALID_PARAMETERS);
2328 }
2329 break;
2330 case FFA_MEM_LEND_32:
2331 /*
2332 * For operations with multiple borrowers only permit XN
2333 * permissions, and both Sender and borrower should have used
2334 * FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED.
2335 */
2336 if (multiple_borrowers) {
2337 if (requested_instruction_access !=
2338 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
2339 dlog_verbose(
2340 "%s: lend/share/donate with multiple "
2341 "borrowers "
2342 "instruction permissions must NOT be "
2343 "specified.\n",
2344 __func__);
2345 return ffa_error(FFA_INVALID_PARAMETERS);
2346 }
2347 break;
2348 }
2349 /* Fall through if the operation targets a single borrower. */
2350 case FFA_MEM_DONATE_32:
2351 if (!multiple_borrowers &&
2352 requested_instruction_access ==
2353 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
2354 dlog_verbose(
2355 "%s: for lend/donate with single borrower "
2356 "instruction permissions must be speficified "
2357 "by borrower\n",
2358 __func__);
2359 return ffa_error(FFA_INVALID_PARAMETERS);
2360 }
2361 break;
2362 default:
2363 panic("%s: Wrong func id provided.\n", __func__);
2364 }
2365
J-Alves96de29f2022-04-26 16:05:24 +01002366 switch (sent_instruction_access) {
2367 case FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED:
2368 case FFA_INSTRUCTION_ACCESS_X:
J-Alvesdcad8992023-09-15 14:10:35 +01002369 if (requested_instruction_access == FFA_INSTRUCTION_ACCESS_X) {
J-Alves96de29f2022-04-26 16:05:24 +01002370 if (permissions != NULL) {
2371 ffa_set_instruction_access_attr(
2372 permissions, FFA_INSTRUCTION_ACCESS_X);
2373 }
2374 break;
2375 }
J-Alvesdcad8992023-09-15 14:10:35 +01002376 /*
2377 * Fall through if requested permissions are less
2378 * permissive than those provided by the sender.
2379 */
J-Alves96de29f2022-04-26 16:05:24 +01002380 case FFA_INSTRUCTION_ACCESS_NX:
2381 if (requested_instruction_access ==
2382 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED ||
2383 requested_instruction_access == FFA_INSTRUCTION_ACCESS_NX) {
2384 if (permissions != NULL) {
2385 ffa_set_instruction_access_attr(
2386 permissions, FFA_INSTRUCTION_ACCESS_NX);
2387 }
2388 break;
2389 }
2390 dlog_verbose(
2391 "Invalid instruction access requested; sender "
2392 "specified permissions %#x but receiver requested "
2393 "%#x.\n",
2394 sent_instruction_access, requested_instruction_access);
J-Alvesdcad8992023-09-15 14:10:35 +01002395 return ffa_error(FFA_DENIED);
J-Alves96de29f2022-04-26 16:05:24 +01002396 case FFA_INSTRUCTION_ACCESS_RESERVED:
2397 panic("Got unexpected FFA_INSTRUCTION_ACCESS_RESERVED. Should "
2398 "be checked before this point.");
2399 }
2400
J-Alvesdcad8992023-09-15 14:10:35 +01002401 return (struct ffa_value){.func = FFA_SUCCESS_32};
J-Alves96de29f2022-04-26 16:05:24 +01002402}
2403
2404/**
2405 * Validate the receivers' permissions in the retrieve request against those
2406 * specified by the lender.
2407 * In the `permissions` argument returns the permissions to set at S2 for the
2408 * caller to the FFA_MEMORY_RETRIEVE_REQ.
J-Alves3456e032023-07-20 12:20:05 +01002409 * The function looks into the flag to bypass multiple borrower checks:
2410 * - If not set returns FFA_SUCCESS if all specified permissions are valid.
2411 * - If set returns FFA_SUCCESS if the descriptor contains the permissions
2412 * to the caller of FFA_MEM_RETRIEVE_REQ and they are valid. Other permissions
2413 * are ignored, if provided.
J-Alves96de29f2022-04-26 16:05:24 +01002414 */
2415static struct ffa_value ffa_memory_retrieve_validate_memory_access_list(
2416 struct ffa_memory_region *memory_region,
J-Alves19e20cf2023-08-02 12:48:55 +01002417 struct ffa_memory_region *retrieve_request, ffa_id_t to_vm_id,
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002418 ffa_memory_access_permissions_t *permissions,
2419 struct ffa_memory_access **receiver_ret, uint32_t func_id)
J-Alves96de29f2022-04-26 16:05:24 +01002420{
2421 uint32_t retrieve_receiver_index;
J-Alves3456e032023-07-20 12:20:05 +01002422 bool bypass_multi_receiver_check =
2423 (retrieve_request->flags &
2424 FFA_MEMORY_REGION_FLAG_BYPASS_BORROWERS_CHECK) != 0U;
J-Alvesdcad8992023-09-15 14:10:35 +01002425 const uint32_t region_receiver_count = memory_region->receiver_count;
2426 struct ffa_value ret;
J-Alves96de29f2022-04-26 16:05:24 +01002427
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002428 assert(receiver_ret != NULL);
J-Alves96de29f2022-04-26 16:05:24 +01002429 assert(permissions != NULL);
2430
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002431 *permissions = 0;
2432
J-Alves3456e032023-07-20 12:20:05 +01002433 if (!bypass_multi_receiver_check) {
J-Alvesdcad8992023-09-15 14:10:35 +01002434 if (retrieve_request->receiver_count != region_receiver_count) {
J-Alves3456e032023-07-20 12:20:05 +01002435 dlog_verbose(
2436 "Retrieve request should contain same list of "
2437 "borrowers, as specified by the lender.\n");
2438 return ffa_error(FFA_INVALID_PARAMETERS);
2439 }
2440 } else {
2441 if (retrieve_request->receiver_count != 1) {
2442 dlog_verbose(
2443 "Set bypass multiple borrower check, receiver "
2444 "list must be sized 1 (%x)\n",
2445 memory_region->receiver_count);
2446 return ffa_error(FFA_INVALID_PARAMETERS);
2447 }
J-Alves96de29f2022-04-26 16:05:24 +01002448 }
2449
2450 retrieve_receiver_index = retrieve_request->receiver_count;
2451
J-Alves96de29f2022-04-26 16:05:24 +01002452 for (uint32_t i = 0U; i < retrieve_request->receiver_count; i++) {
2453 ffa_memory_access_permissions_t sent_permissions;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002454 struct ffa_memory_access *retrieve_request_receiver =
2455 ffa_memory_region_get_receiver(retrieve_request, i);
2456 assert(retrieve_request_receiver != NULL);
J-Alves96de29f2022-04-26 16:05:24 +01002457 ffa_memory_access_permissions_t requested_permissions =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002458 retrieve_request_receiver->receiver_permissions
2459 .permissions;
J-Alves19e20cf2023-08-02 12:48:55 +01002460 ffa_id_t current_receiver_id =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002461 retrieve_request_receiver->receiver_permissions
2462 .receiver;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002463 struct ffa_memory_access *receiver;
2464 uint32_t mem_region_receiver_index;
2465 bool permissions_RO;
2466 bool clear_memory_flags;
J-Alves96de29f2022-04-26 16:05:24 +01002467 bool found_to_id = current_receiver_id == to_vm_id;
2468
J-Alves3456e032023-07-20 12:20:05 +01002469 if (bypass_multi_receiver_check && !found_to_id) {
2470 dlog_verbose(
2471 "Bypass multiple borrower check for id %x.\n",
2472 current_receiver_id);
2473 continue;
2474 }
2475
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002476 if (retrieve_request_receiver->composite_memory_region_offset !=
2477 0U) {
2478 dlog_verbose(
2479 "Retriever specified address ranges not "
2480 "supported (got offset %d).\n",
2481 retrieve_request_receiver
2482 ->composite_memory_region_offset);
2483 return ffa_error(FFA_INVALID_PARAMETERS);
2484 }
2485
J-Alves96de29f2022-04-26 16:05:24 +01002486 /*
2487 * Find the current receiver in the transaction descriptor from
2488 * sender.
2489 */
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002490 mem_region_receiver_index =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002491 ffa_memory_region_get_receiver_index(
2492 memory_region, current_receiver_id);
J-Alves96de29f2022-04-26 16:05:24 +01002493
2494 if (mem_region_receiver_index ==
2495 memory_region->receiver_count) {
2496 dlog_verbose("%s: receiver %x not found\n", __func__,
2497 current_receiver_id);
2498 return ffa_error(FFA_DENIED);
2499 }
2500
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002501 receiver = ffa_memory_region_get_receiver(
2502 memory_region, mem_region_receiver_index);
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002503 assert(receiver != NULL);
2504
2505 sent_permissions = receiver->receiver_permissions.permissions;
J-Alves96de29f2022-04-26 16:05:24 +01002506
2507 if (found_to_id) {
2508 retrieve_receiver_index = i;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002509
2510 *receiver_ret = receiver;
J-Alves96de29f2022-04-26 16:05:24 +01002511 }
2512
2513 /*
J-Alvesdcad8992023-09-15 14:10:35 +01002514 * Check if retrieve request memory access list is valid:
2515 * - The retrieve request complies with the specification.
2516 * - Permissions are within those specified by the sender.
J-Alves96de29f2022-04-26 16:05:24 +01002517 */
J-Alvesdcad8992023-09-15 14:10:35 +01002518 ret = ffa_memory_retrieve_is_memory_access_valid(
2519 func_id, ffa_get_data_access_attr(sent_permissions),
2520 ffa_get_data_access_attr(requested_permissions),
2521 ffa_get_instruction_access_attr(sent_permissions),
2522 ffa_get_instruction_access_attr(requested_permissions),
2523 found_to_id ? permissions : NULL,
2524 region_receiver_count > 1);
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002525
J-Alvesdcad8992023-09-15 14:10:35 +01002526 if (ret.func != FFA_SUCCESS_32) {
2527 return ret;
J-Alves96de29f2022-04-26 16:05:24 +01002528 }
2529
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002530 permissions_RO = (ffa_get_data_access_attr(*permissions) ==
2531 FFA_DATA_ACCESS_RO);
2532 clear_memory_flags = (retrieve_request->flags &
2533 FFA_MEMORY_REGION_FLAG_CLEAR) != 0U;
2534
J-Alves96de29f2022-04-26 16:05:24 +01002535 /*
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002536 * Can't request PM to clear memory if only provided
2537 * with RO permissions.
J-Alves96de29f2022-04-26 16:05:24 +01002538 */
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002539 if (found_to_id && permissions_RO && clear_memory_flags) {
J-Alves96de29f2022-04-26 16:05:24 +01002540 dlog_verbose(
2541 "Receiver has RO permissions can not request "
2542 "clear.\n");
2543 return ffa_error(FFA_DENIED);
2544 }
Daniel Boulbyde974ca2023-12-12 13:53:31 +00002545
2546 /*
2547 * Check the impdef in the retrieve_request matches the value in
2548 * the original memory send.
2549 */
2550 if (ffa_version_from_memory_access_desc_size(
2551 memory_region->memory_access_desc_size) >=
2552 MAKE_FFA_VERSION(1, 2) &&
2553 ffa_version_from_memory_access_desc_size(
2554 retrieve_request->memory_access_desc_size) >=
2555 MAKE_FFA_VERSION(1, 2)) {
Daniel Boulbyde974ca2023-12-12 13:53:31 +00002556 if (receiver->impdef.val[0] !=
2557 retrieve_request_receiver->impdef.val[0] ||
2558 receiver->impdef.val[1] !=
2559 retrieve_request_receiver->impdef.val[1]) {
2560 dlog_verbose(
2561 "Impdef value in memory send does not "
2562 "match retrieve request value "
2563 "send value %#x %#x retrieve request "
2564 "value %#x %#x\n",
2565 receiver->impdef.val[0],
2566 receiver->impdef.val[1],
2567 retrieve_request_receiver->impdef
2568 .val[0],
2569 retrieve_request_receiver->impdef
2570 .val[1]);
2571 return ffa_error(FFA_INVALID_PARAMETERS);
2572 }
2573 }
J-Alves96de29f2022-04-26 16:05:24 +01002574 }
2575
2576 if (retrieve_receiver_index == retrieve_request->receiver_count) {
2577 dlog_verbose(
2578 "Retrieve request does not contain caller's (%x) "
2579 "permissions\n",
2580 to_vm_id);
2581 return ffa_error(FFA_INVALID_PARAMETERS);
2582 }
2583
2584 return (struct ffa_value){.func = FFA_SUCCESS_32};
2585}
2586
J-Alvesa9cd7e32022-07-01 13:49:33 +01002587/*
2588 * According to section 16.4.3 of FF-A v1.1 EAC0 specification, the hypervisor
2589 * may issue an FFA_MEM_RETRIEVE_REQ to obtain the memory region description
2590 * of a pending memory sharing operation whose allocator is the SPM, for
2591 * validation purposes before forwarding an FFA_MEM_RECLAIM call. In doing so
2592 * the memory region descriptor of the retrieve request must be zeroed with the
2593 * exception of the sender ID and handle.
2594 */
J-Alves4f0d9c12024-01-17 17:23:11 +00002595bool is_ffa_hypervisor_retrieve_request(struct ffa_memory_region *request,
2596 struct vm_locked to_locked)
J-Alvesa9cd7e32022-07-01 13:49:33 +01002597{
2598 return to_locked.vm->id == HF_HYPERVISOR_VM_ID &&
2599 request->attributes == 0U && request->flags == 0U &&
2600 request->tag == 0U && request->receiver_count == 0U &&
2601 plat_ffa_memory_handle_allocated_by_current_world(
2602 request->handle);
2603}
2604
2605/*
2606 * Helper to reset count of fragments retrieved by the hypervisor.
2607 */
2608static void ffa_memory_retrieve_complete_from_hyp(
2609 struct ffa_memory_share_state *share_state)
2610{
2611 if (share_state->hypervisor_fragment_count ==
2612 share_state->fragment_count) {
2613 share_state->hypervisor_fragment_count = 0;
2614 }
2615}
2616
J-Alves089004f2022-07-13 14:25:44 +01002617/**
J-Alves4f0d9c12024-01-17 17:23:11 +00002618 * Prepares the return of the ffa_value for the memory retrieve response.
2619 */
2620static struct ffa_value ffa_memory_retrieve_resp(uint32_t total_length,
2621 uint32_t fragment_length)
2622{
2623 return (struct ffa_value){.func = FFA_MEM_RETRIEVE_RESP_32,
2624 .arg1 = total_length,
2625 .arg2 = fragment_length};
2626}
2627
2628/**
J-Alves089004f2022-07-13 14:25:44 +01002629 * Validate that the memory region descriptor provided by the borrower on
2630 * FFA_MEM_RETRIEVE_REQ, against saved memory region provided by lender at the
2631 * memory sharing call.
2632 */
2633static struct ffa_value ffa_memory_retrieve_validate(
J-Alves4f0d9c12024-01-17 17:23:11 +00002634 ffa_id_t to_id, struct ffa_memory_region *retrieve_request,
2635 uint32_t retrieve_request_length,
J-Alves089004f2022-07-13 14:25:44 +01002636 struct ffa_memory_region *memory_region, uint32_t *receiver_index,
2637 uint32_t share_func)
2638{
2639 ffa_memory_region_flags_t transaction_type =
2640 retrieve_request->flags &
2641 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK;
Olivier Deprez4342a3c2022-02-28 09:37:25 +01002642 enum ffa_memory_security security_state;
J-Alves4f0d9c12024-01-17 17:23:11 +00002643 const uint64_t memory_access_desc_size =
2644 retrieve_request->memory_access_desc_size;
2645 const uint32_t expected_retrieve_request_length =
2646 retrieve_request->receivers_offset +
2647 (uint32_t)(retrieve_request->receiver_count *
2648 memory_access_desc_size);
J-Alves089004f2022-07-13 14:25:44 +01002649
2650 assert(retrieve_request != NULL);
2651 assert(memory_region != NULL);
2652 assert(receiver_index != NULL);
2653 assert(retrieve_request->sender == memory_region->sender);
2654
J-Alves4f0d9c12024-01-17 17:23:11 +00002655 if (retrieve_request_length != expected_retrieve_request_length) {
2656 dlog_verbose(
2657 "Invalid length for FFA_MEM_RETRIEVE_REQ, expected %d "
2658 "but was %d.\n",
2659 expected_retrieve_request_length,
2660 retrieve_request_length);
2661 return ffa_error(FFA_INVALID_PARAMETERS);
2662 }
2663
2664 if (retrieve_request->sender != memory_region->sender) {
2665 dlog_verbose(
2666 "Memory with handle %#x not fully sent, can't "
2667 "retrieve.\n",
2668 memory_region->handle);
2669 return ffa_error(FFA_DENIED);
2670 }
2671
2672 /*
2673 * The SPMC can only process retrieve requests to memory share
2674 * operations with one borrower from the other world. It can't
2675 * determine the ID of the NWd VM that invoked the retrieve
2676 * request interface call. It relies on the hypervisor to
2677 * validate the caller's ID against that provided in the
2678 * `receivers` list of the retrieve response.
2679 * In case there is only one borrower from the NWd in the
2680 * transaction descriptor, record that in the `receiver_id` for
2681 * later use, and validate in the retrieve request message.
2682 * This limitation is due to the fact SPMC can't determine the
2683 * index in the memory share structures state to update.
2684 */
2685 if (to_id == HF_HYPERVISOR_VM_ID) {
2686 uint32_t other_world_count = 0;
2687
2688 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
2689 struct ffa_memory_access *receiver =
2690 ffa_memory_region_get_receiver(retrieve_request,
2691 0);
2692 assert(receiver != NULL);
2693
2694 to_id = receiver->receiver_permissions.receiver;
2695
2696 if (!vm_id_is_current_world(to_id)) {
2697 other_world_count++;
2698 }
2699 }
2700
2701 if (other_world_count > 1) {
2702 dlog_verbose(
2703 "Support one receiver from the other "
2704 "world.\n");
2705 return ffa_error(FFA_NOT_SUPPORTED);
2706 }
2707 }
J-Alves089004f2022-07-13 14:25:44 +01002708 /*
2709 * Check that the transaction type expected by the receiver is
2710 * correct, if it has been specified.
2711 */
2712 if (transaction_type !=
2713 FFA_MEMORY_REGION_TRANSACTION_TYPE_UNSPECIFIED &&
2714 transaction_type != (memory_region->flags &
2715 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK)) {
2716 dlog_verbose(
2717 "Incorrect transaction type %#x for "
2718 "FFA_MEM_RETRIEVE_REQ, expected %#x for handle %#x.\n",
2719 transaction_type,
2720 memory_region->flags &
2721 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK,
2722 retrieve_request->handle);
2723 return ffa_error(FFA_INVALID_PARAMETERS);
2724 }
2725
2726 if (retrieve_request->tag != memory_region->tag) {
2727 dlog_verbose(
2728 "Incorrect tag %d for FFA_MEM_RETRIEVE_REQ, expected "
2729 "%d for handle %#x.\n",
2730 retrieve_request->tag, memory_region->tag,
2731 retrieve_request->handle);
2732 return ffa_error(FFA_INVALID_PARAMETERS);
2733 }
2734
J-Alves4f0d9c12024-01-17 17:23:11 +00002735 *receiver_index =
2736 ffa_memory_region_get_receiver_index(memory_region, to_id);
J-Alves089004f2022-07-13 14:25:44 +01002737
2738 if (*receiver_index == memory_region->receiver_count) {
2739 dlog_verbose(
2740 "Incorrect receiver VM ID %d for "
2741 "FFA_MEM_RETRIEVE_REQ, for handle %#x.\n",
J-Alves4f0d9c12024-01-17 17:23:11 +00002742 to_id, memory_region->handle);
J-Alves089004f2022-07-13 14:25:44 +01002743 return ffa_error(FFA_INVALID_PARAMETERS);
2744 }
2745
2746 if ((retrieve_request->flags &
2747 FFA_MEMORY_REGION_ADDRESS_RANGE_HINT_VALID) != 0U) {
2748 dlog_verbose(
2749 "Retriever specified 'address range alignment 'hint' "
2750 "not supported.\n");
2751 return ffa_error(FFA_INVALID_PARAMETERS);
2752 }
2753 if ((retrieve_request->flags &
2754 FFA_MEMORY_REGION_ADDRESS_RANGE_HINT_MASK) != 0) {
2755 dlog_verbose(
2756 "Bits 8-5 must be zero in memory region's flags "
2757 "(address range alignment hint not supported).\n");
2758 return ffa_error(FFA_INVALID_PARAMETERS);
2759 }
2760
2761 if ((retrieve_request->flags & ~0x7FF) != 0U) {
2762 dlog_verbose(
2763 "Bits 31-10 must be zero in memory region's flags.\n");
2764 return ffa_error(FFA_INVALID_PARAMETERS);
2765 }
2766
2767 if (share_func == FFA_MEM_SHARE_32 &&
2768 (retrieve_request->flags &
2769 (FFA_MEMORY_REGION_FLAG_CLEAR |
2770 FFA_MEMORY_REGION_FLAG_CLEAR_RELINQUISH)) != 0U) {
2771 dlog_verbose(
2772 "Memory Share operation can't clean after relinquish "
2773 "memory region.\n");
2774 return ffa_error(FFA_INVALID_PARAMETERS);
2775 }
2776
2777 /*
2778 * If the borrower needs the memory to be cleared before mapping
2779 * to its address space, the sender should have set the flag
2780 * when calling FFA_MEM_LEND/FFA_MEM_DONATE, else return
2781 * FFA_DENIED.
2782 */
2783 if ((retrieve_request->flags & FFA_MEMORY_REGION_FLAG_CLEAR) != 0U &&
2784 (memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR) == 0U) {
2785 dlog_verbose(
2786 "Borrower needs memory cleared. Sender needs to set "
2787 "flag for clearing memory.\n");
2788 return ffa_error(FFA_DENIED);
2789 }
2790
Olivier Deprez4342a3c2022-02-28 09:37:25 +01002791 /* Memory region attributes NS-Bit MBZ for FFA_MEM_RETRIEVE_REQ. */
2792 security_state =
2793 ffa_get_memory_security_attr(retrieve_request->attributes);
2794 if (security_state != FFA_MEMORY_SECURITY_UNSPECIFIED) {
2795 dlog_verbose(
2796 "Invalid security state for memory retrieve request "
2797 "operation.\n");
2798 return ffa_error(FFA_INVALID_PARAMETERS);
2799 }
2800
J-Alves089004f2022-07-13 14:25:44 +01002801 /*
2802 * If memory type is not specified, bypass validation of memory
2803 * attributes in the retrieve request. The retriever is expecting to
2804 * obtain this information from the SPMC.
2805 */
2806 if (ffa_get_memory_type_attr(retrieve_request->attributes) ==
2807 FFA_MEMORY_NOT_SPECIFIED_MEM) {
2808 return (struct ffa_value){.func = FFA_SUCCESS_32};
2809 }
2810
2811 /*
2812 * Ensure receiver's attributes are compatible with how
2813 * Hafnium maps memory: Normal Memory, Inner shareable,
2814 * Write-Back Read-Allocate Write-Allocate Cacheable.
2815 */
2816 return ffa_memory_attributes_validate(retrieve_request->attributes);
2817}
2818
J-Alves4f0d9c12024-01-17 17:23:11 +00002819static struct ffa_value ffa_partition_retrieve_request(
2820 struct share_states_locked share_states,
2821 struct ffa_memory_share_state *share_state, struct vm_locked to_locked,
2822 struct ffa_memory_region *retrieve_request,
2823 uint32_t retrieve_request_length, struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002824{
J-Alvesa9cd7e32022-07-01 13:49:33 +01002825 ffa_memory_access_permissions_t permissions = 0;
Olivier Deprez878bd5b2021-04-15 19:05:10 +02002826 uint32_t memory_to_mode;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002827 struct ffa_value ret;
Andrew Walbranca808b12020-05-15 17:22:28 +01002828 struct ffa_composite_memory_region *composite;
2829 uint32_t total_length;
2830 uint32_t fragment_length;
J-Alves19e20cf2023-08-02 12:48:55 +01002831 ffa_id_t receiver_id = to_locked.vm->id;
J-Alves4f0d9c12024-01-17 17:23:11 +00002832 bool is_retrieve_complete = false;
Olivier Deprez878bd5b2021-04-15 19:05:10 +02002833 ffa_memory_attributes_t attributes;
J-Alves4f0d9c12024-01-17 17:23:11 +00002834 const uint64_t memory_access_desc_size =
Daniel Boulbyde974ca2023-12-12 13:53:31 +00002835 retrieve_request->memory_access_desc_size;
J-Alves4f0d9c12024-01-17 17:23:11 +00002836 uint32_t receiver_index;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002837 struct ffa_memory_access *receiver;
J-Alves4f0d9c12024-01-17 17:23:11 +00002838 ffa_memory_handle_t handle = retrieve_request->handle;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002839
J-Alves4f0d9c12024-01-17 17:23:11 +00002840 struct ffa_memory_region *memory_region = share_state->memory_region;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002841
J-Alves96de29f2022-04-26 16:05:24 +01002842 if (!share_state->sending_complete) {
2843 dlog_verbose(
2844 "Memory with handle %#x not fully sent, can't "
2845 "retrieve.\n",
2846 handle);
J-Alves4f0d9c12024-01-17 17:23:11 +00002847 return ffa_error(FFA_INVALID_PARAMETERS);
J-Alves96de29f2022-04-26 16:05:24 +01002848 }
2849
J-Alves4f0d9c12024-01-17 17:23:11 +00002850 /*
2851 * Validate retrieve request, according to what was sent by the
2852 * sender. Function will output the `receiver_index` from the
2853 * provided memory region.
2854 */
2855 ret = ffa_memory_retrieve_validate(
2856 receiver_id, retrieve_request, retrieve_request_length,
2857 memory_region, &receiver_index, share_state->share_func);
J-Alves089004f2022-07-13 14:25:44 +01002858
J-Alves4f0d9c12024-01-17 17:23:11 +00002859 if (ret.func != FFA_SUCCESS_32) {
2860 return ret;
J-Alves089004f2022-07-13 14:25:44 +01002861 }
J-Alves96de29f2022-04-26 16:05:24 +01002862
J-Alves4f0d9c12024-01-17 17:23:11 +00002863 if (share_state->retrieved_fragment_count[receiver_index] != 0U) {
2864 dlog_verbose("Memory with handle %#x already retrieved.\n",
2865 handle);
2866 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002867 }
2868
J-Alves4f0d9c12024-01-17 17:23:11 +00002869 /*
2870 * Validate the requested permissions against the sent
2871 * permissions.
2872 * Outputs the permissions to give to retriever at S2
2873 * PTs.
2874 */
2875 ret = ffa_memory_retrieve_validate_memory_access_list(
2876 memory_region, retrieve_request, receiver_id, &permissions,
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002877 &receiver, share_state->share_func);
J-Alves4f0d9c12024-01-17 17:23:11 +00002878 if (ret.func != FFA_SUCCESS_32) {
2879 return ret;
2880 }
2881
2882 memory_to_mode = ffa_memory_permissions_to_mode(
2883 permissions, share_state->sender_orig_mode);
2884
2885 ret = ffa_retrieve_check_update(
2886 to_locked, share_state->fragments,
2887 share_state->fragment_constituent_counts,
2888 share_state->fragment_count, memory_to_mode,
2889 share_state->share_func, false, page_pool);
2890
2891 if (ret.func != FFA_SUCCESS_32) {
2892 return ret;
2893 }
2894
2895 share_state->retrieved_fragment_count[receiver_index] = 1;
2896
2897 is_retrieve_complete =
2898 share_state->retrieved_fragment_count[receiver_index] ==
2899 share_state->fragment_count;
2900
2901 share_state->clear_after_relinquish =
2902 (retrieve_request->flags &
2903 FFA_MEMORY_REGION_FLAG_CLEAR_RELINQUISH) != 0U;
2904
J-Alvesb5084cf2022-07-06 14:20:12 +01002905 /* VMs acquire the RX buffer from SPMC. */
2906 CHECK(plat_ffa_acquire_receiver_rx(to_locked, &ret));
2907
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002908 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01002909 * Copy response to RX buffer of caller and deliver the message.
2910 * This must be done before the share_state is (possibly) freed.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002911 */
Andrew Walbranca808b12020-05-15 17:22:28 +01002912 composite = ffa_memory_region_get_composite(memory_region, 0);
J-Alves4f0d9c12024-01-17 17:23:11 +00002913
Andrew Walbranca808b12020-05-15 17:22:28 +01002914 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01002915 * Constituents which we received in the first fragment should
2916 * always fit in the first fragment we are sending, because the
2917 * header is the same size in both cases and we have a fixed
2918 * message buffer size. So `ffa_retrieved_memory_region_init`
2919 * should never fail.
Andrew Walbranca808b12020-05-15 17:22:28 +01002920 */
Olivier Deprez878bd5b2021-04-15 19:05:10 +02002921
2922 /*
2923 * Set the security state in the memory retrieve response attributes
2924 * if specified by the target mode.
2925 */
2926 attributes = plat_ffa_memory_security_mode(
2927 memory_region->attributes, share_state->sender_orig_mode);
2928
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002929 /* Provide the permissions that had been provided. */
2930 receiver->receiver_permissions.permissions = permissions;
2931
2932 /*
2933 * Prepare the memory region descriptor for the retrieve response.
2934 * Provide the pointer to the receiver tracked in the share state
2935 * strucutures.
2936 */
Andrew Walbranca808b12020-05-15 17:22:28 +01002937 CHECK(ffa_retrieved_memory_region_init(
J-Alves2d8457f2022-10-05 11:06:41 +01002938 to_locked.vm->mailbox.recv, to_locked.vm->ffa_version,
Olivier Deprez878bd5b2021-04-15 19:05:10 +02002939 HF_MAILBOX_SIZE, memory_region->sender, attributes,
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002940 memory_region->flags, handle, permissions, receiver, 1,
2941 memory_access_desc_size, composite->page_count,
2942 composite->constituent_count, share_state->fragments[0],
Andrew Walbranca808b12020-05-15 17:22:28 +01002943 share_state->fragment_constituent_counts[0], &total_length,
2944 &fragment_length));
J-Alvesb5084cf2022-07-06 14:20:12 +01002945
J-Alves4f0d9c12024-01-17 17:23:11 +00002946 if (is_retrieve_complete) {
Andrew Walbranca808b12020-05-15 17:22:28 +01002947 ffa_memory_retrieve_complete(share_states, share_state,
2948 page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002949 }
J-Alves4f0d9c12024-01-17 17:23:11 +00002950
2951 return ffa_memory_retrieve_resp(total_length, fragment_length);
2952}
2953
2954static struct ffa_value ffa_hypervisor_retrieve_request(
2955 struct ffa_memory_share_state *share_state, struct vm_locked to_locked,
2956 struct ffa_memory_region *retrieve_request)
2957{
2958 struct ffa_value ret;
2959 struct ffa_composite_memory_region *composite;
2960 uint32_t total_length;
2961 uint32_t fragment_length;
J-Alves4f0d9c12024-01-17 17:23:11 +00002962 ffa_memory_attributes_t attributes;
J-Alves7b6ab612024-01-24 09:54:54 +00002963 uint64_t memory_access_desc_size;
J-Alves4f0d9c12024-01-17 17:23:11 +00002964 struct ffa_memory_region *memory_region;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002965 struct ffa_memory_access *receiver;
J-Alves4f0d9c12024-01-17 17:23:11 +00002966 ffa_memory_handle_t handle = retrieve_request->handle;
2967
J-Alves4f0d9c12024-01-17 17:23:11 +00002968 memory_region = share_state->memory_region;
2969
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002970 assert(to_locked.vm->id == HF_HYPERVISOR_VM_ID);
2971
J-Alves7b6ab612024-01-24 09:54:54 +00002972 switch (to_locked.vm->ffa_version) {
2973 case MAKE_FFA_VERSION(1, 2):
2974 memory_access_desc_size = sizeof(struct ffa_memory_access);
2975 break;
2976 case MAKE_FFA_VERSION(1, 0):
2977 case MAKE_FFA_VERSION(1, 1):
2978 memory_access_desc_size = sizeof(struct ffa_memory_access_v1_0);
2979 break;
2980 default:
2981 panic("version not supported: %x\n", to_locked.vm->ffa_version);
2982 }
2983
J-Alves4f0d9c12024-01-17 17:23:11 +00002984 if (share_state->hypervisor_fragment_count != 0U) {
2985 dlog_verbose(
2986 "Memory with handle %#x already retrieved by "
2987 "the hypervisor.\n",
2988 handle);
2989 return ffa_error(FFA_DENIED);
2990 }
2991
2992 share_state->hypervisor_fragment_count = 1;
2993
2994 ffa_memory_retrieve_complete_from_hyp(share_state);
2995
2996 /* VMs acquire the RX buffer from SPMC. */
2997 CHECK(plat_ffa_acquire_receiver_rx(to_locked, &ret));
2998
2999 /*
3000 * Copy response to RX buffer of caller and deliver the message.
3001 * This must be done before the share_state is (possibly) freed.
3002 */
3003 composite = ffa_memory_region_get_composite(memory_region, 0);
3004
3005 /*
3006 * Constituents which we received in the first fragment should
3007 * always fit in the first fragment we are sending, because the
3008 * header is the same size in both cases and we have a fixed
3009 * message buffer size. So `ffa_retrieved_memory_region_init`
3010 * should never fail.
3011 */
3012
3013 /*
3014 * Set the security state in the memory retrieve response attributes
3015 * if specified by the target mode.
3016 */
3017 attributes = plat_ffa_memory_security_mode(
3018 memory_region->attributes, share_state->sender_orig_mode);
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003019
3020 receiver = ffa_memory_region_get_receiver(memory_region, 0);
3021
J-Alves4f0d9c12024-01-17 17:23:11 +00003022 CHECK(ffa_retrieved_memory_region_init(
3023 to_locked.vm->mailbox.recv, to_locked.vm->ffa_version,
3024 HF_MAILBOX_SIZE, memory_region->sender, attributes,
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003025 memory_region->flags, handle,
3026 receiver->receiver_permissions.permissions, receiver,
3027 memory_region->receiver_count, memory_access_desc_size,
J-Alves4f0d9c12024-01-17 17:23:11 +00003028 composite->page_count, composite->constituent_count,
3029 share_state->fragments[0],
3030 share_state->fragment_constituent_counts[0], &total_length,
3031 &fragment_length));
3032
3033 return ffa_memory_retrieve_resp(total_length, fragment_length);
3034}
3035
3036struct ffa_value ffa_memory_retrieve(struct vm_locked to_locked,
3037 struct ffa_memory_region *retrieve_request,
3038 uint32_t retrieve_request_length,
3039 struct mpool *page_pool)
3040{
3041 ffa_memory_handle_t handle = retrieve_request->handle;
3042 struct share_states_locked share_states;
3043 struct ffa_memory_share_state *share_state;
3044 struct ffa_value ret;
3045
3046 dump_share_states();
3047
3048 share_states = share_states_lock();
3049 share_state = get_share_state(share_states, handle);
3050 if (share_state == NULL) {
3051 dlog_verbose("Invalid handle %#x for FFA_MEM_RETRIEVE_REQ.\n",
3052 handle);
3053 ret = ffa_error(FFA_INVALID_PARAMETERS);
3054 goto out;
3055 }
3056
3057 if (is_ffa_hypervisor_retrieve_request(retrieve_request, to_locked)) {
3058 ret = ffa_hypervisor_retrieve_request(share_state, to_locked,
3059 retrieve_request);
3060 } else {
3061 ret = ffa_partition_retrieve_request(
3062 share_states, share_state, to_locked, retrieve_request,
3063 retrieve_request_length, page_pool);
3064 }
3065
3066 /* Track use of the RX buffer if the handling has succeeded. */
3067 if (ret.func == FFA_MEM_RETRIEVE_RESP_32) {
3068 to_locked.vm->mailbox.recv_func = FFA_MEM_RETRIEVE_RESP_32;
3069 to_locked.vm->mailbox.state = MAILBOX_STATE_FULL;
3070 }
3071
Andrew Walbranca808b12020-05-15 17:22:28 +01003072out:
3073 share_states_unlock(&share_states);
3074 dump_share_states();
3075 return ret;
3076}
3077
J-Alves5da37d92022-10-24 16:33:48 +01003078/**
3079 * Determine expected fragment offset according to the FF-A version of
3080 * the caller.
3081 */
3082static uint32_t ffa_memory_retrieve_expected_offset_per_ffa_version(
3083 struct ffa_memory_region *memory_region,
3084 uint32_t retrieved_constituents_count, uint32_t ffa_version)
3085{
3086 uint32_t expected_fragment_offset;
3087 uint32_t composite_constituents_offset;
3088
Kathleen Capellae4fe2962023-09-01 17:08:47 -04003089 if (ffa_version >= MAKE_FFA_VERSION(1, 1)) {
J-Alves5da37d92022-10-24 16:33:48 +01003090 /*
3091 * Hafnium operates memory regions in FF-A v1.1 format, so we
3092 * can retrieve the constituents offset from descriptor.
3093 */
3094 composite_constituents_offset =
3095 ffa_composite_constituent_offset(memory_region, 0);
3096 } else if (ffa_version == MAKE_FFA_VERSION(1, 0)) {
3097 /*
3098 * If retriever is FF-A v1.0, determine the composite offset
3099 * as it is expected to have been configured in the
3100 * retrieve response.
3101 */
3102 composite_constituents_offset =
3103 sizeof(struct ffa_memory_region_v1_0) +
3104 RECEIVERS_COUNT_IN_RETRIEVE_RESP *
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003105 sizeof(struct ffa_memory_access_v1_0) +
J-Alves5da37d92022-10-24 16:33:48 +01003106 sizeof(struct ffa_composite_memory_region);
3107 } else {
3108 panic("%s received an invalid FF-A version.\n", __func__);
3109 }
3110
3111 expected_fragment_offset =
3112 composite_constituents_offset +
3113 retrieved_constituents_count *
3114 sizeof(struct ffa_memory_region_constituent) -
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003115 (uint32_t)(memory_region->memory_access_desc_size *
3116 (memory_region->receiver_count - 1));
J-Alves5da37d92022-10-24 16:33:48 +01003117
3118 return expected_fragment_offset;
3119}
3120
Andrew Walbranca808b12020-05-15 17:22:28 +01003121struct ffa_value ffa_memory_retrieve_continue(struct vm_locked to_locked,
3122 ffa_memory_handle_t handle,
3123 uint32_t fragment_offset,
J-Alves19e20cf2023-08-02 12:48:55 +01003124 ffa_id_t sender_vm_id,
Andrew Walbranca808b12020-05-15 17:22:28 +01003125 struct mpool *page_pool)
3126{
3127 struct ffa_memory_region *memory_region;
3128 struct share_states_locked share_states;
3129 struct ffa_memory_share_state *share_state;
3130 struct ffa_value ret;
3131 uint32_t fragment_index;
3132 uint32_t retrieved_constituents_count;
3133 uint32_t i;
3134 uint32_t expected_fragment_offset;
3135 uint32_t remaining_constituent_count;
3136 uint32_t fragment_length;
J-Alvesc7484f12022-05-13 12:41:14 +01003137 uint32_t receiver_index;
J-Alves59ed0042022-07-28 18:26:41 +01003138 bool continue_ffa_hyp_mem_retrieve_req;
Andrew Walbranca808b12020-05-15 17:22:28 +01003139
3140 dump_share_states();
3141
3142 share_states = share_states_lock();
Karl Meakin4a2854a2023-06-30 16:26:52 +01003143 share_state = get_share_state(share_states, handle);
J-Alvesb56aac82023-11-10 09:44:43 +00003144 if (share_state == NULL) {
Andrew Walbranca808b12020-05-15 17:22:28 +01003145 dlog_verbose("Invalid handle %#x for FFA_MEM_FRAG_RX.\n",
3146 handle);
3147 ret = ffa_error(FFA_INVALID_PARAMETERS);
3148 goto out;
3149 }
3150
3151 memory_region = share_state->memory_region;
3152 CHECK(memory_region != NULL);
3153
Andrew Walbranca808b12020-05-15 17:22:28 +01003154 if (!share_state->sending_complete) {
3155 dlog_verbose(
3156 "Memory with handle %#x not fully sent, can't "
3157 "retrieve.\n",
3158 handle);
3159 ret = ffa_error(FFA_INVALID_PARAMETERS);
3160 goto out;
3161 }
3162
J-Alves59ed0042022-07-28 18:26:41 +01003163 /*
3164 * If retrieve request from the hypervisor has been initiated in the
3165 * given share_state, continue it, else assume it is a continuation of
3166 * retrieve request from a NWd VM.
3167 */
3168 continue_ffa_hyp_mem_retrieve_req =
3169 (to_locked.vm->id == HF_HYPERVISOR_VM_ID) &&
3170 (share_state->hypervisor_fragment_count != 0U) &&
J-Alves661e1b72023-08-02 13:39:40 +01003171 ffa_is_vm_id(sender_vm_id);
Andrew Walbranca808b12020-05-15 17:22:28 +01003172
J-Alves59ed0042022-07-28 18:26:41 +01003173 if (!continue_ffa_hyp_mem_retrieve_req) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003174 receiver_index = ffa_memory_region_get_receiver_index(
J-Alves59ed0042022-07-28 18:26:41 +01003175 memory_region, to_locked.vm->id);
3176
3177 if (receiver_index == memory_region->receiver_count) {
3178 dlog_verbose(
3179 "Caller of FFA_MEM_FRAG_RX (%x) is not a "
3180 "borrower to memory sharing transaction (%x)\n",
3181 to_locked.vm->id, handle);
3182 ret = ffa_error(FFA_INVALID_PARAMETERS);
3183 goto out;
3184 }
3185
3186 if (share_state->retrieved_fragment_count[receiver_index] ==
3187 0 ||
3188 share_state->retrieved_fragment_count[receiver_index] >=
3189 share_state->fragment_count) {
3190 dlog_verbose(
3191 "Retrieval of memory with handle %#x not yet "
3192 "started or already completed (%d/%d fragments "
3193 "retrieved).\n",
3194 handle,
3195 share_state->retrieved_fragment_count
3196 [receiver_index],
3197 share_state->fragment_count);
3198 ret = ffa_error(FFA_INVALID_PARAMETERS);
3199 goto out;
3200 }
3201
3202 fragment_index =
3203 share_state->retrieved_fragment_count[receiver_index];
3204 } else {
3205 if (share_state->hypervisor_fragment_count == 0 ||
3206 share_state->hypervisor_fragment_count >=
3207 share_state->fragment_count) {
3208 dlog_verbose(
3209 "Retrieve of memory with handle %x not "
3210 "started from hypervisor.\n",
3211 handle);
3212 ret = ffa_error(FFA_INVALID_PARAMETERS);
3213 goto out;
3214 }
3215
3216 if (memory_region->sender != sender_vm_id) {
3217 dlog_verbose(
3218 "Sender ID (%x) is not as expected for memory "
3219 "handle %x\n",
3220 sender_vm_id, handle);
3221 ret = ffa_error(FFA_INVALID_PARAMETERS);
3222 goto out;
3223 }
3224
3225 fragment_index = share_state->hypervisor_fragment_count;
3226
3227 receiver_index = 0;
3228 }
Andrew Walbranca808b12020-05-15 17:22:28 +01003229
3230 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01003231 * Check that the given fragment offset is correct by counting
3232 * how many constituents were in the fragments previously sent.
Andrew Walbranca808b12020-05-15 17:22:28 +01003233 */
3234 retrieved_constituents_count = 0;
3235 for (i = 0; i < fragment_index; ++i) {
3236 retrieved_constituents_count +=
3237 share_state->fragment_constituent_counts[i];
3238 }
J-Alvesc7484f12022-05-13 12:41:14 +01003239
3240 CHECK(memory_region->receiver_count > 0);
3241
Andrew Walbranca808b12020-05-15 17:22:28 +01003242 expected_fragment_offset =
J-Alves5da37d92022-10-24 16:33:48 +01003243 ffa_memory_retrieve_expected_offset_per_ffa_version(
3244 memory_region, retrieved_constituents_count,
3245 to_locked.vm->ffa_version);
3246
Andrew Walbranca808b12020-05-15 17:22:28 +01003247 if (fragment_offset != expected_fragment_offset) {
3248 dlog_verbose("Fragment offset was %d but expected %d.\n",
3249 fragment_offset, expected_fragment_offset);
3250 ret = ffa_error(FFA_INVALID_PARAMETERS);
3251 goto out;
3252 }
3253
J-Alves4f0d9c12024-01-17 17:23:11 +00003254 /*
3255 * When hafnium is the hypervisor, acquire the RX buffer of a VM, that
3256 * is currently ownder by the SPMC.
3257 */
3258 assert(plat_ffa_acquire_receiver_rx(to_locked, &ret));
J-Alves59ed0042022-07-28 18:26:41 +01003259
Andrew Walbranca808b12020-05-15 17:22:28 +01003260 remaining_constituent_count = ffa_memory_fragment_init(
3261 to_locked.vm->mailbox.recv, HF_MAILBOX_SIZE,
3262 share_state->fragments[fragment_index],
3263 share_state->fragment_constituent_counts[fragment_index],
3264 &fragment_length);
3265 CHECK(remaining_constituent_count == 0);
J-Alves674e4de2024-01-17 16:20:32 +00003266
Andrew Walbranca808b12020-05-15 17:22:28 +01003267 to_locked.vm->mailbox.recv_func = FFA_MEM_FRAG_TX_32;
J-Alvese8c8c2b2022-12-16 15:34:48 +00003268 to_locked.vm->mailbox.state = MAILBOX_STATE_FULL;
Andrew Walbranca808b12020-05-15 17:22:28 +01003269
J-Alves59ed0042022-07-28 18:26:41 +01003270 if (!continue_ffa_hyp_mem_retrieve_req) {
3271 share_state->retrieved_fragment_count[receiver_index]++;
3272 if (share_state->retrieved_fragment_count[receiver_index] ==
3273 share_state->fragment_count) {
3274 ffa_memory_retrieve_complete(share_states, share_state,
3275 page_pool);
3276 }
3277 } else {
3278 share_state->hypervisor_fragment_count++;
3279
3280 ffa_memory_retrieve_complete_from_hyp(share_state);
3281 }
Andrew Walbranca808b12020-05-15 17:22:28 +01003282 ret = (struct ffa_value){.func = FFA_MEM_FRAG_TX_32,
3283 .arg1 = (uint32_t)handle,
3284 .arg2 = (uint32_t)(handle >> 32),
3285 .arg3 = fragment_length};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003286
3287out:
3288 share_states_unlock(&share_states);
3289 dump_share_states();
3290 return ret;
3291}
3292
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003293struct ffa_value ffa_memory_relinquish(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003294 struct vm_locked from_locked,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003295 struct ffa_mem_relinquish *relinquish_request, struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003296{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003297 ffa_memory_handle_t handle = relinquish_request->handle;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003298 struct share_states_locked share_states;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003299 struct ffa_memory_share_state *share_state;
3300 struct ffa_memory_region *memory_region;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003301 bool clear;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003302 struct ffa_value ret;
J-Alves8eb19162022-04-28 10:56:48 +01003303 uint32_t receiver_index;
J-Alves3c5b2072022-11-21 12:45:40 +00003304 bool receivers_relinquished_memory;
J-Alves639ddfc2023-11-21 14:17:26 +00003305 ffa_memory_access_permissions_t receiver_permissions = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003306
Andrew Walbrana65a1322020-04-06 19:32:32 +01003307 if (relinquish_request->endpoint_count != 1) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003308 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01003309 "Stream endpoints not supported (got %d "
J-Alves668a86e2023-05-10 11:53:25 +01003310 "endpoints on FFA_MEM_RELINQUISH, expected 1).\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003311 relinquish_request->endpoint_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003312 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003313 }
3314
Andrew Walbrana65a1322020-04-06 19:32:32 +01003315 if (relinquish_request->endpoints[0] != from_locked.vm->id) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003316 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01003317 "VM ID %d in relinquish message doesn't match "
J-Alves668a86e2023-05-10 11:53:25 +01003318 "calling VM ID %d.\n",
Andrew Walbrana65a1322020-04-06 19:32:32 +01003319 relinquish_request->endpoints[0], from_locked.vm->id);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003320 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003321 }
3322
3323 dump_share_states();
3324
3325 share_states = share_states_lock();
Karl Meakin4a2854a2023-06-30 16:26:52 +01003326 share_state = get_share_state(share_states, handle);
J-Alvesb56aac82023-11-10 09:44:43 +00003327 if (share_state == NULL) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003328 dlog_verbose("Invalid handle %#x for FFA_MEM_RELINQUISH.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003329 handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003330 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003331 goto out;
3332 }
3333
Andrew Walbranca808b12020-05-15 17:22:28 +01003334 if (!share_state->sending_complete) {
3335 dlog_verbose(
3336 "Memory with handle %#x not fully sent, can't "
3337 "relinquish.\n",
3338 handle);
3339 ret = ffa_error(FFA_INVALID_PARAMETERS);
3340 goto out;
3341 }
3342
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003343 memory_region = share_state->memory_region;
3344 CHECK(memory_region != NULL);
3345
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003346 receiver_index = ffa_memory_region_get_receiver_index(
3347 memory_region, from_locked.vm->id);
J-Alves8eb19162022-04-28 10:56:48 +01003348
3349 if (receiver_index == memory_region->receiver_count) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003350 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01003351 "VM ID %d tried to relinquish memory region "
J-Alves668a86e2023-05-10 11:53:25 +01003352 "with handle %#x and it is not a valid borrower.\n",
J-Alves8eb19162022-04-28 10:56:48 +01003353 from_locked.vm->id, handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003354 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003355 goto out;
3356 }
3357
J-Alves8eb19162022-04-28 10:56:48 +01003358 if (share_state->retrieved_fragment_count[receiver_index] !=
Andrew Walbranca808b12020-05-15 17:22:28 +01003359 share_state->fragment_count) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003360 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01003361 "Memory with handle %#x not yet fully "
3362 "retrieved, "
J-Alves8eb19162022-04-28 10:56:48 +01003363 "receiver %x can't relinquish.\n",
3364 handle, from_locked.vm->id);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003365 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003366 goto out;
3367 }
3368
J-Alves3c5b2072022-11-21 12:45:40 +00003369 /*
3370 * Either clear if requested in relinquish call, or in a retrieve
3371 * request from one of the borrowers.
3372 */
3373 receivers_relinquished_memory = true;
3374
3375 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
3376 struct ffa_memory_access *receiver =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003377 ffa_memory_region_get_receiver(memory_region, i);
3378 assert(receiver != NULL);
J-Alves3c5b2072022-11-21 12:45:40 +00003379 if (receiver->receiver_permissions.receiver ==
3380 from_locked.vm->id) {
J-Alves639ddfc2023-11-21 14:17:26 +00003381 receiver_permissions =
3382 receiver->receiver_permissions.permissions;
J-Alves3c5b2072022-11-21 12:45:40 +00003383 continue;
3384 }
3385
3386 if (share_state->retrieved_fragment_count[i] != 0U) {
3387 receivers_relinquished_memory = false;
3388 break;
3389 }
3390 }
3391
3392 clear = receivers_relinquished_memory &&
3393 (share_state->clear_after_relinquish ||
3394 (relinquish_request->flags & FFA_MEMORY_REGION_FLAG_CLEAR) !=
3395 0U);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003396
3397 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01003398 * Clear is not allowed for memory that was shared, as the
3399 * original sender still has access to the memory.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003400 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003401 if (clear && share_state->share_func == FFA_MEM_SHARE_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003402 dlog_verbose("Memory which was shared can't be cleared.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003403 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003404 goto out;
3405 }
3406
J-Alves639ddfc2023-11-21 14:17:26 +00003407 if (clear && receiver_permissions == FFA_DATA_ACCESS_RO) {
3408 dlog_verbose("%s: RO memory can't use clear memory flag.\n",
3409 __func__);
3410 ret = ffa_error(FFA_DENIED);
3411 goto out;
3412 }
3413
Andrew Walbranca808b12020-05-15 17:22:28 +01003414 ret = ffa_relinquish_check_update(
J-Alves26483382023-04-20 12:01:49 +01003415 from_locked, share_state->fragments,
Andrew Walbranca808b12020-05-15 17:22:28 +01003416 share_state->fragment_constituent_counts,
3417 share_state->fragment_count, page_pool, clear);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003418
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003419 if (ret.func == FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003420 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01003421 * Mark memory handle as not retrieved, so it can be
3422 * reclaimed (or retrieved again).
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003423 */
J-Alves8eb19162022-04-28 10:56:48 +01003424 share_state->retrieved_fragment_count[receiver_index] = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003425 }
3426
3427out:
3428 share_states_unlock(&share_states);
3429 dump_share_states();
3430 return ret;
3431}
3432
3433/**
J-Alvesa9cd7e32022-07-01 13:49:33 +01003434 * Validates that the reclaim transition is allowed for the given
3435 * handle, updates the page table of the reclaiming VM, and frees the
3436 * internal state associated with the handle.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003437 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003438struct ffa_value ffa_memory_reclaim(struct vm_locked to_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01003439 ffa_memory_handle_t handle,
3440 ffa_memory_region_flags_t flags,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003441 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003442{
3443 struct share_states_locked share_states;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003444 struct ffa_memory_share_state *share_state;
3445 struct ffa_memory_region *memory_region;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003446 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003447
3448 dump_share_states();
3449
3450 share_states = share_states_lock();
Karl Meakin52cdfe72023-06-30 14:49:10 +01003451
Karl Meakin4a2854a2023-06-30 16:26:52 +01003452 share_state = get_share_state(share_states, handle);
J-Alvesb56aac82023-11-10 09:44:43 +00003453 if (share_state == NULL) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003454 dlog_verbose("Invalid handle %#x for FFA_MEM_RECLAIM.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003455 handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003456 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003457 goto out;
3458 }
Karl Meakin4a2854a2023-06-30 16:26:52 +01003459 memory_region = share_state->memory_region;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003460
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003461 CHECK(memory_region != NULL);
3462
J-Alvesa9cd7e32022-07-01 13:49:33 +01003463 if (vm_id_is_current_world(to_locked.vm->id) &&
3464 to_locked.vm->id != memory_region->sender) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003465 dlog_verbose(
Olivier Deprezf92e5d42020-11-13 16:00:54 +01003466 "VM %#x attempted to reclaim memory handle %#x "
3467 "originally sent by VM %#x.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003468 to_locked.vm->id, handle, memory_region->sender);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003469 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003470 goto out;
3471 }
3472
Andrew Walbranca808b12020-05-15 17:22:28 +01003473 if (!share_state->sending_complete) {
3474 dlog_verbose(
3475 "Memory with handle %#x not fully sent, can't "
3476 "reclaim.\n",
3477 handle);
3478 ret = ffa_error(FFA_INVALID_PARAMETERS);
3479 goto out;
3480 }
3481
J-Alves752236c2022-04-28 11:07:47 +01003482 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
3483 if (share_state->retrieved_fragment_count[i] != 0) {
3484 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01003485 "Tried to reclaim memory handle %#x "
J-Alves3c5b2072022-11-21 12:45:40 +00003486 "that has not been relinquished by all "
J-Alvesa9cd7e32022-07-01 13:49:33 +01003487 "borrowers(%x).\n",
J-Alves752236c2022-04-28 11:07:47 +01003488 handle,
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003489 ffa_memory_region_get_receiver(memory_region, i)
3490 ->receiver_permissions.receiver);
J-Alves752236c2022-04-28 11:07:47 +01003491 ret = ffa_error(FFA_DENIED);
3492 goto out;
3493 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003494 }
3495
Andrew Walbranca808b12020-05-15 17:22:28 +01003496 ret = ffa_retrieve_check_update(
J-Alves26483382023-04-20 12:01:49 +01003497 to_locked, share_state->fragments,
Andrew Walbranca808b12020-05-15 17:22:28 +01003498 share_state->fragment_constituent_counts,
J-Alves2a0d2882020-10-29 14:49:50 +00003499 share_state->fragment_count, share_state->sender_orig_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +01003500 FFA_MEM_RECLAIM_32, flags & FFA_MEM_RECLAIM_CLEAR, page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003501
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003502 if (ret.func == FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003503 share_state_free(share_states, share_state, page_pool);
J-Alves3c5b2072022-11-21 12:45:40 +00003504 dlog_verbose("Freed share state after successful reclaim.\n");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003505 }
3506
3507out:
3508 share_states_unlock(&share_states);
3509 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01003510}