blob: ed9fe17b0e5d2f9d13d307b067a50b2c0380bd4d [file] [log] [blame]
Jose Marinho75509b42019-04-09 09:34:59 +01001/*
2 * Copyright 2019 The Hafnium Authors.
3 *
Andrew Walbrane959ec12020-06-17 15:01:09 +01004 * Use of this source code is governed by a BSD-style
5 * license that can be found in the LICENSE file or at
6 * https://opensource.org/licenses/BSD-3-Clause.
Jose Marinho75509b42019-04-09 09:34:59 +01007 */
8
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01009#include "hf/ffa_memory.h"
Andrew Walbran475c1452020-02-07 13:22:22 +000010
J-Alves7b9cc432024-04-04 10:57:17 +010011#include "hf/arch/memcpy_trapped.h"
Federico Recanati4fd065d2021-12-13 20:06:23 +010012#include "hf/arch/mm.h"
Andrew Walbran290b0c92020-02-03 16:37:14 +000013
J-Alves5952d942022-12-22 16:03:00 +000014#include "hf/addr.h"
Jose Marinho09b1db82019-08-08 09:16:59 +010015#include "hf/check.h"
Jose Marinho75509b42019-04-09 09:34:59 +010016#include "hf/dlog.h"
J-Alves3456e032023-07-20 12:20:05 +010017#include "hf/ffa.h"
Karl Meakin902af082024-11-28 14:58:38 +000018#include "hf/ffa/ffa_memory.h"
19#include "hf/ffa/setup_and_discovery.h"
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010020#include "hf/ffa_internal.h"
J-Alves66652252022-07-06 09:49:51 +010021#include "hf/ffa_memory_internal.h"
J-Alves5952d942022-12-22 16:03:00 +000022#include "hf/mm.h"
Andrew Walbran475c1452020-02-07 13:22:22 +000023#include "hf/mpool.h"
J-Alvescf6253e2024-01-03 13:48:48 +000024#include "hf/panic.h"
25#include "hf/plat/memory_protect.h"
Jose Marinho75509b42019-04-09 09:34:59 +010026#include "hf/std.h"
Andrew Scull3c257452019-11-26 13:32:50 +000027#include "hf/vm.h"
Daniel Boulby44e9b3b2024-01-17 12:21:44 +000028#include "hf/vm_ids.h"
Jose Marinho75509b42019-04-09 09:34:59 +010029
J-Alves2d8457f2022-10-05 11:06:41 +010030#include "vmapi/hf/ffa_v1_0.h"
31
J-Alves5da37d92022-10-24 16:33:48 +010032#define RECEIVERS_COUNT_IN_RETRIEVE_RESP 1
33
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000034/**
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010035 * All access to members of a `struct ffa_memory_share_state` must be guarded
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000036 * by this lock.
37 */
38static struct spinlock share_states_lock_instance = SPINLOCK_INIT;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010039static struct ffa_memory_share_state share_states[MAX_MEM_SHARES];
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000040
41/**
J-Alvesed508c82023-05-04 16:09:48 +010042 * Return the offset to the first constituent within the
43 * `ffa_composite_memory_region` for the given receiver from an
44 * `ffa_memory_region`. The caller must check that the receiver_index is within
45 * bounds, and that it has a composite memory region offset.
46 */
47static uint32_t ffa_composite_constituent_offset(
48 struct ffa_memory_region *memory_region, uint32_t receiver_index)
49{
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +000050 struct ffa_memory_access *receiver;
51 uint32_t composite_offset;
J-Alvesed508c82023-05-04 16:09:48 +010052
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +000053 CHECK(receiver_index < memory_region->receiver_count);
54
55 receiver =
56 ffa_memory_region_get_receiver(memory_region, receiver_index);
57 CHECK(receiver != NULL);
58
59 composite_offset = receiver->composite_memory_region_offset;
60
61 CHECK(composite_offset != 0);
62
63 return composite_offset + sizeof(struct ffa_composite_memory_region);
J-Alvesed508c82023-05-04 16:09:48 +010064}
65
66/**
Karl Meakin52cdfe72023-06-30 14:49:10 +010067 * Initialises the next available `struct ffa_memory_share_state`. If `handle`
68 * is `FFA_MEMORY_HANDLE_INVALID` then allocates an appropriate handle,
69 * otherwise uses the provided handle which is assumed to be globally unique.
Andrew Walbranca808b12020-05-15 17:22:28 +010070 *
Karl Meakin52cdfe72023-06-30 14:49:10 +010071 * Returns a pointer to the allocated `ffa_memory_share_state` on success or
72 * `NULL` if none are available.
Andrew Walbranca808b12020-05-15 17:22:28 +010073 */
Karl Meakin52cdfe72023-06-30 14:49:10 +010074struct ffa_memory_share_state *allocate_share_state(
75 struct share_states_locked share_states, uint32_t share_func,
76 struct ffa_memory_region *memory_region, uint32_t fragment_length,
77 ffa_memory_handle_t handle)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000078{
Daniel Boulbya2f8c662021-11-26 17:52:53 +000079 assert(share_states.share_states != NULL);
80 assert(memory_region != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000081
Karl Meakin52cdfe72023-06-30 14:49:10 +010082 for (uint64_t i = 0; i < MAX_MEM_SHARES; ++i) {
Andrew Walbranca808b12020-05-15 17:22:28 +010083 if (share_states.share_states[i].share_func == 0) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010084 struct ffa_memory_share_state *allocated_state =
Andrew Walbranca808b12020-05-15 17:22:28 +010085 &share_states.share_states[i];
86 struct ffa_composite_memory_region *composite =
87 ffa_memory_region_get_composite(memory_region,
88 0);
89
90 if (handle == FFA_MEMORY_HANDLE_INVALID) {
J-Alvesee68c542020-10-29 17:48:20 +000091 memory_region->handle =
Karl Meakin117c8082024-12-04 16:03:28 +000092 ffa_memory_make_handle(i);
Andrew Walbranca808b12020-05-15 17:22:28 +010093 } else {
J-Alvesee68c542020-10-29 17:48:20 +000094 memory_region->handle = handle;
Andrew Walbranca808b12020-05-15 17:22:28 +010095 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000096 allocated_state->share_func = share_func;
97 allocated_state->memory_region = memory_region;
Andrew Walbranca808b12020-05-15 17:22:28 +010098 allocated_state->fragment_count = 1;
99 allocated_state->fragments[0] = composite->constituents;
100 allocated_state->fragment_constituent_counts[0] =
101 (fragment_length -
102 ffa_composite_constituent_offset(memory_region,
103 0)) /
104 sizeof(struct ffa_memory_region_constituent);
105 allocated_state->sending_complete = false;
Karl Meakin52cdfe72023-06-30 14:49:10 +0100106 for (uint32_t j = 0; j < MAX_MEM_SHARE_RECIPIENTS;
107 ++j) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100108 allocated_state->retrieved_fragment_count[j] =
109 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000110 }
Karl Meakin52cdfe72023-06-30 14:49:10 +0100111 return allocated_state;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000112 }
113 }
114
Karl Meakin52cdfe72023-06-30 14:49:10 +0100115 return NULL;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000116}
117
118/** Locks the share states lock. */
119struct share_states_locked share_states_lock(void)
120{
121 sl_lock(&share_states_lock_instance);
122
123 return (struct share_states_locked){.share_states = share_states};
124}
125
126/** Unlocks the share states lock. */
J-Alves66652252022-07-06 09:49:51 +0100127void share_states_unlock(struct share_states_locked *share_states)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000128{
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000129 assert(share_states->share_states != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000130 share_states->share_states = NULL;
131 sl_unlock(&share_states_lock_instance);
132}
133
134/**
Andrew Walbranca808b12020-05-15 17:22:28 +0100135 * If the given handle is a valid handle for an allocated share state then
Karl Meakin4a2854a2023-06-30 16:26:52 +0100136 * returns a pointer to the share state. Otherwise returns NULL.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000137 */
Karl Meakin4a2854a2023-06-30 16:26:52 +0100138struct ffa_memory_share_state *get_share_state(
139 struct share_states_locked share_states, ffa_memory_handle_t handle)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000140{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100141 struct ffa_memory_share_state *share_state;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000142
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000143 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100144
145 /*
146 * First look for a share_state allocated by us, in which case the
147 * handle is based on the index.
148 */
Karl Meakin117c8082024-12-04 16:03:28 +0000149 if (ffa_memory_is_handle_allocated_by_current_world(handle)) {
Karl Meakin1a760e72024-07-25 18:58:37 +0100150 uint64_t index = ffa_memory_handle_index(handle);
Karl Meakin4a2854a2023-06-30 16:26:52 +0100151
Andrew Walbranca808b12020-05-15 17:22:28 +0100152 if (index < MAX_MEM_SHARES) {
153 share_state = &share_states.share_states[index];
154 if (share_state->share_func != 0) {
Karl Meakin4a2854a2023-06-30 16:26:52 +0100155 return share_state;
Andrew Walbranca808b12020-05-15 17:22:28 +0100156 }
157 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000158 }
159
Andrew Walbranca808b12020-05-15 17:22:28 +0100160 /* Fall back to a linear scan. */
Karl Meakin4a2854a2023-06-30 16:26:52 +0100161 for (uint64_t index = 0; index < MAX_MEM_SHARES; ++index) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100162 share_state = &share_states.share_states[index];
J-Alvesee68c542020-10-29 17:48:20 +0000163 if (share_state->memory_region != NULL &&
164 share_state->memory_region->handle == handle &&
Andrew Walbranca808b12020-05-15 17:22:28 +0100165 share_state->share_func != 0) {
Karl Meakin4a2854a2023-06-30 16:26:52 +0100166 return share_state;
Andrew Walbranca808b12020-05-15 17:22:28 +0100167 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000168 }
169
Karl Meakin4a2854a2023-06-30 16:26:52 +0100170 return NULL;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000171}
172
173/** Marks a share state as unallocated. */
J-Alvesfdd29272022-07-19 13:16:31 +0100174void share_state_free(struct share_states_locked share_states,
175 struct ffa_memory_share_state *share_state,
176 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000177{
Andrew Walbranca808b12020-05-15 17:22:28 +0100178 uint32_t i;
179
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000180 assert(share_states.share_states != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000181 share_state->share_func = 0;
Andrew Walbranca808b12020-05-15 17:22:28 +0100182 share_state->sending_complete = false;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000183 mpool_free(page_pool, share_state->memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +0100184 /*
185 * First fragment is part of the same page as the `memory_region`, so it
186 * doesn't need to be freed separately.
187 */
188 share_state->fragments[0] = NULL;
189 share_state->fragment_constituent_counts[0] = 0;
190 for (i = 1; i < share_state->fragment_count; ++i) {
191 mpool_free(page_pool, share_state->fragments[i]);
192 share_state->fragments[i] = NULL;
193 share_state->fragment_constituent_counts[i] = 0;
194 }
195 share_state->fragment_count = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000196 share_state->memory_region = NULL;
J-Alvesa9cd7e32022-07-01 13:49:33 +0100197 share_state->hypervisor_fragment_count = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000198}
199
Andrew Walbranca808b12020-05-15 17:22:28 +0100200/** Checks whether the given share state has been fully sent. */
J-Alvesfdd29272022-07-19 13:16:31 +0100201bool share_state_sending_complete(struct share_states_locked share_states,
202 struct ffa_memory_share_state *share_state)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000203{
Andrew Walbranca808b12020-05-15 17:22:28 +0100204 struct ffa_composite_memory_region *composite;
205 uint32_t expected_constituent_count;
206 uint32_t fragment_constituent_count_total = 0;
207 uint32_t i;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000208
Andrew Walbranca808b12020-05-15 17:22:28 +0100209 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000210 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100211
212 /*
213 * Share state must already be valid, or it's not possible to get hold
214 * of it.
215 */
216 CHECK(share_state->memory_region != NULL &&
217 share_state->share_func != 0);
218
219 composite =
220 ffa_memory_region_get_composite(share_state->memory_region, 0);
221 expected_constituent_count = composite->constituent_count;
222 for (i = 0; i < share_state->fragment_count; ++i) {
223 fragment_constituent_count_total +=
224 share_state->fragment_constituent_counts[i];
225 }
226 dlog_verbose(
227 "Checking completion: constituent count %d/%d from %d "
228 "fragments.\n",
229 fragment_constituent_count_total, expected_constituent_count,
230 share_state->fragment_count);
231
232 return fragment_constituent_count_total == expected_constituent_count;
233}
234
235/**
236 * Calculates the offset of the next fragment expected for the given share
237 * state.
238 */
J-Alvesfdd29272022-07-19 13:16:31 +0100239uint32_t share_state_next_fragment_offset(
Andrew Walbranca808b12020-05-15 17:22:28 +0100240 struct share_states_locked share_states,
241 struct ffa_memory_share_state *share_state)
242{
243 uint32_t next_fragment_offset;
244 uint32_t i;
245
246 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000247 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100248
249 next_fragment_offset =
250 ffa_composite_constituent_offset(share_state->memory_region, 0);
251 for (i = 0; i < share_state->fragment_count; ++i) {
252 next_fragment_offset +=
253 share_state->fragment_constituent_counts[i] *
254 sizeof(struct ffa_memory_region_constituent);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000255 }
256
Andrew Walbranca808b12020-05-15 17:22:28 +0100257 return next_fragment_offset;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000258}
259
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100260static void dump_memory_region(struct ffa_memory_region *memory_region)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000261{
262 uint32_t i;
263
264 if (LOG_LEVEL < LOG_LEVEL_VERBOSE) {
265 return;
266 }
267
Karl Meakine8937d92024-03-19 16:04:25 +0000268 dlog("from VM %#x, attributes (shareability = %s, cacheability = %s, "
269 "type = %s, security = %s), flags %#x, handle %#lx "
270 "tag %lu, memory access descriptor size %u, to %u "
Andrew Walbrana65a1322020-04-06 19:32:32 +0100271 "recipients [",
Karl Meakine8937d92024-03-19 16:04:25 +0000272 memory_region->sender,
273 ffa_memory_shareability_name(
274 memory_region->attributes.shareability),
275 ffa_memory_cacheability_name(
276 memory_region->attributes.cacheability),
277 ffa_memory_type_name(memory_region->attributes.type),
278 ffa_memory_security_name(memory_region->attributes.security),
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000279 memory_region->flags, memory_region->handle, memory_region->tag,
280 memory_region->memory_access_desc_size,
Andrew Walbrana65a1322020-04-06 19:32:32 +0100281 memory_region->receiver_count);
282 for (i = 0; i < memory_region->receiver_count; ++i) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +0000283 struct ffa_memory_access *receiver =
284 ffa_memory_region_get_receiver(memory_region, i);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000285 if (i != 0) {
286 dlog(", ");
287 }
Karl Meakine8937d92024-03-19 16:04:25 +0000288 dlog("Receiver %#x: permissions (%s, %s) (offset %u)",
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +0000289 receiver->receiver_permissions.receiver,
Karl Meakine8937d92024-03-19 16:04:25 +0000290 ffa_data_access_name(receiver->receiver_permissions
291 .permissions.data_access),
292 ffa_instruction_access_name(
293 receiver->receiver_permissions.permissions
294 .instruction_access),
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +0000295 receiver->composite_memory_region_offset);
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000296 /* The impdef field is only present from v1.2 and later */
297 if (ffa_version_from_memory_access_desc_size(
298 memory_region->memory_access_desc_size) >=
Karl Meakin0e617d92024-04-05 12:55:22 +0100299 FFA_VERSION_1_2) {
Karl Meakine8937d92024-03-19 16:04:25 +0000300 dlog(", impdef: %#lx %#lx", receiver->impdef.val[0],
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000301 receiver->impdef.val[1]);
302 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000303 }
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000304 dlog("] at offset %u", memory_region->receivers_offset);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000305}
306
J-Alves66652252022-07-06 09:49:51 +0100307void dump_share_states(void)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000308{
309 uint32_t i;
310
311 if (LOG_LEVEL < LOG_LEVEL_VERBOSE) {
312 return;
313 }
314
315 dlog("Current share states:\n");
316 sl_lock(&share_states_lock_instance);
317 for (i = 0; i < MAX_MEM_SHARES; ++i) {
318 if (share_states[i].share_func != 0) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000319 switch (share_states[i].share_func) {
J-Alves95fbb312024-03-20 15:19:16 +0000320 case FFA_MEM_SHARE_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100321 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000322 dlog("SHARE");
323 break;
J-Alves95fbb312024-03-20 15:19:16 +0000324 case FFA_MEM_LEND_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100325 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000326 dlog("LEND");
327 break;
J-Alves95fbb312024-03-20 15:19:16 +0000328 case FFA_MEM_DONATE_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100329 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000330 dlog("DONATE");
331 break;
332 default:
333 dlog("invalid share_func %#x",
334 share_states[i].share_func);
335 }
Karl Meakine8937d92024-03-19 16:04:25 +0000336 dlog(" %#lx (", share_states[i].memory_region->handle);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000337 dump_memory_region(share_states[i].memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +0100338 if (share_states[i].sending_complete) {
339 dlog("): fully sent");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000340 } else {
Andrew Walbranca808b12020-05-15 17:22:28 +0100341 dlog("): partially sent");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000342 }
J-Alves2a0d2882020-10-29 14:49:50 +0000343 dlog(" with %d fragments, %d retrieved, "
344 " sender's original mode: %#x\n",
Andrew Walbranca808b12020-05-15 17:22:28 +0100345 share_states[i].fragment_count,
J-Alves2a0d2882020-10-29 14:49:50 +0000346 share_states[i].retrieved_fragment_count[0],
347 share_states[i].sender_orig_mode);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000348 }
349 }
350 sl_unlock(&share_states_lock_instance);
351}
352
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100353static inline uint32_t ffa_memory_permissions_to_mode(
J-Alves7cd5eb32020-10-16 19:06:10 +0100354 ffa_memory_access_permissions_t permissions, uint32_t default_mode)
Andrew Walbran475c1452020-02-07 13:22:22 +0000355{
Karl Meakin07a69ab2025-02-07 14:53:19 +0000356 mm_mode_t mode = 0;
Andrew Walbran475c1452020-02-07 13:22:22 +0000357
Karl Meakin84710f32023-10-12 15:14:49 +0100358 switch (permissions.data_access) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100359 case FFA_DATA_ACCESS_RO:
Andrew Walbran475c1452020-02-07 13:22:22 +0000360 mode = MM_MODE_R;
361 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100362 case FFA_DATA_ACCESS_RW:
Andrew Walbran475c1452020-02-07 13:22:22 +0000363 mode = MM_MODE_R | MM_MODE_W;
364 break;
J-Alves7cd5eb32020-10-16 19:06:10 +0100365 case FFA_DATA_ACCESS_NOT_SPECIFIED:
366 mode = (default_mode & (MM_MODE_R | MM_MODE_W));
367 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100368 case FFA_DATA_ACCESS_RESERVED:
369 panic("Tried to convert FFA_DATA_ACCESS_RESERVED.");
Karl Meakina5ea9092024-05-28 15:40:33 +0100370 default:
371 panic("Unknown data access %#x\n", permissions.data_access);
Andrew Walbrana65a1322020-04-06 19:32:32 +0100372 }
373
Karl Meakin84710f32023-10-12 15:14:49 +0100374 switch (permissions.instruction_access) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100375 case FFA_INSTRUCTION_ACCESS_NX:
Andrew Walbran475c1452020-02-07 13:22:22 +0000376 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100377 case FFA_INSTRUCTION_ACCESS_X:
Andrew Walbrana65a1322020-04-06 19:32:32 +0100378 mode |= MM_MODE_X;
379 break;
J-Alves7cd5eb32020-10-16 19:06:10 +0100380 case FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED:
381 mode |= (default_mode & MM_MODE_X);
382 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100383 case FFA_INSTRUCTION_ACCESS_RESERVED:
384 panic("Tried to convert FFA_INSTRUCTION_ACCESS_RESVERVED.");
Karl Meakina5ea9092024-05-28 15:40:33 +0100385 default:
386 panic("Unknown instruction access %#x\n",
387 permissions.instruction_access);
Andrew Walbran475c1452020-02-07 13:22:22 +0000388 }
389
Olivier Deprez878bd5b2021-04-15 19:05:10 +0200390 /* Set the security state bit if necessary. */
Karl Meakin117c8082024-12-04 16:03:28 +0000391 if ((default_mode & ffa_memory_get_other_world_mode()) != 0) {
392 mode |= ffa_memory_get_other_world_mode();
Olivier Deprez878bd5b2021-04-15 19:05:10 +0200393 }
394
Daniel Boulby6e261362024-06-13 16:53:00 +0100395 mode |= default_mode & MM_MODE_D;
396
Andrew Walbran475c1452020-02-07 13:22:22 +0000397 return mode;
398}
399
Jose Marinho75509b42019-04-09 09:34:59 +0100400/**
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000401 * Get the current mode in the stage-2 page table of the given vm of all the
402 * pages in the given constituents, if they all have the same mode, or return
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100403 * an appropriate FF-A error if not.
Jose Marinho75509b42019-04-09 09:34:59 +0100404 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100405static struct ffa_value constituents_get_mode(
Karl Meakin07a69ab2025-02-07 14:53:19 +0000406 struct vm_locked vm, mm_mode_t *orig_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100407 struct ffa_memory_region_constituent **fragments,
408 const uint32_t *fragment_constituent_counts, uint32_t fragment_count)
Jose Marinho75509b42019-04-09 09:34:59 +0100409{
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100410 uint32_t i;
Andrew Walbranca808b12020-05-15 17:22:28 +0100411 uint32_t j;
Jose Marinho75509b42019-04-09 09:34:59 +0100412
Andrew Walbranca808b12020-05-15 17:22:28 +0100413 if (fragment_count == 0 || fragment_constituent_counts[0] == 0) {
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100414 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000415 * Fail if there are no constituents. Otherwise we would get an
416 * uninitialised *orig_mode.
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100417 */
Karl Meakin5df422c2023-07-11 17:31:38 +0100418 dlog_verbose("%s: no constituents\n", __func__);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100419 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho75509b42019-04-09 09:34:59 +0100420 }
421
Andrew Walbranca808b12020-05-15 17:22:28 +0100422 for (i = 0; i < fragment_count; ++i) {
423 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
424 ipaddr_t begin = ipa_init(fragments[i][j].address);
425 size_t size = fragments[i][j].page_count * PAGE_SIZE;
426 ipaddr_t end = ipa_add(begin, size);
427 uint32_t current_mode;
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100428
Andrew Walbranca808b12020-05-15 17:22:28 +0100429 /* Fail if addresses are not page-aligned. */
430 if (!is_aligned(ipa_addr(begin), PAGE_SIZE) ||
431 !is_aligned(ipa_addr(end), PAGE_SIZE)) {
Karl Meakin5df422c2023-07-11 17:31:38 +0100432 dlog_verbose("%s: addresses not page-aligned\n",
433 __func__);
Andrew Walbranca808b12020-05-15 17:22:28 +0100434 return ffa_error(FFA_INVALID_PARAMETERS);
435 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100436
Andrew Walbranca808b12020-05-15 17:22:28 +0100437 /*
438 * Ensure that this constituent memory range is all
439 * mapped with the same mode.
440 */
Raghu Krishnamurthy785d52f2021-02-13 00:02:40 -0800441 if (!vm_mem_get_mode(vm, begin, end, &current_mode)) {
Karl Meakin5df422c2023-07-11 17:31:38 +0100442 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +0000443 "%s: constituent memory range "
444 "%#lx..%#lx "
Karl Meakin5df422c2023-07-11 17:31:38 +0100445 "not mapped with the same mode\n",
Karl Meakine8937d92024-03-19 16:04:25 +0000446 __func__, begin.ipa, end.ipa);
Andrew Walbranca808b12020-05-15 17:22:28 +0100447 return ffa_error(FFA_DENIED);
448 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100449
Andrew Walbranca808b12020-05-15 17:22:28 +0100450 /*
451 * Ensure that all constituents are mapped with the same
452 * mode.
453 */
454 if (i == 0) {
455 *orig_mode = current_mode;
456 } else if (current_mode != *orig_mode) {
457 dlog_verbose(
Karl Meakin5df422c2023-07-11 17:31:38 +0100458 "%s: expected mode %#x but was %#x for "
Karl Meakine8937d92024-03-19 16:04:25 +0000459 "%d pages at %#lx.\n",
Karl Meakin5df422c2023-07-11 17:31:38 +0100460 __func__, *orig_mode, current_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100461 fragments[i][j].page_count,
462 ipa_addr(begin));
463 return ffa_error(FFA_DENIED);
464 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100465 }
Jose Marinho75509b42019-04-09 09:34:59 +0100466 }
467
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100468 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000469}
470
Karl Meakin0e617d92024-04-05 12:55:22 +0100471enum ffa_version ffa_version_from_memory_access_desc_size(
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100472 uint32_t memory_access_desc_size)
473{
474 switch (memory_access_desc_size) {
475 /*
476 * v1.0 and v1.1 memory access descriptors are the same size however
477 * v1.1 is the first version to include the memory access descriptor
478 * size field so return v1.1.
479 */
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000480 case sizeof(struct ffa_memory_access_v1_0):
Karl Meakin0e617d92024-04-05 12:55:22 +0100481 return FFA_VERSION_1_1;
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000482 case sizeof(struct ffa_memory_access):
Karl Meakin0e617d92024-04-05 12:55:22 +0100483 return FFA_VERSION_1_2;
Karl Meakina5ea9092024-05-28 15:40:33 +0100484 default:
485 return 0;
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100486 }
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100487}
488
489/**
490 * Check if the receivers size and offset given is valid for the senders
491 * FF-A version.
492 */
493static bool receiver_size_and_offset_valid_for_version(
494 uint32_t receivers_size, uint32_t receivers_offset,
Karl Meakin0e617d92024-04-05 12:55:22 +0100495 enum ffa_version ffa_version)
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100496{
497 /*
498 * Check that the version that the memory access descriptor size belongs
499 * to is compatible with the FF-A version we believe the sender to be.
500 */
Karl Meakin0e617d92024-04-05 12:55:22 +0100501 enum ffa_version expected_ffa_version =
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100502 ffa_version_from_memory_access_desc_size(receivers_size);
Karl Meakin0e617d92024-04-05 12:55:22 +0100503 if (!ffa_versions_are_compatible(expected_ffa_version, ffa_version)) {
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100504 return false;
505 }
506
507 /*
508 * Check the receivers_offset matches the version we found from
509 * memory access descriptor size.
510 */
511 switch (expected_ffa_version) {
Karl Meakin0e617d92024-04-05 12:55:22 +0100512 case FFA_VERSION_1_1:
513 case FFA_VERSION_1_2:
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100514 return receivers_offset == sizeof(struct ffa_memory_region);
515 default:
516 return false;
517 }
518}
519
520/**
521 * Check the values set for fields in the memory region are valid and safe.
522 * Offset values are within safe bounds, receiver count will not cause overflows
523 * and reserved fields are 0.
524 */
525bool ffa_memory_region_sanity_check(struct ffa_memory_region *memory_region,
Karl Meakin0e617d92024-04-05 12:55:22 +0100526 enum ffa_version ffa_version,
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100527 uint32_t fragment_length,
528 bool send_transaction)
529{
530 uint32_t receiver_count;
531 struct ffa_memory_access *receiver;
532 uint32_t composite_offset_0;
Daniel Boulbyf06b5232024-02-22 16:26:43 +0000533 struct ffa_memory_region_v1_0 *memory_region_v1_0 =
534 (struct ffa_memory_region_v1_0 *)memory_region;
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100535
Karl Meakin0e617d92024-04-05 12:55:22 +0100536 if (ffa_version == FFA_VERSION_1_0) {
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100537 /* Check the reserved fields are 0. */
538 if (memory_region_v1_0->reserved_0 != 0 ||
539 memory_region_v1_0->reserved_1 != 0) {
540 dlog_verbose("Reserved fields must be 0.\n");
541 return false;
542 }
543
544 receiver_count = memory_region_v1_0->receiver_count;
545 } else {
546 uint32_t receivers_size =
547 memory_region->memory_access_desc_size;
548 uint32_t receivers_offset = memory_region->receivers_offset;
549
550 /* Check the reserved field is 0. */
551 if (memory_region->reserved[0] != 0 ||
552 memory_region->reserved[1] != 0 ||
553 memory_region->reserved[2] != 0) {
554 dlog_verbose("Reserved fields must be 0.\n");
555 return false;
556 }
557
558 /*
559 * Check memory_access_desc_size matches the size of the struct
560 * for the senders FF-A version.
561 */
562 if (!receiver_size_and_offset_valid_for_version(
563 receivers_size, receivers_offset, ffa_version)) {
564 dlog_verbose(
J-Alvesa0049022025-03-21 10:37:48 +0000565 "Invalid memory access descriptor size %d, or "
566 "receiver offset %d, for FF-A version %#x\n",
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100567 receivers_size, receivers_offset, ffa_version);
568 return false;
569 }
570
571 receiver_count = memory_region->receiver_count;
572 }
573
574 /* Check receiver count is not too large. */
Daniel Boulbyf06b5232024-02-22 16:26:43 +0000575 if (receiver_count > MAX_MEM_SHARE_RECIPIENTS || receiver_count < 1) {
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100576 dlog_verbose(
Daniel Boulbyf06b5232024-02-22 16:26:43 +0000577 "Receiver count must be 0 < receiver_count < %u "
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100578 "specified %u\n",
579 MAX_MEM_SHARE_RECIPIENTS, receiver_count);
580 return false;
581 }
582
583 /* Check values in the memory access descriptors. */
584 /*
585 * The composite offset values must be the same for all recievers so
586 * check the first one is valid and then they are all the same.
587 */
Karl Meakin0e617d92024-04-05 12:55:22 +0100588 receiver = ffa_version == FFA_VERSION_1_0
Daniel Boulbyf06b5232024-02-22 16:26:43 +0000589 ? (struct ffa_memory_access *)&memory_region_v1_0
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100590 ->receivers[0]
591 : ffa_memory_region_get_receiver(memory_region, 0);
592 assert(receiver != NULL);
593 composite_offset_0 = receiver->composite_memory_region_offset;
594
595 if (!send_transaction) {
596 if (composite_offset_0 != 0) {
597 dlog_verbose(
598 "Composite offset memory region descriptor "
599 "offset must be 0 for retrieve requests. "
600 "Currently %d",
601 composite_offset_0);
602 return false;
603 }
604 } else {
605 bool comp_offset_is_zero = composite_offset_0 == 0U;
606 bool comp_offset_lt_transaction_descriptor_size =
607 composite_offset_0 <
608 (sizeof(struct ffa_memory_region) +
Karl Meakin66a38bd2024-05-28 16:00:56 +0100609 (size_t)(memory_region->memory_access_desc_size *
610 memory_region->receiver_count));
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100611 bool comp_offset_with_comp_gt_fragment_length =
612 composite_offset_0 +
613 sizeof(struct ffa_composite_memory_region) >
614 fragment_length;
615 if (comp_offset_is_zero ||
616 comp_offset_lt_transaction_descriptor_size ||
617 comp_offset_with_comp_gt_fragment_length) {
618 dlog_verbose(
619 "Invalid composite memory region descriptor "
620 "offset for send transaction %u\n",
621 composite_offset_0);
622 return false;
623 }
624 }
625
Karl Meakin824b63d2024-06-03 19:04:53 +0100626 for (size_t i = 0; i < memory_region->receiver_count; i++) {
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100627 uint32_t composite_offset;
628
Karl Meakin0e617d92024-04-05 12:55:22 +0100629 if (ffa_version == FFA_VERSION_1_0) {
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100630 struct ffa_memory_access_v1_0 *receiver_v1_0 =
631 &memory_region_v1_0->receivers[i];
632 /* Check reserved fields are 0 */
633 if (receiver_v1_0->reserved_0 != 0) {
634 dlog_verbose(
635 "Reserved field in the memory access "
Karl Meakine8937d92024-03-19 16:04:25 +0000636 "descriptor must be zero. Currently "
637 "reciever %zu has a reserved field "
638 "with a value of %lu\n",
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100639 i, receiver_v1_0->reserved_0);
640 return false;
641 }
642 /*
643 * We can cast to the current version receiver as the
644 * remaining fields we are checking have the same
645 * offsets for all versions since memory access
646 * descriptors are forwards compatible.
647 */
648 receiver = (struct ffa_memory_access *)receiver_v1_0;
649 } else {
650 receiver = ffa_memory_region_get_receiver(memory_region,
651 i);
652 assert(receiver != NULL);
653
Daniel Boulbyfd374b82024-07-31 14:31:16 +0100654 if (ffa_version == FFA_VERSION_1_1) {
655 /*
656 * Since the reserved field is at the end of the
657 * Endpoint Memory Access Descriptor we must
658 * cast to ffa_memory_access_v1_0 as they match.
659 * Since all fields except reserved in the
660 * Endpoint Memory Access Descriptor have the
661 * same offsets across all versions this cast is
662 * not required when accessing other fields in
663 * the future.
664 */
665 struct ffa_memory_access_v1_0 *receiver_v1_0 =
666 (struct ffa_memory_access_v1_0 *)
667 receiver;
668 if (receiver_v1_0->reserved_0 != 0) {
669 dlog_verbose(
670 "Reserved field in the memory "
671 "access descriptor must be "
672 "zero. Currently reciever %zu "
673 "has a reserved field with a "
674 "value of %lu\n",
675 i, receiver_v1_0->reserved_0);
676 return false;
677 }
678
679 } else {
680 if (receiver->reserved_0 != 0) {
681 dlog_verbose(
682 "Reserved field in the memory "
683 "access descriptor must be "
684 "zero. Currently reciever %zu "
685 "has a reserved field with a "
686 "value of %lu\n",
687 i, receiver->reserved_0);
688 return false;
689 }
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100690 }
691 }
692
693 /* Check composite offset values are equal for all receivers. */
694 composite_offset = receiver->composite_memory_region_offset;
695 if (composite_offset != composite_offset_0) {
696 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +0000697 "Composite offset %x differs from %x in "
698 "index\n",
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100699 composite_offset, composite_offset_0);
700 return false;
701 }
702 }
703 return true;
704}
705
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000706/**
J-Alves460d36c2023-10-12 17:02:15 +0100707 * If the receivers for the memory management operation are all from the
Daniel Boulby734981e2024-07-22 11:06:35 +0100708 * secure world, the memory is not device memory (as it isn't covered by the
709 * granule page table) and this isn't a FFA_MEM_SHARE, then request memory
710 * security state update by returning MAP_ACTION_CHECK_PROTECT.
J-Alves460d36c2023-10-12 17:02:15 +0100711 */
712static enum ffa_map_action ffa_mem_send_get_map_action(
713 bool all_receivers_from_current_world, ffa_id_t sender_id,
Daniel Boulby734981e2024-07-22 11:06:35 +0100714 uint32_t mem_func_id, bool is_normal_memory)
J-Alves460d36c2023-10-12 17:02:15 +0100715{
J-Alves95fbb312024-03-20 15:19:16 +0000716 const bool is_memory_share_abi = mem_func_id == FFA_MEM_SHARE_32 ||
717 mem_func_id == FFA_MEM_SHARE_64;
718 const bool protect_memory =
719 (!is_memory_share_abi && all_receivers_from_current_world &&
Daniel Boulby734981e2024-07-22 11:06:35 +0100720 ffa_is_vm_id(sender_id) && is_normal_memory);
J-Alves460d36c2023-10-12 17:02:15 +0100721
722 return protect_memory ? MAP_ACTION_CHECK_PROTECT : MAP_ACTION_CHECK;
723}
724
725/**
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000726 * Verify that all pages have the same mode, that the starting mode
727 * constitutes a valid state and obtain the next mode to apply
J-Alves460d36c2023-10-12 17:02:15 +0100728 * to the sending VM. It outputs the mapping action that needs to be
729 * invoked for the given memory range. On memory lend/donate there
730 * could be a need to protect the memory from the normal world.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000731 *
732 * Returns:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100733 * 1) FFA_DENIED if a state transition was not found;
734 * 2) FFA_DENIED if the pages being shared do not have the same mode within
Andrew Walbrana65a1322020-04-06 19:32:32 +0100735 * the <from> VM;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100736 * 3) FFA_INVALID_PARAMETERS if the beginning and end IPAs are not page
Andrew Walbrana65a1322020-04-06 19:32:32 +0100737 * aligned;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100738 * 4) FFA_INVALID_PARAMETERS if the requested share type was not handled.
739 * Or FFA_SUCCESS on success.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000740 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100741static struct ffa_value ffa_send_check_transition(
Andrew Walbrana65a1322020-04-06 19:32:32 +0100742 struct vm_locked from, uint32_t share_func,
Karl Meakin07a69ab2025-02-07 14:53:19 +0000743 struct ffa_memory_region *memory_region, mm_mode_t *orig_from_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100744 struct ffa_memory_region_constituent **fragments,
745 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
Karl Meakin07a69ab2025-02-07 14:53:19 +0000746 mm_mode_t *from_mode, enum ffa_map_action *map_action, bool zero)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000747{
Karl Meakin07a69ab2025-02-07 14:53:19 +0000748 const mm_mode_t state_mask =
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000749 MM_MODE_INVALID | MM_MODE_UNOWNED | MM_MODE_SHARED;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100750 struct ffa_value ret;
J-Alves460d36c2023-10-12 17:02:15 +0100751 bool all_receivers_from_current_world = true;
Daniel Boulbya76fd912024-02-22 14:22:15 +0000752 uint32_t receivers_count = memory_region->receiver_count;
J-Alves95fbb312024-03-20 15:19:16 +0000753 const bool is_memory_lend = (share_func == FFA_MEM_LEND_32) ||
754 (share_func == FFA_MEM_LEND_64);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000755
Andrew Walbranca808b12020-05-15 17:22:28 +0100756 ret = constituents_get_mode(from, orig_from_mode, fragments,
757 fragment_constituent_counts,
758 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100759 if (ret.func != FFA_SUCCESS_32) {
Olivier Depreze7eb1682022-03-16 17:09:03 +0100760 dlog_verbose("Inconsistent modes.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100761 return ret;
Andrew Scullb5f49e02019-10-02 13:20:47 +0100762 }
763
Daniel Boulby6e261362024-06-13 16:53:00 +0100764 /*
765 * Check requested memory type is valid with the memory type of the
766 * owner. E.g. they follow the memory type precedence where Normal
767 * memory is more permissive than device and therefore device memory
768 * can only be shared as device memory.
769 */
770 if (memory_region->attributes.type == FFA_MEMORY_NORMAL_MEM &&
771 (*orig_from_mode & MM_MODE_D) != 0U) {
772 dlog_verbose(
773 "Send device memory as Normal memory is not allowed\n");
774 return ffa_error(FFA_DENIED);
775 }
776
Daniel Boulby63af1fa2024-03-18 14:17:31 +0000777 /* Device memory regions can only be lent a single borrower. */
Daniel Boulby9764ff62024-01-30 17:47:39 +0000778 if ((*orig_from_mode & MM_MODE_D) != 0U &&
J-Alves95fbb312024-03-20 15:19:16 +0000779 !(is_memory_lend && receivers_count == 1)) {
Daniel Boulby9764ff62024-01-30 17:47:39 +0000780 dlog_verbose(
Daniel Boulby63af1fa2024-03-18 14:17:31 +0000781 "Device memory can only be lent to a single borrower "
782 "(mode is %#x).\n",
Daniel Boulby9764ff62024-01-30 17:47:39 +0000783 *orig_from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100784 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000785 }
786
787 /*
788 * Ensure the sender is the owner and has exclusive access to the
789 * memory.
790 */
791 if ((*orig_from_mode & state_mask) != 0) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100792 return ffa_error(FFA_DENIED);
Andrew Walbrana65a1322020-04-06 19:32:32 +0100793 }
794
Daniel Boulby4b846eb2024-05-23 17:32:23 +0100795 /*
796 * Memory cannot be zeroed during the lend/donate operation if the
797 * sender only has RO access.
798 */
799 if ((*orig_from_mode & MM_MODE_W) == 0 && zero == true) {
800 dlog_verbose(
801 "Cannot zero memory when the sender doesn't have "
802 "write access\n");
803 return ffa_error(FFA_DENIED);
804 }
805
Daniel Boulbya76fd912024-02-22 14:22:15 +0000806 assert(receivers_count > 0U);
J-Alves7cd5eb32020-10-16 19:06:10 +0100807
J-Alves363f5722022-04-25 17:37:37 +0100808 for (uint32_t i = 0U; i < receivers_count; i++) {
Daniel Boulbya76fd912024-02-22 14:22:15 +0000809 struct ffa_memory_access *receiver =
810 ffa_memory_region_get_receiver(memory_region, i);
811 assert(receiver != NULL);
J-Alves363f5722022-04-25 17:37:37 +0100812 ffa_memory_access_permissions_t permissions =
Daniel Boulbya76fd912024-02-22 14:22:15 +0000813 receiver->receiver_permissions.permissions;
J-Alves363f5722022-04-25 17:37:37 +0100814 uint32_t required_from_mode = ffa_memory_permissions_to_mode(
815 permissions, *orig_from_mode);
816
J-Alves788b4492023-04-18 14:01:23 +0100817 /*
818 * The assumption is that at this point, the operation from
819 * SP to a receiver VM, should have returned an FFA_ERROR
820 * already.
821 */
822 if (!ffa_is_vm_id(from.vm->id)) {
823 assert(!ffa_is_vm_id(
Daniel Boulbya76fd912024-02-22 14:22:15 +0000824 receiver->receiver_permissions.receiver));
J-Alves788b4492023-04-18 14:01:23 +0100825 }
826
J-Alves460d36c2023-10-12 17:02:15 +0100827 /* Track if all senders are from current world. */
828 all_receivers_from_current_world =
829 all_receivers_from_current_world &&
830 vm_id_is_current_world(
Daniel Boulbya76fd912024-02-22 14:22:15 +0000831 receiver->receiver_permissions.receiver);
J-Alves460d36c2023-10-12 17:02:15 +0100832
J-Alves363f5722022-04-25 17:37:37 +0100833 if ((*orig_from_mode & required_from_mode) !=
834 required_from_mode) {
835 dlog_verbose(
836 "Sender tried to send memory with permissions "
J-Alves788b4492023-04-18 14:01:23 +0100837 "which required mode %#x but only had %#x "
838 "itself.\n",
J-Alves363f5722022-04-25 17:37:37 +0100839 required_from_mode, *orig_from_mode);
840 return ffa_error(FFA_DENIED);
841 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000842 }
843
J-Alves460d36c2023-10-12 17:02:15 +0100844 *map_action = ffa_mem_send_get_map_action(
Daniel Boulby734981e2024-07-22 11:06:35 +0100845 all_receivers_from_current_world, from.vm->id, share_func,
846 (*orig_from_mode & MM_MODE_D) == 0U);
J-Alves460d36c2023-10-12 17:02:15 +0100847
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000848 /* Find the appropriate new mode. */
849 *from_mode = ~state_mask & *orig_from_mode;
Andrew Walbrane7ad3c02019-12-24 17:03:04 +0000850 switch (share_func) {
J-Alves95fbb312024-03-20 15:19:16 +0000851 case FFA_MEM_DONATE_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100852 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000853 *from_mode |= MM_MODE_INVALID | MM_MODE_UNOWNED;
Jose Marinho75509b42019-04-09 09:34:59 +0100854 break;
J-Alves95fbb312024-03-20 15:19:16 +0000855 case FFA_MEM_LEND_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100856 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000857 *from_mode |= MM_MODE_INVALID;
Andrew Walbran648fc3e2019-10-22 16:23:05 +0100858 break;
J-Alves95fbb312024-03-20 15:19:16 +0000859 case FFA_MEM_SHARE_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100860 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000861 *from_mode |= MM_MODE_SHARED;
Jose Marinho56c25732019-05-20 09:48:53 +0100862 break;
863
Jose Marinho75509b42019-04-09 09:34:59 +0100864 default:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100865 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho75509b42019-04-09 09:34:59 +0100866 }
867
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100868 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000869}
870
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100871static struct ffa_value ffa_relinquish_check_transition(
Karl Meakin07a69ab2025-02-07 14:53:19 +0000872 struct vm_locked from, mm_mode_t *orig_from_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100873 struct ffa_memory_region_constituent **fragments,
874 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
Karl Meakin07a69ab2025-02-07 14:53:19 +0000875 mm_mode_t *from_mode, enum ffa_map_action *map_action)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000876{
877 const uint32_t state_mask =
878 MM_MODE_INVALID | MM_MODE_UNOWNED | MM_MODE_SHARED;
879 uint32_t orig_from_state;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100880 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000881
J-Alves69cdfd92024-04-26 11:40:59 +0100882 assert(map_action != NULL);
883 if (vm_id_is_current_world(from.vm->id)) {
884 *map_action = MAP_ACTION_COMMIT;
885 } else {
886 /*
887 * No need to check the attributes of caller.
888 * The assumption is that the retrieve request of the receiver
889 * also used the MAP_ACTION_NONE, and no update was done to the
890 * page tables. When the receiver is not at the secure virtual
891 * instance SPMC doesn't manage its S2 translation (i.e. when
892 * the receiver is a VM).
893 */
894 *map_action = MAP_ACTION_NONE;
895
896 return (struct ffa_value){.func = FFA_SUCCESS_32};
897 }
898
Andrew Walbranca808b12020-05-15 17:22:28 +0100899 ret = constituents_get_mode(from, orig_from_mode, fragments,
900 fragment_constituent_counts,
901 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100902 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbrana65a1322020-04-06 19:32:32 +0100903 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000904 }
905
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000906 /*
907 * Ensure the relinquishing VM is not the owner but has access to the
908 * memory.
909 */
910 orig_from_state = *orig_from_mode & state_mask;
911 if ((orig_from_state & ~MM_MODE_SHARED) != MM_MODE_UNOWNED) {
912 dlog_verbose(
913 "Tried to relinquish memory in state %#x (masked %#x "
Andrew Walbranca808b12020-05-15 17:22:28 +0100914 "but should be %#x).\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000915 *orig_from_mode, orig_from_state, MM_MODE_UNOWNED);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100916 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000917 }
918
919 /* Find the appropriate new mode. */
920 *from_mode = (~state_mask & *orig_from_mode) | MM_MODE_UNMAPPED_MASK;
921
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100922 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000923}
924
925/**
926 * Verify that all pages have the same mode, that the starting mode
927 * constitutes a valid state and obtain the next mode to apply
928 * to the retrieving VM.
929 *
930 * Returns:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100931 * 1) FFA_DENIED if a state transition was not found;
932 * 2) FFA_DENIED if the pages being shared do not have the same mode within
Andrew Walbrana65a1322020-04-06 19:32:32 +0100933 * the <to> VM;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100934 * 3) FFA_INVALID_PARAMETERS if the beginning and end IPAs are not page
Andrew Walbrana65a1322020-04-06 19:32:32 +0100935 * aligned;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100936 * 4) FFA_INVALID_PARAMETERS if the requested share type was not handled.
937 * Or FFA_SUCCESS on success.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000938 */
J-Alvesfc19b372022-07-06 12:17:35 +0100939struct ffa_value ffa_retrieve_check_transition(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000940 struct vm_locked to, uint32_t share_func,
Andrew Walbranca808b12020-05-15 17:22:28 +0100941 struct ffa_memory_region_constituent **fragments,
942 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
Karl Meakin07a69ab2025-02-07 14:53:19 +0000943 mm_mode_t sender_orig_mode, mm_mode_t *to_mode, bool memory_protected,
J-Alvesfd206052023-05-22 16:45:00 +0100944 enum ffa_map_action *map_action)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000945{
Karl Meakin07a69ab2025-02-07 14:53:19 +0000946 mm_mode_t orig_to_mode;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100947 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000948
Andrew Walbranca808b12020-05-15 17:22:28 +0100949 ret = constituents_get_mode(to, &orig_to_mode, fragments,
950 fragment_constituent_counts,
951 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100952 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100953 dlog_verbose("Inconsistent modes.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100954 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000955 }
956
J-Alves460d36c2023-10-12 17:02:15 +0100957 /* Find the appropriate new mode. */
Daniel Boulby71d887b2024-06-28 16:38:06 +0100958 *to_mode = sender_orig_mode;
J-Alves460d36c2023-10-12 17:02:15 +0100959
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100960 if (share_func == FFA_MEM_RECLAIM_32) {
J-Alves9256f162021-12-09 13:18:43 +0000961 /*
962 * If the original ffa memory send call has been processed
963 * successfully, it is expected the orig_to_mode would overlay
964 * with `state_mask`, as a result of the function
965 * `ffa_send_check_transition`.
J-Alvesfd206052023-05-22 16:45:00 +0100966 *
967 * If Hafnium is the SPMC:
968 * - Caller of the reclaim interface is an SP, the memory shall
969 * have been protected throughout the flow.
970 * - Caller of the reclaim is from the NWd, the memory may have
971 * been protected at the time of lending/donating the memory.
972 * In such case, set action to unprotect memory in the
973 * handling of reclaim operation.
974 * - If Hafnium is the hypervisor memory shall never have been
975 * protected in memory lend/share/donate.
976 *
977 * More details in the doc comment of the function
978 * `ffa_region_group_identity_map`.
J-Alves9256f162021-12-09 13:18:43 +0000979 */
J-Alves59ed0042022-07-28 18:26:41 +0100980 if (vm_id_is_current_world(to.vm->id)) {
981 assert((orig_to_mode &
982 (MM_MODE_INVALID | MM_MODE_UNOWNED |
983 MM_MODE_SHARED)) != 0U);
J-Alvesfd206052023-05-22 16:45:00 +0100984 assert(!memory_protected);
985 } else if (to.vm->id == HF_OTHER_WORLD_ID &&
986 map_action != NULL && memory_protected) {
987 *map_action = MAP_ACTION_COMMIT_UNPROTECT;
J-Alves59ed0042022-07-28 18:26:41 +0100988 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000989 } else {
J-Alves69cdfd92024-04-26 11:40:59 +0100990 if (!vm_id_is_current_world(to.vm->id)) {
991 assert(map_action != NULL);
992 *map_action = MAP_ACTION_NONE;
993 return (struct ffa_value){.func = FFA_SUCCESS_32};
994 }
995
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000996 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +0100997 * If the retriever is from virtual FF-A instance:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000998 * Ensure the retriever has the expected state. We don't care
999 * about the MM_MODE_SHARED bit; either with or without it set
1000 * are both valid representations of the !O-NA state.
1001 */
J-Alvesa9cd7e32022-07-01 13:49:33 +01001002 if (vm_id_is_current_world(to.vm->id) &&
Karl Meakin5e996992024-05-20 11:27:07 +01001003 !vm_is_primary(to.vm) &&
J-Alvesa9cd7e32022-07-01 13:49:33 +01001004 (orig_to_mode & MM_MODE_UNMAPPED_MASK) !=
1005 MM_MODE_UNMAPPED_MASK) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001006 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001007 }
J-Alves460d36c2023-10-12 17:02:15 +01001008
1009 /*
1010 * If memory has been protected before, clear the NS bit to
1011 * allow the secure access from the SP.
1012 */
1013 if (memory_protected) {
Karl Meakin117c8082024-12-04 16:03:28 +00001014 *to_mode &= ~ffa_memory_get_other_world_mode();
J-Alves460d36c2023-10-12 17:02:15 +01001015 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001016 }
1017
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001018 switch (share_func) {
J-Alves95fbb312024-03-20 15:19:16 +00001019 case FFA_MEM_DONATE_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001020 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001021 *to_mode |= 0;
1022 break;
J-Alves95fbb312024-03-20 15:19:16 +00001023 case FFA_MEM_LEND_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001024 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001025 *to_mode |= MM_MODE_UNOWNED;
1026 break;
J-Alves95fbb312024-03-20 15:19:16 +00001027 case FFA_MEM_SHARE_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001028 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001029 *to_mode |= MM_MODE_UNOWNED | MM_MODE_SHARED;
1030 break;
1031
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001032 case FFA_MEM_RECLAIM_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001033 *to_mode |= 0;
1034 break;
1035
1036 default:
Andrew Walbranca808b12020-05-15 17:22:28 +01001037 dlog_error("Invalid share_func %#x.\n", share_func);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001038 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001039 }
1040
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001041 return (struct ffa_value){.func = FFA_SUCCESS_32};
Jose Marinho75509b42019-04-09 09:34:59 +01001042}
Jose Marinho09b1db82019-08-08 09:16:59 +01001043
J-Alvescf6253e2024-01-03 13:48:48 +00001044/*
1045 * Performs the operations related to the `action` MAP_ACTION_CHECK*.
1046 * Returns:
1047 * - FFA_SUCCESS_32: if all goes well.
1048 * - FFA_ERROR_32: with FFA_NO_MEMORY, if there is no memory to manage
1049 * the page table update. Or error code provided by the function
1050 * `arch_memory_protect`.
1051 */
1052static struct ffa_value ffa_region_group_check_actions(
1053 struct vm_locked vm_locked, paddr_t pa_begin, paddr_t pa_end,
Karl Meakin07a69ab2025-02-07 14:53:19 +00001054 struct mpool *ppool, mm_mode_t mode, enum ffa_map_action action,
J-Alvescf6253e2024-01-03 13:48:48 +00001055 bool *memory_protected)
1056{
1057 struct ffa_value ret;
1058 bool is_memory_protected;
1059
1060 if (!vm_identity_prepare(vm_locked, pa_begin, pa_end, mode, ppool)) {
1061 dlog_verbose(
1062 "%s: memory can't be mapped to %x due to lack of "
Karl Meakine8937d92024-03-19 16:04:25 +00001063 "memory. Base: %lx end: %lx\n",
J-Alvescf6253e2024-01-03 13:48:48 +00001064 __func__, vm_locked.vm->id, pa_addr(pa_begin),
1065 pa_addr(pa_end));
1066 return ffa_error(FFA_NO_MEMORY);
1067 }
1068
1069 switch (action) {
1070 case MAP_ACTION_CHECK:
1071 /* No protect requested. */
1072 is_memory_protected = false;
1073 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
1074 break;
1075 case MAP_ACTION_CHECK_PROTECT: {
1076 paddr_t last_protected_pa = pa_init(0);
1077
1078 ret = arch_memory_protect(pa_begin, pa_end, &last_protected_pa);
1079
1080 is_memory_protected = (ret.func == FFA_SUCCESS_32);
1081
1082 /*
1083 * - If protect memory has failed with FFA_DENIED, means some
1084 * range of memory was in the wrong state. In such case, SPM
1085 * reverts the state of the pages that were successfully
1086 * updated.
1087 * - If protect memory has failed with FFA_NOT_SUPPORTED, it
1088 * means the platform doesn't support the protection mechanism.
1089 * That said, it still permits the page table update to go
1090 * through. The variable
1091 * `is_memory_protected` will be equal to false.
1092 * - If protect memory has failed with FFA_INVALID_PARAMETERS,
1093 * break from switch and return the error.
1094 */
1095 if (ret.func == FFA_ERROR_32) {
1096 assert(!is_memory_protected);
1097 if (ffa_error_code(ret) == FFA_DENIED &&
1098 pa_addr(last_protected_pa) != (uintptr_t)0) {
1099 CHECK(arch_memory_unprotect(
1100 pa_begin,
1101 pa_add(last_protected_pa, PAGE_SIZE)));
1102 } else if (ffa_error_code(ret) == FFA_NOT_SUPPORTED) {
1103 ret = (struct ffa_value){
1104 .func = FFA_SUCCESS_32,
1105 };
1106 }
1107 }
1108 } break;
1109 default:
1110 panic("%s: invalid action to process %x\n", __func__, action);
1111 }
1112
1113 if (memory_protected != NULL) {
1114 *memory_protected = is_memory_protected;
1115 }
1116
1117 return ret;
1118}
1119
1120static void ffa_region_group_commit_actions(struct vm_locked vm_locked,
1121 paddr_t pa_begin, paddr_t pa_end,
Karl Meakin07a69ab2025-02-07 14:53:19 +00001122 struct mpool *ppool, mm_mode_t mode,
J-Alvescf6253e2024-01-03 13:48:48 +00001123 enum ffa_map_action action)
1124{
1125 switch (action) {
1126 case MAP_ACTION_COMMIT_UNPROTECT:
1127 /*
1128 * Checking that it should succeed because SPM should be
1129 * unprotecting memory that it had protected before.
1130 */
1131 CHECK(arch_memory_unprotect(pa_begin, pa_end));
1132 case MAP_ACTION_COMMIT:
1133 vm_identity_commit(vm_locked, pa_begin, pa_end, mode, ppool,
1134 NULL);
1135 break;
1136 default:
1137 panic("%s: invalid action to process %x\n", __func__, action);
1138 }
1139}
1140
Jose Marinho09b1db82019-08-08 09:16:59 +01001141/**
J-Alves063ad832023-10-03 18:05:40 +01001142 * Helper function to revert a failed "Protect" action from the SPMC:
1143 * - `fragment_count`: should specify the number of fragments to traverse from
1144 * `fragments`. This may not be the full amount of fragments that are part of
1145 * the share_state structure.
1146 * - `fragment_constituent_counts`: array holding the amount of constituents
1147 * per fragment.
1148 * - `end`: pointer to the constituent that failed the "protect" action. It
1149 * shall be part of the last fragment, and it shall make the loop below break.
1150 */
1151static void ffa_region_group_fragments_revert_protect(
1152 struct ffa_memory_region_constituent **fragments,
1153 const uint32_t *fragment_constituent_counts, uint32_t fragment_count,
1154 const struct ffa_memory_region_constituent *end)
1155{
1156 for (uint32_t i = 0; i < fragment_count; ++i) {
1157 for (uint32_t j = 0; j < fragment_constituent_counts[i]; ++j) {
1158 struct ffa_memory_region_constituent *constituent =
1159 &fragments[i][j];
1160 size_t size = constituent->page_count * PAGE_SIZE;
1161 paddr_t pa_begin =
1162 pa_from_ipa(ipa_init(constituent->address));
1163 paddr_t pa_end = pa_add(pa_begin, size);
1164
Karl Meakine8937d92024-03-19 16:04:25 +00001165 dlog_verbose("%s: reverting fragment %lx size %zx\n",
J-Alves063ad832023-10-03 18:05:40 +01001166 __func__, pa_addr(pa_begin), size);
1167
1168 if (constituent == end) {
1169 /*
1170 * The last constituent is expected to be in the
1171 * last fragment.
1172 */
1173 assert(i == fragment_count - 1);
1174 break;
1175 }
1176
1177 CHECK(arch_memory_unprotect(pa_begin, pa_end));
1178 }
1179 }
1180}
1181
1182/**
Jose Marinho09b1db82019-08-08 09:16:59 +01001183 * Updates a VM's page table such that the given set of physical address ranges
1184 * are mapped in the address space at the corresponding address ranges, in the
1185 * mode provided.
1186 *
J-Alves0a83dc22023-05-05 09:50:37 +01001187 * The enum ffa_map_action determines the action taken from a call to the
1188 * function below:
1189 * - If action is MAP_ACTION_CHECK, the page tables will be allocated from the
1190 * mpool but no mappings will actually be updated. This function must always
1191 * be called first with action set to MAP_ACTION_CHECK to check that it will
1192 * succeed before calling ffa_region_group_identity_map with whichever one of
1193 * the remaining actions, to avoid leaving the page table in a half-updated
1194 * state.
1195 * - The action MAP_ACTION_COMMIT allocates the page tables from the mpool, and
1196 * changes the memory mappings.
J-Alvescf6253e2024-01-03 13:48:48 +00001197 * - The action MAP_ACTION_CHECK_PROTECT extends the MAP_ACTION_CHECK with an
1198 * invocation to the monitor to update the security state of the memory,
1199 * to that of the SPMC.
1200 * - The action MAP_ACTION_COMMIT_UNPROTECT extends the MAP_ACTION_COMMIT
1201 * with a call into the monitor, to reset the security state of memory
1202 * that has priorly been mapped with the MAP_ACTION_CHECK_PROTECT action.
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001203 * vm_ptable_defrag should always be called after a series of page table
1204 * updates, whether they succeed or fail.
Jose Marinho09b1db82019-08-08 09:16:59 +01001205 *
J-Alvescf6253e2024-01-03 13:48:48 +00001206 * If all goes well, returns FFA_SUCCESS_32; or FFA_ERROR, with following
1207 * error codes:
1208 * - FFA_INVALID_PARAMETERS: invalid range of memory.
1209 * - FFA_DENIED:
1210 *
Jose Marinho09b1db82019-08-08 09:16:59 +01001211 * made to memory mappings.
1212 */
J-Alvescf6253e2024-01-03 13:48:48 +00001213struct ffa_value ffa_region_group_identity_map(
Andrew Walbranf4b51af2020-02-03 14:44:54 +00001214 struct vm_locked vm_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01001215 struct ffa_memory_region_constituent **fragments,
1216 const uint32_t *fragment_constituent_counts, uint32_t fragment_count,
Karl Meakin07a69ab2025-02-07 14:53:19 +00001217 mm_mode_t mode, struct mpool *ppool, enum ffa_map_action action,
J-Alvescf6253e2024-01-03 13:48:48 +00001218 bool *memory_protected)
Jose Marinho09b1db82019-08-08 09:16:59 +01001219{
Andrew Walbranca808b12020-05-15 17:22:28 +01001220 uint32_t i;
1221 uint32_t j;
J-Alvescf6253e2024-01-03 13:48:48 +00001222 struct ffa_value ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01001223
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001224 if (vm_locked.vm->el0_partition) {
1225 mode |= MM_MODE_USER | MM_MODE_NG;
1226 }
1227
Andrew Walbranca808b12020-05-15 17:22:28 +01001228 /* Iterate over the memory region constituents within each fragment. */
1229 for (i = 0; i < fragment_count; ++i) {
1230 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
J-Alves063ad832023-10-03 18:05:40 +01001231 struct ffa_memory_region_constituent *constituent =
1232 &fragments[i][j];
1233 size_t size = constituent->page_count * PAGE_SIZE;
Andrew Walbranca808b12020-05-15 17:22:28 +01001234 paddr_t pa_begin =
J-Alves063ad832023-10-03 18:05:40 +01001235 pa_from_ipa(ipa_init(constituent->address));
Andrew Walbranca808b12020-05-15 17:22:28 +01001236 paddr_t pa_end = pa_add(pa_begin, size);
Jens Wiklander4f1880c2022-10-19 17:00:14 +02001237 uint32_t pa_bits =
1238 arch_mm_get_pa_bits(arch_mm_get_pa_range());
Federico Recanati4fd065d2021-12-13 20:06:23 +01001239
1240 /*
1241 * Ensure the requested region falls into system's PA
1242 * range.
1243 */
Jens Wiklander4f1880c2022-10-19 17:00:14 +02001244 if (((pa_addr(pa_begin) >> pa_bits) > 0) ||
1245 ((pa_addr(pa_end) >> pa_bits) > 0)) {
Federico Recanati4fd065d2021-12-13 20:06:23 +01001246 dlog_error("Region is outside of PA Range\n");
J-Alvescf6253e2024-01-03 13:48:48 +00001247 return ffa_error(FFA_INVALID_PARAMETERS);
Federico Recanati4fd065d2021-12-13 20:06:23 +01001248 }
Andrew Walbranca808b12020-05-15 17:22:28 +01001249
J-Alvescf6253e2024-01-03 13:48:48 +00001250 if (action <= MAP_ACTION_CHECK_PROTECT) {
1251 ret = ffa_region_group_check_actions(
1252 vm_locked, pa_begin, pa_end, ppool,
1253 mode, action, memory_protected);
J-Alves063ad832023-10-03 18:05:40 +01001254
1255 if (ret.func == FFA_ERROR_32 &&
1256 ffa_error_code(ret) == FFA_DENIED) {
1257 if (memory_protected != NULL) {
1258 assert(!*memory_protected);
1259 }
1260
1261 ffa_region_group_fragments_revert_protect(
1262 fragments,
1263 fragment_constituent_counts,
1264 i + 1, constituent);
1265 break;
1266 }
J-Alvescf6253e2024-01-03 13:48:48 +00001267 } else if (action >= MAP_ACTION_COMMIT &&
1268 action < MAP_ACTION_MAX) {
1269 ffa_region_group_commit_actions(
1270 vm_locked, pa_begin, pa_end, ppool,
1271 mode, action);
1272 ret = (struct ffa_value){
1273 .func = FFA_SUCCESS_32};
1274 } else {
1275 panic("%s: Unknown ffa_map_action.\n",
1276 __func__);
Andrew Walbranca808b12020-05-15 17:22:28 +01001277 }
Jose Marinho09b1db82019-08-08 09:16:59 +01001278 }
1279 }
1280
J-Alvescf6253e2024-01-03 13:48:48 +00001281 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01001282}
1283
1284/**
1285 * Clears a region of physical memory by overwriting it with zeros. The data is
1286 * flushed from the cache so the memory has been cleared across the system.
1287 */
J-Alves7db32002021-12-14 14:44:50 +00001288static bool clear_memory(paddr_t begin, paddr_t end, struct mpool *ppool,
Karl Meakin07a69ab2025-02-07 14:53:19 +00001289 mm_mode_t extra_mode)
Jose Marinho09b1db82019-08-08 09:16:59 +01001290{
1291 /*
Fuad Tabbaed294af2019-12-20 10:43:01 +00001292 * TODO: change this to a CPU local single page window rather than a
Jose Marinho09b1db82019-08-08 09:16:59 +01001293 * global mapping of the whole range. Such an approach will limit
1294 * the changes to stage-1 tables and will allow only local
1295 * invalidation.
1296 */
1297 bool ret;
1298 struct mm_stage1_locked stage1_locked = mm_lock_stage1();
Karl Meakin07a69ab2025-02-07 14:53:19 +00001299 void *ptr = mm_identity_map(
1300 stage1_locked, begin, end,
1301 MM_MODE_W | (extra_mode & ffa_memory_get_other_world_mode()),
1302 ppool);
Jose Marinho09b1db82019-08-08 09:16:59 +01001303 size_t size = pa_difference(begin, end);
1304
1305 if (!ptr) {
Jose Marinho09b1db82019-08-08 09:16:59 +01001306 goto fail;
1307 }
1308
1309 memset_s(ptr, size, 0, size);
1310 arch_mm_flush_dcache(ptr, size);
1311 mm_unmap(stage1_locked, begin, end, ppool);
1312
1313 ret = true;
1314 goto out;
1315
1316fail:
1317 ret = false;
1318
1319out:
1320 mm_unlock_stage1(&stage1_locked);
1321
1322 return ret;
1323}
1324
1325/**
1326 * Clears a region of physical memory by overwriting it with zeros. The data is
1327 * flushed from the cache so the memory has been cleared across the system.
1328 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001329static bool ffa_clear_memory_constituents(
Karl Meakin07a69ab2025-02-07 14:53:19 +00001330 mm_mode_t security_state_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +01001331 struct ffa_memory_region_constituent **fragments,
1332 const uint32_t *fragment_constituent_counts, uint32_t fragment_count,
1333 struct mpool *page_pool)
Jose Marinho09b1db82019-08-08 09:16:59 +01001334{
1335 struct mpool local_page_pool;
Andrew Walbranca808b12020-05-15 17:22:28 +01001336 uint32_t i;
Jose Marinho09b1db82019-08-08 09:16:59 +01001337 bool ret = false;
1338
1339 /*
1340 * Create a local pool so any freed memory can't be used by another
1341 * thread. This is to ensure each constituent that is mapped can be
1342 * unmapped again afterwards.
1343 */
Andrew Walbran475c1452020-02-07 13:22:22 +00001344 mpool_init_with_fallback(&local_page_pool, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +01001345
Andrew Walbranca808b12020-05-15 17:22:28 +01001346 /* Iterate over the memory region constituents within each fragment. */
1347 for (i = 0; i < fragment_count; ++i) {
1348 uint32_t j;
Jose Marinho09b1db82019-08-08 09:16:59 +01001349
J-Alves8457f932023-10-11 16:41:45 +01001350 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001351 size_t size = fragments[i][j].page_count * PAGE_SIZE;
1352 paddr_t begin =
1353 pa_from_ipa(ipa_init(fragments[i][j].address));
1354 paddr_t end = pa_add(begin, size);
1355
J-Alves7db32002021-12-14 14:44:50 +00001356 if (!clear_memory(begin, end, &local_page_pool,
1357 security_state_mode)) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001358 /*
1359 * api_clear_memory will defrag on failure, so
1360 * no need to do it here.
1361 */
1362 goto out;
1363 }
Jose Marinho09b1db82019-08-08 09:16:59 +01001364 }
1365 }
1366
Jose Marinho09b1db82019-08-08 09:16:59 +01001367 ret = true;
1368
1369out:
1370 mpool_fini(&local_page_pool);
1371 return ret;
1372}
1373
J-Alves5952d942022-12-22 16:03:00 +00001374static bool is_memory_range_within(ipaddr_t begin, ipaddr_t end,
1375 ipaddr_t in_begin, ipaddr_t in_end)
1376{
1377 return (ipa_addr(begin) >= ipa_addr(in_begin) &&
1378 ipa_addr(begin) < ipa_addr(in_end)) ||
1379 (ipa_addr(end) <= ipa_addr(in_end) &&
1380 ipa_addr(end) > ipa_addr(in_begin));
1381}
1382
1383/**
1384 * Receives a memory range and looks for overlaps with the remainder
1385 * constituents of the memory share/lend/donate operation. Assumes they are
1386 * passed in order to avoid having to loop over all the elements at each call.
1387 * The function only compares the received memory ranges with those that follow
1388 * within the same fragment, and subsequent fragments from the same operation.
1389 */
1390static bool ffa_memory_check_overlap(
1391 struct ffa_memory_region_constituent **fragments,
1392 const uint32_t *fragment_constituent_counts,
1393 const uint32_t fragment_count, const uint32_t current_fragment,
1394 const uint32_t current_constituent)
1395{
1396 uint32_t i = current_fragment;
1397 uint32_t j = current_constituent;
1398 ipaddr_t current_begin = ipa_init(fragments[i][j].address);
1399 const uint32_t current_page_count = fragments[i][j].page_count;
1400 size_t current_size = current_page_count * PAGE_SIZE;
1401 ipaddr_t current_end = ipa_add(current_begin, current_size - 1);
1402
1403 if (current_size == 0 ||
1404 current_size > UINT64_MAX - ipa_addr(current_begin)) {
Karl Meakine8937d92024-03-19 16:04:25 +00001405 dlog_verbose("Invalid page count. Addr: %zx page_count: %x\n",
1406 current_begin.ipa, current_page_count);
J-Alves5952d942022-12-22 16:03:00 +00001407 return false;
1408 }
1409
1410 for (; i < fragment_count; i++) {
1411 j = (i == current_fragment) ? j + 1 : 0;
1412
1413 for (; j < fragment_constituent_counts[i]; j++) {
1414 ipaddr_t begin = ipa_init(fragments[i][j].address);
1415 const uint32_t page_count = fragments[i][j].page_count;
1416 size_t size = page_count * PAGE_SIZE;
1417 ipaddr_t end = ipa_add(begin, size - 1);
1418
1419 if (size == 0 || size > UINT64_MAX - ipa_addr(begin)) {
1420 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00001421 "Invalid page count. Addr: %lx "
J-Alves5952d942022-12-22 16:03:00 +00001422 "page_count: %x\n",
Karl Meakine8937d92024-03-19 16:04:25 +00001423 begin.ipa, page_count);
J-Alves5952d942022-12-22 16:03:00 +00001424 return false;
1425 }
1426
1427 /*
1428 * Check if current ranges is within begin and end, as
1429 * well as the reverse. This should help optimize the
1430 * loop, and reduce the number of iterations.
1431 */
1432 if (is_memory_range_within(begin, end, current_begin,
1433 current_end) ||
1434 is_memory_range_within(current_begin, current_end,
1435 begin, end)) {
1436 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00001437 "Overlapping memory ranges: %#lx - "
1438 "%#lx with %#lx - %#lx\n",
J-Alves5952d942022-12-22 16:03:00 +00001439 ipa_addr(begin), ipa_addr(end),
1440 ipa_addr(current_begin),
1441 ipa_addr(current_end));
1442 return true;
1443 }
1444 }
1445 }
1446
1447 return false;
1448}
1449
Jose Marinho09b1db82019-08-08 09:16:59 +01001450/**
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001451 * Validates and prepares memory to be sent from the calling VM to another.
Jose Marinho09b1db82019-08-08 09:16:59 +01001452 *
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001453 * This function requires the calling context to hold the <from> VM lock.
Jose Marinho09b1db82019-08-08 09:16:59 +01001454 *
1455 * Returns:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001456 * In case of error, one of the following values is returned:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001457 * 1) FFA_INVALID_PARAMETERS - The endpoint provided parameters were
Jose Marinho09b1db82019-08-08 09:16:59 +01001458 * erroneous;
Andrew Walbranf07f04d2020-05-01 18:09:00 +01001459 * 2) FFA_NO_MEMORY - Hafnium did not have sufficient memory to complete the
1460 * request.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001461 * 3) FFA_DENIED - The sender doesn't have sufficient access to send the
Andrew Walbrana65a1322020-04-06 19:32:32 +01001462 * memory with the given permissions.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001463 * Success is indicated by FFA_SUCCESS.
Jose Marinho09b1db82019-08-08 09:16:59 +01001464 */
Daniel Boulbya76fd912024-02-22 14:22:15 +00001465static struct ffa_value ffa_send_check_update(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001466 struct vm_locked from_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01001467 struct ffa_memory_region_constituent **fragments,
1468 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
J-Alves8f11cde2022-12-21 16:18:22 +00001469 uint32_t composite_total_page_count, uint32_t share_func,
Daniel Boulbya76fd912024-02-22 14:22:15 +00001470 struct ffa_memory_region *memory_region, struct mpool *page_pool,
Karl Meakin07a69ab2025-02-07 14:53:19 +00001471 mm_mode_t *orig_from_mode_ret, bool *memory_protected)
Jose Marinho09b1db82019-08-08 09:16:59 +01001472{
Andrew Walbranca808b12020-05-15 17:22:28 +01001473 uint32_t i;
J-Alves8f11cde2022-12-21 16:18:22 +00001474 uint32_t j;
Karl Meakin07a69ab2025-02-07 14:53:19 +00001475 mm_mode_t orig_from_mode;
1476 mm_mode_t clean_mode;
1477 mm_mode_t from_mode;
Jose Marinho09b1db82019-08-08 09:16:59 +01001478 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001479 struct ffa_value ret;
J-Alves8f11cde2022-12-21 16:18:22 +00001480 uint32_t constituents_total_page_count = 0;
J-Alves460d36c2023-10-12 17:02:15 +01001481 enum ffa_map_action map_action = MAP_ACTION_CHECK;
Daniel Boulbya76fd912024-02-22 14:22:15 +00001482 bool clear = memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR;
Jose Marinho09b1db82019-08-08 09:16:59 +01001483
1484 /*
Andrew Walbrana65a1322020-04-06 19:32:32 +01001485 * Make sure constituents are properly aligned to a 64-bit boundary. If
1486 * not we would get alignment faults trying to read (64-bit) values.
Jose Marinho09b1db82019-08-08 09:16:59 +01001487 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001488 for (i = 0; i < fragment_count; ++i) {
1489 if (!is_aligned(fragments[i], 8)) {
1490 dlog_verbose("Constituents not aligned.\n");
1491 return ffa_error(FFA_INVALID_PARAMETERS);
1492 }
J-Alves8f11cde2022-12-21 16:18:22 +00001493 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
1494 constituents_total_page_count +=
1495 fragments[i][j].page_count;
J-Alves5952d942022-12-22 16:03:00 +00001496 if (ffa_memory_check_overlap(
1497 fragments, fragment_constituent_counts,
1498 fragment_count, i, j)) {
1499 return ffa_error(FFA_INVALID_PARAMETERS);
1500 }
J-Alves8f11cde2022-12-21 16:18:22 +00001501 }
1502 }
1503
1504 if (constituents_total_page_count != composite_total_page_count) {
1505 dlog_verbose(
1506 "Composite page count differs from calculated page "
1507 "count from constituents.\n");
1508 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho09b1db82019-08-08 09:16:59 +01001509 }
1510
1511 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001512 * Check if the state transition is lawful for the sender, ensure that
1513 * all constituents of a memory region being shared are at the same
1514 * state.
Jose Marinho09b1db82019-08-08 09:16:59 +01001515 */
J-Alves460d36c2023-10-12 17:02:15 +01001516 ret = ffa_send_check_transition(
Daniel Boulbya76fd912024-02-22 14:22:15 +00001517 from_locked, share_func, memory_region, &orig_from_mode,
1518 fragments, fragment_constituent_counts, fragment_count,
Daniel Boulby4b846eb2024-05-23 17:32:23 +01001519 &from_mode, &map_action, clear);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001520 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001521 dlog_verbose("Invalid transition for send.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +01001522 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01001523 }
1524
Andrew Walbran37c574e2020-06-03 11:45:46 +01001525 if (orig_from_mode_ret != NULL) {
1526 *orig_from_mode_ret = orig_from_mode;
1527 }
1528
Jose Marinho09b1db82019-08-08 09:16:59 +01001529 /*
1530 * Create a local pool so any freed memory can't be used by another
1531 * thread. This is to ensure the original mapping can be restored if the
1532 * clear fails.
1533 */
Andrew Walbran475c1452020-02-07 13:22:22 +00001534 mpool_init_with_fallback(&local_page_pool, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +01001535
1536 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001537 * First reserve all required memory for the new page table entries
1538 * without committing, to make sure the entire operation will succeed
1539 * without exhausting the page pool.
J-Alves460d36c2023-10-12 17:02:15 +01001540 * Provide the map_action as populated by 'ffa_send_check_transition'.
1541 * It may request memory to be protected.
Jose Marinho09b1db82019-08-08 09:16:59 +01001542 */
J-Alvescf6253e2024-01-03 13:48:48 +00001543 ret = ffa_region_group_identity_map(
1544 from_locked, fragments, fragment_constituent_counts,
J-Alves460d36c2023-10-12 17:02:15 +01001545 fragment_count, from_mode, page_pool, map_action,
1546 memory_protected);
J-Alvescf6253e2024-01-03 13:48:48 +00001547 if (ret.func == FFA_ERROR_32) {
Jose Marinho09b1db82019-08-08 09:16:59 +01001548 goto out;
1549 }
1550
1551 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001552 * Update the mapping for the sender. This won't allocate because the
1553 * transaction was already prepared above, but may free pages in the
1554 * case that a whole block is being unmapped that was previously
1555 * partially mapped.
Jose Marinho09b1db82019-08-08 09:16:59 +01001556 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001557 CHECK(ffa_region_group_identity_map(
J-Alvescf6253e2024-01-03 13:48:48 +00001558 from_locked, fragments, fragment_constituent_counts,
1559 fragment_count, from_mode, &local_page_pool,
1560 MAP_ACTION_COMMIT, NULL)
1561 .func == FFA_SUCCESS_32);
Jose Marinho09b1db82019-08-08 09:16:59 +01001562
J-Alves460d36c2023-10-12 17:02:15 +01001563 /*
1564 * If memory has been protected, it is now part of the secure PAS
1565 * (happens for lend/donate from NWd to SWd), and the `orig_from_mode`
1566 * should have the MM_MODE_NS set, as such mask it in `clean_mode` for
1567 * SPM's S1 translation.
1568 * In case memory hasn't been protected, and it is in the non-secure
1569 * PAS (e.g. memory share from NWd to SWd), as such the SPM needs to
1570 * perform a non-secure memory access. In such case `clean_mode` takes
1571 * the same mode as `orig_from_mode`.
1572 */
Karl Meakin117c8082024-12-04 16:03:28 +00001573 clean_mode =
1574 (memory_protected != NULL && *memory_protected)
1575 ? orig_from_mode & ~ffa_memory_get_other_world_mode()
1576 : orig_from_mode;
J-Alves460d36c2023-10-12 17:02:15 +01001577
Jose Marinho09b1db82019-08-08 09:16:59 +01001578 /* Clear the memory so no VM or device can see the previous contents. */
J-Alves460d36c2023-10-12 17:02:15 +01001579 if (clear && !ffa_clear_memory_constituents(
1580 clean_mode, fragments, fragment_constituent_counts,
1581 fragment_count, page_pool)) {
1582 map_action = (memory_protected != NULL && *memory_protected)
1583 ? MAP_ACTION_COMMIT_UNPROTECT
1584 : MAP_ACTION_COMMIT;
1585
Jose Marinho09b1db82019-08-08 09:16:59 +01001586 /*
1587 * On failure, roll back by returning memory to the sender. This
1588 * may allocate pages which were previously freed into
1589 * `local_page_pool` by the call above, but will never allocate
1590 * more pages than that so can never fail.
1591 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001592 CHECK(ffa_region_group_identity_map(
J-Alvescf6253e2024-01-03 13:48:48 +00001593 from_locked, fragments,
1594 fragment_constituent_counts, fragment_count,
1595 orig_from_mode, &local_page_pool,
1596 MAP_ACTION_COMMIT, NULL)
1597 .func == FFA_SUCCESS_32);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001598 ret = ffa_error(FFA_NO_MEMORY);
Jose Marinho09b1db82019-08-08 09:16:59 +01001599 goto out;
1600 }
1601
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001602 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001603
1604out:
1605 mpool_fini(&local_page_pool);
1606
1607 /*
1608 * Tidy up the page table by reclaiming failed mappings (if there was an
1609 * error) or merging entries into blocks where possible (on success).
1610 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001611 vm_ptable_defrag(from_locked, page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001612
1613 return ret;
1614}
1615
1616/**
1617 * Validates and maps memory shared from one VM to another.
1618 *
1619 * This function requires the calling context to hold the <to> lock.
1620 *
1621 * Returns:
1622 * In case of error, one of the following values is returned:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001623 * 1) FFA_INVALID_PARAMETERS - The endpoint provided parameters were
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001624 * erroneous;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001625 * 2) FFA_NO_MEMORY - Hafnium did not have sufficient memory to complete
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001626 * the request.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001627 * Success is indicated by FFA_SUCCESS.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001628 */
J-Alvesb5084cf2022-07-06 14:20:12 +01001629struct ffa_value ffa_retrieve_check_update(
J-Alves26483382023-04-20 12:01:49 +01001630 struct vm_locked to_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01001631 struct ffa_memory_region_constituent **fragments,
1632 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
Karl Meakin07a69ab2025-02-07 14:53:19 +00001633 mm_mode_t sender_orig_mode, uint32_t share_func, bool clear,
1634 struct mpool *page_pool, mm_mode_t *response_mode,
1635 bool memory_protected)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001636{
Andrew Walbranca808b12020-05-15 17:22:28 +01001637 uint32_t i;
Karl Meakin07a69ab2025-02-07 14:53:19 +00001638 mm_mode_t to_mode;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001639 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001640 struct ffa_value ret;
J-Alvesfd206052023-05-22 16:45:00 +01001641 enum ffa_map_action map_action = MAP_ACTION_COMMIT;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001642
1643 /*
Andrew Walbranca808b12020-05-15 17:22:28 +01001644 * Make sure constituents are properly aligned to a 64-bit boundary. If
1645 * not we would get alignment faults trying to read (64-bit) values.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001646 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001647 for (i = 0; i < fragment_count; ++i) {
1648 if (!is_aligned(fragments[i], 8)) {
J-Alvesb5084cf2022-07-06 14:20:12 +01001649 dlog_verbose("Fragment not properly aligned.\n");
Andrew Walbranca808b12020-05-15 17:22:28 +01001650 return ffa_error(FFA_INVALID_PARAMETERS);
1651 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001652 }
1653
1654 /*
Daniel Boulby4b846eb2024-05-23 17:32:23 +01001655 * Ensure the sender has write permissions if the memory needs to be
1656 * cleared.
1657 */
1658 if ((sender_orig_mode & MM_MODE_W) == 0 && clear == true) {
1659 dlog_verbose(
1660 "Cannot zero memory when the sender does not have "
1661 "write access\n");
1662 return ffa_error(FFA_DENIED);
1663 }
1664
1665 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001666 * Check if the state transition is lawful for the recipient, and ensure
1667 * that all constituents of the memory region being retrieved are at the
1668 * same state.
1669 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001670 ret = ffa_retrieve_check_transition(
1671 to_locked, share_func, fragments, fragment_constituent_counts,
J-Alvesfd206052023-05-22 16:45:00 +01001672 fragment_count, sender_orig_mode, &to_mode, memory_protected,
1673 &map_action);
J-Alves460d36c2023-10-12 17:02:15 +01001674
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001675 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001676 dlog_verbose("Invalid transition for retrieve.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +01001677 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001678 }
1679
1680 /*
J-Alves69cdfd92024-04-26 11:40:59 +01001681 * Create a local pool so any freed memory can't be used by
1682 * another thread. This is to ensure the original mapping can be
1683 * restored if the clear fails.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001684 */
1685 mpool_init_with_fallback(&local_page_pool, page_pool);
1686
1687 /*
J-Alves69cdfd92024-04-26 11:40:59 +01001688 * Memory retrieves from the NWd VMs don't require update to S2 PTs on
1689 * retrieve request.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001690 */
J-Alves69cdfd92024-04-26 11:40:59 +01001691 if (map_action != MAP_ACTION_NONE) {
1692 /*
1693 * First reserve all required memory for the new page table
1694 * entries in the recipient page tables without committing, to
1695 * make sure the entire operation will succeed without
1696 * exhausting the page pool.
1697 */
1698 ret = ffa_region_group_identity_map(
1699 to_locked, fragments, fragment_constituent_counts,
1700 fragment_count, to_mode, page_pool, MAP_ACTION_CHECK,
1701 NULL);
1702 if (ret.func == FFA_ERROR_32) {
1703 /* TODO: partial defrag of failed range. */
1704 goto out;
1705 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001706 }
1707
1708 /* Clear the memory so no VM or device can see the previous contents. */
J-Alves7db32002021-12-14 14:44:50 +00001709 if (clear &&
J-Alves26483382023-04-20 12:01:49 +01001710 !ffa_clear_memory_constituents(sender_orig_mode, fragments,
1711 fragment_constituent_counts,
1712 fragment_count, page_pool)) {
J-Alvesb5084cf2022-07-06 14:20:12 +01001713 dlog_verbose("Couldn't clear constituents.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001714 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001715 goto out;
1716 }
1717
J-Alves69cdfd92024-04-26 11:40:59 +01001718 if (map_action != MAP_ACTION_NONE) {
1719 /*
1720 * Complete the transfer by mapping the memory into the
1721 * recipient. This won't allocate because the transaction was
1722 * already prepared above, so it doesn't need to use the
1723 * `local_page_pool`.
1724 */
1725 CHECK(ffa_region_group_identity_map(to_locked, fragments,
1726 fragment_constituent_counts,
1727 fragment_count, to_mode,
1728 page_pool, map_action, NULL)
1729 .func == FFA_SUCCESS_32);
Jose Marinho09b1db82019-08-08 09:16:59 +01001730
J-Alves69cdfd92024-04-26 11:40:59 +01001731 /*
1732 * Return the mode used in mapping the memory in retriever's PT.
1733 */
1734 if (response_mode != NULL) {
1735 *response_mode = to_mode;
1736 }
J-Alves460d36c2023-10-12 17:02:15 +01001737 }
1738
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001739 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Jose Marinho09b1db82019-08-08 09:16:59 +01001740
1741out:
1742 mpool_fini(&local_page_pool);
1743
1744 /*
Andrew Walbranf07f04d2020-05-01 18:09:00 +01001745 * Tidy up the page table by reclaiming failed mappings (if there was an
1746 * error) or merging entries into blocks where possible (on success).
Jose Marinho09b1db82019-08-08 09:16:59 +01001747 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001748 vm_ptable_defrag(to_locked, page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001749
1750 return ret;
1751}
1752
Andrew Walbran996d1d12020-05-27 14:08:43 +01001753static struct ffa_value ffa_relinquish_check_update(
J-Alves26483382023-04-20 12:01:49 +01001754 struct vm_locked from_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01001755 struct ffa_memory_region_constituent **fragments,
1756 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
Karl Meakin07a69ab2025-02-07 14:53:19 +00001757 mm_mode_t sender_orig_mode, struct mpool *page_pool, bool clear)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001758{
Karl Meakin07a69ab2025-02-07 14:53:19 +00001759 mm_mode_t orig_from_mode;
1760 mm_mode_t clearing_mode;
1761 mm_mode_t from_mode;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001762 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001763 struct ffa_value ret;
J-Alves69cdfd92024-04-26 11:40:59 +01001764 enum ffa_map_action map_action;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001765
Andrew Walbranca808b12020-05-15 17:22:28 +01001766 ret = ffa_relinquish_check_transition(
1767 from_locked, &orig_from_mode, fragments,
J-Alves69cdfd92024-04-26 11:40:59 +01001768 fragment_constituent_counts, fragment_count, &from_mode,
1769 &map_action);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001770 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001771 dlog_verbose("Invalid transition for relinquish.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +01001772 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001773 }
1774
1775 /*
1776 * Create a local pool so any freed memory can't be used by another
1777 * thread. This is to ensure the original mapping can be restored if the
1778 * clear fails.
1779 */
1780 mpool_init_with_fallback(&local_page_pool, page_pool);
1781
J-Alves69cdfd92024-04-26 11:40:59 +01001782 if (map_action != MAP_ACTION_NONE) {
1783 clearing_mode = orig_from_mode;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001784
J-Alves69cdfd92024-04-26 11:40:59 +01001785 /*
1786 * First reserve all required memory for the new page table
1787 * entries without committing, to make sure the entire operation
1788 * will succeed without exhausting the page pool.
1789 */
1790 ret = ffa_region_group_identity_map(
1791 from_locked, fragments, fragment_constituent_counts,
1792 fragment_count, from_mode, page_pool, MAP_ACTION_CHECK,
1793 NULL);
1794 if (ret.func == FFA_ERROR_32) {
1795 goto out;
1796 }
1797
1798 /*
1799 * Update the mapping for the sender. This won't allocate
1800 * because the transaction was already prepared above, but may
1801 * free pages in the case that a whole block is being unmapped
1802 * that was previously partially mapped.
1803 */
1804 CHECK(ffa_region_group_identity_map(from_locked, fragments,
1805 fragment_constituent_counts,
1806 fragment_count, from_mode,
1807 &local_page_pool,
1808 MAP_ACTION_COMMIT, NULL)
1809 .func == FFA_SUCCESS_32);
1810 } else {
1811 /*
1812 * If the `map_action` is set to `MAP_ACTION_NONE`, S2 PTs
1813 * were not updated on retrieve/relinquish. These were updating
1814 * only the `share_state` structures. As such, use the sender's
1815 * original mode.
1816 */
1817 clearing_mode = sender_orig_mode;
1818 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001819
1820 /* Clear the memory so no VM or device can see the previous contents. */
J-Alves7db32002021-12-14 14:44:50 +00001821 if (clear &&
J-Alves69cdfd92024-04-26 11:40:59 +01001822 !ffa_clear_memory_constituents(clearing_mode, fragments,
J-Alves26483382023-04-20 12:01:49 +01001823 fragment_constituent_counts,
1824 fragment_count, page_pool)) {
J-Alves69cdfd92024-04-26 11:40:59 +01001825 if (map_action != MAP_ACTION_NONE) {
1826 /*
1827 * On failure, roll back by returning memory to the
1828 * sender. This may allocate pages which were previously
1829 * freed into `local_page_pool` by the call above, but
1830 * will never allocate more pages than that so can never
1831 * fail.
1832 */
1833 CHECK(ffa_region_group_identity_map(
1834 from_locked, fragments,
1835 fragment_constituent_counts,
1836 fragment_count, orig_from_mode,
1837 &local_page_pool, MAP_ACTION_COMMIT, NULL)
1838 .func == FFA_SUCCESS_32);
1839 }
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001840 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001841 goto out;
1842 }
1843
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001844 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001845
1846out:
1847 mpool_fini(&local_page_pool);
1848
1849 /*
1850 * Tidy up the page table by reclaiming failed mappings (if there was an
1851 * error) or merging entries into blocks where possible (on success).
1852 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001853 vm_ptable_defrag(from_locked, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +01001854
1855 return ret;
1856}
1857
1858/**
Andrew Walbranca808b12020-05-15 17:22:28 +01001859 * Complete a memory sending operation by checking that it is valid, updating
1860 * the sender page table, and then either marking the share state as having
1861 * completed sending (on success) or freeing it (on failure).
1862 *
1863 * Returns FFA_SUCCESS with the handle encoded, or the relevant FFA_ERROR.
1864 */
J-Alvesfdd29272022-07-19 13:16:31 +01001865struct ffa_value ffa_memory_send_complete(
Andrew Walbranca808b12020-05-15 17:22:28 +01001866 struct vm_locked from_locked, struct share_states_locked share_states,
Andrew Walbran37c574e2020-06-03 11:45:46 +01001867 struct ffa_memory_share_state *share_state, struct mpool *page_pool,
Karl Meakin07a69ab2025-02-07 14:53:19 +00001868 mm_mode_t *orig_from_mode_ret)
Andrew Walbranca808b12020-05-15 17:22:28 +01001869{
1870 struct ffa_memory_region *memory_region = share_state->memory_region;
J-Alves8f11cde2022-12-21 16:18:22 +00001871 struct ffa_composite_memory_region *composite;
Andrew Walbranca808b12020-05-15 17:22:28 +01001872 struct ffa_value ret;
1873
1874 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +00001875 assert(share_states.share_states != NULL);
J-Alves8f11cde2022-12-21 16:18:22 +00001876 assert(memory_region != NULL);
1877 composite = ffa_memory_region_get_composite(memory_region, 0);
1878 assert(composite != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +01001879
1880 /* Check that state is valid in sender page table and update. */
1881 ret = ffa_send_check_update(
1882 from_locked, share_state->fragments,
1883 share_state->fragment_constituent_counts,
J-Alves8f11cde2022-12-21 16:18:22 +00001884 share_state->fragment_count, composite->page_count,
Daniel Boulbya76fd912024-02-22 14:22:15 +00001885 share_state->share_func, memory_region, page_pool,
J-Alves460d36c2023-10-12 17:02:15 +01001886 orig_from_mode_ret, &share_state->memory_protected);
Andrew Walbranca808b12020-05-15 17:22:28 +01001887 if (ret.func != FFA_SUCCESS_32) {
1888 /*
1889 * Free share state, it failed to send so it can't be retrieved.
1890 */
Karl Meakin4cec5e82023-06-30 16:30:22 +01001891 dlog_verbose("%s: failed to send check update: %s(%s)\n",
1892 __func__, ffa_func_name(ret.func),
1893 ffa_error_name(ffa_error_code(ret)));
Andrew Walbranca808b12020-05-15 17:22:28 +01001894 share_state_free(share_states, share_state, page_pool);
1895 return ret;
1896 }
1897
1898 share_state->sending_complete = true;
Karl Meakin4cec5e82023-06-30 16:30:22 +01001899 dlog_verbose("%s: marked sending complete.\n", __func__);
Andrew Walbranca808b12020-05-15 17:22:28 +01001900
J-Alvesee68c542020-10-29 17:48:20 +00001901 return ffa_mem_success(share_state->memory_region->handle);
Andrew Walbranca808b12020-05-15 17:22:28 +01001902}
1903
1904/**
Daniel Boulby9764ff62024-01-30 17:47:39 +00001905 * Check that the memory attributes match Hafnium expectations.
1906 * Cacheability:
1907 * - Normal Memory as `FFA_MEMORY_CACHE_WRITE_BACK`.
1908 * - Device memory as `FFA_MEMORY_DEV_NGNRNE`.
1909 *
1910 * Shareability:
1911 * - Inner Shareable.
Federico Recanatia98603a2021-12-20 18:04:03 +01001912 */
1913static struct ffa_value ffa_memory_attributes_validate(
J-Alves7a99d0d2023-02-08 13:49:48 +00001914 ffa_memory_attributes_t attributes)
Federico Recanatia98603a2021-12-20 18:04:03 +01001915{
1916 enum ffa_memory_type memory_type;
1917 enum ffa_memory_cacheability cacheability;
1918 enum ffa_memory_shareability shareability;
1919
Karl Meakin84710f32023-10-12 15:14:49 +01001920 memory_type = attributes.type;
Daniel Boulby9764ff62024-01-30 17:47:39 +00001921 cacheability = attributes.cacheability;
1922 if (memory_type == FFA_MEMORY_NORMAL_MEM &&
1923 cacheability != FFA_MEMORY_CACHE_WRITE_BACK) {
1924 dlog_verbose(
1925 "Normal Memory: Invalid cacheability %s, "
1926 "expected %s.\n",
1927 ffa_memory_cacheability_name(cacheability),
1928 ffa_memory_cacheability_name(
1929 FFA_MEMORY_CACHE_WRITE_BACK));
Federico Recanati3d953f32022-02-17 09:31:29 +01001930 return ffa_error(FFA_DENIED);
Federico Recanatia98603a2021-12-20 18:04:03 +01001931 }
Daniel Boulby9764ff62024-01-30 17:47:39 +00001932 if (memory_type == FFA_MEMORY_DEVICE_MEM &&
1933 cacheability != FFA_MEMORY_DEV_NGNRNE) {
1934 dlog_verbose(
1935 "Device Memory: Invalid cacheability %s, "
1936 "expected %s.\n",
1937 ffa_device_memory_cacheability_name(cacheability),
1938 ffa_device_memory_cacheability_name(
1939 FFA_MEMORY_DEV_NGNRNE));
Federico Recanati3d953f32022-02-17 09:31:29 +01001940 return ffa_error(FFA_DENIED);
Federico Recanatia98603a2021-12-20 18:04:03 +01001941 }
1942
Karl Meakin84710f32023-10-12 15:14:49 +01001943 shareability = attributes.shareability;
Federico Recanatia98603a2021-12-20 18:04:03 +01001944 if (shareability != FFA_MEMORY_INNER_SHAREABLE) {
Karl Meakinf98b2aa2023-10-12 16:09:59 +01001945 dlog_verbose("Invalid shareability %s, expected %s.\n",
1946 ffa_memory_shareability_name(shareability),
1947 ffa_memory_shareability_name(
1948 FFA_MEMORY_INNER_SHAREABLE));
Federico Recanati3d953f32022-02-17 09:31:29 +01001949 return ffa_error(FFA_DENIED);
Federico Recanatia98603a2021-12-20 18:04:03 +01001950 }
1951
1952 return (struct ffa_value){.func = FFA_SUCCESS_32};
1953}
1954
1955/**
Andrew Walbrana65a1322020-04-06 19:32:32 +01001956 * Check that the given `memory_region` represents a valid memory send request
1957 * of the given `share_func` type, return the clear flag and permissions via the
1958 * respective output parameters, and update the permissions if necessary.
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001959 *
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001960 * Returns FFA_SUCCESS if the request was valid, or the relevant FFA_ERROR if
Andrew Walbrana65a1322020-04-06 19:32:32 +01001961 * not.
1962 */
J-Alves66652252022-07-06 09:49:51 +01001963struct ffa_value ffa_memory_send_validate(
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001964 struct vm_locked from_locked, struct ffa_memory_region *memory_region,
1965 uint32_t memory_share_length, uint32_t fragment_length,
J-Alves363f5722022-04-25 17:37:37 +01001966 uint32_t share_func)
Andrew Walbrana65a1322020-04-06 19:32:32 +01001967{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001968 struct ffa_composite_memory_region *composite;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001969 struct ffa_memory_access *receiver =
1970 ffa_memory_region_get_receiver(memory_region, 0);
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001971 uint64_t receivers_end;
1972 uint64_t min_length;
Federico Recanati872cd692022-01-05 13:10:10 +01001973 uint32_t composite_memory_region_offset;
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001974 uint32_t constituents_start;
Andrew Walbran130a8ae2020-05-15 16:27:15 +01001975 uint32_t constituents_length;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001976 enum ffa_data_access data_access;
1977 enum ffa_instruction_access instruction_access;
Olivier Deprez4342a3c2022-02-28 09:37:25 +01001978 enum ffa_memory_security security_state;
Karl Meakinf98b2aa2023-10-12 16:09:59 +01001979 enum ffa_memory_type type;
Federico Recanatia98603a2021-12-20 18:04:03 +01001980 struct ffa_value ret;
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001981 const size_t minimum_first_fragment_length =
Daniel Boulby41ef8ba2023-10-13 17:01:22 +01001982 memory_region->receivers_offset +
1983 memory_region->memory_access_desc_size +
1984 sizeof(struct ffa_composite_memory_region);
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001985
1986 if (fragment_length < minimum_first_fragment_length) {
Karl Meakine8937d92024-03-19 16:04:25 +00001987 dlog_verbose("Fragment length %u too short (min %zu).\n",
1988 fragment_length, minimum_first_fragment_length);
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001989 return ffa_error(FFA_INVALID_PARAMETERS);
1990 }
1991
Demi Marie Obenour73a1e942023-02-04 14:09:18 -05001992 static_assert(sizeof(struct ffa_memory_region_constituent) == 16,
1993 "struct ffa_memory_region_constituent must be 16 bytes");
1994 if (!is_aligned(fragment_length,
1995 sizeof(struct ffa_memory_region_constituent)) ||
1996 !is_aligned(memory_share_length,
1997 sizeof(struct ffa_memory_region_constituent))) {
1998 dlog_verbose(
1999 "Fragment length %u or total length %u"
2000 " is not 16-byte aligned.\n",
2001 fragment_length, memory_share_length);
2002 return ffa_error(FFA_INVALID_PARAMETERS);
2003 }
2004
Demi Marie Obenourd4677412023-02-03 20:35:12 -05002005 if (fragment_length > memory_share_length) {
2006 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00002007 "Fragment length %zu greater than total length %zu.\n",
Demi Marie Obenourd4677412023-02-03 20:35:12 -05002008 (size_t)fragment_length, (size_t)memory_share_length);
2009 return ffa_error(FFA_INVALID_PARAMETERS);
2010 }
Andrew Walbrana65a1322020-04-06 19:32:32 +01002011
J-Alves95df0ef2022-12-07 10:09:48 +00002012 /* The sender must match the caller. */
2013 if ((!vm_id_is_current_world(from_locked.vm->id) &&
2014 vm_id_is_current_world(memory_region->sender)) ||
2015 (vm_id_is_current_world(from_locked.vm->id) &&
2016 memory_region->sender != from_locked.vm->id)) {
2017 dlog_verbose("Invalid memory sender ID.\n");
2018 return ffa_error(FFA_DENIED);
Andrew Walbrana65a1322020-04-06 19:32:32 +01002019 }
2020
Demi Marie Obenourd4677412023-02-03 20:35:12 -05002021 if (memory_region->receiver_count <= 0) {
2022 dlog_verbose("No receivers!\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002023 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01002024 }
2025
Demi Marie Obenourd4677412023-02-03 20:35:12 -05002026 /*
2027 * Ensure that the composite header is within the memory bounds and
2028 * doesn't overlap the first part of the message. Cast to uint64_t
2029 * to prevent overflow.
2030 */
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002031 receivers_end = ((uint64_t)memory_region->memory_access_desc_size *
Demi Marie Obenourd4677412023-02-03 20:35:12 -05002032 (uint64_t)memory_region->receiver_count) +
Daniel Boulby41ef8ba2023-10-13 17:01:22 +01002033 memory_region->receivers_offset;
Demi Marie Obenourd4677412023-02-03 20:35:12 -05002034 min_length = receivers_end +
2035 sizeof(struct ffa_composite_memory_region) +
2036 sizeof(struct ffa_memory_region_constituent);
2037 if (min_length > memory_share_length) {
Karl Meakine8937d92024-03-19 16:04:25 +00002038 dlog_verbose("Share too short: got %zu but minimum is %zu.\n",
Demi Marie Obenourd4677412023-02-03 20:35:12 -05002039 (size_t)memory_share_length, (size_t)min_length);
2040 return ffa_error(FFA_INVALID_PARAMETERS);
2041 }
2042
2043 composite_memory_region_offset =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002044 receiver->composite_memory_region_offset;
Andrew Walbrana65a1322020-04-06 19:32:32 +01002045
2046 /*
Demi Marie Obenourd4677412023-02-03 20:35:12 -05002047 * Check that the composite memory region descriptor is after the access
2048 * descriptors, is at least 16-byte aligned, and fits in the first
2049 * fragment.
Andrew Walbrana65a1322020-04-06 19:32:32 +01002050 */
Demi Marie Obenourd4677412023-02-03 20:35:12 -05002051 if ((composite_memory_region_offset < receivers_end) ||
2052 (composite_memory_region_offset % 16 != 0) ||
2053 (composite_memory_region_offset >
2054 fragment_length - sizeof(struct ffa_composite_memory_region))) {
2055 dlog_verbose(
2056 "Invalid composite memory region descriptor offset "
Karl Meakine8937d92024-03-19 16:04:25 +00002057 "%zu.\n",
Demi Marie Obenourd4677412023-02-03 20:35:12 -05002058 (size_t)composite_memory_region_offset);
2059 return ffa_error(FFA_INVALID_PARAMETERS);
2060 }
2061
2062 /*
2063 * Compute the start of the constituent regions. Already checked
2064 * to be not more than fragment_length and thus not more than
2065 * memory_share_length.
2066 */
2067 constituents_start = composite_memory_region_offset +
2068 sizeof(struct ffa_composite_memory_region);
2069 constituents_length = memory_share_length - constituents_start;
2070
2071 /*
2072 * Check that the number of constituents is consistent with the length
2073 * of the constituent region.
2074 */
2075 composite = ffa_memory_region_get_composite(memory_region, 0);
2076 if ((constituents_length %
2077 sizeof(struct ffa_memory_region_constituent) !=
2078 0) ||
2079 ((constituents_length /
2080 sizeof(struct ffa_memory_region_constituent)) !=
2081 composite->constituent_count)) {
Karl Meakine8937d92024-03-19 16:04:25 +00002082 dlog_verbose("Invalid length %zu or composite offset %zu.\n",
Demi Marie Obenourd4677412023-02-03 20:35:12 -05002083 (size_t)memory_share_length,
2084 (size_t)composite_memory_region_offset);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002085 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01002086 }
Andrew Walbranca808b12020-05-15 17:22:28 +01002087 if (fragment_length < memory_share_length &&
2088 fragment_length < HF_MAILBOX_SIZE) {
2089 dlog_warning(
2090 "Initial fragment length %d smaller than mailbox "
2091 "size.\n",
2092 fragment_length);
2093 }
Andrew Walbrana65a1322020-04-06 19:32:32 +01002094
Andrew Walbrana65a1322020-04-06 19:32:32 +01002095 /*
2096 * Clear is not allowed for memory sharing, as the sender still has
2097 * access to the memory.
2098 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002099 if ((memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR) &&
J-Alves95fbb312024-03-20 15:19:16 +00002100 (share_func == FFA_MEM_SHARE_32 ||
2101 share_func == FFA_MEM_SHARE_64)) {
Andrew Walbrana65a1322020-04-06 19:32:32 +01002102 dlog_verbose("Memory can't be cleared while being shared.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002103 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01002104 }
2105
2106 /* No other flags are allowed/supported here. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002107 if (memory_region->flags & ~FFA_MEMORY_REGION_FLAG_CLEAR) {
Andrew Walbrana65a1322020-04-06 19:32:32 +01002108 dlog_verbose("Invalid flags %#x.\n", memory_region->flags);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002109 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01002110 }
2111
J-Alves363f5722022-04-25 17:37:37 +01002112 /* Check that the permissions are valid, for each specified receiver. */
2113 for (uint32_t i = 0U; i < memory_region->receiver_count; i++) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002114 struct ffa_memory_region_attributes receiver_permissions;
2115
2116 receiver = ffa_memory_region_get_receiver(memory_region, i);
2117 assert(receiver != NULL);
2118 receiver_permissions = receiver->receiver_permissions;
J-Alves363f5722022-04-25 17:37:37 +01002119 ffa_memory_access_permissions_t permissions =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002120 receiver_permissions.permissions;
2121 ffa_id_t receiver_id = receiver_permissions.receiver;
J-Alves363f5722022-04-25 17:37:37 +01002122
2123 if (memory_region->sender == receiver_id) {
2124 dlog_verbose("Can't share memory with itself.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002125 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01002126 }
Federico Recanati85090c42021-12-15 13:17:54 +01002127
J-Alves363f5722022-04-25 17:37:37 +01002128 for (uint32_t j = i + 1; j < memory_region->receiver_count;
2129 j++) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002130 struct ffa_memory_access *other_receiver =
2131 ffa_memory_region_get_receiver(memory_region,
2132 j);
2133 assert(other_receiver != NULL);
2134
J-Alves363f5722022-04-25 17:37:37 +01002135 if (receiver_id ==
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002136 other_receiver->receiver_permissions.receiver) {
J-Alves363f5722022-04-25 17:37:37 +01002137 dlog_verbose(
2138 "Repeated receiver(%x) in memory send "
2139 "operation.\n",
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002140 other_receiver->receiver_permissions
2141 .receiver);
J-Alves363f5722022-04-25 17:37:37 +01002142 return ffa_error(FFA_INVALID_PARAMETERS);
2143 }
2144 }
2145
2146 if (composite_memory_region_offset !=
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002147 receiver->composite_memory_region_offset) {
J-Alves363f5722022-04-25 17:37:37 +01002148 dlog_verbose(
2149 "All ffa_memory_access should point to the "
2150 "same composite memory region offset.\n");
2151 return ffa_error(FFA_INVALID_PARAMETERS);
2152 }
2153
Karl Meakin84710f32023-10-12 15:14:49 +01002154 data_access = permissions.data_access;
2155 instruction_access = permissions.instruction_access;
J-Alves363f5722022-04-25 17:37:37 +01002156 if (data_access == FFA_DATA_ACCESS_RESERVED ||
2157 instruction_access == FFA_INSTRUCTION_ACCESS_RESERVED) {
2158 dlog_verbose(
2159 "Reserved value for receiver permissions "
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002160 "(data_access = %s, instruction_access = %s)\n",
2161 ffa_data_access_name(data_access),
2162 ffa_instruction_access_name(
2163 instruction_access));
J-Alves363f5722022-04-25 17:37:37 +01002164 return ffa_error(FFA_INVALID_PARAMETERS);
2165 }
2166 if (instruction_access !=
2167 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
2168 dlog_verbose(
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002169 "Invalid instruction access permissions %s "
2170 "for sending memory, expected %s.\n",
2171 ffa_instruction_access_name(instruction_access),
2172 ffa_instruction_access_name(
Daniel Boulby91052c32024-05-21 14:09:48 +01002173 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED));
J-Alves363f5722022-04-25 17:37:37 +01002174 return ffa_error(FFA_INVALID_PARAMETERS);
2175 }
J-Alves95fbb312024-03-20 15:19:16 +00002176 if (share_func == FFA_MEM_SHARE_32 ||
2177 share_func == FFA_MEM_SHARE_64) {
J-Alves363f5722022-04-25 17:37:37 +01002178 if (data_access == FFA_DATA_ACCESS_NOT_SPECIFIED) {
2179 dlog_verbose(
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002180 "Invalid data access permissions %s "
2181 "for sharing memory, expected %s.\n",
2182 ffa_data_access_name(data_access),
2183 ffa_data_access_name(
2184 FFA_DATA_ACCESS_NOT_SPECIFIED));
J-Alves363f5722022-04-25 17:37:37 +01002185 return ffa_error(FFA_INVALID_PARAMETERS);
2186 }
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002187 /*
2188 * According to section 10.10.3 of the FF-A v1.1 EAC0
2189 * spec, NX is required for share operations (but must
2190 * not be specified by the sender) so set it in the
2191 * copy that we store, ready to be returned to the
2192 * retriever.
2193 */
2194 if (vm_id_is_current_world(receiver_id)) {
Karl Meakin84710f32023-10-12 15:14:49 +01002195 permissions.instruction_access =
2196 FFA_INSTRUCTION_ACCESS_NX;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002197 receiver_permissions.permissions = permissions;
2198 }
J-Alves363f5722022-04-25 17:37:37 +01002199 }
J-Alves95fbb312024-03-20 15:19:16 +00002200 if ((share_func == FFA_MEM_LEND_32 ||
2201 share_func == FFA_MEM_LEND_64) &&
J-Alves363f5722022-04-25 17:37:37 +01002202 data_access == FFA_DATA_ACCESS_NOT_SPECIFIED) {
2203 dlog_verbose(
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002204 "Invalid data access permissions %s for "
2205 "lending memory, expected %s.\n",
2206 ffa_data_access_name(data_access),
2207 ffa_data_access_name(
2208 FFA_DATA_ACCESS_NOT_SPECIFIED));
J-Alves363f5722022-04-25 17:37:37 +01002209 return ffa_error(FFA_INVALID_PARAMETERS);
2210 }
2211
J-Alves95fbb312024-03-20 15:19:16 +00002212 if ((share_func == FFA_MEM_DONATE_32 ||
2213 share_func == FFA_MEM_DONATE_64) &&
J-Alves363f5722022-04-25 17:37:37 +01002214 data_access != FFA_DATA_ACCESS_NOT_SPECIFIED) {
2215 dlog_verbose(
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002216 "Invalid data access permissions %s for "
2217 "donating memory, expected %s.\n",
2218 ffa_data_access_name(data_access),
2219 ffa_data_access_name(
2220 FFA_DATA_ACCESS_NOT_SPECIFIED));
J-Alves363f5722022-04-25 17:37:37 +01002221 return ffa_error(FFA_INVALID_PARAMETERS);
2222 }
Andrew Walbrana65a1322020-04-06 19:32:32 +01002223 }
2224
Olivier Deprez4342a3c2022-02-28 09:37:25 +01002225 /* Memory region attributes NS-Bit MBZ for FFA_MEM_SHARE/LEND/DONATE. */
Karl Meakin84710f32023-10-12 15:14:49 +01002226 security_state = memory_region->attributes.security;
Olivier Deprez4342a3c2022-02-28 09:37:25 +01002227 if (security_state != FFA_MEMORY_SECURITY_UNSPECIFIED) {
2228 dlog_verbose(
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002229 "Invalid security state %s for memory share operation, "
2230 "expected %s.\n",
2231 ffa_memory_security_name(security_state),
2232 ffa_memory_security_name(
2233 FFA_MEMORY_SECURITY_UNSPECIFIED));
Olivier Deprez4342a3c2022-02-28 09:37:25 +01002234 return ffa_error(FFA_INVALID_PARAMETERS);
2235 }
2236
Federico Recanatid937f5e2021-12-20 17:38:23 +01002237 /*
J-Alves807794e2022-06-16 13:42:47 +01002238 * If a memory donate or lend with single borrower, the memory type
2239 * shall not be specified by the sender.
Federico Recanatid937f5e2021-12-20 17:38:23 +01002240 */
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002241 type = memory_region->attributes.type;
J-Alves807794e2022-06-16 13:42:47 +01002242 if (share_func == FFA_MEM_DONATE_32 ||
J-Alves95fbb312024-03-20 15:19:16 +00002243 share_func == FFA_MEM_DONATE_64 ||
2244 ((share_func == FFA_MEM_LEND_32 || share_func == FFA_MEM_LEND_64) &&
J-Alves807794e2022-06-16 13:42:47 +01002245 memory_region->receiver_count == 1)) {
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002246 if (type != FFA_MEMORY_NOT_SPECIFIED_MEM) {
J-Alves807794e2022-06-16 13:42:47 +01002247 dlog_verbose(
Karl Meakinf98b2aa2023-10-12 16:09:59 +01002248 "Invalid memory type %s for memory share "
2249 "operation, expected %s.\n",
2250 ffa_memory_type_name(type),
2251 ffa_memory_type_name(
2252 FFA_MEMORY_NOT_SPECIFIED_MEM));
J-Alves807794e2022-06-16 13:42:47 +01002253 return ffa_error(FFA_INVALID_PARAMETERS);
2254 }
2255 } else {
2256 /*
2257 * Check that sender's memory attributes match Hafnium
2258 * expectations: Normal Memory, Inner shareable, Write-Back
2259 * Read-Allocate Write-Allocate Cacheable.
2260 */
2261 ret = ffa_memory_attributes_validate(memory_region->attributes);
2262 if (ret.func != FFA_SUCCESS_32) {
2263 return ret;
2264 }
Federico Recanatid937f5e2021-12-20 17:38:23 +01002265 }
2266
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002267 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbrana65a1322020-04-06 19:32:32 +01002268}
2269
2270/**
Andrew Walbranca808b12020-05-15 17:22:28 +01002271 * Gets the share state for continuing an operation to donate, lend or share
2272 * memory, and checks that it is a valid request.
2273 *
2274 * Returns FFA_SUCCESS if the request was valid, or the relevant FFA_ERROR if
2275 * not.
2276 */
J-Alvesfdd29272022-07-19 13:16:31 +01002277struct ffa_value ffa_memory_send_continue_validate(
Andrew Walbranca808b12020-05-15 17:22:28 +01002278 struct share_states_locked share_states, ffa_memory_handle_t handle,
J-Alves19e20cf2023-08-02 12:48:55 +01002279 struct ffa_memory_share_state **share_state_ret, ffa_id_t from_vm_id,
Andrew Walbranca808b12020-05-15 17:22:28 +01002280 struct mpool *page_pool)
2281{
2282 struct ffa_memory_share_state *share_state;
2283 struct ffa_memory_region *memory_region;
2284
Daniel Boulbya2f8c662021-11-26 17:52:53 +00002285 assert(share_state_ret != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +01002286
2287 /*
2288 * Look up the share state by handle and make sure that the VM ID
2289 * matches.
2290 */
Karl Meakin4a2854a2023-06-30 16:26:52 +01002291 share_state = get_share_state(share_states, handle);
J-Alvesb56aac82023-11-10 09:44:43 +00002292 if (share_state == NULL) {
Andrew Walbranca808b12020-05-15 17:22:28 +01002293 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00002294 "Invalid handle %#lx for memory send continuation.\n",
Andrew Walbranca808b12020-05-15 17:22:28 +01002295 handle);
2296 return ffa_error(FFA_INVALID_PARAMETERS);
2297 }
2298 memory_region = share_state->memory_region;
2299
J-Alvesfdd29272022-07-19 13:16:31 +01002300 if (vm_id_is_current_world(from_vm_id) &&
2301 memory_region->sender != from_vm_id) {
Andrew Walbranca808b12020-05-15 17:22:28 +01002302 dlog_verbose("Invalid sender %d.\n", memory_region->sender);
2303 return ffa_error(FFA_INVALID_PARAMETERS);
2304 }
2305
2306 if (share_state->sending_complete) {
2307 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00002308 "Sending of memory handle %#lx is already complete.\n",
Andrew Walbranca808b12020-05-15 17:22:28 +01002309 handle);
2310 return ffa_error(FFA_INVALID_PARAMETERS);
2311 }
2312
2313 if (share_state->fragment_count == MAX_FRAGMENTS) {
2314 /*
2315 * Log a warning as this is a sign that MAX_FRAGMENTS should
2316 * probably be increased.
2317 */
2318 dlog_warning(
Karl Meakine8937d92024-03-19 16:04:25 +00002319 "Too many fragments for memory share with handle %#lx; "
Andrew Walbranca808b12020-05-15 17:22:28 +01002320 "only %d supported.\n",
2321 handle, MAX_FRAGMENTS);
2322 /* Free share state, as it's not possible to complete it. */
2323 share_state_free(share_states, share_state, page_pool);
2324 return ffa_error(FFA_NO_MEMORY);
2325 }
2326
2327 *share_state_ret = share_state;
2328
2329 return (struct ffa_value){.func = FFA_SUCCESS_32};
2330}
2331
2332/**
J-Alves95df0ef2022-12-07 10:09:48 +00002333 * Checks if there is at least one receiver from the other world.
2334 */
J-Alvesfdd29272022-07-19 13:16:31 +01002335bool memory_region_receivers_from_other_world(
J-Alves95df0ef2022-12-07 10:09:48 +00002336 struct ffa_memory_region *memory_region)
2337{
2338 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002339 struct ffa_memory_access *receiver =
2340 ffa_memory_region_get_receiver(memory_region, i);
2341 assert(receiver != NULL);
2342 ffa_id_t receiver_id = receiver->receiver_permissions.receiver;
2343
2344 if (!vm_id_is_current_world(receiver_id)) {
J-Alves95df0ef2022-12-07 10:09:48 +00002345 return true;
2346 }
2347 }
2348 return false;
2349}
2350
2351/**
J-Alves9da280b2022-12-21 14:55:39 +00002352 * Validates a call to donate, lend or share memory in which Hafnium is the
2353 * designated allocator of the memory handle. In practice, this also means
2354 * Hafnium is responsible for managing the state structures for the transaction.
2355 * If Hafnium is the SPMC, it should allocate the memory handle when either the
2356 * sender is an SP or there is at least one borrower that is an SP.
2357 * If Hafnium is the hypervisor, it should allocate the memory handle when
2358 * operation involves only NWd VMs.
2359 *
2360 * If validation goes well, Hafnium updates the stage-2 page tables of the
2361 * sender. Validation consists of checking if the message length and number of
2362 * memory region constituents match, and if the transition is valid for the
2363 * type of memory sending operation.
Andrew Walbran475c1452020-02-07 13:22:22 +00002364 *
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002365 * Assumes that the caller has already found and locked the sender VM and copied
2366 * the memory region descriptor from the sender's TX buffer to a freshly
2367 * allocated page from Hafnium's internal pool. The caller must have also
2368 * validated that the receiver VM ID is valid.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002369 *
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002370 * This function takes ownership of the `memory_region` passed in and will free
2371 * it when necessary; it must not be freed by the caller.
Jose Marinho09b1db82019-08-08 09:16:59 +01002372 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002373struct ffa_value ffa_memory_send(struct vm_locked from_locked,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002374 struct ffa_memory_region *memory_region,
Andrew Walbran130a8ae2020-05-15 16:27:15 +01002375 uint32_t memory_share_length,
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002376 uint32_t fragment_length, uint32_t share_func,
2377 struct mpool *page_pool)
Jose Marinho09b1db82019-08-08 09:16:59 +01002378{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002379 struct ffa_value ret;
Andrew Walbranca808b12020-05-15 17:22:28 +01002380 struct share_states_locked share_states;
2381 struct ffa_memory_share_state *share_state;
Jose Marinho09b1db82019-08-08 09:16:59 +01002382
2383 /*
Andrew Walbrana65a1322020-04-06 19:32:32 +01002384 * If there is an error validating the `memory_region` then we need to
2385 * free it because we own it but we won't be storing it in a share state
2386 * after all.
Jose Marinho09b1db82019-08-08 09:16:59 +01002387 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002388 ret = ffa_memory_send_validate(from_locked, memory_region,
2389 memory_share_length, fragment_length,
J-Alves363f5722022-04-25 17:37:37 +01002390 share_func);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002391 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002392 mpool_free(page_pool, memory_region);
Andrew Walbrana65a1322020-04-06 19:32:32 +01002393 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01002394 }
2395
Andrew Walbrana65a1322020-04-06 19:32:32 +01002396 /* Set flag for share function, ready to be retrieved later. */
2397 switch (share_func) {
J-Alves95fbb312024-03-20 15:19:16 +00002398 case FFA_MEM_SHARE_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002399 case FFA_MEM_SHARE_32:
Andrew Walbrana65a1322020-04-06 19:32:32 +01002400 memory_region->flags |=
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002401 FFA_MEMORY_REGION_TRANSACTION_TYPE_SHARE;
Andrew Walbrana65a1322020-04-06 19:32:32 +01002402 break;
J-Alves95fbb312024-03-20 15:19:16 +00002403 case FFA_MEM_LEND_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002404 case FFA_MEM_LEND_32:
2405 memory_region->flags |= FFA_MEMORY_REGION_TRANSACTION_TYPE_LEND;
Andrew Walbrana65a1322020-04-06 19:32:32 +01002406 break;
J-Alves95fbb312024-03-20 15:19:16 +00002407 case FFA_MEM_DONATE_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002408 case FFA_MEM_DONATE_32:
Andrew Walbrana65a1322020-04-06 19:32:32 +01002409 memory_region->flags |=
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002410 FFA_MEMORY_REGION_TRANSACTION_TYPE_DONATE;
Andrew Walbrana65a1322020-04-06 19:32:32 +01002411 break;
Karl Meakina5ea9092024-05-28 15:40:33 +01002412 default:
2413 dlog_verbose("Unknown share func %#x (%s)\n", share_func,
2414 ffa_func_name(share_func));
2415 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho09b1db82019-08-08 09:16:59 +01002416 }
2417
Andrew Walbranca808b12020-05-15 17:22:28 +01002418 share_states = share_states_lock();
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002419 /*
2420 * Allocate a share state before updating the page table. Otherwise if
2421 * updating the page table succeeded but allocating the share state
2422 * failed then it would leave the memory in a state where nobody could
2423 * get it back.
2424 */
Karl Meakin52cdfe72023-06-30 14:49:10 +01002425 share_state = allocate_share_state(share_states, share_func,
2426 memory_region, fragment_length,
2427 FFA_MEMORY_HANDLE_INVALID);
J-Alvesb56aac82023-11-10 09:44:43 +00002428 if (share_state == NULL) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002429 dlog_verbose("Failed to allocate share state.\n");
2430 mpool_free(page_pool, memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +01002431 ret = ffa_error(FFA_NO_MEMORY);
2432 goto out;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002433 }
2434
Andrew Walbranca808b12020-05-15 17:22:28 +01002435 if (fragment_length == memory_share_length) {
2436 /* No more fragments to come, everything fit in one message. */
J-Alves2a0d2882020-10-29 14:49:50 +00002437 ret = ffa_memory_send_complete(
2438 from_locked, share_states, share_state, page_pool,
2439 &(share_state->sender_orig_mode));
Andrew Walbranca808b12020-05-15 17:22:28 +01002440 } else {
J-Alvesfdd29272022-07-19 13:16:31 +01002441 /*
2442 * Use sender ID from 'memory_region' assuming
2443 * that at this point it has been validated:
2444 * - MBZ at virtual FF-A instance.
2445 */
J-Alves19e20cf2023-08-02 12:48:55 +01002446 ffa_id_t sender_to_ret =
J-Alvesfdd29272022-07-19 13:16:31 +01002447 (from_locked.vm->id == HF_OTHER_WORLD_ID)
2448 ? memory_region->sender
2449 : 0;
Andrew Walbranca808b12020-05-15 17:22:28 +01002450 ret = (struct ffa_value){
2451 .func = FFA_MEM_FRAG_RX_32,
J-Alvesee68c542020-10-29 17:48:20 +00002452 .arg1 = (uint32_t)memory_region->handle,
2453 .arg2 = (uint32_t)(memory_region->handle >> 32),
J-Alvesfdd29272022-07-19 13:16:31 +01002454 .arg3 = fragment_length,
2455 .arg4 = (uint32_t)(sender_to_ret & 0xffff) << 16};
Andrew Walbranca808b12020-05-15 17:22:28 +01002456 }
2457
2458out:
2459 share_states_unlock(&share_states);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002460 dump_share_states();
Andrew Walbranca808b12020-05-15 17:22:28 +01002461 return ret;
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002462}
2463
2464/**
J-Alves8505a8a2022-06-15 18:10:18 +01002465 * Continues an operation to donate, lend or share memory to a VM from current
2466 * world. If this is the last fragment then checks that the transition is valid
2467 * for the type of memory sending operation and updates the stage-2 page tables
2468 * of the sender.
Andrew Walbranca808b12020-05-15 17:22:28 +01002469 *
2470 * Assumes that the caller has already found and locked the sender VM and copied
2471 * the memory region descriptor from the sender's TX buffer to a freshly
2472 * allocated page from Hafnium's internal pool.
2473 *
2474 * This function takes ownership of the `fragment` passed in; it must not be
2475 * freed by the caller.
2476 */
2477struct ffa_value ffa_memory_send_continue(struct vm_locked from_locked,
2478 void *fragment,
2479 uint32_t fragment_length,
2480 ffa_memory_handle_t handle,
2481 struct mpool *page_pool)
2482{
2483 struct share_states_locked share_states = share_states_lock();
2484 struct ffa_memory_share_state *share_state;
2485 struct ffa_value ret;
2486 struct ffa_memory_region *memory_region;
2487
Demi Marie Obenour73a1e942023-02-04 14:09:18 -05002488 CHECK(is_aligned(fragment,
2489 alignof(struct ffa_memory_region_constituent)));
2490 if (fragment_length % sizeof(struct ffa_memory_region_constituent) !=
2491 0) {
2492 dlog_verbose("Fragment length %u misaligned.\n",
2493 fragment_length);
2494 ret = ffa_error(FFA_INVALID_PARAMETERS);
2495 goto out_free_fragment;
2496 }
2497
Andrew Walbranca808b12020-05-15 17:22:28 +01002498 ret = ffa_memory_send_continue_validate(share_states, handle,
2499 &share_state,
2500 from_locked.vm->id, page_pool);
2501 if (ret.func != FFA_SUCCESS_32) {
2502 goto out_free_fragment;
2503 }
2504 memory_region = share_state->memory_region;
2505
J-Alves95df0ef2022-12-07 10:09:48 +00002506 if (memory_region_receivers_from_other_world(memory_region)) {
Andrew Walbranca808b12020-05-15 17:22:28 +01002507 dlog_error(
2508 "Got hypervisor-allocated handle for memory send to "
J-Alves8505a8a2022-06-15 18:10:18 +01002509 "other world. This should never happen, and indicates "
2510 "a bug in "
Andrew Walbranca808b12020-05-15 17:22:28 +01002511 "EL3 code.\n");
2512 ret = ffa_error(FFA_INVALID_PARAMETERS);
2513 goto out_free_fragment;
2514 }
2515
2516 /* Add this fragment. */
2517 share_state->fragments[share_state->fragment_count] = fragment;
2518 share_state->fragment_constituent_counts[share_state->fragment_count] =
2519 fragment_length / sizeof(struct ffa_memory_region_constituent);
2520 share_state->fragment_count++;
2521
2522 /* Check whether the memory send operation is now ready to complete. */
2523 if (share_state_sending_complete(share_states, share_state)) {
J-Alves2a0d2882020-10-29 14:49:50 +00002524 ret = ffa_memory_send_complete(
2525 from_locked, share_states, share_state, page_pool,
2526 &(share_state->sender_orig_mode));
Andrew Walbranca808b12020-05-15 17:22:28 +01002527 } else {
2528 ret = (struct ffa_value){
2529 .func = FFA_MEM_FRAG_RX_32,
2530 .arg1 = (uint32_t)handle,
2531 .arg2 = (uint32_t)(handle >> 32),
2532 .arg3 = share_state_next_fragment_offset(share_states,
2533 share_state)};
2534 }
2535 goto out;
2536
2537out_free_fragment:
2538 mpool_free(page_pool, fragment);
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002539
2540out:
Andrew Walbranca808b12020-05-15 17:22:28 +01002541 share_states_unlock(&share_states);
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002542 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002543}
2544
Andrew Walbranca808b12020-05-15 17:22:28 +01002545/** Clean up after the receiver has finished retrieving a memory region. */
2546static void ffa_memory_retrieve_complete(
2547 struct share_states_locked share_states,
2548 struct ffa_memory_share_state *share_state, struct mpool *page_pool)
2549{
J-Alves95fbb312024-03-20 15:19:16 +00002550 if (share_state->share_func == FFA_MEM_DONATE_32 ||
2551 share_state->share_func == FFA_MEM_DONATE_64) {
Andrew Walbranca808b12020-05-15 17:22:28 +01002552 /*
2553 * Memory that has been donated can't be relinquished,
2554 * so no need to keep the share state around.
2555 */
2556 share_state_free(share_states, share_state, page_pool);
2557 dlog_verbose("Freed share state for donate.\n");
2558 }
2559}
2560
J-Alves2d8457f2022-10-05 11:06:41 +01002561/**
2562 * Initialises the given memory region descriptor to be used for an
2563 * `FFA_MEM_RETRIEVE_RESP`, including the given constituents for the first
2564 * fragment.
2565 * The memory region descriptor is initialized according to retriever's
2566 * FF-A version.
2567 *
2568 * Returns true on success, or false if the given constituents won't all fit in
2569 * the first fragment.
2570 */
2571static bool ffa_retrieved_memory_region_init(
Karl Meakin0e617d92024-04-05 12:55:22 +01002572 void *response, enum ffa_version ffa_version, size_t response_max_size,
J-Alves19e20cf2023-08-02 12:48:55 +01002573 ffa_id_t sender, ffa_memory_attributes_t attributes,
J-Alves2d8457f2022-10-05 11:06:41 +01002574 ffa_memory_region_flags_t flags, ffa_memory_handle_t handle,
Daniel Boulbyde974ca2023-12-12 13:53:31 +00002575 ffa_memory_access_permissions_t permissions,
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002576 struct ffa_memory_access *receivers, size_t receiver_count,
2577 uint32_t memory_access_desc_size, uint32_t page_count,
2578 uint32_t total_constituent_count,
J-Alves2d8457f2022-10-05 11:06:41 +01002579 const struct ffa_memory_region_constituent constituents[],
2580 uint32_t fragment_constituent_count, uint32_t *total_length,
2581 uint32_t *fragment_length)
2582{
2583 struct ffa_composite_memory_region *composite_memory_region;
J-Alves2d8457f2022-10-05 11:06:41 +01002584 uint32_t i;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002585 uint32_t composite_offset;
J-Alves2d8457f2022-10-05 11:06:41 +01002586 uint32_t constituents_offset;
J-Alves2d8457f2022-10-05 11:06:41 +01002587
2588 assert(response != NULL);
2589
Karl Meakin0e617d92024-04-05 12:55:22 +01002590 if (ffa_version == FFA_VERSION_1_0) {
J-Alves2d8457f2022-10-05 11:06:41 +01002591 struct ffa_memory_region_v1_0 *retrieve_response =
2592 (struct ffa_memory_region_v1_0 *)response;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002593 struct ffa_memory_access_v1_0 *receiver;
J-Alves2d8457f2022-10-05 11:06:41 +01002594
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002595 ffa_memory_region_init_header_v1_0(retrieve_response, sender,
2596 attributes, flags, handle, 0,
2597 receiver_count);
J-Alves2d8457f2022-10-05 11:06:41 +01002598
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002599 receiver = (struct ffa_memory_access_v1_0 *)
2600 retrieve_response->receivers;
J-Alves2d8457f2022-10-05 11:06:41 +01002601 receiver_count = retrieve_response->receiver_count;
2602
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002603 for (uint32_t i = 0; i < receiver_count; i++) {
2604 ffa_id_t receiver_id =
2605 receivers[i].receiver_permissions.receiver;
2606 ffa_memory_receiver_flags_t recv_flags =
2607 receivers[i].receiver_permissions.flags;
2608
2609 /*
2610 * Initialized here as in memory retrieve responses we
2611 * currently expect one borrower to be specified.
2612 */
2613 ffa_memory_access_init_v1_0(
Karl Meakin84710f32023-10-12 15:14:49 +01002614 receiver, receiver_id, permissions.data_access,
2615 permissions.instruction_access, recv_flags);
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002616 }
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002617
2618 composite_offset =
J-Alves2d8457f2022-10-05 11:06:41 +01002619 sizeof(struct ffa_memory_region_v1_0) +
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002620 receiver_count * sizeof(struct ffa_memory_access_v1_0);
2621 receiver->composite_memory_region_offset = composite_offset;
J-Alves2d8457f2022-10-05 11:06:41 +01002622
2623 composite_memory_region = ffa_memory_region_get_composite_v1_0(
2624 retrieve_response, 0);
2625 } else {
J-Alves2d8457f2022-10-05 11:06:41 +01002626 struct ffa_memory_region *retrieve_response =
2627 (struct ffa_memory_region *)response;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002628 struct ffa_memory_access *retrieve_response_receivers;
J-Alves2d8457f2022-10-05 11:06:41 +01002629
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002630 ffa_memory_region_init_header(
2631 retrieve_response, sender, attributes, flags, handle, 0,
2632 receiver_count, memory_access_desc_size);
J-Alves2d8457f2022-10-05 11:06:41 +01002633
2634 /*
2635 * Note that `sizeof(struct_ffa_memory_region)` and
2636 * `sizeof(struct ffa_memory_access)` must both be multiples of
2637 * 16 (as verified by the asserts in `ffa_memory.c`, so it is
2638 * guaranteed that the offset we calculate here is aligned to a
2639 * 64-bit boundary and so 64-bit values can be copied without
2640 * alignment faults.
2641 */
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002642 composite_offset =
Daniel Boulby41ef8ba2023-10-13 17:01:22 +01002643 retrieve_response->receivers_offset +
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002644 (uint32_t)(receiver_count *
2645 retrieve_response->memory_access_desc_size);
J-Alves2d8457f2022-10-05 11:06:41 +01002646
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002647 retrieve_response_receivers =
2648 ffa_memory_region_get_receiver(retrieve_response, 0);
2649 assert(retrieve_response_receivers != NULL);
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002650
2651 /*
2652 * Initialized here as in memory retrieve responses we currently
2653 * expect one borrower to be specified.
2654 */
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002655 memcpy_s(retrieve_response_receivers,
2656 sizeof(struct ffa_memory_access) * receiver_count,
2657 receivers,
2658 sizeof(struct ffa_memory_access) * receiver_count);
2659
2660 retrieve_response_receivers->composite_memory_region_offset =
2661 composite_offset;
2662
J-Alves2d8457f2022-10-05 11:06:41 +01002663 composite_memory_region =
2664 ffa_memory_region_get_composite(retrieve_response, 0);
2665 }
2666
J-Alves2d8457f2022-10-05 11:06:41 +01002667 assert(composite_memory_region != NULL);
2668
J-Alves2d8457f2022-10-05 11:06:41 +01002669 composite_memory_region->page_count = page_count;
2670 composite_memory_region->constituent_count = total_constituent_count;
2671 composite_memory_region->reserved_0 = 0;
2672
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002673 constituents_offset =
2674 composite_offset + sizeof(struct ffa_composite_memory_region);
J-Alves2d8457f2022-10-05 11:06:41 +01002675 if (constituents_offset +
2676 fragment_constituent_count *
2677 sizeof(struct ffa_memory_region_constituent) >
2678 response_max_size) {
2679 return false;
2680 }
2681
2682 for (i = 0; i < fragment_constituent_count; ++i) {
2683 composite_memory_region->constituents[i] = constituents[i];
2684 }
2685
2686 if (total_length != NULL) {
2687 *total_length =
2688 constituents_offset +
2689 composite_memory_region->constituent_count *
2690 sizeof(struct ffa_memory_region_constituent);
2691 }
2692 if (fragment_length != NULL) {
2693 *fragment_length =
2694 constituents_offset +
2695 fragment_constituent_count *
2696 sizeof(struct ffa_memory_region_constituent);
2697 }
2698
2699 return true;
2700}
2701
J-Alves96de29f2022-04-26 16:05:24 +01002702/**
2703 * Validates the retrieved permissions against those specified by the lender
2704 * of memory share operation. Optionally can help set the permissions to be used
2705 * for the S2 mapping, through the `permissions` argument.
J-Alvesdcad8992023-09-15 14:10:35 +01002706 * Returns FFA_SUCCESS if all the fields are valid. FFA_ERROR, with error code:
2707 * - FFA_INVALID_PARAMETERS -> if the fields have invalid values as per the
2708 * specification for each ABI.
2709 * - FFA_DENIED -> if the permissions specified by the retriever are not
2710 * less permissive than those provided by the sender.
J-Alves96de29f2022-04-26 16:05:24 +01002711 */
J-Alvesdcad8992023-09-15 14:10:35 +01002712static struct ffa_value ffa_memory_retrieve_is_memory_access_valid(
2713 uint32_t share_func, enum ffa_data_access sent_data_access,
J-Alves96de29f2022-04-26 16:05:24 +01002714 enum ffa_data_access requested_data_access,
2715 enum ffa_instruction_access sent_instruction_access,
2716 enum ffa_instruction_access requested_instruction_access,
J-Alvesdcad8992023-09-15 14:10:35 +01002717 ffa_memory_access_permissions_t *permissions, bool multiple_borrowers)
J-Alves96de29f2022-04-26 16:05:24 +01002718{
2719 switch (sent_data_access) {
2720 case FFA_DATA_ACCESS_NOT_SPECIFIED:
2721 case FFA_DATA_ACCESS_RW:
2722 if (requested_data_access == FFA_DATA_ACCESS_NOT_SPECIFIED ||
2723 requested_data_access == FFA_DATA_ACCESS_RW) {
2724 if (permissions != NULL) {
Karl Meakin84710f32023-10-12 15:14:49 +01002725 permissions->data_access = FFA_DATA_ACCESS_RW;
J-Alves96de29f2022-04-26 16:05:24 +01002726 }
2727 break;
2728 }
2729 /* Intentional fall-through. */
2730 case FFA_DATA_ACCESS_RO:
2731 if (requested_data_access == FFA_DATA_ACCESS_NOT_SPECIFIED ||
2732 requested_data_access == FFA_DATA_ACCESS_RO) {
2733 if (permissions != NULL) {
Karl Meakin84710f32023-10-12 15:14:49 +01002734 permissions->data_access = FFA_DATA_ACCESS_RO;
J-Alves96de29f2022-04-26 16:05:24 +01002735 }
2736 break;
2737 }
2738 dlog_verbose(
2739 "Invalid data access requested; sender specified "
2740 "permissions %#x but receiver requested %#x.\n",
2741 sent_data_access, requested_data_access);
J-Alvesdcad8992023-09-15 14:10:35 +01002742 return ffa_error(FFA_DENIED);
J-Alves96de29f2022-04-26 16:05:24 +01002743 case FFA_DATA_ACCESS_RESERVED:
2744 panic("Got unexpected FFA_DATA_ACCESS_RESERVED. Should be "
2745 "checked before this point.");
2746 }
2747
J-Alvesdcad8992023-09-15 14:10:35 +01002748 /*
2749 * For operations with a single borrower, If it is an FFA_MEMORY_LEND
2750 * or FFA_MEMORY_DONATE the retriever should have specifed the
2751 * instruction permissions it wishes to receive.
2752 */
2753 switch (share_func) {
J-Alves95fbb312024-03-20 15:19:16 +00002754 case FFA_MEM_SHARE_64:
J-Alvesdcad8992023-09-15 14:10:35 +01002755 case FFA_MEM_SHARE_32:
2756 if (requested_instruction_access !=
2757 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
2758 dlog_verbose(
2759 "%s: for share instruction permissions must "
2760 "NOT be specified.\n",
2761 __func__);
2762 return ffa_error(FFA_INVALID_PARAMETERS);
2763 }
2764 break;
J-Alves95fbb312024-03-20 15:19:16 +00002765 case FFA_MEM_LEND_64:
J-Alvesdcad8992023-09-15 14:10:35 +01002766 case FFA_MEM_LEND_32:
2767 /*
2768 * For operations with multiple borrowers only permit XN
2769 * permissions, and both Sender and borrower should have used
2770 * FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED.
2771 */
2772 if (multiple_borrowers) {
2773 if (requested_instruction_access !=
2774 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
2775 dlog_verbose(
2776 "%s: lend/share/donate with multiple "
2777 "borrowers "
2778 "instruction permissions must NOT be "
2779 "specified.\n",
2780 __func__);
2781 return ffa_error(FFA_INVALID_PARAMETERS);
2782 }
2783 break;
2784 }
2785 /* Fall through if the operation targets a single borrower. */
J-Alves95fbb312024-03-20 15:19:16 +00002786 case FFA_MEM_DONATE_64:
J-Alvesdcad8992023-09-15 14:10:35 +01002787 case FFA_MEM_DONATE_32:
2788 if (!multiple_borrowers &&
2789 requested_instruction_access ==
2790 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
2791 dlog_verbose(
2792 "%s: for lend/donate with single borrower "
2793 "instruction permissions must be speficified "
2794 "by borrower\n",
2795 __func__);
2796 return ffa_error(FFA_INVALID_PARAMETERS);
2797 }
2798 break;
2799 default:
2800 panic("%s: Wrong func id provided.\n", __func__);
2801 }
2802
J-Alves96de29f2022-04-26 16:05:24 +01002803 switch (sent_instruction_access) {
2804 case FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED:
2805 case FFA_INSTRUCTION_ACCESS_X:
J-Alvesdcad8992023-09-15 14:10:35 +01002806 if (requested_instruction_access == FFA_INSTRUCTION_ACCESS_X) {
J-Alves96de29f2022-04-26 16:05:24 +01002807 if (permissions != NULL) {
Karl Meakin84710f32023-10-12 15:14:49 +01002808 permissions->instruction_access =
2809 FFA_INSTRUCTION_ACCESS_X;
J-Alves96de29f2022-04-26 16:05:24 +01002810 }
2811 break;
2812 }
J-Alvesdcad8992023-09-15 14:10:35 +01002813 /*
2814 * Fall through if requested permissions are less
2815 * permissive than those provided by the sender.
2816 */
J-Alves96de29f2022-04-26 16:05:24 +01002817 case FFA_INSTRUCTION_ACCESS_NX:
2818 if (requested_instruction_access ==
2819 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED ||
2820 requested_instruction_access == FFA_INSTRUCTION_ACCESS_NX) {
2821 if (permissions != NULL) {
Karl Meakin84710f32023-10-12 15:14:49 +01002822 permissions->instruction_access =
2823 FFA_INSTRUCTION_ACCESS_NX;
J-Alves96de29f2022-04-26 16:05:24 +01002824 }
2825 break;
2826 }
2827 dlog_verbose(
2828 "Invalid instruction access requested; sender "
2829 "specified permissions %#x but receiver requested "
2830 "%#x.\n",
2831 sent_instruction_access, requested_instruction_access);
J-Alvesdcad8992023-09-15 14:10:35 +01002832 return ffa_error(FFA_DENIED);
J-Alves96de29f2022-04-26 16:05:24 +01002833 case FFA_INSTRUCTION_ACCESS_RESERVED:
2834 panic("Got unexpected FFA_INSTRUCTION_ACCESS_RESERVED. Should "
2835 "be checked before this point.");
2836 }
2837
J-Alvesdcad8992023-09-15 14:10:35 +01002838 return (struct ffa_value){.func = FFA_SUCCESS_32};
J-Alves96de29f2022-04-26 16:05:24 +01002839}
2840
2841/**
2842 * Validate the receivers' permissions in the retrieve request against those
2843 * specified by the lender.
2844 * In the `permissions` argument returns the permissions to set at S2 for the
2845 * caller to the FFA_MEMORY_RETRIEVE_REQ.
J-Alves3456e032023-07-20 12:20:05 +01002846 * The function looks into the flag to bypass multiple borrower checks:
2847 * - If not set returns FFA_SUCCESS if all specified permissions are valid.
2848 * - If set returns FFA_SUCCESS if the descriptor contains the permissions
2849 * to the caller of FFA_MEM_RETRIEVE_REQ and they are valid. Other permissions
2850 * are ignored, if provided.
J-Alves96de29f2022-04-26 16:05:24 +01002851 */
2852static struct ffa_value ffa_memory_retrieve_validate_memory_access_list(
2853 struct ffa_memory_region *memory_region,
J-Alves19e20cf2023-08-02 12:48:55 +01002854 struct ffa_memory_region *retrieve_request, ffa_id_t to_vm_id,
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002855 ffa_memory_access_permissions_t *permissions,
2856 struct ffa_memory_access **receiver_ret, uint32_t func_id)
J-Alves96de29f2022-04-26 16:05:24 +01002857{
2858 uint32_t retrieve_receiver_index;
J-Alves3456e032023-07-20 12:20:05 +01002859 bool bypass_multi_receiver_check =
2860 (retrieve_request->flags &
2861 FFA_MEMORY_REGION_FLAG_BYPASS_BORROWERS_CHECK) != 0U;
J-Alvesdcad8992023-09-15 14:10:35 +01002862 const uint32_t region_receiver_count = memory_region->receiver_count;
2863 struct ffa_value ret;
J-Alves96de29f2022-04-26 16:05:24 +01002864
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002865 assert(receiver_ret != NULL);
J-Alves96de29f2022-04-26 16:05:24 +01002866 assert(permissions != NULL);
2867
Karl Meakin84710f32023-10-12 15:14:49 +01002868 *permissions = (ffa_memory_access_permissions_t){0};
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002869
J-Alves3456e032023-07-20 12:20:05 +01002870 if (!bypass_multi_receiver_check) {
J-Alvesdcad8992023-09-15 14:10:35 +01002871 if (retrieve_request->receiver_count != region_receiver_count) {
J-Alves3456e032023-07-20 12:20:05 +01002872 dlog_verbose(
2873 "Retrieve request should contain same list of "
2874 "borrowers, as specified by the lender.\n");
2875 return ffa_error(FFA_INVALID_PARAMETERS);
2876 }
2877 } else {
2878 if (retrieve_request->receiver_count != 1) {
2879 dlog_verbose(
2880 "Set bypass multiple borrower check, receiver "
Daniel Boulbyc0c8f8c2024-12-31 10:51:35 +00002881 "list must be sized 1 in the retrieve request "
2882 "not %x.\n",
J-Alves3456e032023-07-20 12:20:05 +01002883 memory_region->receiver_count);
2884 return ffa_error(FFA_INVALID_PARAMETERS);
2885 }
Daniel Boulbyc0c8f8c2024-12-31 10:51:35 +00002886 if (memory_region->receiver_count == 1) {
2887 dlog_verbose(
2888 "Setting the bypass multiple borrower check "
2889 "flag for a transaction with a single borrower "
2890 "is not allowed.\n");
2891 return ffa_error(FFA_INVALID_PARAMETERS);
2892 }
J-Alves96de29f2022-04-26 16:05:24 +01002893 }
2894
2895 retrieve_receiver_index = retrieve_request->receiver_count;
2896
J-Alves96de29f2022-04-26 16:05:24 +01002897 for (uint32_t i = 0U; i < retrieve_request->receiver_count; i++) {
2898 ffa_memory_access_permissions_t sent_permissions;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002899 struct ffa_memory_access *retrieve_request_receiver =
2900 ffa_memory_region_get_receiver(retrieve_request, i);
2901 assert(retrieve_request_receiver != NULL);
J-Alves96de29f2022-04-26 16:05:24 +01002902 ffa_memory_access_permissions_t requested_permissions =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002903 retrieve_request_receiver->receiver_permissions
2904 .permissions;
J-Alves19e20cf2023-08-02 12:48:55 +01002905 ffa_id_t current_receiver_id =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002906 retrieve_request_receiver->receiver_permissions
2907 .receiver;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002908 struct ffa_memory_access *receiver;
2909 uint32_t mem_region_receiver_index;
2910 bool permissions_RO;
2911 bool clear_memory_flags;
J-Alvesf220d572024-04-24 22:15:14 +01002912 /*
2913 * If the call is at the virtual FF-A instance the caller's
2914 * ID must match an entry in the memory access list.
2915 * In the SPMC, one of the specified receivers could be from
2916 * the NWd.
2917 */
2918 bool found_to_id = vm_id_is_current_world(to_vm_id)
2919 ? (current_receiver_id == to_vm_id)
2920 : (!vm_id_is_current_world(
2921 current_receiver_id));
J-Alves96de29f2022-04-26 16:05:24 +01002922
J-Alves3456e032023-07-20 12:20:05 +01002923 if (bypass_multi_receiver_check && !found_to_id) {
2924 dlog_verbose(
2925 "Bypass multiple borrower check for id %x.\n",
2926 current_receiver_id);
2927 continue;
2928 }
2929
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002930 if (retrieve_request_receiver->composite_memory_region_offset !=
2931 0U) {
2932 dlog_verbose(
2933 "Retriever specified address ranges not "
2934 "supported (got offset %d).\n",
2935 retrieve_request_receiver
2936 ->composite_memory_region_offset);
2937 return ffa_error(FFA_INVALID_PARAMETERS);
2938 }
2939
J-Alves96de29f2022-04-26 16:05:24 +01002940 /*
2941 * Find the current receiver in the transaction descriptor from
2942 * sender.
2943 */
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002944 mem_region_receiver_index =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002945 ffa_memory_region_get_receiver_index(
2946 memory_region, current_receiver_id);
J-Alves96de29f2022-04-26 16:05:24 +01002947
2948 if (mem_region_receiver_index ==
2949 memory_region->receiver_count) {
2950 dlog_verbose("%s: receiver %x not found\n", __func__,
2951 current_receiver_id);
2952 return ffa_error(FFA_DENIED);
2953 }
2954
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002955 receiver = ffa_memory_region_get_receiver(
2956 memory_region, mem_region_receiver_index);
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002957 assert(receiver != NULL);
2958
2959 sent_permissions = receiver->receiver_permissions.permissions;
J-Alves96de29f2022-04-26 16:05:24 +01002960
2961 if (found_to_id) {
2962 retrieve_receiver_index = i;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002963
2964 *receiver_ret = receiver;
J-Alves96de29f2022-04-26 16:05:24 +01002965 }
2966
2967 /*
J-Alvesdcad8992023-09-15 14:10:35 +01002968 * Check if retrieve request memory access list is valid:
2969 * - The retrieve request complies with the specification.
2970 * - Permissions are within those specified by the sender.
J-Alves96de29f2022-04-26 16:05:24 +01002971 */
J-Alvesdcad8992023-09-15 14:10:35 +01002972 ret = ffa_memory_retrieve_is_memory_access_valid(
Karl Meakin84710f32023-10-12 15:14:49 +01002973 func_id, sent_permissions.data_access,
2974 requested_permissions.data_access,
2975 sent_permissions.instruction_access,
2976 requested_permissions.instruction_access,
J-Alvesdcad8992023-09-15 14:10:35 +01002977 found_to_id ? permissions : NULL,
2978 region_receiver_count > 1);
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002979
J-Alvesdcad8992023-09-15 14:10:35 +01002980 if (ret.func != FFA_SUCCESS_32) {
2981 return ret;
J-Alves96de29f2022-04-26 16:05:24 +01002982 }
2983
Karl Meakin84710f32023-10-12 15:14:49 +01002984 permissions_RO =
2985 (permissions->data_access == FFA_DATA_ACCESS_RO);
J-Alvese5262372024-03-27 11:02:03 +00002986 clear_memory_flags =
2987 (retrieve_request->flags &
2988 (FFA_MEMORY_REGION_FLAG_CLEAR |
2989 FFA_MEMORY_REGION_FLAG_CLEAR_RELINQUISH)) != 0U;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002990
J-Alves96de29f2022-04-26 16:05:24 +01002991 /*
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002992 * Can't request PM to clear memory if only provided
2993 * with RO permissions.
J-Alves96de29f2022-04-26 16:05:24 +01002994 */
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002995 if (found_to_id && permissions_RO && clear_memory_flags) {
J-Alves96de29f2022-04-26 16:05:24 +01002996 dlog_verbose(
2997 "Receiver has RO permissions can not request "
2998 "clear.\n");
2999 return ffa_error(FFA_DENIED);
3000 }
Daniel Boulbyde974ca2023-12-12 13:53:31 +00003001
3002 /*
3003 * Check the impdef in the retrieve_request matches the value in
3004 * the original memory send.
3005 */
3006 if (ffa_version_from_memory_access_desc_size(
3007 memory_region->memory_access_desc_size) >=
Karl Meakin0e617d92024-04-05 12:55:22 +01003008 FFA_VERSION_1_2 &&
Daniel Boulbyde974ca2023-12-12 13:53:31 +00003009 ffa_version_from_memory_access_desc_size(
3010 retrieve_request->memory_access_desc_size) >=
Karl Meakin0e617d92024-04-05 12:55:22 +01003011 FFA_VERSION_1_2) {
Daniel Boulbyde974ca2023-12-12 13:53:31 +00003012 if (receiver->impdef.val[0] !=
3013 retrieve_request_receiver->impdef.val[0] ||
3014 receiver->impdef.val[1] !=
3015 retrieve_request_receiver->impdef.val[1]) {
3016 dlog_verbose(
3017 "Impdef value in memory send does not "
J-Alves0a824e92024-04-26 16:20:12 +01003018 "match retrieve request value send "
3019 "value %#lx %#lx retrieve request "
Karl Meakine8937d92024-03-19 16:04:25 +00003020 "value %#lx %#lx\n",
Daniel Boulbyde974ca2023-12-12 13:53:31 +00003021 receiver->impdef.val[0],
3022 receiver->impdef.val[1],
3023 retrieve_request_receiver->impdef
3024 .val[0],
3025 retrieve_request_receiver->impdef
3026 .val[1]);
3027 return ffa_error(FFA_INVALID_PARAMETERS);
3028 }
3029 }
J-Alves96de29f2022-04-26 16:05:24 +01003030 }
3031
3032 if (retrieve_receiver_index == retrieve_request->receiver_count) {
3033 dlog_verbose(
3034 "Retrieve request does not contain caller's (%x) "
3035 "permissions\n",
3036 to_vm_id);
3037 return ffa_error(FFA_INVALID_PARAMETERS);
3038 }
3039
3040 return (struct ffa_value){.func = FFA_SUCCESS_32};
3041}
3042
Daniel Boulby296ee702023-11-28 13:36:55 +00003043/**
3044 * According to section 17.4.3 of the FF-A v1.2 ALP0 specification, the
3045 * hypervisor may issue an FFA_MEM_RETRIEVE_REQ to obtain the memory region
3046 * description of a pending memory sharing operation whose allocator is the SPM,
3047 * for validation purposes before forwarding an FFA_MEM_RECLAIM call. For a
3048 * hypervisor retrieve request the endpoint memory access descriptor count must
3049 * be 0 (for any other retrieve request it must be >= 1).
J-Alvesa9cd7e32022-07-01 13:49:33 +01003050 */
Daniel Boulby296ee702023-11-28 13:36:55 +00003051bool is_ffa_hypervisor_retrieve_request(struct ffa_memory_region *request)
J-Alvesa9cd7e32022-07-01 13:49:33 +01003052{
Daniel Boulby296ee702023-11-28 13:36:55 +00003053 return request->receiver_count == 0U;
3054}
3055
J-Alvesa9cd7e32022-07-01 13:49:33 +01003056/*
3057 * Helper to reset count of fragments retrieved by the hypervisor.
3058 */
3059static void ffa_memory_retrieve_complete_from_hyp(
3060 struct ffa_memory_share_state *share_state)
3061{
3062 if (share_state->hypervisor_fragment_count ==
3063 share_state->fragment_count) {
3064 share_state->hypervisor_fragment_count = 0;
3065 }
3066}
3067
J-Alves089004f2022-07-13 14:25:44 +01003068/**
J-Alves4f0d9c12024-01-17 17:23:11 +00003069 * Prepares the return of the ffa_value for the memory retrieve response.
3070 */
3071static struct ffa_value ffa_memory_retrieve_resp(uint32_t total_length,
3072 uint32_t fragment_length)
3073{
3074 return (struct ffa_value){.func = FFA_MEM_RETRIEVE_RESP_32,
3075 .arg1 = total_length,
3076 .arg2 = fragment_length};
3077}
3078
3079/**
J-Alves089004f2022-07-13 14:25:44 +01003080 * Validate that the memory region descriptor provided by the borrower on
3081 * FFA_MEM_RETRIEVE_REQ, against saved memory region provided by lender at the
3082 * memory sharing call.
3083 */
3084static struct ffa_value ffa_memory_retrieve_validate(
J-Alves4f0d9c12024-01-17 17:23:11 +00003085 ffa_id_t to_id, struct ffa_memory_region *retrieve_request,
3086 uint32_t retrieve_request_length,
J-Alves089004f2022-07-13 14:25:44 +01003087 struct ffa_memory_region *memory_region, uint32_t *receiver_index,
3088 uint32_t share_func)
3089{
3090 ffa_memory_region_flags_t transaction_type =
3091 retrieve_request->flags &
3092 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK;
Olivier Deprez4342a3c2022-02-28 09:37:25 +01003093 enum ffa_memory_security security_state;
J-Alves4f0d9c12024-01-17 17:23:11 +00003094 const uint64_t memory_access_desc_size =
3095 retrieve_request->memory_access_desc_size;
3096 const uint32_t expected_retrieve_request_length =
3097 retrieve_request->receivers_offset +
3098 (uint32_t)(retrieve_request->receiver_count *
3099 memory_access_desc_size);
J-Alves089004f2022-07-13 14:25:44 +01003100
3101 assert(retrieve_request != NULL);
3102 assert(memory_region != NULL);
3103 assert(receiver_index != NULL);
J-Alves089004f2022-07-13 14:25:44 +01003104
J-Alves4f0d9c12024-01-17 17:23:11 +00003105 if (retrieve_request_length != expected_retrieve_request_length) {
3106 dlog_verbose(
3107 "Invalid length for FFA_MEM_RETRIEVE_REQ, expected %d "
3108 "but was %d.\n",
3109 expected_retrieve_request_length,
3110 retrieve_request_length);
3111 return ffa_error(FFA_INVALID_PARAMETERS);
3112 }
3113
3114 if (retrieve_request->sender != memory_region->sender) {
3115 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003116 "Memory with handle %#lx not fully sent, can't "
J-Alves4f0d9c12024-01-17 17:23:11 +00003117 "retrieve.\n",
3118 memory_region->handle);
3119 return ffa_error(FFA_DENIED);
3120 }
3121
3122 /*
3123 * The SPMC can only process retrieve requests to memory share
3124 * operations with one borrower from the other world. It can't
3125 * determine the ID of the NWd VM that invoked the retrieve
3126 * request interface call. It relies on the hypervisor to
3127 * validate the caller's ID against that provided in the
3128 * `receivers` list of the retrieve response.
3129 * In case there is only one borrower from the NWd in the
3130 * transaction descriptor, record that in the `receiver_id` for
3131 * later use, and validate in the retrieve request message.
3132 * This limitation is due to the fact SPMC can't determine the
3133 * index in the memory share structures state to update.
3134 */
3135 if (to_id == HF_HYPERVISOR_VM_ID) {
3136 uint32_t other_world_count = 0;
3137
3138 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
3139 struct ffa_memory_access *receiver =
3140 ffa_memory_region_get_receiver(retrieve_request,
J-Alvesf220d572024-04-24 22:15:14 +01003141 i);
J-Alves4f0d9c12024-01-17 17:23:11 +00003142 assert(receiver != NULL);
3143
J-Alvesf220d572024-04-24 22:15:14 +01003144 if (!vm_id_is_current_world(
3145 receiver->receiver_permissions.receiver)) {
J-Alves4f0d9c12024-01-17 17:23:11 +00003146 other_world_count++;
J-Alvesf220d572024-04-24 22:15:14 +01003147 /* Set it to be used later. */
3148 to_id = receiver->receiver_permissions.receiver;
J-Alves4f0d9c12024-01-17 17:23:11 +00003149 }
3150 }
3151
3152 if (other_world_count > 1) {
3153 dlog_verbose(
J-Alves0a824e92024-04-26 16:20:12 +01003154 "Support one receiver from the other world.\n");
J-Alves4f0d9c12024-01-17 17:23:11 +00003155 return ffa_error(FFA_NOT_SUPPORTED);
3156 }
3157 }
J-Alves089004f2022-07-13 14:25:44 +01003158 /*
3159 * Check that the transaction type expected by the receiver is
3160 * correct, if it has been specified.
3161 */
3162 if (transaction_type !=
3163 FFA_MEMORY_REGION_TRANSACTION_TYPE_UNSPECIFIED &&
3164 transaction_type != (memory_region->flags &
3165 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK)) {
3166 dlog_verbose(
3167 "Incorrect transaction type %#x for "
Karl Meakine8937d92024-03-19 16:04:25 +00003168 "FFA_MEM_RETRIEVE_REQ, expected %#x for handle %#lx.\n",
J-Alves089004f2022-07-13 14:25:44 +01003169 transaction_type,
3170 memory_region->flags &
3171 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK,
3172 retrieve_request->handle);
3173 return ffa_error(FFA_INVALID_PARAMETERS);
3174 }
3175
3176 if (retrieve_request->tag != memory_region->tag) {
3177 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003178 "Incorrect tag %lu for FFA_MEM_RETRIEVE_REQ, expected "
3179 "%lu for handle %#lx.\n",
J-Alves089004f2022-07-13 14:25:44 +01003180 retrieve_request->tag, memory_region->tag,
3181 retrieve_request->handle);
3182 return ffa_error(FFA_INVALID_PARAMETERS);
3183 }
3184
J-Alves4f0d9c12024-01-17 17:23:11 +00003185 *receiver_index =
3186 ffa_memory_region_get_receiver_index(memory_region, to_id);
J-Alves089004f2022-07-13 14:25:44 +01003187
3188 if (*receiver_index == memory_region->receiver_count) {
3189 dlog_verbose(
3190 "Incorrect receiver VM ID %d for "
Karl Meakine8937d92024-03-19 16:04:25 +00003191 "FFA_MEM_RETRIEVE_REQ, for handle %#lx.\n",
J-Alves4f0d9c12024-01-17 17:23:11 +00003192 to_id, memory_region->handle);
J-Alves089004f2022-07-13 14:25:44 +01003193 return ffa_error(FFA_INVALID_PARAMETERS);
3194 }
3195
3196 if ((retrieve_request->flags &
3197 FFA_MEMORY_REGION_ADDRESS_RANGE_HINT_VALID) != 0U) {
3198 dlog_verbose(
3199 "Retriever specified 'address range alignment 'hint' "
3200 "not supported.\n");
3201 return ffa_error(FFA_INVALID_PARAMETERS);
3202 }
3203 if ((retrieve_request->flags &
3204 FFA_MEMORY_REGION_ADDRESS_RANGE_HINT_MASK) != 0) {
3205 dlog_verbose(
3206 "Bits 8-5 must be zero in memory region's flags "
3207 "(address range alignment hint not supported).\n");
3208 return ffa_error(FFA_INVALID_PARAMETERS);
3209 }
3210
3211 if ((retrieve_request->flags & ~0x7FF) != 0U) {
3212 dlog_verbose(
3213 "Bits 31-10 must be zero in memory region's flags.\n");
3214 return ffa_error(FFA_INVALID_PARAMETERS);
3215 }
3216
J-Alves95fbb312024-03-20 15:19:16 +00003217 if ((share_func == FFA_MEM_SHARE_32 ||
3218 share_func == FFA_MEM_SHARE_64) &&
J-Alves089004f2022-07-13 14:25:44 +01003219 (retrieve_request->flags &
3220 (FFA_MEMORY_REGION_FLAG_CLEAR |
3221 FFA_MEMORY_REGION_FLAG_CLEAR_RELINQUISH)) != 0U) {
3222 dlog_verbose(
3223 "Memory Share operation can't clean after relinquish "
3224 "memory region.\n");
3225 return ffa_error(FFA_INVALID_PARAMETERS);
3226 }
3227
3228 /*
3229 * If the borrower needs the memory to be cleared before mapping
3230 * to its address space, the sender should have set the flag
3231 * when calling FFA_MEM_LEND/FFA_MEM_DONATE, else return
3232 * FFA_DENIED.
3233 */
3234 if ((retrieve_request->flags & FFA_MEMORY_REGION_FLAG_CLEAR) != 0U &&
3235 (memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR) == 0U) {
3236 dlog_verbose(
3237 "Borrower needs memory cleared. Sender needs to set "
3238 "flag for clearing memory.\n");
3239 return ffa_error(FFA_DENIED);
3240 }
3241
Olivier Deprez4342a3c2022-02-28 09:37:25 +01003242 /* Memory region attributes NS-Bit MBZ for FFA_MEM_RETRIEVE_REQ. */
Karl Meakin84710f32023-10-12 15:14:49 +01003243 security_state = retrieve_request->attributes.security;
Olivier Deprez4342a3c2022-02-28 09:37:25 +01003244 if (security_state != FFA_MEMORY_SECURITY_UNSPECIFIED) {
3245 dlog_verbose(
3246 "Invalid security state for memory retrieve request "
3247 "operation.\n");
3248 return ffa_error(FFA_INVALID_PARAMETERS);
3249 }
3250
J-Alves089004f2022-07-13 14:25:44 +01003251 /*
3252 * If memory type is not specified, bypass validation of memory
3253 * attributes in the retrieve request. The retriever is expecting to
3254 * obtain this information from the SPMC.
3255 */
Karl Meakin84710f32023-10-12 15:14:49 +01003256 if (retrieve_request->attributes.type == FFA_MEMORY_NOT_SPECIFIED_MEM) {
J-Alves089004f2022-07-13 14:25:44 +01003257 return (struct ffa_value){.func = FFA_SUCCESS_32};
3258 }
3259
3260 /*
3261 * Ensure receiver's attributes are compatible with how
3262 * Hafnium maps memory: Normal Memory, Inner shareable,
3263 * Write-Back Read-Allocate Write-Allocate Cacheable.
3264 */
3265 return ffa_memory_attributes_validate(retrieve_request->attributes);
3266}
3267
J-Alves3f6527c2024-04-25 17:10:57 +01003268/**
3269 * Whilst processing the retrieve request, the operation could be aborted, and
3270 * changes to page tables and the share state structures need to be reverted.
3271 */
3272static void ffa_partition_memory_retrieve_request_undo(
3273 struct vm_locked from_locked,
3274 struct ffa_memory_share_state *share_state, uint32_t receiver_index)
3275{
3276 /*
3277 * Currently this operation is expected for operations involving the
3278 * 'other_world' vm.
3279 */
3280 assert(from_locked.vm->id == HF_OTHER_WORLD_ID);
3281 assert(share_state->retrieved_fragment_count[receiver_index] > 0);
3282
3283 /* Decrement the retrieved fragment count for the given receiver. */
3284 share_state->retrieved_fragment_count[receiver_index]--;
3285}
3286
3287/**
3288 * Whilst processing an hypervisor retrieve request the operation could be
3289 * aborted. There were no updates to PTs in this case, so decrementing the
3290 * fragment count retrieved by the hypervisor should be enough.
3291 */
3292static void ffa_hypervisor_memory_retrieve_request_undo(
3293 struct ffa_memory_share_state *share_state)
3294{
3295 assert(share_state->hypervisor_fragment_count > 0);
3296 share_state->hypervisor_fragment_count--;
3297}
3298
J-Alves4f0d9c12024-01-17 17:23:11 +00003299static struct ffa_value ffa_partition_retrieve_request(
3300 struct share_states_locked share_states,
3301 struct ffa_memory_share_state *share_state, struct vm_locked to_locked,
3302 struct ffa_memory_region *retrieve_request,
3303 uint32_t retrieve_request_length, struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003304{
Karl Meakin84710f32023-10-12 15:14:49 +01003305 ffa_memory_access_permissions_t permissions = {0};
Karl Meakin07a69ab2025-02-07 14:53:19 +00003306 mm_mode_t memory_to_mode;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003307 struct ffa_value ret;
Andrew Walbranca808b12020-05-15 17:22:28 +01003308 struct ffa_composite_memory_region *composite;
3309 uint32_t total_length;
3310 uint32_t fragment_length;
J-Alves19e20cf2023-08-02 12:48:55 +01003311 ffa_id_t receiver_id = to_locked.vm->id;
J-Alves4f0d9c12024-01-17 17:23:11 +00003312 bool is_retrieve_complete = false;
J-Alves4f0d9c12024-01-17 17:23:11 +00003313 const uint64_t memory_access_desc_size =
Daniel Boulbyde974ca2023-12-12 13:53:31 +00003314 retrieve_request->memory_access_desc_size;
J-Alves4f0d9c12024-01-17 17:23:11 +00003315 uint32_t receiver_index;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003316 struct ffa_memory_access *receiver;
J-Alves4f0d9c12024-01-17 17:23:11 +00003317 ffa_memory_handle_t handle = retrieve_request->handle;
Karl Meakin84710f32023-10-12 15:14:49 +01003318 ffa_memory_attributes_t attributes = {0};
Karl Meakin07a69ab2025-02-07 14:53:19 +00003319 mm_mode_t retrieve_mode = 0;
J-Alves4f0d9c12024-01-17 17:23:11 +00003320 struct ffa_memory_region *memory_region = share_state->memory_region;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003321
J-Alves96de29f2022-04-26 16:05:24 +01003322 if (!share_state->sending_complete) {
3323 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003324 "Memory with handle %#lx not fully sent, can't "
J-Alves96de29f2022-04-26 16:05:24 +01003325 "retrieve.\n",
3326 handle);
J-Alves4f0d9c12024-01-17 17:23:11 +00003327 return ffa_error(FFA_INVALID_PARAMETERS);
J-Alves96de29f2022-04-26 16:05:24 +01003328 }
3329
J-Alves4f0d9c12024-01-17 17:23:11 +00003330 /*
3331 * Validate retrieve request, according to what was sent by the
3332 * sender. Function will output the `receiver_index` from the
3333 * provided memory region.
3334 */
3335 ret = ffa_memory_retrieve_validate(
3336 receiver_id, retrieve_request, retrieve_request_length,
3337 memory_region, &receiver_index, share_state->share_func);
J-Alves089004f2022-07-13 14:25:44 +01003338
J-Alves4f0d9c12024-01-17 17:23:11 +00003339 if (ret.func != FFA_SUCCESS_32) {
3340 return ret;
J-Alves089004f2022-07-13 14:25:44 +01003341 }
J-Alves96de29f2022-04-26 16:05:24 +01003342
J-Alves4f0d9c12024-01-17 17:23:11 +00003343 /*
3344 * Validate the requested permissions against the sent
3345 * permissions.
3346 * Outputs the permissions to give to retriever at S2
3347 * PTs.
3348 */
3349 ret = ffa_memory_retrieve_validate_memory_access_list(
3350 memory_region, retrieve_request, receiver_id, &permissions,
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003351 &receiver, share_state->share_func);
J-Alves4f0d9c12024-01-17 17:23:11 +00003352 if (ret.func != FFA_SUCCESS_32) {
3353 return ret;
3354 }
3355
3356 memory_to_mode = ffa_memory_permissions_to_mode(
3357 permissions, share_state->sender_orig_mode);
3358
Daniel Boulby6e261362024-06-13 16:53:00 +01003359 /*
3360 * Check requested memory type is valid with the memory type of the
3361 * owner. E.g. they follow the memory type precedence where Normal
3362 * memory is more permissive than device and therefore device memory
3363 * can only be shared as device memory.
3364 */
3365 if (retrieve_request->attributes.type == FFA_MEMORY_NORMAL_MEM &&
3366 ((share_state->sender_orig_mode & MM_MODE_D) != 0U ||
3367 memory_region->attributes.type == FFA_MEMORY_DEVICE_MEM)) {
3368 dlog_verbose(
3369 "Retrieving device memory as Normal memory is not "
3370 "allowed\n");
3371 return ffa_error(FFA_DENIED);
3372 }
3373
J-Alves4f0d9c12024-01-17 17:23:11 +00003374 ret = ffa_retrieve_check_update(
3375 to_locked, share_state->fragments,
3376 share_state->fragment_constituent_counts,
3377 share_state->fragment_count, memory_to_mode,
J-Alves460d36c2023-10-12 17:02:15 +01003378 share_state->share_func, false, page_pool, &retrieve_mode,
3379 share_state->memory_protected);
J-Alves4f0d9c12024-01-17 17:23:11 +00003380
3381 if (ret.func != FFA_SUCCESS_32) {
3382 return ret;
3383 }
3384
3385 share_state->retrieved_fragment_count[receiver_index] = 1;
3386
3387 is_retrieve_complete =
3388 share_state->retrieved_fragment_count[receiver_index] ==
3389 share_state->fragment_count;
3390
J-Alvesb5084cf2022-07-06 14:20:12 +01003391 /* VMs acquire the RX buffer from SPMC. */
Karl Meakin117c8082024-12-04 16:03:28 +00003392 CHECK(ffa_setup_acquire_receiver_rx(to_locked, &ret));
J-Alvesb5084cf2022-07-06 14:20:12 +01003393
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003394 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01003395 * Copy response to RX buffer of caller and deliver the message.
3396 * This must be done before the share_state is (possibly) freed.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003397 */
Andrew Walbranca808b12020-05-15 17:22:28 +01003398 composite = ffa_memory_region_get_composite(memory_region, 0);
J-Alves4f0d9c12024-01-17 17:23:11 +00003399
Andrew Walbranca808b12020-05-15 17:22:28 +01003400 /*
J-Alves460d36c2023-10-12 17:02:15 +01003401 * Set the security state in the memory retrieve response attributes
3402 * if specified by the target mode.
3403 */
Karl Meakin117c8082024-12-04 16:03:28 +00003404 attributes = ffa_memory_add_security_bit_from_mode(
Karl Meakin3d32eef2024-11-25 16:40:09 +00003405 memory_region->attributes, retrieve_mode);
J-Alves460d36c2023-10-12 17:02:15 +01003406
3407 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01003408 * Constituents which we received in the first fragment should
3409 * always fit in the first fragment we are sending, because the
3410 * header is the same size in both cases and we have a fixed
3411 * message buffer size. So `ffa_retrieved_memory_region_init`
3412 * should never fail.
Andrew Walbranca808b12020-05-15 17:22:28 +01003413 */
Olivier Deprez878bd5b2021-04-15 19:05:10 +02003414
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003415 /* Provide the permissions that had been provided. */
3416 receiver->receiver_permissions.permissions = permissions;
3417
3418 /*
3419 * Prepare the memory region descriptor for the retrieve response.
3420 * Provide the pointer to the receiver tracked in the share state
J-Alves7b9cc432024-04-04 10:57:17 +01003421 * structures.
3422 * At this point the retrieve request descriptor from the partition
3423 * has been processed. The `retrieve_request` is expected to be in
3424 * a region that is handled by the SPMC/Hyp. Reuse the same buffer to
3425 * prepare the retrieve response before copying it to the RX buffer of
3426 * the caller.
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003427 */
Andrew Walbranca808b12020-05-15 17:22:28 +01003428 CHECK(ffa_retrieved_memory_region_init(
J-Alves7b9cc432024-04-04 10:57:17 +01003429 retrieve_request, to_locked.vm->ffa_version, HF_MAILBOX_SIZE,
3430 memory_region->sender, attributes, memory_region->flags, handle,
3431 permissions, receiver, 1, memory_access_desc_size,
3432 composite->page_count, composite->constituent_count,
3433 share_state->fragments[0],
Andrew Walbranca808b12020-05-15 17:22:28 +01003434 share_state->fragment_constituent_counts[0], &total_length,
3435 &fragment_length));
J-Alvesb5084cf2022-07-06 14:20:12 +01003436
J-Alves7b9cc432024-04-04 10:57:17 +01003437 /*
3438 * Copy the message from the buffer into the partition's mailbox.
3439 * The operation might fail unexpectedly due to change in PAS address
3440 * space, or improper values to the sizes of the structures.
3441 */
3442 if (!memcpy_trapped(to_locked.vm->mailbox.recv, HF_MAILBOX_SIZE,
3443 retrieve_request, fragment_length)) {
3444 dlog_error(
3445 "%s: aborted the copy of response to RX buffer of "
3446 "%x.\n",
3447 __func__, to_locked.vm->id);
J-Alves3f6527c2024-04-25 17:10:57 +01003448
3449 ffa_partition_memory_retrieve_request_undo(
3450 to_locked, share_state, receiver_index);
3451
J-Alves7b9cc432024-04-04 10:57:17 +01003452 return ffa_error(FFA_ABORTED);
3453 }
3454
J-Alves4f0d9c12024-01-17 17:23:11 +00003455 if (is_retrieve_complete) {
Andrew Walbranca808b12020-05-15 17:22:28 +01003456 ffa_memory_retrieve_complete(share_states, share_state,
3457 page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003458 }
J-Alves4f0d9c12024-01-17 17:23:11 +00003459
3460 return ffa_memory_retrieve_resp(total_length, fragment_length);
3461}
3462
3463static struct ffa_value ffa_hypervisor_retrieve_request(
3464 struct ffa_memory_share_state *share_state, struct vm_locked to_locked,
3465 struct ffa_memory_region *retrieve_request)
3466{
3467 struct ffa_value ret;
3468 struct ffa_composite_memory_region *composite;
3469 uint32_t total_length;
3470 uint32_t fragment_length;
J-Alves4f0d9c12024-01-17 17:23:11 +00003471 ffa_memory_attributes_t attributes;
J-Alves7b6ab612024-01-24 09:54:54 +00003472 uint64_t memory_access_desc_size;
J-Alves4f0d9c12024-01-17 17:23:11 +00003473 struct ffa_memory_region *memory_region;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003474 struct ffa_memory_access *receiver;
J-Alves4f0d9c12024-01-17 17:23:11 +00003475 ffa_memory_handle_t handle = retrieve_request->handle;
3476
J-Alves4f0d9c12024-01-17 17:23:11 +00003477 memory_region = share_state->memory_region;
3478
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003479 assert(to_locked.vm->id == HF_HYPERVISOR_VM_ID);
3480
J-Alves7b6ab612024-01-24 09:54:54 +00003481 switch (to_locked.vm->ffa_version) {
Karl Meakin0e617d92024-04-05 12:55:22 +01003482 case FFA_VERSION_1_2:
J-Alves7b6ab612024-01-24 09:54:54 +00003483 memory_access_desc_size = sizeof(struct ffa_memory_access);
3484 break;
Karl Meakin0e617d92024-04-05 12:55:22 +01003485 case FFA_VERSION_1_0:
3486 case FFA_VERSION_1_1:
J-Alves7b6ab612024-01-24 09:54:54 +00003487 memory_access_desc_size = sizeof(struct ffa_memory_access_v1_0);
3488 break;
3489 default:
3490 panic("version not supported: %x\n", to_locked.vm->ffa_version);
3491 }
3492
J-Alves4f0d9c12024-01-17 17:23:11 +00003493 if (share_state->hypervisor_fragment_count != 0U) {
3494 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003495 "Memory with handle %#lx already retrieved by "
J-Alves4f0d9c12024-01-17 17:23:11 +00003496 "the hypervisor.\n",
3497 handle);
3498 return ffa_error(FFA_DENIED);
3499 }
3500
3501 share_state->hypervisor_fragment_count = 1;
3502
J-Alves4f0d9c12024-01-17 17:23:11 +00003503 /* VMs acquire the RX buffer from SPMC. */
Karl Meakin117c8082024-12-04 16:03:28 +00003504 CHECK(ffa_setup_acquire_receiver_rx(to_locked, &ret));
J-Alves4f0d9c12024-01-17 17:23:11 +00003505
3506 /*
3507 * Copy response to RX buffer of caller and deliver the message.
3508 * This must be done before the share_state is (possibly) freed.
3509 */
3510 composite = ffa_memory_region_get_composite(memory_region, 0);
3511
3512 /*
3513 * Constituents which we received in the first fragment should
3514 * always fit in the first fragment we are sending, because the
3515 * header is the same size in both cases and we have a fixed
3516 * message buffer size. So `ffa_retrieved_memory_region_init`
3517 * should never fail.
3518 */
3519
3520 /*
3521 * Set the security state in the memory retrieve response attributes
3522 * if specified by the target mode.
3523 */
Karl Meakin117c8082024-12-04 16:03:28 +00003524 attributes = ffa_memory_add_security_bit_from_mode(
J-Alves4f0d9c12024-01-17 17:23:11 +00003525 memory_region->attributes, share_state->sender_orig_mode);
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003526
3527 receiver = ffa_memory_region_get_receiver(memory_region, 0);
3528
J-Alves7b9cc432024-04-04 10:57:17 +01003529 /*
3530 * At this point the `retrieve_request` is expected to be in a section
3531 * managed by the hypervisor.
3532 */
J-Alves4f0d9c12024-01-17 17:23:11 +00003533 CHECK(ffa_retrieved_memory_region_init(
J-Alves7b9cc432024-04-04 10:57:17 +01003534 retrieve_request, to_locked.vm->ffa_version, HF_MAILBOX_SIZE,
3535 memory_region->sender, attributes, memory_region->flags, handle,
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003536 receiver->receiver_permissions.permissions, receiver,
3537 memory_region->receiver_count, memory_access_desc_size,
J-Alves4f0d9c12024-01-17 17:23:11 +00003538 composite->page_count, composite->constituent_count,
3539 share_state->fragments[0],
3540 share_state->fragment_constituent_counts[0], &total_length,
3541 &fragment_length));
3542
J-Alves7b9cc432024-04-04 10:57:17 +01003543 /*
3544 * Copy the message from the buffer into the hypervisor's mailbox.
3545 * The operation might fail unexpectedly due to change in PAS, or
3546 * improper values for the sizes of the structures.
3547 */
3548 if (!memcpy_trapped(to_locked.vm->mailbox.recv, HF_MAILBOX_SIZE,
3549 retrieve_request, fragment_length)) {
3550 dlog_error(
3551 "%s: aborted the copy of response to RX buffer of "
3552 "%x.\n",
3553 __func__, to_locked.vm->id);
J-Alves3f6527c2024-04-25 17:10:57 +01003554
3555 ffa_hypervisor_memory_retrieve_request_undo(share_state);
3556
J-Alves7b9cc432024-04-04 10:57:17 +01003557 return ffa_error(FFA_ABORTED);
3558 }
3559
J-Alves3f6527c2024-04-25 17:10:57 +01003560 ffa_memory_retrieve_complete_from_hyp(share_state);
3561
J-Alves4f0d9c12024-01-17 17:23:11 +00003562 return ffa_memory_retrieve_resp(total_length, fragment_length);
3563}
3564
3565struct ffa_value ffa_memory_retrieve(struct vm_locked to_locked,
3566 struct ffa_memory_region *retrieve_request,
3567 uint32_t retrieve_request_length,
3568 struct mpool *page_pool)
3569{
3570 ffa_memory_handle_t handle = retrieve_request->handle;
3571 struct share_states_locked share_states;
3572 struct ffa_memory_share_state *share_state;
3573 struct ffa_value ret;
3574
3575 dump_share_states();
3576
3577 share_states = share_states_lock();
3578 share_state = get_share_state(share_states, handle);
3579 if (share_state == NULL) {
Karl Meakine8937d92024-03-19 16:04:25 +00003580 dlog_verbose("Invalid handle %#lx for FFA_MEM_RETRIEVE_REQ.\n",
J-Alves4f0d9c12024-01-17 17:23:11 +00003581 handle);
3582 ret = ffa_error(FFA_INVALID_PARAMETERS);
3583 goto out;
3584 }
3585
Daniel Boulby296ee702023-11-28 13:36:55 +00003586 if (is_ffa_hypervisor_retrieve_request(retrieve_request)) {
J-Alves4f0d9c12024-01-17 17:23:11 +00003587 ret = ffa_hypervisor_retrieve_request(share_state, to_locked,
3588 retrieve_request);
3589 } else {
3590 ret = ffa_partition_retrieve_request(
3591 share_states, share_state, to_locked, retrieve_request,
3592 retrieve_request_length, page_pool);
3593 }
3594
3595 /* Track use of the RX buffer if the handling has succeeded. */
3596 if (ret.func == FFA_MEM_RETRIEVE_RESP_32) {
3597 to_locked.vm->mailbox.recv_func = FFA_MEM_RETRIEVE_RESP_32;
3598 to_locked.vm->mailbox.state = MAILBOX_STATE_FULL;
3599 }
3600
Andrew Walbranca808b12020-05-15 17:22:28 +01003601out:
3602 share_states_unlock(&share_states);
3603 dump_share_states();
3604 return ret;
3605}
3606
J-Alves5da37d92022-10-24 16:33:48 +01003607/**
3608 * Determine expected fragment offset according to the FF-A version of
3609 * the caller.
3610 */
3611static uint32_t ffa_memory_retrieve_expected_offset_per_ffa_version(
3612 struct ffa_memory_region *memory_region,
Karl Meakin0e617d92024-04-05 12:55:22 +01003613 uint32_t retrieved_constituents_count, enum ffa_version ffa_version)
J-Alves5da37d92022-10-24 16:33:48 +01003614{
3615 uint32_t expected_fragment_offset;
3616 uint32_t composite_constituents_offset;
3617
Karl Meakin0e617d92024-04-05 12:55:22 +01003618 if (ffa_version >= FFA_VERSION_1_1) {
J-Alves5da37d92022-10-24 16:33:48 +01003619 /*
3620 * Hafnium operates memory regions in FF-A v1.1 format, so we
3621 * can retrieve the constituents offset from descriptor.
3622 */
3623 composite_constituents_offset =
3624 ffa_composite_constituent_offset(memory_region, 0);
Karl Meakin0e617d92024-04-05 12:55:22 +01003625 } else if (ffa_version == FFA_VERSION_1_0) {
J-Alves5da37d92022-10-24 16:33:48 +01003626 /*
3627 * If retriever is FF-A v1.0, determine the composite offset
3628 * as it is expected to have been configured in the
3629 * retrieve response.
3630 */
3631 composite_constituents_offset =
3632 sizeof(struct ffa_memory_region_v1_0) +
3633 RECEIVERS_COUNT_IN_RETRIEVE_RESP *
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003634 sizeof(struct ffa_memory_access_v1_0) +
J-Alves5da37d92022-10-24 16:33:48 +01003635 sizeof(struct ffa_composite_memory_region);
3636 } else {
3637 panic("%s received an invalid FF-A version.\n", __func__);
3638 }
3639
3640 expected_fragment_offset =
3641 composite_constituents_offset +
3642 retrieved_constituents_count *
3643 sizeof(struct ffa_memory_region_constituent) -
Karl Meakin66a38bd2024-05-28 16:00:56 +01003644 (size_t)(memory_region->memory_access_desc_size *
3645 (memory_region->receiver_count - 1));
J-Alves5da37d92022-10-24 16:33:48 +01003646
3647 return expected_fragment_offset;
3648}
3649
Andrew Walbranca808b12020-05-15 17:22:28 +01003650struct ffa_value ffa_memory_retrieve_continue(struct vm_locked to_locked,
3651 ffa_memory_handle_t handle,
3652 uint32_t fragment_offset,
J-Alves19e20cf2023-08-02 12:48:55 +01003653 ffa_id_t sender_vm_id,
J-Alvesc3fd9752024-04-04 11:45:33 +01003654 void *retrieve_continue_page,
Andrew Walbranca808b12020-05-15 17:22:28 +01003655 struct mpool *page_pool)
3656{
3657 struct ffa_memory_region *memory_region;
3658 struct share_states_locked share_states;
3659 struct ffa_memory_share_state *share_state;
3660 struct ffa_value ret;
3661 uint32_t fragment_index;
3662 uint32_t retrieved_constituents_count;
3663 uint32_t i;
3664 uint32_t expected_fragment_offset;
3665 uint32_t remaining_constituent_count;
3666 uint32_t fragment_length;
J-Alvesc7484f12022-05-13 12:41:14 +01003667 uint32_t receiver_index;
J-Alves59ed0042022-07-28 18:26:41 +01003668 bool continue_ffa_hyp_mem_retrieve_req;
Andrew Walbranca808b12020-05-15 17:22:28 +01003669
3670 dump_share_states();
3671
3672 share_states = share_states_lock();
Karl Meakin4a2854a2023-06-30 16:26:52 +01003673 share_state = get_share_state(share_states, handle);
J-Alvesb56aac82023-11-10 09:44:43 +00003674 if (share_state == NULL) {
Karl Meakine8937d92024-03-19 16:04:25 +00003675 dlog_verbose("Invalid handle %#lx for FFA_MEM_FRAG_RX.\n",
Andrew Walbranca808b12020-05-15 17:22:28 +01003676 handle);
3677 ret = ffa_error(FFA_INVALID_PARAMETERS);
3678 goto out;
3679 }
3680
3681 memory_region = share_state->memory_region;
3682 CHECK(memory_region != NULL);
3683
Andrew Walbranca808b12020-05-15 17:22:28 +01003684 if (!share_state->sending_complete) {
3685 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003686 "Memory with handle %#lx not fully sent, can't "
Andrew Walbranca808b12020-05-15 17:22:28 +01003687 "retrieve.\n",
3688 handle);
3689 ret = ffa_error(FFA_INVALID_PARAMETERS);
3690 goto out;
3691 }
3692
J-Alves59ed0042022-07-28 18:26:41 +01003693 /*
3694 * If retrieve request from the hypervisor has been initiated in the
3695 * given share_state, continue it, else assume it is a continuation of
J-Alvesc3fd9752024-04-04 11:45:33 +01003696 * retrieve request from a partition.
J-Alves59ed0042022-07-28 18:26:41 +01003697 */
3698 continue_ffa_hyp_mem_retrieve_req =
3699 (to_locked.vm->id == HF_HYPERVISOR_VM_ID) &&
3700 (share_state->hypervisor_fragment_count != 0U) &&
J-Alves661e1b72023-08-02 13:39:40 +01003701 ffa_is_vm_id(sender_vm_id);
Andrew Walbranca808b12020-05-15 17:22:28 +01003702
J-Alves59ed0042022-07-28 18:26:41 +01003703 if (!continue_ffa_hyp_mem_retrieve_req) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003704 receiver_index = ffa_memory_region_get_receiver_index(
J-Alves59ed0042022-07-28 18:26:41 +01003705 memory_region, to_locked.vm->id);
3706
3707 if (receiver_index == memory_region->receiver_count) {
3708 dlog_verbose(
3709 "Caller of FFA_MEM_FRAG_RX (%x) is not a "
Karl Meakine8937d92024-03-19 16:04:25 +00003710 "borrower to memory sharing transaction "
3711 "(%lx)\n",
J-Alves59ed0042022-07-28 18:26:41 +01003712 to_locked.vm->id, handle);
3713 ret = ffa_error(FFA_INVALID_PARAMETERS);
3714 goto out;
3715 }
3716
J-Alvesc3fd9752024-04-04 11:45:33 +01003717 fragment_index =
3718 share_state->retrieved_fragment_count[receiver_index];
3719
3720 if (fragment_index == 0 ||
3721 fragment_index >= share_state->fragment_count) {
J-Alves59ed0042022-07-28 18:26:41 +01003722 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003723 "Retrieval of memory with handle %#lx not yet "
J-Alves59ed0042022-07-28 18:26:41 +01003724 "started or already completed (%d/%d fragments "
3725 "retrieved).\n",
3726 handle,
3727 share_state->retrieved_fragment_count
3728 [receiver_index],
3729 share_state->fragment_count);
3730 ret = ffa_error(FFA_INVALID_PARAMETERS);
3731 goto out;
3732 }
J-Alves59ed0042022-07-28 18:26:41 +01003733 } else {
J-Alvesc3fd9752024-04-04 11:45:33 +01003734 fragment_index = share_state->hypervisor_fragment_count;
3735
3736 if (fragment_index == 0 ||
3737 fragment_index >= share_state->fragment_count) {
J-Alves59ed0042022-07-28 18:26:41 +01003738 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003739 "Retrieve of memory with handle %lx not "
J-Alves59ed0042022-07-28 18:26:41 +01003740 "started from hypervisor.\n",
3741 handle);
3742 ret = ffa_error(FFA_INVALID_PARAMETERS);
3743 goto out;
3744 }
3745
3746 if (memory_region->sender != sender_vm_id) {
3747 dlog_verbose(
3748 "Sender ID (%x) is not as expected for memory "
Karl Meakine8937d92024-03-19 16:04:25 +00003749 "handle %lx\n",
J-Alves59ed0042022-07-28 18:26:41 +01003750 sender_vm_id, handle);
3751 ret = ffa_error(FFA_INVALID_PARAMETERS);
3752 goto out;
3753 }
3754
J-Alves59ed0042022-07-28 18:26:41 +01003755 receiver_index = 0;
3756 }
Andrew Walbranca808b12020-05-15 17:22:28 +01003757
3758 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01003759 * Check that the given fragment offset is correct by counting
3760 * how many constituents were in the fragments previously sent.
Andrew Walbranca808b12020-05-15 17:22:28 +01003761 */
3762 retrieved_constituents_count = 0;
3763 for (i = 0; i < fragment_index; ++i) {
3764 retrieved_constituents_count +=
3765 share_state->fragment_constituent_counts[i];
3766 }
J-Alvesc7484f12022-05-13 12:41:14 +01003767
3768 CHECK(memory_region->receiver_count > 0);
3769
Andrew Walbranca808b12020-05-15 17:22:28 +01003770 expected_fragment_offset =
J-Alves5da37d92022-10-24 16:33:48 +01003771 ffa_memory_retrieve_expected_offset_per_ffa_version(
3772 memory_region, retrieved_constituents_count,
3773 to_locked.vm->ffa_version);
3774
Andrew Walbranca808b12020-05-15 17:22:28 +01003775 if (fragment_offset != expected_fragment_offset) {
3776 dlog_verbose("Fragment offset was %d but expected %d.\n",
3777 fragment_offset, expected_fragment_offset);
3778 ret = ffa_error(FFA_INVALID_PARAMETERS);
3779 goto out;
3780 }
3781
J-Alves4f0d9c12024-01-17 17:23:11 +00003782 /*
3783 * When hafnium is the hypervisor, acquire the RX buffer of a VM, that
3784 * is currently ownder by the SPMC.
3785 */
Karl Meakin117c8082024-12-04 16:03:28 +00003786 assert(ffa_setup_acquire_receiver_rx(to_locked, &ret));
J-Alves59ed0042022-07-28 18:26:41 +01003787
Andrew Walbranca808b12020-05-15 17:22:28 +01003788 remaining_constituent_count = ffa_memory_fragment_init(
J-Alvesc3fd9752024-04-04 11:45:33 +01003789 (struct ffa_memory_region_constituent *)retrieve_continue_page,
3790 HF_MAILBOX_SIZE, share_state->fragments[fragment_index],
Andrew Walbranca808b12020-05-15 17:22:28 +01003791 share_state->fragment_constituent_counts[fragment_index],
3792 &fragment_length);
3793 CHECK(remaining_constituent_count == 0);
J-Alves674e4de2024-01-17 16:20:32 +00003794
J-Alvesc3fd9752024-04-04 11:45:33 +01003795 /*
3796 * Return FFA_ERROR(FFA_ABORTED) in case the access to the partition's
3797 * RX buffer results in a GPF exception. Could happen if the retrieve
3798 * request is for a VM or the Hypervisor retrieve request, if the PAS
3799 * has been changed externally.
3800 */
3801 if (!memcpy_trapped(to_locked.vm->mailbox.recv, HF_MAILBOX_SIZE,
3802 retrieve_continue_page, fragment_length)) {
3803 dlog_error(
3804 "%s: aborted copying fragment to RX buffer of %#x.\n",
3805 __func__, to_locked.vm->id);
3806 ret = ffa_error(FFA_ABORTED);
3807 goto out;
3808 }
3809
Andrew Walbranca808b12020-05-15 17:22:28 +01003810 to_locked.vm->mailbox.recv_func = FFA_MEM_FRAG_TX_32;
J-Alvese8c8c2b2022-12-16 15:34:48 +00003811 to_locked.vm->mailbox.state = MAILBOX_STATE_FULL;
Andrew Walbranca808b12020-05-15 17:22:28 +01003812
J-Alves59ed0042022-07-28 18:26:41 +01003813 if (!continue_ffa_hyp_mem_retrieve_req) {
3814 share_state->retrieved_fragment_count[receiver_index]++;
3815 if (share_state->retrieved_fragment_count[receiver_index] ==
3816 share_state->fragment_count) {
3817 ffa_memory_retrieve_complete(share_states, share_state,
3818 page_pool);
3819 }
3820 } else {
3821 share_state->hypervisor_fragment_count++;
3822
3823 ffa_memory_retrieve_complete_from_hyp(share_state);
3824 }
Andrew Walbranca808b12020-05-15 17:22:28 +01003825 ret = (struct ffa_value){.func = FFA_MEM_FRAG_TX_32,
3826 .arg1 = (uint32_t)handle,
3827 .arg2 = (uint32_t)(handle >> 32),
3828 .arg3 = fragment_length};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003829
3830out:
3831 share_states_unlock(&share_states);
3832 dump_share_states();
3833 return ret;
3834}
3835
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003836struct ffa_value ffa_memory_relinquish(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003837 struct vm_locked from_locked,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003838 struct ffa_mem_relinquish *relinquish_request, struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003839{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003840 ffa_memory_handle_t handle = relinquish_request->handle;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003841 struct share_states_locked share_states;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003842 struct ffa_memory_share_state *share_state;
3843 struct ffa_memory_region *memory_region;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003844 bool clear;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003845 struct ffa_value ret;
J-Alves8eb19162022-04-28 10:56:48 +01003846 uint32_t receiver_index;
J-Alves3c5b2072022-11-21 12:45:40 +00003847 bool receivers_relinquished_memory;
Karl Meakin84710f32023-10-12 15:14:49 +01003848 ffa_memory_access_permissions_t receiver_permissions = {0};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003849
Andrew Walbrana65a1322020-04-06 19:32:32 +01003850 if (relinquish_request->endpoint_count != 1) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003851 dlog_verbose(
J-Alves0a824e92024-04-26 16:20:12 +01003852 "Stream endpoints not supported (got %d endpoints on "
3853 "FFA_MEM_RELINQUISH, expected 1).\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003854 relinquish_request->endpoint_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003855 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003856 }
3857
J-Alvesbd060342024-04-26 18:44:31 +01003858 if (vm_id_is_current_world(from_locked.vm->id) &&
3859 relinquish_request->endpoints[0] != from_locked.vm->id) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003860 dlog_verbose(
J-Alves0a824e92024-04-26 16:20:12 +01003861 "VM ID %d in relinquish message doesn't match calling "
3862 "VM ID %d.\n",
Andrew Walbrana65a1322020-04-06 19:32:32 +01003863 relinquish_request->endpoints[0], from_locked.vm->id);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003864 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003865 }
3866
3867 dump_share_states();
3868
3869 share_states = share_states_lock();
Karl Meakin4a2854a2023-06-30 16:26:52 +01003870 share_state = get_share_state(share_states, handle);
J-Alvesb56aac82023-11-10 09:44:43 +00003871 if (share_state == NULL) {
Karl Meakine8937d92024-03-19 16:04:25 +00003872 dlog_verbose("Invalid handle %#lx for FFA_MEM_RELINQUISH.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003873 handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003874 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003875 goto out;
3876 }
3877
Andrew Walbranca808b12020-05-15 17:22:28 +01003878 if (!share_state->sending_complete) {
3879 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003880 "Memory with handle %#lx not fully sent, can't "
Andrew Walbranca808b12020-05-15 17:22:28 +01003881 "relinquish.\n",
3882 handle);
3883 ret = ffa_error(FFA_INVALID_PARAMETERS);
3884 goto out;
3885 }
3886
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003887 memory_region = share_state->memory_region;
3888 CHECK(memory_region != NULL);
3889
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003890 receiver_index = ffa_memory_region_get_receiver_index(
J-Alvesbd060342024-04-26 18:44:31 +01003891 memory_region, relinquish_request->endpoints[0]);
J-Alves8eb19162022-04-28 10:56:48 +01003892
3893 if (receiver_index == memory_region->receiver_count) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003894 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01003895 "VM ID %d tried to relinquish memory region "
Karl Meakine8937d92024-03-19 16:04:25 +00003896 "with handle %#lx and it is not a valid borrower.\n",
J-Alves8eb19162022-04-28 10:56:48 +01003897 from_locked.vm->id, handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003898 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003899 goto out;
3900 }
3901
J-Alves8eb19162022-04-28 10:56:48 +01003902 if (share_state->retrieved_fragment_count[receiver_index] !=
Andrew Walbranca808b12020-05-15 17:22:28 +01003903 share_state->fragment_count) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003904 dlog_verbose(
J-Alves0a824e92024-04-26 16:20:12 +01003905 "Memory with handle %#lx not yet fully retrieved, "
J-Alves8eb19162022-04-28 10:56:48 +01003906 "receiver %x can't relinquish.\n",
3907 handle, from_locked.vm->id);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003908 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003909 goto out;
3910 }
3911
J-Alves3c5b2072022-11-21 12:45:40 +00003912 /*
3913 * Either clear if requested in relinquish call, or in a retrieve
3914 * request from one of the borrowers.
3915 */
3916 receivers_relinquished_memory = true;
3917
3918 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
3919 struct ffa_memory_access *receiver =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003920 ffa_memory_region_get_receiver(memory_region, i);
3921 assert(receiver != NULL);
J-Alves3c5b2072022-11-21 12:45:40 +00003922 if (receiver->receiver_permissions.receiver ==
3923 from_locked.vm->id) {
J-Alves639ddfc2023-11-21 14:17:26 +00003924 receiver_permissions =
3925 receiver->receiver_permissions.permissions;
J-Alves3c5b2072022-11-21 12:45:40 +00003926 continue;
3927 }
3928
3929 if (share_state->retrieved_fragment_count[i] != 0U) {
3930 receivers_relinquished_memory = false;
3931 break;
3932 }
3933 }
3934
3935 clear = receivers_relinquished_memory &&
Daniel Boulby2e14ebe2024-01-15 16:21:44 +00003936 ((relinquish_request->flags & FFA_MEMORY_REGION_FLAG_CLEAR) !=
3937 0U);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003938
3939 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01003940 * Clear is not allowed for memory that was shared, as the
3941 * original sender still has access to the memory.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003942 */
J-Alves95fbb312024-03-20 15:19:16 +00003943 if (clear && (share_state->share_func == FFA_MEM_SHARE_32 ||
3944 share_state->share_func == FFA_MEM_SHARE_64)) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003945 dlog_verbose("Memory which was shared can't be cleared.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003946 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003947 goto out;
3948 }
3949
J-Alvesb886d492024-04-15 10:55:29 +01003950 if (clear && receiver_permissions.data_access == FFA_DATA_ACCESS_RO) {
J-Alves639ddfc2023-11-21 14:17:26 +00003951 dlog_verbose("%s: RO memory can't use clear memory flag.\n",
3952 __func__);
3953 ret = ffa_error(FFA_DENIED);
3954 goto out;
3955 }
3956
Andrew Walbranca808b12020-05-15 17:22:28 +01003957 ret = ffa_relinquish_check_update(
J-Alves26483382023-04-20 12:01:49 +01003958 from_locked, share_state->fragments,
Andrew Walbranca808b12020-05-15 17:22:28 +01003959 share_state->fragment_constituent_counts,
J-Alves69cdfd92024-04-26 11:40:59 +01003960 share_state->fragment_count, share_state->sender_orig_mode,
3961 page_pool, clear);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003962
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003963 if (ret.func == FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003964 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01003965 * Mark memory handle as not retrieved, so it can be
3966 * reclaimed (or retrieved again).
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003967 */
J-Alves8eb19162022-04-28 10:56:48 +01003968 share_state->retrieved_fragment_count[receiver_index] = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003969 }
3970
3971out:
3972 share_states_unlock(&share_states);
3973 dump_share_states();
3974 return ret;
3975}
3976
3977/**
J-Alvesa9cd7e32022-07-01 13:49:33 +01003978 * Validates that the reclaim transition is allowed for the given
3979 * handle, updates the page table of the reclaiming VM, and frees the
3980 * internal state associated with the handle.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003981 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003982struct ffa_value ffa_memory_reclaim(struct vm_locked to_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01003983 ffa_memory_handle_t handle,
3984 ffa_memory_region_flags_t flags,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003985 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003986{
3987 struct share_states_locked share_states;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003988 struct ffa_memory_share_state *share_state;
3989 struct ffa_memory_region *memory_region;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003990 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003991
3992 dump_share_states();
3993
3994 share_states = share_states_lock();
Karl Meakin52cdfe72023-06-30 14:49:10 +01003995
Karl Meakin4a2854a2023-06-30 16:26:52 +01003996 share_state = get_share_state(share_states, handle);
J-Alvesb56aac82023-11-10 09:44:43 +00003997 if (share_state == NULL) {
Karl Meakine8937d92024-03-19 16:04:25 +00003998 dlog_verbose("Invalid handle %#lx for FFA_MEM_RECLAIM.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003999 handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01004000 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00004001 goto out;
4002 }
Karl Meakin4a2854a2023-06-30 16:26:52 +01004003 memory_region = share_state->memory_region;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00004004
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00004005 CHECK(memory_region != NULL);
4006
J-Alvesa9cd7e32022-07-01 13:49:33 +01004007 if (vm_id_is_current_world(to_locked.vm->id) &&
4008 to_locked.vm->id != memory_region->sender) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00004009 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00004010 "VM %#x attempted to reclaim memory handle %#lx "
Olivier Deprezf92e5d42020-11-13 16:00:54 +01004011 "originally sent by VM %#x.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00004012 to_locked.vm->id, handle, memory_region->sender);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01004013 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00004014 goto out;
4015 }
4016
Andrew Walbranca808b12020-05-15 17:22:28 +01004017 if (!share_state->sending_complete) {
4018 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00004019 "Memory with handle %#lx not fully sent, can't "
Andrew Walbranca808b12020-05-15 17:22:28 +01004020 "reclaim.\n",
4021 handle);
4022 ret = ffa_error(FFA_INVALID_PARAMETERS);
4023 goto out;
4024 }
4025
J-Alves752236c2022-04-28 11:07:47 +01004026 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
4027 if (share_state->retrieved_fragment_count[i] != 0) {
J-Alves9bbcb872024-04-25 17:19:00 +01004028 struct ffa_memory_access *receiver =
4029 ffa_memory_region_get_receiver(memory_region,
4030 i);
4031
4032 assert(receiver != NULL);
4033 (void)receiver;
J-Alves752236c2022-04-28 11:07:47 +01004034 dlog_verbose(
J-Alves0a824e92024-04-26 16:20:12 +01004035 "Tried to reclaim memory handle %#lx that has "
4036 "not been relinquished by all borrowers(%x).\n",
J-Alves752236c2022-04-28 11:07:47 +01004037 handle,
J-Alves9bbcb872024-04-25 17:19:00 +01004038 receiver->receiver_permissions.receiver);
J-Alves752236c2022-04-28 11:07:47 +01004039 ret = ffa_error(FFA_DENIED);
4040 goto out;
4041 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00004042 }
4043
Andrew Walbranca808b12020-05-15 17:22:28 +01004044 ret = ffa_retrieve_check_update(
J-Alves26483382023-04-20 12:01:49 +01004045 to_locked, share_state->fragments,
Andrew Walbranca808b12020-05-15 17:22:28 +01004046 share_state->fragment_constituent_counts,
J-Alves2a0d2882020-10-29 14:49:50 +00004047 share_state->fragment_count, share_state->sender_orig_mode,
J-Alves460d36c2023-10-12 17:02:15 +01004048 FFA_MEM_RECLAIM_32, flags & FFA_MEM_RECLAIM_CLEAR, page_pool,
J-Alvesfd206052023-05-22 16:45:00 +01004049 NULL, share_state->memory_protected);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00004050
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01004051 if (ret.func == FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00004052 share_state_free(share_states, share_state, page_pool);
J-Alves3c5b2072022-11-21 12:45:40 +00004053 dlog_verbose("Freed share state after successful reclaim.\n");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00004054 }
4055
4056out:
4057 share_states_unlock(&share_states);
4058 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01004059}