blob: 01b591257bfe8c48941922396a196552a89dccdf [file] [log] [blame]
Jose Marinho75509b42019-04-09 09:34:59 +01001/*
2 * Copyright 2019 The Hafnium Authors.
3 *
Andrew Walbrane959ec12020-06-17 15:01:09 +01004 * Use of this source code is governed by a BSD-style
5 * license that can be found in the LICENSE file or at
6 * https://opensource.org/licenses/BSD-3-Clause.
Jose Marinho75509b42019-04-09 09:34:59 +01007 */
8
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01009#include "hf/ffa_memory.h"
Andrew Walbran475c1452020-02-07 13:22:22 +000010
J-Alves460d36c2023-10-12 17:02:15 +010011#include <stdint.h>
12
Federico Recanati4fd065d2021-12-13 20:06:23 +010013#include "hf/arch/mm.h"
Olivier Deprez112d2b52020-09-30 07:39:23 +020014#include "hf/arch/other_world.h"
Olivier Deprez55a189e2021-06-09 15:45:27 +020015#include "hf/arch/plat/ffa.h"
Andrew Walbran290b0c92020-02-03 16:37:14 +000016
J-Alves5952d942022-12-22 16:03:00 +000017#include "hf/addr.h"
Jose Marinho75509b42019-04-09 09:34:59 +010018#include "hf/api.h"
Daniel Boulbya2f8c662021-11-26 17:52:53 +000019#include "hf/assert.h"
Jose Marinho09b1db82019-08-08 09:16:59 +010020#include "hf/check.h"
Jose Marinho75509b42019-04-09 09:34:59 +010021#include "hf/dlog.h"
J-Alves3456e032023-07-20 12:20:05 +010022#include "hf/ffa.h"
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010023#include "hf/ffa_internal.h"
J-Alves66652252022-07-06 09:49:51 +010024#include "hf/ffa_memory_internal.h"
J-Alves3456e032023-07-20 12:20:05 +010025#include "hf/ffa_partition_manifest.h"
J-Alves5952d942022-12-22 16:03:00 +000026#include "hf/mm.h"
Andrew Walbran475c1452020-02-07 13:22:22 +000027#include "hf/mpool.h"
J-Alvescf6253e2024-01-03 13:48:48 +000028#include "hf/panic.h"
29#include "hf/plat/memory_protect.h"
Jose Marinho75509b42019-04-09 09:34:59 +010030#include "hf/std.h"
Andrew Scull3c257452019-11-26 13:32:50 +000031#include "hf/vm.h"
Daniel Boulby44e9b3b2024-01-17 12:21:44 +000032#include "hf/vm_ids.h"
Jose Marinho75509b42019-04-09 09:34:59 +010033
J-Alves2d8457f2022-10-05 11:06:41 +010034#include "vmapi/hf/ffa_v1_0.h"
35
J-Alves5da37d92022-10-24 16:33:48 +010036#define RECEIVERS_COUNT_IN_RETRIEVE_RESP 1
37
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000038/**
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010039 * All access to members of a `struct ffa_memory_share_state` must be guarded
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000040 * by this lock.
41 */
42static struct spinlock share_states_lock_instance = SPINLOCK_INIT;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010043static struct ffa_memory_share_state share_states[MAX_MEM_SHARES];
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000044
45/**
J-Alvesed508c82023-05-04 16:09:48 +010046 * Return the offset to the first constituent within the
47 * `ffa_composite_memory_region` for the given receiver from an
48 * `ffa_memory_region`. The caller must check that the receiver_index is within
49 * bounds, and that it has a composite memory region offset.
50 */
51static uint32_t ffa_composite_constituent_offset(
52 struct ffa_memory_region *memory_region, uint32_t receiver_index)
53{
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +000054 struct ffa_memory_access *receiver;
55 uint32_t composite_offset;
J-Alvesed508c82023-05-04 16:09:48 +010056
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +000057 CHECK(receiver_index < memory_region->receiver_count);
58
59 receiver =
60 ffa_memory_region_get_receiver(memory_region, receiver_index);
61 CHECK(receiver != NULL);
62
63 composite_offset = receiver->composite_memory_region_offset;
64
65 CHECK(composite_offset != 0);
66
67 return composite_offset + sizeof(struct ffa_composite_memory_region);
J-Alvesed508c82023-05-04 16:09:48 +010068}
69
70/**
J-Alves917d2f22020-10-30 18:39:30 +000071 * Extracts the index from a memory handle allocated by Hafnium's current world.
72 */
73uint64_t ffa_memory_handle_get_index(ffa_memory_handle_t handle)
74{
75 return handle & ~FFA_MEMORY_HANDLE_ALLOCATOR_MASK;
76}
77
78/**
Karl Meakin52cdfe72023-06-30 14:49:10 +010079 * Initialises the next available `struct ffa_memory_share_state`. If `handle`
80 * is `FFA_MEMORY_HANDLE_INVALID` then allocates an appropriate handle,
81 * otherwise uses the provided handle which is assumed to be globally unique.
Andrew Walbranca808b12020-05-15 17:22:28 +010082 *
Karl Meakin52cdfe72023-06-30 14:49:10 +010083 * Returns a pointer to the allocated `ffa_memory_share_state` on success or
84 * `NULL` if none are available.
Andrew Walbranca808b12020-05-15 17:22:28 +010085 */
Karl Meakin52cdfe72023-06-30 14:49:10 +010086struct ffa_memory_share_state *allocate_share_state(
87 struct share_states_locked share_states, uint32_t share_func,
88 struct ffa_memory_region *memory_region, uint32_t fragment_length,
89 ffa_memory_handle_t handle)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000090{
Daniel Boulbya2f8c662021-11-26 17:52:53 +000091 assert(share_states.share_states != NULL);
92 assert(memory_region != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000093
Karl Meakin52cdfe72023-06-30 14:49:10 +010094 for (uint64_t i = 0; i < MAX_MEM_SHARES; ++i) {
Andrew Walbranca808b12020-05-15 17:22:28 +010095 if (share_states.share_states[i].share_func == 0) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010096 struct ffa_memory_share_state *allocated_state =
Andrew Walbranca808b12020-05-15 17:22:28 +010097 &share_states.share_states[i];
98 struct ffa_composite_memory_region *composite =
99 ffa_memory_region_get_composite(memory_region,
100 0);
101
102 if (handle == FFA_MEMORY_HANDLE_INVALID) {
J-Alvesee68c542020-10-29 17:48:20 +0000103 memory_region->handle =
Olivier Deprez55a189e2021-06-09 15:45:27 +0200104 plat_ffa_memory_handle_make(i);
Andrew Walbranca808b12020-05-15 17:22:28 +0100105 } else {
J-Alvesee68c542020-10-29 17:48:20 +0000106 memory_region->handle = handle;
Andrew Walbranca808b12020-05-15 17:22:28 +0100107 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000108 allocated_state->share_func = share_func;
109 allocated_state->memory_region = memory_region;
Andrew Walbranca808b12020-05-15 17:22:28 +0100110 allocated_state->fragment_count = 1;
111 allocated_state->fragments[0] = composite->constituents;
112 allocated_state->fragment_constituent_counts[0] =
113 (fragment_length -
114 ffa_composite_constituent_offset(memory_region,
115 0)) /
116 sizeof(struct ffa_memory_region_constituent);
117 allocated_state->sending_complete = false;
Karl Meakin52cdfe72023-06-30 14:49:10 +0100118 for (uint32_t j = 0; j < MAX_MEM_SHARE_RECIPIENTS;
119 ++j) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100120 allocated_state->retrieved_fragment_count[j] =
121 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000122 }
Karl Meakin52cdfe72023-06-30 14:49:10 +0100123 return allocated_state;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000124 }
125 }
126
Karl Meakin52cdfe72023-06-30 14:49:10 +0100127 return NULL;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000128}
129
130/** Locks the share states lock. */
131struct share_states_locked share_states_lock(void)
132{
133 sl_lock(&share_states_lock_instance);
134
135 return (struct share_states_locked){.share_states = share_states};
136}
137
138/** Unlocks the share states lock. */
J-Alves66652252022-07-06 09:49:51 +0100139void share_states_unlock(struct share_states_locked *share_states)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000140{
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000141 assert(share_states->share_states != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000142 share_states->share_states = NULL;
143 sl_unlock(&share_states_lock_instance);
144}
145
146/**
Andrew Walbranca808b12020-05-15 17:22:28 +0100147 * If the given handle is a valid handle for an allocated share state then
Karl Meakin4a2854a2023-06-30 16:26:52 +0100148 * returns a pointer to the share state. Otherwise returns NULL.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000149 */
Karl Meakin4a2854a2023-06-30 16:26:52 +0100150struct ffa_memory_share_state *get_share_state(
151 struct share_states_locked share_states, ffa_memory_handle_t handle)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000152{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100153 struct ffa_memory_share_state *share_state;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000154
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000155 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100156
157 /*
158 * First look for a share_state allocated by us, in which case the
159 * handle is based on the index.
160 */
Olivier Deprez55a189e2021-06-09 15:45:27 +0200161 if (plat_ffa_memory_handle_allocated_by_current_world(handle)) {
Karl Meakin4a2854a2023-06-30 16:26:52 +0100162 uint64_t index = ffa_memory_handle_get_index(handle);
163
Andrew Walbranca808b12020-05-15 17:22:28 +0100164 if (index < MAX_MEM_SHARES) {
165 share_state = &share_states.share_states[index];
166 if (share_state->share_func != 0) {
Karl Meakin4a2854a2023-06-30 16:26:52 +0100167 return share_state;
Andrew Walbranca808b12020-05-15 17:22:28 +0100168 }
169 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000170 }
171
Andrew Walbranca808b12020-05-15 17:22:28 +0100172 /* Fall back to a linear scan. */
Karl Meakin4a2854a2023-06-30 16:26:52 +0100173 for (uint64_t index = 0; index < MAX_MEM_SHARES; ++index) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100174 share_state = &share_states.share_states[index];
J-Alvesee68c542020-10-29 17:48:20 +0000175 if (share_state->memory_region != NULL &&
176 share_state->memory_region->handle == handle &&
Andrew Walbranca808b12020-05-15 17:22:28 +0100177 share_state->share_func != 0) {
Karl Meakin4a2854a2023-06-30 16:26:52 +0100178 return share_state;
Andrew Walbranca808b12020-05-15 17:22:28 +0100179 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000180 }
181
Karl Meakin4a2854a2023-06-30 16:26:52 +0100182 return NULL;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000183}
184
185/** Marks a share state as unallocated. */
J-Alvesfdd29272022-07-19 13:16:31 +0100186void share_state_free(struct share_states_locked share_states,
187 struct ffa_memory_share_state *share_state,
188 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000189{
Andrew Walbranca808b12020-05-15 17:22:28 +0100190 uint32_t i;
191
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000192 assert(share_states.share_states != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000193 share_state->share_func = 0;
Andrew Walbranca808b12020-05-15 17:22:28 +0100194 share_state->sending_complete = false;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000195 mpool_free(page_pool, share_state->memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +0100196 /*
197 * First fragment is part of the same page as the `memory_region`, so it
198 * doesn't need to be freed separately.
199 */
200 share_state->fragments[0] = NULL;
201 share_state->fragment_constituent_counts[0] = 0;
202 for (i = 1; i < share_state->fragment_count; ++i) {
203 mpool_free(page_pool, share_state->fragments[i]);
204 share_state->fragments[i] = NULL;
205 share_state->fragment_constituent_counts[i] = 0;
206 }
207 share_state->fragment_count = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000208 share_state->memory_region = NULL;
J-Alvesa9cd7e32022-07-01 13:49:33 +0100209 share_state->hypervisor_fragment_count = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000210}
211
Andrew Walbranca808b12020-05-15 17:22:28 +0100212/** Checks whether the given share state has been fully sent. */
J-Alvesfdd29272022-07-19 13:16:31 +0100213bool share_state_sending_complete(struct share_states_locked share_states,
214 struct ffa_memory_share_state *share_state)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000215{
Andrew Walbranca808b12020-05-15 17:22:28 +0100216 struct ffa_composite_memory_region *composite;
217 uint32_t expected_constituent_count;
218 uint32_t fragment_constituent_count_total = 0;
219 uint32_t i;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000220
Andrew Walbranca808b12020-05-15 17:22:28 +0100221 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000222 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100223
224 /*
225 * Share state must already be valid, or it's not possible to get hold
226 * of it.
227 */
228 CHECK(share_state->memory_region != NULL &&
229 share_state->share_func != 0);
230
231 composite =
232 ffa_memory_region_get_composite(share_state->memory_region, 0);
233 expected_constituent_count = composite->constituent_count;
234 for (i = 0; i < share_state->fragment_count; ++i) {
235 fragment_constituent_count_total +=
236 share_state->fragment_constituent_counts[i];
237 }
238 dlog_verbose(
239 "Checking completion: constituent count %d/%d from %d "
240 "fragments.\n",
241 fragment_constituent_count_total, expected_constituent_count,
242 share_state->fragment_count);
243
244 return fragment_constituent_count_total == expected_constituent_count;
245}
246
247/**
248 * Calculates the offset of the next fragment expected for the given share
249 * state.
250 */
J-Alvesfdd29272022-07-19 13:16:31 +0100251uint32_t share_state_next_fragment_offset(
Andrew Walbranca808b12020-05-15 17:22:28 +0100252 struct share_states_locked share_states,
253 struct ffa_memory_share_state *share_state)
254{
255 uint32_t next_fragment_offset;
256 uint32_t i;
257
258 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000259 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100260
261 next_fragment_offset =
262 ffa_composite_constituent_offset(share_state->memory_region, 0);
263 for (i = 0; i < share_state->fragment_count; ++i) {
264 next_fragment_offset +=
265 share_state->fragment_constituent_counts[i] *
266 sizeof(struct ffa_memory_region_constituent);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000267 }
268
Andrew Walbranca808b12020-05-15 17:22:28 +0100269 return next_fragment_offset;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000270}
271
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100272static void dump_memory_region(struct ffa_memory_region *memory_region)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000273{
274 uint32_t i;
275
276 if (LOG_LEVEL < LOG_LEVEL_VERBOSE) {
277 return;
278 }
279
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000280 dlog("from VM %#x, attributes %#x, flags %#x, handle %#x "
281 "tag %u, memory access descriptor size %u, to %u "
Andrew Walbrana65a1322020-04-06 19:32:32 +0100282 "recipients [",
283 memory_region->sender, memory_region->attributes,
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000284 memory_region->flags, memory_region->handle, memory_region->tag,
285 memory_region->memory_access_desc_size,
Andrew Walbrana65a1322020-04-06 19:32:32 +0100286 memory_region->receiver_count);
287 for (i = 0; i < memory_region->receiver_count; ++i) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +0000288 struct ffa_memory_access *receiver =
289 ffa_memory_region_get_receiver(memory_region, i);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000290 if (i != 0) {
291 dlog(", ");
292 }
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +0000293 dlog("Receiver %#x: %#x (offset %u)",
294 receiver->receiver_permissions.receiver,
295 receiver->receiver_permissions.permissions,
296 receiver->composite_memory_region_offset);
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000297 /* The impdef field is only present from v1.2 and later */
298 if (ffa_version_from_memory_access_desc_size(
299 memory_region->memory_access_desc_size) >=
300 MAKE_FFA_VERSION(1, 2)) {
301 dlog(", impdef: %#x %#x", receiver->impdef.val[0],
302 receiver->impdef.val[1]);
303 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000304 }
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000305 dlog("] at offset %u", memory_region->receivers_offset);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000306}
307
J-Alves66652252022-07-06 09:49:51 +0100308void dump_share_states(void)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000309{
310 uint32_t i;
311
312 if (LOG_LEVEL < LOG_LEVEL_VERBOSE) {
313 return;
314 }
315
316 dlog("Current share states:\n");
317 sl_lock(&share_states_lock_instance);
318 for (i = 0; i < MAX_MEM_SHARES; ++i) {
319 if (share_states[i].share_func != 0) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000320 switch (share_states[i].share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100321 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000322 dlog("SHARE");
323 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100324 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000325 dlog("LEND");
326 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100327 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000328 dlog("DONATE");
329 break;
330 default:
331 dlog("invalid share_func %#x",
332 share_states[i].share_func);
333 }
Olivier Deprez935e1b12020-12-22 18:01:29 +0100334 dlog(" %#x (", share_states[i].memory_region->handle);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000335 dump_memory_region(share_states[i].memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +0100336 if (share_states[i].sending_complete) {
337 dlog("): fully sent");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000338 } else {
Andrew Walbranca808b12020-05-15 17:22:28 +0100339 dlog("): partially sent");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000340 }
J-Alves2a0d2882020-10-29 14:49:50 +0000341 dlog(" with %d fragments, %d retrieved, "
342 " sender's original mode: %#x\n",
Andrew Walbranca808b12020-05-15 17:22:28 +0100343 share_states[i].fragment_count,
J-Alves2a0d2882020-10-29 14:49:50 +0000344 share_states[i].retrieved_fragment_count[0],
345 share_states[i].sender_orig_mode);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000346 }
347 }
348 sl_unlock(&share_states_lock_instance);
349}
350
Andrew Walbran475c1452020-02-07 13:22:22 +0000351/* TODO: Add device attributes: GRE, cacheability, shareability. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100352static inline uint32_t ffa_memory_permissions_to_mode(
J-Alves7cd5eb32020-10-16 19:06:10 +0100353 ffa_memory_access_permissions_t permissions, uint32_t default_mode)
Andrew Walbran475c1452020-02-07 13:22:22 +0000354{
355 uint32_t mode = 0;
356
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100357 switch (ffa_get_data_access_attr(permissions)) {
358 case FFA_DATA_ACCESS_RO:
Andrew Walbran475c1452020-02-07 13:22:22 +0000359 mode = MM_MODE_R;
360 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100361 case FFA_DATA_ACCESS_RW:
Andrew Walbran475c1452020-02-07 13:22:22 +0000362 mode = MM_MODE_R | MM_MODE_W;
363 break;
J-Alves7cd5eb32020-10-16 19:06:10 +0100364 case FFA_DATA_ACCESS_NOT_SPECIFIED:
365 mode = (default_mode & (MM_MODE_R | MM_MODE_W));
366 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100367 case FFA_DATA_ACCESS_RESERVED:
368 panic("Tried to convert FFA_DATA_ACCESS_RESERVED.");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100369 }
370
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100371 switch (ffa_get_instruction_access_attr(permissions)) {
372 case FFA_INSTRUCTION_ACCESS_NX:
Andrew Walbran475c1452020-02-07 13:22:22 +0000373 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100374 case FFA_INSTRUCTION_ACCESS_X:
Andrew Walbrana65a1322020-04-06 19:32:32 +0100375 mode |= MM_MODE_X;
376 break;
J-Alves7cd5eb32020-10-16 19:06:10 +0100377 case FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED:
378 mode |= (default_mode & MM_MODE_X);
379 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100380 case FFA_INSTRUCTION_ACCESS_RESERVED:
381 panic("Tried to convert FFA_INSTRUCTION_ACCESS_RESVERVED.");
Andrew Walbran475c1452020-02-07 13:22:22 +0000382 }
383
Olivier Deprez878bd5b2021-04-15 19:05:10 +0200384 /* Set the security state bit if necessary. */
385 if ((default_mode & plat_ffa_other_world_mode()) != 0) {
386 mode |= plat_ffa_other_world_mode();
387 }
388
Andrew Walbran475c1452020-02-07 13:22:22 +0000389 return mode;
390}
391
Jose Marinho75509b42019-04-09 09:34:59 +0100392/**
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000393 * Get the current mode in the stage-2 page table of the given vm of all the
394 * pages in the given constituents, if they all have the same mode, or return
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100395 * an appropriate FF-A error if not.
Jose Marinho75509b42019-04-09 09:34:59 +0100396 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100397static struct ffa_value constituents_get_mode(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000398 struct vm_locked vm, uint32_t *orig_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100399 struct ffa_memory_region_constituent **fragments,
400 const uint32_t *fragment_constituent_counts, uint32_t fragment_count)
Jose Marinho75509b42019-04-09 09:34:59 +0100401{
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100402 uint32_t i;
Andrew Walbranca808b12020-05-15 17:22:28 +0100403 uint32_t j;
Jose Marinho75509b42019-04-09 09:34:59 +0100404
Andrew Walbranca808b12020-05-15 17:22:28 +0100405 if (fragment_count == 0 || fragment_constituent_counts[0] == 0) {
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100406 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000407 * Fail if there are no constituents. Otherwise we would get an
408 * uninitialised *orig_mode.
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100409 */
Karl Meakin5df422c2023-07-11 17:31:38 +0100410 dlog_verbose("%s: no constituents\n", __func__);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100411 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho75509b42019-04-09 09:34:59 +0100412 }
413
Andrew Walbranca808b12020-05-15 17:22:28 +0100414 for (i = 0; i < fragment_count; ++i) {
415 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
416 ipaddr_t begin = ipa_init(fragments[i][j].address);
417 size_t size = fragments[i][j].page_count * PAGE_SIZE;
418 ipaddr_t end = ipa_add(begin, size);
419 uint32_t current_mode;
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100420
Andrew Walbranca808b12020-05-15 17:22:28 +0100421 /* Fail if addresses are not page-aligned. */
422 if (!is_aligned(ipa_addr(begin), PAGE_SIZE) ||
423 !is_aligned(ipa_addr(end), PAGE_SIZE)) {
Karl Meakin5df422c2023-07-11 17:31:38 +0100424 dlog_verbose("%s: addresses not page-aligned\n",
425 __func__);
Andrew Walbranca808b12020-05-15 17:22:28 +0100426 return ffa_error(FFA_INVALID_PARAMETERS);
427 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100428
Andrew Walbranca808b12020-05-15 17:22:28 +0100429 /*
430 * Ensure that this constituent memory range is all
431 * mapped with the same mode.
432 */
Raghu Krishnamurthy785d52f2021-02-13 00:02:40 -0800433 if (!vm_mem_get_mode(vm, begin, end, &current_mode)) {
Karl Meakin5df422c2023-07-11 17:31:38 +0100434 dlog_verbose(
435 "%s: constituent memory range %#x..%#x "
436 "not mapped with the same mode\n",
437 __func__, begin, end);
Andrew Walbranca808b12020-05-15 17:22:28 +0100438 return ffa_error(FFA_DENIED);
439 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100440
Andrew Walbranca808b12020-05-15 17:22:28 +0100441 /*
442 * Ensure that all constituents are mapped with the same
443 * mode.
444 */
445 if (i == 0) {
446 *orig_mode = current_mode;
447 } else if (current_mode != *orig_mode) {
448 dlog_verbose(
Karl Meakin5df422c2023-07-11 17:31:38 +0100449 "%s: expected mode %#x but was %#x for "
450 "%d pages at %#x.\n",
451 __func__, *orig_mode, current_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100452 fragments[i][j].page_count,
453 ipa_addr(begin));
454 return ffa_error(FFA_DENIED);
455 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100456 }
Jose Marinho75509b42019-04-09 09:34:59 +0100457 }
458
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100459 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000460}
461
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100462uint32_t ffa_version_from_memory_access_desc_size(
463 uint32_t memory_access_desc_size)
464{
465 switch (memory_access_desc_size) {
466 /*
467 * v1.0 and v1.1 memory access descriptors are the same size however
468 * v1.1 is the first version to include the memory access descriptor
469 * size field so return v1.1.
470 */
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000471 case sizeof(struct ffa_memory_access_v1_0):
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100472 return MAKE_FFA_VERSION(1, 1);
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000473 case sizeof(struct ffa_memory_access):
474 return MAKE_FFA_VERSION(1, 2);
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100475 }
476 return 0;
477}
478
479/**
480 * Check if the receivers size and offset given is valid for the senders
481 * FF-A version.
482 */
483static bool receiver_size_and_offset_valid_for_version(
484 uint32_t receivers_size, uint32_t receivers_offset,
485 uint32_t ffa_version)
486{
487 /*
488 * Check that the version that the memory access descriptor size belongs
489 * to is compatible with the FF-A version we believe the sender to be.
490 */
491 uint32_t expected_ffa_version =
492 ffa_version_from_memory_access_desc_size(receivers_size);
493 if (!FFA_VERSIONS_ARE_COMPATIBLE(expected_ffa_version, ffa_version)) {
494 return false;
495 }
496
497 /*
498 * Check the receivers_offset matches the version we found from
499 * memory access descriptor size.
500 */
501 switch (expected_ffa_version) {
502 case MAKE_FFA_VERSION(1, 1):
Daniel Boulbyde974ca2023-12-12 13:53:31 +0000503 case MAKE_FFA_VERSION(1, 2):
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100504 return receivers_offset == sizeof(struct ffa_memory_region);
505 default:
506 return false;
507 }
508}
509
510/**
511 * Check the values set for fields in the memory region are valid and safe.
512 * Offset values are within safe bounds, receiver count will not cause overflows
513 * and reserved fields are 0.
514 */
515bool ffa_memory_region_sanity_check(struct ffa_memory_region *memory_region,
516 uint32_t ffa_version,
517 uint32_t fragment_length,
518 bool send_transaction)
519{
520 uint32_t receiver_count;
521 struct ffa_memory_access *receiver;
522 uint32_t composite_offset_0;
523
524 if (ffa_version == MAKE_FFA_VERSION(1, 0)) {
525 struct ffa_memory_region_v1_0 *memory_region_v1_0 =
526 (struct ffa_memory_region_v1_0 *)memory_region;
527 /* Check the reserved fields are 0. */
528 if (memory_region_v1_0->reserved_0 != 0 ||
529 memory_region_v1_0->reserved_1 != 0) {
530 dlog_verbose("Reserved fields must be 0.\n");
531 return false;
532 }
533
534 receiver_count = memory_region_v1_0->receiver_count;
535 } else {
536 uint32_t receivers_size =
537 memory_region->memory_access_desc_size;
538 uint32_t receivers_offset = memory_region->receivers_offset;
539
540 /* Check the reserved field is 0. */
541 if (memory_region->reserved[0] != 0 ||
542 memory_region->reserved[1] != 0 ||
543 memory_region->reserved[2] != 0) {
544 dlog_verbose("Reserved fields must be 0.\n");
545 return false;
546 }
547
548 /*
549 * Check memory_access_desc_size matches the size of the struct
550 * for the senders FF-A version.
551 */
552 if (!receiver_size_and_offset_valid_for_version(
553 receivers_size, receivers_offset, ffa_version)) {
554 dlog_verbose(
555 "Invalid memory access descriptor size %d, "
556 " or receiver offset %d, "
557 "for FF-A version %#x\n",
558 receivers_size, receivers_offset, ffa_version);
559 return false;
560 }
561
562 receiver_count = memory_region->receiver_count;
563 }
564
565 /* Check receiver count is not too large. */
566 if (receiver_count > MAX_MEM_SHARE_RECIPIENTS) {
567 dlog_verbose(
568 "Max number of recipients supported is %u "
569 "specified %u\n",
570 MAX_MEM_SHARE_RECIPIENTS, receiver_count);
571 return false;
572 }
573
574 /* Check values in the memory access descriptors. */
575 /*
576 * The composite offset values must be the same for all recievers so
577 * check the first one is valid and then they are all the same.
578 */
579 receiver = ffa_version == MAKE_FFA_VERSION(1, 0)
580 ? (struct ffa_memory_access *)&(
581 (struct ffa_memory_region_v1_0 *)
582 memory_region)
583 ->receivers[0]
584 : ffa_memory_region_get_receiver(memory_region, 0);
585 assert(receiver != NULL);
586 composite_offset_0 = receiver->composite_memory_region_offset;
587
588 if (!send_transaction) {
589 if (composite_offset_0 != 0) {
590 dlog_verbose(
591 "Composite offset memory region descriptor "
592 "offset must be 0 for retrieve requests. "
593 "Currently %d",
594 composite_offset_0);
595 return false;
596 }
597 } else {
598 bool comp_offset_is_zero = composite_offset_0 == 0U;
599 bool comp_offset_lt_transaction_descriptor_size =
600 composite_offset_0 <
601 (sizeof(struct ffa_memory_region) +
602 (uint32_t)(memory_region->memory_access_desc_size *
603 memory_region->receiver_count));
604 bool comp_offset_with_comp_gt_fragment_length =
605 composite_offset_0 +
606 sizeof(struct ffa_composite_memory_region) >
607 fragment_length;
608 if (comp_offset_is_zero ||
609 comp_offset_lt_transaction_descriptor_size ||
610 comp_offset_with_comp_gt_fragment_length) {
611 dlog_verbose(
612 "Invalid composite memory region descriptor "
613 "offset for send transaction %u\n",
614 composite_offset_0);
615 return false;
616 }
617 }
618
619 for (int i = 0; i < memory_region->receiver_count; i++) {
620 uint32_t composite_offset;
621
622 if (ffa_version == MAKE_FFA_VERSION(1, 0)) {
623 struct ffa_memory_region_v1_0 *memory_region_v1_0 =
624 (struct ffa_memory_region_v1_0 *)memory_region;
625
626 struct ffa_memory_access_v1_0 *receiver_v1_0 =
627 &memory_region_v1_0->receivers[i];
628 /* Check reserved fields are 0 */
629 if (receiver_v1_0->reserved_0 != 0) {
630 dlog_verbose(
631 "Reserved field in the memory access "
632 " descriptor must be zero "
633 " Currently reciever %d has a reserved "
634 " field with a value of %d\n",
635 i, receiver_v1_0->reserved_0);
636 return false;
637 }
638 /*
639 * We can cast to the current version receiver as the
640 * remaining fields we are checking have the same
641 * offsets for all versions since memory access
642 * descriptors are forwards compatible.
643 */
644 receiver = (struct ffa_memory_access *)receiver_v1_0;
645 } else {
646 receiver = ffa_memory_region_get_receiver(memory_region,
647 i);
648 assert(receiver != NULL);
649
650 if (receiver->reserved_0 != 0) {
651 dlog_verbose(
652 "Reserved field in the memory access "
653 " descriptor must be zero "
654 " Currently reciever %d has a reserved "
655 " field with a value of %d\n",
656 i, receiver->reserved_0);
657 return false;
658 }
659 }
660
661 /* Check composite offset values are equal for all receivers. */
662 composite_offset = receiver->composite_memory_region_offset;
663 if (composite_offset != composite_offset_0) {
664 dlog_verbose(
665 "Composite offset %x differs from %x in index "
666 "%u\n",
667 composite_offset, composite_offset_0);
668 return false;
669 }
670 }
671 return true;
672}
673
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000674/**
J-Alves460d36c2023-10-12 17:02:15 +0100675 * If the receivers for the memory management operation are all from the
676 * secure world and this isn't a FFA_MEM_SHARE, then request memory security
677 * state update by returning MAP_ACTION_CHECK_PROTECT.
678 */
679static enum ffa_map_action ffa_mem_send_get_map_action(
680 bool all_receivers_from_current_world, ffa_id_t sender_id,
681 uint32_t mem_func_id)
682{
683 bool protect_memory =
684 (mem_func_id != FFA_MEM_SHARE_32 &&
685 all_receivers_from_current_world && ffa_is_vm_id(sender_id));
686
687 return protect_memory ? MAP_ACTION_CHECK_PROTECT : MAP_ACTION_CHECK;
688}
689
690/**
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000691 * Verify that all pages have the same mode, that the starting mode
692 * constitutes a valid state and obtain the next mode to apply
J-Alves460d36c2023-10-12 17:02:15 +0100693 * to the sending VM. It outputs the mapping action that needs to be
694 * invoked for the given memory range. On memory lend/donate there
695 * could be a need to protect the memory from the normal world.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000696 *
697 * Returns:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100698 * 1) FFA_DENIED if a state transition was not found;
699 * 2) FFA_DENIED if the pages being shared do not have the same mode within
Andrew Walbrana65a1322020-04-06 19:32:32 +0100700 * the <from> VM;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100701 * 3) FFA_INVALID_PARAMETERS if the beginning and end IPAs are not page
Andrew Walbrana65a1322020-04-06 19:32:32 +0100702 * aligned;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100703 * 4) FFA_INVALID_PARAMETERS if the requested share type was not handled.
704 * Or FFA_SUCCESS on success.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000705 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100706static struct ffa_value ffa_send_check_transition(
Andrew Walbrana65a1322020-04-06 19:32:32 +0100707 struct vm_locked from, uint32_t share_func,
J-Alves363f5722022-04-25 17:37:37 +0100708 struct ffa_memory_access *receivers, uint32_t receivers_count,
709 uint32_t *orig_from_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100710 struct ffa_memory_region_constituent **fragments,
711 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
J-Alves460d36c2023-10-12 17:02:15 +0100712 uint32_t *from_mode, enum ffa_map_action *map_action)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000713{
714 const uint32_t state_mask =
715 MM_MODE_INVALID | MM_MODE_UNOWNED | MM_MODE_SHARED;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100716 struct ffa_value ret;
J-Alves460d36c2023-10-12 17:02:15 +0100717 bool all_receivers_from_current_world = true;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000718
Andrew Walbranca808b12020-05-15 17:22:28 +0100719 ret = constituents_get_mode(from, orig_from_mode, fragments,
720 fragment_constituent_counts,
721 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100722 if (ret.func != FFA_SUCCESS_32) {
Olivier Depreze7eb1682022-03-16 17:09:03 +0100723 dlog_verbose("Inconsistent modes.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100724 return ret;
Andrew Scullb5f49e02019-10-02 13:20:47 +0100725 }
726
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000727 /* Ensure the address range is normal memory and not a device. */
J-Alves788b4492023-04-18 14:01:23 +0100728 if ((*orig_from_mode & MM_MODE_D) != 0U) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000729 dlog_verbose("Can't share device memory (mode is %#x).\n",
730 *orig_from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100731 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000732 }
733
734 /*
735 * Ensure the sender is the owner and has exclusive access to the
736 * memory.
737 */
738 if ((*orig_from_mode & state_mask) != 0) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100739 return ffa_error(FFA_DENIED);
Andrew Walbrana65a1322020-04-06 19:32:32 +0100740 }
741
J-Alves363f5722022-04-25 17:37:37 +0100742 assert(receivers != NULL && receivers_count > 0U);
J-Alves7cd5eb32020-10-16 19:06:10 +0100743
J-Alves363f5722022-04-25 17:37:37 +0100744 for (uint32_t i = 0U; i < receivers_count; i++) {
745 ffa_memory_access_permissions_t permissions =
746 receivers[i].receiver_permissions.permissions;
747 uint32_t required_from_mode = ffa_memory_permissions_to_mode(
748 permissions, *orig_from_mode);
749
J-Alves788b4492023-04-18 14:01:23 +0100750 /*
751 * The assumption is that at this point, the operation from
752 * SP to a receiver VM, should have returned an FFA_ERROR
753 * already.
754 */
755 if (!ffa_is_vm_id(from.vm->id)) {
756 assert(!ffa_is_vm_id(
757 receivers[i].receiver_permissions.receiver));
758 }
759
J-Alves460d36c2023-10-12 17:02:15 +0100760 /* Track if all senders are from current world. */
761 all_receivers_from_current_world =
762 all_receivers_from_current_world &&
763 vm_id_is_current_world(
764 receivers[i].receiver_permissions.receiver);
765
J-Alves363f5722022-04-25 17:37:37 +0100766 if ((*orig_from_mode & required_from_mode) !=
767 required_from_mode) {
768 dlog_verbose(
769 "Sender tried to send memory with permissions "
J-Alves788b4492023-04-18 14:01:23 +0100770 "which required mode %#x but only had %#x "
771 "itself.\n",
J-Alves363f5722022-04-25 17:37:37 +0100772 required_from_mode, *orig_from_mode);
773 return ffa_error(FFA_DENIED);
774 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000775 }
776
J-Alves460d36c2023-10-12 17:02:15 +0100777 *map_action = ffa_mem_send_get_map_action(
778 all_receivers_from_current_world, from.vm->id, share_func);
779
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000780 /* Find the appropriate new mode. */
781 *from_mode = ~state_mask & *orig_from_mode;
Andrew Walbrane7ad3c02019-12-24 17:03:04 +0000782 switch (share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100783 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000784 *from_mode |= MM_MODE_INVALID | MM_MODE_UNOWNED;
Jose Marinho75509b42019-04-09 09:34:59 +0100785 break;
786
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100787 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000788 *from_mode |= MM_MODE_INVALID;
Andrew Walbran648fc3e2019-10-22 16:23:05 +0100789 break;
790
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100791 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000792 *from_mode |= MM_MODE_SHARED;
Jose Marinho56c25732019-05-20 09:48:53 +0100793 break;
794
Jose Marinho75509b42019-04-09 09:34:59 +0100795 default:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100796 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho75509b42019-04-09 09:34:59 +0100797 }
798
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100799 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000800}
801
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100802static struct ffa_value ffa_relinquish_check_transition(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000803 struct vm_locked from, uint32_t *orig_from_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100804 struct ffa_memory_region_constituent **fragments,
805 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
806 uint32_t *from_mode)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000807{
808 const uint32_t state_mask =
809 MM_MODE_INVALID | MM_MODE_UNOWNED | MM_MODE_SHARED;
810 uint32_t orig_from_state;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100811 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000812
Andrew Walbranca808b12020-05-15 17:22:28 +0100813 ret = constituents_get_mode(from, orig_from_mode, fragments,
814 fragment_constituent_counts,
815 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100816 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbrana65a1322020-04-06 19:32:32 +0100817 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000818 }
819
820 /* Ensure the address range is normal memory and not a device. */
821 if (*orig_from_mode & MM_MODE_D) {
822 dlog_verbose("Can't relinquish device memory (mode is %#x).\n",
823 *orig_from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100824 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000825 }
826
827 /*
828 * Ensure the relinquishing VM is not the owner but has access to the
829 * memory.
830 */
831 orig_from_state = *orig_from_mode & state_mask;
832 if ((orig_from_state & ~MM_MODE_SHARED) != MM_MODE_UNOWNED) {
833 dlog_verbose(
834 "Tried to relinquish memory in state %#x (masked %#x "
Andrew Walbranca808b12020-05-15 17:22:28 +0100835 "but should be %#x).\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000836 *orig_from_mode, orig_from_state, MM_MODE_UNOWNED);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100837 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000838 }
839
840 /* Find the appropriate new mode. */
841 *from_mode = (~state_mask & *orig_from_mode) | MM_MODE_UNMAPPED_MASK;
842
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100843 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000844}
845
846/**
847 * Verify that all pages have the same mode, that the starting mode
848 * constitutes a valid state and obtain the next mode to apply
849 * to the retrieving VM.
850 *
851 * Returns:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100852 * 1) FFA_DENIED if a state transition was not found;
853 * 2) FFA_DENIED if the pages being shared do not have the same mode within
Andrew Walbrana65a1322020-04-06 19:32:32 +0100854 * the <to> VM;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100855 * 3) FFA_INVALID_PARAMETERS if the beginning and end IPAs are not page
Andrew Walbrana65a1322020-04-06 19:32:32 +0100856 * aligned;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100857 * 4) FFA_INVALID_PARAMETERS if the requested share type was not handled.
858 * Or FFA_SUCCESS on success.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000859 */
J-Alvesfc19b372022-07-06 12:17:35 +0100860struct ffa_value ffa_retrieve_check_transition(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000861 struct vm_locked to, uint32_t share_func,
Andrew Walbranca808b12020-05-15 17:22:28 +0100862 struct ffa_memory_region_constituent **fragments,
863 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
J-Alvesfd206052023-05-22 16:45:00 +0100864 uint32_t memory_to_attributes, uint32_t *to_mode, bool memory_protected,
865 enum ffa_map_action *map_action)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000866{
867 uint32_t orig_to_mode;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100868 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000869
Andrew Walbranca808b12020-05-15 17:22:28 +0100870 ret = constituents_get_mode(to, &orig_to_mode, fragments,
871 fragment_constituent_counts,
872 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100873 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100874 dlog_verbose("Inconsistent modes.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100875 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000876 }
877
J-Alves460d36c2023-10-12 17:02:15 +0100878 /* Find the appropriate new mode. */
879 *to_mode = memory_to_attributes;
880
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100881 if (share_func == FFA_MEM_RECLAIM_32) {
J-Alves9256f162021-12-09 13:18:43 +0000882 /*
883 * If the original ffa memory send call has been processed
884 * successfully, it is expected the orig_to_mode would overlay
885 * with `state_mask`, as a result of the function
886 * `ffa_send_check_transition`.
J-Alvesfd206052023-05-22 16:45:00 +0100887 *
888 * If Hafnium is the SPMC:
889 * - Caller of the reclaim interface is an SP, the memory shall
890 * have been protected throughout the flow.
891 * - Caller of the reclaim is from the NWd, the memory may have
892 * been protected at the time of lending/donating the memory.
893 * In such case, set action to unprotect memory in the
894 * handling of reclaim operation.
895 * - If Hafnium is the hypervisor memory shall never have been
896 * protected in memory lend/share/donate.
897 *
898 * More details in the doc comment of the function
899 * `ffa_region_group_identity_map`.
J-Alves9256f162021-12-09 13:18:43 +0000900 */
J-Alves59ed0042022-07-28 18:26:41 +0100901 if (vm_id_is_current_world(to.vm->id)) {
902 assert((orig_to_mode &
903 (MM_MODE_INVALID | MM_MODE_UNOWNED |
904 MM_MODE_SHARED)) != 0U);
J-Alvesfd206052023-05-22 16:45:00 +0100905 assert(!memory_protected);
906 } else if (to.vm->id == HF_OTHER_WORLD_ID &&
907 map_action != NULL && memory_protected) {
908 *map_action = MAP_ACTION_COMMIT_UNPROTECT;
J-Alves59ed0042022-07-28 18:26:41 +0100909 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000910 } else {
911 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +0100912 * If the retriever is from virtual FF-A instance:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000913 * Ensure the retriever has the expected state. We don't care
914 * about the MM_MODE_SHARED bit; either with or without it set
915 * are both valid representations of the !O-NA state.
916 */
J-Alvesa9cd7e32022-07-01 13:49:33 +0100917 if (vm_id_is_current_world(to.vm->id) &&
918 to.vm->id != HF_PRIMARY_VM_ID &&
919 (orig_to_mode & MM_MODE_UNMAPPED_MASK) !=
920 MM_MODE_UNMAPPED_MASK) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100921 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000922 }
J-Alves460d36c2023-10-12 17:02:15 +0100923
924 /*
925 * If memory has been protected before, clear the NS bit to
926 * allow the secure access from the SP.
927 */
928 if (memory_protected) {
929 *to_mode &= ~plat_ffa_other_world_mode();
930 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000931 }
932
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000933 switch (share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100934 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000935 *to_mode |= 0;
936 break;
937
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100938 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000939 *to_mode |= MM_MODE_UNOWNED;
940 break;
941
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100942 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000943 *to_mode |= MM_MODE_UNOWNED | MM_MODE_SHARED;
944 break;
945
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100946 case FFA_MEM_RECLAIM_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000947 *to_mode |= 0;
948 break;
949
950 default:
Andrew Walbranca808b12020-05-15 17:22:28 +0100951 dlog_error("Invalid share_func %#x.\n", share_func);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100952 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000953 }
954
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100955 return (struct ffa_value){.func = FFA_SUCCESS_32};
Jose Marinho75509b42019-04-09 09:34:59 +0100956}
Jose Marinho09b1db82019-08-08 09:16:59 +0100957
J-Alvescf6253e2024-01-03 13:48:48 +0000958/*
959 * Performs the operations related to the `action` MAP_ACTION_CHECK*.
960 * Returns:
961 * - FFA_SUCCESS_32: if all goes well.
962 * - FFA_ERROR_32: with FFA_NO_MEMORY, if there is no memory to manage
963 * the page table update. Or error code provided by the function
964 * `arch_memory_protect`.
965 */
966static struct ffa_value ffa_region_group_check_actions(
967 struct vm_locked vm_locked, paddr_t pa_begin, paddr_t pa_end,
968 struct mpool *ppool, uint32_t mode, enum ffa_map_action action,
969 bool *memory_protected)
970{
971 struct ffa_value ret;
972 bool is_memory_protected;
973
974 if (!vm_identity_prepare(vm_locked, pa_begin, pa_end, mode, ppool)) {
975 dlog_verbose(
976 "%s: memory can't be mapped to %x due to lack of "
977 "memory. Base: %lx end: %x\n",
978 __func__, vm_locked.vm->id, pa_addr(pa_begin),
979 pa_addr(pa_end));
980 return ffa_error(FFA_NO_MEMORY);
981 }
982
983 switch (action) {
984 case MAP_ACTION_CHECK:
985 /* No protect requested. */
986 is_memory_protected = false;
987 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
988 break;
989 case MAP_ACTION_CHECK_PROTECT: {
990 paddr_t last_protected_pa = pa_init(0);
991
992 ret = arch_memory_protect(pa_begin, pa_end, &last_protected_pa);
993
994 is_memory_protected = (ret.func == FFA_SUCCESS_32);
995
996 /*
997 * - If protect memory has failed with FFA_DENIED, means some
998 * range of memory was in the wrong state. In such case, SPM
999 * reverts the state of the pages that were successfully
1000 * updated.
1001 * - If protect memory has failed with FFA_NOT_SUPPORTED, it
1002 * means the platform doesn't support the protection mechanism.
1003 * That said, it still permits the page table update to go
1004 * through. The variable
1005 * `is_memory_protected` will be equal to false.
1006 * - If protect memory has failed with FFA_INVALID_PARAMETERS,
1007 * break from switch and return the error.
1008 */
1009 if (ret.func == FFA_ERROR_32) {
1010 assert(!is_memory_protected);
1011 if (ffa_error_code(ret) == FFA_DENIED &&
1012 pa_addr(last_protected_pa) != (uintptr_t)0) {
1013 CHECK(arch_memory_unprotect(
1014 pa_begin,
1015 pa_add(last_protected_pa, PAGE_SIZE)));
1016 } else if (ffa_error_code(ret) == FFA_NOT_SUPPORTED) {
1017 ret = (struct ffa_value){
1018 .func = FFA_SUCCESS_32,
1019 };
1020 }
1021 }
1022 } break;
1023 default:
1024 panic("%s: invalid action to process %x\n", __func__, action);
1025 }
1026
1027 if (memory_protected != NULL) {
1028 *memory_protected = is_memory_protected;
1029 }
1030
1031 return ret;
1032}
1033
1034static void ffa_region_group_commit_actions(struct vm_locked vm_locked,
1035 paddr_t pa_begin, paddr_t pa_end,
1036 struct mpool *ppool, uint32_t mode,
1037 enum ffa_map_action action)
1038{
1039 switch (action) {
1040 case MAP_ACTION_COMMIT_UNPROTECT:
1041 /*
1042 * Checking that it should succeed because SPM should be
1043 * unprotecting memory that it had protected before.
1044 */
1045 CHECK(arch_memory_unprotect(pa_begin, pa_end));
1046 case MAP_ACTION_COMMIT:
1047 vm_identity_commit(vm_locked, pa_begin, pa_end, mode, ppool,
1048 NULL);
1049 break;
1050 default:
1051 panic("%s: invalid action to process %x\n", __func__, action);
1052 }
1053}
1054
Jose Marinho09b1db82019-08-08 09:16:59 +01001055/**
J-Alves063ad832023-10-03 18:05:40 +01001056 * Helper function to revert a failed "Protect" action from the SPMC:
1057 * - `fragment_count`: should specify the number of fragments to traverse from
1058 * `fragments`. This may not be the full amount of fragments that are part of
1059 * the share_state structure.
1060 * - `fragment_constituent_counts`: array holding the amount of constituents
1061 * per fragment.
1062 * - `end`: pointer to the constituent that failed the "protect" action. It
1063 * shall be part of the last fragment, and it shall make the loop below break.
1064 */
1065static void ffa_region_group_fragments_revert_protect(
1066 struct ffa_memory_region_constituent **fragments,
1067 const uint32_t *fragment_constituent_counts, uint32_t fragment_count,
1068 const struct ffa_memory_region_constituent *end)
1069{
1070 for (uint32_t i = 0; i < fragment_count; ++i) {
1071 for (uint32_t j = 0; j < fragment_constituent_counts[i]; ++j) {
1072 struct ffa_memory_region_constituent *constituent =
1073 &fragments[i][j];
1074 size_t size = constituent->page_count * PAGE_SIZE;
1075 paddr_t pa_begin =
1076 pa_from_ipa(ipa_init(constituent->address));
1077 paddr_t pa_end = pa_add(pa_begin, size);
1078
1079 dlog_verbose("%s: reverting fragment %x size %x\n",
1080 __func__, pa_addr(pa_begin), size);
1081
1082 if (constituent == end) {
1083 /*
1084 * The last constituent is expected to be in the
1085 * last fragment.
1086 */
1087 assert(i == fragment_count - 1);
1088 break;
1089 }
1090
1091 CHECK(arch_memory_unprotect(pa_begin, pa_end));
1092 }
1093 }
1094}
1095
1096/**
Jose Marinho09b1db82019-08-08 09:16:59 +01001097 * Updates a VM's page table such that the given set of physical address ranges
1098 * are mapped in the address space at the corresponding address ranges, in the
1099 * mode provided.
1100 *
J-Alves0a83dc22023-05-05 09:50:37 +01001101 * The enum ffa_map_action determines the action taken from a call to the
1102 * function below:
1103 * - If action is MAP_ACTION_CHECK, the page tables will be allocated from the
1104 * mpool but no mappings will actually be updated. This function must always
1105 * be called first with action set to MAP_ACTION_CHECK to check that it will
1106 * succeed before calling ffa_region_group_identity_map with whichever one of
1107 * the remaining actions, to avoid leaving the page table in a half-updated
1108 * state.
1109 * - The action MAP_ACTION_COMMIT allocates the page tables from the mpool, and
1110 * changes the memory mappings.
J-Alvescf6253e2024-01-03 13:48:48 +00001111 * - The action MAP_ACTION_CHECK_PROTECT extends the MAP_ACTION_CHECK with an
1112 * invocation to the monitor to update the security state of the memory,
1113 * to that of the SPMC.
1114 * - The action MAP_ACTION_COMMIT_UNPROTECT extends the MAP_ACTION_COMMIT
1115 * with a call into the monitor, to reset the security state of memory
1116 * that has priorly been mapped with the MAP_ACTION_CHECK_PROTECT action.
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001117 * vm_ptable_defrag should always be called after a series of page table
1118 * updates, whether they succeed or fail.
Jose Marinho09b1db82019-08-08 09:16:59 +01001119 *
J-Alvescf6253e2024-01-03 13:48:48 +00001120 * If all goes well, returns FFA_SUCCESS_32; or FFA_ERROR, with following
1121 * error codes:
1122 * - FFA_INVALID_PARAMETERS: invalid range of memory.
1123 * - FFA_DENIED:
1124 *
Jose Marinho09b1db82019-08-08 09:16:59 +01001125 * made to memory mappings.
1126 */
J-Alvescf6253e2024-01-03 13:48:48 +00001127struct ffa_value ffa_region_group_identity_map(
Andrew Walbranf4b51af2020-02-03 14:44:54 +00001128 struct vm_locked vm_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01001129 struct ffa_memory_region_constituent **fragments,
1130 const uint32_t *fragment_constituent_counts, uint32_t fragment_count,
J-Alvescf6253e2024-01-03 13:48:48 +00001131 uint32_t mode, struct mpool *ppool, enum ffa_map_action action,
1132 bool *memory_protected)
Jose Marinho09b1db82019-08-08 09:16:59 +01001133{
Andrew Walbranca808b12020-05-15 17:22:28 +01001134 uint32_t i;
1135 uint32_t j;
J-Alvescf6253e2024-01-03 13:48:48 +00001136 struct ffa_value ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01001137
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001138 if (vm_locked.vm->el0_partition) {
1139 mode |= MM_MODE_USER | MM_MODE_NG;
1140 }
1141
Andrew Walbranca808b12020-05-15 17:22:28 +01001142 /* Iterate over the memory region constituents within each fragment. */
1143 for (i = 0; i < fragment_count; ++i) {
1144 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
J-Alves063ad832023-10-03 18:05:40 +01001145 struct ffa_memory_region_constituent *constituent =
1146 &fragments[i][j];
1147 size_t size = constituent->page_count * PAGE_SIZE;
Andrew Walbranca808b12020-05-15 17:22:28 +01001148 paddr_t pa_begin =
J-Alves063ad832023-10-03 18:05:40 +01001149 pa_from_ipa(ipa_init(constituent->address));
Andrew Walbranca808b12020-05-15 17:22:28 +01001150 paddr_t pa_end = pa_add(pa_begin, size);
Jens Wiklander4f1880c2022-10-19 17:00:14 +02001151 uint32_t pa_bits =
1152 arch_mm_get_pa_bits(arch_mm_get_pa_range());
Federico Recanati4fd065d2021-12-13 20:06:23 +01001153
1154 /*
1155 * Ensure the requested region falls into system's PA
1156 * range.
1157 */
Jens Wiklander4f1880c2022-10-19 17:00:14 +02001158 if (((pa_addr(pa_begin) >> pa_bits) > 0) ||
1159 ((pa_addr(pa_end) >> pa_bits) > 0)) {
Federico Recanati4fd065d2021-12-13 20:06:23 +01001160 dlog_error("Region is outside of PA Range\n");
J-Alvescf6253e2024-01-03 13:48:48 +00001161 return ffa_error(FFA_INVALID_PARAMETERS);
Federico Recanati4fd065d2021-12-13 20:06:23 +01001162 }
Andrew Walbranca808b12020-05-15 17:22:28 +01001163
J-Alvescf6253e2024-01-03 13:48:48 +00001164 if (action <= MAP_ACTION_CHECK_PROTECT) {
1165 ret = ffa_region_group_check_actions(
1166 vm_locked, pa_begin, pa_end, ppool,
1167 mode, action, memory_protected);
J-Alves063ad832023-10-03 18:05:40 +01001168
1169 if (ret.func == FFA_ERROR_32 &&
1170 ffa_error_code(ret) == FFA_DENIED) {
1171 if (memory_protected != NULL) {
1172 assert(!*memory_protected);
1173 }
1174
1175 ffa_region_group_fragments_revert_protect(
1176 fragments,
1177 fragment_constituent_counts,
1178 i + 1, constituent);
1179 break;
1180 }
J-Alvescf6253e2024-01-03 13:48:48 +00001181 } else if (action >= MAP_ACTION_COMMIT &&
1182 action < MAP_ACTION_MAX) {
1183 ffa_region_group_commit_actions(
1184 vm_locked, pa_begin, pa_end, ppool,
1185 mode, action);
1186 ret = (struct ffa_value){
1187 .func = FFA_SUCCESS_32};
1188 } else {
1189 panic("%s: Unknown ffa_map_action.\n",
1190 __func__);
Andrew Walbranca808b12020-05-15 17:22:28 +01001191 }
Jose Marinho09b1db82019-08-08 09:16:59 +01001192 }
1193 }
1194
J-Alvescf6253e2024-01-03 13:48:48 +00001195 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01001196}
1197
1198/**
1199 * Clears a region of physical memory by overwriting it with zeros. The data is
1200 * flushed from the cache so the memory has been cleared across the system.
1201 */
J-Alves7db32002021-12-14 14:44:50 +00001202static bool clear_memory(paddr_t begin, paddr_t end, struct mpool *ppool,
1203 uint32_t extra_mode_attributes)
Jose Marinho09b1db82019-08-08 09:16:59 +01001204{
1205 /*
Fuad Tabbaed294af2019-12-20 10:43:01 +00001206 * TODO: change this to a CPU local single page window rather than a
Jose Marinho09b1db82019-08-08 09:16:59 +01001207 * global mapping of the whole range. Such an approach will limit
1208 * the changes to stage-1 tables and will allow only local
1209 * invalidation.
1210 */
1211 bool ret;
1212 struct mm_stage1_locked stage1_locked = mm_lock_stage1();
J-Alves7db32002021-12-14 14:44:50 +00001213 void *ptr = mm_identity_map(stage1_locked, begin, end,
1214 MM_MODE_W | (extra_mode_attributes &
1215 plat_ffa_other_world_mode()),
1216 ppool);
Jose Marinho09b1db82019-08-08 09:16:59 +01001217 size_t size = pa_difference(begin, end);
1218
1219 if (!ptr) {
Jose Marinho09b1db82019-08-08 09:16:59 +01001220 goto fail;
1221 }
1222
1223 memset_s(ptr, size, 0, size);
1224 arch_mm_flush_dcache(ptr, size);
1225 mm_unmap(stage1_locked, begin, end, ppool);
1226
1227 ret = true;
1228 goto out;
1229
1230fail:
1231 ret = false;
1232
1233out:
1234 mm_unlock_stage1(&stage1_locked);
1235
1236 return ret;
1237}
1238
1239/**
1240 * Clears a region of physical memory by overwriting it with zeros. The data is
1241 * flushed from the cache so the memory has been cleared across the system.
1242 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001243static bool ffa_clear_memory_constituents(
J-Alves7db32002021-12-14 14:44:50 +00001244 uint32_t security_state_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +01001245 struct ffa_memory_region_constituent **fragments,
1246 const uint32_t *fragment_constituent_counts, uint32_t fragment_count,
1247 struct mpool *page_pool)
Jose Marinho09b1db82019-08-08 09:16:59 +01001248{
1249 struct mpool local_page_pool;
Andrew Walbranca808b12020-05-15 17:22:28 +01001250 uint32_t i;
Jose Marinho09b1db82019-08-08 09:16:59 +01001251 bool ret = false;
1252
1253 /*
1254 * Create a local pool so any freed memory can't be used by another
1255 * thread. This is to ensure each constituent that is mapped can be
1256 * unmapped again afterwards.
1257 */
Andrew Walbran475c1452020-02-07 13:22:22 +00001258 mpool_init_with_fallback(&local_page_pool, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +01001259
Andrew Walbranca808b12020-05-15 17:22:28 +01001260 /* Iterate over the memory region constituents within each fragment. */
1261 for (i = 0; i < fragment_count; ++i) {
1262 uint32_t j;
Jose Marinho09b1db82019-08-08 09:16:59 +01001263
J-Alves8457f932023-10-11 16:41:45 +01001264 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001265 size_t size = fragments[i][j].page_count * PAGE_SIZE;
1266 paddr_t begin =
1267 pa_from_ipa(ipa_init(fragments[i][j].address));
1268 paddr_t end = pa_add(begin, size);
1269
J-Alves7db32002021-12-14 14:44:50 +00001270 if (!clear_memory(begin, end, &local_page_pool,
1271 security_state_mode)) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001272 /*
1273 * api_clear_memory will defrag on failure, so
1274 * no need to do it here.
1275 */
1276 goto out;
1277 }
Jose Marinho09b1db82019-08-08 09:16:59 +01001278 }
1279 }
1280
Jose Marinho09b1db82019-08-08 09:16:59 +01001281 ret = true;
1282
1283out:
1284 mpool_fini(&local_page_pool);
1285 return ret;
1286}
1287
J-Alves5952d942022-12-22 16:03:00 +00001288static bool is_memory_range_within(ipaddr_t begin, ipaddr_t end,
1289 ipaddr_t in_begin, ipaddr_t in_end)
1290{
1291 return (ipa_addr(begin) >= ipa_addr(in_begin) &&
1292 ipa_addr(begin) < ipa_addr(in_end)) ||
1293 (ipa_addr(end) <= ipa_addr(in_end) &&
1294 ipa_addr(end) > ipa_addr(in_begin));
1295}
1296
1297/**
1298 * Receives a memory range and looks for overlaps with the remainder
1299 * constituents of the memory share/lend/donate operation. Assumes they are
1300 * passed in order to avoid having to loop over all the elements at each call.
1301 * The function only compares the received memory ranges with those that follow
1302 * within the same fragment, and subsequent fragments from the same operation.
1303 */
1304static bool ffa_memory_check_overlap(
1305 struct ffa_memory_region_constituent **fragments,
1306 const uint32_t *fragment_constituent_counts,
1307 const uint32_t fragment_count, const uint32_t current_fragment,
1308 const uint32_t current_constituent)
1309{
1310 uint32_t i = current_fragment;
1311 uint32_t j = current_constituent;
1312 ipaddr_t current_begin = ipa_init(fragments[i][j].address);
1313 const uint32_t current_page_count = fragments[i][j].page_count;
1314 size_t current_size = current_page_count * PAGE_SIZE;
1315 ipaddr_t current_end = ipa_add(current_begin, current_size - 1);
1316
1317 if (current_size == 0 ||
1318 current_size > UINT64_MAX - ipa_addr(current_begin)) {
1319 dlog_verbose("Invalid page count. Addr: %x page_count: %x\n",
1320 current_begin, current_page_count);
1321 return false;
1322 }
1323
1324 for (; i < fragment_count; i++) {
1325 j = (i == current_fragment) ? j + 1 : 0;
1326
1327 for (; j < fragment_constituent_counts[i]; j++) {
1328 ipaddr_t begin = ipa_init(fragments[i][j].address);
1329 const uint32_t page_count = fragments[i][j].page_count;
1330 size_t size = page_count * PAGE_SIZE;
1331 ipaddr_t end = ipa_add(begin, size - 1);
1332
1333 if (size == 0 || size > UINT64_MAX - ipa_addr(begin)) {
1334 dlog_verbose(
1335 "Invalid page count. Addr: %x "
1336 "page_count: %x\n",
1337 begin, page_count);
1338 return false;
1339 }
1340
1341 /*
1342 * Check if current ranges is within begin and end, as
1343 * well as the reverse. This should help optimize the
1344 * loop, and reduce the number of iterations.
1345 */
1346 if (is_memory_range_within(begin, end, current_begin,
1347 current_end) ||
1348 is_memory_range_within(current_begin, current_end,
1349 begin, end)) {
1350 dlog_verbose(
1351 "Overlapping memory ranges: %#x - %#x "
1352 "with %#x - %#x\n",
1353 ipa_addr(begin), ipa_addr(end),
1354 ipa_addr(current_begin),
1355 ipa_addr(current_end));
1356 return true;
1357 }
1358 }
1359 }
1360
1361 return false;
1362}
1363
Jose Marinho09b1db82019-08-08 09:16:59 +01001364/**
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001365 * Validates and prepares memory to be sent from the calling VM to another.
Jose Marinho09b1db82019-08-08 09:16:59 +01001366 *
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001367 * This function requires the calling context to hold the <from> VM lock.
Jose Marinho09b1db82019-08-08 09:16:59 +01001368 *
1369 * Returns:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001370 * In case of error, one of the following values is returned:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001371 * 1) FFA_INVALID_PARAMETERS - The endpoint provided parameters were
Jose Marinho09b1db82019-08-08 09:16:59 +01001372 * erroneous;
Andrew Walbranf07f04d2020-05-01 18:09:00 +01001373 * 2) FFA_NO_MEMORY - Hafnium did not have sufficient memory to complete the
1374 * request.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001375 * 3) FFA_DENIED - The sender doesn't have sufficient access to send the
Andrew Walbrana65a1322020-04-06 19:32:32 +01001376 * memory with the given permissions.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001377 * Success is indicated by FFA_SUCCESS.
Jose Marinho09b1db82019-08-08 09:16:59 +01001378 */
J-Alves66652252022-07-06 09:49:51 +01001379struct ffa_value ffa_send_check_update(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001380 struct vm_locked from_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01001381 struct ffa_memory_region_constituent **fragments,
1382 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
J-Alves8f11cde2022-12-21 16:18:22 +00001383 uint32_t composite_total_page_count, uint32_t share_func,
1384 struct ffa_memory_access *receivers, uint32_t receivers_count,
J-Alves460d36c2023-10-12 17:02:15 +01001385 struct mpool *page_pool, bool clear, uint32_t *orig_from_mode_ret,
1386 bool *memory_protected)
Jose Marinho09b1db82019-08-08 09:16:59 +01001387{
Andrew Walbranca808b12020-05-15 17:22:28 +01001388 uint32_t i;
J-Alves8f11cde2022-12-21 16:18:22 +00001389 uint32_t j;
Jose Marinho09b1db82019-08-08 09:16:59 +01001390 uint32_t orig_from_mode;
J-Alves460d36c2023-10-12 17:02:15 +01001391 uint32_t clean_mode;
Jose Marinho09b1db82019-08-08 09:16:59 +01001392 uint32_t from_mode;
Jose Marinho09b1db82019-08-08 09:16:59 +01001393 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001394 struct ffa_value ret;
J-Alves8f11cde2022-12-21 16:18:22 +00001395 uint32_t constituents_total_page_count = 0;
J-Alves460d36c2023-10-12 17:02:15 +01001396 enum ffa_map_action map_action = MAP_ACTION_CHECK;
Jose Marinho09b1db82019-08-08 09:16:59 +01001397
1398 /*
Andrew Walbrana65a1322020-04-06 19:32:32 +01001399 * Make sure constituents are properly aligned to a 64-bit boundary. If
1400 * not we would get alignment faults trying to read (64-bit) values.
Jose Marinho09b1db82019-08-08 09:16:59 +01001401 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001402 for (i = 0; i < fragment_count; ++i) {
1403 if (!is_aligned(fragments[i], 8)) {
1404 dlog_verbose("Constituents not aligned.\n");
1405 return ffa_error(FFA_INVALID_PARAMETERS);
1406 }
J-Alves8f11cde2022-12-21 16:18:22 +00001407 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
1408 constituents_total_page_count +=
1409 fragments[i][j].page_count;
J-Alves5952d942022-12-22 16:03:00 +00001410 if (ffa_memory_check_overlap(
1411 fragments, fragment_constituent_counts,
1412 fragment_count, i, j)) {
1413 return ffa_error(FFA_INVALID_PARAMETERS);
1414 }
J-Alves8f11cde2022-12-21 16:18:22 +00001415 }
1416 }
1417
1418 if (constituents_total_page_count != composite_total_page_count) {
1419 dlog_verbose(
1420 "Composite page count differs from calculated page "
1421 "count from constituents.\n");
1422 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho09b1db82019-08-08 09:16:59 +01001423 }
1424
1425 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001426 * Check if the state transition is lawful for the sender, ensure that
1427 * all constituents of a memory region being shared are at the same
1428 * state.
Jose Marinho09b1db82019-08-08 09:16:59 +01001429 */
J-Alves460d36c2023-10-12 17:02:15 +01001430 ret = ffa_send_check_transition(
1431 from_locked, share_func, receivers, receivers_count,
1432 &orig_from_mode, fragments, fragment_constituent_counts,
1433 fragment_count, &from_mode, &map_action);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001434 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001435 dlog_verbose("Invalid transition for send.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +01001436 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01001437 }
1438
Andrew Walbran37c574e2020-06-03 11:45:46 +01001439 if (orig_from_mode_ret != NULL) {
1440 *orig_from_mode_ret = orig_from_mode;
1441 }
1442
Jose Marinho09b1db82019-08-08 09:16:59 +01001443 /*
1444 * Create a local pool so any freed memory can't be used by another
1445 * thread. This is to ensure the original mapping can be restored if the
1446 * clear fails.
1447 */
Andrew Walbran475c1452020-02-07 13:22:22 +00001448 mpool_init_with_fallback(&local_page_pool, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +01001449
1450 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001451 * First reserve all required memory for the new page table entries
1452 * without committing, to make sure the entire operation will succeed
1453 * without exhausting the page pool.
J-Alves460d36c2023-10-12 17:02:15 +01001454 * Provide the map_action as populated by 'ffa_send_check_transition'.
1455 * It may request memory to be protected.
Jose Marinho09b1db82019-08-08 09:16:59 +01001456 */
J-Alvescf6253e2024-01-03 13:48:48 +00001457 ret = ffa_region_group_identity_map(
1458 from_locked, fragments, fragment_constituent_counts,
J-Alves460d36c2023-10-12 17:02:15 +01001459 fragment_count, from_mode, page_pool, map_action,
1460 memory_protected);
J-Alvescf6253e2024-01-03 13:48:48 +00001461 if (ret.func == FFA_ERROR_32) {
Jose Marinho09b1db82019-08-08 09:16:59 +01001462 goto out;
1463 }
1464
1465 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001466 * Update the mapping for the sender. This won't allocate because the
1467 * transaction was already prepared above, but may free pages in the
1468 * case that a whole block is being unmapped that was previously
1469 * partially mapped.
Jose Marinho09b1db82019-08-08 09:16:59 +01001470 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001471 CHECK(ffa_region_group_identity_map(
J-Alvescf6253e2024-01-03 13:48:48 +00001472 from_locked, fragments, fragment_constituent_counts,
1473 fragment_count, from_mode, &local_page_pool,
1474 MAP_ACTION_COMMIT, NULL)
1475 .func == FFA_SUCCESS_32);
Jose Marinho09b1db82019-08-08 09:16:59 +01001476
J-Alves460d36c2023-10-12 17:02:15 +01001477 /*
1478 * If memory has been protected, it is now part of the secure PAS
1479 * (happens for lend/donate from NWd to SWd), and the `orig_from_mode`
1480 * should have the MM_MODE_NS set, as such mask it in `clean_mode` for
1481 * SPM's S1 translation.
1482 * In case memory hasn't been protected, and it is in the non-secure
1483 * PAS (e.g. memory share from NWd to SWd), as such the SPM needs to
1484 * perform a non-secure memory access. In such case `clean_mode` takes
1485 * the same mode as `orig_from_mode`.
1486 */
1487 clean_mode = (memory_protected != NULL && *memory_protected)
1488 ? orig_from_mode & ~plat_ffa_other_world_mode()
1489 : orig_from_mode;
1490
Jose Marinho09b1db82019-08-08 09:16:59 +01001491 /* Clear the memory so no VM or device can see the previous contents. */
J-Alves460d36c2023-10-12 17:02:15 +01001492 if (clear && !ffa_clear_memory_constituents(
1493 clean_mode, fragments, fragment_constituent_counts,
1494 fragment_count, page_pool)) {
1495 map_action = (memory_protected != NULL && *memory_protected)
1496 ? MAP_ACTION_COMMIT_UNPROTECT
1497 : MAP_ACTION_COMMIT;
1498
Jose Marinho09b1db82019-08-08 09:16:59 +01001499 /*
1500 * On failure, roll back by returning memory to the sender. This
1501 * may allocate pages which were previously freed into
1502 * `local_page_pool` by the call above, but will never allocate
1503 * more pages than that so can never fail.
1504 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001505 CHECK(ffa_region_group_identity_map(
J-Alvescf6253e2024-01-03 13:48:48 +00001506 from_locked, fragments,
1507 fragment_constituent_counts, fragment_count,
1508 orig_from_mode, &local_page_pool,
1509 MAP_ACTION_COMMIT, NULL)
1510 .func == FFA_SUCCESS_32);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001511 ret = ffa_error(FFA_NO_MEMORY);
Jose Marinho09b1db82019-08-08 09:16:59 +01001512 goto out;
1513 }
1514
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001515 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001516
1517out:
1518 mpool_fini(&local_page_pool);
1519
1520 /*
1521 * Tidy up the page table by reclaiming failed mappings (if there was an
1522 * error) or merging entries into blocks where possible (on success).
1523 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001524 vm_ptable_defrag(from_locked, page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001525
1526 return ret;
1527}
1528
1529/**
1530 * Validates and maps memory shared from one VM to another.
1531 *
1532 * This function requires the calling context to hold the <to> lock.
1533 *
1534 * Returns:
1535 * In case of error, one of the following values is returned:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001536 * 1) FFA_INVALID_PARAMETERS - The endpoint provided parameters were
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001537 * erroneous;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001538 * 2) FFA_NO_MEMORY - Hafnium did not have sufficient memory to complete
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001539 * the request.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001540 * Success is indicated by FFA_SUCCESS.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001541 */
J-Alvesb5084cf2022-07-06 14:20:12 +01001542struct ffa_value ffa_retrieve_check_update(
J-Alves26483382023-04-20 12:01:49 +01001543 struct vm_locked to_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01001544 struct ffa_memory_region_constituent **fragments,
1545 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
J-Alves26483382023-04-20 12:01:49 +01001546 uint32_t sender_orig_mode, uint32_t share_func, bool clear,
J-Alves460d36c2023-10-12 17:02:15 +01001547 struct mpool *page_pool, uint32_t *response_mode, bool memory_protected)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001548{
Andrew Walbranca808b12020-05-15 17:22:28 +01001549 uint32_t i;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001550 uint32_t to_mode;
1551 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001552 struct ffa_value ret;
J-Alvesfd206052023-05-22 16:45:00 +01001553 enum ffa_map_action map_action = MAP_ACTION_COMMIT;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001554
1555 /*
Andrew Walbranca808b12020-05-15 17:22:28 +01001556 * Make sure constituents are properly aligned to a 64-bit boundary. If
1557 * not we would get alignment faults trying to read (64-bit) values.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001558 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001559 for (i = 0; i < fragment_count; ++i) {
1560 if (!is_aligned(fragments[i], 8)) {
J-Alvesb5084cf2022-07-06 14:20:12 +01001561 dlog_verbose("Fragment not properly aligned.\n");
Andrew Walbranca808b12020-05-15 17:22:28 +01001562 return ffa_error(FFA_INVALID_PARAMETERS);
1563 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001564 }
1565
1566 /*
1567 * Check if the state transition is lawful for the recipient, and ensure
1568 * that all constituents of the memory region being retrieved are at the
1569 * same state.
1570 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001571 ret = ffa_retrieve_check_transition(
1572 to_locked, share_func, fragments, fragment_constituent_counts,
J-Alvesfd206052023-05-22 16:45:00 +01001573 fragment_count, sender_orig_mode, &to_mode, memory_protected,
1574 &map_action);
J-Alves460d36c2023-10-12 17:02:15 +01001575
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001576 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001577 dlog_verbose("Invalid transition for retrieve.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +01001578 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001579 }
1580
1581 /*
1582 * Create a local pool so any freed memory can't be used by another
1583 * thread. This is to ensure the original mapping can be restored if the
1584 * clear fails.
1585 */
1586 mpool_init_with_fallback(&local_page_pool, page_pool);
1587
1588 /*
1589 * First reserve all required memory for the new page table entries in
1590 * the recipient page tables without committing, to make sure the entire
1591 * operation will succeed without exhausting the page pool.
1592 */
J-Alvescf6253e2024-01-03 13:48:48 +00001593 ret = ffa_region_group_identity_map(
1594 to_locked, fragments, fragment_constituent_counts,
1595 fragment_count, to_mode, page_pool, MAP_ACTION_CHECK, NULL);
1596 if (ret.func == FFA_ERROR_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001597 /* TODO: partial defrag of failed range. */
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001598 goto out;
1599 }
1600
1601 /* Clear the memory so no VM or device can see the previous contents. */
J-Alves7db32002021-12-14 14:44:50 +00001602 if (clear &&
J-Alves26483382023-04-20 12:01:49 +01001603 !ffa_clear_memory_constituents(sender_orig_mode, fragments,
1604 fragment_constituent_counts,
1605 fragment_count, page_pool)) {
J-Alvesb5084cf2022-07-06 14:20:12 +01001606 dlog_verbose("Couldn't clear constituents.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001607 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001608 goto out;
1609 }
1610
Jose Marinho09b1db82019-08-08 09:16:59 +01001611 /*
1612 * Complete the transfer by mapping the memory into the recipient. This
1613 * won't allocate because the transaction was already prepared above, so
1614 * it doesn't need to use the `local_page_pool`.
1615 */
J-Alvesfd206052023-05-22 16:45:00 +01001616 CHECK(ffa_region_group_identity_map(
1617 to_locked, fragments, fragment_constituent_counts,
1618 fragment_count, to_mode, page_pool, map_action, NULL)
J-Alvescf6253e2024-01-03 13:48:48 +00001619 .func == FFA_SUCCESS_32);
Jose Marinho09b1db82019-08-08 09:16:59 +01001620
J-Alves460d36c2023-10-12 17:02:15 +01001621 /* Return the mode used in mapping the memory in retriever's PT. */
1622 if (response_mode != NULL) {
1623 *response_mode = to_mode;
1624 }
1625
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001626 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Jose Marinho09b1db82019-08-08 09:16:59 +01001627
1628out:
1629 mpool_fini(&local_page_pool);
1630
1631 /*
Andrew Walbranf07f04d2020-05-01 18:09:00 +01001632 * Tidy up the page table by reclaiming failed mappings (if there was an
1633 * error) or merging entries into blocks where possible (on success).
Jose Marinho09b1db82019-08-08 09:16:59 +01001634 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001635 vm_ptable_defrag(to_locked, page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001636
1637 return ret;
1638}
1639
Andrew Walbran996d1d12020-05-27 14:08:43 +01001640static struct ffa_value ffa_relinquish_check_update(
J-Alves26483382023-04-20 12:01:49 +01001641 struct vm_locked from_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01001642 struct ffa_memory_region_constituent **fragments,
1643 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
1644 struct mpool *page_pool, bool clear)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001645{
1646 uint32_t orig_from_mode;
1647 uint32_t from_mode;
1648 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001649 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001650
Andrew Walbranca808b12020-05-15 17:22:28 +01001651 ret = ffa_relinquish_check_transition(
1652 from_locked, &orig_from_mode, fragments,
1653 fragment_constituent_counts, fragment_count, &from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001654 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001655 dlog_verbose("Invalid transition for relinquish.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +01001656 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001657 }
1658
1659 /*
1660 * Create a local pool so any freed memory can't be used by another
1661 * thread. This is to ensure the original mapping can be restored if the
1662 * clear fails.
1663 */
1664 mpool_init_with_fallback(&local_page_pool, page_pool);
1665
1666 /*
1667 * First reserve all required memory for the new page table entries
1668 * without committing, to make sure the entire operation will succeed
1669 * without exhausting the page pool.
1670 */
J-Alvescf6253e2024-01-03 13:48:48 +00001671 ret = ffa_region_group_identity_map(
1672 from_locked, fragments, fragment_constituent_counts,
1673 fragment_count, from_mode, page_pool, MAP_ACTION_CHECK, NULL);
1674 if (ret.func == FFA_ERROR_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001675 goto out;
1676 }
1677
1678 /*
1679 * Update the mapping for the sender. This won't allocate because the
1680 * transaction was already prepared above, but may free pages in the
1681 * case that a whole block is being unmapped that was previously
1682 * partially mapped.
1683 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001684 CHECK(ffa_region_group_identity_map(
J-Alvescf6253e2024-01-03 13:48:48 +00001685 from_locked, fragments, fragment_constituent_counts,
1686 fragment_count, from_mode, &local_page_pool,
1687 MAP_ACTION_COMMIT, NULL)
1688 .func == FFA_SUCCESS_32);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001689
1690 /* Clear the memory so no VM or device can see the previous contents. */
J-Alves7db32002021-12-14 14:44:50 +00001691 if (clear &&
J-Alves26483382023-04-20 12:01:49 +01001692 !ffa_clear_memory_constituents(orig_from_mode, fragments,
1693 fragment_constituent_counts,
1694 fragment_count, page_pool)) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001695 /*
1696 * On failure, roll back by returning memory to the sender. This
1697 * may allocate pages which were previously freed into
1698 * `local_page_pool` by the call above, but will never allocate
1699 * more pages than that so can never fail.
1700 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001701 CHECK(ffa_region_group_identity_map(
J-Alvescf6253e2024-01-03 13:48:48 +00001702 from_locked, fragments,
1703 fragment_constituent_counts, fragment_count,
1704 orig_from_mode, &local_page_pool,
1705 MAP_ACTION_COMMIT, NULL)
1706 .func == FFA_SUCCESS_32);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001707
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001708 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001709 goto out;
1710 }
1711
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001712 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001713
1714out:
1715 mpool_fini(&local_page_pool);
1716
1717 /*
1718 * Tidy up the page table by reclaiming failed mappings (if there was an
1719 * error) or merging entries into blocks where possible (on success).
1720 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001721 vm_ptable_defrag(from_locked, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +01001722
1723 return ret;
1724}
1725
1726/**
Andrew Walbranca808b12020-05-15 17:22:28 +01001727 * Complete a memory sending operation by checking that it is valid, updating
1728 * the sender page table, and then either marking the share state as having
1729 * completed sending (on success) or freeing it (on failure).
1730 *
1731 * Returns FFA_SUCCESS with the handle encoded, or the relevant FFA_ERROR.
1732 */
J-Alvesfdd29272022-07-19 13:16:31 +01001733struct ffa_value ffa_memory_send_complete(
Andrew Walbranca808b12020-05-15 17:22:28 +01001734 struct vm_locked from_locked, struct share_states_locked share_states,
Andrew Walbran37c574e2020-06-03 11:45:46 +01001735 struct ffa_memory_share_state *share_state, struct mpool *page_pool,
1736 uint32_t *orig_from_mode_ret)
Andrew Walbranca808b12020-05-15 17:22:28 +01001737{
1738 struct ffa_memory_region *memory_region = share_state->memory_region;
J-Alves8f11cde2022-12-21 16:18:22 +00001739 struct ffa_composite_memory_region *composite;
Daniel Boulby41ef8ba2023-10-13 17:01:22 +01001740 struct ffa_memory_access *receiver;
Andrew Walbranca808b12020-05-15 17:22:28 +01001741 struct ffa_value ret;
1742
1743 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +00001744 assert(share_states.share_states != NULL);
J-Alves8f11cde2022-12-21 16:18:22 +00001745 assert(memory_region != NULL);
1746 composite = ffa_memory_region_get_composite(memory_region, 0);
1747 assert(composite != NULL);
Daniel Boulby41ef8ba2023-10-13 17:01:22 +01001748 receiver = ffa_memory_region_get_receiver(memory_region, 0);
1749 assert(receiver != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +01001750
1751 /* Check that state is valid in sender page table and update. */
1752 ret = ffa_send_check_update(
1753 from_locked, share_state->fragments,
1754 share_state->fragment_constituent_counts,
J-Alves8f11cde2022-12-21 16:18:22 +00001755 share_state->fragment_count, composite->page_count,
Daniel Boulby41ef8ba2023-10-13 17:01:22 +01001756 share_state->share_func, receiver,
J-Alves8f11cde2022-12-21 16:18:22 +00001757 memory_region->receiver_count, page_pool,
1758 memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR,
J-Alves460d36c2023-10-12 17:02:15 +01001759 orig_from_mode_ret, &share_state->memory_protected);
Andrew Walbranca808b12020-05-15 17:22:28 +01001760 if (ret.func != FFA_SUCCESS_32) {
1761 /*
1762 * Free share state, it failed to send so it can't be retrieved.
1763 */
Karl Meakin4cec5e82023-06-30 16:30:22 +01001764 dlog_verbose("%s: failed to send check update: %s(%s)\n",
1765 __func__, ffa_func_name(ret.func),
1766 ffa_error_name(ffa_error_code(ret)));
Andrew Walbranca808b12020-05-15 17:22:28 +01001767 share_state_free(share_states, share_state, page_pool);
1768 return ret;
1769 }
1770
1771 share_state->sending_complete = true;
Karl Meakin4cec5e82023-06-30 16:30:22 +01001772 dlog_verbose("%s: marked sending complete.\n", __func__);
Andrew Walbranca808b12020-05-15 17:22:28 +01001773
J-Alvesee68c542020-10-29 17:48:20 +00001774 return ffa_mem_success(share_state->memory_region->handle);
Andrew Walbranca808b12020-05-15 17:22:28 +01001775}
1776
1777/**
Federico Recanatia98603a2021-12-20 18:04:03 +01001778 * Check that the memory attributes match Hafnium expectations:
1779 * Normal Memory, Inner shareable, Write-Back Read-Allocate
1780 * Write-Allocate Cacheable.
1781 */
1782static struct ffa_value ffa_memory_attributes_validate(
J-Alves7a99d0d2023-02-08 13:49:48 +00001783 ffa_memory_attributes_t attributes)
Federico Recanatia98603a2021-12-20 18:04:03 +01001784{
1785 enum ffa_memory_type memory_type;
1786 enum ffa_memory_cacheability cacheability;
1787 enum ffa_memory_shareability shareability;
1788
1789 memory_type = ffa_get_memory_type_attr(attributes);
1790 if (memory_type != FFA_MEMORY_NORMAL_MEM) {
1791 dlog_verbose("Invalid memory type %#x, expected %#x.\n",
1792 memory_type, FFA_MEMORY_NORMAL_MEM);
Federico Recanati3d953f32022-02-17 09:31:29 +01001793 return ffa_error(FFA_DENIED);
Federico Recanatia98603a2021-12-20 18:04:03 +01001794 }
1795
1796 cacheability = ffa_get_memory_cacheability_attr(attributes);
1797 if (cacheability != FFA_MEMORY_CACHE_WRITE_BACK) {
1798 dlog_verbose("Invalid cacheability %#x, expected %#x.\n",
1799 cacheability, FFA_MEMORY_CACHE_WRITE_BACK);
Federico Recanati3d953f32022-02-17 09:31:29 +01001800 return ffa_error(FFA_DENIED);
Federico Recanatia98603a2021-12-20 18:04:03 +01001801 }
1802
1803 shareability = ffa_get_memory_shareability_attr(attributes);
1804 if (shareability != FFA_MEMORY_INNER_SHAREABLE) {
1805 dlog_verbose("Invalid shareability %#x, expected #%x.\n",
1806 shareability, FFA_MEMORY_INNER_SHAREABLE);
Federico Recanati3d953f32022-02-17 09:31:29 +01001807 return ffa_error(FFA_DENIED);
Federico Recanatia98603a2021-12-20 18:04:03 +01001808 }
1809
1810 return (struct ffa_value){.func = FFA_SUCCESS_32};
1811}
1812
1813/**
Andrew Walbrana65a1322020-04-06 19:32:32 +01001814 * Check that the given `memory_region` represents a valid memory send request
1815 * of the given `share_func` type, return the clear flag and permissions via the
1816 * respective output parameters, and update the permissions if necessary.
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001817 *
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001818 * Returns FFA_SUCCESS if the request was valid, or the relevant FFA_ERROR if
Andrew Walbrana65a1322020-04-06 19:32:32 +01001819 * not.
1820 */
J-Alves66652252022-07-06 09:49:51 +01001821struct ffa_value ffa_memory_send_validate(
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001822 struct vm_locked from_locked, struct ffa_memory_region *memory_region,
1823 uint32_t memory_share_length, uint32_t fragment_length,
J-Alves363f5722022-04-25 17:37:37 +01001824 uint32_t share_func)
Andrew Walbrana65a1322020-04-06 19:32:32 +01001825{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001826 struct ffa_composite_memory_region *composite;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001827 struct ffa_memory_access *receiver =
1828 ffa_memory_region_get_receiver(memory_region, 0);
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001829 uint64_t receivers_end;
1830 uint64_t min_length;
Federico Recanati872cd692022-01-05 13:10:10 +01001831 uint32_t composite_memory_region_offset;
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001832 uint32_t constituents_start;
Andrew Walbran130a8ae2020-05-15 16:27:15 +01001833 uint32_t constituents_length;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001834 enum ffa_data_access data_access;
1835 enum ffa_instruction_access instruction_access;
Olivier Deprez4342a3c2022-02-28 09:37:25 +01001836 enum ffa_memory_security security_state;
Federico Recanatia98603a2021-12-20 18:04:03 +01001837 struct ffa_value ret;
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001838 const size_t minimum_first_fragment_length =
Daniel Boulby41ef8ba2023-10-13 17:01:22 +01001839 memory_region->receivers_offset +
1840 memory_region->memory_access_desc_size +
1841 sizeof(struct ffa_composite_memory_region);
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001842
1843 if (fragment_length < minimum_first_fragment_length) {
1844 dlog_verbose("Fragment length %u too short (min %u).\n",
1845 (size_t)fragment_length,
1846 minimum_first_fragment_length);
1847 return ffa_error(FFA_INVALID_PARAMETERS);
1848 }
1849
Demi Marie Obenour73a1e942023-02-04 14:09:18 -05001850 static_assert(sizeof(struct ffa_memory_region_constituent) == 16,
1851 "struct ffa_memory_region_constituent must be 16 bytes");
1852 if (!is_aligned(fragment_length,
1853 sizeof(struct ffa_memory_region_constituent)) ||
1854 !is_aligned(memory_share_length,
1855 sizeof(struct ffa_memory_region_constituent))) {
1856 dlog_verbose(
1857 "Fragment length %u or total length %u"
1858 " is not 16-byte aligned.\n",
1859 fragment_length, memory_share_length);
1860 return ffa_error(FFA_INVALID_PARAMETERS);
1861 }
1862
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001863 if (fragment_length > memory_share_length) {
1864 dlog_verbose(
1865 "Fragment length %u greater than total length %u.\n",
1866 (size_t)fragment_length, (size_t)memory_share_length);
1867 return ffa_error(FFA_INVALID_PARAMETERS);
1868 }
Andrew Walbrana65a1322020-04-06 19:32:32 +01001869
J-Alves95df0ef2022-12-07 10:09:48 +00001870 /* The sender must match the caller. */
1871 if ((!vm_id_is_current_world(from_locked.vm->id) &&
1872 vm_id_is_current_world(memory_region->sender)) ||
1873 (vm_id_is_current_world(from_locked.vm->id) &&
1874 memory_region->sender != from_locked.vm->id)) {
1875 dlog_verbose("Invalid memory sender ID.\n");
1876 return ffa_error(FFA_DENIED);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001877 }
1878
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001879 if (memory_region->receiver_count <= 0) {
1880 dlog_verbose("No receivers!\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001881 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001882 }
1883
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001884 /*
1885 * Ensure that the composite header is within the memory bounds and
1886 * doesn't overlap the first part of the message. Cast to uint64_t
1887 * to prevent overflow.
1888 */
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001889 receivers_end = ((uint64_t)memory_region->memory_access_desc_size *
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001890 (uint64_t)memory_region->receiver_count) +
Daniel Boulby41ef8ba2023-10-13 17:01:22 +01001891 memory_region->receivers_offset;
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001892 min_length = receivers_end +
1893 sizeof(struct ffa_composite_memory_region) +
1894 sizeof(struct ffa_memory_region_constituent);
1895 if (min_length > memory_share_length) {
1896 dlog_verbose("Share too short: got %u but minimum is %u.\n",
1897 (size_t)memory_share_length, (size_t)min_length);
1898 return ffa_error(FFA_INVALID_PARAMETERS);
1899 }
1900
1901 composite_memory_region_offset =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001902 receiver->composite_memory_region_offset;
Andrew Walbrana65a1322020-04-06 19:32:32 +01001903
1904 /*
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001905 * Check that the composite memory region descriptor is after the access
1906 * descriptors, is at least 16-byte aligned, and fits in the first
1907 * fragment.
Andrew Walbrana65a1322020-04-06 19:32:32 +01001908 */
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001909 if ((composite_memory_region_offset < receivers_end) ||
1910 (composite_memory_region_offset % 16 != 0) ||
1911 (composite_memory_region_offset >
1912 fragment_length - sizeof(struct ffa_composite_memory_region))) {
1913 dlog_verbose(
1914 "Invalid composite memory region descriptor offset "
1915 "%u.\n",
1916 (size_t)composite_memory_region_offset);
1917 return ffa_error(FFA_INVALID_PARAMETERS);
1918 }
1919
1920 /*
1921 * Compute the start of the constituent regions. Already checked
1922 * to be not more than fragment_length and thus not more than
1923 * memory_share_length.
1924 */
1925 constituents_start = composite_memory_region_offset +
1926 sizeof(struct ffa_composite_memory_region);
1927 constituents_length = memory_share_length - constituents_start;
1928
1929 /*
1930 * Check that the number of constituents is consistent with the length
1931 * of the constituent region.
1932 */
1933 composite = ffa_memory_region_get_composite(memory_region, 0);
1934 if ((constituents_length %
1935 sizeof(struct ffa_memory_region_constituent) !=
1936 0) ||
1937 ((constituents_length /
1938 sizeof(struct ffa_memory_region_constituent)) !=
1939 composite->constituent_count)) {
1940 dlog_verbose("Invalid length %u or composite offset %u.\n",
1941 (size_t)memory_share_length,
1942 (size_t)composite_memory_region_offset);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001943 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001944 }
Andrew Walbranca808b12020-05-15 17:22:28 +01001945 if (fragment_length < memory_share_length &&
1946 fragment_length < HF_MAILBOX_SIZE) {
1947 dlog_warning(
1948 "Initial fragment length %d smaller than mailbox "
1949 "size.\n",
1950 fragment_length);
1951 }
Andrew Walbrana65a1322020-04-06 19:32:32 +01001952
Andrew Walbrana65a1322020-04-06 19:32:32 +01001953 /*
1954 * Clear is not allowed for memory sharing, as the sender still has
1955 * access to the memory.
1956 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001957 if ((memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR) &&
1958 share_func == FFA_MEM_SHARE_32) {
Andrew Walbrana65a1322020-04-06 19:32:32 +01001959 dlog_verbose("Memory can't be cleared while being shared.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001960 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001961 }
1962
1963 /* No other flags are allowed/supported here. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001964 if (memory_region->flags & ~FFA_MEMORY_REGION_FLAG_CLEAR) {
Andrew Walbrana65a1322020-04-06 19:32:32 +01001965 dlog_verbose("Invalid flags %#x.\n", memory_region->flags);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001966 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001967 }
1968
J-Alves363f5722022-04-25 17:37:37 +01001969 /* Check that the permissions are valid, for each specified receiver. */
1970 for (uint32_t i = 0U; i < memory_region->receiver_count; i++) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001971 struct ffa_memory_region_attributes receiver_permissions;
1972
1973 receiver = ffa_memory_region_get_receiver(memory_region, i);
1974 assert(receiver != NULL);
1975 receiver_permissions = receiver->receiver_permissions;
J-Alves363f5722022-04-25 17:37:37 +01001976 ffa_memory_access_permissions_t permissions =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001977 receiver_permissions.permissions;
1978 ffa_id_t receiver_id = receiver_permissions.receiver;
J-Alves363f5722022-04-25 17:37:37 +01001979
1980 if (memory_region->sender == receiver_id) {
1981 dlog_verbose("Can't share memory with itself.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001982 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001983 }
Federico Recanati85090c42021-12-15 13:17:54 +01001984
J-Alves363f5722022-04-25 17:37:37 +01001985 for (uint32_t j = i + 1; j < memory_region->receiver_count;
1986 j++) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001987 struct ffa_memory_access *other_receiver =
1988 ffa_memory_region_get_receiver(memory_region,
1989 j);
1990 assert(other_receiver != NULL);
1991
J-Alves363f5722022-04-25 17:37:37 +01001992 if (receiver_id ==
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001993 other_receiver->receiver_permissions.receiver) {
J-Alves363f5722022-04-25 17:37:37 +01001994 dlog_verbose(
1995 "Repeated receiver(%x) in memory send "
1996 "operation.\n",
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001997 other_receiver->receiver_permissions
1998 .receiver);
J-Alves363f5722022-04-25 17:37:37 +01001999 return ffa_error(FFA_INVALID_PARAMETERS);
2000 }
2001 }
2002
2003 if (composite_memory_region_offset !=
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002004 receiver->composite_memory_region_offset) {
J-Alves363f5722022-04-25 17:37:37 +01002005 dlog_verbose(
2006 "All ffa_memory_access should point to the "
2007 "same composite memory region offset.\n");
2008 return ffa_error(FFA_INVALID_PARAMETERS);
2009 }
2010
2011 data_access = ffa_get_data_access_attr(permissions);
2012 instruction_access =
2013 ffa_get_instruction_access_attr(permissions);
2014 if (data_access == FFA_DATA_ACCESS_RESERVED ||
2015 instruction_access == FFA_INSTRUCTION_ACCESS_RESERVED) {
2016 dlog_verbose(
2017 "Reserved value for receiver permissions "
2018 "%#x.\n",
2019 permissions);
2020 return ffa_error(FFA_INVALID_PARAMETERS);
2021 }
2022 if (instruction_access !=
2023 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
2024 dlog_verbose(
2025 "Invalid instruction access permissions %#x "
2026 "for sending memory.\n",
2027 permissions);
2028 return ffa_error(FFA_INVALID_PARAMETERS);
2029 }
2030 if (share_func == FFA_MEM_SHARE_32) {
2031 if (data_access == FFA_DATA_ACCESS_NOT_SPECIFIED) {
2032 dlog_verbose(
2033 "Invalid data access permissions %#x "
2034 "for sharing memory.\n",
2035 permissions);
2036 return ffa_error(FFA_INVALID_PARAMETERS);
2037 }
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002038 /*
2039 * According to section 10.10.3 of the FF-A v1.1 EAC0
2040 * spec, NX is required for share operations (but must
2041 * not be specified by the sender) so set it in the
2042 * copy that we store, ready to be returned to the
2043 * retriever.
2044 */
2045 if (vm_id_is_current_world(receiver_id)) {
2046 ffa_set_instruction_access_attr(
2047 &permissions,
2048 FFA_INSTRUCTION_ACCESS_NX);
2049 receiver_permissions.permissions = permissions;
2050 }
J-Alves363f5722022-04-25 17:37:37 +01002051 }
2052 if (share_func == FFA_MEM_LEND_32 &&
2053 data_access == FFA_DATA_ACCESS_NOT_SPECIFIED) {
2054 dlog_verbose(
2055 "Invalid data access permissions %#x for "
2056 "lending memory.\n",
2057 permissions);
2058 return ffa_error(FFA_INVALID_PARAMETERS);
2059 }
2060
2061 if (share_func == FFA_MEM_DONATE_32 &&
2062 data_access != FFA_DATA_ACCESS_NOT_SPECIFIED) {
2063 dlog_verbose(
2064 "Invalid data access permissions %#x for "
2065 "donating memory.\n",
2066 permissions);
2067 return ffa_error(FFA_INVALID_PARAMETERS);
2068 }
Andrew Walbrana65a1322020-04-06 19:32:32 +01002069 }
2070
Olivier Deprez4342a3c2022-02-28 09:37:25 +01002071 /* Memory region attributes NS-Bit MBZ for FFA_MEM_SHARE/LEND/DONATE. */
2072 security_state =
2073 ffa_get_memory_security_attr(memory_region->attributes);
2074 if (security_state != FFA_MEMORY_SECURITY_UNSPECIFIED) {
2075 dlog_verbose(
2076 "Invalid security state for memory share operation.\n");
2077 return ffa_error(FFA_INVALID_PARAMETERS);
2078 }
2079
Federico Recanatid937f5e2021-12-20 17:38:23 +01002080 /*
J-Alves807794e2022-06-16 13:42:47 +01002081 * If a memory donate or lend with single borrower, the memory type
2082 * shall not be specified by the sender.
Federico Recanatid937f5e2021-12-20 17:38:23 +01002083 */
J-Alves807794e2022-06-16 13:42:47 +01002084 if (share_func == FFA_MEM_DONATE_32 ||
2085 (share_func == FFA_MEM_LEND_32 &&
2086 memory_region->receiver_count == 1)) {
2087 if (ffa_get_memory_type_attr(memory_region->attributes) !=
2088 FFA_MEMORY_NOT_SPECIFIED_MEM) {
2089 dlog_verbose(
2090 "Memory type shall not be specified by "
2091 "sender.\n");
2092 return ffa_error(FFA_INVALID_PARAMETERS);
2093 }
2094 } else {
2095 /*
2096 * Check that sender's memory attributes match Hafnium
2097 * expectations: Normal Memory, Inner shareable, Write-Back
2098 * Read-Allocate Write-Allocate Cacheable.
2099 */
2100 ret = ffa_memory_attributes_validate(memory_region->attributes);
2101 if (ret.func != FFA_SUCCESS_32) {
2102 return ret;
2103 }
Federico Recanatid937f5e2021-12-20 17:38:23 +01002104 }
2105
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002106 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbrana65a1322020-04-06 19:32:32 +01002107}
2108
2109/**
Andrew Walbranca808b12020-05-15 17:22:28 +01002110 * Gets the share state for continuing an operation to donate, lend or share
2111 * memory, and checks that it is a valid request.
2112 *
2113 * Returns FFA_SUCCESS if the request was valid, or the relevant FFA_ERROR if
2114 * not.
2115 */
J-Alvesfdd29272022-07-19 13:16:31 +01002116struct ffa_value ffa_memory_send_continue_validate(
Andrew Walbranca808b12020-05-15 17:22:28 +01002117 struct share_states_locked share_states, ffa_memory_handle_t handle,
J-Alves19e20cf2023-08-02 12:48:55 +01002118 struct ffa_memory_share_state **share_state_ret, ffa_id_t from_vm_id,
Andrew Walbranca808b12020-05-15 17:22:28 +01002119 struct mpool *page_pool)
2120{
2121 struct ffa_memory_share_state *share_state;
2122 struct ffa_memory_region *memory_region;
2123
Daniel Boulbya2f8c662021-11-26 17:52:53 +00002124 assert(share_state_ret != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +01002125
2126 /*
2127 * Look up the share state by handle and make sure that the VM ID
2128 * matches.
2129 */
Karl Meakin4a2854a2023-06-30 16:26:52 +01002130 share_state = get_share_state(share_states, handle);
J-Alvesb56aac82023-11-10 09:44:43 +00002131 if (share_state == NULL) {
Andrew Walbranca808b12020-05-15 17:22:28 +01002132 dlog_verbose(
2133 "Invalid handle %#x for memory send continuation.\n",
2134 handle);
2135 return ffa_error(FFA_INVALID_PARAMETERS);
2136 }
2137 memory_region = share_state->memory_region;
2138
J-Alvesfdd29272022-07-19 13:16:31 +01002139 if (vm_id_is_current_world(from_vm_id) &&
2140 memory_region->sender != from_vm_id) {
Andrew Walbranca808b12020-05-15 17:22:28 +01002141 dlog_verbose("Invalid sender %d.\n", memory_region->sender);
2142 return ffa_error(FFA_INVALID_PARAMETERS);
2143 }
2144
2145 if (share_state->sending_complete) {
2146 dlog_verbose(
2147 "Sending of memory handle %#x is already complete.\n",
2148 handle);
2149 return ffa_error(FFA_INVALID_PARAMETERS);
2150 }
2151
2152 if (share_state->fragment_count == MAX_FRAGMENTS) {
2153 /*
2154 * Log a warning as this is a sign that MAX_FRAGMENTS should
2155 * probably be increased.
2156 */
2157 dlog_warning(
2158 "Too many fragments for memory share with handle %#x; "
2159 "only %d supported.\n",
2160 handle, MAX_FRAGMENTS);
2161 /* Free share state, as it's not possible to complete it. */
2162 share_state_free(share_states, share_state, page_pool);
2163 return ffa_error(FFA_NO_MEMORY);
2164 }
2165
2166 *share_state_ret = share_state;
2167
2168 return (struct ffa_value){.func = FFA_SUCCESS_32};
2169}
2170
2171/**
J-Alves95df0ef2022-12-07 10:09:48 +00002172 * Checks if there is at least one receiver from the other world.
2173 */
J-Alvesfdd29272022-07-19 13:16:31 +01002174bool memory_region_receivers_from_other_world(
J-Alves95df0ef2022-12-07 10:09:48 +00002175 struct ffa_memory_region *memory_region)
2176{
2177 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002178 struct ffa_memory_access *receiver =
2179 ffa_memory_region_get_receiver(memory_region, i);
2180 assert(receiver != NULL);
2181 ffa_id_t receiver_id = receiver->receiver_permissions.receiver;
2182
2183 if (!vm_id_is_current_world(receiver_id)) {
J-Alves95df0ef2022-12-07 10:09:48 +00002184 return true;
2185 }
2186 }
2187 return false;
2188}
2189
2190/**
J-Alves9da280b2022-12-21 14:55:39 +00002191 * Validates a call to donate, lend or share memory in which Hafnium is the
2192 * designated allocator of the memory handle. In practice, this also means
2193 * Hafnium is responsible for managing the state structures for the transaction.
2194 * If Hafnium is the SPMC, it should allocate the memory handle when either the
2195 * sender is an SP or there is at least one borrower that is an SP.
2196 * If Hafnium is the hypervisor, it should allocate the memory handle when
2197 * operation involves only NWd VMs.
2198 *
2199 * If validation goes well, Hafnium updates the stage-2 page tables of the
2200 * sender. Validation consists of checking if the message length and number of
2201 * memory region constituents match, and if the transition is valid for the
2202 * type of memory sending operation.
Andrew Walbran475c1452020-02-07 13:22:22 +00002203 *
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002204 * Assumes that the caller has already found and locked the sender VM and copied
2205 * the memory region descriptor from the sender's TX buffer to a freshly
2206 * allocated page from Hafnium's internal pool. The caller must have also
2207 * validated that the receiver VM ID is valid.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002208 *
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002209 * This function takes ownership of the `memory_region` passed in and will free
2210 * it when necessary; it must not be freed by the caller.
Jose Marinho09b1db82019-08-08 09:16:59 +01002211 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002212struct ffa_value ffa_memory_send(struct vm_locked from_locked,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002213 struct ffa_memory_region *memory_region,
Andrew Walbran130a8ae2020-05-15 16:27:15 +01002214 uint32_t memory_share_length,
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002215 uint32_t fragment_length, uint32_t share_func,
2216 struct mpool *page_pool)
Jose Marinho09b1db82019-08-08 09:16:59 +01002217{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002218 struct ffa_value ret;
Andrew Walbranca808b12020-05-15 17:22:28 +01002219 struct share_states_locked share_states;
2220 struct ffa_memory_share_state *share_state;
Jose Marinho09b1db82019-08-08 09:16:59 +01002221
2222 /*
Andrew Walbrana65a1322020-04-06 19:32:32 +01002223 * If there is an error validating the `memory_region` then we need to
2224 * free it because we own it but we won't be storing it in a share state
2225 * after all.
Jose Marinho09b1db82019-08-08 09:16:59 +01002226 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002227 ret = ffa_memory_send_validate(from_locked, memory_region,
2228 memory_share_length, fragment_length,
J-Alves363f5722022-04-25 17:37:37 +01002229 share_func);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002230 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002231 mpool_free(page_pool, memory_region);
Andrew Walbrana65a1322020-04-06 19:32:32 +01002232 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01002233 }
2234
Andrew Walbrana65a1322020-04-06 19:32:32 +01002235 /* Set flag for share function, ready to be retrieved later. */
2236 switch (share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002237 case FFA_MEM_SHARE_32:
Andrew Walbrana65a1322020-04-06 19:32:32 +01002238 memory_region->flags |=
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002239 FFA_MEMORY_REGION_TRANSACTION_TYPE_SHARE;
Andrew Walbrana65a1322020-04-06 19:32:32 +01002240 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002241 case FFA_MEM_LEND_32:
2242 memory_region->flags |= FFA_MEMORY_REGION_TRANSACTION_TYPE_LEND;
Andrew Walbrana65a1322020-04-06 19:32:32 +01002243 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002244 case FFA_MEM_DONATE_32:
Andrew Walbrana65a1322020-04-06 19:32:32 +01002245 memory_region->flags |=
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002246 FFA_MEMORY_REGION_TRANSACTION_TYPE_DONATE;
Andrew Walbrana65a1322020-04-06 19:32:32 +01002247 break;
Jose Marinho09b1db82019-08-08 09:16:59 +01002248 }
2249
Andrew Walbranca808b12020-05-15 17:22:28 +01002250 share_states = share_states_lock();
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002251 /*
2252 * Allocate a share state before updating the page table. Otherwise if
2253 * updating the page table succeeded but allocating the share state
2254 * failed then it would leave the memory in a state where nobody could
2255 * get it back.
2256 */
Karl Meakin52cdfe72023-06-30 14:49:10 +01002257 share_state = allocate_share_state(share_states, share_func,
2258 memory_region, fragment_length,
2259 FFA_MEMORY_HANDLE_INVALID);
J-Alvesb56aac82023-11-10 09:44:43 +00002260 if (share_state == NULL) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002261 dlog_verbose("Failed to allocate share state.\n");
2262 mpool_free(page_pool, memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +01002263 ret = ffa_error(FFA_NO_MEMORY);
2264 goto out;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002265 }
2266
Andrew Walbranca808b12020-05-15 17:22:28 +01002267 if (fragment_length == memory_share_length) {
2268 /* No more fragments to come, everything fit in one message. */
J-Alves2a0d2882020-10-29 14:49:50 +00002269 ret = ffa_memory_send_complete(
2270 from_locked, share_states, share_state, page_pool,
2271 &(share_state->sender_orig_mode));
Andrew Walbranca808b12020-05-15 17:22:28 +01002272 } else {
J-Alvesfdd29272022-07-19 13:16:31 +01002273 /*
2274 * Use sender ID from 'memory_region' assuming
2275 * that at this point it has been validated:
2276 * - MBZ at virtual FF-A instance.
2277 */
J-Alves19e20cf2023-08-02 12:48:55 +01002278 ffa_id_t sender_to_ret =
J-Alvesfdd29272022-07-19 13:16:31 +01002279 (from_locked.vm->id == HF_OTHER_WORLD_ID)
2280 ? memory_region->sender
2281 : 0;
Andrew Walbranca808b12020-05-15 17:22:28 +01002282 ret = (struct ffa_value){
2283 .func = FFA_MEM_FRAG_RX_32,
J-Alvesee68c542020-10-29 17:48:20 +00002284 .arg1 = (uint32_t)memory_region->handle,
2285 .arg2 = (uint32_t)(memory_region->handle >> 32),
J-Alvesfdd29272022-07-19 13:16:31 +01002286 .arg3 = fragment_length,
2287 .arg4 = (uint32_t)(sender_to_ret & 0xffff) << 16};
Andrew Walbranca808b12020-05-15 17:22:28 +01002288 }
2289
2290out:
2291 share_states_unlock(&share_states);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002292 dump_share_states();
Andrew Walbranca808b12020-05-15 17:22:28 +01002293 return ret;
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002294}
2295
2296/**
J-Alves8505a8a2022-06-15 18:10:18 +01002297 * Continues an operation to donate, lend or share memory to a VM from current
2298 * world. If this is the last fragment then checks that the transition is valid
2299 * for the type of memory sending operation and updates the stage-2 page tables
2300 * of the sender.
Andrew Walbranca808b12020-05-15 17:22:28 +01002301 *
2302 * Assumes that the caller has already found and locked the sender VM and copied
2303 * the memory region descriptor from the sender's TX buffer to a freshly
2304 * allocated page from Hafnium's internal pool.
2305 *
2306 * This function takes ownership of the `fragment` passed in; it must not be
2307 * freed by the caller.
2308 */
2309struct ffa_value ffa_memory_send_continue(struct vm_locked from_locked,
2310 void *fragment,
2311 uint32_t fragment_length,
2312 ffa_memory_handle_t handle,
2313 struct mpool *page_pool)
2314{
2315 struct share_states_locked share_states = share_states_lock();
2316 struct ffa_memory_share_state *share_state;
2317 struct ffa_value ret;
2318 struct ffa_memory_region *memory_region;
2319
Demi Marie Obenour73a1e942023-02-04 14:09:18 -05002320 CHECK(is_aligned(fragment,
2321 alignof(struct ffa_memory_region_constituent)));
2322 if (fragment_length % sizeof(struct ffa_memory_region_constituent) !=
2323 0) {
2324 dlog_verbose("Fragment length %u misaligned.\n",
2325 fragment_length);
2326 ret = ffa_error(FFA_INVALID_PARAMETERS);
2327 goto out_free_fragment;
2328 }
2329
Andrew Walbranca808b12020-05-15 17:22:28 +01002330 ret = ffa_memory_send_continue_validate(share_states, handle,
2331 &share_state,
2332 from_locked.vm->id, page_pool);
2333 if (ret.func != FFA_SUCCESS_32) {
2334 goto out_free_fragment;
2335 }
2336 memory_region = share_state->memory_region;
2337
J-Alves95df0ef2022-12-07 10:09:48 +00002338 if (memory_region_receivers_from_other_world(memory_region)) {
Andrew Walbranca808b12020-05-15 17:22:28 +01002339 dlog_error(
2340 "Got hypervisor-allocated handle for memory send to "
J-Alves8505a8a2022-06-15 18:10:18 +01002341 "other world. This should never happen, and indicates "
2342 "a bug in "
Andrew Walbranca808b12020-05-15 17:22:28 +01002343 "EL3 code.\n");
2344 ret = ffa_error(FFA_INVALID_PARAMETERS);
2345 goto out_free_fragment;
2346 }
2347
2348 /* Add this fragment. */
2349 share_state->fragments[share_state->fragment_count] = fragment;
2350 share_state->fragment_constituent_counts[share_state->fragment_count] =
2351 fragment_length / sizeof(struct ffa_memory_region_constituent);
2352 share_state->fragment_count++;
2353
2354 /* Check whether the memory send operation is now ready to complete. */
2355 if (share_state_sending_complete(share_states, share_state)) {
J-Alves2a0d2882020-10-29 14:49:50 +00002356 ret = ffa_memory_send_complete(
2357 from_locked, share_states, share_state, page_pool,
2358 &(share_state->sender_orig_mode));
Andrew Walbranca808b12020-05-15 17:22:28 +01002359 } else {
2360 ret = (struct ffa_value){
2361 .func = FFA_MEM_FRAG_RX_32,
2362 .arg1 = (uint32_t)handle,
2363 .arg2 = (uint32_t)(handle >> 32),
2364 .arg3 = share_state_next_fragment_offset(share_states,
2365 share_state)};
2366 }
2367 goto out;
2368
2369out_free_fragment:
2370 mpool_free(page_pool, fragment);
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002371
2372out:
Andrew Walbranca808b12020-05-15 17:22:28 +01002373 share_states_unlock(&share_states);
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002374 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002375}
2376
Andrew Walbranca808b12020-05-15 17:22:28 +01002377/** Clean up after the receiver has finished retrieving a memory region. */
2378static void ffa_memory_retrieve_complete(
2379 struct share_states_locked share_states,
2380 struct ffa_memory_share_state *share_state, struct mpool *page_pool)
2381{
2382 if (share_state->share_func == FFA_MEM_DONATE_32) {
2383 /*
2384 * Memory that has been donated can't be relinquished,
2385 * so no need to keep the share state around.
2386 */
2387 share_state_free(share_states, share_state, page_pool);
2388 dlog_verbose("Freed share state for donate.\n");
2389 }
2390}
2391
J-Alves2d8457f2022-10-05 11:06:41 +01002392/**
2393 * Initialises the given memory region descriptor to be used for an
2394 * `FFA_MEM_RETRIEVE_RESP`, including the given constituents for the first
2395 * fragment.
2396 * The memory region descriptor is initialized according to retriever's
2397 * FF-A version.
2398 *
2399 * Returns true on success, or false if the given constituents won't all fit in
2400 * the first fragment.
2401 */
2402static bool ffa_retrieved_memory_region_init(
2403 void *response, uint32_t ffa_version, size_t response_max_size,
J-Alves19e20cf2023-08-02 12:48:55 +01002404 ffa_id_t sender, ffa_memory_attributes_t attributes,
J-Alves2d8457f2022-10-05 11:06:41 +01002405 ffa_memory_region_flags_t flags, ffa_memory_handle_t handle,
Daniel Boulbyde974ca2023-12-12 13:53:31 +00002406 ffa_memory_access_permissions_t permissions,
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002407 struct ffa_memory_access *receivers, size_t receiver_count,
2408 uint32_t memory_access_desc_size, uint32_t page_count,
2409 uint32_t total_constituent_count,
J-Alves2d8457f2022-10-05 11:06:41 +01002410 const struct ffa_memory_region_constituent constituents[],
2411 uint32_t fragment_constituent_count, uint32_t *total_length,
2412 uint32_t *fragment_length)
2413{
2414 struct ffa_composite_memory_region *composite_memory_region;
J-Alves2d8457f2022-10-05 11:06:41 +01002415 uint32_t i;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002416 uint32_t composite_offset;
J-Alves2d8457f2022-10-05 11:06:41 +01002417 uint32_t constituents_offset;
J-Alves2d8457f2022-10-05 11:06:41 +01002418
2419 assert(response != NULL);
2420
2421 if (ffa_version == MAKE_FFA_VERSION(1, 0)) {
2422 struct ffa_memory_region_v1_0 *retrieve_response =
2423 (struct ffa_memory_region_v1_0 *)response;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002424 struct ffa_memory_access_v1_0 *receiver;
J-Alves2d8457f2022-10-05 11:06:41 +01002425
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002426 ffa_memory_region_init_header_v1_0(retrieve_response, sender,
2427 attributes, flags, handle, 0,
2428 receiver_count);
J-Alves2d8457f2022-10-05 11:06:41 +01002429
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002430 receiver = (struct ffa_memory_access_v1_0 *)
2431 retrieve_response->receivers;
J-Alves2d8457f2022-10-05 11:06:41 +01002432 receiver_count = retrieve_response->receiver_count;
2433
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002434 for (uint32_t i = 0; i < receiver_count; i++) {
2435 ffa_id_t receiver_id =
2436 receivers[i].receiver_permissions.receiver;
2437 ffa_memory_receiver_flags_t recv_flags =
2438 receivers[i].receiver_permissions.flags;
2439
2440 /*
2441 * Initialized here as in memory retrieve responses we
2442 * currently expect one borrower to be specified.
2443 */
2444 ffa_memory_access_init_v1_0(
2445 receiver, receiver_id,
2446 ffa_get_data_access_attr(permissions),
2447 ffa_get_instruction_access_attr(permissions),
2448 recv_flags);
2449 }
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002450
2451 composite_offset =
J-Alves2d8457f2022-10-05 11:06:41 +01002452 sizeof(struct ffa_memory_region_v1_0) +
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002453 receiver_count * sizeof(struct ffa_memory_access_v1_0);
2454 receiver->composite_memory_region_offset = composite_offset;
J-Alves2d8457f2022-10-05 11:06:41 +01002455
2456 composite_memory_region = ffa_memory_region_get_composite_v1_0(
2457 retrieve_response, 0);
2458 } else {
J-Alves2d8457f2022-10-05 11:06:41 +01002459 struct ffa_memory_region *retrieve_response =
2460 (struct ffa_memory_region *)response;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002461 struct ffa_memory_access *retrieve_response_receivers;
J-Alves2d8457f2022-10-05 11:06:41 +01002462
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002463 ffa_memory_region_init_header(
2464 retrieve_response, sender, attributes, flags, handle, 0,
2465 receiver_count, memory_access_desc_size);
J-Alves2d8457f2022-10-05 11:06:41 +01002466
2467 /*
2468 * Note that `sizeof(struct_ffa_memory_region)` and
2469 * `sizeof(struct ffa_memory_access)` must both be multiples of
2470 * 16 (as verified by the asserts in `ffa_memory.c`, so it is
2471 * guaranteed that the offset we calculate here is aligned to a
2472 * 64-bit boundary and so 64-bit values can be copied without
2473 * alignment faults.
2474 */
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002475 composite_offset =
Daniel Boulby41ef8ba2023-10-13 17:01:22 +01002476 retrieve_response->receivers_offset +
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002477 (uint32_t)(receiver_count *
2478 retrieve_response->memory_access_desc_size);
J-Alves2d8457f2022-10-05 11:06:41 +01002479
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002480 retrieve_response_receivers =
2481 ffa_memory_region_get_receiver(retrieve_response, 0);
2482 assert(retrieve_response_receivers != NULL);
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002483
2484 /*
2485 * Initialized here as in memory retrieve responses we currently
2486 * expect one borrower to be specified.
2487 */
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002488 memcpy_s(retrieve_response_receivers,
2489 sizeof(struct ffa_memory_access) * receiver_count,
2490 receivers,
2491 sizeof(struct ffa_memory_access) * receiver_count);
2492
2493 retrieve_response_receivers->composite_memory_region_offset =
2494 composite_offset;
2495
J-Alves2d8457f2022-10-05 11:06:41 +01002496 composite_memory_region =
2497 ffa_memory_region_get_composite(retrieve_response, 0);
2498 }
2499
J-Alves2d8457f2022-10-05 11:06:41 +01002500 assert(composite_memory_region != NULL);
2501
J-Alves2d8457f2022-10-05 11:06:41 +01002502 composite_memory_region->page_count = page_count;
2503 composite_memory_region->constituent_count = total_constituent_count;
2504 composite_memory_region->reserved_0 = 0;
2505
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002506 constituents_offset =
2507 composite_offset + sizeof(struct ffa_composite_memory_region);
J-Alves2d8457f2022-10-05 11:06:41 +01002508 if (constituents_offset +
2509 fragment_constituent_count *
2510 sizeof(struct ffa_memory_region_constituent) >
2511 response_max_size) {
2512 return false;
2513 }
2514
2515 for (i = 0; i < fragment_constituent_count; ++i) {
2516 composite_memory_region->constituents[i] = constituents[i];
2517 }
2518
2519 if (total_length != NULL) {
2520 *total_length =
2521 constituents_offset +
2522 composite_memory_region->constituent_count *
2523 sizeof(struct ffa_memory_region_constituent);
2524 }
2525 if (fragment_length != NULL) {
2526 *fragment_length =
2527 constituents_offset +
2528 fragment_constituent_count *
2529 sizeof(struct ffa_memory_region_constituent);
2530 }
2531
2532 return true;
2533}
2534
J-Alves96de29f2022-04-26 16:05:24 +01002535/**
2536 * Validates the retrieved permissions against those specified by the lender
2537 * of memory share operation. Optionally can help set the permissions to be used
2538 * for the S2 mapping, through the `permissions` argument.
J-Alvesdcad8992023-09-15 14:10:35 +01002539 * Returns FFA_SUCCESS if all the fields are valid. FFA_ERROR, with error code:
2540 * - FFA_INVALID_PARAMETERS -> if the fields have invalid values as per the
2541 * specification for each ABI.
2542 * - FFA_DENIED -> if the permissions specified by the retriever are not
2543 * less permissive than those provided by the sender.
J-Alves96de29f2022-04-26 16:05:24 +01002544 */
J-Alvesdcad8992023-09-15 14:10:35 +01002545static struct ffa_value ffa_memory_retrieve_is_memory_access_valid(
2546 uint32_t share_func, enum ffa_data_access sent_data_access,
J-Alves96de29f2022-04-26 16:05:24 +01002547 enum ffa_data_access requested_data_access,
2548 enum ffa_instruction_access sent_instruction_access,
2549 enum ffa_instruction_access requested_instruction_access,
J-Alvesdcad8992023-09-15 14:10:35 +01002550 ffa_memory_access_permissions_t *permissions, bool multiple_borrowers)
J-Alves96de29f2022-04-26 16:05:24 +01002551{
2552 switch (sent_data_access) {
2553 case FFA_DATA_ACCESS_NOT_SPECIFIED:
2554 case FFA_DATA_ACCESS_RW:
2555 if (requested_data_access == FFA_DATA_ACCESS_NOT_SPECIFIED ||
2556 requested_data_access == FFA_DATA_ACCESS_RW) {
2557 if (permissions != NULL) {
2558 ffa_set_data_access_attr(permissions,
2559 FFA_DATA_ACCESS_RW);
2560 }
2561 break;
2562 }
2563 /* Intentional fall-through. */
2564 case FFA_DATA_ACCESS_RO:
2565 if (requested_data_access == FFA_DATA_ACCESS_NOT_SPECIFIED ||
2566 requested_data_access == FFA_DATA_ACCESS_RO) {
2567 if (permissions != NULL) {
2568 ffa_set_data_access_attr(permissions,
2569 FFA_DATA_ACCESS_RO);
2570 }
2571 break;
2572 }
2573 dlog_verbose(
2574 "Invalid data access requested; sender specified "
2575 "permissions %#x but receiver requested %#x.\n",
2576 sent_data_access, requested_data_access);
J-Alvesdcad8992023-09-15 14:10:35 +01002577 return ffa_error(FFA_DENIED);
J-Alves96de29f2022-04-26 16:05:24 +01002578 case FFA_DATA_ACCESS_RESERVED:
2579 panic("Got unexpected FFA_DATA_ACCESS_RESERVED. Should be "
2580 "checked before this point.");
2581 }
2582
J-Alvesdcad8992023-09-15 14:10:35 +01002583 /*
2584 * For operations with a single borrower, If it is an FFA_MEMORY_LEND
2585 * or FFA_MEMORY_DONATE the retriever should have specifed the
2586 * instruction permissions it wishes to receive.
2587 */
2588 switch (share_func) {
2589 case FFA_MEM_SHARE_32:
2590 if (requested_instruction_access !=
2591 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
2592 dlog_verbose(
2593 "%s: for share instruction permissions must "
2594 "NOT be specified.\n",
2595 __func__);
2596 return ffa_error(FFA_INVALID_PARAMETERS);
2597 }
2598 break;
2599 case FFA_MEM_LEND_32:
2600 /*
2601 * For operations with multiple borrowers only permit XN
2602 * permissions, and both Sender and borrower should have used
2603 * FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED.
2604 */
2605 if (multiple_borrowers) {
2606 if (requested_instruction_access !=
2607 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
2608 dlog_verbose(
2609 "%s: lend/share/donate with multiple "
2610 "borrowers "
2611 "instruction permissions must NOT be "
2612 "specified.\n",
2613 __func__);
2614 return ffa_error(FFA_INVALID_PARAMETERS);
2615 }
2616 break;
2617 }
2618 /* Fall through if the operation targets a single borrower. */
2619 case FFA_MEM_DONATE_32:
2620 if (!multiple_borrowers &&
2621 requested_instruction_access ==
2622 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
2623 dlog_verbose(
2624 "%s: for lend/donate with single borrower "
2625 "instruction permissions must be speficified "
2626 "by borrower\n",
2627 __func__);
2628 return ffa_error(FFA_INVALID_PARAMETERS);
2629 }
2630 break;
2631 default:
2632 panic("%s: Wrong func id provided.\n", __func__);
2633 }
2634
J-Alves96de29f2022-04-26 16:05:24 +01002635 switch (sent_instruction_access) {
2636 case FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED:
2637 case FFA_INSTRUCTION_ACCESS_X:
J-Alvesdcad8992023-09-15 14:10:35 +01002638 if (requested_instruction_access == FFA_INSTRUCTION_ACCESS_X) {
J-Alves96de29f2022-04-26 16:05:24 +01002639 if (permissions != NULL) {
2640 ffa_set_instruction_access_attr(
2641 permissions, FFA_INSTRUCTION_ACCESS_X);
2642 }
2643 break;
2644 }
J-Alvesdcad8992023-09-15 14:10:35 +01002645 /*
2646 * Fall through if requested permissions are less
2647 * permissive than those provided by the sender.
2648 */
J-Alves96de29f2022-04-26 16:05:24 +01002649 case FFA_INSTRUCTION_ACCESS_NX:
2650 if (requested_instruction_access ==
2651 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED ||
2652 requested_instruction_access == FFA_INSTRUCTION_ACCESS_NX) {
2653 if (permissions != NULL) {
2654 ffa_set_instruction_access_attr(
2655 permissions, FFA_INSTRUCTION_ACCESS_NX);
2656 }
2657 break;
2658 }
2659 dlog_verbose(
2660 "Invalid instruction access requested; sender "
2661 "specified permissions %#x but receiver requested "
2662 "%#x.\n",
2663 sent_instruction_access, requested_instruction_access);
J-Alvesdcad8992023-09-15 14:10:35 +01002664 return ffa_error(FFA_DENIED);
J-Alves96de29f2022-04-26 16:05:24 +01002665 case FFA_INSTRUCTION_ACCESS_RESERVED:
2666 panic("Got unexpected FFA_INSTRUCTION_ACCESS_RESERVED. Should "
2667 "be checked before this point.");
2668 }
2669
J-Alvesdcad8992023-09-15 14:10:35 +01002670 return (struct ffa_value){.func = FFA_SUCCESS_32};
J-Alves96de29f2022-04-26 16:05:24 +01002671}
2672
2673/**
2674 * Validate the receivers' permissions in the retrieve request against those
2675 * specified by the lender.
2676 * In the `permissions` argument returns the permissions to set at S2 for the
2677 * caller to the FFA_MEMORY_RETRIEVE_REQ.
J-Alves3456e032023-07-20 12:20:05 +01002678 * The function looks into the flag to bypass multiple borrower checks:
2679 * - If not set returns FFA_SUCCESS if all specified permissions are valid.
2680 * - If set returns FFA_SUCCESS if the descriptor contains the permissions
2681 * to the caller of FFA_MEM_RETRIEVE_REQ and they are valid. Other permissions
2682 * are ignored, if provided.
J-Alves96de29f2022-04-26 16:05:24 +01002683 */
2684static struct ffa_value ffa_memory_retrieve_validate_memory_access_list(
2685 struct ffa_memory_region *memory_region,
J-Alves19e20cf2023-08-02 12:48:55 +01002686 struct ffa_memory_region *retrieve_request, ffa_id_t to_vm_id,
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002687 ffa_memory_access_permissions_t *permissions,
2688 struct ffa_memory_access **receiver_ret, uint32_t func_id)
J-Alves96de29f2022-04-26 16:05:24 +01002689{
2690 uint32_t retrieve_receiver_index;
J-Alves3456e032023-07-20 12:20:05 +01002691 bool bypass_multi_receiver_check =
2692 (retrieve_request->flags &
2693 FFA_MEMORY_REGION_FLAG_BYPASS_BORROWERS_CHECK) != 0U;
J-Alvesdcad8992023-09-15 14:10:35 +01002694 const uint32_t region_receiver_count = memory_region->receiver_count;
2695 struct ffa_value ret;
J-Alves96de29f2022-04-26 16:05:24 +01002696
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002697 assert(receiver_ret != NULL);
J-Alves96de29f2022-04-26 16:05:24 +01002698 assert(permissions != NULL);
2699
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002700 *permissions = 0;
2701
J-Alves3456e032023-07-20 12:20:05 +01002702 if (!bypass_multi_receiver_check) {
J-Alvesdcad8992023-09-15 14:10:35 +01002703 if (retrieve_request->receiver_count != region_receiver_count) {
J-Alves3456e032023-07-20 12:20:05 +01002704 dlog_verbose(
2705 "Retrieve request should contain same list of "
2706 "borrowers, as specified by the lender.\n");
2707 return ffa_error(FFA_INVALID_PARAMETERS);
2708 }
2709 } else {
2710 if (retrieve_request->receiver_count != 1) {
2711 dlog_verbose(
2712 "Set bypass multiple borrower check, receiver "
2713 "list must be sized 1 (%x)\n",
2714 memory_region->receiver_count);
2715 return ffa_error(FFA_INVALID_PARAMETERS);
2716 }
J-Alves96de29f2022-04-26 16:05:24 +01002717 }
2718
2719 retrieve_receiver_index = retrieve_request->receiver_count;
2720
J-Alves96de29f2022-04-26 16:05:24 +01002721 for (uint32_t i = 0U; i < retrieve_request->receiver_count; i++) {
2722 ffa_memory_access_permissions_t sent_permissions;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002723 struct ffa_memory_access *retrieve_request_receiver =
2724 ffa_memory_region_get_receiver(retrieve_request, i);
2725 assert(retrieve_request_receiver != NULL);
J-Alves96de29f2022-04-26 16:05:24 +01002726 ffa_memory_access_permissions_t requested_permissions =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002727 retrieve_request_receiver->receiver_permissions
2728 .permissions;
J-Alves19e20cf2023-08-02 12:48:55 +01002729 ffa_id_t current_receiver_id =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002730 retrieve_request_receiver->receiver_permissions
2731 .receiver;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002732 struct ffa_memory_access *receiver;
2733 uint32_t mem_region_receiver_index;
2734 bool permissions_RO;
2735 bool clear_memory_flags;
J-Alves96de29f2022-04-26 16:05:24 +01002736 bool found_to_id = current_receiver_id == to_vm_id;
2737
J-Alves3456e032023-07-20 12:20:05 +01002738 if (bypass_multi_receiver_check && !found_to_id) {
2739 dlog_verbose(
2740 "Bypass multiple borrower check for id %x.\n",
2741 current_receiver_id);
2742 continue;
2743 }
2744
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002745 if (retrieve_request_receiver->composite_memory_region_offset !=
2746 0U) {
2747 dlog_verbose(
2748 "Retriever specified address ranges not "
2749 "supported (got offset %d).\n",
2750 retrieve_request_receiver
2751 ->composite_memory_region_offset);
2752 return ffa_error(FFA_INVALID_PARAMETERS);
2753 }
2754
J-Alves96de29f2022-04-26 16:05:24 +01002755 /*
2756 * Find the current receiver in the transaction descriptor from
2757 * sender.
2758 */
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002759 mem_region_receiver_index =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002760 ffa_memory_region_get_receiver_index(
2761 memory_region, current_receiver_id);
J-Alves96de29f2022-04-26 16:05:24 +01002762
2763 if (mem_region_receiver_index ==
2764 memory_region->receiver_count) {
2765 dlog_verbose("%s: receiver %x not found\n", __func__,
2766 current_receiver_id);
2767 return ffa_error(FFA_DENIED);
2768 }
2769
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002770 receiver = ffa_memory_region_get_receiver(
2771 memory_region, mem_region_receiver_index);
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002772 assert(receiver != NULL);
2773
2774 sent_permissions = receiver->receiver_permissions.permissions;
J-Alves96de29f2022-04-26 16:05:24 +01002775
2776 if (found_to_id) {
2777 retrieve_receiver_index = i;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002778
2779 *receiver_ret = receiver;
J-Alves96de29f2022-04-26 16:05:24 +01002780 }
2781
2782 /*
J-Alvesdcad8992023-09-15 14:10:35 +01002783 * Check if retrieve request memory access list is valid:
2784 * - The retrieve request complies with the specification.
2785 * - Permissions are within those specified by the sender.
J-Alves96de29f2022-04-26 16:05:24 +01002786 */
J-Alvesdcad8992023-09-15 14:10:35 +01002787 ret = ffa_memory_retrieve_is_memory_access_valid(
2788 func_id, ffa_get_data_access_attr(sent_permissions),
2789 ffa_get_data_access_attr(requested_permissions),
2790 ffa_get_instruction_access_attr(sent_permissions),
2791 ffa_get_instruction_access_attr(requested_permissions),
2792 found_to_id ? permissions : NULL,
2793 region_receiver_count > 1);
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002794
J-Alvesdcad8992023-09-15 14:10:35 +01002795 if (ret.func != FFA_SUCCESS_32) {
2796 return ret;
J-Alves96de29f2022-04-26 16:05:24 +01002797 }
2798
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002799 permissions_RO = (ffa_get_data_access_attr(*permissions) ==
2800 FFA_DATA_ACCESS_RO);
2801 clear_memory_flags = (retrieve_request->flags &
2802 FFA_MEMORY_REGION_FLAG_CLEAR) != 0U;
2803
J-Alves96de29f2022-04-26 16:05:24 +01002804 /*
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002805 * Can't request PM to clear memory if only provided
2806 * with RO permissions.
J-Alves96de29f2022-04-26 16:05:24 +01002807 */
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00002808 if (found_to_id && permissions_RO && clear_memory_flags) {
J-Alves96de29f2022-04-26 16:05:24 +01002809 dlog_verbose(
2810 "Receiver has RO permissions can not request "
2811 "clear.\n");
2812 return ffa_error(FFA_DENIED);
2813 }
Daniel Boulbyde974ca2023-12-12 13:53:31 +00002814
2815 /*
2816 * Check the impdef in the retrieve_request matches the value in
2817 * the original memory send.
2818 */
2819 if (ffa_version_from_memory_access_desc_size(
2820 memory_region->memory_access_desc_size) >=
2821 MAKE_FFA_VERSION(1, 2) &&
2822 ffa_version_from_memory_access_desc_size(
2823 retrieve_request->memory_access_desc_size) >=
2824 MAKE_FFA_VERSION(1, 2)) {
Daniel Boulbyde974ca2023-12-12 13:53:31 +00002825 if (receiver->impdef.val[0] !=
2826 retrieve_request_receiver->impdef.val[0] ||
2827 receiver->impdef.val[1] !=
2828 retrieve_request_receiver->impdef.val[1]) {
2829 dlog_verbose(
2830 "Impdef value in memory send does not "
2831 "match retrieve request value "
2832 "send value %#x %#x retrieve request "
2833 "value %#x %#x\n",
2834 receiver->impdef.val[0],
2835 receiver->impdef.val[1],
2836 retrieve_request_receiver->impdef
2837 .val[0],
2838 retrieve_request_receiver->impdef
2839 .val[1]);
2840 return ffa_error(FFA_INVALID_PARAMETERS);
2841 }
2842 }
J-Alves96de29f2022-04-26 16:05:24 +01002843 }
2844
2845 if (retrieve_receiver_index == retrieve_request->receiver_count) {
2846 dlog_verbose(
2847 "Retrieve request does not contain caller's (%x) "
2848 "permissions\n",
2849 to_vm_id);
2850 return ffa_error(FFA_INVALID_PARAMETERS);
2851 }
2852
2853 return (struct ffa_value){.func = FFA_SUCCESS_32};
2854}
2855
J-Alvesa9cd7e32022-07-01 13:49:33 +01002856/*
2857 * According to section 16.4.3 of FF-A v1.1 EAC0 specification, the hypervisor
2858 * may issue an FFA_MEM_RETRIEVE_REQ to obtain the memory region description
2859 * of a pending memory sharing operation whose allocator is the SPM, for
2860 * validation purposes before forwarding an FFA_MEM_RECLAIM call. In doing so
2861 * the memory region descriptor of the retrieve request must be zeroed with the
2862 * exception of the sender ID and handle.
2863 */
J-Alves4f0d9c12024-01-17 17:23:11 +00002864bool is_ffa_hypervisor_retrieve_request(struct ffa_memory_region *request,
2865 struct vm_locked to_locked)
J-Alvesa9cd7e32022-07-01 13:49:33 +01002866{
2867 return to_locked.vm->id == HF_HYPERVISOR_VM_ID &&
2868 request->attributes == 0U && request->flags == 0U &&
2869 request->tag == 0U && request->receiver_count == 0U &&
2870 plat_ffa_memory_handle_allocated_by_current_world(
2871 request->handle);
2872}
2873
2874/*
2875 * Helper to reset count of fragments retrieved by the hypervisor.
2876 */
2877static void ffa_memory_retrieve_complete_from_hyp(
2878 struct ffa_memory_share_state *share_state)
2879{
2880 if (share_state->hypervisor_fragment_count ==
2881 share_state->fragment_count) {
2882 share_state->hypervisor_fragment_count = 0;
2883 }
2884}
2885
J-Alves089004f2022-07-13 14:25:44 +01002886/**
J-Alves4f0d9c12024-01-17 17:23:11 +00002887 * Prepares the return of the ffa_value for the memory retrieve response.
2888 */
2889static struct ffa_value ffa_memory_retrieve_resp(uint32_t total_length,
2890 uint32_t fragment_length)
2891{
2892 return (struct ffa_value){.func = FFA_MEM_RETRIEVE_RESP_32,
2893 .arg1 = total_length,
2894 .arg2 = fragment_length};
2895}
2896
2897/**
J-Alves089004f2022-07-13 14:25:44 +01002898 * Validate that the memory region descriptor provided by the borrower on
2899 * FFA_MEM_RETRIEVE_REQ, against saved memory region provided by lender at the
2900 * memory sharing call.
2901 */
2902static struct ffa_value ffa_memory_retrieve_validate(
J-Alves4f0d9c12024-01-17 17:23:11 +00002903 ffa_id_t to_id, struct ffa_memory_region *retrieve_request,
2904 uint32_t retrieve_request_length,
J-Alves089004f2022-07-13 14:25:44 +01002905 struct ffa_memory_region *memory_region, uint32_t *receiver_index,
2906 uint32_t share_func)
2907{
2908 ffa_memory_region_flags_t transaction_type =
2909 retrieve_request->flags &
2910 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK;
Olivier Deprez4342a3c2022-02-28 09:37:25 +01002911 enum ffa_memory_security security_state;
J-Alves4f0d9c12024-01-17 17:23:11 +00002912 const uint64_t memory_access_desc_size =
2913 retrieve_request->memory_access_desc_size;
2914 const uint32_t expected_retrieve_request_length =
2915 retrieve_request->receivers_offset +
2916 (uint32_t)(retrieve_request->receiver_count *
2917 memory_access_desc_size);
J-Alves089004f2022-07-13 14:25:44 +01002918
2919 assert(retrieve_request != NULL);
2920 assert(memory_region != NULL);
2921 assert(receiver_index != NULL);
2922 assert(retrieve_request->sender == memory_region->sender);
2923
J-Alves4f0d9c12024-01-17 17:23:11 +00002924 if (retrieve_request_length != expected_retrieve_request_length) {
2925 dlog_verbose(
2926 "Invalid length for FFA_MEM_RETRIEVE_REQ, expected %d "
2927 "but was %d.\n",
2928 expected_retrieve_request_length,
2929 retrieve_request_length);
2930 return ffa_error(FFA_INVALID_PARAMETERS);
2931 }
2932
2933 if (retrieve_request->sender != memory_region->sender) {
2934 dlog_verbose(
2935 "Memory with handle %#x not fully sent, can't "
2936 "retrieve.\n",
2937 memory_region->handle);
2938 return ffa_error(FFA_DENIED);
2939 }
2940
2941 /*
2942 * The SPMC can only process retrieve requests to memory share
2943 * operations with one borrower from the other world. It can't
2944 * determine the ID of the NWd VM that invoked the retrieve
2945 * request interface call. It relies on the hypervisor to
2946 * validate the caller's ID against that provided in the
2947 * `receivers` list of the retrieve response.
2948 * In case there is only one borrower from the NWd in the
2949 * transaction descriptor, record that in the `receiver_id` for
2950 * later use, and validate in the retrieve request message.
2951 * This limitation is due to the fact SPMC can't determine the
2952 * index in the memory share structures state to update.
2953 */
2954 if (to_id == HF_HYPERVISOR_VM_ID) {
2955 uint32_t other_world_count = 0;
2956
2957 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
2958 struct ffa_memory_access *receiver =
2959 ffa_memory_region_get_receiver(retrieve_request,
2960 0);
2961 assert(receiver != NULL);
2962
2963 to_id = receiver->receiver_permissions.receiver;
2964
2965 if (!vm_id_is_current_world(to_id)) {
2966 other_world_count++;
2967 }
2968 }
2969
2970 if (other_world_count > 1) {
2971 dlog_verbose(
2972 "Support one receiver from the other "
2973 "world.\n");
2974 return ffa_error(FFA_NOT_SUPPORTED);
2975 }
2976 }
J-Alves089004f2022-07-13 14:25:44 +01002977 /*
2978 * Check that the transaction type expected by the receiver is
2979 * correct, if it has been specified.
2980 */
2981 if (transaction_type !=
2982 FFA_MEMORY_REGION_TRANSACTION_TYPE_UNSPECIFIED &&
2983 transaction_type != (memory_region->flags &
2984 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK)) {
2985 dlog_verbose(
2986 "Incorrect transaction type %#x for "
2987 "FFA_MEM_RETRIEVE_REQ, expected %#x for handle %#x.\n",
2988 transaction_type,
2989 memory_region->flags &
2990 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK,
2991 retrieve_request->handle);
2992 return ffa_error(FFA_INVALID_PARAMETERS);
2993 }
2994
2995 if (retrieve_request->tag != memory_region->tag) {
2996 dlog_verbose(
2997 "Incorrect tag %d for FFA_MEM_RETRIEVE_REQ, expected "
2998 "%d for handle %#x.\n",
2999 retrieve_request->tag, memory_region->tag,
3000 retrieve_request->handle);
3001 return ffa_error(FFA_INVALID_PARAMETERS);
3002 }
3003
J-Alves4f0d9c12024-01-17 17:23:11 +00003004 *receiver_index =
3005 ffa_memory_region_get_receiver_index(memory_region, to_id);
J-Alves089004f2022-07-13 14:25:44 +01003006
3007 if (*receiver_index == memory_region->receiver_count) {
3008 dlog_verbose(
3009 "Incorrect receiver VM ID %d for "
3010 "FFA_MEM_RETRIEVE_REQ, for handle %#x.\n",
J-Alves4f0d9c12024-01-17 17:23:11 +00003011 to_id, memory_region->handle);
J-Alves089004f2022-07-13 14:25:44 +01003012 return ffa_error(FFA_INVALID_PARAMETERS);
3013 }
3014
3015 if ((retrieve_request->flags &
3016 FFA_MEMORY_REGION_ADDRESS_RANGE_HINT_VALID) != 0U) {
3017 dlog_verbose(
3018 "Retriever specified 'address range alignment 'hint' "
3019 "not supported.\n");
3020 return ffa_error(FFA_INVALID_PARAMETERS);
3021 }
3022 if ((retrieve_request->flags &
3023 FFA_MEMORY_REGION_ADDRESS_RANGE_HINT_MASK) != 0) {
3024 dlog_verbose(
3025 "Bits 8-5 must be zero in memory region's flags "
3026 "(address range alignment hint not supported).\n");
3027 return ffa_error(FFA_INVALID_PARAMETERS);
3028 }
3029
3030 if ((retrieve_request->flags & ~0x7FF) != 0U) {
3031 dlog_verbose(
3032 "Bits 31-10 must be zero in memory region's flags.\n");
3033 return ffa_error(FFA_INVALID_PARAMETERS);
3034 }
3035
3036 if (share_func == FFA_MEM_SHARE_32 &&
3037 (retrieve_request->flags &
3038 (FFA_MEMORY_REGION_FLAG_CLEAR |
3039 FFA_MEMORY_REGION_FLAG_CLEAR_RELINQUISH)) != 0U) {
3040 dlog_verbose(
3041 "Memory Share operation can't clean after relinquish "
3042 "memory region.\n");
3043 return ffa_error(FFA_INVALID_PARAMETERS);
3044 }
3045
3046 /*
3047 * If the borrower needs the memory to be cleared before mapping
3048 * to its address space, the sender should have set the flag
3049 * when calling FFA_MEM_LEND/FFA_MEM_DONATE, else return
3050 * FFA_DENIED.
3051 */
3052 if ((retrieve_request->flags & FFA_MEMORY_REGION_FLAG_CLEAR) != 0U &&
3053 (memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR) == 0U) {
3054 dlog_verbose(
3055 "Borrower needs memory cleared. Sender needs to set "
3056 "flag for clearing memory.\n");
3057 return ffa_error(FFA_DENIED);
3058 }
3059
Olivier Deprez4342a3c2022-02-28 09:37:25 +01003060 /* Memory region attributes NS-Bit MBZ for FFA_MEM_RETRIEVE_REQ. */
3061 security_state =
3062 ffa_get_memory_security_attr(retrieve_request->attributes);
3063 if (security_state != FFA_MEMORY_SECURITY_UNSPECIFIED) {
3064 dlog_verbose(
3065 "Invalid security state for memory retrieve request "
3066 "operation.\n");
3067 return ffa_error(FFA_INVALID_PARAMETERS);
3068 }
3069
J-Alves089004f2022-07-13 14:25:44 +01003070 /*
3071 * If memory type is not specified, bypass validation of memory
3072 * attributes in the retrieve request. The retriever is expecting to
3073 * obtain this information from the SPMC.
3074 */
3075 if (ffa_get_memory_type_attr(retrieve_request->attributes) ==
3076 FFA_MEMORY_NOT_SPECIFIED_MEM) {
3077 return (struct ffa_value){.func = FFA_SUCCESS_32};
3078 }
3079
3080 /*
3081 * Ensure receiver's attributes are compatible with how
3082 * Hafnium maps memory: Normal Memory, Inner shareable,
3083 * Write-Back Read-Allocate Write-Allocate Cacheable.
3084 */
3085 return ffa_memory_attributes_validate(retrieve_request->attributes);
3086}
3087
J-Alves4f0d9c12024-01-17 17:23:11 +00003088static struct ffa_value ffa_partition_retrieve_request(
3089 struct share_states_locked share_states,
3090 struct ffa_memory_share_state *share_state, struct vm_locked to_locked,
3091 struct ffa_memory_region *retrieve_request,
3092 uint32_t retrieve_request_length, struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003093{
J-Alvesa9cd7e32022-07-01 13:49:33 +01003094 ffa_memory_access_permissions_t permissions = 0;
Olivier Deprez878bd5b2021-04-15 19:05:10 +02003095 uint32_t memory_to_mode;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003096 struct ffa_value ret;
Andrew Walbranca808b12020-05-15 17:22:28 +01003097 struct ffa_composite_memory_region *composite;
3098 uint32_t total_length;
3099 uint32_t fragment_length;
J-Alves19e20cf2023-08-02 12:48:55 +01003100 ffa_id_t receiver_id = to_locked.vm->id;
J-Alves4f0d9c12024-01-17 17:23:11 +00003101 bool is_retrieve_complete = false;
J-Alves4f0d9c12024-01-17 17:23:11 +00003102 const uint64_t memory_access_desc_size =
Daniel Boulbyde974ca2023-12-12 13:53:31 +00003103 retrieve_request->memory_access_desc_size;
J-Alves4f0d9c12024-01-17 17:23:11 +00003104 uint32_t receiver_index;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003105 struct ffa_memory_access *receiver;
J-Alves4f0d9c12024-01-17 17:23:11 +00003106 ffa_memory_handle_t handle = retrieve_request->handle;
J-Alves460d36c2023-10-12 17:02:15 +01003107 ffa_memory_attributes_t attributes = 0;
3108 uint32_t retrieve_mode = 0;
J-Alves4f0d9c12024-01-17 17:23:11 +00003109 struct ffa_memory_region *memory_region = share_state->memory_region;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003110
J-Alves96de29f2022-04-26 16:05:24 +01003111 if (!share_state->sending_complete) {
3112 dlog_verbose(
3113 "Memory with handle %#x not fully sent, can't "
3114 "retrieve.\n",
3115 handle);
J-Alves4f0d9c12024-01-17 17:23:11 +00003116 return ffa_error(FFA_INVALID_PARAMETERS);
J-Alves96de29f2022-04-26 16:05:24 +01003117 }
3118
J-Alves4f0d9c12024-01-17 17:23:11 +00003119 /*
3120 * Validate retrieve request, according to what was sent by the
3121 * sender. Function will output the `receiver_index` from the
3122 * provided memory region.
3123 */
3124 ret = ffa_memory_retrieve_validate(
3125 receiver_id, retrieve_request, retrieve_request_length,
3126 memory_region, &receiver_index, share_state->share_func);
J-Alves089004f2022-07-13 14:25:44 +01003127
J-Alves4f0d9c12024-01-17 17:23:11 +00003128 if (ret.func != FFA_SUCCESS_32) {
3129 return ret;
J-Alves089004f2022-07-13 14:25:44 +01003130 }
J-Alves96de29f2022-04-26 16:05:24 +01003131
J-Alves4f0d9c12024-01-17 17:23:11 +00003132 /*
3133 * Validate the requested permissions against the sent
3134 * permissions.
3135 * Outputs the permissions to give to retriever at S2
3136 * PTs.
3137 */
3138 ret = ffa_memory_retrieve_validate_memory_access_list(
3139 memory_region, retrieve_request, receiver_id, &permissions,
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003140 &receiver, share_state->share_func);
J-Alves4f0d9c12024-01-17 17:23:11 +00003141 if (ret.func != FFA_SUCCESS_32) {
3142 return ret;
3143 }
3144
3145 memory_to_mode = ffa_memory_permissions_to_mode(
3146 permissions, share_state->sender_orig_mode);
3147
3148 ret = ffa_retrieve_check_update(
3149 to_locked, share_state->fragments,
3150 share_state->fragment_constituent_counts,
3151 share_state->fragment_count, memory_to_mode,
J-Alves460d36c2023-10-12 17:02:15 +01003152 share_state->share_func, false, page_pool, &retrieve_mode,
3153 share_state->memory_protected);
J-Alves4f0d9c12024-01-17 17:23:11 +00003154
3155 if (ret.func != FFA_SUCCESS_32) {
3156 return ret;
3157 }
3158
3159 share_state->retrieved_fragment_count[receiver_index] = 1;
3160
3161 is_retrieve_complete =
3162 share_state->retrieved_fragment_count[receiver_index] ==
3163 share_state->fragment_count;
3164
J-Alvesb5084cf2022-07-06 14:20:12 +01003165 /* VMs acquire the RX buffer from SPMC. */
3166 CHECK(plat_ffa_acquire_receiver_rx(to_locked, &ret));
3167
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003168 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01003169 * Copy response to RX buffer of caller and deliver the message.
3170 * This must be done before the share_state is (possibly) freed.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003171 */
Andrew Walbranca808b12020-05-15 17:22:28 +01003172 composite = ffa_memory_region_get_composite(memory_region, 0);
J-Alves4f0d9c12024-01-17 17:23:11 +00003173
Andrew Walbranca808b12020-05-15 17:22:28 +01003174 /*
J-Alves460d36c2023-10-12 17:02:15 +01003175 * Set the security state in the memory retrieve response attributes
3176 * if specified by the target mode.
3177 */
3178 attributes = plat_ffa_memory_security_mode(memory_region->attributes,
3179 retrieve_mode);
3180
3181 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01003182 * Constituents which we received in the first fragment should
3183 * always fit in the first fragment we are sending, because the
3184 * header is the same size in both cases and we have a fixed
3185 * message buffer size. So `ffa_retrieved_memory_region_init`
3186 * should never fail.
Andrew Walbranca808b12020-05-15 17:22:28 +01003187 */
Olivier Deprez878bd5b2021-04-15 19:05:10 +02003188
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003189 /* Provide the permissions that had been provided. */
3190 receiver->receiver_permissions.permissions = permissions;
3191
3192 /*
3193 * Prepare the memory region descriptor for the retrieve response.
3194 * Provide the pointer to the receiver tracked in the share state
3195 * strucutures.
3196 */
Andrew Walbranca808b12020-05-15 17:22:28 +01003197 CHECK(ffa_retrieved_memory_region_init(
J-Alves2d8457f2022-10-05 11:06:41 +01003198 to_locked.vm->mailbox.recv, to_locked.vm->ffa_version,
Olivier Deprez878bd5b2021-04-15 19:05:10 +02003199 HF_MAILBOX_SIZE, memory_region->sender, attributes,
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003200 memory_region->flags, handle, permissions, receiver, 1,
3201 memory_access_desc_size, composite->page_count,
3202 composite->constituent_count, share_state->fragments[0],
Andrew Walbranca808b12020-05-15 17:22:28 +01003203 share_state->fragment_constituent_counts[0], &total_length,
3204 &fragment_length));
J-Alvesb5084cf2022-07-06 14:20:12 +01003205
J-Alves4f0d9c12024-01-17 17:23:11 +00003206 if (is_retrieve_complete) {
Andrew Walbranca808b12020-05-15 17:22:28 +01003207 ffa_memory_retrieve_complete(share_states, share_state,
3208 page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003209 }
J-Alves4f0d9c12024-01-17 17:23:11 +00003210
3211 return ffa_memory_retrieve_resp(total_length, fragment_length);
3212}
3213
3214static struct ffa_value ffa_hypervisor_retrieve_request(
3215 struct ffa_memory_share_state *share_state, struct vm_locked to_locked,
3216 struct ffa_memory_region *retrieve_request)
3217{
3218 struct ffa_value ret;
3219 struct ffa_composite_memory_region *composite;
3220 uint32_t total_length;
3221 uint32_t fragment_length;
J-Alves4f0d9c12024-01-17 17:23:11 +00003222 ffa_memory_attributes_t attributes;
J-Alves7b6ab612024-01-24 09:54:54 +00003223 uint64_t memory_access_desc_size;
J-Alves4f0d9c12024-01-17 17:23:11 +00003224 struct ffa_memory_region *memory_region;
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003225 struct ffa_memory_access *receiver;
J-Alves4f0d9c12024-01-17 17:23:11 +00003226 ffa_memory_handle_t handle = retrieve_request->handle;
3227
J-Alves4f0d9c12024-01-17 17:23:11 +00003228 memory_region = share_state->memory_region;
3229
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003230 assert(to_locked.vm->id == HF_HYPERVISOR_VM_ID);
3231
J-Alves7b6ab612024-01-24 09:54:54 +00003232 switch (to_locked.vm->ffa_version) {
3233 case MAKE_FFA_VERSION(1, 2):
3234 memory_access_desc_size = sizeof(struct ffa_memory_access);
3235 break;
3236 case MAKE_FFA_VERSION(1, 0):
3237 case MAKE_FFA_VERSION(1, 1):
3238 memory_access_desc_size = sizeof(struct ffa_memory_access_v1_0);
3239 break;
3240 default:
3241 panic("version not supported: %x\n", to_locked.vm->ffa_version);
3242 }
3243
J-Alves4f0d9c12024-01-17 17:23:11 +00003244 if (share_state->hypervisor_fragment_count != 0U) {
3245 dlog_verbose(
3246 "Memory with handle %#x already retrieved by "
3247 "the hypervisor.\n",
3248 handle);
3249 return ffa_error(FFA_DENIED);
3250 }
3251
3252 share_state->hypervisor_fragment_count = 1;
3253
3254 ffa_memory_retrieve_complete_from_hyp(share_state);
3255
3256 /* VMs acquire the RX buffer from SPMC. */
3257 CHECK(plat_ffa_acquire_receiver_rx(to_locked, &ret));
3258
3259 /*
3260 * Copy response to RX buffer of caller and deliver the message.
3261 * This must be done before the share_state is (possibly) freed.
3262 */
3263 composite = ffa_memory_region_get_composite(memory_region, 0);
3264
3265 /*
3266 * Constituents which we received in the first fragment should
3267 * always fit in the first fragment we are sending, because the
3268 * header is the same size in both cases and we have a fixed
3269 * message buffer size. So `ffa_retrieved_memory_region_init`
3270 * should never fail.
3271 */
3272
3273 /*
3274 * Set the security state in the memory retrieve response attributes
3275 * if specified by the target mode.
3276 */
3277 attributes = plat_ffa_memory_security_mode(
3278 memory_region->attributes, share_state->sender_orig_mode);
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003279
3280 receiver = ffa_memory_region_get_receiver(memory_region, 0);
3281
J-Alves4f0d9c12024-01-17 17:23:11 +00003282 CHECK(ffa_retrieved_memory_region_init(
3283 to_locked.vm->mailbox.recv, to_locked.vm->ffa_version,
3284 HF_MAILBOX_SIZE, memory_region->sender, attributes,
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003285 memory_region->flags, handle,
3286 receiver->receiver_permissions.permissions, receiver,
3287 memory_region->receiver_count, memory_access_desc_size,
J-Alves4f0d9c12024-01-17 17:23:11 +00003288 composite->page_count, composite->constituent_count,
3289 share_state->fragments[0],
3290 share_state->fragment_constituent_counts[0], &total_length,
3291 &fragment_length));
3292
3293 return ffa_memory_retrieve_resp(total_length, fragment_length);
3294}
3295
3296struct ffa_value ffa_memory_retrieve(struct vm_locked to_locked,
3297 struct ffa_memory_region *retrieve_request,
3298 uint32_t retrieve_request_length,
3299 struct mpool *page_pool)
3300{
3301 ffa_memory_handle_t handle = retrieve_request->handle;
3302 struct share_states_locked share_states;
3303 struct ffa_memory_share_state *share_state;
3304 struct ffa_value ret;
3305
3306 dump_share_states();
3307
3308 share_states = share_states_lock();
3309 share_state = get_share_state(share_states, handle);
3310 if (share_state == NULL) {
3311 dlog_verbose("Invalid handle %#x for FFA_MEM_RETRIEVE_REQ.\n",
3312 handle);
3313 ret = ffa_error(FFA_INVALID_PARAMETERS);
3314 goto out;
3315 }
3316
3317 if (is_ffa_hypervisor_retrieve_request(retrieve_request, to_locked)) {
3318 ret = ffa_hypervisor_retrieve_request(share_state, to_locked,
3319 retrieve_request);
3320 } else {
3321 ret = ffa_partition_retrieve_request(
3322 share_states, share_state, to_locked, retrieve_request,
3323 retrieve_request_length, page_pool);
3324 }
3325
3326 /* Track use of the RX buffer if the handling has succeeded. */
3327 if (ret.func == FFA_MEM_RETRIEVE_RESP_32) {
3328 to_locked.vm->mailbox.recv_func = FFA_MEM_RETRIEVE_RESP_32;
3329 to_locked.vm->mailbox.state = MAILBOX_STATE_FULL;
3330 }
3331
Andrew Walbranca808b12020-05-15 17:22:28 +01003332out:
3333 share_states_unlock(&share_states);
3334 dump_share_states();
3335 return ret;
3336}
3337
J-Alves5da37d92022-10-24 16:33:48 +01003338/**
3339 * Determine expected fragment offset according to the FF-A version of
3340 * the caller.
3341 */
3342static uint32_t ffa_memory_retrieve_expected_offset_per_ffa_version(
3343 struct ffa_memory_region *memory_region,
3344 uint32_t retrieved_constituents_count, uint32_t ffa_version)
3345{
3346 uint32_t expected_fragment_offset;
3347 uint32_t composite_constituents_offset;
3348
Kathleen Capellae4fe2962023-09-01 17:08:47 -04003349 if (ffa_version >= MAKE_FFA_VERSION(1, 1)) {
J-Alves5da37d92022-10-24 16:33:48 +01003350 /*
3351 * Hafnium operates memory regions in FF-A v1.1 format, so we
3352 * can retrieve the constituents offset from descriptor.
3353 */
3354 composite_constituents_offset =
3355 ffa_composite_constituent_offset(memory_region, 0);
3356 } else if (ffa_version == MAKE_FFA_VERSION(1, 0)) {
3357 /*
3358 * If retriever is FF-A v1.0, determine the composite offset
3359 * as it is expected to have been configured in the
3360 * retrieve response.
3361 */
3362 composite_constituents_offset =
3363 sizeof(struct ffa_memory_region_v1_0) +
3364 RECEIVERS_COUNT_IN_RETRIEVE_RESP *
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003365 sizeof(struct ffa_memory_access_v1_0) +
J-Alves5da37d92022-10-24 16:33:48 +01003366 sizeof(struct ffa_composite_memory_region);
3367 } else {
3368 panic("%s received an invalid FF-A version.\n", __func__);
3369 }
3370
3371 expected_fragment_offset =
3372 composite_constituents_offset +
3373 retrieved_constituents_count *
3374 sizeof(struct ffa_memory_region_constituent) -
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003375 (uint32_t)(memory_region->memory_access_desc_size *
3376 (memory_region->receiver_count - 1));
J-Alves5da37d92022-10-24 16:33:48 +01003377
3378 return expected_fragment_offset;
3379}
3380
Andrew Walbranca808b12020-05-15 17:22:28 +01003381struct ffa_value ffa_memory_retrieve_continue(struct vm_locked to_locked,
3382 ffa_memory_handle_t handle,
3383 uint32_t fragment_offset,
J-Alves19e20cf2023-08-02 12:48:55 +01003384 ffa_id_t sender_vm_id,
Andrew Walbranca808b12020-05-15 17:22:28 +01003385 struct mpool *page_pool)
3386{
3387 struct ffa_memory_region *memory_region;
3388 struct share_states_locked share_states;
3389 struct ffa_memory_share_state *share_state;
3390 struct ffa_value ret;
3391 uint32_t fragment_index;
3392 uint32_t retrieved_constituents_count;
3393 uint32_t i;
3394 uint32_t expected_fragment_offset;
3395 uint32_t remaining_constituent_count;
3396 uint32_t fragment_length;
J-Alvesc7484f12022-05-13 12:41:14 +01003397 uint32_t receiver_index;
J-Alves59ed0042022-07-28 18:26:41 +01003398 bool continue_ffa_hyp_mem_retrieve_req;
Andrew Walbranca808b12020-05-15 17:22:28 +01003399
3400 dump_share_states();
3401
3402 share_states = share_states_lock();
Karl Meakin4a2854a2023-06-30 16:26:52 +01003403 share_state = get_share_state(share_states, handle);
J-Alvesb56aac82023-11-10 09:44:43 +00003404 if (share_state == NULL) {
Andrew Walbranca808b12020-05-15 17:22:28 +01003405 dlog_verbose("Invalid handle %#x for FFA_MEM_FRAG_RX.\n",
3406 handle);
3407 ret = ffa_error(FFA_INVALID_PARAMETERS);
3408 goto out;
3409 }
3410
3411 memory_region = share_state->memory_region;
3412 CHECK(memory_region != NULL);
3413
Andrew Walbranca808b12020-05-15 17:22:28 +01003414 if (!share_state->sending_complete) {
3415 dlog_verbose(
3416 "Memory with handle %#x not fully sent, can't "
3417 "retrieve.\n",
3418 handle);
3419 ret = ffa_error(FFA_INVALID_PARAMETERS);
3420 goto out;
3421 }
3422
J-Alves59ed0042022-07-28 18:26:41 +01003423 /*
3424 * If retrieve request from the hypervisor has been initiated in the
3425 * given share_state, continue it, else assume it is a continuation of
3426 * retrieve request from a NWd VM.
3427 */
3428 continue_ffa_hyp_mem_retrieve_req =
3429 (to_locked.vm->id == HF_HYPERVISOR_VM_ID) &&
3430 (share_state->hypervisor_fragment_count != 0U) &&
J-Alves661e1b72023-08-02 13:39:40 +01003431 ffa_is_vm_id(sender_vm_id);
Andrew Walbranca808b12020-05-15 17:22:28 +01003432
J-Alves59ed0042022-07-28 18:26:41 +01003433 if (!continue_ffa_hyp_mem_retrieve_req) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003434 receiver_index = ffa_memory_region_get_receiver_index(
J-Alves59ed0042022-07-28 18:26:41 +01003435 memory_region, to_locked.vm->id);
3436
3437 if (receiver_index == memory_region->receiver_count) {
3438 dlog_verbose(
3439 "Caller of FFA_MEM_FRAG_RX (%x) is not a "
3440 "borrower to memory sharing transaction (%x)\n",
3441 to_locked.vm->id, handle);
3442 ret = ffa_error(FFA_INVALID_PARAMETERS);
3443 goto out;
3444 }
3445
3446 if (share_state->retrieved_fragment_count[receiver_index] ==
3447 0 ||
3448 share_state->retrieved_fragment_count[receiver_index] >=
3449 share_state->fragment_count) {
3450 dlog_verbose(
3451 "Retrieval of memory with handle %#x not yet "
3452 "started or already completed (%d/%d fragments "
3453 "retrieved).\n",
3454 handle,
3455 share_state->retrieved_fragment_count
3456 [receiver_index],
3457 share_state->fragment_count);
3458 ret = ffa_error(FFA_INVALID_PARAMETERS);
3459 goto out;
3460 }
3461
3462 fragment_index =
3463 share_state->retrieved_fragment_count[receiver_index];
3464 } else {
3465 if (share_state->hypervisor_fragment_count == 0 ||
3466 share_state->hypervisor_fragment_count >=
3467 share_state->fragment_count) {
3468 dlog_verbose(
3469 "Retrieve of memory with handle %x not "
3470 "started from hypervisor.\n",
3471 handle);
3472 ret = ffa_error(FFA_INVALID_PARAMETERS);
3473 goto out;
3474 }
3475
3476 if (memory_region->sender != sender_vm_id) {
3477 dlog_verbose(
3478 "Sender ID (%x) is not as expected for memory "
3479 "handle %x\n",
3480 sender_vm_id, handle);
3481 ret = ffa_error(FFA_INVALID_PARAMETERS);
3482 goto out;
3483 }
3484
3485 fragment_index = share_state->hypervisor_fragment_count;
3486
3487 receiver_index = 0;
3488 }
Andrew Walbranca808b12020-05-15 17:22:28 +01003489
3490 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01003491 * Check that the given fragment offset is correct by counting
3492 * how many constituents were in the fragments previously sent.
Andrew Walbranca808b12020-05-15 17:22:28 +01003493 */
3494 retrieved_constituents_count = 0;
3495 for (i = 0; i < fragment_index; ++i) {
3496 retrieved_constituents_count +=
3497 share_state->fragment_constituent_counts[i];
3498 }
J-Alvesc7484f12022-05-13 12:41:14 +01003499
3500 CHECK(memory_region->receiver_count > 0);
3501
Andrew Walbranca808b12020-05-15 17:22:28 +01003502 expected_fragment_offset =
J-Alves5da37d92022-10-24 16:33:48 +01003503 ffa_memory_retrieve_expected_offset_per_ffa_version(
3504 memory_region, retrieved_constituents_count,
3505 to_locked.vm->ffa_version);
3506
Andrew Walbranca808b12020-05-15 17:22:28 +01003507 if (fragment_offset != expected_fragment_offset) {
3508 dlog_verbose("Fragment offset was %d but expected %d.\n",
3509 fragment_offset, expected_fragment_offset);
3510 ret = ffa_error(FFA_INVALID_PARAMETERS);
3511 goto out;
3512 }
3513
J-Alves4f0d9c12024-01-17 17:23:11 +00003514 /*
3515 * When hafnium is the hypervisor, acquire the RX buffer of a VM, that
3516 * is currently ownder by the SPMC.
3517 */
3518 assert(plat_ffa_acquire_receiver_rx(to_locked, &ret));
J-Alves59ed0042022-07-28 18:26:41 +01003519
Andrew Walbranca808b12020-05-15 17:22:28 +01003520 remaining_constituent_count = ffa_memory_fragment_init(
3521 to_locked.vm->mailbox.recv, HF_MAILBOX_SIZE,
3522 share_state->fragments[fragment_index],
3523 share_state->fragment_constituent_counts[fragment_index],
3524 &fragment_length);
3525 CHECK(remaining_constituent_count == 0);
J-Alves674e4de2024-01-17 16:20:32 +00003526
Andrew Walbranca808b12020-05-15 17:22:28 +01003527 to_locked.vm->mailbox.recv_func = FFA_MEM_FRAG_TX_32;
J-Alvese8c8c2b2022-12-16 15:34:48 +00003528 to_locked.vm->mailbox.state = MAILBOX_STATE_FULL;
Andrew Walbranca808b12020-05-15 17:22:28 +01003529
J-Alves59ed0042022-07-28 18:26:41 +01003530 if (!continue_ffa_hyp_mem_retrieve_req) {
3531 share_state->retrieved_fragment_count[receiver_index]++;
3532 if (share_state->retrieved_fragment_count[receiver_index] ==
3533 share_state->fragment_count) {
3534 ffa_memory_retrieve_complete(share_states, share_state,
3535 page_pool);
3536 }
3537 } else {
3538 share_state->hypervisor_fragment_count++;
3539
3540 ffa_memory_retrieve_complete_from_hyp(share_state);
3541 }
Andrew Walbranca808b12020-05-15 17:22:28 +01003542 ret = (struct ffa_value){.func = FFA_MEM_FRAG_TX_32,
3543 .arg1 = (uint32_t)handle,
3544 .arg2 = (uint32_t)(handle >> 32),
3545 .arg3 = fragment_length};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003546
3547out:
3548 share_states_unlock(&share_states);
3549 dump_share_states();
3550 return ret;
3551}
3552
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003553struct ffa_value ffa_memory_relinquish(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003554 struct vm_locked from_locked,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003555 struct ffa_mem_relinquish *relinquish_request, struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003556{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003557 ffa_memory_handle_t handle = relinquish_request->handle;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003558 struct share_states_locked share_states;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003559 struct ffa_memory_share_state *share_state;
3560 struct ffa_memory_region *memory_region;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003561 bool clear;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003562 struct ffa_value ret;
J-Alves8eb19162022-04-28 10:56:48 +01003563 uint32_t receiver_index;
J-Alves3c5b2072022-11-21 12:45:40 +00003564 bool receivers_relinquished_memory;
J-Alves639ddfc2023-11-21 14:17:26 +00003565 ffa_memory_access_permissions_t receiver_permissions = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003566
Andrew Walbrana65a1322020-04-06 19:32:32 +01003567 if (relinquish_request->endpoint_count != 1) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003568 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01003569 "Stream endpoints not supported (got %d "
J-Alves668a86e2023-05-10 11:53:25 +01003570 "endpoints on FFA_MEM_RELINQUISH, expected 1).\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003571 relinquish_request->endpoint_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003572 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003573 }
3574
Andrew Walbrana65a1322020-04-06 19:32:32 +01003575 if (relinquish_request->endpoints[0] != from_locked.vm->id) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003576 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01003577 "VM ID %d in relinquish message doesn't match "
J-Alves668a86e2023-05-10 11:53:25 +01003578 "calling VM ID %d.\n",
Andrew Walbrana65a1322020-04-06 19:32:32 +01003579 relinquish_request->endpoints[0], from_locked.vm->id);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003580 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003581 }
3582
3583 dump_share_states();
3584
3585 share_states = share_states_lock();
Karl Meakin4a2854a2023-06-30 16:26:52 +01003586 share_state = get_share_state(share_states, handle);
J-Alvesb56aac82023-11-10 09:44:43 +00003587 if (share_state == NULL) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003588 dlog_verbose("Invalid handle %#x for FFA_MEM_RELINQUISH.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003589 handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003590 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003591 goto out;
3592 }
3593
Andrew Walbranca808b12020-05-15 17:22:28 +01003594 if (!share_state->sending_complete) {
3595 dlog_verbose(
3596 "Memory with handle %#x not fully sent, can't "
3597 "relinquish.\n",
3598 handle);
3599 ret = ffa_error(FFA_INVALID_PARAMETERS);
3600 goto out;
3601 }
3602
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003603 memory_region = share_state->memory_region;
3604 CHECK(memory_region != NULL);
3605
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003606 receiver_index = ffa_memory_region_get_receiver_index(
3607 memory_region, from_locked.vm->id);
J-Alves8eb19162022-04-28 10:56:48 +01003608
3609 if (receiver_index == memory_region->receiver_count) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003610 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01003611 "VM ID %d tried to relinquish memory region "
J-Alves668a86e2023-05-10 11:53:25 +01003612 "with handle %#x and it is not a valid borrower.\n",
J-Alves8eb19162022-04-28 10:56:48 +01003613 from_locked.vm->id, handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003614 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003615 goto out;
3616 }
3617
J-Alves8eb19162022-04-28 10:56:48 +01003618 if (share_state->retrieved_fragment_count[receiver_index] !=
Andrew Walbranca808b12020-05-15 17:22:28 +01003619 share_state->fragment_count) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003620 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01003621 "Memory with handle %#x not yet fully "
3622 "retrieved, "
J-Alves8eb19162022-04-28 10:56:48 +01003623 "receiver %x can't relinquish.\n",
3624 handle, from_locked.vm->id);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003625 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003626 goto out;
3627 }
3628
J-Alves3c5b2072022-11-21 12:45:40 +00003629 /*
3630 * Either clear if requested in relinquish call, or in a retrieve
3631 * request from one of the borrowers.
3632 */
3633 receivers_relinquished_memory = true;
3634
3635 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
3636 struct ffa_memory_access *receiver =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003637 ffa_memory_region_get_receiver(memory_region, i);
3638 assert(receiver != NULL);
J-Alves3c5b2072022-11-21 12:45:40 +00003639 if (receiver->receiver_permissions.receiver ==
3640 from_locked.vm->id) {
J-Alves639ddfc2023-11-21 14:17:26 +00003641 receiver_permissions =
3642 receiver->receiver_permissions.permissions;
J-Alves3c5b2072022-11-21 12:45:40 +00003643 continue;
3644 }
3645
3646 if (share_state->retrieved_fragment_count[i] != 0U) {
3647 receivers_relinquished_memory = false;
3648 break;
3649 }
3650 }
3651
3652 clear = receivers_relinquished_memory &&
Daniel Boulby2e14ebe2024-01-15 16:21:44 +00003653 ((relinquish_request->flags & FFA_MEMORY_REGION_FLAG_CLEAR) !=
3654 0U);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003655
3656 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01003657 * Clear is not allowed for memory that was shared, as the
3658 * original sender still has access to the memory.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003659 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003660 if (clear && share_state->share_func == FFA_MEM_SHARE_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003661 dlog_verbose("Memory which was shared can't be cleared.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003662 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003663 goto out;
3664 }
3665
J-Alves639ddfc2023-11-21 14:17:26 +00003666 if (clear && receiver_permissions == FFA_DATA_ACCESS_RO) {
3667 dlog_verbose("%s: RO memory can't use clear memory flag.\n",
3668 __func__);
3669 ret = ffa_error(FFA_DENIED);
3670 goto out;
3671 }
3672
Andrew Walbranca808b12020-05-15 17:22:28 +01003673 ret = ffa_relinquish_check_update(
J-Alves26483382023-04-20 12:01:49 +01003674 from_locked, share_state->fragments,
Andrew Walbranca808b12020-05-15 17:22:28 +01003675 share_state->fragment_constituent_counts,
3676 share_state->fragment_count, page_pool, clear);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003677
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003678 if (ret.func == FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003679 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01003680 * Mark memory handle as not retrieved, so it can be
3681 * reclaimed (or retrieved again).
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003682 */
J-Alves8eb19162022-04-28 10:56:48 +01003683 share_state->retrieved_fragment_count[receiver_index] = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003684 }
3685
3686out:
3687 share_states_unlock(&share_states);
3688 dump_share_states();
3689 return ret;
3690}
3691
3692/**
J-Alvesa9cd7e32022-07-01 13:49:33 +01003693 * Validates that the reclaim transition is allowed for the given
3694 * handle, updates the page table of the reclaiming VM, and frees the
3695 * internal state associated with the handle.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003696 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003697struct ffa_value ffa_memory_reclaim(struct vm_locked to_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01003698 ffa_memory_handle_t handle,
3699 ffa_memory_region_flags_t flags,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003700 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003701{
3702 struct share_states_locked share_states;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003703 struct ffa_memory_share_state *share_state;
3704 struct ffa_memory_region *memory_region;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003705 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003706
3707 dump_share_states();
3708
3709 share_states = share_states_lock();
Karl Meakin52cdfe72023-06-30 14:49:10 +01003710
Karl Meakin4a2854a2023-06-30 16:26:52 +01003711 share_state = get_share_state(share_states, handle);
J-Alvesb56aac82023-11-10 09:44:43 +00003712 if (share_state == NULL) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003713 dlog_verbose("Invalid handle %#x for FFA_MEM_RECLAIM.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003714 handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003715 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003716 goto out;
3717 }
Karl Meakin4a2854a2023-06-30 16:26:52 +01003718 memory_region = share_state->memory_region;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003719
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003720 CHECK(memory_region != NULL);
3721
J-Alvesa9cd7e32022-07-01 13:49:33 +01003722 if (vm_id_is_current_world(to_locked.vm->id) &&
3723 to_locked.vm->id != memory_region->sender) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003724 dlog_verbose(
Olivier Deprezf92e5d42020-11-13 16:00:54 +01003725 "VM %#x attempted to reclaim memory handle %#x "
3726 "originally sent by VM %#x.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003727 to_locked.vm->id, handle, memory_region->sender);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003728 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003729 goto out;
3730 }
3731
Andrew Walbranca808b12020-05-15 17:22:28 +01003732 if (!share_state->sending_complete) {
3733 dlog_verbose(
3734 "Memory with handle %#x not fully sent, can't "
3735 "reclaim.\n",
3736 handle);
3737 ret = ffa_error(FFA_INVALID_PARAMETERS);
3738 goto out;
3739 }
3740
J-Alves752236c2022-04-28 11:07:47 +01003741 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
3742 if (share_state->retrieved_fragment_count[i] != 0) {
3743 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01003744 "Tried to reclaim memory handle %#x "
J-Alves3c5b2072022-11-21 12:45:40 +00003745 "that has not been relinquished by all "
J-Alvesa9cd7e32022-07-01 13:49:33 +01003746 "borrowers(%x).\n",
J-Alves752236c2022-04-28 11:07:47 +01003747 handle,
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003748 ffa_memory_region_get_receiver(memory_region, i)
3749 ->receiver_permissions.receiver);
J-Alves752236c2022-04-28 11:07:47 +01003750 ret = ffa_error(FFA_DENIED);
3751 goto out;
3752 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003753 }
3754
Andrew Walbranca808b12020-05-15 17:22:28 +01003755 ret = ffa_retrieve_check_update(
J-Alves26483382023-04-20 12:01:49 +01003756 to_locked, share_state->fragments,
Andrew Walbranca808b12020-05-15 17:22:28 +01003757 share_state->fragment_constituent_counts,
J-Alves2a0d2882020-10-29 14:49:50 +00003758 share_state->fragment_count, share_state->sender_orig_mode,
J-Alves460d36c2023-10-12 17:02:15 +01003759 FFA_MEM_RECLAIM_32, flags & FFA_MEM_RECLAIM_CLEAR, page_pool,
J-Alvesfd206052023-05-22 16:45:00 +01003760 NULL, share_state->memory_protected);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003761
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003762 if (ret.func == FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003763 share_state_free(share_states, share_state, page_pool);
J-Alves3c5b2072022-11-21 12:45:40 +00003764 dlog_verbose("Freed share state after successful reclaim.\n");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003765 }
3766
3767out:
3768 share_states_unlock(&share_states);
3769 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01003770}