blob: 546d18842e9c7ef0ea6b99a94a6d125163dd1339 [file] [log] [blame]
Jose Marinho75509b42019-04-09 09:34:59 +01001/*
2 * Copyright 2019 The Hafnium Authors.
3 *
Andrew Walbrane959ec12020-06-17 15:01:09 +01004 * Use of this source code is governed by a BSD-style
5 * license that can be found in the LICENSE file or at
6 * https://opensource.org/licenses/BSD-3-Clause.
Jose Marinho75509b42019-04-09 09:34:59 +01007 */
8
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01009#include "hf/ffa_memory.h"
Andrew Walbran475c1452020-02-07 13:22:22 +000010
Federico Recanati4fd065d2021-12-13 20:06:23 +010011#include "hf/arch/mm.h"
Olivier Deprez112d2b52020-09-30 07:39:23 +020012#include "hf/arch/other_world.h"
Olivier Deprez55a189e2021-06-09 15:45:27 +020013#include "hf/arch/plat/ffa.h"
Andrew Walbran290b0c92020-02-03 16:37:14 +000014
J-Alves5952d942022-12-22 16:03:00 +000015#include "hf/addr.h"
Jose Marinho75509b42019-04-09 09:34:59 +010016#include "hf/api.h"
Daniel Boulbya2f8c662021-11-26 17:52:53 +000017#include "hf/assert.h"
Jose Marinho09b1db82019-08-08 09:16:59 +010018#include "hf/check.h"
Jose Marinho75509b42019-04-09 09:34:59 +010019#include "hf/dlog.h"
J-Alves3456e032023-07-20 12:20:05 +010020#include "hf/ffa.h"
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010021#include "hf/ffa_internal.h"
J-Alves66652252022-07-06 09:49:51 +010022#include "hf/ffa_memory_internal.h"
J-Alves3456e032023-07-20 12:20:05 +010023#include "hf/ffa_partition_manifest.h"
J-Alves5952d942022-12-22 16:03:00 +000024#include "hf/mm.h"
Andrew Walbran475c1452020-02-07 13:22:22 +000025#include "hf/mpool.h"
Jose Marinho75509b42019-04-09 09:34:59 +010026#include "hf/std.h"
Andrew Scull3c257452019-11-26 13:32:50 +000027#include "hf/vm.h"
Jose Marinho75509b42019-04-09 09:34:59 +010028
J-Alves2d8457f2022-10-05 11:06:41 +010029#include "vmapi/hf/ffa_v1_0.h"
30
J-Alves5da37d92022-10-24 16:33:48 +010031#define RECEIVERS_COUNT_IN_RETRIEVE_RESP 1
32
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000033/**
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010034 * All access to members of a `struct ffa_memory_share_state` must be guarded
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000035 * by this lock.
36 */
37static struct spinlock share_states_lock_instance = SPINLOCK_INIT;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010038static struct ffa_memory_share_state share_states[MAX_MEM_SHARES];
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000039
40/**
J-Alvesed508c82023-05-04 16:09:48 +010041 * Return the offset to the first constituent within the
42 * `ffa_composite_memory_region` for the given receiver from an
43 * `ffa_memory_region`. The caller must check that the receiver_index is within
44 * bounds, and that it has a composite memory region offset.
45 */
46static uint32_t ffa_composite_constituent_offset(
47 struct ffa_memory_region *memory_region, uint32_t receiver_index)
48{
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +000049 struct ffa_memory_access *receiver;
50 uint32_t composite_offset;
J-Alvesed508c82023-05-04 16:09:48 +010051
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +000052 CHECK(receiver_index < memory_region->receiver_count);
53
54 receiver =
55 ffa_memory_region_get_receiver(memory_region, receiver_index);
56 CHECK(receiver != NULL);
57
58 composite_offset = receiver->composite_memory_region_offset;
59
60 CHECK(composite_offset != 0);
61
62 return composite_offset + sizeof(struct ffa_composite_memory_region);
J-Alvesed508c82023-05-04 16:09:48 +010063}
64
65/**
J-Alves917d2f22020-10-30 18:39:30 +000066 * Extracts the index from a memory handle allocated by Hafnium's current world.
67 */
68uint64_t ffa_memory_handle_get_index(ffa_memory_handle_t handle)
69{
70 return handle & ~FFA_MEMORY_HANDLE_ALLOCATOR_MASK;
71}
72
73/**
Karl Meakin52cdfe72023-06-30 14:49:10 +010074 * Initialises the next available `struct ffa_memory_share_state`. If `handle`
75 * is `FFA_MEMORY_HANDLE_INVALID` then allocates an appropriate handle,
76 * otherwise uses the provided handle which is assumed to be globally unique.
Andrew Walbranca808b12020-05-15 17:22:28 +010077 *
Karl Meakin52cdfe72023-06-30 14:49:10 +010078 * Returns a pointer to the allocated `ffa_memory_share_state` on success or
79 * `NULL` if none are available.
Andrew Walbranca808b12020-05-15 17:22:28 +010080 */
Karl Meakin52cdfe72023-06-30 14:49:10 +010081struct ffa_memory_share_state *allocate_share_state(
82 struct share_states_locked share_states, uint32_t share_func,
83 struct ffa_memory_region *memory_region, uint32_t fragment_length,
84 ffa_memory_handle_t handle)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000085{
Daniel Boulbya2f8c662021-11-26 17:52:53 +000086 assert(share_states.share_states != NULL);
87 assert(memory_region != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000088
Karl Meakin52cdfe72023-06-30 14:49:10 +010089 for (uint64_t i = 0; i < MAX_MEM_SHARES; ++i) {
Andrew Walbranca808b12020-05-15 17:22:28 +010090 if (share_states.share_states[i].share_func == 0) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010091 struct ffa_memory_share_state *allocated_state =
Andrew Walbranca808b12020-05-15 17:22:28 +010092 &share_states.share_states[i];
93 struct ffa_composite_memory_region *composite =
94 ffa_memory_region_get_composite(memory_region,
95 0);
96
97 if (handle == FFA_MEMORY_HANDLE_INVALID) {
J-Alvesee68c542020-10-29 17:48:20 +000098 memory_region->handle =
Olivier Deprez55a189e2021-06-09 15:45:27 +020099 plat_ffa_memory_handle_make(i);
Andrew Walbranca808b12020-05-15 17:22:28 +0100100 } else {
J-Alvesee68c542020-10-29 17:48:20 +0000101 memory_region->handle = handle;
Andrew Walbranca808b12020-05-15 17:22:28 +0100102 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000103 allocated_state->share_func = share_func;
104 allocated_state->memory_region = memory_region;
Andrew Walbranca808b12020-05-15 17:22:28 +0100105 allocated_state->fragment_count = 1;
106 allocated_state->fragments[0] = composite->constituents;
107 allocated_state->fragment_constituent_counts[0] =
108 (fragment_length -
109 ffa_composite_constituent_offset(memory_region,
110 0)) /
111 sizeof(struct ffa_memory_region_constituent);
112 allocated_state->sending_complete = false;
Karl Meakin52cdfe72023-06-30 14:49:10 +0100113 for (uint32_t j = 0; j < MAX_MEM_SHARE_RECIPIENTS;
114 ++j) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100115 allocated_state->retrieved_fragment_count[j] =
116 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000117 }
Karl Meakin52cdfe72023-06-30 14:49:10 +0100118 return allocated_state;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000119 }
120 }
121
Karl Meakin52cdfe72023-06-30 14:49:10 +0100122 return NULL;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000123}
124
125/** Locks the share states lock. */
126struct share_states_locked share_states_lock(void)
127{
128 sl_lock(&share_states_lock_instance);
129
130 return (struct share_states_locked){.share_states = share_states};
131}
132
133/** Unlocks the share states lock. */
J-Alves66652252022-07-06 09:49:51 +0100134void share_states_unlock(struct share_states_locked *share_states)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000135{
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000136 assert(share_states->share_states != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000137 share_states->share_states = NULL;
138 sl_unlock(&share_states_lock_instance);
139}
140
141/**
Andrew Walbranca808b12020-05-15 17:22:28 +0100142 * If the given handle is a valid handle for an allocated share state then
Karl Meakin4a2854a2023-06-30 16:26:52 +0100143 * returns a pointer to the share state. Otherwise returns NULL.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000144 */
Karl Meakin4a2854a2023-06-30 16:26:52 +0100145struct ffa_memory_share_state *get_share_state(
146 struct share_states_locked share_states, ffa_memory_handle_t handle)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000147{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100148 struct ffa_memory_share_state *share_state;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000149
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000150 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100151
152 /*
153 * First look for a share_state allocated by us, in which case the
154 * handle is based on the index.
155 */
Olivier Deprez55a189e2021-06-09 15:45:27 +0200156 if (plat_ffa_memory_handle_allocated_by_current_world(handle)) {
Karl Meakin4a2854a2023-06-30 16:26:52 +0100157 uint64_t index = ffa_memory_handle_get_index(handle);
158
Andrew Walbranca808b12020-05-15 17:22:28 +0100159 if (index < MAX_MEM_SHARES) {
160 share_state = &share_states.share_states[index];
161 if (share_state->share_func != 0) {
Karl Meakin4a2854a2023-06-30 16:26:52 +0100162 return share_state;
Andrew Walbranca808b12020-05-15 17:22:28 +0100163 }
164 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000165 }
166
Andrew Walbranca808b12020-05-15 17:22:28 +0100167 /* Fall back to a linear scan. */
Karl Meakin4a2854a2023-06-30 16:26:52 +0100168 for (uint64_t index = 0; index < MAX_MEM_SHARES; ++index) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100169 share_state = &share_states.share_states[index];
J-Alvesee68c542020-10-29 17:48:20 +0000170 if (share_state->memory_region != NULL &&
171 share_state->memory_region->handle == handle &&
Andrew Walbranca808b12020-05-15 17:22:28 +0100172 share_state->share_func != 0) {
Karl Meakin4a2854a2023-06-30 16:26:52 +0100173 return share_state;
Andrew Walbranca808b12020-05-15 17:22:28 +0100174 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000175 }
176
Karl Meakin4a2854a2023-06-30 16:26:52 +0100177 return NULL;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000178}
179
180/** Marks a share state as unallocated. */
J-Alvesfdd29272022-07-19 13:16:31 +0100181void share_state_free(struct share_states_locked share_states,
182 struct ffa_memory_share_state *share_state,
183 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000184{
Andrew Walbranca808b12020-05-15 17:22:28 +0100185 uint32_t i;
186
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000187 assert(share_states.share_states != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000188 share_state->share_func = 0;
Andrew Walbranca808b12020-05-15 17:22:28 +0100189 share_state->sending_complete = false;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000190 mpool_free(page_pool, share_state->memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +0100191 /*
192 * First fragment is part of the same page as the `memory_region`, so it
193 * doesn't need to be freed separately.
194 */
195 share_state->fragments[0] = NULL;
196 share_state->fragment_constituent_counts[0] = 0;
197 for (i = 1; i < share_state->fragment_count; ++i) {
198 mpool_free(page_pool, share_state->fragments[i]);
199 share_state->fragments[i] = NULL;
200 share_state->fragment_constituent_counts[i] = 0;
201 }
202 share_state->fragment_count = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000203 share_state->memory_region = NULL;
J-Alvesa9cd7e32022-07-01 13:49:33 +0100204 share_state->hypervisor_fragment_count = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000205}
206
Andrew Walbranca808b12020-05-15 17:22:28 +0100207/** Checks whether the given share state has been fully sent. */
J-Alvesfdd29272022-07-19 13:16:31 +0100208bool share_state_sending_complete(struct share_states_locked share_states,
209 struct ffa_memory_share_state *share_state)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000210{
Andrew Walbranca808b12020-05-15 17:22:28 +0100211 struct ffa_composite_memory_region *composite;
212 uint32_t expected_constituent_count;
213 uint32_t fragment_constituent_count_total = 0;
214 uint32_t i;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000215
Andrew Walbranca808b12020-05-15 17:22:28 +0100216 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000217 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100218
219 /*
220 * Share state must already be valid, or it's not possible to get hold
221 * of it.
222 */
223 CHECK(share_state->memory_region != NULL &&
224 share_state->share_func != 0);
225
226 composite =
227 ffa_memory_region_get_composite(share_state->memory_region, 0);
228 expected_constituent_count = composite->constituent_count;
229 for (i = 0; i < share_state->fragment_count; ++i) {
230 fragment_constituent_count_total +=
231 share_state->fragment_constituent_counts[i];
232 }
233 dlog_verbose(
234 "Checking completion: constituent count %d/%d from %d "
235 "fragments.\n",
236 fragment_constituent_count_total, expected_constituent_count,
237 share_state->fragment_count);
238
239 return fragment_constituent_count_total == expected_constituent_count;
240}
241
242/**
243 * Calculates the offset of the next fragment expected for the given share
244 * state.
245 */
J-Alvesfdd29272022-07-19 13:16:31 +0100246uint32_t share_state_next_fragment_offset(
Andrew Walbranca808b12020-05-15 17:22:28 +0100247 struct share_states_locked share_states,
248 struct ffa_memory_share_state *share_state)
249{
250 uint32_t next_fragment_offset;
251 uint32_t i;
252
253 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000254 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100255
256 next_fragment_offset =
257 ffa_composite_constituent_offset(share_state->memory_region, 0);
258 for (i = 0; i < share_state->fragment_count; ++i) {
259 next_fragment_offset +=
260 share_state->fragment_constituent_counts[i] *
261 sizeof(struct ffa_memory_region_constituent);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000262 }
263
Andrew Walbranca808b12020-05-15 17:22:28 +0100264 return next_fragment_offset;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000265}
266
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100267static void dump_memory_region(struct ffa_memory_region *memory_region)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000268{
269 uint32_t i;
270
271 if (LOG_LEVEL < LOG_LEVEL_VERBOSE) {
272 return;
273 }
274
Olivier Deprez935e1b12020-12-22 18:01:29 +0100275 dlog("from VM %#x, attributes %#x, flags %#x, tag %u, to "
Olivier Deprezf92e5d42020-11-13 16:00:54 +0100276 "%u "
Andrew Walbrana65a1322020-04-06 19:32:32 +0100277 "recipients [",
278 memory_region->sender, memory_region->attributes,
Olivier Deprez935e1b12020-12-22 18:01:29 +0100279 memory_region->flags, memory_region->tag,
Andrew Walbrana65a1322020-04-06 19:32:32 +0100280 memory_region->receiver_count);
281 for (i = 0; i < memory_region->receiver_count; ++i) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +0000282 struct ffa_memory_access *receiver =
283 ffa_memory_region_get_receiver(memory_region, i);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000284 if (i != 0) {
285 dlog(", ");
286 }
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +0000287 dlog("Receiver %#x: %#x (offset %u)",
288 receiver->receiver_permissions.receiver,
289 receiver->receiver_permissions.permissions,
290 receiver->composite_memory_region_offset);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000291 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000292}
293
J-Alves66652252022-07-06 09:49:51 +0100294void dump_share_states(void)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000295{
296 uint32_t i;
297
298 if (LOG_LEVEL < LOG_LEVEL_VERBOSE) {
299 return;
300 }
301
302 dlog("Current share states:\n");
303 sl_lock(&share_states_lock_instance);
304 for (i = 0; i < MAX_MEM_SHARES; ++i) {
305 if (share_states[i].share_func != 0) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000306 switch (share_states[i].share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100307 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000308 dlog("SHARE");
309 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100310 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000311 dlog("LEND");
312 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100313 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000314 dlog("DONATE");
315 break;
316 default:
317 dlog("invalid share_func %#x",
318 share_states[i].share_func);
319 }
Olivier Deprez935e1b12020-12-22 18:01:29 +0100320 dlog(" %#x (", share_states[i].memory_region->handle);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000321 dump_memory_region(share_states[i].memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +0100322 if (share_states[i].sending_complete) {
323 dlog("): fully sent");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000324 } else {
Andrew Walbranca808b12020-05-15 17:22:28 +0100325 dlog("): partially sent");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000326 }
J-Alves2a0d2882020-10-29 14:49:50 +0000327 dlog(" with %d fragments, %d retrieved, "
328 " sender's original mode: %#x\n",
Andrew Walbranca808b12020-05-15 17:22:28 +0100329 share_states[i].fragment_count,
J-Alves2a0d2882020-10-29 14:49:50 +0000330 share_states[i].retrieved_fragment_count[0],
331 share_states[i].sender_orig_mode);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000332 }
333 }
334 sl_unlock(&share_states_lock_instance);
335}
336
Andrew Walbran475c1452020-02-07 13:22:22 +0000337/* TODO: Add device attributes: GRE, cacheability, shareability. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100338static inline uint32_t ffa_memory_permissions_to_mode(
J-Alves7cd5eb32020-10-16 19:06:10 +0100339 ffa_memory_access_permissions_t permissions, uint32_t default_mode)
Andrew Walbran475c1452020-02-07 13:22:22 +0000340{
341 uint32_t mode = 0;
342
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100343 switch (ffa_get_data_access_attr(permissions)) {
344 case FFA_DATA_ACCESS_RO:
Andrew Walbran475c1452020-02-07 13:22:22 +0000345 mode = MM_MODE_R;
346 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100347 case FFA_DATA_ACCESS_RW:
Andrew Walbran475c1452020-02-07 13:22:22 +0000348 mode = MM_MODE_R | MM_MODE_W;
349 break;
J-Alves7cd5eb32020-10-16 19:06:10 +0100350 case FFA_DATA_ACCESS_NOT_SPECIFIED:
351 mode = (default_mode & (MM_MODE_R | MM_MODE_W));
352 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100353 case FFA_DATA_ACCESS_RESERVED:
354 panic("Tried to convert FFA_DATA_ACCESS_RESERVED.");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100355 }
356
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100357 switch (ffa_get_instruction_access_attr(permissions)) {
358 case FFA_INSTRUCTION_ACCESS_NX:
Andrew Walbran475c1452020-02-07 13:22:22 +0000359 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100360 case FFA_INSTRUCTION_ACCESS_X:
Andrew Walbrana65a1322020-04-06 19:32:32 +0100361 mode |= MM_MODE_X;
362 break;
J-Alves7cd5eb32020-10-16 19:06:10 +0100363 case FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED:
364 mode |= (default_mode & MM_MODE_X);
365 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100366 case FFA_INSTRUCTION_ACCESS_RESERVED:
367 panic("Tried to convert FFA_INSTRUCTION_ACCESS_RESVERVED.");
Andrew Walbran475c1452020-02-07 13:22:22 +0000368 }
369
Olivier Deprez878bd5b2021-04-15 19:05:10 +0200370 /* Set the security state bit if necessary. */
371 if ((default_mode & plat_ffa_other_world_mode()) != 0) {
372 mode |= plat_ffa_other_world_mode();
373 }
374
Andrew Walbran475c1452020-02-07 13:22:22 +0000375 return mode;
376}
377
Jose Marinho75509b42019-04-09 09:34:59 +0100378/**
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000379 * Get the current mode in the stage-2 page table of the given vm of all the
380 * pages in the given constituents, if they all have the same mode, or return
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100381 * an appropriate FF-A error if not.
Jose Marinho75509b42019-04-09 09:34:59 +0100382 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100383static struct ffa_value constituents_get_mode(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000384 struct vm_locked vm, uint32_t *orig_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100385 struct ffa_memory_region_constituent **fragments,
386 const uint32_t *fragment_constituent_counts, uint32_t fragment_count)
Jose Marinho75509b42019-04-09 09:34:59 +0100387{
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100388 uint32_t i;
Andrew Walbranca808b12020-05-15 17:22:28 +0100389 uint32_t j;
Jose Marinho75509b42019-04-09 09:34:59 +0100390
Andrew Walbranca808b12020-05-15 17:22:28 +0100391 if (fragment_count == 0 || fragment_constituent_counts[0] == 0) {
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100392 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000393 * Fail if there are no constituents. Otherwise we would get an
394 * uninitialised *orig_mode.
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100395 */
Karl Meakin5df422c2023-07-11 17:31:38 +0100396 dlog_verbose("%s: no constituents\n", __func__);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100397 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho75509b42019-04-09 09:34:59 +0100398 }
399
Andrew Walbranca808b12020-05-15 17:22:28 +0100400 for (i = 0; i < fragment_count; ++i) {
401 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
402 ipaddr_t begin = ipa_init(fragments[i][j].address);
403 size_t size = fragments[i][j].page_count * PAGE_SIZE;
404 ipaddr_t end = ipa_add(begin, size);
405 uint32_t current_mode;
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100406
Andrew Walbranca808b12020-05-15 17:22:28 +0100407 /* Fail if addresses are not page-aligned. */
408 if (!is_aligned(ipa_addr(begin), PAGE_SIZE) ||
409 !is_aligned(ipa_addr(end), PAGE_SIZE)) {
Karl Meakin5df422c2023-07-11 17:31:38 +0100410 dlog_verbose("%s: addresses not page-aligned\n",
411 __func__);
Andrew Walbranca808b12020-05-15 17:22:28 +0100412 return ffa_error(FFA_INVALID_PARAMETERS);
413 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100414
Andrew Walbranca808b12020-05-15 17:22:28 +0100415 /*
416 * Ensure that this constituent memory range is all
417 * mapped with the same mode.
418 */
Raghu Krishnamurthy785d52f2021-02-13 00:02:40 -0800419 if (!vm_mem_get_mode(vm, begin, end, &current_mode)) {
Karl Meakin5df422c2023-07-11 17:31:38 +0100420 dlog_verbose(
421 "%s: constituent memory range %#x..%#x "
422 "not mapped with the same mode\n",
423 __func__, begin, end);
Andrew Walbranca808b12020-05-15 17:22:28 +0100424 return ffa_error(FFA_DENIED);
425 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100426
Andrew Walbranca808b12020-05-15 17:22:28 +0100427 /*
428 * Ensure that all constituents are mapped with the same
429 * mode.
430 */
431 if (i == 0) {
432 *orig_mode = current_mode;
433 } else if (current_mode != *orig_mode) {
434 dlog_verbose(
Karl Meakin5df422c2023-07-11 17:31:38 +0100435 "%s: expected mode %#x but was %#x for "
436 "%d pages at %#x.\n",
437 __func__, *orig_mode, current_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100438 fragments[i][j].page_count,
439 ipa_addr(begin));
440 return ffa_error(FFA_DENIED);
441 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100442 }
Jose Marinho75509b42019-04-09 09:34:59 +0100443 }
444
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100445 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000446}
447
Daniel Boulbyc7dc9322023-10-27 15:12:07 +0100448uint32_t ffa_version_from_memory_access_desc_size(
449 uint32_t memory_access_desc_size)
450{
451 switch (memory_access_desc_size) {
452 /*
453 * v1.0 and v1.1 memory access descriptors are the same size however
454 * v1.1 is the first version to include the memory access descriptor
455 * size field so return v1.1.
456 */
457 case sizeof(struct ffa_memory_access):
458 return MAKE_FFA_VERSION(1, 1);
459 }
460 return 0;
461}
462
463/**
464 * Check if the receivers size and offset given is valid for the senders
465 * FF-A version.
466 */
467static bool receiver_size_and_offset_valid_for_version(
468 uint32_t receivers_size, uint32_t receivers_offset,
469 uint32_t ffa_version)
470{
471 /*
472 * Check that the version that the memory access descriptor size belongs
473 * to is compatible with the FF-A version we believe the sender to be.
474 */
475 uint32_t expected_ffa_version =
476 ffa_version_from_memory_access_desc_size(receivers_size);
477 if (!FFA_VERSIONS_ARE_COMPATIBLE(expected_ffa_version, ffa_version)) {
478 return false;
479 }
480
481 /*
482 * Check the receivers_offset matches the version we found from
483 * memory access descriptor size.
484 */
485 switch (expected_ffa_version) {
486 case MAKE_FFA_VERSION(1, 1):
487 return receivers_offset == sizeof(struct ffa_memory_region);
488 default:
489 return false;
490 }
491}
492
493/**
494 * Check the values set for fields in the memory region are valid and safe.
495 * Offset values are within safe bounds, receiver count will not cause overflows
496 * and reserved fields are 0.
497 */
498bool ffa_memory_region_sanity_check(struct ffa_memory_region *memory_region,
499 uint32_t ffa_version,
500 uint32_t fragment_length,
501 bool send_transaction)
502{
503 uint32_t receiver_count;
504 struct ffa_memory_access *receiver;
505 uint32_t composite_offset_0;
506
507 if (ffa_version == MAKE_FFA_VERSION(1, 0)) {
508 struct ffa_memory_region_v1_0 *memory_region_v1_0 =
509 (struct ffa_memory_region_v1_0 *)memory_region;
510 /* Check the reserved fields are 0. */
511 if (memory_region_v1_0->reserved_0 != 0 ||
512 memory_region_v1_0->reserved_1 != 0) {
513 dlog_verbose("Reserved fields must be 0.\n");
514 return false;
515 }
516
517 receiver_count = memory_region_v1_0->receiver_count;
518 } else {
519 uint32_t receivers_size =
520 memory_region->memory_access_desc_size;
521 uint32_t receivers_offset = memory_region->receivers_offset;
522
523 /* Check the reserved field is 0. */
524 if (memory_region->reserved[0] != 0 ||
525 memory_region->reserved[1] != 0 ||
526 memory_region->reserved[2] != 0) {
527 dlog_verbose("Reserved fields must be 0.\n");
528 return false;
529 }
530
531 /*
532 * Check memory_access_desc_size matches the size of the struct
533 * for the senders FF-A version.
534 */
535 if (!receiver_size_and_offset_valid_for_version(
536 receivers_size, receivers_offset, ffa_version)) {
537 dlog_verbose(
538 "Invalid memory access descriptor size %d, "
539 " or receiver offset %d, "
540 "for FF-A version %#x\n",
541 receivers_size, receivers_offset, ffa_version);
542 return false;
543 }
544
545 receiver_count = memory_region->receiver_count;
546 }
547
548 /* Check receiver count is not too large. */
549 if (receiver_count > MAX_MEM_SHARE_RECIPIENTS) {
550 dlog_verbose(
551 "Max number of recipients supported is %u "
552 "specified %u\n",
553 MAX_MEM_SHARE_RECIPIENTS, receiver_count);
554 return false;
555 }
556
557 /* Check values in the memory access descriptors. */
558 /*
559 * The composite offset values must be the same for all recievers so
560 * check the first one is valid and then they are all the same.
561 */
562 receiver = ffa_version == MAKE_FFA_VERSION(1, 0)
563 ? (struct ffa_memory_access *)&(
564 (struct ffa_memory_region_v1_0 *)
565 memory_region)
566 ->receivers[0]
567 : ffa_memory_region_get_receiver(memory_region, 0);
568 assert(receiver != NULL);
569 composite_offset_0 = receiver->composite_memory_region_offset;
570
571 if (!send_transaction) {
572 if (composite_offset_0 != 0) {
573 dlog_verbose(
574 "Composite offset memory region descriptor "
575 "offset must be 0 for retrieve requests. "
576 "Currently %d",
577 composite_offset_0);
578 return false;
579 }
580 } else {
581 bool comp_offset_is_zero = composite_offset_0 == 0U;
582 bool comp_offset_lt_transaction_descriptor_size =
583 composite_offset_0 <
584 (sizeof(struct ffa_memory_region) +
585 (uint32_t)(memory_region->memory_access_desc_size *
586 memory_region->receiver_count));
587 bool comp_offset_with_comp_gt_fragment_length =
588 composite_offset_0 +
589 sizeof(struct ffa_composite_memory_region) >
590 fragment_length;
591 if (comp_offset_is_zero ||
592 comp_offset_lt_transaction_descriptor_size ||
593 comp_offset_with_comp_gt_fragment_length) {
594 dlog_verbose(
595 "Invalid composite memory region descriptor "
596 "offset for send transaction %u\n",
597 composite_offset_0);
598 return false;
599 }
600 }
601
602 for (int i = 0; i < memory_region->receiver_count; i++) {
603 uint32_t composite_offset;
604
605 if (ffa_version == MAKE_FFA_VERSION(1, 0)) {
606 struct ffa_memory_region_v1_0 *memory_region_v1_0 =
607 (struct ffa_memory_region_v1_0 *)memory_region;
608
609 struct ffa_memory_access_v1_0 *receiver_v1_0 =
610 &memory_region_v1_0->receivers[i];
611 /* Check reserved fields are 0 */
612 if (receiver_v1_0->reserved_0 != 0) {
613 dlog_verbose(
614 "Reserved field in the memory access "
615 " descriptor must be zero "
616 " Currently reciever %d has a reserved "
617 " field with a value of %d\n",
618 i, receiver_v1_0->reserved_0);
619 return false;
620 }
621 /*
622 * We can cast to the current version receiver as the
623 * remaining fields we are checking have the same
624 * offsets for all versions since memory access
625 * descriptors are forwards compatible.
626 */
627 receiver = (struct ffa_memory_access *)receiver_v1_0;
628 } else {
629 receiver = ffa_memory_region_get_receiver(memory_region,
630 i);
631 assert(receiver != NULL);
632
633 if (receiver->reserved_0 != 0) {
634 dlog_verbose(
635 "Reserved field in the memory access "
636 " descriptor must be zero "
637 " Currently reciever %d has a reserved "
638 " field with a value of %d\n",
639 i, receiver->reserved_0);
640 return false;
641 }
642 }
643
644 /* Check composite offset values are equal for all receivers. */
645 composite_offset = receiver->composite_memory_region_offset;
646 if (composite_offset != composite_offset_0) {
647 dlog_verbose(
648 "Composite offset %x differs from %x in index "
649 "%u\n",
650 composite_offset, composite_offset_0);
651 return false;
652 }
653 }
654 return true;
655}
656
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000657/**
658 * Verify that all pages have the same mode, that the starting mode
659 * constitutes a valid state and obtain the next mode to apply
660 * to the sending VM.
661 *
662 * Returns:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100663 * 1) FFA_DENIED if a state transition was not found;
664 * 2) FFA_DENIED if the pages being shared do not have the same mode within
Andrew Walbrana65a1322020-04-06 19:32:32 +0100665 * the <from> VM;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100666 * 3) FFA_INVALID_PARAMETERS if the beginning and end IPAs are not page
Andrew Walbrana65a1322020-04-06 19:32:32 +0100667 * aligned;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100668 * 4) FFA_INVALID_PARAMETERS if the requested share type was not handled.
669 * Or FFA_SUCCESS on success.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000670 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100671static struct ffa_value ffa_send_check_transition(
Andrew Walbrana65a1322020-04-06 19:32:32 +0100672 struct vm_locked from, uint32_t share_func,
J-Alves363f5722022-04-25 17:37:37 +0100673 struct ffa_memory_access *receivers, uint32_t receivers_count,
674 uint32_t *orig_from_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100675 struct ffa_memory_region_constituent **fragments,
676 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
677 uint32_t *from_mode)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000678{
679 const uint32_t state_mask =
680 MM_MODE_INVALID | MM_MODE_UNOWNED | MM_MODE_SHARED;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100681 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000682
Andrew Walbranca808b12020-05-15 17:22:28 +0100683 ret = constituents_get_mode(from, orig_from_mode, fragments,
684 fragment_constituent_counts,
685 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100686 if (ret.func != FFA_SUCCESS_32) {
Olivier Depreze7eb1682022-03-16 17:09:03 +0100687 dlog_verbose("Inconsistent modes.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100688 return ret;
Andrew Scullb5f49e02019-10-02 13:20:47 +0100689 }
690
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000691 /* Ensure the address range is normal memory and not a device. */
J-Alves788b4492023-04-18 14:01:23 +0100692 if ((*orig_from_mode & MM_MODE_D) != 0U) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000693 dlog_verbose("Can't share device memory (mode is %#x).\n",
694 *orig_from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100695 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000696 }
697
698 /*
699 * Ensure the sender is the owner and has exclusive access to the
700 * memory.
701 */
702 if ((*orig_from_mode & state_mask) != 0) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100703 return ffa_error(FFA_DENIED);
Andrew Walbrana65a1322020-04-06 19:32:32 +0100704 }
705
J-Alves363f5722022-04-25 17:37:37 +0100706 assert(receivers != NULL && receivers_count > 0U);
J-Alves7cd5eb32020-10-16 19:06:10 +0100707
J-Alves363f5722022-04-25 17:37:37 +0100708 for (uint32_t i = 0U; i < receivers_count; i++) {
709 ffa_memory_access_permissions_t permissions =
710 receivers[i].receiver_permissions.permissions;
711 uint32_t required_from_mode = ffa_memory_permissions_to_mode(
712 permissions, *orig_from_mode);
713
J-Alves788b4492023-04-18 14:01:23 +0100714 /*
715 * The assumption is that at this point, the operation from
716 * SP to a receiver VM, should have returned an FFA_ERROR
717 * already.
718 */
719 if (!ffa_is_vm_id(from.vm->id)) {
720 assert(!ffa_is_vm_id(
721 receivers[i].receiver_permissions.receiver));
722 }
723
J-Alves363f5722022-04-25 17:37:37 +0100724 if ((*orig_from_mode & required_from_mode) !=
725 required_from_mode) {
726 dlog_verbose(
727 "Sender tried to send memory with permissions "
J-Alves788b4492023-04-18 14:01:23 +0100728 "which required mode %#x but only had %#x "
729 "itself.\n",
J-Alves363f5722022-04-25 17:37:37 +0100730 required_from_mode, *orig_from_mode);
731 return ffa_error(FFA_DENIED);
732 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000733 }
734
735 /* Find the appropriate new mode. */
736 *from_mode = ~state_mask & *orig_from_mode;
Andrew Walbrane7ad3c02019-12-24 17:03:04 +0000737 switch (share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100738 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000739 *from_mode |= MM_MODE_INVALID | MM_MODE_UNOWNED;
Jose Marinho75509b42019-04-09 09:34:59 +0100740 break;
741
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100742 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000743 *from_mode |= MM_MODE_INVALID;
Andrew Walbran648fc3e2019-10-22 16:23:05 +0100744 break;
745
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100746 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000747 *from_mode |= MM_MODE_SHARED;
Jose Marinho56c25732019-05-20 09:48:53 +0100748 break;
749
Jose Marinho75509b42019-04-09 09:34:59 +0100750 default:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100751 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho75509b42019-04-09 09:34:59 +0100752 }
753
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100754 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000755}
756
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100757static struct ffa_value ffa_relinquish_check_transition(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000758 struct vm_locked from, uint32_t *orig_from_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100759 struct ffa_memory_region_constituent **fragments,
760 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
761 uint32_t *from_mode)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000762{
763 const uint32_t state_mask =
764 MM_MODE_INVALID | MM_MODE_UNOWNED | MM_MODE_SHARED;
765 uint32_t orig_from_state;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100766 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000767
Andrew Walbranca808b12020-05-15 17:22:28 +0100768 ret = constituents_get_mode(from, orig_from_mode, fragments,
769 fragment_constituent_counts,
770 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100771 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbrana65a1322020-04-06 19:32:32 +0100772 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000773 }
774
775 /* Ensure the address range is normal memory and not a device. */
776 if (*orig_from_mode & MM_MODE_D) {
777 dlog_verbose("Can't relinquish device memory (mode is %#x).\n",
778 *orig_from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100779 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000780 }
781
782 /*
783 * Ensure the relinquishing VM is not the owner but has access to the
784 * memory.
785 */
786 orig_from_state = *orig_from_mode & state_mask;
787 if ((orig_from_state & ~MM_MODE_SHARED) != MM_MODE_UNOWNED) {
788 dlog_verbose(
789 "Tried to relinquish memory in state %#x (masked %#x "
Andrew Walbranca808b12020-05-15 17:22:28 +0100790 "but should be %#x).\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000791 *orig_from_mode, orig_from_state, MM_MODE_UNOWNED);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100792 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000793 }
794
795 /* Find the appropriate new mode. */
796 *from_mode = (~state_mask & *orig_from_mode) | MM_MODE_UNMAPPED_MASK;
797
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100798 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000799}
800
801/**
802 * Verify that all pages have the same mode, that the starting mode
803 * constitutes a valid state and obtain the next mode to apply
804 * to the retrieving VM.
805 *
806 * Returns:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100807 * 1) FFA_DENIED if a state transition was not found;
808 * 2) FFA_DENIED if the pages being shared do not have the same mode within
Andrew Walbrana65a1322020-04-06 19:32:32 +0100809 * the <to> VM;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100810 * 3) FFA_INVALID_PARAMETERS if the beginning and end IPAs are not page
Andrew Walbrana65a1322020-04-06 19:32:32 +0100811 * aligned;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100812 * 4) FFA_INVALID_PARAMETERS if the requested share type was not handled.
813 * Or FFA_SUCCESS on success.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000814 */
J-Alvesfc19b372022-07-06 12:17:35 +0100815struct ffa_value ffa_retrieve_check_transition(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000816 struct vm_locked to, uint32_t share_func,
Andrew Walbranca808b12020-05-15 17:22:28 +0100817 struct ffa_memory_region_constituent **fragments,
818 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
819 uint32_t memory_to_attributes, uint32_t *to_mode)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000820{
821 uint32_t orig_to_mode;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100822 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000823
Andrew Walbranca808b12020-05-15 17:22:28 +0100824 ret = constituents_get_mode(to, &orig_to_mode, fragments,
825 fragment_constituent_counts,
826 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100827 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100828 dlog_verbose("Inconsistent modes.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100829 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000830 }
831
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100832 if (share_func == FFA_MEM_RECLAIM_32) {
J-Alves9256f162021-12-09 13:18:43 +0000833 /*
834 * If the original ffa memory send call has been processed
835 * successfully, it is expected the orig_to_mode would overlay
836 * with `state_mask`, as a result of the function
837 * `ffa_send_check_transition`.
838 */
J-Alves59ed0042022-07-28 18:26:41 +0100839 if (vm_id_is_current_world(to.vm->id)) {
840 assert((orig_to_mode &
841 (MM_MODE_INVALID | MM_MODE_UNOWNED |
842 MM_MODE_SHARED)) != 0U);
843 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000844 } else {
845 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +0100846 * If the retriever is from virtual FF-A instance:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000847 * Ensure the retriever has the expected state. We don't care
848 * about the MM_MODE_SHARED bit; either with or without it set
849 * are both valid representations of the !O-NA state.
850 */
J-Alvesa9cd7e32022-07-01 13:49:33 +0100851 if (vm_id_is_current_world(to.vm->id) &&
852 to.vm->id != HF_PRIMARY_VM_ID &&
853 (orig_to_mode & MM_MODE_UNMAPPED_MASK) !=
854 MM_MODE_UNMAPPED_MASK) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100855 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000856 }
857 }
858
859 /* Find the appropriate new mode. */
860 *to_mode = memory_to_attributes;
861 switch (share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100862 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000863 *to_mode |= 0;
864 break;
865
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100866 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000867 *to_mode |= MM_MODE_UNOWNED;
868 break;
869
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100870 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000871 *to_mode |= MM_MODE_UNOWNED | MM_MODE_SHARED;
872 break;
873
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100874 case FFA_MEM_RECLAIM_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000875 *to_mode |= 0;
876 break;
877
878 default:
Andrew Walbranca808b12020-05-15 17:22:28 +0100879 dlog_error("Invalid share_func %#x.\n", share_func);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100880 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000881 }
882
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100883 return (struct ffa_value){.func = FFA_SUCCESS_32};
Jose Marinho75509b42019-04-09 09:34:59 +0100884}
Jose Marinho09b1db82019-08-08 09:16:59 +0100885
886/**
887 * Updates a VM's page table such that the given set of physical address ranges
888 * are mapped in the address space at the corresponding address ranges, in the
889 * mode provided.
890 *
891 * If commit is false, the page tables will be allocated from the mpool but no
892 * mappings will actually be updated. This function must always be called first
893 * with commit false to check that it will succeed before calling with commit
894 * true, to avoid leaving the page table in a half-updated state. To make a
895 * series of changes atomically you can call them all with commit false before
896 * calling them all with commit true.
897 *
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -0700898 * vm_ptable_defrag should always be called after a series of page table
899 * updates, whether they succeed or fail.
Jose Marinho09b1db82019-08-08 09:16:59 +0100900 *
901 * Returns true on success, or false if the update failed and no changes were
902 * made to memory mappings.
903 */
J-Alves66652252022-07-06 09:49:51 +0100904bool ffa_region_group_identity_map(
Andrew Walbranf4b51af2020-02-03 14:44:54 +0000905 struct vm_locked vm_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +0100906 struct ffa_memory_region_constituent **fragments,
907 const uint32_t *fragment_constituent_counts, uint32_t fragment_count,
Daniel Boulby4dd3f532021-09-21 09:57:08 +0100908 uint32_t mode, struct mpool *ppool, bool commit)
Jose Marinho09b1db82019-08-08 09:16:59 +0100909{
Andrew Walbranca808b12020-05-15 17:22:28 +0100910 uint32_t i;
911 uint32_t j;
Jose Marinho09b1db82019-08-08 09:16:59 +0100912
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -0700913 if (vm_locked.vm->el0_partition) {
914 mode |= MM_MODE_USER | MM_MODE_NG;
915 }
916
Andrew Walbranca808b12020-05-15 17:22:28 +0100917 /* Iterate over the memory region constituents within each fragment. */
918 for (i = 0; i < fragment_count; ++i) {
919 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
920 size_t size = fragments[i][j].page_count * PAGE_SIZE;
921 paddr_t pa_begin =
922 pa_from_ipa(ipa_init(fragments[i][j].address));
923 paddr_t pa_end = pa_add(pa_begin, size);
Jens Wiklander4f1880c2022-10-19 17:00:14 +0200924 uint32_t pa_bits =
925 arch_mm_get_pa_bits(arch_mm_get_pa_range());
Federico Recanati4fd065d2021-12-13 20:06:23 +0100926
927 /*
928 * Ensure the requested region falls into system's PA
929 * range.
930 */
Jens Wiklander4f1880c2022-10-19 17:00:14 +0200931 if (((pa_addr(pa_begin) >> pa_bits) > 0) ||
932 ((pa_addr(pa_end) >> pa_bits) > 0)) {
Federico Recanati4fd065d2021-12-13 20:06:23 +0100933 dlog_error("Region is outside of PA Range\n");
934 return false;
935 }
Andrew Walbranca808b12020-05-15 17:22:28 +0100936
937 if (commit) {
938 vm_identity_commit(vm_locked, pa_begin, pa_end,
939 mode, ppool, NULL);
940 } else if (!vm_identity_prepare(vm_locked, pa_begin,
941 pa_end, mode, ppool)) {
942 return false;
943 }
Jose Marinho09b1db82019-08-08 09:16:59 +0100944 }
945 }
946
947 return true;
948}
949
950/**
951 * Clears a region of physical memory by overwriting it with zeros. The data is
952 * flushed from the cache so the memory has been cleared across the system.
953 */
J-Alves7db32002021-12-14 14:44:50 +0000954static bool clear_memory(paddr_t begin, paddr_t end, struct mpool *ppool,
955 uint32_t extra_mode_attributes)
Jose Marinho09b1db82019-08-08 09:16:59 +0100956{
957 /*
Fuad Tabbaed294af2019-12-20 10:43:01 +0000958 * TODO: change this to a CPU local single page window rather than a
Jose Marinho09b1db82019-08-08 09:16:59 +0100959 * global mapping of the whole range. Such an approach will limit
960 * the changes to stage-1 tables and will allow only local
961 * invalidation.
962 */
963 bool ret;
964 struct mm_stage1_locked stage1_locked = mm_lock_stage1();
J-Alves7db32002021-12-14 14:44:50 +0000965 void *ptr = mm_identity_map(stage1_locked, begin, end,
966 MM_MODE_W | (extra_mode_attributes &
967 plat_ffa_other_world_mode()),
968 ppool);
Jose Marinho09b1db82019-08-08 09:16:59 +0100969 size_t size = pa_difference(begin, end);
970
971 if (!ptr) {
Jose Marinho09b1db82019-08-08 09:16:59 +0100972 goto fail;
973 }
974
975 memset_s(ptr, size, 0, size);
976 arch_mm_flush_dcache(ptr, size);
977 mm_unmap(stage1_locked, begin, end, ppool);
978
979 ret = true;
980 goto out;
981
982fail:
983 ret = false;
984
985out:
986 mm_unlock_stage1(&stage1_locked);
987
988 return ret;
989}
990
991/**
992 * Clears a region of physical memory by overwriting it with zeros. The data is
993 * flushed from the cache so the memory has been cleared across the system.
994 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100995static bool ffa_clear_memory_constituents(
J-Alves7db32002021-12-14 14:44:50 +0000996 uint32_t security_state_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100997 struct ffa_memory_region_constituent **fragments,
998 const uint32_t *fragment_constituent_counts, uint32_t fragment_count,
999 struct mpool *page_pool)
Jose Marinho09b1db82019-08-08 09:16:59 +01001000{
1001 struct mpool local_page_pool;
Andrew Walbranca808b12020-05-15 17:22:28 +01001002 uint32_t i;
Jose Marinho09b1db82019-08-08 09:16:59 +01001003 bool ret = false;
1004
1005 /*
1006 * Create a local pool so any freed memory can't be used by another
1007 * thread. This is to ensure each constituent that is mapped can be
1008 * unmapped again afterwards.
1009 */
Andrew Walbran475c1452020-02-07 13:22:22 +00001010 mpool_init_with_fallback(&local_page_pool, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +01001011
Andrew Walbranca808b12020-05-15 17:22:28 +01001012 /* Iterate over the memory region constituents within each fragment. */
1013 for (i = 0; i < fragment_count; ++i) {
1014 uint32_t j;
Jose Marinho09b1db82019-08-08 09:16:59 +01001015
J-Alves8457f932023-10-11 16:41:45 +01001016 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001017 size_t size = fragments[i][j].page_count * PAGE_SIZE;
1018 paddr_t begin =
1019 pa_from_ipa(ipa_init(fragments[i][j].address));
1020 paddr_t end = pa_add(begin, size);
1021
J-Alves7db32002021-12-14 14:44:50 +00001022 if (!clear_memory(begin, end, &local_page_pool,
1023 security_state_mode)) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001024 /*
1025 * api_clear_memory will defrag on failure, so
1026 * no need to do it here.
1027 */
1028 goto out;
1029 }
Jose Marinho09b1db82019-08-08 09:16:59 +01001030 }
1031 }
1032
Jose Marinho09b1db82019-08-08 09:16:59 +01001033 ret = true;
1034
1035out:
1036 mpool_fini(&local_page_pool);
1037 return ret;
1038}
1039
J-Alves5952d942022-12-22 16:03:00 +00001040static bool is_memory_range_within(ipaddr_t begin, ipaddr_t end,
1041 ipaddr_t in_begin, ipaddr_t in_end)
1042{
1043 return (ipa_addr(begin) >= ipa_addr(in_begin) &&
1044 ipa_addr(begin) < ipa_addr(in_end)) ||
1045 (ipa_addr(end) <= ipa_addr(in_end) &&
1046 ipa_addr(end) > ipa_addr(in_begin));
1047}
1048
1049/**
1050 * Receives a memory range and looks for overlaps with the remainder
1051 * constituents of the memory share/lend/donate operation. Assumes they are
1052 * passed in order to avoid having to loop over all the elements at each call.
1053 * The function only compares the received memory ranges with those that follow
1054 * within the same fragment, and subsequent fragments from the same operation.
1055 */
1056static bool ffa_memory_check_overlap(
1057 struct ffa_memory_region_constituent **fragments,
1058 const uint32_t *fragment_constituent_counts,
1059 const uint32_t fragment_count, const uint32_t current_fragment,
1060 const uint32_t current_constituent)
1061{
1062 uint32_t i = current_fragment;
1063 uint32_t j = current_constituent;
1064 ipaddr_t current_begin = ipa_init(fragments[i][j].address);
1065 const uint32_t current_page_count = fragments[i][j].page_count;
1066 size_t current_size = current_page_count * PAGE_SIZE;
1067 ipaddr_t current_end = ipa_add(current_begin, current_size - 1);
1068
1069 if (current_size == 0 ||
1070 current_size > UINT64_MAX - ipa_addr(current_begin)) {
1071 dlog_verbose("Invalid page count. Addr: %x page_count: %x\n",
1072 current_begin, current_page_count);
1073 return false;
1074 }
1075
1076 for (; i < fragment_count; i++) {
1077 j = (i == current_fragment) ? j + 1 : 0;
1078
1079 for (; j < fragment_constituent_counts[i]; j++) {
1080 ipaddr_t begin = ipa_init(fragments[i][j].address);
1081 const uint32_t page_count = fragments[i][j].page_count;
1082 size_t size = page_count * PAGE_SIZE;
1083 ipaddr_t end = ipa_add(begin, size - 1);
1084
1085 if (size == 0 || size > UINT64_MAX - ipa_addr(begin)) {
1086 dlog_verbose(
1087 "Invalid page count. Addr: %x "
1088 "page_count: %x\n",
1089 begin, page_count);
1090 return false;
1091 }
1092
1093 /*
1094 * Check if current ranges is within begin and end, as
1095 * well as the reverse. This should help optimize the
1096 * loop, and reduce the number of iterations.
1097 */
1098 if (is_memory_range_within(begin, end, current_begin,
1099 current_end) ||
1100 is_memory_range_within(current_begin, current_end,
1101 begin, end)) {
1102 dlog_verbose(
1103 "Overlapping memory ranges: %#x - %#x "
1104 "with %#x - %#x\n",
1105 ipa_addr(begin), ipa_addr(end),
1106 ipa_addr(current_begin),
1107 ipa_addr(current_end));
1108 return true;
1109 }
1110 }
1111 }
1112
1113 return false;
1114}
1115
Jose Marinho09b1db82019-08-08 09:16:59 +01001116/**
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001117 * Validates and prepares memory to be sent from the calling VM to another.
Jose Marinho09b1db82019-08-08 09:16:59 +01001118 *
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001119 * This function requires the calling context to hold the <from> VM lock.
Jose Marinho09b1db82019-08-08 09:16:59 +01001120 *
1121 * Returns:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001122 * In case of error, one of the following values is returned:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001123 * 1) FFA_INVALID_PARAMETERS - The endpoint provided parameters were
Jose Marinho09b1db82019-08-08 09:16:59 +01001124 * erroneous;
Andrew Walbranf07f04d2020-05-01 18:09:00 +01001125 * 2) FFA_NO_MEMORY - Hafnium did not have sufficient memory to complete the
1126 * request.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001127 * 3) FFA_DENIED - The sender doesn't have sufficient access to send the
Andrew Walbrana65a1322020-04-06 19:32:32 +01001128 * memory with the given permissions.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001129 * Success is indicated by FFA_SUCCESS.
Jose Marinho09b1db82019-08-08 09:16:59 +01001130 */
J-Alves66652252022-07-06 09:49:51 +01001131struct ffa_value ffa_send_check_update(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001132 struct vm_locked from_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01001133 struct ffa_memory_region_constituent **fragments,
1134 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
J-Alves8f11cde2022-12-21 16:18:22 +00001135 uint32_t composite_total_page_count, uint32_t share_func,
1136 struct ffa_memory_access *receivers, uint32_t receivers_count,
1137 struct mpool *page_pool, bool clear, uint32_t *orig_from_mode_ret)
Jose Marinho09b1db82019-08-08 09:16:59 +01001138{
Andrew Walbranca808b12020-05-15 17:22:28 +01001139 uint32_t i;
J-Alves8f11cde2022-12-21 16:18:22 +00001140 uint32_t j;
Jose Marinho09b1db82019-08-08 09:16:59 +01001141 uint32_t orig_from_mode;
1142 uint32_t from_mode;
Jose Marinho09b1db82019-08-08 09:16:59 +01001143 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001144 struct ffa_value ret;
J-Alves8f11cde2022-12-21 16:18:22 +00001145 uint32_t constituents_total_page_count = 0;
Jose Marinho09b1db82019-08-08 09:16:59 +01001146
1147 /*
Andrew Walbrana65a1322020-04-06 19:32:32 +01001148 * Make sure constituents are properly aligned to a 64-bit boundary. If
1149 * not we would get alignment faults trying to read (64-bit) values.
Jose Marinho09b1db82019-08-08 09:16:59 +01001150 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001151 for (i = 0; i < fragment_count; ++i) {
1152 if (!is_aligned(fragments[i], 8)) {
1153 dlog_verbose("Constituents not aligned.\n");
1154 return ffa_error(FFA_INVALID_PARAMETERS);
1155 }
J-Alves8f11cde2022-12-21 16:18:22 +00001156 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
1157 constituents_total_page_count +=
1158 fragments[i][j].page_count;
J-Alves5952d942022-12-22 16:03:00 +00001159 if (ffa_memory_check_overlap(
1160 fragments, fragment_constituent_counts,
1161 fragment_count, i, j)) {
1162 return ffa_error(FFA_INVALID_PARAMETERS);
1163 }
J-Alves8f11cde2022-12-21 16:18:22 +00001164 }
1165 }
1166
1167 if (constituents_total_page_count != composite_total_page_count) {
1168 dlog_verbose(
1169 "Composite page count differs from calculated page "
1170 "count from constituents.\n");
1171 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho09b1db82019-08-08 09:16:59 +01001172 }
1173
1174 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001175 * Check if the state transition is lawful for the sender, ensure that
1176 * all constituents of a memory region being shared are at the same
1177 * state.
Jose Marinho09b1db82019-08-08 09:16:59 +01001178 */
J-Alves363f5722022-04-25 17:37:37 +01001179 ret = ffa_send_check_transition(from_locked, share_func, receivers,
1180 receivers_count, &orig_from_mode,
1181 fragments, fragment_constituent_counts,
Andrew Walbranca808b12020-05-15 17:22:28 +01001182 fragment_count, &from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001183 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001184 dlog_verbose("Invalid transition for send.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +01001185 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01001186 }
1187
Andrew Walbran37c574e2020-06-03 11:45:46 +01001188 if (orig_from_mode_ret != NULL) {
1189 *orig_from_mode_ret = orig_from_mode;
1190 }
1191
Jose Marinho09b1db82019-08-08 09:16:59 +01001192 /*
1193 * Create a local pool so any freed memory can't be used by another
1194 * thread. This is to ensure the original mapping can be restored if the
1195 * clear fails.
1196 */
Andrew Walbran475c1452020-02-07 13:22:22 +00001197 mpool_init_with_fallback(&local_page_pool, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +01001198
1199 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001200 * First reserve all required memory for the new page table entries
1201 * without committing, to make sure the entire operation will succeed
1202 * without exhausting the page pool.
Jose Marinho09b1db82019-08-08 09:16:59 +01001203 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001204 if (!ffa_region_group_identity_map(
1205 from_locked, fragments, fragment_constituent_counts,
1206 fragment_count, from_mode, page_pool, false)) {
Jose Marinho09b1db82019-08-08 09:16:59 +01001207 /* TODO: partial defrag of failed range. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001208 ret = ffa_error(FFA_NO_MEMORY);
Jose Marinho09b1db82019-08-08 09:16:59 +01001209 goto out;
1210 }
1211
1212 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001213 * Update the mapping for the sender. This won't allocate because the
1214 * transaction was already prepared above, but may free pages in the
1215 * case that a whole block is being unmapped that was previously
1216 * partially mapped.
Jose Marinho09b1db82019-08-08 09:16:59 +01001217 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001218 CHECK(ffa_region_group_identity_map(
1219 from_locked, fragments, fragment_constituent_counts,
1220 fragment_count, from_mode, &local_page_pool, true));
Jose Marinho09b1db82019-08-08 09:16:59 +01001221
1222 /* Clear the memory so no VM or device can see the previous contents. */
J-Alves7db32002021-12-14 14:44:50 +00001223 if (clear &&
J-Alves26483382023-04-20 12:01:49 +01001224 !ffa_clear_memory_constituents(orig_from_mode, fragments,
1225 fragment_constituent_counts,
1226 fragment_count, page_pool)) {
Jose Marinho09b1db82019-08-08 09:16:59 +01001227 /*
1228 * On failure, roll back by returning memory to the sender. This
1229 * may allocate pages which were previously freed into
1230 * `local_page_pool` by the call above, but will never allocate
1231 * more pages than that so can never fail.
1232 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001233 CHECK(ffa_region_group_identity_map(
Andrew Walbranca808b12020-05-15 17:22:28 +01001234 from_locked, fragments, fragment_constituent_counts,
1235 fragment_count, orig_from_mode, &local_page_pool,
1236 true));
Jose Marinho09b1db82019-08-08 09:16:59 +01001237
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001238 ret = ffa_error(FFA_NO_MEMORY);
Jose Marinho09b1db82019-08-08 09:16:59 +01001239 goto out;
1240 }
1241
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001242 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001243
1244out:
1245 mpool_fini(&local_page_pool);
1246
1247 /*
1248 * Tidy up the page table by reclaiming failed mappings (if there was an
1249 * error) or merging entries into blocks where possible (on success).
1250 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001251 vm_ptable_defrag(from_locked, page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001252
1253 return ret;
1254}
1255
1256/**
1257 * Validates and maps memory shared from one VM to another.
1258 *
1259 * This function requires the calling context to hold the <to> lock.
1260 *
1261 * Returns:
1262 * In case of error, one of the following values is returned:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001263 * 1) FFA_INVALID_PARAMETERS - The endpoint provided parameters were
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001264 * erroneous;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001265 * 2) FFA_NO_MEMORY - Hafnium did not have sufficient memory to complete
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001266 * the request.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001267 * Success is indicated by FFA_SUCCESS.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001268 */
J-Alvesb5084cf2022-07-06 14:20:12 +01001269struct ffa_value ffa_retrieve_check_update(
J-Alves26483382023-04-20 12:01:49 +01001270 struct vm_locked to_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01001271 struct ffa_memory_region_constituent **fragments,
1272 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
J-Alves26483382023-04-20 12:01:49 +01001273 uint32_t sender_orig_mode, uint32_t share_func, bool clear,
Andrew Walbranca808b12020-05-15 17:22:28 +01001274 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001275{
Andrew Walbranca808b12020-05-15 17:22:28 +01001276 uint32_t i;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001277 uint32_t to_mode;
1278 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001279 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001280
1281 /*
Andrew Walbranca808b12020-05-15 17:22:28 +01001282 * Make sure constituents are properly aligned to a 64-bit boundary. If
1283 * not we would get alignment faults trying to read (64-bit) values.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001284 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001285 for (i = 0; i < fragment_count; ++i) {
1286 if (!is_aligned(fragments[i], 8)) {
J-Alvesb5084cf2022-07-06 14:20:12 +01001287 dlog_verbose("Fragment not properly aligned.\n");
Andrew Walbranca808b12020-05-15 17:22:28 +01001288 return ffa_error(FFA_INVALID_PARAMETERS);
1289 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001290 }
1291
1292 /*
1293 * Check if the state transition is lawful for the recipient, and ensure
1294 * that all constituents of the memory region being retrieved are at the
1295 * same state.
1296 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001297 ret = ffa_retrieve_check_transition(
1298 to_locked, share_func, fragments, fragment_constituent_counts,
J-Alves26483382023-04-20 12:01:49 +01001299 fragment_count, sender_orig_mode, &to_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001300 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001301 dlog_verbose("Invalid transition for retrieve.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +01001302 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001303 }
1304
1305 /*
1306 * Create a local pool so any freed memory can't be used by another
1307 * thread. This is to ensure the original mapping can be restored if the
1308 * clear fails.
1309 */
1310 mpool_init_with_fallback(&local_page_pool, page_pool);
1311
1312 /*
1313 * First reserve all required memory for the new page table entries in
1314 * the recipient page tables without committing, to make sure the entire
1315 * operation will succeed without exhausting the page pool.
1316 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001317 if (!ffa_region_group_identity_map(
1318 to_locked, fragments, fragment_constituent_counts,
1319 fragment_count, to_mode, page_pool, false)) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001320 /* TODO: partial defrag of failed range. */
1321 dlog_verbose(
1322 "Insufficient memory to update recipient page "
1323 "table.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001324 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001325 goto out;
1326 }
1327
1328 /* Clear the memory so no VM or device can see the previous contents. */
J-Alves7db32002021-12-14 14:44:50 +00001329 if (clear &&
J-Alves26483382023-04-20 12:01:49 +01001330 !ffa_clear_memory_constituents(sender_orig_mode, fragments,
1331 fragment_constituent_counts,
1332 fragment_count, page_pool)) {
J-Alvesb5084cf2022-07-06 14:20:12 +01001333 dlog_verbose("Couldn't clear constituents.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001334 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001335 goto out;
1336 }
1337
Jose Marinho09b1db82019-08-08 09:16:59 +01001338 /*
1339 * Complete the transfer by mapping the memory into the recipient. This
1340 * won't allocate because the transaction was already prepared above, so
1341 * it doesn't need to use the `local_page_pool`.
1342 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001343 CHECK(ffa_region_group_identity_map(
1344 to_locked, fragments, fragment_constituent_counts,
1345 fragment_count, to_mode, page_pool, true));
Jose Marinho09b1db82019-08-08 09:16:59 +01001346
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001347 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Jose Marinho09b1db82019-08-08 09:16:59 +01001348
1349out:
1350 mpool_fini(&local_page_pool);
1351
1352 /*
Andrew Walbranf07f04d2020-05-01 18:09:00 +01001353 * Tidy up the page table by reclaiming failed mappings (if there was an
1354 * error) or merging entries into blocks where possible (on success).
Jose Marinho09b1db82019-08-08 09:16:59 +01001355 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001356 vm_ptable_defrag(to_locked, page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001357
1358 return ret;
1359}
1360
Andrew Walbran996d1d12020-05-27 14:08:43 +01001361static struct ffa_value ffa_relinquish_check_update(
J-Alves26483382023-04-20 12:01:49 +01001362 struct vm_locked from_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01001363 struct ffa_memory_region_constituent **fragments,
1364 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
1365 struct mpool *page_pool, bool clear)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001366{
1367 uint32_t orig_from_mode;
1368 uint32_t from_mode;
1369 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001370 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001371
Andrew Walbranca808b12020-05-15 17:22:28 +01001372 ret = ffa_relinquish_check_transition(
1373 from_locked, &orig_from_mode, fragments,
1374 fragment_constituent_counts, fragment_count, &from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001375 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001376 dlog_verbose("Invalid transition for relinquish.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +01001377 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001378 }
1379
1380 /*
1381 * Create a local pool so any freed memory can't be used by another
1382 * thread. This is to ensure the original mapping can be restored if the
1383 * clear fails.
1384 */
1385 mpool_init_with_fallback(&local_page_pool, page_pool);
1386
1387 /*
1388 * First reserve all required memory for the new page table entries
1389 * without committing, to make sure the entire operation will succeed
1390 * without exhausting the page pool.
1391 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001392 if (!ffa_region_group_identity_map(
1393 from_locked, fragments, fragment_constituent_counts,
1394 fragment_count, from_mode, page_pool, false)) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001395 /* TODO: partial defrag of failed range. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001396 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001397 goto out;
1398 }
1399
1400 /*
1401 * Update the mapping for the sender. This won't allocate because the
1402 * transaction was already prepared above, but may free pages in the
1403 * case that a whole block is being unmapped that was previously
1404 * partially mapped.
1405 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001406 CHECK(ffa_region_group_identity_map(
1407 from_locked, fragments, fragment_constituent_counts,
1408 fragment_count, from_mode, &local_page_pool, true));
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001409
1410 /* Clear the memory so no VM or device can see the previous contents. */
J-Alves7db32002021-12-14 14:44:50 +00001411 if (clear &&
J-Alves26483382023-04-20 12:01:49 +01001412 !ffa_clear_memory_constituents(orig_from_mode, fragments,
1413 fragment_constituent_counts,
1414 fragment_count, page_pool)) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001415 /*
1416 * On failure, roll back by returning memory to the sender. This
1417 * may allocate pages which were previously freed into
1418 * `local_page_pool` by the call above, but will never allocate
1419 * more pages than that so can never fail.
1420 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001421 CHECK(ffa_region_group_identity_map(
Andrew Walbranca808b12020-05-15 17:22:28 +01001422 from_locked, fragments, fragment_constituent_counts,
1423 fragment_count, orig_from_mode, &local_page_pool,
1424 true));
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001425
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001426 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001427 goto out;
1428 }
1429
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001430 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001431
1432out:
1433 mpool_fini(&local_page_pool);
1434
1435 /*
1436 * Tidy up the page table by reclaiming failed mappings (if there was an
1437 * error) or merging entries into blocks where possible (on success).
1438 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001439 vm_ptable_defrag(from_locked, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +01001440
1441 return ret;
1442}
1443
1444/**
Andrew Walbranca808b12020-05-15 17:22:28 +01001445 * Complete a memory sending operation by checking that it is valid, updating
1446 * the sender page table, and then either marking the share state as having
1447 * completed sending (on success) or freeing it (on failure).
1448 *
1449 * Returns FFA_SUCCESS with the handle encoded, or the relevant FFA_ERROR.
1450 */
J-Alvesfdd29272022-07-19 13:16:31 +01001451struct ffa_value ffa_memory_send_complete(
Andrew Walbranca808b12020-05-15 17:22:28 +01001452 struct vm_locked from_locked, struct share_states_locked share_states,
Andrew Walbran37c574e2020-06-03 11:45:46 +01001453 struct ffa_memory_share_state *share_state, struct mpool *page_pool,
1454 uint32_t *orig_from_mode_ret)
Andrew Walbranca808b12020-05-15 17:22:28 +01001455{
1456 struct ffa_memory_region *memory_region = share_state->memory_region;
J-Alves8f11cde2022-12-21 16:18:22 +00001457 struct ffa_composite_memory_region *composite;
Andrew Walbranca808b12020-05-15 17:22:28 +01001458 struct ffa_value ret;
1459
1460 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +00001461 assert(share_states.share_states != NULL);
J-Alves8f11cde2022-12-21 16:18:22 +00001462 assert(memory_region != NULL);
1463 composite = ffa_memory_region_get_composite(memory_region, 0);
1464 assert(composite != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +01001465
1466 /* Check that state is valid in sender page table and update. */
1467 ret = ffa_send_check_update(
1468 from_locked, share_state->fragments,
1469 share_state->fragment_constituent_counts,
J-Alves8f11cde2022-12-21 16:18:22 +00001470 share_state->fragment_count, composite->page_count,
1471 share_state->share_func, memory_region->receivers,
1472 memory_region->receiver_count, page_pool,
1473 memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR,
Andrew Walbran37c574e2020-06-03 11:45:46 +01001474 orig_from_mode_ret);
Andrew Walbranca808b12020-05-15 17:22:28 +01001475 if (ret.func != FFA_SUCCESS_32) {
1476 /*
1477 * Free share state, it failed to send so it can't be retrieved.
1478 */
Karl Meakin4cec5e82023-06-30 16:30:22 +01001479 dlog_verbose("%s: failed to send check update: %s(%s)\n",
1480 __func__, ffa_func_name(ret.func),
1481 ffa_error_name(ffa_error_code(ret)));
Andrew Walbranca808b12020-05-15 17:22:28 +01001482 share_state_free(share_states, share_state, page_pool);
1483 return ret;
1484 }
1485
1486 share_state->sending_complete = true;
Karl Meakin4cec5e82023-06-30 16:30:22 +01001487 dlog_verbose("%s: marked sending complete.\n", __func__);
Andrew Walbranca808b12020-05-15 17:22:28 +01001488
J-Alvesee68c542020-10-29 17:48:20 +00001489 return ffa_mem_success(share_state->memory_region->handle);
Andrew Walbranca808b12020-05-15 17:22:28 +01001490}
1491
1492/**
Federico Recanatia98603a2021-12-20 18:04:03 +01001493 * Check that the memory attributes match Hafnium expectations:
1494 * Normal Memory, Inner shareable, Write-Back Read-Allocate
1495 * Write-Allocate Cacheable.
1496 */
1497static struct ffa_value ffa_memory_attributes_validate(
J-Alves7a99d0d2023-02-08 13:49:48 +00001498 ffa_memory_attributes_t attributes)
Federico Recanatia98603a2021-12-20 18:04:03 +01001499{
1500 enum ffa_memory_type memory_type;
1501 enum ffa_memory_cacheability cacheability;
1502 enum ffa_memory_shareability shareability;
1503
1504 memory_type = ffa_get_memory_type_attr(attributes);
1505 if (memory_type != FFA_MEMORY_NORMAL_MEM) {
1506 dlog_verbose("Invalid memory type %#x, expected %#x.\n",
1507 memory_type, FFA_MEMORY_NORMAL_MEM);
Federico Recanati3d953f32022-02-17 09:31:29 +01001508 return ffa_error(FFA_DENIED);
Federico Recanatia98603a2021-12-20 18:04:03 +01001509 }
1510
1511 cacheability = ffa_get_memory_cacheability_attr(attributes);
1512 if (cacheability != FFA_MEMORY_CACHE_WRITE_BACK) {
1513 dlog_verbose("Invalid cacheability %#x, expected %#x.\n",
1514 cacheability, FFA_MEMORY_CACHE_WRITE_BACK);
Federico Recanati3d953f32022-02-17 09:31:29 +01001515 return ffa_error(FFA_DENIED);
Federico Recanatia98603a2021-12-20 18:04:03 +01001516 }
1517
1518 shareability = ffa_get_memory_shareability_attr(attributes);
1519 if (shareability != FFA_MEMORY_INNER_SHAREABLE) {
1520 dlog_verbose("Invalid shareability %#x, expected #%x.\n",
1521 shareability, FFA_MEMORY_INNER_SHAREABLE);
Federico Recanati3d953f32022-02-17 09:31:29 +01001522 return ffa_error(FFA_DENIED);
Federico Recanatia98603a2021-12-20 18:04:03 +01001523 }
1524
1525 return (struct ffa_value){.func = FFA_SUCCESS_32};
1526}
1527
1528/**
Andrew Walbrana65a1322020-04-06 19:32:32 +01001529 * Check that the given `memory_region` represents a valid memory send request
1530 * of the given `share_func` type, return the clear flag and permissions via the
1531 * respective output parameters, and update the permissions if necessary.
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001532 *
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001533 * Returns FFA_SUCCESS if the request was valid, or the relevant FFA_ERROR if
Andrew Walbrana65a1322020-04-06 19:32:32 +01001534 * not.
1535 */
J-Alves66652252022-07-06 09:49:51 +01001536struct ffa_value ffa_memory_send_validate(
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001537 struct vm_locked from_locked, struct ffa_memory_region *memory_region,
1538 uint32_t memory_share_length, uint32_t fragment_length,
J-Alves363f5722022-04-25 17:37:37 +01001539 uint32_t share_func)
Andrew Walbrana65a1322020-04-06 19:32:32 +01001540{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001541 struct ffa_composite_memory_region *composite;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001542 struct ffa_memory_access *receiver =
1543 ffa_memory_region_get_receiver(memory_region, 0);
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001544 uint64_t receivers_end;
1545 uint64_t min_length;
Federico Recanati872cd692022-01-05 13:10:10 +01001546 uint32_t composite_memory_region_offset;
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001547 uint32_t constituents_start;
Andrew Walbran130a8ae2020-05-15 16:27:15 +01001548 uint32_t constituents_length;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001549 enum ffa_data_access data_access;
1550 enum ffa_instruction_access instruction_access;
Olivier Deprez4342a3c2022-02-28 09:37:25 +01001551 enum ffa_memory_security security_state;
Federico Recanatia98603a2021-12-20 18:04:03 +01001552 struct ffa_value ret;
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001553 const size_t minimum_first_fragment_length =
1554 (sizeof(struct ffa_memory_region) +
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001555 memory_region->memory_access_desc_size +
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001556 sizeof(struct ffa_composite_memory_region));
1557
1558 if (fragment_length < minimum_first_fragment_length) {
1559 dlog_verbose("Fragment length %u too short (min %u).\n",
1560 (size_t)fragment_length,
1561 minimum_first_fragment_length);
1562 return ffa_error(FFA_INVALID_PARAMETERS);
1563 }
1564
Demi Marie Obenour73a1e942023-02-04 14:09:18 -05001565 static_assert(sizeof(struct ffa_memory_region_constituent) == 16,
1566 "struct ffa_memory_region_constituent must be 16 bytes");
1567 if (!is_aligned(fragment_length,
1568 sizeof(struct ffa_memory_region_constituent)) ||
1569 !is_aligned(memory_share_length,
1570 sizeof(struct ffa_memory_region_constituent))) {
1571 dlog_verbose(
1572 "Fragment length %u or total length %u"
1573 " is not 16-byte aligned.\n",
1574 fragment_length, memory_share_length);
1575 return ffa_error(FFA_INVALID_PARAMETERS);
1576 }
1577
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001578 if (fragment_length > memory_share_length) {
1579 dlog_verbose(
1580 "Fragment length %u greater than total length %u.\n",
1581 (size_t)fragment_length, (size_t)memory_share_length);
1582 return ffa_error(FFA_INVALID_PARAMETERS);
1583 }
Andrew Walbrana65a1322020-04-06 19:32:32 +01001584
J-Alves0b6653d2022-04-22 13:17:38 +01001585 assert(memory_region->receivers_offset ==
1586 offsetof(struct ffa_memory_region, receivers));
J-Alves0b6653d2022-04-22 13:17:38 +01001587
J-Alves95df0ef2022-12-07 10:09:48 +00001588 /* The sender must match the caller. */
1589 if ((!vm_id_is_current_world(from_locked.vm->id) &&
1590 vm_id_is_current_world(memory_region->sender)) ||
1591 (vm_id_is_current_world(from_locked.vm->id) &&
1592 memory_region->sender != from_locked.vm->id)) {
1593 dlog_verbose("Invalid memory sender ID.\n");
1594 return ffa_error(FFA_DENIED);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001595 }
1596
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001597 if (memory_region->receiver_count <= 0) {
1598 dlog_verbose("No receivers!\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001599 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001600 }
1601
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001602 /*
1603 * Ensure that the composite header is within the memory bounds and
1604 * doesn't overlap the first part of the message. Cast to uint64_t
1605 * to prevent overflow.
1606 */
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001607 receivers_end = ((uint64_t)memory_region->memory_access_desc_size *
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001608 (uint64_t)memory_region->receiver_count) +
1609 sizeof(struct ffa_memory_region);
1610 min_length = receivers_end +
1611 sizeof(struct ffa_composite_memory_region) +
1612 sizeof(struct ffa_memory_region_constituent);
1613 if (min_length > memory_share_length) {
1614 dlog_verbose("Share too short: got %u but minimum is %u.\n",
1615 (size_t)memory_share_length, (size_t)min_length);
1616 return ffa_error(FFA_INVALID_PARAMETERS);
1617 }
1618
1619 composite_memory_region_offset =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001620 receiver->composite_memory_region_offset;
Andrew Walbrana65a1322020-04-06 19:32:32 +01001621
1622 /*
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001623 * Check that the composite memory region descriptor is after the access
1624 * descriptors, is at least 16-byte aligned, and fits in the first
1625 * fragment.
Andrew Walbrana65a1322020-04-06 19:32:32 +01001626 */
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001627 if ((composite_memory_region_offset < receivers_end) ||
1628 (composite_memory_region_offset % 16 != 0) ||
1629 (composite_memory_region_offset >
1630 fragment_length - sizeof(struct ffa_composite_memory_region))) {
1631 dlog_verbose(
1632 "Invalid composite memory region descriptor offset "
1633 "%u.\n",
1634 (size_t)composite_memory_region_offset);
1635 return ffa_error(FFA_INVALID_PARAMETERS);
1636 }
1637
1638 /*
1639 * Compute the start of the constituent regions. Already checked
1640 * to be not more than fragment_length and thus not more than
1641 * memory_share_length.
1642 */
1643 constituents_start = composite_memory_region_offset +
1644 sizeof(struct ffa_composite_memory_region);
1645 constituents_length = memory_share_length - constituents_start;
1646
1647 /*
1648 * Check that the number of constituents is consistent with the length
1649 * of the constituent region.
1650 */
1651 composite = ffa_memory_region_get_composite(memory_region, 0);
1652 if ((constituents_length %
1653 sizeof(struct ffa_memory_region_constituent) !=
1654 0) ||
1655 ((constituents_length /
1656 sizeof(struct ffa_memory_region_constituent)) !=
1657 composite->constituent_count)) {
1658 dlog_verbose("Invalid length %u or composite offset %u.\n",
1659 (size_t)memory_share_length,
1660 (size_t)composite_memory_region_offset);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001661 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001662 }
Andrew Walbranca808b12020-05-15 17:22:28 +01001663 if (fragment_length < memory_share_length &&
1664 fragment_length < HF_MAILBOX_SIZE) {
1665 dlog_warning(
1666 "Initial fragment length %d smaller than mailbox "
1667 "size.\n",
1668 fragment_length);
1669 }
Andrew Walbrana65a1322020-04-06 19:32:32 +01001670
Andrew Walbrana65a1322020-04-06 19:32:32 +01001671 /*
1672 * Clear is not allowed for memory sharing, as the sender still has
1673 * access to the memory.
1674 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001675 if ((memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR) &&
1676 share_func == FFA_MEM_SHARE_32) {
Andrew Walbrana65a1322020-04-06 19:32:32 +01001677 dlog_verbose("Memory can't be cleared while being shared.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001678 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001679 }
1680
1681 /* No other flags are allowed/supported here. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001682 if (memory_region->flags & ~FFA_MEMORY_REGION_FLAG_CLEAR) {
Andrew Walbrana65a1322020-04-06 19:32:32 +01001683 dlog_verbose("Invalid flags %#x.\n", memory_region->flags);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001684 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001685 }
1686
J-Alves363f5722022-04-25 17:37:37 +01001687 /* Check that the permissions are valid, for each specified receiver. */
1688 for (uint32_t i = 0U; i < memory_region->receiver_count; i++) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001689 struct ffa_memory_region_attributes receiver_permissions;
1690
1691 receiver = ffa_memory_region_get_receiver(memory_region, i);
1692 assert(receiver != NULL);
1693 receiver_permissions = receiver->receiver_permissions;
J-Alves363f5722022-04-25 17:37:37 +01001694 ffa_memory_access_permissions_t permissions =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001695 receiver_permissions.permissions;
1696 ffa_id_t receiver_id = receiver_permissions.receiver;
J-Alves363f5722022-04-25 17:37:37 +01001697
1698 if (memory_region->sender == receiver_id) {
1699 dlog_verbose("Can't share memory with itself.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001700 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001701 }
Federico Recanati85090c42021-12-15 13:17:54 +01001702
J-Alves363f5722022-04-25 17:37:37 +01001703 for (uint32_t j = i + 1; j < memory_region->receiver_count;
1704 j++) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001705 struct ffa_memory_access *other_receiver =
1706 ffa_memory_region_get_receiver(memory_region,
1707 j);
1708 assert(other_receiver != NULL);
1709
J-Alves363f5722022-04-25 17:37:37 +01001710 if (receiver_id ==
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001711 other_receiver->receiver_permissions.receiver) {
J-Alves363f5722022-04-25 17:37:37 +01001712 dlog_verbose(
1713 "Repeated receiver(%x) in memory send "
1714 "operation.\n",
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001715 other_receiver->receiver_permissions
1716 .receiver);
J-Alves363f5722022-04-25 17:37:37 +01001717 return ffa_error(FFA_INVALID_PARAMETERS);
1718 }
1719 }
1720
1721 if (composite_memory_region_offset !=
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001722 receiver->composite_memory_region_offset) {
J-Alves363f5722022-04-25 17:37:37 +01001723 dlog_verbose(
1724 "All ffa_memory_access should point to the "
1725 "same composite memory region offset.\n");
1726 return ffa_error(FFA_INVALID_PARAMETERS);
1727 }
1728
1729 data_access = ffa_get_data_access_attr(permissions);
1730 instruction_access =
1731 ffa_get_instruction_access_attr(permissions);
1732 if (data_access == FFA_DATA_ACCESS_RESERVED ||
1733 instruction_access == FFA_INSTRUCTION_ACCESS_RESERVED) {
1734 dlog_verbose(
1735 "Reserved value for receiver permissions "
1736 "%#x.\n",
1737 permissions);
1738 return ffa_error(FFA_INVALID_PARAMETERS);
1739 }
1740 if (instruction_access !=
1741 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
1742 dlog_verbose(
1743 "Invalid instruction access permissions %#x "
1744 "for sending memory.\n",
1745 permissions);
1746 return ffa_error(FFA_INVALID_PARAMETERS);
1747 }
1748 if (share_func == FFA_MEM_SHARE_32) {
1749 if (data_access == FFA_DATA_ACCESS_NOT_SPECIFIED) {
1750 dlog_verbose(
1751 "Invalid data access permissions %#x "
1752 "for sharing memory.\n",
1753 permissions);
1754 return ffa_error(FFA_INVALID_PARAMETERS);
1755 }
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001756 /*
1757 * According to section 10.10.3 of the FF-A v1.1 EAC0
1758 * spec, NX is required for share operations (but must
1759 * not be specified by the sender) so set it in the
1760 * copy that we store, ready to be returned to the
1761 * retriever.
1762 */
1763 if (vm_id_is_current_world(receiver_id)) {
1764 ffa_set_instruction_access_attr(
1765 &permissions,
1766 FFA_INSTRUCTION_ACCESS_NX);
1767 receiver_permissions.permissions = permissions;
1768 }
J-Alves363f5722022-04-25 17:37:37 +01001769 }
1770 if (share_func == FFA_MEM_LEND_32 &&
1771 data_access == FFA_DATA_ACCESS_NOT_SPECIFIED) {
1772 dlog_verbose(
1773 "Invalid data access permissions %#x for "
1774 "lending memory.\n",
1775 permissions);
1776 return ffa_error(FFA_INVALID_PARAMETERS);
1777 }
1778
1779 if (share_func == FFA_MEM_DONATE_32 &&
1780 data_access != FFA_DATA_ACCESS_NOT_SPECIFIED) {
1781 dlog_verbose(
1782 "Invalid data access permissions %#x for "
1783 "donating memory.\n",
1784 permissions);
1785 return ffa_error(FFA_INVALID_PARAMETERS);
1786 }
Andrew Walbrana65a1322020-04-06 19:32:32 +01001787 }
1788
Olivier Deprez4342a3c2022-02-28 09:37:25 +01001789 /* Memory region attributes NS-Bit MBZ for FFA_MEM_SHARE/LEND/DONATE. */
1790 security_state =
1791 ffa_get_memory_security_attr(memory_region->attributes);
1792 if (security_state != FFA_MEMORY_SECURITY_UNSPECIFIED) {
1793 dlog_verbose(
1794 "Invalid security state for memory share operation.\n");
1795 return ffa_error(FFA_INVALID_PARAMETERS);
1796 }
1797
Federico Recanatid937f5e2021-12-20 17:38:23 +01001798 /*
J-Alves807794e2022-06-16 13:42:47 +01001799 * If a memory donate or lend with single borrower, the memory type
1800 * shall not be specified by the sender.
Federico Recanatid937f5e2021-12-20 17:38:23 +01001801 */
J-Alves807794e2022-06-16 13:42:47 +01001802 if (share_func == FFA_MEM_DONATE_32 ||
1803 (share_func == FFA_MEM_LEND_32 &&
1804 memory_region->receiver_count == 1)) {
1805 if (ffa_get_memory_type_attr(memory_region->attributes) !=
1806 FFA_MEMORY_NOT_SPECIFIED_MEM) {
1807 dlog_verbose(
1808 "Memory type shall not be specified by "
1809 "sender.\n");
1810 return ffa_error(FFA_INVALID_PARAMETERS);
1811 }
1812 } else {
1813 /*
1814 * Check that sender's memory attributes match Hafnium
1815 * expectations: Normal Memory, Inner shareable, Write-Back
1816 * Read-Allocate Write-Allocate Cacheable.
1817 */
1818 ret = ffa_memory_attributes_validate(memory_region->attributes);
1819 if (ret.func != FFA_SUCCESS_32) {
1820 return ret;
1821 }
Federico Recanatid937f5e2021-12-20 17:38:23 +01001822 }
1823
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001824 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbrana65a1322020-04-06 19:32:32 +01001825}
1826
1827/**
Andrew Walbranca808b12020-05-15 17:22:28 +01001828 * Gets the share state for continuing an operation to donate, lend or share
1829 * memory, and checks that it is a valid request.
1830 *
1831 * Returns FFA_SUCCESS if the request was valid, or the relevant FFA_ERROR if
1832 * not.
1833 */
J-Alvesfdd29272022-07-19 13:16:31 +01001834struct ffa_value ffa_memory_send_continue_validate(
Andrew Walbranca808b12020-05-15 17:22:28 +01001835 struct share_states_locked share_states, ffa_memory_handle_t handle,
J-Alves19e20cf2023-08-02 12:48:55 +01001836 struct ffa_memory_share_state **share_state_ret, ffa_id_t from_vm_id,
Andrew Walbranca808b12020-05-15 17:22:28 +01001837 struct mpool *page_pool)
1838{
1839 struct ffa_memory_share_state *share_state;
1840 struct ffa_memory_region *memory_region;
1841
Daniel Boulbya2f8c662021-11-26 17:52:53 +00001842 assert(share_state_ret != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +01001843
1844 /*
1845 * Look up the share state by handle and make sure that the VM ID
1846 * matches.
1847 */
Karl Meakin4a2854a2023-06-30 16:26:52 +01001848 share_state = get_share_state(share_states, handle);
J-Alvesb56aac82023-11-10 09:44:43 +00001849 if (share_state == NULL) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001850 dlog_verbose(
1851 "Invalid handle %#x for memory send continuation.\n",
1852 handle);
1853 return ffa_error(FFA_INVALID_PARAMETERS);
1854 }
1855 memory_region = share_state->memory_region;
1856
J-Alvesfdd29272022-07-19 13:16:31 +01001857 if (vm_id_is_current_world(from_vm_id) &&
1858 memory_region->sender != from_vm_id) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001859 dlog_verbose("Invalid sender %d.\n", memory_region->sender);
1860 return ffa_error(FFA_INVALID_PARAMETERS);
1861 }
1862
1863 if (share_state->sending_complete) {
1864 dlog_verbose(
1865 "Sending of memory handle %#x is already complete.\n",
1866 handle);
1867 return ffa_error(FFA_INVALID_PARAMETERS);
1868 }
1869
1870 if (share_state->fragment_count == MAX_FRAGMENTS) {
1871 /*
1872 * Log a warning as this is a sign that MAX_FRAGMENTS should
1873 * probably be increased.
1874 */
1875 dlog_warning(
1876 "Too many fragments for memory share with handle %#x; "
1877 "only %d supported.\n",
1878 handle, MAX_FRAGMENTS);
1879 /* Free share state, as it's not possible to complete it. */
1880 share_state_free(share_states, share_state, page_pool);
1881 return ffa_error(FFA_NO_MEMORY);
1882 }
1883
1884 *share_state_ret = share_state;
1885
1886 return (struct ffa_value){.func = FFA_SUCCESS_32};
1887}
1888
1889/**
J-Alves95df0ef2022-12-07 10:09:48 +00001890 * Checks if there is at least one receiver from the other world.
1891 */
J-Alvesfdd29272022-07-19 13:16:31 +01001892bool memory_region_receivers_from_other_world(
J-Alves95df0ef2022-12-07 10:09:48 +00001893 struct ffa_memory_region *memory_region)
1894{
1895 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00001896 struct ffa_memory_access *receiver =
1897 ffa_memory_region_get_receiver(memory_region, i);
1898 assert(receiver != NULL);
1899 ffa_id_t receiver_id = receiver->receiver_permissions.receiver;
1900
1901 if (!vm_id_is_current_world(receiver_id)) {
J-Alves95df0ef2022-12-07 10:09:48 +00001902 return true;
1903 }
1904 }
1905 return false;
1906}
1907
1908/**
J-Alves9da280b2022-12-21 14:55:39 +00001909 * Validates a call to donate, lend or share memory in which Hafnium is the
1910 * designated allocator of the memory handle. In practice, this also means
1911 * Hafnium is responsible for managing the state structures for the transaction.
1912 * If Hafnium is the SPMC, it should allocate the memory handle when either the
1913 * sender is an SP or there is at least one borrower that is an SP.
1914 * If Hafnium is the hypervisor, it should allocate the memory handle when
1915 * operation involves only NWd VMs.
1916 *
1917 * If validation goes well, Hafnium updates the stage-2 page tables of the
1918 * sender. Validation consists of checking if the message length and number of
1919 * memory region constituents match, and if the transition is valid for the
1920 * type of memory sending operation.
Andrew Walbran475c1452020-02-07 13:22:22 +00001921 *
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001922 * Assumes that the caller has already found and locked the sender VM and copied
1923 * the memory region descriptor from the sender's TX buffer to a freshly
1924 * allocated page from Hafnium's internal pool. The caller must have also
1925 * validated that the receiver VM ID is valid.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001926 *
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001927 * This function takes ownership of the `memory_region` passed in and will free
1928 * it when necessary; it must not be freed by the caller.
Jose Marinho09b1db82019-08-08 09:16:59 +01001929 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001930struct ffa_value ffa_memory_send(struct vm_locked from_locked,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001931 struct ffa_memory_region *memory_region,
Andrew Walbran130a8ae2020-05-15 16:27:15 +01001932 uint32_t memory_share_length,
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001933 uint32_t fragment_length, uint32_t share_func,
1934 struct mpool *page_pool)
Jose Marinho09b1db82019-08-08 09:16:59 +01001935{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001936 struct ffa_value ret;
Andrew Walbranca808b12020-05-15 17:22:28 +01001937 struct share_states_locked share_states;
1938 struct ffa_memory_share_state *share_state;
Jose Marinho09b1db82019-08-08 09:16:59 +01001939
1940 /*
Andrew Walbrana65a1322020-04-06 19:32:32 +01001941 * If there is an error validating the `memory_region` then we need to
1942 * free it because we own it but we won't be storing it in a share state
1943 * after all.
Jose Marinho09b1db82019-08-08 09:16:59 +01001944 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001945 ret = ffa_memory_send_validate(from_locked, memory_region,
1946 memory_share_length, fragment_length,
J-Alves363f5722022-04-25 17:37:37 +01001947 share_func);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001948 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001949 mpool_free(page_pool, memory_region);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001950 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01001951 }
1952
Andrew Walbrana65a1322020-04-06 19:32:32 +01001953 /* Set flag for share function, ready to be retrieved later. */
1954 switch (share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001955 case FFA_MEM_SHARE_32:
Andrew Walbrana65a1322020-04-06 19:32:32 +01001956 memory_region->flags |=
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001957 FFA_MEMORY_REGION_TRANSACTION_TYPE_SHARE;
Andrew Walbrana65a1322020-04-06 19:32:32 +01001958 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001959 case FFA_MEM_LEND_32:
1960 memory_region->flags |= FFA_MEMORY_REGION_TRANSACTION_TYPE_LEND;
Andrew Walbrana65a1322020-04-06 19:32:32 +01001961 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001962 case FFA_MEM_DONATE_32:
Andrew Walbrana65a1322020-04-06 19:32:32 +01001963 memory_region->flags |=
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001964 FFA_MEMORY_REGION_TRANSACTION_TYPE_DONATE;
Andrew Walbrana65a1322020-04-06 19:32:32 +01001965 break;
Jose Marinho09b1db82019-08-08 09:16:59 +01001966 }
1967
Andrew Walbranca808b12020-05-15 17:22:28 +01001968 share_states = share_states_lock();
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001969 /*
1970 * Allocate a share state before updating the page table. Otherwise if
1971 * updating the page table succeeded but allocating the share state
1972 * failed then it would leave the memory in a state where nobody could
1973 * get it back.
1974 */
Karl Meakin52cdfe72023-06-30 14:49:10 +01001975 share_state = allocate_share_state(share_states, share_func,
1976 memory_region, fragment_length,
1977 FFA_MEMORY_HANDLE_INVALID);
J-Alvesb56aac82023-11-10 09:44:43 +00001978 if (share_state == NULL) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001979 dlog_verbose("Failed to allocate share state.\n");
1980 mpool_free(page_pool, memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +01001981 ret = ffa_error(FFA_NO_MEMORY);
1982 goto out;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001983 }
1984
Andrew Walbranca808b12020-05-15 17:22:28 +01001985 if (fragment_length == memory_share_length) {
1986 /* No more fragments to come, everything fit in one message. */
J-Alves2a0d2882020-10-29 14:49:50 +00001987 ret = ffa_memory_send_complete(
1988 from_locked, share_states, share_state, page_pool,
1989 &(share_state->sender_orig_mode));
Andrew Walbranca808b12020-05-15 17:22:28 +01001990 } else {
J-Alvesfdd29272022-07-19 13:16:31 +01001991 /*
1992 * Use sender ID from 'memory_region' assuming
1993 * that at this point it has been validated:
1994 * - MBZ at virtual FF-A instance.
1995 */
J-Alves19e20cf2023-08-02 12:48:55 +01001996 ffa_id_t sender_to_ret =
J-Alvesfdd29272022-07-19 13:16:31 +01001997 (from_locked.vm->id == HF_OTHER_WORLD_ID)
1998 ? memory_region->sender
1999 : 0;
Andrew Walbranca808b12020-05-15 17:22:28 +01002000 ret = (struct ffa_value){
2001 .func = FFA_MEM_FRAG_RX_32,
J-Alvesee68c542020-10-29 17:48:20 +00002002 .arg1 = (uint32_t)memory_region->handle,
2003 .arg2 = (uint32_t)(memory_region->handle >> 32),
J-Alvesfdd29272022-07-19 13:16:31 +01002004 .arg3 = fragment_length,
2005 .arg4 = (uint32_t)(sender_to_ret & 0xffff) << 16};
Andrew Walbranca808b12020-05-15 17:22:28 +01002006 }
2007
2008out:
2009 share_states_unlock(&share_states);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002010 dump_share_states();
Andrew Walbranca808b12020-05-15 17:22:28 +01002011 return ret;
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002012}
2013
2014/**
J-Alves8505a8a2022-06-15 18:10:18 +01002015 * Continues an operation to donate, lend or share memory to a VM from current
2016 * world. If this is the last fragment then checks that the transition is valid
2017 * for the type of memory sending operation and updates the stage-2 page tables
2018 * of the sender.
Andrew Walbranca808b12020-05-15 17:22:28 +01002019 *
2020 * Assumes that the caller has already found and locked the sender VM and copied
2021 * the memory region descriptor from the sender's TX buffer to a freshly
2022 * allocated page from Hafnium's internal pool.
2023 *
2024 * This function takes ownership of the `fragment` passed in; it must not be
2025 * freed by the caller.
2026 */
2027struct ffa_value ffa_memory_send_continue(struct vm_locked from_locked,
2028 void *fragment,
2029 uint32_t fragment_length,
2030 ffa_memory_handle_t handle,
2031 struct mpool *page_pool)
2032{
2033 struct share_states_locked share_states = share_states_lock();
2034 struct ffa_memory_share_state *share_state;
2035 struct ffa_value ret;
2036 struct ffa_memory_region *memory_region;
2037
Demi Marie Obenour73a1e942023-02-04 14:09:18 -05002038 CHECK(is_aligned(fragment,
2039 alignof(struct ffa_memory_region_constituent)));
2040 if (fragment_length % sizeof(struct ffa_memory_region_constituent) !=
2041 0) {
2042 dlog_verbose("Fragment length %u misaligned.\n",
2043 fragment_length);
2044 ret = ffa_error(FFA_INVALID_PARAMETERS);
2045 goto out_free_fragment;
2046 }
2047
Andrew Walbranca808b12020-05-15 17:22:28 +01002048 ret = ffa_memory_send_continue_validate(share_states, handle,
2049 &share_state,
2050 from_locked.vm->id, page_pool);
2051 if (ret.func != FFA_SUCCESS_32) {
2052 goto out_free_fragment;
2053 }
2054 memory_region = share_state->memory_region;
2055
J-Alves95df0ef2022-12-07 10:09:48 +00002056 if (memory_region_receivers_from_other_world(memory_region)) {
Andrew Walbranca808b12020-05-15 17:22:28 +01002057 dlog_error(
2058 "Got hypervisor-allocated handle for memory send to "
J-Alves8505a8a2022-06-15 18:10:18 +01002059 "other world. This should never happen, and indicates "
2060 "a bug in "
Andrew Walbranca808b12020-05-15 17:22:28 +01002061 "EL3 code.\n");
2062 ret = ffa_error(FFA_INVALID_PARAMETERS);
2063 goto out_free_fragment;
2064 }
2065
2066 /* Add this fragment. */
2067 share_state->fragments[share_state->fragment_count] = fragment;
2068 share_state->fragment_constituent_counts[share_state->fragment_count] =
2069 fragment_length / sizeof(struct ffa_memory_region_constituent);
2070 share_state->fragment_count++;
2071
2072 /* Check whether the memory send operation is now ready to complete. */
2073 if (share_state_sending_complete(share_states, share_state)) {
J-Alves2a0d2882020-10-29 14:49:50 +00002074 ret = ffa_memory_send_complete(
2075 from_locked, share_states, share_state, page_pool,
2076 &(share_state->sender_orig_mode));
Andrew Walbranca808b12020-05-15 17:22:28 +01002077 } else {
2078 ret = (struct ffa_value){
2079 .func = FFA_MEM_FRAG_RX_32,
2080 .arg1 = (uint32_t)handle,
2081 .arg2 = (uint32_t)(handle >> 32),
2082 .arg3 = share_state_next_fragment_offset(share_states,
2083 share_state)};
2084 }
2085 goto out;
2086
2087out_free_fragment:
2088 mpool_free(page_pool, fragment);
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002089
2090out:
Andrew Walbranca808b12020-05-15 17:22:28 +01002091 share_states_unlock(&share_states);
Andrew Walbran1a86aa92020-05-15 17:22:28 +01002092 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002093}
2094
Andrew Walbranca808b12020-05-15 17:22:28 +01002095/** Clean up after the receiver has finished retrieving a memory region. */
2096static void ffa_memory_retrieve_complete(
2097 struct share_states_locked share_states,
2098 struct ffa_memory_share_state *share_state, struct mpool *page_pool)
2099{
2100 if (share_state->share_func == FFA_MEM_DONATE_32) {
2101 /*
2102 * Memory that has been donated can't be relinquished,
2103 * so no need to keep the share state around.
2104 */
2105 share_state_free(share_states, share_state, page_pool);
2106 dlog_verbose("Freed share state for donate.\n");
2107 }
2108}
2109
J-Alves2d8457f2022-10-05 11:06:41 +01002110/**
2111 * Initialises the given memory region descriptor to be used for an
2112 * `FFA_MEM_RETRIEVE_RESP`, including the given constituents for the first
2113 * fragment.
2114 * The memory region descriptor is initialized according to retriever's
2115 * FF-A version.
2116 *
2117 * Returns true on success, or false if the given constituents won't all fit in
2118 * the first fragment.
2119 */
2120static bool ffa_retrieved_memory_region_init(
2121 void *response, uint32_t ffa_version, size_t response_max_size,
J-Alves19e20cf2023-08-02 12:48:55 +01002122 ffa_id_t sender, ffa_memory_attributes_t attributes,
J-Alves2d8457f2022-10-05 11:06:41 +01002123 ffa_memory_region_flags_t flags, ffa_memory_handle_t handle,
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002124 ffa_id_t receiver_id, uint32_t memory_access_desc_size,
2125 ffa_memory_access_permissions_t permissions, uint32_t page_count,
2126 uint32_t total_constituent_count,
J-Alves2d8457f2022-10-05 11:06:41 +01002127 const struct ffa_memory_region_constituent constituents[],
2128 uint32_t fragment_constituent_count, uint32_t *total_length,
2129 uint32_t *fragment_length)
2130{
2131 struct ffa_composite_memory_region *composite_memory_region;
J-Alves2d8457f2022-10-05 11:06:41 +01002132 uint32_t i;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002133 uint32_t composite_offset;
J-Alves2d8457f2022-10-05 11:06:41 +01002134 uint32_t constituents_offset;
2135 uint32_t receiver_count;
2136
2137 assert(response != NULL);
2138
2139 if (ffa_version == MAKE_FFA_VERSION(1, 0)) {
2140 struct ffa_memory_region_v1_0 *retrieve_response =
2141 (struct ffa_memory_region_v1_0 *)response;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002142 struct ffa_memory_access_v1_0 *receiver;
J-Alves2d8457f2022-10-05 11:06:41 +01002143
J-Alves5da37d92022-10-24 16:33:48 +01002144 ffa_memory_region_init_header_v1_0(
2145 retrieve_response, sender, attributes, flags, handle, 0,
2146 RECEIVERS_COUNT_IN_RETRIEVE_RESP);
J-Alves2d8457f2022-10-05 11:06:41 +01002147
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002148 receiver = (struct ffa_memory_access_v1_0 *)
2149 retrieve_response->receivers;
J-Alves2d8457f2022-10-05 11:06:41 +01002150 receiver_count = retrieve_response->receiver_count;
2151
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002152 /*
2153 * Initialized here as in memory retrieve responses we currently
2154 * expect one borrower to be specified.
2155 */
2156 ffa_memory_access_v1_0_init_permissions(
2157 receiver, receiver_id,
2158 ffa_get_data_access_attr(permissions),
2159 ffa_get_instruction_access_attr(permissions), flags);
2160
2161 composite_offset =
J-Alves2d8457f2022-10-05 11:06:41 +01002162 sizeof(struct ffa_memory_region_v1_0) +
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002163 receiver_count * sizeof(struct ffa_memory_access_v1_0);
2164 receiver->composite_memory_region_offset = composite_offset;
J-Alves2d8457f2022-10-05 11:06:41 +01002165
2166 composite_memory_region = ffa_memory_region_get_composite_v1_0(
2167 retrieve_response, 0);
2168 } else {
J-Alves2d8457f2022-10-05 11:06:41 +01002169 struct ffa_memory_region *retrieve_response =
2170 (struct ffa_memory_region *)response;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002171 struct ffa_memory_access *receiver;
J-Alves2d8457f2022-10-05 11:06:41 +01002172
2173 ffa_memory_region_init_header(retrieve_response, sender,
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002174 attributes, flags, handle, 0, 1,
2175 memory_access_desc_size);
J-Alves2d8457f2022-10-05 11:06:41 +01002176
J-Alves2d8457f2022-10-05 11:06:41 +01002177 receiver_count = retrieve_response->receiver_count;
2178
2179 /*
2180 * Note that `sizeof(struct_ffa_memory_region)` and
2181 * `sizeof(struct ffa_memory_access)` must both be multiples of
2182 * 16 (as verified by the asserts in `ffa_memory.c`, so it is
2183 * guaranteed that the offset we calculate here is aligned to a
2184 * 64-bit boundary and so 64-bit values can be copied without
2185 * alignment faults.
2186 */
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002187 composite_offset =
J-Alves2d8457f2022-10-05 11:06:41 +01002188 sizeof(struct ffa_memory_region) +
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002189 (uint32_t)(receiver_count *
2190 retrieve_response->memory_access_desc_size);
J-Alves2d8457f2022-10-05 11:06:41 +01002191
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002192 receiver = ffa_memory_region_get_receiver(retrieve_response, 0);
2193 assert(receiver != NULL);
2194
2195 /*
2196 * Initialized here as in memory retrieve responses we currently
2197 * expect one borrower to be specified.
2198 */
2199 ffa_memory_access_init_permissions(
2200 receiver, receiver_id,
2201 ffa_get_data_access_attr(permissions),
2202 ffa_get_instruction_access_attr(permissions), flags);
2203 receiver->composite_memory_region_offset = composite_offset;
J-Alves2d8457f2022-10-05 11:06:41 +01002204 composite_memory_region =
2205 ffa_memory_region_get_composite(retrieve_response, 0);
2206 }
2207
J-Alves2d8457f2022-10-05 11:06:41 +01002208 assert(composite_memory_region != NULL);
2209
J-Alves2d8457f2022-10-05 11:06:41 +01002210 composite_memory_region->page_count = page_count;
2211 composite_memory_region->constituent_count = total_constituent_count;
2212 composite_memory_region->reserved_0 = 0;
2213
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002214 constituents_offset =
2215 composite_offset + sizeof(struct ffa_composite_memory_region);
J-Alves2d8457f2022-10-05 11:06:41 +01002216 if (constituents_offset +
2217 fragment_constituent_count *
2218 sizeof(struct ffa_memory_region_constituent) >
2219 response_max_size) {
2220 return false;
2221 }
2222
2223 for (i = 0; i < fragment_constituent_count; ++i) {
2224 composite_memory_region->constituents[i] = constituents[i];
2225 }
2226
2227 if (total_length != NULL) {
2228 *total_length =
2229 constituents_offset +
2230 composite_memory_region->constituent_count *
2231 sizeof(struct ffa_memory_region_constituent);
2232 }
2233 if (fragment_length != NULL) {
2234 *fragment_length =
2235 constituents_offset +
2236 fragment_constituent_count *
2237 sizeof(struct ffa_memory_region_constituent);
2238 }
2239
2240 return true;
2241}
2242
J-Alves96de29f2022-04-26 16:05:24 +01002243/*
2244 * Gets the receiver's access permissions from 'struct ffa_memory_region' and
2245 * returns its index in the receiver's array. If receiver's ID doesn't exist
2246 * in the array, return the region's 'receiver_count'.
2247 */
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002248uint32_t ffa_memory_region_get_receiver_index(
2249 struct ffa_memory_region *memory_region, ffa_id_t receiver_id)
J-Alves96de29f2022-04-26 16:05:24 +01002250{
J-Alves96de29f2022-04-26 16:05:24 +01002251 uint32_t i;
2252
2253 assert(memory_region != NULL);
2254
J-Alves96de29f2022-04-26 16:05:24 +01002255 for (i = 0U; i < memory_region->receiver_count; i++) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002256 struct ffa_memory_access *receiver =
2257 ffa_memory_region_get_receiver(memory_region, i);
2258 assert(receiver != NULL);
2259 if (receiver->receiver_permissions.receiver == receiver_id) {
J-Alves96de29f2022-04-26 16:05:24 +01002260 break;
2261 }
2262 }
2263
2264 return i;
2265}
2266
2267/**
2268 * Validates the retrieved permissions against those specified by the lender
2269 * of memory share operation. Optionally can help set the permissions to be used
2270 * for the S2 mapping, through the `permissions` argument.
J-Alvesdcad8992023-09-15 14:10:35 +01002271 * Returns FFA_SUCCESS if all the fields are valid. FFA_ERROR, with error code:
2272 * - FFA_INVALID_PARAMETERS -> if the fields have invalid values as per the
2273 * specification for each ABI.
2274 * - FFA_DENIED -> if the permissions specified by the retriever are not
2275 * less permissive than those provided by the sender.
J-Alves96de29f2022-04-26 16:05:24 +01002276 */
J-Alvesdcad8992023-09-15 14:10:35 +01002277static struct ffa_value ffa_memory_retrieve_is_memory_access_valid(
2278 uint32_t share_func, enum ffa_data_access sent_data_access,
J-Alves96de29f2022-04-26 16:05:24 +01002279 enum ffa_data_access requested_data_access,
2280 enum ffa_instruction_access sent_instruction_access,
2281 enum ffa_instruction_access requested_instruction_access,
J-Alvesdcad8992023-09-15 14:10:35 +01002282 ffa_memory_access_permissions_t *permissions, bool multiple_borrowers)
J-Alves96de29f2022-04-26 16:05:24 +01002283{
2284 switch (sent_data_access) {
2285 case FFA_DATA_ACCESS_NOT_SPECIFIED:
2286 case FFA_DATA_ACCESS_RW:
2287 if (requested_data_access == FFA_DATA_ACCESS_NOT_SPECIFIED ||
2288 requested_data_access == FFA_DATA_ACCESS_RW) {
2289 if (permissions != NULL) {
2290 ffa_set_data_access_attr(permissions,
2291 FFA_DATA_ACCESS_RW);
2292 }
2293 break;
2294 }
2295 /* Intentional fall-through. */
2296 case FFA_DATA_ACCESS_RO:
2297 if (requested_data_access == FFA_DATA_ACCESS_NOT_SPECIFIED ||
2298 requested_data_access == FFA_DATA_ACCESS_RO) {
2299 if (permissions != NULL) {
2300 ffa_set_data_access_attr(permissions,
2301 FFA_DATA_ACCESS_RO);
2302 }
2303 break;
2304 }
2305 dlog_verbose(
2306 "Invalid data access requested; sender specified "
2307 "permissions %#x but receiver requested %#x.\n",
2308 sent_data_access, requested_data_access);
J-Alvesdcad8992023-09-15 14:10:35 +01002309 return ffa_error(FFA_DENIED);
J-Alves96de29f2022-04-26 16:05:24 +01002310 case FFA_DATA_ACCESS_RESERVED:
2311 panic("Got unexpected FFA_DATA_ACCESS_RESERVED. Should be "
2312 "checked before this point.");
2313 }
2314
J-Alvesdcad8992023-09-15 14:10:35 +01002315 /*
2316 * For operations with a single borrower, If it is an FFA_MEMORY_LEND
2317 * or FFA_MEMORY_DONATE the retriever should have specifed the
2318 * instruction permissions it wishes to receive.
2319 */
2320 switch (share_func) {
2321 case FFA_MEM_SHARE_32:
2322 if (requested_instruction_access !=
2323 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
2324 dlog_verbose(
2325 "%s: for share instruction permissions must "
2326 "NOT be specified.\n",
2327 __func__);
2328 return ffa_error(FFA_INVALID_PARAMETERS);
2329 }
2330 break;
2331 case FFA_MEM_LEND_32:
2332 /*
2333 * For operations with multiple borrowers only permit XN
2334 * permissions, and both Sender and borrower should have used
2335 * FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED.
2336 */
2337 if (multiple_borrowers) {
2338 if (requested_instruction_access !=
2339 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
2340 dlog_verbose(
2341 "%s: lend/share/donate with multiple "
2342 "borrowers "
2343 "instruction permissions must NOT be "
2344 "specified.\n",
2345 __func__);
2346 return ffa_error(FFA_INVALID_PARAMETERS);
2347 }
2348 break;
2349 }
2350 /* Fall through if the operation targets a single borrower. */
2351 case FFA_MEM_DONATE_32:
2352 if (!multiple_borrowers &&
2353 requested_instruction_access ==
2354 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
2355 dlog_verbose(
2356 "%s: for lend/donate with single borrower "
2357 "instruction permissions must be speficified "
2358 "by borrower\n",
2359 __func__);
2360 return ffa_error(FFA_INVALID_PARAMETERS);
2361 }
2362 break;
2363 default:
2364 panic("%s: Wrong func id provided.\n", __func__);
2365 }
2366
J-Alves96de29f2022-04-26 16:05:24 +01002367 switch (sent_instruction_access) {
2368 case FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED:
2369 case FFA_INSTRUCTION_ACCESS_X:
J-Alvesdcad8992023-09-15 14:10:35 +01002370 if (requested_instruction_access == FFA_INSTRUCTION_ACCESS_X) {
J-Alves96de29f2022-04-26 16:05:24 +01002371 if (permissions != NULL) {
2372 ffa_set_instruction_access_attr(
2373 permissions, FFA_INSTRUCTION_ACCESS_X);
2374 }
2375 break;
2376 }
J-Alvesdcad8992023-09-15 14:10:35 +01002377 /*
2378 * Fall through if requested permissions are less
2379 * permissive than those provided by the sender.
2380 */
J-Alves96de29f2022-04-26 16:05:24 +01002381 case FFA_INSTRUCTION_ACCESS_NX:
2382 if (requested_instruction_access ==
2383 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED ||
2384 requested_instruction_access == FFA_INSTRUCTION_ACCESS_NX) {
2385 if (permissions != NULL) {
2386 ffa_set_instruction_access_attr(
2387 permissions, FFA_INSTRUCTION_ACCESS_NX);
2388 }
2389 break;
2390 }
2391 dlog_verbose(
2392 "Invalid instruction access requested; sender "
2393 "specified permissions %#x but receiver requested "
2394 "%#x.\n",
2395 sent_instruction_access, requested_instruction_access);
J-Alvesdcad8992023-09-15 14:10:35 +01002396 return ffa_error(FFA_DENIED);
J-Alves96de29f2022-04-26 16:05:24 +01002397 case FFA_INSTRUCTION_ACCESS_RESERVED:
2398 panic("Got unexpected FFA_INSTRUCTION_ACCESS_RESERVED. Should "
2399 "be checked before this point.");
2400 }
2401
J-Alvesdcad8992023-09-15 14:10:35 +01002402 return (struct ffa_value){.func = FFA_SUCCESS_32};
J-Alves96de29f2022-04-26 16:05:24 +01002403}
2404
2405/**
2406 * Validate the receivers' permissions in the retrieve request against those
2407 * specified by the lender.
2408 * In the `permissions` argument returns the permissions to set at S2 for the
2409 * caller to the FFA_MEMORY_RETRIEVE_REQ.
J-Alves3456e032023-07-20 12:20:05 +01002410 * The function looks into the flag to bypass multiple borrower checks:
2411 * - If not set returns FFA_SUCCESS if all specified permissions are valid.
2412 * - If set returns FFA_SUCCESS if the descriptor contains the permissions
2413 * to the caller of FFA_MEM_RETRIEVE_REQ and they are valid. Other permissions
2414 * are ignored, if provided.
J-Alves96de29f2022-04-26 16:05:24 +01002415 */
2416static struct ffa_value ffa_memory_retrieve_validate_memory_access_list(
2417 struct ffa_memory_region *memory_region,
J-Alves19e20cf2023-08-02 12:48:55 +01002418 struct ffa_memory_region *retrieve_request, ffa_id_t to_vm_id,
J-Alvesdcad8992023-09-15 14:10:35 +01002419 ffa_memory_access_permissions_t *permissions, uint32_t func_id)
J-Alves96de29f2022-04-26 16:05:24 +01002420{
2421 uint32_t retrieve_receiver_index;
J-Alves3456e032023-07-20 12:20:05 +01002422 bool bypass_multi_receiver_check =
2423 (retrieve_request->flags &
2424 FFA_MEMORY_REGION_FLAG_BYPASS_BORROWERS_CHECK) != 0U;
J-Alvesdcad8992023-09-15 14:10:35 +01002425 const uint32_t region_receiver_count = memory_region->receiver_count;
2426 struct ffa_value ret;
J-Alves96de29f2022-04-26 16:05:24 +01002427
2428 assert(permissions != NULL);
2429
J-Alves3456e032023-07-20 12:20:05 +01002430 if (!bypass_multi_receiver_check) {
J-Alvesdcad8992023-09-15 14:10:35 +01002431 if (retrieve_request->receiver_count != region_receiver_count) {
J-Alves3456e032023-07-20 12:20:05 +01002432 dlog_verbose(
2433 "Retrieve request should contain same list of "
2434 "borrowers, as specified by the lender.\n");
2435 return ffa_error(FFA_INVALID_PARAMETERS);
2436 }
2437 } else {
2438 if (retrieve_request->receiver_count != 1) {
2439 dlog_verbose(
2440 "Set bypass multiple borrower check, receiver "
2441 "list must be sized 1 (%x)\n",
2442 memory_region->receiver_count);
2443 return ffa_error(FFA_INVALID_PARAMETERS);
2444 }
J-Alves96de29f2022-04-26 16:05:24 +01002445 }
2446
2447 retrieve_receiver_index = retrieve_request->receiver_count;
2448
2449 /* Should be populated with the permissions of the retriever. */
2450 *permissions = 0;
2451
2452 for (uint32_t i = 0U; i < retrieve_request->receiver_count; i++) {
2453 ffa_memory_access_permissions_t sent_permissions;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002454 struct ffa_memory_access *retrieve_request_receiver =
2455 ffa_memory_region_get_receiver(retrieve_request, i);
2456 assert(retrieve_request_receiver != NULL);
J-Alves96de29f2022-04-26 16:05:24 +01002457 ffa_memory_access_permissions_t requested_permissions =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002458 retrieve_request_receiver->receiver_permissions
2459 .permissions;
J-Alves19e20cf2023-08-02 12:48:55 +01002460 ffa_id_t current_receiver_id =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002461 retrieve_request_receiver->receiver_permissions
2462 .receiver;
J-Alves96de29f2022-04-26 16:05:24 +01002463 bool found_to_id = current_receiver_id == to_vm_id;
2464
J-Alves3456e032023-07-20 12:20:05 +01002465 if (bypass_multi_receiver_check && !found_to_id) {
2466 dlog_verbose(
2467 "Bypass multiple borrower check for id %x.\n",
2468 current_receiver_id);
2469 continue;
2470 }
2471
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002472 if (retrieve_request_receiver->composite_memory_region_offset !=
2473 0U) {
2474 dlog_verbose(
2475 "Retriever specified address ranges not "
2476 "supported (got offset %d).\n",
2477 retrieve_request_receiver
2478 ->composite_memory_region_offset);
2479 return ffa_error(FFA_INVALID_PARAMETERS);
2480 }
2481
J-Alves96de29f2022-04-26 16:05:24 +01002482 /*
2483 * Find the current receiver in the transaction descriptor from
2484 * sender.
2485 */
2486 uint32_t mem_region_receiver_index =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002487 ffa_memory_region_get_receiver_index(
2488 memory_region, current_receiver_id);
J-Alves96de29f2022-04-26 16:05:24 +01002489
2490 if (mem_region_receiver_index ==
2491 memory_region->receiver_count) {
2492 dlog_verbose("%s: receiver %x not found\n", __func__,
2493 current_receiver_id);
2494 return ffa_error(FFA_DENIED);
2495 }
2496
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002497 struct ffa_memory_access *receiver =
2498 ffa_memory_region_get_receiver(
2499 memory_region, mem_region_receiver_index);
2500 assert(receiver != NULL);
2501
2502 sent_permissions = receiver->receiver_permissions.permissions;
J-Alves96de29f2022-04-26 16:05:24 +01002503
2504 if (found_to_id) {
2505 retrieve_receiver_index = i;
2506 }
2507
2508 /*
J-Alvesdcad8992023-09-15 14:10:35 +01002509 * Check if retrieve request memory access list is valid:
2510 * - The retrieve request complies with the specification.
2511 * - Permissions are within those specified by the sender.
J-Alves96de29f2022-04-26 16:05:24 +01002512 */
J-Alvesdcad8992023-09-15 14:10:35 +01002513 ret = ffa_memory_retrieve_is_memory_access_valid(
2514 func_id, ffa_get_data_access_attr(sent_permissions),
2515 ffa_get_data_access_attr(requested_permissions),
2516 ffa_get_instruction_access_attr(sent_permissions),
2517 ffa_get_instruction_access_attr(requested_permissions),
2518 found_to_id ? permissions : NULL,
2519 region_receiver_count > 1);
2520 if (ret.func != FFA_SUCCESS_32) {
2521 return ret;
J-Alves96de29f2022-04-26 16:05:24 +01002522 }
2523
2524 /*
2525 * Can't request PM to clear memory if only provided with RO
2526 * permissions.
2527 */
2528 if (found_to_id &&
2529 (ffa_get_data_access_attr(*permissions) ==
2530 FFA_DATA_ACCESS_RO) &&
2531 (retrieve_request->flags & FFA_MEMORY_REGION_FLAG_CLEAR) !=
2532 0U) {
2533 dlog_verbose(
2534 "Receiver has RO permissions can not request "
2535 "clear.\n");
2536 return ffa_error(FFA_DENIED);
2537 }
2538 }
2539
2540 if (retrieve_receiver_index == retrieve_request->receiver_count) {
2541 dlog_verbose(
2542 "Retrieve request does not contain caller's (%x) "
2543 "permissions\n",
2544 to_vm_id);
2545 return ffa_error(FFA_INVALID_PARAMETERS);
2546 }
2547
2548 return (struct ffa_value){.func = FFA_SUCCESS_32};
2549}
2550
J-Alvesa9cd7e32022-07-01 13:49:33 +01002551/*
2552 * According to section 16.4.3 of FF-A v1.1 EAC0 specification, the hypervisor
2553 * may issue an FFA_MEM_RETRIEVE_REQ to obtain the memory region description
2554 * of a pending memory sharing operation whose allocator is the SPM, for
2555 * validation purposes before forwarding an FFA_MEM_RECLAIM call. In doing so
2556 * the memory region descriptor of the retrieve request must be zeroed with the
2557 * exception of the sender ID and handle.
2558 */
2559bool is_ffa_memory_retrieve_borrower_request(struct ffa_memory_region *request,
2560 struct vm_locked to_locked)
2561{
2562 return to_locked.vm->id == HF_HYPERVISOR_VM_ID &&
2563 request->attributes == 0U && request->flags == 0U &&
2564 request->tag == 0U && request->receiver_count == 0U &&
2565 plat_ffa_memory_handle_allocated_by_current_world(
2566 request->handle);
2567}
2568
2569/*
2570 * Helper to reset count of fragments retrieved by the hypervisor.
2571 */
2572static void ffa_memory_retrieve_complete_from_hyp(
2573 struct ffa_memory_share_state *share_state)
2574{
2575 if (share_state->hypervisor_fragment_count ==
2576 share_state->fragment_count) {
2577 share_state->hypervisor_fragment_count = 0;
2578 }
2579}
2580
J-Alves089004f2022-07-13 14:25:44 +01002581/**
2582 * Validate that the memory region descriptor provided by the borrower on
2583 * FFA_MEM_RETRIEVE_REQ, against saved memory region provided by lender at the
2584 * memory sharing call.
2585 */
2586static struct ffa_value ffa_memory_retrieve_validate(
J-Alves19e20cf2023-08-02 12:48:55 +01002587 ffa_id_t receiver_id, struct ffa_memory_region *retrieve_request,
J-Alves089004f2022-07-13 14:25:44 +01002588 struct ffa_memory_region *memory_region, uint32_t *receiver_index,
2589 uint32_t share_func)
2590{
2591 ffa_memory_region_flags_t transaction_type =
2592 retrieve_request->flags &
2593 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK;
Olivier Deprez4342a3c2022-02-28 09:37:25 +01002594 enum ffa_memory_security security_state;
J-Alves089004f2022-07-13 14:25:44 +01002595
2596 assert(retrieve_request != NULL);
2597 assert(memory_region != NULL);
2598 assert(receiver_index != NULL);
2599 assert(retrieve_request->sender == memory_region->sender);
2600
2601 /*
2602 * Check that the transaction type expected by the receiver is
2603 * correct, if it has been specified.
2604 */
2605 if (transaction_type !=
2606 FFA_MEMORY_REGION_TRANSACTION_TYPE_UNSPECIFIED &&
2607 transaction_type != (memory_region->flags &
2608 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK)) {
2609 dlog_verbose(
2610 "Incorrect transaction type %#x for "
2611 "FFA_MEM_RETRIEVE_REQ, expected %#x for handle %#x.\n",
2612 transaction_type,
2613 memory_region->flags &
2614 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK,
2615 retrieve_request->handle);
2616 return ffa_error(FFA_INVALID_PARAMETERS);
2617 }
2618
2619 if (retrieve_request->tag != memory_region->tag) {
2620 dlog_verbose(
2621 "Incorrect tag %d for FFA_MEM_RETRIEVE_REQ, expected "
2622 "%d for handle %#x.\n",
2623 retrieve_request->tag, memory_region->tag,
2624 retrieve_request->handle);
2625 return ffa_error(FFA_INVALID_PARAMETERS);
2626 }
2627
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002628 *receiver_index = ffa_memory_region_get_receiver_index(memory_region,
2629 receiver_id);
J-Alves089004f2022-07-13 14:25:44 +01002630
2631 if (*receiver_index == memory_region->receiver_count) {
2632 dlog_verbose(
2633 "Incorrect receiver VM ID %d for "
2634 "FFA_MEM_RETRIEVE_REQ, for handle %#x.\n",
J-Alves59ed0042022-07-28 18:26:41 +01002635 receiver_id, memory_region->handle);
J-Alves089004f2022-07-13 14:25:44 +01002636 return ffa_error(FFA_INVALID_PARAMETERS);
2637 }
2638
2639 if ((retrieve_request->flags &
2640 FFA_MEMORY_REGION_ADDRESS_RANGE_HINT_VALID) != 0U) {
2641 dlog_verbose(
2642 "Retriever specified 'address range alignment 'hint' "
2643 "not supported.\n");
2644 return ffa_error(FFA_INVALID_PARAMETERS);
2645 }
2646 if ((retrieve_request->flags &
2647 FFA_MEMORY_REGION_ADDRESS_RANGE_HINT_MASK) != 0) {
2648 dlog_verbose(
2649 "Bits 8-5 must be zero in memory region's flags "
2650 "(address range alignment hint not supported).\n");
2651 return ffa_error(FFA_INVALID_PARAMETERS);
2652 }
2653
2654 if ((retrieve_request->flags & ~0x7FF) != 0U) {
2655 dlog_verbose(
2656 "Bits 31-10 must be zero in memory region's flags.\n");
2657 return ffa_error(FFA_INVALID_PARAMETERS);
2658 }
2659
2660 if (share_func == FFA_MEM_SHARE_32 &&
2661 (retrieve_request->flags &
2662 (FFA_MEMORY_REGION_FLAG_CLEAR |
2663 FFA_MEMORY_REGION_FLAG_CLEAR_RELINQUISH)) != 0U) {
2664 dlog_verbose(
2665 "Memory Share operation can't clean after relinquish "
2666 "memory region.\n");
2667 return ffa_error(FFA_INVALID_PARAMETERS);
2668 }
2669
2670 /*
2671 * If the borrower needs the memory to be cleared before mapping
2672 * to its address space, the sender should have set the flag
2673 * when calling FFA_MEM_LEND/FFA_MEM_DONATE, else return
2674 * FFA_DENIED.
2675 */
2676 if ((retrieve_request->flags & FFA_MEMORY_REGION_FLAG_CLEAR) != 0U &&
2677 (memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR) == 0U) {
2678 dlog_verbose(
2679 "Borrower needs memory cleared. Sender needs to set "
2680 "flag for clearing memory.\n");
2681 return ffa_error(FFA_DENIED);
2682 }
2683
Olivier Deprez4342a3c2022-02-28 09:37:25 +01002684 /* Memory region attributes NS-Bit MBZ for FFA_MEM_RETRIEVE_REQ. */
2685 security_state =
2686 ffa_get_memory_security_attr(retrieve_request->attributes);
2687 if (security_state != FFA_MEMORY_SECURITY_UNSPECIFIED) {
2688 dlog_verbose(
2689 "Invalid security state for memory retrieve request "
2690 "operation.\n");
2691 return ffa_error(FFA_INVALID_PARAMETERS);
2692 }
2693
J-Alves089004f2022-07-13 14:25:44 +01002694 /*
2695 * If memory type is not specified, bypass validation of memory
2696 * attributes in the retrieve request. The retriever is expecting to
2697 * obtain this information from the SPMC.
2698 */
2699 if (ffa_get_memory_type_attr(retrieve_request->attributes) ==
2700 FFA_MEMORY_NOT_SPECIFIED_MEM) {
2701 return (struct ffa_value){.func = FFA_SUCCESS_32};
2702 }
2703
2704 /*
2705 * Ensure receiver's attributes are compatible with how
2706 * Hafnium maps memory: Normal Memory, Inner shareable,
2707 * Write-Back Read-Allocate Write-Allocate Cacheable.
2708 */
2709 return ffa_memory_attributes_validate(retrieve_request->attributes);
2710}
2711
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002712struct ffa_value ffa_memory_retrieve(struct vm_locked to_locked,
2713 struct ffa_memory_region *retrieve_request,
Andrew Walbran130a8ae2020-05-15 16:27:15 +01002714 uint32_t retrieve_request_length,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002715 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002716{
Andrew Walbran130a8ae2020-05-15 16:27:15 +01002717 uint32_t expected_retrieve_request_length =
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002718 sizeof(struct ffa_memory_region) +
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002719 (uint32_t)(retrieve_request->receiver_count *
2720 retrieve_request->memory_access_desc_size);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002721 ffa_memory_handle_t handle = retrieve_request->handle;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002722 struct ffa_memory_region *memory_region;
J-Alvesa9cd7e32022-07-01 13:49:33 +01002723 ffa_memory_access_permissions_t permissions = 0;
Olivier Deprez878bd5b2021-04-15 19:05:10 +02002724 uint32_t memory_to_mode;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002725 struct share_states_locked share_states;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002726 struct ffa_memory_share_state *share_state;
2727 struct ffa_value ret;
Andrew Walbranca808b12020-05-15 17:22:28 +01002728 struct ffa_composite_memory_region *composite;
2729 uint32_t total_length;
2730 uint32_t fragment_length;
J-Alves19e20cf2023-08-02 12:48:55 +01002731 ffa_id_t receiver_id = to_locked.vm->id;
J-Alvesa9cd7e32022-07-01 13:49:33 +01002732 bool is_send_complete = false;
Olivier Deprez878bd5b2021-04-15 19:05:10 +02002733 ffa_memory_attributes_t attributes;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002734
2735 dump_share_states();
2736
Andrew Walbran130a8ae2020-05-15 16:27:15 +01002737 if (retrieve_request_length != expected_retrieve_request_length) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002738 dlog_verbose(
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002739 "Invalid length for FFA_MEM_RETRIEVE_REQ, expected %d "
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002740 "but was %d.\n",
Andrew Walbran130a8ae2020-05-15 16:27:15 +01002741 expected_retrieve_request_length,
2742 retrieve_request_length);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002743 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002744 }
2745
2746 share_states = share_states_lock();
Karl Meakin4a2854a2023-06-30 16:26:52 +01002747 share_state = get_share_state(share_states, handle);
J-Alvesb56aac82023-11-10 09:44:43 +00002748 if (share_state == NULL) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002749 dlog_verbose("Invalid handle %#x for FFA_MEM_RETRIEVE_REQ.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002750 handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002751 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002752 goto out;
2753 }
2754
J-Alves96de29f2022-04-26 16:05:24 +01002755 if (!share_state->sending_complete) {
2756 dlog_verbose(
2757 "Memory with handle %#x not fully sent, can't "
2758 "retrieve.\n",
2759 handle);
2760 ret = ffa_error(FFA_INVALID_PARAMETERS);
2761 goto out;
2762 }
2763
Andrew Walbrana65a1322020-04-06 19:32:32 +01002764 memory_region = share_state->memory_region;
J-Alves089004f2022-07-13 14:25:44 +01002765
Andrew Walbrana65a1322020-04-06 19:32:32 +01002766 CHECK(memory_region != NULL);
2767
J-Alves089004f2022-07-13 14:25:44 +01002768 if (retrieve_request->sender != memory_region->sender) {
2769 dlog_verbose(
2770 "Memory with handle %#x not fully sent, can't "
2771 "retrieve.\n",
2772 handle);
J-Alves41d4fef2023-11-16 16:20:09 +00002773 ret = ffa_error(FFA_DENIED);
J-Alves089004f2022-07-13 14:25:44 +01002774 goto out;
2775 }
J-Alves96de29f2022-04-26 16:05:24 +01002776
J-Alvesa9cd7e32022-07-01 13:49:33 +01002777 if (!is_ffa_memory_retrieve_borrower_request(retrieve_request,
2778 to_locked)) {
2779 uint32_t receiver_index;
J-Alvesa9cd7e32022-07-01 13:49:33 +01002780
J-Alvesb5084cf2022-07-06 14:20:12 +01002781 /*
2782 * The SPMC can only process retrieve requests to memory share
2783 * operations with one borrower from the other world. It can't
2784 * determine the ID of the NWd VM that invoked the retrieve
2785 * request interface call. It relies on the hypervisor to
2786 * validate the caller's ID against that provided in the
2787 * `receivers` list of the retrieve response.
2788 * In case there is only one borrower from the NWd in the
2789 * transaction descriptor, record that in the `receiver_id` for
2790 * later use, and validate in the retrieve request message.
J-Alves3fa82aa2023-09-20 18:19:21 +01002791 * This limitation is due to the fact SPMC can't determine the
2792 * index in the memory share structures state to update.
J-Alvesb5084cf2022-07-06 14:20:12 +01002793 */
2794 if (to_locked.vm->id == HF_HYPERVISOR_VM_ID) {
2795 uint32_t other_world_count = 0;
2796
2797 for (uint32_t i = 0; i < memory_region->receiver_count;
2798 i++) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002799 struct ffa_memory_access *receiver =
2800 ffa_memory_region_get_receiver(
2801 retrieve_request, 0);
2802
2803 assert(receiver != NULL);
J-Alvesb5084cf2022-07-06 14:20:12 +01002804 receiver_id =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002805 receiver->receiver_permissions.receiver;
J-Alvesb5084cf2022-07-06 14:20:12 +01002806 if (!vm_id_is_current_world(receiver_id)) {
2807 other_world_count++;
2808 }
2809 }
2810 if (other_world_count > 1) {
2811 dlog_verbose(
2812 "Support one receiver from the other "
2813 "world.\n");
2814 return ffa_error(FFA_NOT_SUPPORTED);
2815 }
2816 }
2817
2818 /*
2819 * Validate retrieve request, according to what was sent by the
2820 * sender. Function will output the `receiver_index` from the
J-Alves3fa82aa2023-09-20 18:19:21 +01002821 * provided memory region.
J-Alvesb5084cf2022-07-06 14:20:12 +01002822 */
J-Alves089004f2022-07-13 14:25:44 +01002823 ret = ffa_memory_retrieve_validate(
2824 receiver_id, retrieve_request, memory_region,
2825 &receiver_index, share_state->share_func);
2826 if (ret.func != FFA_SUCCESS_32) {
J-Alvesa9cd7e32022-07-01 13:49:33 +01002827 goto out;
2828 }
2829
2830 if (share_state->retrieved_fragment_count[receiver_index] !=
2831 0U) {
2832 dlog_verbose(
2833 "Memory with handle %#x already retrieved.\n",
2834 handle);
2835 ret = ffa_error(FFA_DENIED);
2836 goto out;
2837 }
2838
J-Alves3fa82aa2023-09-20 18:19:21 +01002839 /*
2840 * Validate the requested permissions against the sent
2841 * permissions.
2842 * Outputs the permissions to give to retriever at S2
2843 * PTs.
2844 */
J-Alvesa9cd7e32022-07-01 13:49:33 +01002845 ret = ffa_memory_retrieve_validate_memory_access_list(
2846 memory_region, retrieve_request, receiver_id,
J-Alvesdcad8992023-09-15 14:10:35 +01002847 &permissions, share_state->share_func);
J-Alves614d9f42022-06-28 14:03:10 +01002848 if (ret.func != FFA_SUCCESS_32) {
2849 goto out;
2850 }
Federico Recanatia98603a2021-12-20 18:04:03 +01002851
Olivier Deprez878bd5b2021-04-15 19:05:10 +02002852 memory_to_mode = ffa_memory_permissions_to_mode(
J-Alvesa9cd7e32022-07-01 13:49:33 +01002853 permissions, share_state->sender_orig_mode);
J-Alves40e260e2022-09-22 17:52:43 +01002854
J-Alvesa9cd7e32022-07-01 13:49:33 +01002855 ret = ffa_retrieve_check_update(
J-Alves26483382023-04-20 12:01:49 +01002856 to_locked, share_state->fragments,
J-Alvesa9cd7e32022-07-01 13:49:33 +01002857 share_state->fragment_constituent_counts,
Olivier Deprez878bd5b2021-04-15 19:05:10 +02002858 share_state->fragment_count, memory_to_mode,
J-Alvesa9cd7e32022-07-01 13:49:33 +01002859 share_state->share_func, false, page_pool);
2860
2861 if (ret.func != FFA_SUCCESS_32) {
2862 goto out;
2863 }
2864
2865 share_state->retrieved_fragment_count[receiver_index] = 1;
2866 is_send_complete =
2867 share_state->retrieved_fragment_count[receiver_index] ==
2868 share_state->fragment_count;
J-Alves3c5b2072022-11-21 12:45:40 +00002869
2870 share_state->clear_after_relinquish =
2871 (retrieve_request->flags &
2872 FFA_MEMORY_REGION_FLAG_CLEAR_RELINQUISH) != 0U;
2873
J-Alvesa9cd7e32022-07-01 13:49:33 +01002874 } else {
2875 if (share_state->hypervisor_fragment_count != 0U) {
2876 dlog_verbose(
J-Alvesb5084cf2022-07-06 14:20:12 +01002877 "Memory with handle %#x already retrieved by "
J-Alvesa9cd7e32022-07-01 13:49:33 +01002878 "the hypervisor.\n",
2879 handle);
2880 ret = ffa_error(FFA_DENIED);
2881 goto out;
2882 }
2883
2884 share_state->hypervisor_fragment_count = 1;
2885
2886 ffa_memory_retrieve_complete_from_hyp(share_state);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002887 }
2888
J-Alvesb5084cf2022-07-06 14:20:12 +01002889 /* VMs acquire the RX buffer from SPMC. */
2890 CHECK(plat_ffa_acquire_receiver_rx(to_locked, &ret));
2891
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002892 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01002893 * Copy response to RX buffer of caller and deliver the message.
2894 * This must be done before the share_state is (possibly) freed.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002895 */
Andrew Walbrana65a1322020-04-06 19:32:32 +01002896 /* TODO: combine attributes from sender and request. */
Andrew Walbranca808b12020-05-15 17:22:28 +01002897 composite = ffa_memory_region_get_composite(memory_region, 0);
2898 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01002899 * Constituents which we received in the first fragment should
2900 * always fit in the first fragment we are sending, because the
2901 * header is the same size in both cases and we have a fixed
2902 * message buffer size. So `ffa_retrieved_memory_region_init`
2903 * should never fail.
Andrew Walbranca808b12020-05-15 17:22:28 +01002904 */
Olivier Deprez878bd5b2021-04-15 19:05:10 +02002905
2906 /*
2907 * Set the security state in the memory retrieve response attributes
2908 * if specified by the target mode.
2909 */
2910 attributes = plat_ffa_memory_security_mode(
2911 memory_region->attributes, share_state->sender_orig_mode);
2912
Andrew Walbranca808b12020-05-15 17:22:28 +01002913 CHECK(ffa_retrieved_memory_region_init(
J-Alves2d8457f2022-10-05 11:06:41 +01002914 to_locked.vm->mailbox.recv, to_locked.vm->ffa_version,
Olivier Deprez878bd5b2021-04-15 19:05:10 +02002915 HF_MAILBOX_SIZE, memory_region->sender, attributes,
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002916 memory_region->flags, handle, receiver_id,
2917 memory_region->memory_access_desc_size, permissions,
Olivier Deprez878bd5b2021-04-15 19:05:10 +02002918 composite->page_count, composite->constituent_count,
2919 share_state->fragments[0],
Andrew Walbranca808b12020-05-15 17:22:28 +01002920 share_state->fragment_constituent_counts[0], &total_length,
2921 &fragment_length));
J-Alvesb5084cf2022-07-06 14:20:12 +01002922
Andrew Walbranca808b12020-05-15 17:22:28 +01002923 to_locked.vm->mailbox.recv_size = fragment_length;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002924 to_locked.vm->mailbox.recv_sender = HF_HYPERVISOR_VM_ID;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002925 to_locked.vm->mailbox.recv_func = FFA_MEM_RETRIEVE_RESP_32;
J-Alvese8c8c2b2022-12-16 15:34:48 +00002926 to_locked.vm->mailbox.state = MAILBOX_STATE_FULL;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002927
J-Alvesa9cd7e32022-07-01 13:49:33 +01002928 if (is_send_complete) {
Andrew Walbranca808b12020-05-15 17:22:28 +01002929 ffa_memory_retrieve_complete(share_states, share_state,
2930 page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002931 }
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002932 ret = (struct ffa_value){.func = FFA_MEM_RETRIEVE_RESP_32,
Andrew Walbranca808b12020-05-15 17:22:28 +01002933 .arg1 = total_length,
2934 .arg2 = fragment_length};
Andrew Walbranca808b12020-05-15 17:22:28 +01002935out:
2936 share_states_unlock(&share_states);
2937 dump_share_states();
2938 return ret;
2939}
2940
J-Alves5da37d92022-10-24 16:33:48 +01002941/**
2942 * Determine expected fragment offset according to the FF-A version of
2943 * the caller.
2944 */
2945static uint32_t ffa_memory_retrieve_expected_offset_per_ffa_version(
2946 struct ffa_memory_region *memory_region,
2947 uint32_t retrieved_constituents_count, uint32_t ffa_version)
2948{
2949 uint32_t expected_fragment_offset;
2950 uint32_t composite_constituents_offset;
2951
Kathleen Capellae4fe2962023-09-01 17:08:47 -04002952 if (ffa_version >= MAKE_FFA_VERSION(1, 1)) {
J-Alves5da37d92022-10-24 16:33:48 +01002953 /*
2954 * Hafnium operates memory regions in FF-A v1.1 format, so we
2955 * can retrieve the constituents offset from descriptor.
2956 */
2957 composite_constituents_offset =
2958 ffa_composite_constituent_offset(memory_region, 0);
2959 } else if (ffa_version == MAKE_FFA_VERSION(1, 0)) {
2960 /*
2961 * If retriever is FF-A v1.0, determine the composite offset
2962 * as it is expected to have been configured in the
2963 * retrieve response.
2964 */
2965 composite_constituents_offset =
2966 sizeof(struct ffa_memory_region_v1_0) +
2967 RECEIVERS_COUNT_IN_RETRIEVE_RESP *
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002968 sizeof(struct ffa_memory_access_v1_0) +
J-Alves5da37d92022-10-24 16:33:48 +01002969 sizeof(struct ffa_composite_memory_region);
2970 } else {
2971 panic("%s received an invalid FF-A version.\n", __func__);
2972 }
2973
2974 expected_fragment_offset =
2975 composite_constituents_offset +
2976 retrieved_constituents_count *
2977 sizeof(struct ffa_memory_region_constituent) -
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00002978 (uint32_t)(memory_region->memory_access_desc_size *
2979 (memory_region->receiver_count - 1));
J-Alves5da37d92022-10-24 16:33:48 +01002980
2981 return expected_fragment_offset;
2982}
2983
Andrew Walbranca808b12020-05-15 17:22:28 +01002984struct ffa_value ffa_memory_retrieve_continue(struct vm_locked to_locked,
2985 ffa_memory_handle_t handle,
2986 uint32_t fragment_offset,
J-Alves19e20cf2023-08-02 12:48:55 +01002987 ffa_id_t sender_vm_id,
Andrew Walbranca808b12020-05-15 17:22:28 +01002988 struct mpool *page_pool)
2989{
2990 struct ffa_memory_region *memory_region;
2991 struct share_states_locked share_states;
2992 struct ffa_memory_share_state *share_state;
2993 struct ffa_value ret;
2994 uint32_t fragment_index;
2995 uint32_t retrieved_constituents_count;
2996 uint32_t i;
2997 uint32_t expected_fragment_offset;
2998 uint32_t remaining_constituent_count;
2999 uint32_t fragment_length;
J-Alvesc7484f12022-05-13 12:41:14 +01003000 uint32_t receiver_index;
J-Alves59ed0042022-07-28 18:26:41 +01003001 bool continue_ffa_hyp_mem_retrieve_req;
Andrew Walbranca808b12020-05-15 17:22:28 +01003002
3003 dump_share_states();
3004
3005 share_states = share_states_lock();
Karl Meakin4a2854a2023-06-30 16:26:52 +01003006 share_state = get_share_state(share_states, handle);
J-Alvesb56aac82023-11-10 09:44:43 +00003007 if (share_state == NULL) {
Andrew Walbranca808b12020-05-15 17:22:28 +01003008 dlog_verbose("Invalid handle %#x for FFA_MEM_FRAG_RX.\n",
3009 handle);
3010 ret = ffa_error(FFA_INVALID_PARAMETERS);
3011 goto out;
3012 }
3013
3014 memory_region = share_state->memory_region;
3015 CHECK(memory_region != NULL);
3016
Andrew Walbranca808b12020-05-15 17:22:28 +01003017 if (!share_state->sending_complete) {
3018 dlog_verbose(
3019 "Memory with handle %#x not fully sent, can't "
3020 "retrieve.\n",
3021 handle);
3022 ret = ffa_error(FFA_INVALID_PARAMETERS);
3023 goto out;
3024 }
3025
J-Alves59ed0042022-07-28 18:26:41 +01003026 /*
3027 * If retrieve request from the hypervisor has been initiated in the
3028 * given share_state, continue it, else assume it is a continuation of
3029 * retrieve request from a NWd VM.
3030 */
3031 continue_ffa_hyp_mem_retrieve_req =
3032 (to_locked.vm->id == HF_HYPERVISOR_VM_ID) &&
3033 (share_state->hypervisor_fragment_count != 0U) &&
J-Alves661e1b72023-08-02 13:39:40 +01003034 ffa_is_vm_id(sender_vm_id);
Andrew Walbranca808b12020-05-15 17:22:28 +01003035
J-Alves59ed0042022-07-28 18:26:41 +01003036 if (!continue_ffa_hyp_mem_retrieve_req) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003037 receiver_index = ffa_memory_region_get_receiver_index(
J-Alves59ed0042022-07-28 18:26:41 +01003038 memory_region, to_locked.vm->id);
3039
3040 if (receiver_index == memory_region->receiver_count) {
3041 dlog_verbose(
3042 "Caller of FFA_MEM_FRAG_RX (%x) is not a "
3043 "borrower to memory sharing transaction (%x)\n",
3044 to_locked.vm->id, handle);
3045 ret = ffa_error(FFA_INVALID_PARAMETERS);
3046 goto out;
3047 }
3048
3049 if (share_state->retrieved_fragment_count[receiver_index] ==
3050 0 ||
3051 share_state->retrieved_fragment_count[receiver_index] >=
3052 share_state->fragment_count) {
3053 dlog_verbose(
3054 "Retrieval of memory with handle %#x not yet "
3055 "started or already completed (%d/%d fragments "
3056 "retrieved).\n",
3057 handle,
3058 share_state->retrieved_fragment_count
3059 [receiver_index],
3060 share_state->fragment_count);
3061 ret = ffa_error(FFA_INVALID_PARAMETERS);
3062 goto out;
3063 }
3064
3065 fragment_index =
3066 share_state->retrieved_fragment_count[receiver_index];
3067 } else {
3068 if (share_state->hypervisor_fragment_count == 0 ||
3069 share_state->hypervisor_fragment_count >=
3070 share_state->fragment_count) {
3071 dlog_verbose(
3072 "Retrieve of memory with handle %x not "
3073 "started from hypervisor.\n",
3074 handle);
3075 ret = ffa_error(FFA_INVALID_PARAMETERS);
3076 goto out;
3077 }
3078
3079 if (memory_region->sender != sender_vm_id) {
3080 dlog_verbose(
3081 "Sender ID (%x) is not as expected for memory "
3082 "handle %x\n",
3083 sender_vm_id, handle);
3084 ret = ffa_error(FFA_INVALID_PARAMETERS);
3085 goto out;
3086 }
3087
3088 fragment_index = share_state->hypervisor_fragment_count;
3089
3090 receiver_index = 0;
3091 }
Andrew Walbranca808b12020-05-15 17:22:28 +01003092
3093 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01003094 * Check that the given fragment offset is correct by counting
3095 * how many constituents were in the fragments previously sent.
Andrew Walbranca808b12020-05-15 17:22:28 +01003096 */
3097 retrieved_constituents_count = 0;
3098 for (i = 0; i < fragment_index; ++i) {
3099 retrieved_constituents_count +=
3100 share_state->fragment_constituent_counts[i];
3101 }
J-Alvesc7484f12022-05-13 12:41:14 +01003102
3103 CHECK(memory_region->receiver_count > 0);
3104
Andrew Walbranca808b12020-05-15 17:22:28 +01003105 expected_fragment_offset =
J-Alves5da37d92022-10-24 16:33:48 +01003106 ffa_memory_retrieve_expected_offset_per_ffa_version(
3107 memory_region, retrieved_constituents_count,
3108 to_locked.vm->ffa_version);
3109
Andrew Walbranca808b12020-05-15 17:22:28 +01003110 if (fragment_offset != expected_fragment_offset) {
3111 dlog_verbose("Fragment offset was %d but expected %d.\n",
3112 fragment_offset, expected_fragment_offset);
3113 ret = ffa_error(FFA_INVALID_PARAMETERS);
3114 goto out;
3115 }
3116
J-Alves59ed0042022-07-28 18:26:41 +01003117 /* VMs acquire the RX buffer from SPMC. */
3118 CHECK(plat_ffa_acquire_receiver_rx(to_locked, &ret));
3119
Andrew Walbranca808b12020-05-15 17:22:28 +01003120 remaining_constituent_count = ffa_memory_fragment_init(
3121 to_locked.vm->mailbox.recv, HF_MAILBOX_SIZE,
3122 share_state->fragments[fragment_index],
3123 share_state->fragment_constituent_counts[fragment_index],
3124 &fragment_length);
3125 CHECK(remaining_constituent_count == 0);
3126 to_locked.vm->mailbox.recv_size = fragment_length;
3127 to_locked.vm->mailbox.recv_sender = HF_HYPERVISOR_VM_ID;
3128 to_locked.vm->mailbox.recv_func = FFA_MEM_FRAG_TX_32;
J-Alvese8c8c2b2022-12-16 15:34:48 +00003129 to_locked.vm->mailbox.state = MAILBOX_STATE_FULL;
Andrew Walbranca808b12020-05-15 17:22:28 +01003130
J-Alves59ed0042022-07-28 18:26:41 +01003131 if (!continue_ffa_hyp_mem_retrieve_req) {
3132 share_state->retrieved_fragment_count[receiver_index]++;
3133 if (share_state->retrieved_fragment_count[receiver_index] ==
3134 share_state->fragment_count) {
3135 ffa_memory_retrieve_complete(share_states, share_state,
3136 page_pool);
3137 }
3138 } else {
3139 share_state->hypervisor_fragment_count++;
3140
3141 ffa_memory_retrieve_complete_from_hyp(share_state);
3142 }
Andrew Walbranca808b12020-05-15 17:22:28 +01003143 ret = (struct ffa_value){.func = FFA_MEM_FRAG_TX_32,
3144 .arg1 = (uint32_t)handle,
3145 .arg2 = (uint32_t)(handle >> 32),
3146 .arg3 = fragment_length};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003147
3148out:
3149 share_states_unlock(&share_states);
3150 dump_share_states();
3151 return ret;
3152}
3153
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003154struct ffa_value ffa_memory_relinquish(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003155 struct vm_locked from_locked,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003156 struct ffa_mem_relinquish *relinquish_request, struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003157{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003158 ffa_memory_handle_t handle = relinquish_request->handle;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003159 struct share_states_locked share_states;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003160 struct ffa_memory_share_state *share_state;
3161 struct ffa_memory_region *memory_region;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003162 bool clear;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003163 struct ffa_value ret;
J-Alves8eb19162022-04-28 10:56:48 +01003164 uint32_t receiver_index;
J-Alves3c5b2072022-11-21 12:45:40 +00003165 bool receivers_relinquished_memory;
J-Alves639ddfc2023-11-21 14:17:26 +00003166 ffa_memory_access_permissions_t receiver_permissions = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003167
Andrew Walbrana65a1322020-04-06 19:32:32 +01003168 if (relinquish_request->endpoint_count != 1) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003169 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01003170 "Stream endpoints not supported (got %d "
J-Alves668a86e2023-05-10 11:53:25 +01003171 "endpoints on FFA_MEM_RELINQUISH, expected 1).\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003172 relinquish_request->endpoint_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003173 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003174 }
3175
Andrew Walbrana65a1322020-04-06 19:32:32 +01003176 if (relinquish_request->endpoints[0] != from_locked.vm->id) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003177 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01003178 "VM ID %d in relinquish message doesn't match "
J-Alves668a86e2023-05-10 11:53:25 +01003179 "calling VM ID %d.\n",
Andrew Walbrana65a1322020-04-06 19:32:32 +01003180 relinquish_request->endpoints[0], from_locked.vm->id);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003181 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003182 }
3183
3184 dump_share_states();
3185
3186 share_states = share_states_lock();
Karl Meakin4a2854a2023-06-30 16:26:52 +01003187 share_state = get_share_state(share_states, handle);
J-Alvesb56aac82023-11-10 09:44:43 +00003188 if (share_state == NULL) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003189 dlog_verbose("Invalid handle %#x for FFA_MEM_RELINQUISH.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003190 handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003191 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003192 goto out;
3193 }
3194
Andrew Walbranca808b12020-05-15 17:22:28 +01003195 if (!share_state->sending_complete) {
3196 dlog_verbose(
3197 "Memory with handle %#x not fully sent, can't "
3198 "relinquish.\n",
3199 handle);
3200 ret = ffa_error(FFA_INVALID_PARAMETERS);
3201 goto out;
3202 }
3203
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003204 memory_region = share_state->memory_region;
3205 CHECK(memory_region != NULL);
3206
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003207 receiver_index = ffa_memory_region_get_receiver_index(
3208 memory_region, from_locked.vm->id);
J-Alves8eb19162022-04-28 10:56:48 +01003209
3210 if (receiver_index == memory_region->receiver_count) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003211 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01003212 "VM ID %d tried to relinquish memory region "
J-Alves668a86e2023-05-10 11:53:25 +01003213 "with handle %#x and it is not a valid borrower.\n",
J-Alves8eb19162022-04-28 10:56:48 +01003214 from_locked.vm->id, handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003215 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003216 goto out;
3217 }
3218
J-Alves8eb19162022-04-28 10:56:48 +01003219 if (share_state->retrieved_fragment_count[receiver_index] !=
Andrew Walbranca808b12020-05-15 17:22:28 +01003220 share_state->fragment_count) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003221 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01003222 "Memory with handle %#x not yet fully "
3223 "retrieved, "
J-Alves8eb19162022-04-28 10:56:48 +01003224 "receiver %x can't relinquish.\n",
3225 handle, from_locked.vm->id);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003226 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003227 goto out;
3228 }
3229
J-Alves3c5b2072022-11-21 12:45:40 +00003230 /*
3231 * Either clear if requested in relinquish call, or in a retrieve
3232 * request from one of the borrowers.
3233 */
3234 receivers_relinquished_memory = true;
3235
3236 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
3237 struct ffa_memory_access *receiver =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003238 ffa_memory_region_get_receiver(memory_region, i);
3239 assert(receiver != NULL);
J-Alves3c5b2072022-11-21 12:45:40 +00003240 if (receiver->receiver_permissions.receiver ==
3241 from_locked.vm->id) {
J-Alves639ddfc2023-11-21 14:17:26 +00003242 receiver_permissions =
3243 receiver->receiver_permissions.permissions;
J-Alves3c5b2072022-11-21 12:45:40 +00003244 continue;
3245 }
3246
3247 if (share_state->retrieved_fragment_count[i] != 0U) {
3248 receivers_relinquished_memory = false;
3249 break;
3250 }
3251 }
3252
3253 clear = receivers_relinquished_memory &&
3254 (share_state->clear_after_relinquish ||
3255 (relinquish_request->flags & FFA_MEMORY_REGION_FLAG_CLEAR) !=
3256 0U);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003257
3258 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01003259 * Clear is not allowed for memory that was shared, as the
3260 * original sender still has access to the memory.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003261 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003262 if (clear && share_state->share_func == FFA_MEM_SHARE_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003263 dlog_verbose("Memory which was shared can't be cleared.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003264 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003265 goto out;
3266 }
3267
J-Alves639ddfc2023-11-21 14:17:26 +00003268 if (clear && receiver_permissions == FFA_DATA_ACCESS_RO) {
3269 dlog_verbose("%s: RO memory can't use clear memory flag.\n",
3270 __func__);
3271 ret = ffa_error(FFA_DENIED);
3272 goto out;
3273 }
3274
Andrew Walbranca808b12020-05-15 17:22:28 +01003275 ret = ffa_relinquish_check_update(
J-Alves26483382023-04-20 12:01:49 +01003276 from_locked, share_state->fragments,
Andrew Walbranca808b12020-05-15 17:22:28 +01003277 share_state->fragment_constituent_counts,
3278 share_state->fragment_count, page_pool, clear);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003279
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003280 if (ret.func == FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003281 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01003282 * Mark memory handle as not retrieved, so it can be
3283 * reclaimed (or retrieved again).
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003284 */
J-Alves8eb19162022-04-28 10:56:48 +01003285 share_state->retrieved_fragment_count[receiver_index] = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003286 }
3287
3288out:
3289 share_states_unlock(&share_states);
3290 dump_share_states();
3291 return ret;
3292}
3293
3294/**
J-Alvesa9cd7e32022-07-01 13:49:33 +01003295 * Validates that the reclaim transition is allowed for the given
3296 * handle, updates the page table of the reclaiming VM, and frees the
3297 * internal state associated with the handle.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003298 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003299struct ffa_value ffa_memory_reclaim(struct vm_locked to_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01003300 ffa_memory_handle_t handle,
3301 ffa_memory_region_flags_t flags,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003302 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003303{
3304 struct share_states_locked share_states;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003305 struct ffa_memory_share_state *share_state;
3306 struct ffa_memory_region *memory_region;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003307 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003308
3309 dump_share_states();
3310
3311 share_states = share_states_lock();
Karl Meakin52cdfe72023-06-30 14:49:10 +01003312
Karl Meakin4a2854a2023-06-30 16:26:52 +01003313 share_state = get_share_state(share_states, handle);
J-Alvesb56aac82023-11-10 09:44:43 +00003314 if (share_state == NULL) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003315 dlog_verbose("Invalid handle %#x for FFA_MEM_RECLAIM.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003316 handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003317 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003318 goto out;
3319 }
Karl Meakin4a2854a2023-06-30 16:26:52 +01003320 memory_region = share_state->memory_region;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003321
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003322 CHECK(memory_region != NULL);
3323
J-Alvesa9cd7e32022-07-01 13:49:33 +01003324 if (vm_id_is_current_world(to_locked.vm->id) &&
3325 to_locked.vm->id != memory_region->sender) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003326 dlog_verbose(
Olivier Deprezf92e5d42020-11-13 16:00:54 +01003327 "VM %#x attempted to reclaim memory handle %#x "
3328 "originally sent by VM %#x.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003329 to_locked.vm->id, handle, memory_region->sender);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003330 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003331 goto out;
3332 }
3333
Andrew Walbranca808b12020-05-15 17:22:28 +01003334 if (!share_state->sending_complete) {
3335 dlog_verbose(
3336 "Memory with handle %#x not fully sent, can't "
3337 "reclaim.\n",
3338 handle);
3339 ret = ffa_error(FFA_INVALID_PARAMETERS);
3340 goto out;
3341 }
3342
J-Alves752236c2022-04-28 11:07:47 +01003343 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
3344 if (share_state->retrieved_fragment_count[i] != 0) {
3345 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01003346 "Tried to reclaim memory handle %#x "
J-Alves3c5b2072022-11-21 12:45:40 +00003347 "that has not been relinquished by all "
J-Alvesa9cd7e32022-07-01 13:49:33 +01003348 "borrowers(%x).\n",
J-Alves752236c2022-04-28 11:07:47 +01003349 handle,
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003350 ffa_memory_region_get_receiver(memory_region, i)
3351 ->receiver_permissions.receiver);
J-Alves752236c2022-04-28 11:07:47 +01003352 ret = ffa_error(FFA_DENIED);
3353 goto out;
3354 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003355 }
3356
Andrew Walbranca808b12020-05-15 17:22:28 +01003357 ret = ffa_retrieve_check_update(
J-Alves26483382023-04-20 12:01:49 +01003358 to_locked, share_state->fragments,
Andrew Walbranca808b12020-05-15 17:22:28 +01003359 share_state->fragment_constituent_counts,
J-Alves2a0d2882020-10-29 14:49:50 +00003360 share_state->fragment_count, share_state->sender_orig_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +01003361 FFA_MEM_RECLAIM_32, flags & FFA_MEM_RECLAIM_CLEAR, page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003362
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003363 if (ret.func == FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003364 share_state_free(share_states, share_state, page_pool);
J-Alves3c5b2072022-11-21 12:45:40 +00003365 dlog_verbose("Freed share state after successful reclaim.\n");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003366 }
3367
3368out:
3369 share_states_unlock(&share_states);
3370 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01003371}