blob: 1c68ebbab15bb57d77296abf4e2ae2c18c637b16 [file] [log] [blame]
Jose Marinho75509b42019-04-09 09:34:59 +01001/*
2 * Copyright 2019 The Hafnium Authors.
3 *
Andrew Walbrane959ec12020-06-17 15:01:09 +01004 * Use of this source code is governed by a BSD-style
5 * license that can be found in the LICENSE file or at
6 * https://opensource.org/licenses/BSD-3-Clause.
Jose Marinho75509b42019-04-09 09:34:59 +01007 */
8
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01009#include "hf/ffa_memory.h"
Andrew Walbran475c1452020-02-07 13:22:22 +000010
Federico Recanati4fd065d2021-12-13 20:06:23 +010011#include "hf/arch/mm.h"
Olivier Deprez112d2b52020-09-30 07:39:23 +020012#include "hf/arch/other_world.h"
Olivier Deprez55a189e2021-06-09 15:45:27 +020013#include "hf/arch/plat/ffa.h"
Andrew Walbran290b0c92020-02-03 16:37:14 +000014
Jose Marinho75509b42019-04-09 09:34:59 +010015#include "hf/api.h"
Daniel Boulbya2f8c662021-11-26 17:52:53 +000016#include "hf/assert.h"
Jose Marinho09b1db82019-08-08 09:16:59 +010017#include "hf/check.h"
Jose Marinho75509b42019-04-09 09:34:59 +010018#include "hf/dlog.h"
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010019#include "hf/ffa_internal.h"
J-Alves66652252022-07-06 09:49:51 +010020#include "hf/ffa_memory_internal.h"
Andrew Walbran475c1452020-02-07 13:22:22 +000021#include "hf/mpool.h"
Jose Marinho75509b42019-04-09 09:34:59 +010022#include "hf/std.h"
Andrew Scull3c257452019-11-26 13:32:50 +000023#include "hf/vm.h"
Jose Marinho75509b42019-04-09 09:34:59 +010024
J-Alves2d8457f2022-10-05 11:06:41 +010025#include "vmapi/hf/ffa_v1_0.h"
26
J-Alves5da37d92022-10-24 16:33:48 +010027#define RECEIVERS_COUNT_IN_RETRIEVE_RESP 1
28
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000029/**
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010030 * All access to members of a `struct ffa_memory_share_state` must be guarded
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000031 * by this lock.
32 */
33static struct spinlock share_states_lock_instance = SPINLOCK_INIT;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010034static struct ffa_memory_share_state share_states[MAX_MEM_SHARES];
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000035
36/**
J-Alves917d2f22020-10-30 18:39:30 +000037 * Extracts the index from a memory handle allocated by Hafnium's current world.
38 */
39uint64_t ffa_memory_handle_get_index(ffa_memory_handle_t handle)
40{
41 return handle & ~FFA_MEMORY_HANDLE_ALLOCATOR_MASK;
42}
43
44/**
Andrew Walbranca808b12020-05-15 17:22:28 +010045 * Initialises the next available `struct ffa_memory_share_state` and sets
46 * `share_state_ret` to a pointer to it. If `handle` is
47 * `FFA_MEMORY_HANDLE_INVALID` then allocates an appropriate handle, otherwise
48 * uses the provided handle which is assumed to be globally unique.
49 *
50 * Returns true on success or false if none are available.
51 */
J-Alves66652252022-07-06 09:49:51 +010052bool allocate_share_state(struct share_states_locked share_states,
53 uint32_t share_func,
54 struct ffa_memory_region *memory_region,
55 uint32_t fragment_length, ffa_memory_handle_t handle,
56 struct ffa_memory_share_state **share_state_ret)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000057{
Andrew Walbrana65a1322020-04-06 19:32:32 +010058 uint64_t i;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000059
Daniel Boulbya2f8c662021-11-26 17:52:53 +000060 assert(share_states.share_states != NULL);
61 assert(memory_region != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000062
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000063 for (i = 0; i < MAX_MEM_SHARES; ++i) {
Andrew Walbranca808b12020-05-15 17:22:28 +010064 if (share_states.share_states[i].share_func == 0) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000065 uint32_t j;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010066 struct ffa_memory_share_state *allocated_state =
Andrew Walbranca808b12020-05-15 17:22:28 +010067 &share_states.share_states[i];
68 struct ffa_composite_memory_region *composite =
69 ffa_memory_region_get_composite(memory_region,
70 0);
71
72 if (handle == FFA_MEMORY_HANDLE_INVALID) {
J-Alvesee68c542020-10-29 17:48:20 +000073 memory_region->handle =
Olivier Deprez55a189e2021-06-09 15:45:27 +020074 plat_ffa_memory_handle_make(i);
Andrew Walbranca808b12020-05-15 17:22:28 +010075 } else {
J-Alvesee68c542020-10-29 17:48:20 +000076 memory_region->handle = handle;
Andrew Walbranca808b12020-05-15 17:22:28 +010077 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000078 allocated_state->share_func = share_func;
79 allocated_state->memory_region = memory_region;
Andrew Walbranca808b12020-05-15 17:22:28 +010080 allocated_state->fragment_count = 1;
81 allocated_state->fragments[0] = composite->constituents;
82 allocated_state->fragment_constituent_counts[0] =
83 (fragment_length -
84 ffa_composite_constituent_offset(memory_region,
85 0)) /
86 sizeof(struct ffa_memory_region_constituent);
87 allocated_state->sending_complete = false;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000088 for (j = 0; j < MAX_MEM_SHARE_RECIPIENTS; ++j) {
Andrew Walbranca808b12020-05-15 17:22:28 +010089 allocated_state->retrieved_fragment_count[j] =
90 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000091 }
Andrew Walbranca808b12020-05-15 17:22:28 +010092 if (share_state_ret != NULL) {
93 *share_state_ret = allocated_state;
94 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000095 return true;
96 }
97 }
98
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000099 return false;
100}
101
102/** Locks the share states lock. */
103struct share_states_locked share_states_lock(void)
104{
105 sl_lock(&share_states_lock_instance);
106
107 return (struct share_states_locked){.share_states = share_states};
108}
109
110/** Unlocks the share states lock. */
J-Alves66652252022-07-06 09:49:51 +0100111void share_states_unlock(struct share_states_locked *share_states)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000112{
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000113 assert(share_states->share_states != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000114 share_states->share_states = NULL;
115 sl_unlock(&share_states_lock_instance);
116}
117
118/**
Andrew Walbranca808b12020-05-15 17:22:28 +0100119 * If the given handle is a valid handle for an allocated share state then
120 * initialises `share_state_ret` to point to the share state and returns true.
121 * Otherwise returns false.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000122 */
J-Alvesfdd29272022-07-19 13:16:31 +0100123bool get_share_state(struct share_states_locked share_states,
124 ffa_memory_handle_t handle,
125 struct ffa_memory_share_state **share_state_ret)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000126{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100127 struct ffa_memory_share_state *share_state;
J-Alves917d2f22020-10-30 18:39:30 +0000128 uint64_t index;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000129
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000130 assert(share_states.share_states != NULL);
131 assert(share_state_ret != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100132
133 /*
134 * First look for a share_state allocated by us, in which case the
135 * handle is based on the index.
136 */
Olivier Deprez55a189e2021-06-09 15:45:27 +0200137 if (plat_ffa_memory_handle_allocated_by_current_world(handle)) {
J-Alves917d2f22020-10-30 18:39:30 +0000138 index = ffa_memory_handle_get_index(handle);
Andrew Walbranca808b12020-05-15 17:22:28 +0100139 if (index < MAX_MEM_SHARES) {
140 share_state = &share_states.share_states[index];
141 if (share_state->share_func != 0) {
142 *share_state_ret = share_state;
143 return true;
144 }
145 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000146 }
147
Andrew Walbranca808b12020-05-15 17:22:28 +0100148 /* Fall back to a linear scan. */
149 for (index = 0; index < MAX_MEM_SHARES; ++index) {
150 share_state = &share_states.share_states[index];
J-Alvesee68c542020-10-29 17:48:20 +0000151 if (share_state->memory_region != NULL &&
152 share_state->memory_region->handle == handle &&
Andrew Walbranca808b12020-05-15 17:22:28 +0100153 share_state->share_func != 0) {
154 *share_state_ret = share_state;
155 return true;
156 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000157 }
158
Andrew Walbranca808b12020-05-15 17:22:28 +0100159 return false;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000160}
161
162/** Marks a share state as unallocated. */
J-Alvesfdd29272022-07-19 13:16:31 +0100163void share_state_free(struct share_states_locked share_states,
164 struct ffa_memory_share_state *share_state,
165 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000166{
Andrew Walbranca808b12020-05-15 17:22:28 +0100167 uint32_t i;
168
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000169 assert(share_states.share_states != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000170 share_state->share_func = 0;
Andrew Walbranca808b12020-05-15 17:22:28 +0100171 share_state->sending_complete = false;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000172 mpool_free(page_pool, share_state->memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +0100173 /*
174 * First fragment is part of the same page as the `memory_region`, so it
175 * doesn't need to be freed separately.
176 */
177 share_state->fragments[0] = NULL;
178 share_state->fragment_constituent_counts[0] = 0;
179 for (i = 1; i < share_state->fragment_count; ++i) {
180 mpool_free(page_pool, share_state->fragments[i]);
181 share_state->fragments[i] = NULL;
182 share_state->fragment_constituent_counts[i] = 0;
183 }
184 share_state->fragment_count = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000185 share_state->memory_region = NULL;
J-Alvesa9cd7e32022-07-01 13:49:33 +0100186 share_state->hypervisor_fragment_count = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000187}
188
Andrew Walbranca808b12020-05-15 17:22:28 +0100189/** Checks whether the given share state has been fully sent. */
J-Alvesfdd29272022-07-19 13:16:31 +0100190bool share_state_sending_complete(struct share_states_locked share_states,
191 struct ffa_memory_share_state *share_state)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000192{
Andrew Walbranca808b12020-05-15 17:22:28 +0100193 struct ffa_composite_memory_region *composite;
194 uint32_t expected_constituent_count;
195 uint32_t fragment_constituent_count_total = 0;
196 uint32_t i;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000197
Andrew Walbranca808b12020-05-15 17:22:28 +0100198 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000199 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100200
201 /*
202 * Share state must already be valid, or it's not possible to get hold
203 * of it.
204 */
205 CHECK(share_state->memory_region != NULL &&
206 share_state->share_func != 0);
207
208 composite =
209 ffa_memory_region_get_composite(share_state->memory_region, 0);
210 expected_constituent_count = composite->constituent_count;
211 for (i = 0; i < share_state->fragment_count; ++i) {
212 fragment_constituent_count_total +=
213 share_state->fragment_constituent_counts[i];
214 }
215 dlog_verbose(
216 "Checking completion: constituent count %d/%d from %d "
217 "fragments.\n",
218 fragment_constituent_count_total, expected_constituent_count,
219 share_state->fragment_count);
220
221 return fragment_constituent_count_total == expected_constituent_count;
222}
223
224/**
225 * Calculates the offset of the next fragment expected for the given share
226 * state.
227 */
J-Alvesfdd29272022-07-19 13:16:31 +0100228uint32_t share_state_next_fragment_offset(
Andrew Walbranca808b12020-05-15 17:22:28 +0100229 struct share_states_locked share_states,
230 struct ffa_memory_share_state *share_state)
231{
232 uint32_t next_fragment_offset;
233 uint32_t i;
234
235 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000236 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100237
238 next_fragment_offset =
239 ffa_composite_constituent_offset(share_state->memory_region, 0);
240 for (i = 0; i < share_state->fragment_count; ++i) {
241 next_fragment_offset +=
242 share_state->fragment_constituent_counts[i] *
243 sizeof(struct ffa_memory_region_constituent);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000244 }
245
Andrew Walbranca808b12020-05-15 17:22:28 +0100246 return next_fragment_offset;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000247}
248
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100249static void dump_memory_region(struct ffa_memory_region *memory_region)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000250{
251 uint32_t i;
252
253 if (LOG_LEVEL < LOG_LEVEL_VERBOSE) {
254 return;
255 }
256
Olivier Deprez935e1b12020-12-22 18:01:29 +0100257 dlog("from VM %#x, attributes %#x, flags %#x, tag %u, to "
Olivier Deprezf92e5d42020-11-13 16:00:54 +0100258 "%u "
Andrew Walbrana65a1322020-04-06 19:32:32 +0100259 "recipients [",
260 memory_region->sender, memory_region->attributes,
Olivier Deprez935e1b12020-12-22 18:01:29 +0100261 memory_region->flags, memory_region->tag,
Andrew Walbrana65a1322020-04-06 19:32:32 +0100262 memory_region->receiver_count);
263 for (i = 0; i < memory_region->receiver_count; ++i) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000264 if (i != 0) {
265 dlog(", ");
266 }
Olivier Deprezf92e5d42020-11-13 16:00:54 +0100267 dlog("VM %#x: %#x (offset %u)",
Andrew Walbrana65a1322020-04-06 19:32:32 +0100268 memory_region->receivers[i].receiver_permissions.receiver,
269 memory_region->receivers[i]
270 .receiver_permissions.permissions,
271 memory_region->receivers[i]
272 .composite_memory_region_offset);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000273 }
274 dlog("]");
275}
276
J-Alves66652252022-07-06 09:49:51 +0100277void dump_share_states(void)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000278{
279 uint32_t i;
280
281 if (LOG_LEVEL < LOG_LEVEL_VERBOSE) {
282 return;
283 }
284
285 dlog("Current share states:\n");
286 sl_lock(&share_states_lock_instance);
287 for (i = 0; i < MAX_MEM_SHARES; ++i) {
288 if (share_states[i].share_func != 0) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000289 switch (share_states[i].share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100290 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000291 dlog("SHARE");
292 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100293 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000294 dlog("LEND");
295 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100296 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000297 dlog("DONATE");
298 break;
299 default:
300 dlog("invalid share_func %#x",
301 share_states[i].share_func);
302 }
Olivier Deprez935e1b12020-12-22 18:01:29 +0100303 dlog(" %#x (", share_states[i].memory_region->handle);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000304 dump_memory_region(share_states[i].memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +0100305 if (share_states[i].sending_complete) {
306 dlog("): fully sent");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000307 } else {
Andrew Walbranca808b12020-05-15 17:22:28 +0100308 dlog("): partially sent");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000309 }
J-Alves2a0d2882020-10-29 14:49:50 +0000310 dlog(" with %d fragments, %d retrieved, "
311 " sender's original mode: %#x\n",
Andrew Walbranca808b12020-05-15 17:22:28 +0100312 share_states[i].fragment_count,
J-Alves2a0d2882020-10-29 14:49:50 +0000313 share_states[i].retrieved_fragment_count[0],
314 share_states[i].sender_orig_mode);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000315 }
316 }
317 sl_unlock(&share_states_lock_instance);
318}
319
Andrew Walbran475c1452020-02-07 13:22:22 +0000320/* TODO: Add device attributes: GRE, cacheability, shareability. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100321static inline uint32_t ffa_memory_permissions_to_mode(
J-Alves7cd5eb32020-10-16 19:06:10 +0100322 ffa_memory_access_permissions_t permissions, uint32_t default_mode)
Andrew Walbran475c1452020-02-07 13:22:22 +0000323{
324 uint32_t mode = 0;
325
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100326 switch (ffa_get_data_access_attr(permissions)) {
327 case FFA_DATA_ACCESS_RO:
Andrew Walbran475c1452020-02-07 13:22:22 +0000328 mode = MM_MODE_R;
329 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100330 case FFA_DATA_ACCESS_RW:
Andrew Walbran475c1452020-02-07 13:22:22 +0000331 mode = MM_MODE_R | MM_MODE_W;
332 break;
J-Alves7cd5eb32020-10-16 19:06:10 +0100333 case FFA_DATA_ACCESS_NOT_SPECIFIED:
334 mode = (default_mode & (MM_MODE_R | MM_MODE_W));
335 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100336 case FFA_DATA_ACCESS_RESERVED:
337 panic("Tried to convert FFA_DATA_ACCESS_RESERVED.");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100338 }
339
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100340 switch (ffa_get_instruction_access_attr(permissions)) {
341 case FFA_INSTRUCTION_ACCESS_NX:
Andrew Walbran475c1452020-02-07 13:22:22 +0000342 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100343 case FFA_INSTRUCTION_ACCESS_X:
Andrew Walbrana65a1322020-04-06 19:32:32 +0100344 mode |= MM_MODE_X;
345 break;
J-Alves7cd5eb32020-10-16 19:06:10 +0100346 case FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED:
347 mode |= (default_mode & MM_MODE_X);
348 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100349 case FFA_INSTRUCTION_ACCESS_RESERVED:
350 panic("Tried to convert FFA_INSTRUCTION_ACCESS_RESVERVED.");
Andrew Walbran475c1452020-02-07 13:22:22 +0000351 }
352
353 return mode;
354}
355
Jose Marinho75509b42019-04-09 09:34:59 +0100356/**
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000357 * Get the current mode in the stage-2 page table of the given vm of all the
358 * pages in the given constituents, if they all have the same mode, or return
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100359 * an appropriate FF-A error if not.
Jose Marinho75509b42019-04-09 09:34:59 +0100360 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100361static struct ffa_value constituents_get_mode(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000362 struct vm_locked vm, uint32_t *orig_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100363 struct ffa_memory_region_constituent **fragments,
364 const uint32_t *fragment_constituent_counts, uint32_t fragment_count)
Jose Marinho75509b42019-04-09 09:34:59 +0100365{
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100366 uint32_t i;
Andrew Walbranca808b12020-05-15 17:22:28 +0100367 uint32_t j;
Jose Marinho75509b42019-04-09 09:34:59 +0100368
Andrew Walbranca808b12020-05-15 17:22:28 +0100369 if (fragment_count == 0 || fragment_constituent_counts[0] == 0) {
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100370 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000371 * Fail if there are no constituents. Otherwise we would get an
372 * uninitialised *orig_mode.
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100373 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100374 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho75509b42019-04-09 09:34:59 +0100375 }
376
Andrew Walbranca808b12020-05-15 17:22:28 +0100377 for (i = 0; i < fragment_count; ++i) {
378 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
379 ipaddr_t begin = ipa_init(fragments[i][j].address);
380 size_t size = fragments[i][j].page_count * PAGE_SIZE;
381 ipaddr_t end = ipa_add(begin, size);
382 uint32_t current_mode;
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100383
Andrew Walbranca808b12020-05-15 17:22:28 +0100384 /* Fail if addresses are not page-aligned. */
385 if (!is_aligned(ipa_addr(begin), PAGE_SIZE) ||
386 !is_aligned(ipa_addr(end), PAGE_SIZE)) {
387 return ffa_error(FFA_INVALID_PARAMETERS);
388 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100389
Andrew Walbranca808b12020-05-15 17:22:28 +0100390 /*
391 * Ensure that this constituent memory range is all
392 * mapped with the same mode.
393 */
Raghu Krishnamurthy785d52f2021-02-13 00:02:40 -0800394 if (!vm_mem_get_mode(vm, begin, end, &current_mode)) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100395 return ffa_error(FFA_DENIED);
396 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100397
Andrew Walbranca808b12020-05-15 17:22:28 +0100398 /*
399 * Ensure that all constituents are mapped with the same
400 * mode.
401 */
402 if (i == 0) {
403 *orig_mode = current_mode;
404 } else if (current_mode != *orig_mode) {
405 dlog_verbose(
406 "Expected mode %#x but was %#x for %d "
407 "pages at %#x.\n",
408 *orig_mode, current_mode,
409 fragments[i][j].page_count,
410 ipa_addr(begin));
411 return ffa_error(FFA_DENIED);
412 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100413 }
Jose Marinho75509b42019-04-09 09:34:59 +0100414 }
415
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100416 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000417}
418
419/**
420 * Verify that all pages have the same mode, that the starting mode
421 * constitutes a valid state and obtain the next mode to apply
422 * to the sending VM.
423 *
424 * Returns:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100425 * 1) FFA_DENIED if a state transition was not found;
426 * 2) FFA_DENIED if the pages being shared do not have the same mode within
Andrew Walbrana65a1322020-04-06 19:32:32 +0100427 * the <from> VM;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100428 * 3) FFA_INVALID_PARAMETERS if the beginning and end IPAs are not page
Andrew Walbrana65a1322020-04-06 19:32:32 +0100429 * aligned;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100430 * 4) FFA_INVALID_PARAMETERS if the requested share type was not handled.
431 * Or FFA_SUCCESS on success.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000432 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100433static struct ffa_value ffa_send_check_transition(
Andrew Walbrana65a1322020-04-06 19:32:32 +0100434 struct vm_locked from, uint32_t share_func,
J-Alves363f5722022-04-25 17:37:37 +0100435 struct ffa_memory_access *receivers, uint32_t receivers_count,
436 uint32_t *orig_from_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100437 struct ffa_memory_region_constituent **fragments,
438 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
439 uint32_t *from_mode)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000440{
441 const uint32_t state_mask =
442 MM_MODE_INVALID | MM_MODE_UNOWNED | MM_MODE_SHARED;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100443 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000444
Andrew Walbranca808b12020-05-15 17:22:28 +0100445 ret = constituents_get_mode(from, orig_from_mode, fragments,
446 fragment_constituent_counts,
447 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100448 if (ret.func != FFA_SUCCESS_32) {
Olivier Depreze7eb1682022-03-16 17:09:03 +0100449 dlog_verbose("Inconsistent modes.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100450 return ret;
Andrew Scullb5f49e02019-10-02 13:20:47 +0100451 }
452
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000453 /* Ensure the address range is normal memory and not a device. */
454 if (*orig_from_mode & MM_MODE_D) {
455 dlog_verbose("Can't share device memory (mode is %#x).\n",
456 *orig_from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100457 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000458 }
459
460 /*
461 * Ensure the sender is the owner and has exclusive access to the
462 * memory.
463 */
464 if ((*orig_from_mode & state_mask) != 0) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100465 return ffa_error(FFA_DENIED);
Andrew Walbrana65a1322020-04-06 19:32:32 +0100466 }
467
J-Alves363f5722022-04-25 17:37:37 +0100468 assert(receivers != NULL && receivers_count > 0U);
J-Alves7cd5eb32020-10-16 19:06:10 +0100469
J-Alves363f5722022-04-25 17:37:37 +0100470 for (uint32_t i = 0U; i < receivers_count; i++) {
471 ffa_memory_access_permissions_t permissions =
472 receivers[i].receiver_permissions.permissions;
473 uint32_t required_from_mode = ffa_memory_permissions_to_mode(
474 permissions, *orig_from_mode);
475
476 if ((*orig_from_mode & required_from_mode) !=
477 required_from_mode) {
478 dlog_verbose(
479 "Sender tried to send memory with permissions "
480 "which "
481 "required mode %#x but only had %#x itself.\n",
482 required_from_mode, *orig_from_mode);
483 return ffa_error(FFA_DENIED);
484 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000485 }
486
487 /* Find the appropriate new mode. */
488 *from_mode = ~state_mask & *orig_from_mode;
Andrew Walbrane7ad3c02019-12-24 17:03:04 +0000489 switch (share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100490 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000491 *from_mode |= MM_MODE_INVALID | MM_MODE_UNOWNED;
Jose Marinho75509b42019-04-09 09:34:59 +0100492 break;
493
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100494 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000495 *from_mode |= MM_MODE_INVALID;
Andrew Walbran648fc3e2019-10-22 16:23:05 +0100496 break;
497
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100498 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000499 *from_mode |= MM_MODE_SHARED;
Jose Marinho56c25732019-05-20 09:48:53 +0100500 break;
501
Jose Marinho75509b42019-04-09 09:34:59 +0100502 default:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100503 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho75509b42019-04-09 09:34:59 +0100504 }
505
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100506 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000507}
508
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100509static struct ffa_value ffa_relinquish_check_transition(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000510 struct vm_locked from, uint32_t *orig_from_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100511 struct ffa_memory_region_constituent **fragments,
512 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
513 uint32_t *from_mode)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000514{
515 const uint32_t state_mask =
516 MM_MODE_INVALID | MM_MODE_UNOWNED | MM_MODE_SHARED;
517 uint32_t orig_from_state;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100518 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000519
Andrew Walbranca808b12020-05-15 17:22:28 +0100520 ret = constituents_get_mode(from, orig_from_mode, fragments,
521 fragment_constituent_counts,
522 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100523 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbrana65a1322020-04-06 19:32:32 +0100524 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000525 }
526
527 /* Ensure the address range is normal memory and not a device. */
528 if (*orig_from_mode & MM_MODE_D) {
529 dlog_verbose("Can't relinquish device memory (mode is %#x).\n",
530 *orig_from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100531 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000532 }
533
534 /*
535 * Ensure the relinquishing VM is not the owner but has access to the
536 * memory.
537 */
538 orig_from_state = *orig_from_mode & state_mask;
539 if ((orig_from_state & ~MM_MODE_SHARED) != MM_MODE_UNOWNED) {
540 dlog_verbose(
541 "Tried to relinquish memory in state %#x (masked %#x "
Andrew Walbranca808b12020-05-15 17:22:28 +0100542 "but should be %#x).\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000543 *orig_from_mode, orig_from_state, MM_MODE_UNOWNED);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100544 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000545 }
546
547 /* Find the appropriate new mode. */
548 *from_mode = (~state_mask & *orig_from_mode) | MM_MODE_UNMAPPED_MASK;
549
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100550 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000551}
552
553/**
554 * Verify that all pages have the same mode, that the starting mode
555 * constitutes a valid state and obtain the next mode to apply
556 * to the retrieving VM.
557 *
558 * Returns:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100559 * 1) FFA_DENIED if a state transition was not found;
560 * 2) FFA_DENIED if the pages being shared do not have the same mode within
Andrew Walbrana65a1322020-04-06 19:32:32 +0100561 * the <to> VM;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100562 * 3) FFA_INVALID_PARAMETERS if the beginning and end IPAs are not page
Andrew Walbrana65a1322020-04-06 19:32:32 +0100563 * aligned;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100564 * 4) FFA_INVALID_PARAMETERS if the requested share type was not handled.
565 * Or FFA_SUCCESS on success.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000566 */
J-Alvesfc19b372022-07-06 12:17:35 +0100567struct ffa_value ffa_retrieve_check_transition(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000568 struct vm_locked to, uint32_t share_func,
Andrew Walbranca808b12020-05-15 17:22:28 +0100569 struct ffa_memory_region_constituent **fragments,
570 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
571 uint32_t memory_to_attributes, uint32_t *to_mode)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000572{
573 uint32_t orig_to_mode;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100574 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000575
Andrew Walbranca808b12020-05-15 17:22:28 +0100576 ret = constituents_get_mode(to, &orig_to_mode, fragments,
577 fragment_constituent_counts,
578 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100579 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100580 dlog_verbose("Inconsistent modes.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100581 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000582 }
583
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100584 if (share_func == FFA_MEM_RECLAIM_32) {
J-Alves9256f162021-12-09 13:18:43 +0000585 /*
586 * If the original ffa memory send call has been processed
587 * successfully, it is expected the orig_to_mode would overlay
588 * with `state_mask`, as a result of the function
589 * `ffa_send_check_transition`.
590 */
J-Alves59ed0042022-07-28 18:26:41 +0100591 if (vm_id_is_current_world(to.vm->id)) {
592 assert((orig_to_mode &
593 (MM_MODE_INVALID | MM_MODE_UNOWNED |
594 MM_MODE_SHARED)) != 0U);
595 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000596 } else {
597 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +0100598 * If the retriever is from virtual FF-A instance:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000599 * Ensure the retriever has the expected state. We don't care
600 * about the MM_MODE_SHARED bit; either with or without it set
601 * are both valid representations of the !O-NA state.
602 */
J-Alvesa9cd7e32022-07-01 13:49:33 +0100603 if (vm_id_is_current_world(to.vm->id) &&
604 to.vm->id != HF_PRIMARY_VM_ID &&
605 (orig_to_mode & MM_MODE_UNMAPPED_MASK) !=
606 MM_MODE_UNMAPPED_MASK) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100607 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000608 }
609 }
610
611 /* Find the appropriate new mode. */
612 *to_mode = memory_to_attributes;
613 switch (share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100614 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000615 *to_mode |= 0;
616 break;
617
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100618 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000619 *to_mode |= MM_MODE_UNOWNED;
620 break;
621
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100622 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000623 *to_mode |= MM_MODE_UNOWNED | MM_MODE_SHARED;
624 break;
625
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100626 case FFA_MEM_RECLAIM_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000627 *to_mode |= 0;
628 break;
629
630 default:
Andrew Walbranca808b12020-05-15 17:22:28 +0100631 dlog_error("Invalid share_func %#x.\n", share_func);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100632 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000633 }
634
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100635 return (struct ffa_value){.func = FFA_SUCCESS_32};
Jose Marinho75509b42019-04-09 09:34:59 +0100636}
Jose Marinho09b1db82019-08-08 09:16:59 +0100637
638/**
639 * Updates a VM's page table such that the given set of physical address ranges
640 * are mapped in the address space at the corresponding address ranges, in the
641 * mode provided.
642 *
643 * If commit is false, the page tables will be allocated from the mpool but no
644 * mappings will actually be updated. This function must always be called first
645 * with commit false to check that it will succeed before calling with commit
646 * true, to avoid leaving the page table in a half-updated state. To make a
647 * series of changes atomically you can call them all with commit false before
648 * calling them all with commit true.
649 *
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -0700650 * vm_ptable_defrag should always be called after a series of page table
651 * updates, whether they succeed or fail.
Jose Marinho09b1db82019-08-08 09:16:59 +0100652 *
653 * Returns true on success, or false if the update failed and no changes were
654 * made to memory mappings.
655 */
J-Alves66652252022-07-06 09:49:51 +0100656bool ffa_region_group_identity_map(
Andrew Walbranf4b51af2020-02-03 14:44:54 +0000657 struct vm_locked vm_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +0100658 struct ffa_memory_region_constituent **fragments,
659 const uint32_t *fragment_constituent_counts, uint32_t fragment_count,
Daniel Boulby4dd3f532021-09-21 09:57:08 +0100660 uint32_t mode, struct mpool *ppool, bool commit)
Jose Marinho09b1db82019-08-08 09:16:59 +0100661{
Andrew Walbranca808b12020-05-15 17:22:28 +0100662 uint32_t i;
663 uint32_t j;
Jose Marinho09b1db82019-08-08 09:16:59 +0100664
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -0700665 if (vm_locked.vm->el0_partition) {
666 mode |= MM_MODE_USER | MM_MODE_NG;
667 }
668
Andrew Walbranca808b12020-05-15 17:22:28 +0100669 /* Iterate over the memory region constituents within each fragment. */
670 for (i = 0; i < fragment_count; ++i) {
671 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
672 size_t size = fragments[i][j].page_count * PAGE_SIZE;
673 paddr_t pa_begin =
674 pa_from_ipa(ipa_init(fragments[i][j].address));
675 paddr_t pa_end = pa_add(pa_begin, size);
Jens Wiklander4f1880c2022-10-19 17:00:14 +0200676 uint32_t pa_bits =
677 arch_mm_get_pa_bits(arch_mm_get_pa_range());
Federico Recanati4fd065d2021-12-13 20:06:23 +0100678
679 /*
680 * Ensure the requested region falls into system's PA
681 * range.
682 */
Jens Wiklander4f1880c2022-10-19 17:00:14 +0200683 if (((pa_addr(pa_begin) >> pa_bits) > 0) ||
684 ((pa_addr(pa_end) >> pa_bits) > 0)) {
Federico Recanati4fd065d2021-12-13 20:06:23 +0100685 dlog_error("Region is outside of PA Range\n");
686 return false;
687 }
Andrew Walbranca808b12020-05-15 17:22:28 +0100688
689 if (commit) {
690 vm_identity_commit(vm_locked, pa_begin, pa_end,
691 mode, ppool, NULL);
692 } else if (!vm_identity_prepare(vm_locked, pa_begin,
693 pa_end, mode, ppool)) {
694 return false;
695 }
Jose Marinho09b1db82019-08-08 09:16:59 +0100696 }
697 }
698
699 return true;
700}
701
702/**
703 * Clears a region of physical memory by overwriting it with zeros. The data is
704 * flushed from the cache so the memory has been cleared across the system.
705 */
J-Alves7db32002021-12-14 14:44:50 +0000706static bool clear_memory(paddr_t begin, paddr_t end, struct mpool *ppool,
707 uint32_t extra_mode_attributes)
Jose Marinho09b1db82019-08-08 09:16:59 +0100708{
709 /*
Fuad Tabbaed294af2019-12-20 10:43:01 +0000710 * TODO: change this to a CPU local single page window rather than a
Jose Marinho09b1db82019-08-08 09:16:59 +0100711 * global mapping of the whole range. Such an approach will limit
712 * the changes to stage-1 tables and will allow only local
713 * invalidation.
714 */
715 bool ret;
716 struct mm_stage1_locked stage1_locked = mm_lock_stage1();
J-Alves7db32002021-12-14 14:44:50 +0000717 void *ptr = mm_identity_map(stage1_locked, begin, end,
718 MM_MODE_W | (extra_mode_attributes &
719 plat_ffa_other_world_mode()),
720 ppool);
Jose Marinho09b1db82019-08-08 09:16:59 +0100721 size_t size = pa_difference(begin, end);
722
723 if (!ptr) {
Jose Marinho09b1db82019-08-08 09:16:59 +0100724 goto fail;
725 }
726
727 memset_s(ptr, size, 0, size);
728 arch_mm_flush_dcache(ptr, size);
729 mm_unmap(stage1_locked, begin, end, ppool);
730
731 ret = true;
732 goto out;
733
734fail:
735 ret = false;
736
737out:
738 mm_unlock_stage1(&stage1_locked);
739
740 return ret;
741}
742
743/**
744 * Clears a region of physical memory by overwriting it with zeros. The data is
745 * flushed from the cache so the memory has been cleared across the system.
746 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100747static bool ffa_clear_memory_constituents(
J-Alves7db32002021-12-14 14:44:50 +0000748 uint32_t security_state_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100749 struct ffa_memory_region_constituent **fragments,
750 const uint32_t *fragment_constituent_counts, uint32_t fragment_count,
751 struct mpool *page_pool)
Jose Marinho09b1db82019-08-08 09:16:59 +0100752{
753 struct mpool local_page_pool;
Andrew Walbranca808b12020-05-15 17:22:28 +0100754 uint32_t i;
Jose Marinho09b1db82019-08-08 09:16:59 +0100755 bool ret = false;
756
757 /*
758 * Create a local pool so any freed memory can't be used by another
759 * thread. This is to ensure each constituent that is mapped can be
760 * unmapped again afterwards.
761 */
Andrew Walbran475c1452020-02-07 13:22:22 +0000762 mpool_init_with_fallback(&local_page_pool, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +0100763
Andrew Walbranca808b12020-05-15 17:22:28 +0100764 /* Iterate over the memory region constituents within each fragment. */
765 for (i = 0; i < fragment_count; ++i) {
766 uint32_t j;
Jose Marinho09b1db82019-08-08 09:16:59 +0100767
Andrew Walbranca808b12020-05-15 17:22:28 +0100768 for (j = 0; j < fragment_constituent_counts[j]; ++j) {
769 size_t size = fragments[i][j].page_count * PAGE_SIZE;
770 paddr_t begin =
771 pa_from_ipa(ipa_init(fragments[i][j].address));
772 paddr_t end = pa_add(begin, size);
773
J-Alves7db32002021-12-14 14:44:50 +0000774 if (!clear_memory(begin, end, &local_page_pool,
775 security_state_mode)) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100776 /*
777 * api_clear_memory will defrag on failure, so
778 * no need to do it here.
779 */
780 goto out;
781 }
Jose Marinho09b1db82019-08-08 09:16:59 +0100782 }
783 }
784
Jose Marinho09b1db82019-08-08 09:16:59 +0100785 ret = true;
786
787out:
788 mpool_fini(&local_page_pool);
789 return ret;
790}
791
792/**
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000793 * Validates and prepares memory to be sent from the calling VM to another.
Jose Marinho09b1db82019-08-08 09:16:59 +0100794 *
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000795 * This function requires the calling context to hold the <from> VM lock.
Jose Marinho09b1db82019-08-08 09:16:59 +0100796 *
797 * Returns:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000798 * In case of error, one of the following values is returned:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100799 * 1) FFA_INVALID_PARAMETERS - The endpoint provided parameters were
Jose Marinho09b1db82019-08-08 09:16:59 +0100800 * erroneous;
Andrew Walbranf07f04d2020-05-01 18:09:00 +0100801 * 2) FFA_NO_MEMORY - Hafnium did not have sufficient memory to complete the
802 * request.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100803 * 3) FFA_DENIED - The sender doesn't have sufficient access to send the
Andrew Walbrana65a1322020-04-06 19:32:32 +0100804 * memory with the given permissions.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100805 * Success is indicated by FFA_SUCCESS.
Jose Marinho09b1db82019-08-08 09:16:59 +0100806 */
J-Alves66652252022-07-06 09:49:51 +0100807struct ffa_value ffa_send_check_update(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000808 struct vm_locked from_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +0100809 struct ffa_memory_region_constituent **fragments,
810 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
J-Alves8f11cde2022-12-21 16:18:22 +0000811 uint32_t composite_total_page_count, uint32_t share_func,
812 struct ffa_memory_access *receivers, uint32_t receivers_count,
813 struct mpool *page_pool, bool clear, uint32_t *orig_from_mode_ret)
Jose Marinho09b1db82019-08-08 09:16:59 +0100814{
Andrew Walbranca808b12020-05-15 17:22:28 +0100815 uint32_t i;
J-Alves8f11cde2022-12-21 16:18:22 +0000816 uint32_t j;
Jose Marinho09b1db82019-08-08 09:16:59 +0100817 uint32_t orig_from_mode;
818 uint32_t from_mode;
Jose Marinho09b1db82019-08-08 09:16:59 +0100819 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100820 struct ffa_value ret;
J-Alves8f11cde2022-12-21 16:18:22 +0000821 uint32_t constituents_total_page_count = 0;
Jose Marinho09b1db82019-08-08 09:16:59 +0100822
823 /*
Andrew Walbrana65a1322020-04-06 19:32:32 +0100824 * Make sure constituents are properly aligned to a 64-bit boundary. If
825 * not we would get alignment faults trying to read (64-bit) values.
Jose Marinho09b1db82019-08-08 09:16:59 +0100826 */
Andrew Walbranca808b12020-05-15 17:22:28 +0100827 for (i = 0; i < fragment_count; ++i) {
828 if (!is_aligned(fragments[i], 8)) {
829 dlog_verbose("Constituents not aligned.\n");
830 return ffa_error(FFA_INVALID_PARAMETERS);
831 }
J-Alves8f11cde2022-12-21 16:18:22 +0000832 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
833 constituents_total_page_count +=
834 fragments[i][j].page_count;
835 }
836 }
837
838 if (constituents_total_page_count != composite_total_page_count) {
839 dlog_verbose(
840 "Composite page count differs from calculated page "
841 "count from constituents.\n");
842 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho09b1db82019-08-08 09:16:59 +0100843 }
844
845 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000846 * Check if the state transition is lawful for the sender, ensure that
847 * all constituents of a memory region being shared are at the same
848 * state.
Jose Marinho09b1db82019-08-08 09:16:59 +0100849 */
J-Alves363f5722022-04-25 17:37:37 +0100850 ret = ffa_send_check_transition(from_locked, share_func, receivers,
851 receivers_count, &orig_from_mode,
852 fragments, fragment_constituent_counts,
Andrew Walbranca808b12020-05-15 17:22:28 +0100853 fragment_count, &from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100854 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100855 dlog_verbose("Invalid transition for send.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100856 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +0100857 }
858
Andrew Walbran37c574e2020-06-03 11:45:46 +0100859 if (orig_from_mode_ret != NULL) {
860 *orig_from_mode_ret = orig_from_mode;
861 }
862
Jose Marinho09b1db82019-08-08 09:16:59 +0100863 /*
864 * Create a local pool so any freed memory can't be used by another
865 * thread. This is to ensure the original mapping can be restored if the
866 * clear fails.
867 */
Andrew Walbran475c1452020-02-07 13:22:22 +0000868 mpool_init_with_fallback(&local_page_pool, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +0100869
870 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000871 * First reserve all required memory for the new page table entries
872 * without committing, to make sure the entire operation will succeed
873 * without exhausting the page pool.
Jose Marinho09b1db82019-08-08 09:16:59 +0100874 */
Andrew Walbranca808b12020-05-15 17:22:28 +0100875 if (!ffa_region_group_identity_map(
876 from_locked, fragments, fragment_constituent_counts,
877 fragment_count, from_mode, page_pool, false)) {
Jose Marinho09b1db82019-08-08 09:16:59 +0100878 /* TODO: partial defrag of failed range. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100879 ret = ffa_error(FFA_NO_MEMORY);
Jose Marinho09b1db82019-08-08 09:16:59 +0100880 goto out;
881 }
882
883 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000884 * Update the mapping for the sender. This won't allocate because the
885 * transaction was already prepared above, but may free pages in the
886 * case that a whole block is being unmapped that was previously
887 * partially mapped.
Jose Marinho09b1db82019-08-08 09:16:59 +0100888 */
Andrew Walbranca808b12020-05-15 17:22:28 +0100889 CHECK(ffa_region_group_identity_map(
890 from_locked, fragments, fragment_constituent_counts,
891 fragment_count, from_mode, &local_page_pool, true));
Jose Marinho09b1db82019-08-08 09:16:59 +0100892
893 /* Clear the memory so no VM or device can see the previous contents. */
J-Alves7db32002021-12-14 14:44:50 +0000894 if (clear &&
895 !ffa_clear_memory_constituents(
896 plat_ffa_owner_world_mode(from_locked.vm->id), fragments,
897 fragment_constituent_counts, fragment_count, page_pool)) {
Jose Marinho09b1db82019-08-08 09:16:59 +0100898 /*
899 * On failure, roll back by returning memory to the sender. This
900 * may allocate pages which were previously freed into
901 * `local_page_pool` by the call above, but will never allocate
902 * more pages than that so can never fail.
903 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100904 CHECK(ffa_region_group_identity_map(
Andrew Walbranca808b12020-05-15 17:22:28 +0100905 from_locked, fragments, fragment_constituent_counts,
906 fragment_count, orig_from_mode, &local_page_pool,
907 true));
Jose Marinho09b1db82019-08-08 09:16:59 +0100908
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100909 ret = ffa_error(FFA_NO_MEMORY);
Jose Marinho09b1db82019-08-08 09:16:59 +0100910 goto out;
911 }
912
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100913 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000914
915out:
916 mpool_fini(&local_page_pool);
917
918 /*
919 * Tidy up the page table by reclaiming failed mappings (if there was an
920 * error) or merging entries into blocks where possible (on success).
921 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -0700922 vm_ptable_defrag(from_locked, page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000923
924 return ret;
925}
926
927/**
928 * Validates and maps memory shared from one VM to another.
929 *
930 * This function requires the calling context to hold the <to> lock.
931 *
932 * Returns:
933 * In case of error, one of the following values is returned:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100934 * 1) FFA_INVALID_PARAMETERS - The endpoint provided parameters were
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000935 * erroneous;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100936 * 2) FFA_NO_MEMORY - Hafnium did not have sufficient memory to complete
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000937 * the request.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100938 * Success is indicated by FFA_SUCCESS.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000939 */
J-Alvesb5084cf2022-07-06 14:20:12 +0100940struct ffa_value ffa_retrieve_check_update(
J-Alves7db32002021-12-14 14:44:50 +0000941 struct vm_locked to_locked, ffa_vm_id_t from_id,
Andrew Walbranca808b12020-05-15 17:22:28 +0100942 struct ffa_memory_region_constituent **fragments,
943 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
944 uint32_t memory_to_attributes, uint32_t share_func, bool clear,
945 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000946{
Andrew Walbranca808b12020-05-15 17:22:28 +0100947 uint32_t i;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000948 uint32_t to_mode;
949 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100950 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000951
952 /*
Andrew Walbranca808b12020-05-15 17:22:28 +0100953 * Make sure constituents are properly aligned to a 64-bit boundary. If
954 * not we would get alignment faults trying to read (64-bit) values.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000955 */
Andrew Walbranca808b12020-05-15 17:22:28 +0100956 for (i = 0; i < fragment_count; ++i) {
957 if (!is_aligned(fragments[i], 8)) {
J-Alvesb5084cf2022-07-06 14:20:12 +0100958 dlog_verbose("Fragment not properly aligned.\n");
Andrew Walbranca808b12020-05-15 17:22:28 +0100959 return ffa_error(FFA_INVALID_PARAMETERS);
960 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000961 }
962
963 /*
964 * Check if the state transition is lawful for the recipient, and ensure
965 * that all constituents of the memory region being retrieved are at the
966 * same state.
967 */
Andrew Walbranca808b12020-05-15 17:22:28 +0100968 ret = ffa_retrieve_check_transition(
969 to_locked, share_func, fragments, fragment_constituent_counts,
970 fragment_count, memory_to_attributes, &to_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100971 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100972 dlog_verbose("Invalid transition for retrieve.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100973 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000974 }
975
976 /*
977 * Create a local pool so any freed memory can't be used by another
978 * thread. This is to ensure the original mapping can be restored if the
979 * clear fails.
980 */
981 mpool_init_with_fallback(&local_page_pool, page_pool);
982
983 /*
984 * First reserve all required memory for the new page table entries in
985 * the recipient page tables without committing, to make sure the entire
986 * operation will succeed without exhausting the page pool.
987 */
Andrew Walbranca808b12020-05-15 17:22:28 +0100988 if (!ffa_region_group_identity_map(
989 to_locked, fragments, fragment_constituent_counts,
990 fragment_count, to_mode, page_pool, false)) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000991 /* TODO: partial defrag of failed range. */
992 dlog_verbose(
993 "Insufficient memory to update recipient page "
994 "table.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100995 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000996 goto out;
997 }
998
999 /* Clear the memory so no VM or device can see the previous contents. */
J-Alves7db32002021-12-14 14:44:50 +00001000 if (clear &&
1001 !ffa_clear_memory_constituents(
1002 plat_ffa_owner_world_mode(from_id), fragments,
1003 fragment_constituent_counts, fragment_count, page_pool)) {
J-Alvesb5084cf2022-07-06 14:20:12 +01001004 dlog_verbose("Couldn't clear constituents.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001005 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001006 goto out;
1007 }
1008
Jose Marinho09b1db82019-08-08 09:16:59 +01001009 /*
1010 * Complete the transfer by mapping the memory into the recipient. This
1011 * won't allocate because the transaction was already prepared above, so
1012 * it doesn't need to use the `local_page_pool`.
1013 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001014 CHECK(ffa_region_group_identity_map(
1015 to_locked, fragments, fragment_constituent_counts,
1016 fragment_count, to_mode, page_pool, true));
Jose Marinho09b1db82019-08-08 09:16:59 +01001017
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001018 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Jose Marinho09b1db82019-08-08 09:16:59 +01001019
1020out:
1021 mpool_fini(&local_page_pool);
1022
1023 /*
Andrew Walbranf07f04d2020-05-01 18:09:00 +01001024 * Tidy up the page table by reclaiming failed mappings (if there was an
1025 * error) or merging entries into blocks where possible (on success).
Jose Marinho09b1db82019-08-08 09:16:59 +01001026 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001027 vm_ptable_defrag(to_locked, page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001028
1029 return ret;
1030}
1031
Andrew Walbran996d1d12020-05-27 14:08:43 +01001032static struct ffa_value ffa_relinquish_check_update(
J-Alves3c5b2072022-11-21 12:45:40 +00001033 struct vm_locked from_locked, ffa_vm_id_t owner_id,
Andrew Walbranca808b12020-05-15 17:22:28 +01001034 struct ffa_memory_region_constituent **fragments,
1035 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
1036 struct mpool *page_pool, bool clear)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001037{
1038 uint32_t orig_from_mode;
1039 uint32_t from_mode;
1040 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001041 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001042
Andrew Walbranca808b12020-05-15 17:22:28 +01001043 ret = ffa_relinquish_check_transition(
1044 from_locked, &orig_from_mode, fragments,
1045 fragment_constituent_counts, fragment_count, &from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001046 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001047 dlog_verbose("Invalid transition for relinquish.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +01001048 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001049 }
1050
1051 /*
1052 * Create a local pool so any freed memory can't be used by another
1053 * thread. This is to ensure the original mapping can be restored if the
1054 * clear fails.
1055 */
1056 mpool_init_with_fallback(&local_page_pool, page_pool);
1057
1058 /*
1059 * First reserve all required memory for the new page table entries
1060 * without committing, to make sure the entire operation will succeed
1061 * without exhausting the page pool.
1062 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001063 if (!ffa_region_group_identity_map(
1064 from_locked, fragments, fragment_constituent_counts,
1065 fragment_count, from_mode, page_pool, false)) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001066 /* TODO: partial defrag of failed range. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001067 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001068 goto out;
1069 }
1070
1071 /*
1072 * Update the mapping for the sender. This won't allocate because the
1073 * transaction was already prepared above, but may free pages in the
1074 * case that a whole block is being unmapped that was previously
1075 * partially mapped.
1076 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001077 CHECK(ffa_region_group_identity_map(
1078 from_locked, fragments, fragment_constituent_counts,
1079 fragment_count, from_mode, &local_page_pool, true));
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001080
1081 /* Clear the memory so no VM or device can see the previous contents. */
J-Alves7db32002021-12-14 14:44:50 +00001082 if (clear &&
1083 !ffa_clear_memory_constituents(
J-Alves3c5b2072022-11-21 12:45:40 +00001084 plat_ffa_owner_world_mode(owner_id), fragments,
J-Alves7db32002021-12-14 14:44:50 +00001085 fragment_constituent_counts, fragment_count, page_pool)) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001086 /*
1087 * On failure, roll back by returning memory to the sender. This
1088 * may allocate pages which were previously freed into
1089 * `local_page_pool` by the call above, but will never allocate
1090 * more pages than that so can never fail.
1091 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001092 CHECK(ffa_region_group_identity_map(
Andrew Walbranca808b12020-05-15 17:22:28 +01001093 from_locked, fragments, fragment_constituent_counts,
1094 fragment_count, orig_from_mode, &local_page_pool,
1095 true));
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001096
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001097 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001098 goto out;
1099 }
1100
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001101 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001102
1103out:
1104 mpool_fini(&local_page_pool);
1105
1106 /*
1107 * Tidy up the page table by reclaiming failed mappings (if there was an
1108 * error) or merging entries into blocks where possible (on success).
1109 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001110 vm_ptable_defrag(from_locked, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +01001111
1112 return ret;
1113}
1114
1115/**
Andrew Walbranca808b12020-05-15 17:22:28 +01001116 * Complete a memory sending operation by checking that it is valid, updating
1117 * the sender page table, and then either marking the share state as having
1118 * completed sending (on success) or freeing it (on failure).
1119 *
1120 * Returns FFA_SUCCESS with the handle encoded, or the relevant FFA_ERROR.
1121 */
J-Alvesfdd29272022-07-19 13:16:31 +01001122struct ffa_value ffa_memory_send_complete(
Andrew Walbranca808b12020-05-15 17:22:28 +01001123 struct vm_locked from_locked, struct share_states_locked share_states,
Andrew Walbran37c574e2020-06-03 11:45:46 +01001124 struct ffa_memory_share_state *share_state, struct mpool *page_pool,
1125 uint32_t *orig_from_mode_ret)
Andrew Walbranca808b12020-05-15 17:22:28 +01001126{
1127 struct ffa_memory_region *memory_region = share_state->memory_region;
J-Alves8f11cde2022-12-21 16:18:22 +00001128 struct ffa_composite_memory_region *composite;
Andrew Walbranca808b12020-05-15 17:22:28 +01001129 struct ffa_value ret;
1130
1131 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +00001132 assert(share_states.share_states != NULL);
J-Alves8f11cde2022-12-21 16:18:22 +00001133 assert(memory_region != NULL);
1134 composite = ffa_memory_region_get_composite(memory_region, 0);
1135 assert(composite != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +01001136
1137 /* Check that state is valid in sender page table and update. */
1138 ret = ffa_send_check_update(
1139 from_locked, share_state->fragments,
1140 share_state->fragment_constituent_counts,
J-Alves8f11cde2022-12-21 16:18:22 +00001141 share_state->fragment_count, composite->page_count,
1142 share_state->share_func, memory_region->receivers,
1143 memory_region->receiver_count, page_pool,
1144 memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR,
Andrew Walbran37c574e2020-06-03 11:45:46 +01001145 orig_from_mode_ret);
Andrew Walbranca808b12020-05-15 17:22:28 +01001146 if (ret.func != FFA_SUCCESS_32) {
1147 /*
1148 * Free share state, it failed to send so it can't be retrieved.
1149 */
1150 dlog_verbose("Complete failed, freeing share state.\n");
1151 share_state_free(share_states, share_state, page_pool);
1152 return ret;
1153 }
1154
1155 share_state->sending_complete = true;
1156 dlog_verbose("Marked sending complete.\n");
1157
J-Alvesee68c542020-10-29 17:48:20 +00001158 return ffa_mem_success(share_state->memory_region->handle);
Andrew Walbranca808b12020-05-15 17:22:28 +01001159}
1160
1161/**
Federico Recanatia98603a2021-12-20 18:04:03 +01001162 * Check that the memory attributes match Hafnium expectations:
1163 * Normal Memory, Inner shareable, Write-Back Read-Allocate
1164 * Write-Allocate Cacheable.
1165 */
1166static struct ffa_value ffa_memory_attributes_validate(
J-Alves7a99d0d2023-02-08 13:49:48 +00001167 ffa_memory_attributes_t attributes)
Federico Recanatia98603a2021-12-20 18:04:03 +01001168{
1169 enum ffa_memory_type memory_type;
1170 enum ffa_memory_cacheability cacheability;
1171 enum ffa_memory_shareability shareability;
1172
1173 memory_type = ffa_get_memory_type_attr(attributes);
1174 if (memory_type != FFA_MEMORY_NORMAL_MEM) {
1175 dlog_verbose("Invalid memory type %#x, expected %#x.\n",
1176 memory_type, FFA_MEMORY_NORMAL_MEM);
Federico Recanati3d953f32022-02-17 09:31:29 +01001177 return ffa_error(FFA_DENIED);
Federico Recanatia98603a2021-12-20 18:04:03 +01001178 }
1179
1180 cacheability = ffa_get_memory_cacheability_attr(attributes);
1181 if (cacheability != FFA_MEMORY_CACHE_WRITE_BACK) {
1182 dlog_verbose("Invalid cacheability %#x, expected %#x.\n",
1183 cacheability, FFA_MEMORY_CACHE_WRITE_BACK);
Federico Recanati3d953f32022-02-17 09:31:29 +01001184 return ffa_error(FFA_DENIED);
Federico Recanatia98603a2021-12-20 18:04:03 +01001185 }
1186
1187 shareability = ffa_get_memory_shareability_attr(attributes);
1188 if (shareability != FFA_MEMORY_INNER_SHAREABLE) {
1189 dlog_verbose("Invalid shareability %#x, expected #%x.\n",
1190 shareability, FFA_MEMORY_INNER_SHAREABLE);
Federico Recanati3d953f32022-02-17 09:31:29 +01001191 return ffa_error(FFA_DENIED);
Federico Recanatia98603a2021-12-20 18:04:03 +01001192 }
1193
1194 return (struct ffa_value){.func = FFA_SUCCESS_32};
1195}
1196
1197/**
Andrew Walbrana65a1322020-04-06 19:32:32 +01001198 * Check that the given `memory_region` represents a valid memory send request
1199 * of the given `share_func` type, return the clear flag and permissions via the
1200 * respective output parameters, and update the permissions if necessary.
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001201 *
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001202 * Returns FFA_SUCCESS if the request was valid, or the relevant FFA_ERROR if
Andrew Walbrana65a1322020-04-06 19:32:32 +01001203 * not.
1204 */
J-Alves66652252022-07-06 09:49:51 +01001205struct ffa_value ffa_memory_send_validate(
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001206 struct vm_locked from_locked, struct ffa_memory_region *memory_region,
1207 uint32_t memory_share_length, uint32_t fragment_length,
J-Alves363f5722022-04-25 17:37:37 +01001208 uint32_t share_func)
Andrew Walbrana65a1322020-04-06 19:32:32 +01001209{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001210 struct ffa_composite_memory_region *composite;
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001211 uint64_t receivers_end;
1212 uint64_t min_length;
Federico Recanati872cd692022-01-05 13:10:10 +01001213 uint32_t composite_memory_region_offset;
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001214 uint32_t constituents_start;
Andrew Walbran130a8ae2020-05-15 16:27:15 +01001215 uint32_t constituents_length;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001216 enum ffa_data_access data_access;
1217 enum ffa_instruction_access instruction_access;
Olivier Deprez4342a3c2022-02-28 09:37:25 +01001218 enum ffa_memory_security security_state;
Federico Recanatia98603a2021-12-20 18:04:03 +01001219 struct ffa_value ret;
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001220 const size_t minimum_first_fragment_length =
1221 (sizeof(struct ffa_memory_region) +
1222 sizeof(struct ffa_memory_access) +
1223 sizeof(struct ffa_composite_memory_region));
1224
1225 if (fragment_length < minimum_first_fragment_length) {
1226 dlog_verbose("Fragment length %u too short (min %u).\n",
1227 (size_t)fragment_length,
1228 minimum_first_fragment_length);
1229 return ffa_error(FFA_INVALID_PARAMETERS);
1230 }
1231
Demi Marie Obenour73a1e942023-02-04 14:09:18 -05001232 static_assert(sizeof(struct ffa_memory_region_constituent) == 16,
1233 "struct ffa_memory_region_constituent must be 16 bytes");
1234 if (!is_aligned(fragment_length,
1235 sizeof(struct ffa_memory_region_constituent)) ||
1236 !is_aligned(memory_share_length,
1237 sizeof(struct ffa_memory_region_constituent))) {
1238 dlog_verbose(
1239 "Fragment length %u or total length %u"
1240 " is not 16-byte aligned.\n",
1241 fragment_length, memory_share_length);
1242 return ffa_error(FFA_INVALID_PARAMETERS);
1243 }
1244
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001245 if (fragment_length > memory_share_length) {
1246 dlog_verbose(
1247 "Fragment length %u greater than total length %u.\n",
1248 (size_t)fragment_length, (size_t)memory_share_length);
1249 return ffa_error(FFA_INVALID_PARAMETERS);
1250 }
Andrew Walbrana65a1322020-04-06 19:32:32 +01001251
J-Alves0b6653d2022-04-22 13:17:38 +01001252 assert(memory_region->receivers_offset ==
1253 offsetof(struct ffa_memory_region, receivers));
1254 assert(memory_region->memory_access_desc_size ==
1255 sizeof(struct ffa_memory_access));
1256
J-Alves95df0ef2022-12-07 10:09:48 +00001257 /* The sender must match the caller. */
1258 if ((!vm_id_is_current_world(from_locked.vm->id) &&
1259 vm_id_is_current_world(memory_region->sender)) ||
1260 (vm_id_is_current_world(from_locked.vm->id) &&
1261 memory_region->sender != from_locked.vm->id)) {
1262 dlog_verbose("Invalid memory sender ID.\n");
1263 return ffa_error(FFA_DENIED);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001264 }
1265
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001266 if (memory_region->receiver_count <= 0) {
1267 dlog_verbose("No receivers!\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001268 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001269 }
1270
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001271 /*
1272 * Ensure that the composite header is within the memory bounds and
1273 * doesn't overlap the first part of the message. Cast to uint64_t
1274 * to prevent overflow.
1275 */
1276 receivers_end = ((uint64_t)sizeof(struct ffa_memory_access) *
1277 (uint64_t)memory_region->receiver_count) +
1278 sizeof(struct ffa_memory_region);
1279 min_length = receivers_end +
1280 sizeof(struct ffa_composite_memory_region) +
1281 sizeof(struct ffa_memory_region_constituent);
1282 if (min_length > memory_share_length) {
1283 dlog_verbose("Share too short: got %u but minimum is %u.\n",
1284 (size_t)memory_share_length, (size_t)min_length);
1285 return ffa_error(FFA_INVALID_PARAMETERS);
1286 }
1287
1288 composite_memory_region_offset =
1289 memory_region->receivers[0].composite_memory_region_offset;
Andrew Walbrana65a1322020-04-06 19:32:32 +01001290
1291 /*
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001292 * Check that the composite memory region descriptor is after the access
1293 * descriptors, is at least 16-byte aligned, and fits in the first
1294 * fragment.
Andrew Walbrana65a1322020-04-06 19:32:32 +01001295 */
Demi Marie Obenourd4677412023-02-03 20:35:12 -05001296 if ((composite_memory_region_offset < receivers_end) ||
1297 (composite_memory_region_offset % 16 != 0) ||
1298 (composite_memory_region_offset >
1299 fragment_length - sizeof(struct ffa_composite_memory_region))) {
1300 dlog_verbose(
1301 "Invalid composite memory region descriptor offset "
1302 "%u.\n",
1303 (size_t)composite_memory_region_offset);
1304 return ffa_error(FFA_INVALID_PARAMETERS);
1305 }
1306
1307 /*
1308 * Compute the start of the constituent regions. Already checked
1309 * to be not more than fragment_length and thus not more than
1310 * memory_share_length.
1311 */
1312 constituents_start = composite_memory_region_offset +
1313 sizeof(struct ffa_composite_memory_region);
1314 constituents_length = memory_share_length - constituents_start;
1315
1316 /*
1317 * Check that the number of constituents is consistent with the length
1318 * of the constituent region.
1319 */
1320 composite = ffa_memory_region_get_composite(memory_region, 0);
1321 if ((constituents_length %
1322 sizeof(struct ffa_memory_region_constituent) !=
1323 0) ||
1324 ((constituents_length /
1325 sizeof(struct ffa_memory_region_constituent)) !=
1326 composite->constituent_count)) {
1327 dlog_verbose("Invalid length %u or composite offset %u.\n",
1328 (size_t)memory_share_length,
1329 (size_t)composite_memory_region_offset);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001330 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001331 }
Andrew Walbranca808b12020-05-15 17:22:28 +01001332 if (fragment_length < memory_share_length &&
1333 fragment_length < HF_MAILBOX_SIZE) {
1334 dlog_warning(
1335 "Initial fragment length %d smaller than mailbox "
1336 "size.\n",
1337 fragment_length);
1338 }
Andrew Walbrana65a1322020-04-06 19:32:32 +01001339
Andrew Walbrana65a1322020-04-06 19:32:32 +01001340 /*
1341 * Clear is not allowed for memory sharing, as the sender still has
1342 * access to the memory.
1343 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001344 if ((memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR) &&
1345 share_func == FFA_MEM_SHARE_32) {
Andrew Walbrana65a1322020-04-06 19:32:32 +01001346 dlog_verbose("Memory can't be cleared while being shared.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001347 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001348 }
1349
1350 /* No other flags are allowed/supported here. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001351 if (memory_region->flags & ~FFA_MEMORY_REGION_FLAG_CLEAR) {
Andrew Walbrana65a1322020-04-06 19:32:32 +01001352 dlog_verbose("Invalid flags %#x.\n", memory_region->flags);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001353 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001354 }
1355
J-Alves363f5722022-04-25 17:37:37 +01001356 /* Check that the permissions are valid, for each specified receiver. */
1357 for (uint32_t i = 0U; i < memory_region->receiver_count; i++) {
1358 ffa_memory_access_permissions_t permissions =
1359 memory_region->receivers[i]
1360 .receiver_permissions.permissions;
1361 ffa_vm_id_t receiver_id =
1362 memory_region->receivers[i]
1363 .receiver_permissions.receiver;
1364
1365 if (memory_region->sender == receiver_id) {
1366 dlog_verbose("Can't share memory with itself.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001367 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001368 }
Federico Recanati85090c42021-12-15 13:17:54 +01001369
J-Alves363f5722022-04-25 17:37:37 +01001370 for (uint32_t j = i + 1; j < memory_region->receiver_count;
1371 j++) {
1372 if (receiver_id ==
1373 memory_region->receivers[j]
1374 .receiver_permissions.receiver) {
1375 dlog_verbose(
1376 "Repeated receiver(%x) in memory send "
1377 "operation.\n",
1378 memory_region->receivers[j]
1379 .receiver_permissions.receiver);
1380 return ffa_error(FFA_INVALID_PARAMETERS);
1381 }
1382 }
1383
1384 if (composite_memory_region_offset !=
1385 memory_region->receivers[i]
1386 .composite_memory_region_offset) {
1387 dlog_verbose(
1388 "All ffa_memory_access should point to the "
1389 "same composite memory region offset.\n");
1390 return ffa_error(FFA_INVALID_PARAMETERS);
1391 }
1392
1393 data_access = ffa_get_data_access_attr(permissions);
1394 instruction_access =
1395 ffa_get_instruction_access_attr(permissions);
1396 if (data_access == FFA_DATA_ACCESS_RESERVED ||
1397 instruction_access == FFA_INSTRUCTION_ACCESS_RESERVED) {
1398 dlog_verbose(
1399 "Reserved value for receiver permissions "
1400 "%#x.\n",
1401 permissions);
1402 return ffa_error(FFA_INVALID_PARAMETERS);
1403 }
1404 if (instruction_access !=
1405 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
1406 dlog_verbose(
1407 "Invalid instruction access permissions %#x "
1408 "for sending memory.\n",
1409 permissions);
1410 return ffa_error(FFA_INVALID_PARAMETERS);
1411 }
1412 if (share_func == FFA_MEM_SHARE_32) {
1413 if (data_access == FFA_DATA_ACCESS_NOT_SPECIFIED) {
1414 dlog_verbose(
1415 "Invalid data access permissions %#x "
1416 "for sharing memory.\n",
1417 permissions);
1418 return ffa_error(FFA_INVALID_PARAMETERS);
1419 }
1420 /*
1421 * According to section 10.10.3 of the FF-A v1.1 EAC0
1422 * spec, NX is required for share operations (but must
1423 * not be specified by the sender) so set it in the
1424 * copy that we store, ready to be returned to the
1425 * retriever.
1426 */
J-Alvesb19731a2022-06-20 17:30:33 +01001427 if (vm_id_is_current_world(receiver_id)) {
1428 ffa_set_instruction_access_attr(
1429 &permissions,
1430 FFA_INSTRUCTION_ACCESS_NX);
1431 memory_region->receivers[i]
1432 .receiver_permissions.permissions =
1433 permissions;
1434 }
J-Alves363f5722022-04-25 17:37:37 +01001435 }
1436 if (share_func == FFA_MEM_LEND_32 &&
1437 data_access == FFA_DATA_ACCESS_NOT_SPECIFIED) {
1438 dlog_verbose(
1439 "Invalid data access permissions %#x for "
1440 "lending memory.\n",
1441 permissions);
1442 return ffa_error(FFA_INVALID_PARAMETERS);
1443 }
1444
1445 if (share_func == FFA_MEM_DONATE_32 &&
1446 data_access != FFA_DATA_ACCESS_NOT_SPECIFIED) {
1447 dlog_verbose(
1448 "Invalid data access permissions %#x for "
1449 "donating memory.\n",
1450 permissions);
1451 return ffa_error(FFA_INVALID_PARAMETERS);
1452 }
Andrew Walbrana65a1322020-04-06 19:32:32 +01001453 }
1454
Olivier Deprez4342a3c2022-02-28 09:37:25 +01001455 /* Memory region attributes NS-Bit MBZ for FFA_MEM_SHARE/LEND/DONATE. */
1456 security_state =
1457 ffa_get_memory_security_attr(memory_region->attributes);
1458 if (security_state != FFA_MEMORY_SECURITY_UNSPECIFIED) {
1459 dlog_verbose(
1460 "Invalid security state for memory share operation.\n");
1461 return ffa_error(FFA_INVALID_PARAMETERS);
1462 }
1463
Federico Recanatid937f5e2021-12-20 17:38:23 +01001464 /*
J-Alves807794e2022-06-16 13:42:47 +01001465 * If a memory donate or lend with single borrower, the memory type
1466 * shall not be specified by the sender.
Federico Recanatid937f5e2021-12-20 17:38:23 +01001467 */
J-Alves807794e2022-06-16 13:42:47 +01001468 if (share_func == FFA_MEM_DONATE_32 ||
1469 (share_func == FFA_MEM_LEND_32 &&
1470 memory_region->receiver_count == 1)) {
1471 if (ffa_get_memory_type_attr(memory_region->attributes) !=
1472 FFA_MEMORY_NOT_SPECIFIED_MEM) {
1473 dlog_verbose(
1474 "Memory type shall not be specified by "
1475 "sender.\n");
1476 return ffa_error(FFA_INVALID_PARAMETERS);
1477 }
1478 } else {
1479 /*
1480 * Check that sender's memory attributes match Hafnium
1481 * expectations: Normal Memory, Inner shareable, Write-Back
1482 * Read-Allocate Write-Allocate Cacheable.
1483 */
1484 ret = ffa_memory_attributes_validate(memory_region->attributes);
1485 if (ret.func != FFA_SUCCESS_32) {
1486 return ret;
1487 }
Federico Recanatid937f5e2021-12-20 17:38:23 +01001488 }
1489
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001490 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbrana65a1322020-04-06 19:32:32 +01001491}
1492
1493/**
Andrew Walbranca808b12020-05-15 17:22:28 +01001494 * Gets the share state for continuing an operation to donate, lend or share
1495 * memory, and checks that it is a valid request.
1496 *
1497 * Returns FFA_SUCCESS if the request was valid, or the relevant FFA_ERROR if
1498 * not.
1499 */
J-Alvesfdd29272022-07-19 13:16:31 +01001500struct ffa_value ffa_memory_send_continue_validate(
Andrew Walbranca808b12020-05-15 17:22:28 +01001501 struct share_states_locked share_states, ffa_memory_handle_t handle,
1502 struct ffa_memory_share_state **share_state_ret, ffa_vm_id_t from_vm_id,
1503 struct mpool *page_pool)
1504{
1505 struct ffa_memory_share_state *share_state;
1506 struct ffa_memory_region *memory_region;
1507
Daniel Boulbya2f8c662021-11-26 17:52:53 +00001508 assert(share_state_ret != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +01001509
1510 /*
1511 * Look up the share state by handle and make sure that the VM ID
1512 * matches.
1513 */
1514 if (!get_share_state(share_states, handle, &share_state)) {
1515 dlog_verbose(
1516 "Invalid handle %#x for memory send continuation.\n",
1517 handle);
1518 return ffa_error(FFA_INVALID_PARAMETERS);
1519 }
1520 memory_region = share_state->memory_region;
1521
J-Alvesfdd29272022-07-19 13:16:31 +01001522 if (vm_id_is_current_world(from_vm_id) &&
1523 memory_region->sender != from_vm_id) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001524 dlog_verbose("Invalid sender %d.\n", memory_region->sender);
1525 return ffa_error(FFA_INVALID_PARAMETERS);
1526 }
1527
1528 if (share_state->sending_complete) {
1529 dlog_verbose(
1530 "Sending of memory handle %#x is already complete.\n",
1531 handle);
1532 return ffa_error(FFA_INVALID_PARAMETERS);
1533 }
1534
1535 if (share_state->fragment_count == MAX_FRAGMENTS) {
1536 /*
1537 * Log a warning as this is a sign that MAX_FRAGMENTS should
1538 * probably be increased.
1539 */
1540 dlog_warning(
1541 "Too many fragments for memory share with handle %#x; "
1542 "only %d supported.\n",
1543 handle, MAX_FRAGMENTS);
1544 /* Free share state, as it's not possible to complete it. */
1545 share_state_free(share_states, share_state, page_pool);
1546 return ffa_error(FFA_NO_MEMORY);
1547 }
1548
1549 *share_state_ret = share_state;
1550
1551 return (struct ffa_value){.func = FFA_SUCCESS_32};
1552}
1553
1554/**
J-Alves95df0ef2022-12-07 10:09:48 +00001555 * Checks if there is at least one receiver from the other world.
1556 */
J-Alvesfdd29272022-07-19 13:16:31 +01001557bool memory_region_receivers_from_other_world(
J-Alves95df0ef2022-12-07 10:09:48 +00001558 struct ffa_memory_region *memory_region)
1559{
1560 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
1561 ffa_vm_id_t receiver = memory_region->receivers[i]
1562 .receiver_permissions.receiver;
1563 if (!vm_id_is_current_world(receiver)) {
1564 return true;
1565 }
1566 }
1567 return false;
1568}
1569
1570/**
J-Alves9da280b2022-12-21 14:55:39 +00001571 * Validates a call to donate, lend or share memory in which Hafnium is the
1572 * designated allocator of the memory handle. In practice, this also means
1573 * Hafnium is responsible for managing the state structures for the transaction.
1574 * If Hafnium is the SPMC, it should allocate the memory handle when either the
1575 * sender is an SP or there is at least one borrower that is an SP.
1576 * If Hafnium is the hypervisor, it should allocate the memory handle when
1577 * operation involves only NWd VMs.
1578 *
1579 * If validation goes well, Hafnium updates the stage-2 page tables of the
1580 * sender. Validation consists of checking if the message length and number of
1581 * memory region constituents match, and if the transition is valid for the
1582 * type of memory sending operation.
Andrew Walbran475c1452020-02-07 13:22:22 +00001583 *
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001584 * Assumes that the caller has already found and locked the sender VM and copied
1585 * the memory region descriptor from the sender's TX buffer to a freshly
1586 * allocated page from Hafnium's internal pool. The caller must have also
1587 * validated that the receiver VM ID is valid.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001588 *
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001589 * This function takes ownership of the `memory_region` passed in and will free
1590 * it when necessary; it must not be freed by the caller.
Jose Marinho09b1db82019-08-08 09:16:59 +01001591 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001592struct ffa_value ffa_memory_send(struct vm_locked from_locked,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001593 struct ffa_memory_region *memory_region,
Andrew Walbran130a8ae2020-05-15 16:27:15 +01001594 uint32_t memory_share_length,
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001595 uint32_t fragment_length, uint32_t share_func,
1596 struct mpool *page_pool)
Jose Marinho09b1db82019-08-08 09:16:59 +01001597{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001598 struct ffa_value ret;
Andrew Walbranca808b12020-05-15 17:22:28 +01001599 struct share_states_locked share_states;
1600 struct ffa_memory_share_state *share_state;
Jose Marinho09b1db82019-08-08 09:16:59 +01001601
1602 /*
Andrew Walbrana65a1322020-04-06 19:32:32 +01001603 * If there is an error validating the `memory_region` then we need to
1604 * free it because we own it but we won't be storing it in a share state
1605 * after all.
Jose Marinho09b1db82019-08-08 09:16:59 +01001606 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001607 ret = ffa_memory_send_validate(from_locked, memory_region,
1608 memory_share_length, fragment_length,
J-Alves363f5722022-04-25 17:37:37 +01001609 share_func);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001610 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001611 mpool_free(page_pool, memory_region);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001612 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01001613 }
1614
Andrew Walbrana65a1322020-04-06 19:32:32 +01001615 /* Set flag for share function, ready to be retrieved later. */
1616 switch (share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001617 case FFA_MEM_SHARE_32:
Andrew Walbrana65a1322020-04-06 19:32:32 +01001618 memory_region->flags |=
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001619 FFA_MEMORY_REGION_TRANSACTION_TYPE_SHARE;
Andrew Walbrana65a1322020-04-06 19:32:32 +01001620 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001621 case FFA_MEM_LEND_32:
1622 memory_region->flags |= FFA_MEMORY_REGION_TRANSACTION_TYPE_LEND;
Andrew Walbrana65a1322020-04-06 19:32:32 +01001623 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001624 case FFA_MEM_DONATE_32:
Andrew Walbrana65a1322020-04-06 19:32:32 +01001625 memory_region->flags |=
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001626 FFA_MEMORY_REGION_TRANSACTION_TYPE_DONATE;
Andrew Walbrana65a1322020-04-06 19:32:32 +01001627 break;
Jose Marinho09b1db82019-08-08 09:16:59 +01001628 }
1629
Andrew Walbranca808b12020-05-15 17:22:28 +01001630 share_states = share_states_lock();
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001631 /*
1632 * Allocate a share state before updating the page table. Otherwise if
1633 * updating the page table succeeded but allocating the share state
1634 * failed then it would leave the memory in a state where nobody could
1635 * get it back.
1636 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001637 if (!allocate_share_state(share_states, share_func, memory_region,
1638 fragment_length, FFA_MEMORY_HANDLE_INVALID,
1639 &share_state)) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001640 dlog_verbose("Failed to allocate share state.\n");
1641 mpool_free(page_pool, memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +01001642 ret = ffa_error(FFA_NO_MEMORY);
1643 goto out;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001644 }
1645
Andrew Walbranca808b12020-05-15 17:22:28 +01001646 if (fragment_length == memory_share_length) {
1647 /* No more fragments to come, everything fit in one message. */
J-Alves2a0d2882020-10-29 14:49:50 +00001648 ret = ffa_memory_send_complete(
1649 from_locked, share_states, share_state, page_pool,
1650 &(share_state->sender_orig_mode));
Andrew Walbranca808b12020-05-15 17:22:28 +01001651 } else {
J-Alvesfdd29272022-07-19 13:16:31 +01001652 /*
1653 * Use sender ID from 'memory_region' assuming
1654 * that at this point it has been validated:
1655 * - MBZ at virtual FF-A instance.
1656 */
1657 ffa_vm_id_t sender_to_ret =
1658 (from_locked.vm->id == HF_OTHER_WORLD_ID)
1659 ? memory_region->sender
1660 : 0;
Andrew Walbranca808b12020-05-15 17:22:28 +01001661 ret = (struct ffa_value){
1662 .func = FFA_MEM_FRAG_RX_32,
J-Alvesee68c542020-10-29 17:48:20 +00001663 .arg1 = (uint32_t)memory_region->handle,
1664 .arg2 = (uint32_t)(memory_region->handle >> 32),
J-Alvesfdd29272022-07-19 13:16:31 +01001665 .arg3 = fragment_length,
1666 .arg4 = (uint32_t)(sender_to_ret & 0xffff) << 16};
Andrew Walbranca808b12020-05-15 17:22:28 +01001667 }
1668
1669out:
1670 share_states_unlock(&share_states);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001671 dump_share_states();
Andrew Walbranca808b12020-05-15 17:22:28 +01001672 return ret;
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001673}
1674
1675/**
J-Alves8505a8a2022-06-15 18:10:18 +01001676 * Continues an operation to donate, lend or share memory to a VM from current
1677 * world. If this is the last fragment then checks that the transition is valid
1678 * for the type of memory sending operation and updates the stage-2 page tables
1679 * of the sender.
Andrew Walbranca808b12020-05-15 17:22:28 +01001680 *
1681 * Assumes that the caller has already found and locked the sender VM and copied
1682 * the memory region descriptor from the sender's TX buffer to a freshly
1683 * allocated page from Hafnium's internal pool.
1684 *
1685 * This function takes ownership of the `fragment` passed in; it must not be
1686 * freed by the caller.
1687 */
1688struct ffa_value ffa_memory_send_continue(struct vm_locked from_locked,
1689 void *fragment,
1690 uint32_t fragment_length,
1691 ffa_memory_handle_t handle,
1692 struct mpool *page_pool)
1693{
1694 struct share_states_locked share_states = share_states_lock();
1695 struct ffa_memory_share_state *share_state;
1696 struct ffa_value ret;
1697 struct ffa_memory_region *memory_region;
1698
Demi Marie Obenour73a1e942023-02-04 14:09:18 -05001699 CHECK(is_aligned(fragment,
1700 alignof(struct ffa_memory_region_constituent)));
1701 if (fragment_length % sizeof(struct ffa_memory_region_constituent) !=
1702 0) {
1703 dlog_verbose("Fragment length %u misaligned.\n",
1704 fragment_length);
1705 ret = ffa_error(FFA_INVALID_PARAMETERS);
1706 goto out_free_fragment;
1707 }
1708
Andrew Walbranca808b12020-05-15 17:22:28 +01001709 ret = ffa_memory_send_continue_validate(share_states, handle,
1710 &share_state,
1711 from_locked.vm->id, page_pool);
1712 if (ret.func != FFA_SUCCESS_32) {
1713 goto out_free_fragment;
1714 }
1715 memory_region = share_state->memory_region;
1716
J-Alves95df0ef2022-12-07 10:09:48 +00001717 if (memory_region_receivers_from_other_world(memory_region)) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001718 dlog_error(
1719 "Got hypervisor-allocated handle for memory send to "
J-Alves8505a8a2022-06-15 18:10:18 +01001720 "other world. This should never happen, and indicates "
1721 "a bug in "
Andrew Walbranca808b12020-05-15 17:22:28 +01001722 "EL3 code.\n");
1723 ret = ffa_error(FFA_INVALID_PARAMETERS);
1724 goto out_free_fragment;
1725 }
1726
1727 /* Add this fragment. */
1728 share_state->fragments[share_state->fragment_count] = fragment;
1729 share_state->fragment_constituent_counts[share_state->fragment_count] =
1730 fragment_length / sizeof(struct ffa_memory_region_constituent);
1731 share_state->fragment_count++;
1732
1733 /* Check whether the memory send operation is now ready to complete. */
1734 if (share_state_sending_complete(share_states, share_state)) {
J-Alves2a0d2882020-10-29 14:49:50 +00001735 ret = ffa_memory_send_complete(
1736 from_locked, share_states, share_state, page_pool,
1737 &(share_state->sender_orig_mode));
Andrew Walbranca808b12020-05-15 17:22:28 +01001738 } else {
1739 ret = (struct ffa_value){
1740 .func = FFA_MEM_FRAG_RX_32,
1741 .arg1 = (uint32_t)handle,
1742 .arg2 = (uint32_t)(handle >> 32),
1743 .arg3 = share_state_next_fragment_offset(share_states,
1744 share_state)};
1745 }
1746 goto out;
1747
1748out_free_fragment:
1749 mpool_free(page_pool, fragment);
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001750
1751out:
Andrew Walbranca808b12020-05-15 17:22:28 +01001752 share_states_unlock(&share_states);
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001753 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001754}
1755
Andrew Walbranca808b12020-05-15 17:22:28 +01001756/** Clean up after the receiver has finished retrieving a memory region. */
1757static void ffa_memory_retrieve_complete(
1758 struct share_states_locked share_states,
1759 struct ffa_memory_share_state *share_state, struct mpool *page_pool)
1760{
1761 if (share_state->share_func == FFA_MEM_DONATE_32) {
1762 /*
1763 * Memory that has been donated can't be relinquished,
1764 * so no need to keep the share state around.
1765 */
1766 share_state_free(share_states, share_state, page_pool);
1767 dlog_verbose("Freed share state for donate.\n");
1768 }
1769}
1770
J-Alves2d8457f2022-10-05 11:06:41 +01001771/**
1772 * Initialises the given memory region descriptor to be used for an
1773 * `FFA_MEM_RETRIEVE_RESP`, including the given constituents for the first
1774 * fragment.
1775 * The memory region descriptor is initialized according to retriever's
1776 * FF-A version.
1777 *
1778 * Returns true on success, or false if the given constituents won't all fit in
1779 * the first fragment.
1780 */
1781static bool ffa_retrieved_memory_region_init(
1782 void *response, uint32_t ffa_version, size_t response_max_size,
1783 ffa_vm_id_t sender, ffa_memory_attributes_t attributes,
1784 ffa_memory_region_flags_t flags, ffa_memory_handle_t handle,
1785 ffa_vm_id_t receiver_id, ffa_memory_access_permissions_t permissions,
1786 uint32_t page_count, uint32_t total_constituent_count,
1787 const struct ffa_memory_region_constituent constituents[],
1788 uint32_t fragment_constituent_count, uint32_t *total_length,
1789 uint32_t *fragment_length)
1790{
1791 struct ffa_composite_memory_region *composite_memory_region;
1792 struct ffa_memory_access *receiver;
1793 uint32_t i;
1794 uint32_t constituents_offset;
1795 uint32_t receiver_count;
1796
1797 assert(response != NULL);
1798
1799 if (ffa_version == MAKE_FFA_VERSION(1, 0)) {
1800 struct ffa_memory_region_v1_0 *retrieve_response =
1801 (struct ffa_memory_region_v1_0 *)response;
1802
J-Alves5da37d92022-10-24 16:33:48 +01001803 ffa_memory_region_init_header_v1_0(
1804 retrieve_response, sender, attributes, flags, handle, 0,
1805 RECEIVERS_COUNT_IN_RETRIEVE_RESP);
J-Alves2d8457f2022-10-05 11:06:41 +01001806
1807 receiver = &retrieve_response->receivers[0];
1808 receiver_count = retrieve_response->receiver_count;
1809
1810 receiver->composite_memory_region_offset =
1811 sizeof(struct ffa_memory_region_v1_0) +
1812 receiver_count * sizeof(struct ffa_memory_access);
1813
1814 composite_memory_region = ffa_memory_region_get_composite_v1_0(
1815 retrieve_response, 0);
1816 } else {
1817 /* Default to FF-A v1.1 version. */
1818 struct ffa_memory_region *retrieve_response =
1819 (struct ffa_memory_region *)response;
1820
1821 ffa_memory_region_init_header(retrieve_response, sender,
1822 attributes, flags, handle, 0, 1);
1823
1824 receiver = &retrieve_response->receivers[0];
1825 receiver_count = retrieve_response->receiver_count;
1826
1827 /*
1828 * Note that `sizeof(struct_ffa_memory_region)` and
1829 * `sizeof(struct ffa_memory_access)` must both be multiples of
1830 * 16 (as verified by the asserts in `ffa_memory.c`, so it is
1831 * guaranteed that the offset we calculate here is aligned to a
1832 * 64-bit boundary and so 64-bit values can be copied without
1833 * alignment faults.
1834 */
1835 receiver->composite_memory_region_offset =
1836 sizeof(struct ffa_memory_region) +
1837 receiver_count * sizeof(struct ffa_memory_access);
1838
1839 composite_memory_region =
1840 ffa_memory_region_get_composite(retrieve_response, 0);
1841 }
1842
1843 assert(receiver != NULL);
1844 assert(composite_memory_region != NULL);
1845
1846 /*
1847 * Initialized here as in memory retrieve responses we currently expect
1848 * one borrower to be specified.
1849 */
1850 ffa_memory_access_init_permissions(receiver, receiver_id, 0, 0, flags);
1851 receiver->receiver_permissions.permissions = permissions;
1852
1853 composite_memory_region->page_count = page_count;
1854 composite_memory_region->constituent_count = total_constituent_count;
1855 composite_memory_region->reserved_0 = 0;
1856
1857 constituents_offset = receiver->composite_memory_region_offset +
1858 sizeof(struct ffa_composite_memory_region);
1859 if (constituents_offset +
1860 fragment_constituent_count *
1861 sizeof(struct ffa_memory_region_constituent) >
1862 response_max_size) {
1863 return false;
1864 }
1865
1866 for (i = 0; i < fragment_constituent_count; ++i) {
1867 composite_memory_region->constituents[i] = constituents[i];
1868 }
1869
1870 if (total_length != NULL) {
1871 *total_length =
1872 constituents_offset +
1873 composite_memory_region->constituent_count *
1874 sizeof(struct ffa_memory_region_constituent);
1875 }
1876 if (fragment_length != NULL) {
1877 *fragment_length =
1878 constituents_offset +
1879 fragment_constituent_count *
1880 sizeof(struct ffa_memory_region_constituent);
1881 }
1882
1883 return true;
1884}
1885
J-Alves96de29f2022-04-26 16:05:24 +01001886/*
1887 * Gets the receiver's access permissions from 'struct ffa_memory_region' and
1888 * returns its index in the receiver's array. If receiver's ID doesn't exist
1889 * in the array, return the region's 'receiver_count'.
1890 */
J-Alvesb5084cf2022-07-06 14:20:12 +01001891uint32_t ffa_memory_region_get_receiver(struct ffa_memory_region *memory_region,
1892 ffa_vm_id_t receiver)
J-Alves96de29f2022-04-26 16:05:24 +01001893{
1894 struct ffa_memory_access *receivers;
1895 uint32_t i;
1896
1897 assert(memory_region != NULL);
1898
1899 receivers = memory_region->receivers;
1900
1901 for (i = 0U; i < memory_region->receiver_count; i++) {
1902 if (receivers[i].receiver_permissions.receiver == receiver) {
1903 break;
1904 }
1905 }
1906
1907 return i;
1908}
1909
1910/**
1911 * Validates the retrieved permissions against those specified by the lender
1912 * of memory share operation. Optionally can help set the permissions to be used
1913 * for the S2 mapping, through the `permissions` argument.
1914 * Returns true if permissions are valid, false otherwise.
1915 */
1916static bool ffa_memory_retrieve_is_memory_access_valid(
1917 enum ffa_data_access sent_data_access,
1918 enum ffa_data_access requested_data_access,
1919 enum ffa_instruction_access sent_instruction_access,
1920 enum ffa_instruction_access requested_instruction_access,
1921 ffa_memory_access_permissions_t *permissions)
1922{
1923 switch (sent_data_access) {
1924 case FFA_DATA_ACCESS_NOT_SPECIFIED:
1925 case FFA_DATA_ACCESS_RW:
1926 if (requested_data_access == FFA_DATA_ACCESS_NOT_SPECIFIED ||
1927 requested_data_access == FFA_DATA_ACCESS_RW) {
1928 if (permissions != NULL) {
1929 ffa_set_data_access_attr(permissions,
1930 FFA_DATA_ACCESS_RW);
1931 }
1932 break;
1933 }
1934 /* Intentional fall-through. */
1935 case FFA_DATA_ACCESS_RO:
1936 if (requested_data_access == FFA_DATA_ACCESS_NOT_SPECIFIED ||
1937 requested_data_access == FFA_DATA_ACCESS_RO) {
1938 if (permissions != NULL) {
1939 ffa_set_data_access_attr(permissions,
1940 FFA_DATA_ACCESS_RO);
1941 }
1942 break;
1943 }
1944 dlog_verbose(
1945 "Invalid data access requested; sender specified "
1946 "permissions %#x but receiver requested %#x.\n",
1947 sent_data_access, requested_data_access);
1948 return false;
1949 case FFA_DATA_ACCESS_RESERVED:
1950 panic("Got unexpected FFA_DATA_ACCESS_RESERVED. Should be "
1951 "checked before this point.");
1952 }
1953
1954 switch (sent_instruction_access) {
1955 case FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED:
1956 case FFA_INSTRUCTION_ACCESS_X:
1957 if (requested_instruction_access ==
1958 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED ||
1959 requested_instruction_access == FFA_INSTRUCTION_ACCESS_X) {
1960 if (permissions != NULL) {
1961 ffa_set_instruction_access_attr(
1962 permissions, FFA_INSTRUCTION_ACCESS_X);
1963 }
1964 break;
1965 }
1966 case FFA_INSTRUCTION_ACCESS_NX:
1967 if (requested_instruction_access ==
1968 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED ||
1969 requested_instruction_access == FFA_INSTRUCTION_ACCESS_NX) {
1970 if (permissions != NULL) {
1971 ffa_set_instruction_access_attr(
1972 permissions, FFA_INSTRUCTION_ACCESS_NX);
1973 }
1974 break;
1975 }
1976 dlog_verbose(
1977 "Invalid instruction access requested; sender "
1978 "specified permissions %#x but receiver requested "
1979 "%#x.\n",
1980 sent_instruction_access, requested_instruction_access);
1981 return false;
1982 case FFA_INSTRUCTION_ACCESS_RESERVED:
1983 panic("Got unexpected FFA_INSTRUCTION_ACCESS_RESERVED. Should "
1984 "be checked before this point.");
1985 }
1986
1987 return true;
1988}
1989
1990/**
1991 * Validate the receivers' permissions in the retrieve request against those
1992 * specified by the lender.
1993 * In the `permissions` argument returns the permissions to set at S2 for the
1994 * caller to the FFA_MEMORY_RETRIEVE_REQ.
1995 * Returns FFA_SUCCESS if all specified permissions are valid.
1996 */
1997static struct ffa_value ffa_memory_retrieve_validate_memory_access_list(
1998 struct ffa_memory_region *memory_region,
1999 struct ffa_memory_region *retrieve_request, ffa_vm_id_t to_vm_id,
2000 ffa_memory_access_permissions_t *permissions)
2001{
2002 uint32_t retrieve_receiver_index;
2003
2004 assert(permissions != NULL);
2005
2006 if (retrieve_request->receiver_count != memory_region->receiver_count) {
2007 dlog_verbose(
2008 "Retrieve request should contain same list of "
2009 "borrowers, as specified by the lender.\n");
2010 return ffa_error(FFA_INVALID_PARAMETERS);
2011 }
2012
2013 retrieve_receiver_index = retrieve_request->receiver_count;
2014
2015 /* Should be populated with the permissions of the retriever. */
2016 *permissions = 0;
2017
2018 for (uint32_t i = 0U; i < retrieve_request->receiver_count; i++) {
2019 ffa_memory_access_permissions_t sent_permissions;
2020 struct ffa_memory_access *current_receiver =
2021 &retrieve_request->receivers[i];
2022 ffa_memory_access_permissions_t requested_permissions =
2023 current_receiver->receiver_permissions.permissions;
2024 ffa_vm_id_t current_receiver_id =
2025 current_receiver->receiver_permissions.receiver;
2026 bool found_to_id = current_receiver_id == to_vm_id;
2027
2028 /*
2029 * Find the current receiver in the transaction descriptor from
2030 * sender.
2031 */
2032 uint32_t mem_region_receiver_index =
2033 ffa_memory_region_get_receiver(memory_region,
2034 current_receiver_id);
2035
2036 if (mem_region_receiver_index ==
2037 memory_region->receiver_count) {
2038 dlog_verbose("%s: receiver %x not found\n", __func__,
2039 current_receiver_id);
2040 return ffa_error(FFA_DENIED);
2041 }
2042
2043 sent_permissions =
2044 memory_region->receivers[mem_region_receiver_index]
2045 .receiver_permissions.permissions;
2046
2047 if (found_to_id) {
2048 retrieve_receiver_index = i;
2049 }
2050
2051 /*
2052 * Since we are traversing the list of receivers, save the index
2053 * of the caller. As it needs to be there.
2054 */
2055
2056 if (current_receiver->composite_memory_region_offset != 0U) {
2057 dlog_verbose(
2058 "Retriever specified address ranges not "
2059 "supported (got offset %d).\n",
2060 current_receiver
2061 ->composite_memory_region_offset);
2062 return ffa_error(FFA_INVALID_PARAMETERS);
2063 }
2064
2065 /*
2066 * Check permissions from sender against permissions requested
2067 * by receiver.
2068 */
2069 if (!ffa_memory_retrieve_is_memory_access_valid(
2070 ffa_get_data_access_attr(sent_permissions),
2071 ffa_get_data_access_attr(requested_permissions),
2072 ffa_get_instruction_access_attr(sent_permissions),
2073 ffa_get_instruction_access_attr(
2074 requested_permissions),
2075 found_to_id ? permissions : NULL)) {
2076 return ffa_error(FFA_DENIED);
2077 }
2078
2079 /*
2080 * Can't request PM to clear memory if only provided with RO
2081 * permissions.
2082 */
2083 if (found_to_id &&
2084 (ffa_get_data_access_attr(*permissions) ==
2085 FFA_DATA_ACCESS_RO) &&
2086 (retrieve_request->flags & FFA_MEMORY_REGION_FLAG_CLEAR) !=
2087 0U) {
2088 dlog_verbose(
2089 "Receiver has RO permissions can not request "
2090 "clear.\n");
2091 return ffa_error(FFA_DENIED);
2092 }
2093 }
2094
2095 if (retrieve_receiver_index == retrieve_request->receiver_count) {
2096 dlog_verbose(
2097 "Retrieve request does not contain caller's (%x) "
2098 "permissions\n",
2099 to_vm_id);
2100 return ffa_error(FFA_INVALID_PARAMETERS);
2101 }
2102
2103 return (struct ffa_value){.func = FFA_SUCCESS_32};
2104}
2105
J-Alvesa9cd7e32022-07-01 13:49:33 +01002106/*
2107 * According to section 16.4.3 of FF-A v1.1 EAC0 specification, the hypervisor
2108 * may issue an FFA_MEM_RETRIEVE_REQ to obtain the memory region description
2109 * of a pending memory sharing operation whose allocator is the SPM, for
2110 * validation purposes before forwarding an FFA_MEM_RECLAIM call. In doing so
2111 * the memory region descriptor of the retrieve request must be zeroed with the
2112 * exception of the sender ID and handle.
2113 */
2114bool is_ffa_memory_retrieve_borrower_request(struct ffa_memory_region *request,
2115 struct vm_locked to_locked)
2116{
2117 return to_locked.vm->id == HF_HYPERVISOR_VM_ID &&
2118 request->attributes == 0U && request->flags == 0U &&
2119 request->tag == 0U && request->receiver_count == 0U &&
2120 plat_ffa_memory_handle_allocated_by_current_world(
2121 request->handle);
2122}
2123
2124/*
2125 * Helper to reset count of fragments retrieved by the hypervisor.
2126 */
2127static void ffa_memory_retrieve_complete_from_hyp(
2128 struct ffa_memory_share_state *share_state)
2129{
2130 if (share_state->hypervisor_fragment_count ==
2131 share_state->fragment_count) {
2132 share_state->hypervisor_fragment_count = 0;
2133 }
2134}
2135
J-Alves089004f2022-07-13 14:25:44 +01002136/**
2137 * Validate that the memory region descriptor provided by the borrower on
2138 * FFA_MEM_RETRIEVE_REQ, against saved memory region provided by lender at the
2139 * memory sharing call.
2140 */
2141static struct ffa_value ffa_memory_retrieve_validate(
2142 ffa_vm_id_t receiver_id, struct ffa_memory_region *retrieve_request,
2143 struct ffa_memory_region *memory_region, uint32_t *receiver_index,
2144 uint32_t share_func)
2145{
2146 ffa_memory_region_flags_t transaction_type =
2147 retrieve_request->flags &
2148 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK;
Olivier Deprez4342a3c2022-02-28 09:37:25 +01002149 enum ffa_memory_security security_state;
J-Alves089004f2022-07-13 14:25:44 +01002150
2151 assert(retrieve_request != NULL);
2152 assert(memory_region != NULL);
2153 assert(receiver_index != NULL);
2154 assert(retrieve_request->sender == memory_region->sender);
2155
2156 /*
2157 * Check that the transaction type expected by the receiver is
2158 * correct, if it has been specified.
2159 */
2160 if (transaction_type !=
2161 FFA_MEMORY_REGION_TRANSACTION_TYPE_UNSPECIFIED &&
2162 transaction_type != (memory_region->flags &
2163 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK)) {
2164 dlog_verbose(
2165 "Incorrect transaction type %#x for "
2166 "FFA_MEM_RETRIEVE_REQ, expected %#x for handle %#x.\n",
2167 transaction_type,
2168 memory_region->flags &
2169 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK,
2170 retrieve_request->handle);
2171 return ffa_error(FFA_INVALID_PARAMETERS);
2172 }
2173
2174 if (retrieve_request->tag != memory_region->tag) {
2175 dlog_verbose(
2176 "Incorrect tag %d for FFA_MEM_RETRIEVE_REQ, expected "
2177 "%d for handle %#x.\n",
2178 retrieve_request->tag, memory_region->tag,
2179 retrieve_request->handle);
2180 return ffa_error(FFA_INVALID_PARAMETERS);
2181 }
2182
2183 *receiver_index =
2184 ffa_memory_region_get_receiver(memory_region, receiver_id);
2185
2186 if (*receiver_index == memory_region->receiver_count) {
2187 dlog_verbose(
2188 "Incorrect receiver VM ID %d for "
2189 "FFA_MEM_RETRIEVE_REQ, for handle %#x.\n",
J-Alves59ed0042022-07-28 18:26:41 +01002190 receiver_id, memory_region->handle);
J-Alves089004f2022-07-13 14:25:44 +01002191 return ffa_error(FFA_INVALID_PARAMETERS);
2192 }
2193
2194 if ((retrieve_request->flags &
2195 FFA_MEMORY_REGION_ADDRESS_RANGE_HINT_VALID) != 0U) {
2196 dlog_verbose(
2197 "Retriever specified 'address range alignment 'hint' "
2198 "not supported.\n");
2199 return ffa_error(FFA_INVALID_PARAMETERS);
2200 }
2201 if ((retrieve_request->flags &
2202 FFA_MEMORY_REGION_ADDRESS_RANGE_HINT_MASK) != 0) {
2203 dlog_verbose(
2204 "Bits 8-5 must be zero in memory region's flags "
2205 "(address range alignment hint not supported).\n");
2206 return ffa_error(FFA_INVALID_PARAMETERS);
2207 }
2208
2209 if ((retrieve_request->flags & ~0x7FF) != 0U) {
2210 dlog_verbose(
2211 "Bits 31-10 must be zero in memory region's flags.\n");
2212 return ffa_error(FFA_INVALID_PARAMETERS);
2213 }
2214
2215 if (share_func == FFA_MEM_SHARE_32 &&
2216 (retrieve_request->flags &
2217 (FFA_MEMORY_REGION_FLAG_CLEAR |
2218 FFA_MEMORY_REGION_FLAG_CLEAR_RELINQUISH)) != 0U) {
2219 dlog_verbose(
2220 "Memory Share operation can't clean after relinquish "
2221 "memory region.\n");
2222 return ffa_error(FFA_INVALID_PARAMETERS);
2223 }
2224
2225 /*
2226 * If the borrower needs the memory to be cleared before mapping
2227 * to its address space, the sender should have set the flag
2228 * when calling FFA_MEM_LEND/FFA_MEM_DONATE, else return
2229 * FFA_DENIED.
2230 */
2231 if ((retrieve_request->flags & FFA_MEMORY_REGION_FLAG_CLEAR) != 0U &&
2232 (memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR) == 0U) {
2233 dlog_verbose(
2234 "Borrower needs memory cleared. Sender needs to set "
2235 "flag for clearing memory.\n");
2236 return ffa_error(FFA_DENIED);
2237 }
2238
Olivier Deprez4342a3c2022-02-28 09:37:25 +01002239 /* Memory region attributes NS-Bit MBZ for FFA_MEM_RETRIEVE_REQ. */
2240 security_state =
2241 ffa_get_memory_security_attr(retrieve_request->attributes);
2242 if (security_state != FFA_MEMORY_SECURITY_UNSPECIFIED) {
2243 dlog_verbose(
2244 "Invalid security state for memory retrieve request "
2245 "operation.\n");
2246 return ffa_error(FFA_INVALID_PARAMETERS);
2247 }
2248
J-Alves089004f2022-07-13 14:25:44 +01002249 /*
2250 * If memory type is not specified, bypass validation of memory
2251 * attributes in the retrieve request. The retriever is expecting to
2252 * obtain this information from the SPMC.
2253 */
2254 if (ffa_get_memory_type_attr(retrieve_request->attributes) ==
2255 FFA_MEMORY_NOT_SPECIFIED_MEM) {
2256 return (struct ffa_value){.func = FFA_SUCCESS_32};
2257 }
2258
2259 /*
2260 * Ensure receiver's attributes are compatible with how
2261 * Hafnium maps memory: Normal Memory, Inner shareable,
2262 * Write-Back Read-Allocate Write-Allocate Cacheable.
2263 */
2264 return ffa_memory_attributes_validate(retrieve_request->attributes);
2265}
2266
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002267struct ffa_value ffa_memory_retrieve(struct vm_locked to_locked,
2268 struct ffa_memory_region *retrieve_request,
Andrew Walbran130a8ae2020-05-15 16:27:15 +01002269 uint32_t retrieve_request_length,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002270 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002271{
Andrew Walbran130a8ae2020-05-15 16:27:15 +01002272 uint32_t expected_retrieve_request_length =
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002273 sizeof(struct ffa_memory_region) +
Andrew Walbrana65a1322020-04-06 19:32:32 +01002274 retrieve_request->receiver_count *
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002275 sizeof(struct ffa_memory_access);
2276 ffa_memory_handle_t handle = retrieve_request->handle;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002277 struct ffa_memory_region *memory_region;
J-Alvesa9cd7e32022-07-01 13:49:33 +01002278 ffa_memory_access_permissions_t permissions = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002279 uint32_t memory_to_attributes;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002280 struct share_states_locked share_states;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002281 struct ffa_memory_share_state *share_state;
2282 struct ffa_value ret;
Andrew Walbranca808b12020-05-15 17:22:28 +01002283 struct ffa_composite_memory_region *composite;
2284 uint32_t total_length;
2285 uint32_t fragment_length;
J-Alves089004f2022-07-13 14:25:44 +01002286 ffa_vm_id_t receiver_id = to_locked.vm->id;
J-Alvesa9cd7e32022-07-01 13:49:33 +01002287 bool is_send_complete = false;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002288
2289 dump_share_states();
2290
Andrew Walbran130a8ae2020-05-15 16:27:15 +01002291 if (retrieve_request_length != expected_retrieve_request_length) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002292 dlog_verbose(
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002293 "Invalid length for FFA_MEM_RETRIEVE_REQ, expected %d "
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002294 "but was %d.\n",
Andrew Walbran130a8ae2020-05-15 16:27:15 +01002295 expected_retrieve_request_length,
2296 retrieve_request_length);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002297 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002298 }
2299
2300 share_states = share_states_lock();
2301 if (!get_share_state(share_states, handle, &share_state)) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002302 dlog_verbose("Invalid handle %#x for FFA_MEM_RETRIEVE_REQ.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002303 handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002304 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002305 goto out;
2306 }
2307
J-Alves96de29f2022-04-26 16:05:24 +01002308 if (!share_state->sending_complete) {
2309 dlog_verbose(
2310 "Memory with handle %#x not fully sent, can't "
2311 "retrieve.\n",
2312 handle);
2313 ret = ffa_error(FFA_INVALID_PARAMETERS);
2314 goto out;
2315 }
2316
Andrew Walbrana65a1322020-04-06 19:32:32 +01002317 memory_region = share_state->memory_region;
J-Alves089004f2022-07-13 14:25:44 +01002318
Andrew Walbrana65a1322020-04-06 19:32:32 +01002319 CHECK(memory_region != NULL);
2320
J-Alves089004f2022-07-13 14:25:44 +01002321 if (retrieve_request->sender != memory_region->sender) {
2322 dlog_verbose(
2323 "Memory with handle %#x not fully sent, can't "
2324 "retrieve.\n",
2325 handle);
2326 ret = ffa_error(FFA_INVALID_PARAMETERS);
2327 goto out;
2328 }
J-Alves96de29f2022-04-26 16:05:24 +01002329
J-Alvesa9cd7e32022-07-01 13:49:33 +01002330 if (!is_ffa_memory_retrieve_borrower_request(retrieve_request,
2331 to_locked)) {
2332 uint32_t receiver_index;
J-Alvesa9cd7e32022-07-01 13:49:33 +01002333
J-Alvesb5084cf2022-07-06 14:20:12 +01002334 /*
2335 * The SPMC can only process retrieve requests to memory share
2336 * operations with one borrower from the other world. It can't
2337 * determine the ID of the NWd VM that invoked the retrieve
2338 * request interface call. It relies on the hypervisor to
2339 * validate the caller's ID against that provided in the
2340 * `receivers` list of the retrieve response.
2341 * In case there is only one borrower from the NWd in the
2342 * transaction descriptor, record that in the `receiver_id` for
2343 * later use, and validate in the retrieve request message.
2344 */
2345 if (to_locked.vm->id == HF_HYPERVISOR_VM_ID) {
2346 uint32_t other_world_count = 0;
2347
2348 for (uint32_t i = 0; i < memory_region->receiver_count;
2349 i++) {
2350 receiver_id =
2351 retrieve_request->receivers[0]
2352 .receiver_permissions.receiver;
2353 if (!vm_id_is_current_world(receiver_id)) {
2354 other_world_count++;
2355 }
2356 }
2357 if (other_world_count > 1) {
2358 dlog_verbose(
2359 "Support one receiver from the other "
2360 "world.\n");
2361 return ffa_error(FFA_NOT_SUPPORTED);
2362 }
2363 }
2364
2365 /*
2366 * Validate retrieve request, according to what was sent by the
2367 * sender. Function will output the `receiver_index` from the
2368 * provided memory region, and will output `permissions` from
2369 * the validated requested permissions.
2370 */
J-Alves089004f2022-07-13 14:25:44 +01002371 ret = ffa_memory_retrieve_validate(
2372 receiver_id, retrieve_request, memory_region,
2373 &receiver_index, share_state->share_func);
2374 if (ret.func != FFA_SUCCESS_32) {
J-Alvesa9cd7e32022-07-01 13:49:33 +01002375 goto out;
2376 }
2377
2378 if (share_state->retrieved_fragment_count[receiver_index] !=
2379 0U) {
2380 dlog_verbose(
2381 "Memory with handle %#x already retrieved.\n",
2382 handle);
2383 ret = ffa_error(FFA_DENIED);
2384 goto out;
2385 }
2386
J-Alvesa9cd7e32022-07-01 13:49:33 +01002387 ret = ffa_memory_retrieve_validate_memory_access_list(
2388 memory_region, retrieve_request, receiver_id,
2389 &permissions);
J-Alves614d9f42022-06-28 14:03:10 +01002390 if (ret.func != FFA_SUCCESS_32) {
2391 goto out;
2392 }
Federico Recanatia98603a2021-12-20 18:04:03 +01002393
J-Alvesa9cd7e32022-07-01 13:49:33 +01002394 memory_to_attributes = ffa_memory_permissions_to_mode(
2395 permissions, share_state->sender_orig_mode);
J-Alves40e260e2022-09-22 17:52:43 +01002396
2397 if (to_locked.vm->el0_partition) {
2398 /*
2399 * Get extra mapping attributes for the given VM ID.
2400 * If the memory is shared by a VM executing in non
2401 * secure world, attribute MM_MODE_NS has to be set
2402 * while mapping that in a SP executing in secure world.
2403 */
2404 memory_to_attributes |=
2405 arch_mm_extra_attributes_from_vm(
2406 retrieve_request->sender);
2407 }
2408
J-Alvesa9cd7e32022-07-01 13:49:33 +01002409 ret = ffa_retrieve_check_update(
2410 to_locked, memory_region->sender,
2411 share_state->fragments,
2412 share_state->fragment_constituent_counts,
2413 share_state->fragment_count, memory_to_attributes,
2414 share_state->share_func, false, page_pool);
2415
2416 if (ret.func != FFA_SUCCESS_32) {
2417 goto out;
2418 }
2419
2420 share_state->retrieved_fragment_count[receiver_index] = 1;
2421 is_send_complete =
2422 share_state->retrieved_fragment_count[receiver_index] ==
2423 share_state->fragment_count;
J-Alves3c5b2072022-11-21 12:45:40 +00002424
2425 share_state->clear_after_relinquish =
2426 (retrieve_request->flags &
2427 FFA_MEMORY_REGION_FLAG_CLEAR_RELINQUISH) != 0U;
2428
J-Alvesa9cd7e32022-07-01 13:49:33 +01002429 } else {
2430 if (share_state->hypervisor_fragment_count != 0U) {
2431 dlog_verbose(
J-Alvesb5084cf2022-07-06 14:20:12 +01002432 "Memory with handle %#x already retrieved by "
J-Alvesa9cd7e32022-07-01 13:49:33 +01002433 "the hypervisor.\n",
2434 handle);
2435 ret = ffa_error(FFA_DENIED);
2436 goto out;
2437 }
2438
2439 share_state->hypervisor_fragment_count = 1;
2440
2441 ffa_memory_retrieve_complete_from_hyp(share_state);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002442 }
2443
J-Alvesb5084cf2022-07-06 14:20:12 +01002444 /* VMs acquire the RX buffer from SPMC. */
2445 CHECK(plat_ffa_acquire_receiver_rx(to_locked, &ret));
2446
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002447 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01002448 * Copy response to RX buffer of caller and deliver the message.
2449 * This must be done before the share_state is (possibly) freed.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002450 */
Andrew Walbrana65a1322020-04-06 19:32:32 +01002451 /* TODO: combine attributes from sender and request. */
Andrew Walbranca808b12020-05-15 17:22:28 +01002452 composite = ffa_memory_region_get_composite(memory_region, 0);
2453 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01002454 * Constituents which we received in the first fragment should
2455 * always fit in the first fragment we are sending, because the
2456 * header is the same size in both cases and we have a fixed
2457 * message buffer size. So `ffa_retrieved_memory_region_init`
2458 * should never fail.
Andrew Walbranca808b12020-05-15 17:22:28 +01002459 */
2460 CHECK(ffa_retrieved_memory_region_init(
J-Alves2d8457f2022-10-05 11:06:41 +01002461 to_locked.vm->mailbox.recv, to_locked.vm->ffa_version,
2462 HF_MAILBOX_SIZE, memory_region->sender,
2463 memory_region->attributes, memory_region->flags, handle,
2464 receiver_id, permissions, composite->page_count,
2465 composite->constituent_count, share_state->fragments[0],
Andrew Walbranca808b12020-05-15 17:22:28 +01002466 share_state->fragment_constituent_counts[0], &total_length,
2467 &fragment_length));
J-Alvesb5084cf2022-07-06 14:20:12 +01002468
Andrew Walbranca808b12020-05-15 17:22:28 +01002469 to_locked.vm->mailbox.recv_size = fragment_length;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002470 to_locked.vm->mailbox.recv_sender = HF_HYPERVISOR_VM_ID;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002471 to_locked.vm->mailbox.recv_func = FFA_MEM_RETRIEVE_RESP_32;
J-Alvese8c8c2b2022-12-16 15:34:48 +00002472 to_locked.vm->mailbox.state = MAILBOX_STATE_FULL;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002473
J-Alvesa9cd7e32022-07-01 13:49:33 +01002474 if (is_send_complete) {
Andrew Walbranca808b12020-05-15 17:22:28 +01002475 ffa_memory_retrieve_complete(share_states, share_state,
2476 page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002477 }
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002478 ret = (struct ffa_value){.func = FFA_MEM_RETRIEVE_RESP_32,
Andrew Walbranca808b12020-05-15 17:22:28 +01002479 .arg1 = total_length,
2480 .arg2 = fragment_length};
Andrew Walbranca808b12020-05-15 17:22:28 +01002481out:
2482 share_states_unlock(&share_states);
2483 dump_share_states();
2484 return ret;
2485}
2486
J-Alves5da37d92022-10-24 16:33:48 +01002487/**
2488 * Determine expected fragment offset according to the FF-A version of
2489 * the caller.
2490 */
2491static uint32_t ffa_memory_retrieve_expected_offset_per_ffa_version(
2492 struct ffa_memory_region *memory_region,
2493 uint32_t retrieved_constituents_count, uint32_t ffa_version)
2494{
2495 uint32_t expected_fragment_offset;
2496 uint32_t composite_constituents_offset;
2497
2498 if (ffa_version == MAKE_FFA_VERSION(1, 1)) {
2499 /*
2500 * Hafnium operates memory regions in FF-A v1.1 format, so we
2501 * can retrieve the constituents offset from descriptor.
2502 */
2503 composite_constituents_offset =
2504 ffa_composite_constituent_offset(memory_region, 0);
2505 } else if (ffa_version == MAKE_FFA_VERSION(1, 0)) {
2506 /*
2507 * If retriever is FF-A v1.0, determine the composite offset
2508 * as it is expected to have been configured in the
2509 * retrieve response.
2510 */
2511 composite_constituents_offset =
2512 sizeof(struct ffa_memory_region_v1_0) +
2513 RECEIVERS_COUNT_IN_RETRIEVE_RESP *
2514 sizeof(struct ffa_memory_access) +
2515 sizeof(struct ffa_composite_memory_region);
2516 } else {
2517 panic("%s received an invalid FF-A version.\n", __func__);
2518 }
2519
2520 expected_fragment_offset =
2521 composite_constituents_offset +
2522 retrieved_constituents_count *
2523 sizeof(struct ffa_memory_region_constituent) -
2524 sizeof(struct ffa_memory_access) *
2525 (memory_region->receiver_count - 1);
2526
2527 return expected_fragment_offset;
2528}
2529
Andrew Walbranca808b12020-05-15 17:22:28 +01002530struct ffa_value ffa_memory_retrieve_continue(struct vm_locked to_locked,
2531 ffa_memory_handle_t handle,
2532 uint32_t fragment_offset,
J-Alves59ed0042022-07-28 18:26:41 +01002533 ffa_vm_id_t sender_vm_id,
Andrew Walbranca808b12020-05-15 17:22:28 +01002534 struct mpool *page_pool)
2535{
2536 struct ffa_memory_region *memory_region;
2537 struct share_states_locked share_states;
2538 struct ffa_memory_share_state *share_state;
2539 struct ffa_value ret;
2540 uint32_t fragment_index;
2541 uint32_t retrieved_constituents_count;
2542 uint32_t i;
2543 uint32_t expected_fragment_offset;
2544 uint32_t remaining_constituent_count;
2545 uint32_t fragment_length;
J-Alvesc7484f12022-05-13 12:41:14 +01002546 uint32_t receiver_index;
J-Alves59ed0042022-07-28 18:26:41 +01002547 bool continue_ffa_hyp_mem_retrieve_req;
Andrew Walbranca808b12020-05-15 17:22:28 +01002548
2549 dump_share_states();
2550
2551 share_states = share_states_lock();
2552 if (!get_share_state(share_states, handle, &share_state)) {
2553 dlog_verbose("Invalid handle %#x for FFA_MEM_FRAG_RX.\n",
2554 handle);
2555 ret = ffa_error(FFA_INVALID_PARAMETERS);
2556 goto out;
2557 }
2558
2559 memory_region = share_state->memory_region;
2560 CHECK(memory_region != NULL);
2561
Andrew Walbranca808b12020-05-15 17:22:28 +01002562 if (!share_state->sending_complete) {
2563 dlog_verbose(
2564 "Memory with handle %#x not fully sent, can't "
2565 "retrieve.\n",
2566 handle);
2567 ret = ffa_error(FFA_INVALID_PARAMETERS);
2568 goto out;
2569 }
2570
J-Alves59ed0042022-07-28 18:26:41 +01002571 /*
2572 * If retrieve request from the hypervisor has been initiated in the
2573 * given share_state, continue it, else assume it is a continuation of
2574 * retrieve request from a NWd VM.
2575 */
2576 continue_ffa_hyp_mem_retrieve_req =
2577 (to_locked.vm->id == HF_HYPERVISOR_VM_ID) &&
2578 (share_state->hypervisor_fragment_count != 0U) &&
2579 plat_ffa_is_vm_id(sender_vm_id);
Andrew Walbranca808b12020-05-15 17:22:28 +01002580
J-Alves59ed0042022-07-28 18:26:41 +01002581 if (!continue_ffa_hyp_mem_retrieve_req) {
2582 receiver_index = ffa_memory_region_get_receiver(
2583 memory_region, to_locked.vm->id);
2584
2585 if (receiver_index == memory_region->receiver_count) {
2586 dlog_verbose(
2587 "Caller of FFA_MEM_FRAG_RX (%x) is not a "
2588 "borrower to memory sharing transaction (%x)\n",
2589 to_locked.vm->id, handle);
2590 ret = ffa_error(FFA_INVALID_PARAMETERS);
2591 goto out;
2592 }
2593
2594 if (share_state->retrieved_fragment_count[receiver_index] ==
2595 0 ||
2596 share_state->retrieved_fragment_count[receiver_index] >=
2597 share_state->fragment_count) {
2598 dlog_verbose(
2599 "Retrieval of memory with handle %#x not yet "
2600 "started or already completed (%d/%d fragments "
2601 "retrieved).\n",
2602 handle,
2603 share_state->retrieved_fragment_count
2604 [receiver_index],
2605 share_state->fragment_count);
2606 ret = ffa_error(FFA_INVALID_PARAMETERS);
2607 goto out;
2608 }
2609
2610 fragment_index =
2611 share_state->retrieved_fragment_count[receiver_index];
2612 } else {
2613 if (share_state->hypervisor_fragment_count == 0 ||
2614 share_state->hypervisor_fragment_count >=
2615 share_state->fragment_count) {
2616 dlog_verbose(
2617 "Retrieve of memory with handle %x not "
2618 "started from hypervisor.\n",
2619 handle);
2620 ret = ffa_error(FFA_INVALID_PARAMETERS);
2621 goto out;
2622 }
2623
2624 if (memory_region->sender != sender_vm_id) {
2625 dlog_verbose(
2626 "Sender ID (%x) is not as expected for memory "
2627 "handle %x\n",
2628 sender_vm_id, handle);
2629 ret = ffa_error(FFA_INVALID_PARAMETERS);
2630 goto out;
2631 }
2632
2633 fragment_index = share_state->hypervisor_fragment_count;
2634
2635 receiver_index = 0;
2636 }
Andrew Walbranca808b12020-05-15 17:22:28 +01002637
2638 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01002639 * Check that the given fragment offset is correct by counting
2640 * how many constituents were in the fragments previously sent.
Andrew Walbranca808b12020-05-15 17:22:28 +01002641 */
2642 retrieved_constituents_count = 0;
2643 for (i = 0; i < fragment_index; ++i) {
2644 retrieved_constituents_count +=
2645 share_state->fragment_constituent_counts[i];
2646 }
J-Alvesc7484f12022-05-13 12:41:14 +01002647
2648 CHECK(memory_region->receiver_count > 0);
2649
Andrew Walbranca808b12020-05-15 17:22:28 +01002650 expected_fragment_offset =
J-Alves5da37d92022-10-24 16:33:48 +01002651 ffa_memory_retrieve_expected_offset_per_ffa_version(
2652 memory_region, retrieved_constituents_count,
2653 to_locked.vm->ffa_version);
2654
Andrew Walbranca808b12020-05-15 17:22:28 +01002655 if (fragment_offset != expected_fragment_offset) {
2656 dlog_verbose("Fragment offset was %d but expected %d.\n",
2657 fragment_offset, expected_fragment_offset);
2658 ret = ffa_error(FFA_INVALID_PARAMETERS);
2659 goto out;
2660 }
2661
J-Alves59ed0042022-07-28 18:26:41 +01002662 /* VMs acquire the RX buffer from SPMC. */
2663 CHECK(plat_ffa_acquire_receiver_rx(to_locked, &ret));
2664
Andrew Walbranca808b12020-05-15 17:22:28 +01002665 remaining_constituent_count = ffa_memory_fragment_init(
2666 to_locked.vm->mailbox.recv, HF_MAILBOX_SIZE,
2667 share_state->fragments[fragment_index],
2668 share_state->fragment_constituent_counts[fragment_index],
2669 &fragment_length);
2670 CHECK(remaining_constituent_count == 0);
2671 to_locked.vm->mailbox.recv_size = fragment_length;
2672 to_locked.vm->mailbox.recv_sender = HF_HYPERVISOR_VM_ID;
2673 to_locked.vm->mailbox.recv_func = FFA_MEM_FRAG_TX_32;
J-Alvese8c8c2b2022-12-16 15:34:48 +00002674 to_locked.vm->mailbox.state = MAILBOX_STATE_FULL;
Andrew Walbranca808b12020-05-15 17:22:28 +01002675
J-Alves59ed0042022-07-28 18:26:41 +01002676 if (!continue_ffa_hyp_mem_retrieve_req) {
2677 share_state->retrieved_fragment_count[receiver_index]++;
2678 if (share_state->retrieved_fragment_count[receiver_index] ==
2679 share_state->fragment_count) {
2680 ffa_memory_retrieve_complete(share_states, share_state,
2681 page_pool);
2682 }
2683 } else {
2684 share_state->hypervisor_fragment_count++;
2685
2686 ffa_memory_retrieve_complete_from_hyp(share_state);
2687 }
Andrew Walbranca808b12020-05-15 17:22:28 +01002688 ret = (struct ffa_value){.func = FFA_MEM_FRAG_TX_32,
2689 .arg1 = (uint32_t)handle,
2690 .arg2 = (uint32_t)(handle >> 32),
2691 .arg3 = fragment_length};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002692
2693out:
2694 share_states_unlock(&share_states);
2695 dump_share_states();
2696 return ret;
2697}
2698
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002699struct ffa_value ffa_memory_relinquish(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002700 struct vm_locked from_locked,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002701 struct ffa_mem_relinquish *relinquish_request, struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002702{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002703 ffa_memory_handle_t handle = relinquish_request->handle;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002704 struct share_states_locked share_states;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002705 struct ffa_memory_share_state *share_state;
2706 struct ffa_memory_region *memory_region;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002707 bool clear;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002708 struct ffa_value ret;
J-Alves8eb19162022-04-28 10:56:48 +01002709 uint32_t receiver_index;
J-Alves3c5b2072022-11-21 12:45:40 +00002710 bool receivers_relinquished_memory;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002711
Andrew Walbrana65a1322020-04-06 19:32:32 +01002712 if (relinquish_request->endpoint_count != 1) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002713 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01002714 "Stream endpoints not supported (got %d "
J-Alves668a86e2023-05-10 11:53:25 +01002715 "endpoints on FFA_MEM_RELINQUISH, expected 1).\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002716 relinquish_request->endpoint_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002717 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002718 }
2719
Andrew Walbrana65a1322020-04-06 19:32:32 +01002720 if (relinquish_request->endpoints[0] != from_locked.vm->id) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002721 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01002722 "VM ID %d in relinquish message doesn't match "
J-Alves668a86e2023-05-10 11:53:25 +01002723 "calling VM ID %d.\n",
Andrew Walbrana65a1322020-04-06 19:32:32 +01002724 relinquish_request->endpoints[0], from_locked.vm->id);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002725 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002726 }
2727
2728 dump_share_states();
2729
2730 share_states = share_states_lock();
2731 if (!get_share_state(share_states, handle, &share_state)) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002732 dlog_verbose("Invalid handle %#x for FFA_MEM_RELINQUISH.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002733 handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002734 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002735 goto out;
2736 }
2737
Andrew Walbranca808b12020-05-15 17:22:28 +01002738 if (!share_state->sending_complete) {
2739 dlog_verbose(
2740 "Memory with handle %#x not fully sent, can't "
2741 "relinquish.\n",
2742 handle);
2743 ret = ffa_error(FFA_INVALID_PARAMETERS);
2744 goto out;
2745 }
2746
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002747 memory_region = share_state->memory_region;
2748 CHECK(memory_region != NULL);
2749
J-Alves8eb19162022-04-28 10:56:48 +01002750 receiver_index = ffa_memory_region_get_receiver(memory_region,
2751 from_locked.vm->id);
2752
2753 if (receiver_index == memory_region->receiver_count) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002754 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01002755 "VM ID %d tried to relinquish memory region "
J-Alves668a86e2023-05-10 11:53:25 +01002756 "with handle %#x and it is not a valid borrower.\n",
J-Alves8eb19162022-04-28 10:56:48 +01002757 from_locked.vm->id, handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002758 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002759 goto out;
2760 }
2761
J-Alves8eb19162022-04-28 10:56:48 +01002762 if (share_state->retrieved_fragment_count[receiver_index] !=
Andrew Walbranca808b12020-05-15 17:22:28 +01002763 share_state->fragment_count) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002764 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01002765 "Memory with handle %#x not yet fully "
2766 "retrieved, "
J-Alves8eb19162022-04-28 10:56:48 +01002767 "receiver %x can't relinquish.\n",
2768 handle, from_locked.vm->id);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002769 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002770 goto out;
2771 }
2772
J-Alves3c5b2072022-11-21 12:45:40 +00002773 /*
2774 * Either clear if requested in relinquish call, or in a retrieve
2775 * request from one of the borrowers.
2776 */
2777 receivers_relinquished_memory = true;
2778
2779 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
2780 struct ffa_memory_access *receiver =
2781 &memory_region->receivers[i];
2782
2783 if (receiver->receiver_permissions.receiver ==
2784 from_locked.vm->id) {
2785 continue;
2786 }
2787
2788 if (share_state->retrieved_fragment_count[i] != 0U) {
2789 receivers_relinquished_memory = false;
2790 break;
2791 }
2792 }
2793
2794 clear = receivers_relinquished_memory &&
2795 (share_state->clear_after_relinquish ||
2796 (relinquish_request->flags & FFA_MEMORY_REGION_FLAG_CLEAR) !=
2797 0U);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002798
2799 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01002800 * Clear is not allowed for memory that was shared, as the
2801 * original sender still has access to the memory.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002802 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002803 if (clear && share_state->share_func == FFA_MEM_SHARE_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002804 dlog_verbose("Memory which was shared can't be cleared.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002805 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002806 goto out;
2807 }
2808
Andrew Walbranca808b12020-05-15 17:22:28 +01002809 ret = ffa_relinquish_check_update(
J-Alves3c5b2072022-11-21 12:45:40 +00002810 from_locked, memory_region->sender, share_state->fragments,
Andrew Walbranca808b12020-05-15 17:22:28 +01002811 share_state->fragment_constituent_counts,
2812 share_state->fragment_count, page_pool, clear);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002813
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002814 if (ret.func == FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002815 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01002816 * Mark memory handle as not retrieved, so it can be
2817 * reclaimed (or retrieved again).
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002818 */
J-Alves8eb19162022-04-28 10:56:48 +01002819 share_state->retrieved_fragment_count[receiver_index] = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002820 }
2821
2822out:
2823 share_states_unlock(&share_states);
2824 dump_share_states();
2825 return ret;
2826}
2827
2828/**
J-Alvesa9cd7e32022-07-01 13:49:33 +01002829 * Validates that the reclaim transition is allowed for the given
2830 * handle, updates the page table of the reclaiming VM, and frees the
2831 * internal state associated with the handle.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002832 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002833struct ffa_value ffa_memory_reclaim(struct vm_locked to_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01002834 ffa_memory_handle_t handle,
2835 ffa_memory_region_flags_t flags,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002836 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002837{
2838 struct share_states_locked share_states;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002839 struct ffa_memory_share_state *share_state;
2840 struct ffa_memory_region *memory_region;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002841 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002842
2843 dump_share_states();
2844
2845 share_states = share_states_lock();
J-Alvesb5084cf2022-07-06 14:20:12 +01002846 if (get_share_state(share_states, handle, &share_state)) {
2847 memory_region = share_state->memory_region;
2848 } else {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002849 dlog_verbose("Invalid handle %#x for FFA_MEM_RECLAIM.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002850 handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002851 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002852 goto out;
2853 }
2854
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002855 CHECK(memory_region != NULL);
2856
J-Alvesa9cd7e32022-07-01 13:49:33 +01002857 if (vm_id_is_current_world(to_locked.vm->id) &&
2858 to_locked.vm->id != memory_region->sender) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002859 dlog_verbose(
Olivier Deprezf92e5d42020-11-13 16:00:54 +01002860 "VM %#x attempted to reclaim memory handle %#x "
2861 "originally sent by VM %#x.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002862 to_locked.vm->id, handle, memory_region->sender);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002863 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002864 goto out;
2865 }
2866
Andrew Walbranca808b12020-05-15 17:22:28 +01002867 if (!share_state->sending_complete) {
2868 dlog_verbose(
2869 "Memory with handle %#x not fully sent, can't "
2870 "reclaim.\n",
2871 handle);
2872 ret = ffa_error(FFA_INVALID_PARAMETERS);
2873 goto out;
2874 }
2875
J-Alves752236c2022-04-28 11:07:47 +01002876 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
2877 if (share_state->retrieved_fragment_count[i] != 0) {
2878 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01002879 "Tried to reclaim memory handle %#x "
J-Alves3c5b2072022-11-21 12:45:40 +00002880 "that has not been relinquished by all "
J-Alvesa9cd7e32022-07-01 13:49:33 +01002881 "borrowers(%x).\n",
J-Alves752236c2022-04-28 11:07:47 +01002882 handle,
2883 memory_region->receivers[i]
2884 .receiver_permissions.receiver);
2885 ret = ffa_error(FFA_DENIED);
2886 goto out;
2887 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002888 }
2889
Andrew Walbranca808b12020-05-15 17:22:28 +01002890 ret = ffa_retrieve_check_update(
J-Alves7db32002021-12-14 14:44:50 +00002891 to_locked, memory_region->sender, share_state->fragments,
Andrew Walbranca808b12020-05-15 17:22:28 +01002892 share_state->fragment_constituent_counts,
J-Alves2a0d2882020-10-29 14:49:50 +00002893 share_state->fragment_count, share_state->sender_orig_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +01002894 FFA_MEM_RECLAIM_32, flags & FFA_MEM_RECLAIM_CLEAR, page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002895
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002896 if (ret.func == FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002897 share_state_free(share_states, share_state, page_pool);
J-Alves3c5b2072022-11-21 12:45:40 +00002898 dlog_verbose("Freed share state after successful reclaim.\n");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002899 }
2900
2901out:
2902 share_states_unlock(&share_states);
2903 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01002904}