blob: bf2fe5fc081e8ca99e9fe8cd35b4458c978851db [file] [log] [blame]
Jose Marinho75509b42019-04-09 09:34:59 +01001/*
2 * Copyright 2019 The Hafnium Authors.
3 *
Andrew Walbrane959ec12020-06-17 15:01:09 +01004 * Use of this source code is governed by a BSD-style
5 * license that can be found in the LICENSE file or at
6 * https://opensource.org/licenses/BSD-3-Clause.
Jose Marinho75509b42019-04-09 09:34:59 +01007 */
8
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01009#include "hf/ffa_memory.h"
Andrew Walbran475c1452020-02-07 13:22:22 +000010
Olivier Deprez112d2b52020-09-30 07:39:23 +020011#include "hf/arch/other_world.h"
Olivier Deprez55a189e2021-06-09 15:45:27 +020012#include "hf/arch/plat/ffa.h"
Andrew Walbran290b0c92020-02-03 16:37:14 +000013
Jose Marinho75509b42019-04-09 09:34:59 +010014#include "hf/api.h"
Daniel Boulbya2f8c662021-11-26 17:52:53 +000015#include "hf/assert.h"
Jose Marinho09b1db82019-08-08 09:16:59 +010016#include "hf/check.h"
Jose Marinho75509b42019-04-09 09:34:59 +010017#include "hf/dlog.h"
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010018#include "hf/ffa_internal.h"
Andrew Walbran475c1452020-02-07 13:22:22 +000019#include "hf/mpool.h"
Jose Marinho75509b42019-04-09 09:34:59 +010020#include "hf/std.h"
Andrew Scull3c257452019-11-26 13:32:50 +000021#include "hf/vm.h"
Jose Marinho75509b42019-04-09 09:34:59 +010022
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000023/** The maximum number of recipients a memory region may be sent to. */
24#define MAX_MEM_SHARE_RECIPIENTS 1
25
26/**
27 * The maximum number of memory sharing handles which may be active at once. A
28 * DONATE handle is active from when it is sent to when it is retrieved; a SHARE
29 * or LEND handle is active from when it is sent to when it is reclaimed.
30 */
31#define MAX_MEM_SHARES 100
32
Andrew Walbranca808b12020-05-15 17:22:28 +010033/**
34 * The maximum number of fragments into which a memory sharing message may be
35 * broken.
36 */
37#define MAX_FRAGMENTS 20
38
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010039static_assert(sizeof(struct ffa_memory_region_constituent) % 16 == 0,
40 "struct ffa_memory_region_constituent must be a multiple of 16 "
Andrew Walbranc34c7b22020-02-28 11:16:59 +000041 "bytes long.");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010042static_assert(sizeof(struct ffa_composite_memory_region) % 16 == 0,
43 "struct ffa_composite_memory_region must be a multiple of 16 "
Andrew Walbranc34c7b22020-02-28 11:16:59 +000044 "bytes long.");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010045static_assert(sizeof(struct ffa_memory_region_attributes) == 4,
Andrew Walbran41890ff2020-09-23 15:09:39 +010046 "struct ffa_memory_region_attributes must be 4 bytes long.");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010047static_assert(sizeof(struct ffa_memory_access) % 16 == 0,
48 "struct ffa_memory_access must be a multiple of 16 bytes long.");
49static_assert(sizeof(struct ffa_memory_region) % 16 == 0,
50 "struct ffa_memory_region must be a multiple of 16 bytes long.");
51static_assert(sizeof(struct ffa_mem_relinquish) % 16 == 0,
52 "struct ffa_mem_relinquish must be a multiple of 16 "
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000053 "bytes long.");
Andrew Walbranc34c7b22020-02-28 11:16:59 +000054
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010055struct ffa_memory_share_state {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000056 /**
57 * The memory region being shared, or NULL if this share state is
58 * unallocated.
59 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010060 struct ffa_memory_region *memory_region;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000061
Andrew Walbranca808b12020-05-15 17:22:28 +010062 struct ffa_memory_region_constituent *fragments[MAX_FRAGMENTS];
63
64 /** The number of constituents in each fragment. */
65 uint32_t fragment_constituent_counts[MAX_FRAGMENTS];
66
67 /**
68 * The number of valid elements in the `fragments` and
69 * `fragment_constituent_counts` arrays.
70 */
71 uint32_t fragment_count;
72
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000073 /**
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010074 * The FF-A function used for sharing the memory. Must be one of
75 * FFA_MEM_DONATE_32, FFA_MEM_LEND_32 or FFA_MEM_SHARE_32 if the
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000076 * share state is allocated, or 0.
77 */
78 uint32_t share_func;
79
80 /**
J-Alves2a0d2882020-10-29 14:49:50 +000081 * The sender's original mode before invoking the FF-A function for
82 * sharing the memory.
83 * This is used to reset the original configuration when sender invokes
84 * FFA_MEM_RECLAIM_32.
85 */
86 uint32_t sender_orig_mode;
87
88 /**
Andrew Walbranca808b12020-05-15 17:22:28 +010089 * True if all the fragments of this sharing request have been sent and
90 * Hafnium has updated the sender page table accordingly.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000091 */
Andrew Walbranca808b12020-05-15 17:22:28 +010092 bool sending_complete;
93
94 /**
95 * How many fragments of the memory region each recipient has retrieved
96 * so far. The order of this array matches the order of the endpoint
97 * memory access descriptors in the memory region descriptor. Any
98 * entries beyond the receiver_count will always be 0.
99 */
100 uint32_t retrieved_fragment_count[MAX_MEM_SHARE_RECIPIENTS];
Andrew Walbran475c1452020-02-07 13:22:22 +0000101};
102
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000103/**
104 * Encapsulates the set of share states while the `share_states_lock` is held.
105 */
106struct share_states_locked {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100107 struct ffa_memory_share_state *share_states;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000108};
109
110/**
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100111 * All access to members of a `struct ffa_memory_share_state` must be guarded
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000112 * by this lock.
113 */
114static struct spinlock share_states_lock_instance = SPINLOCK_INIT;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100115static struct ffa_memory_share_state share_states[MAX_MEM_SHARES];
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000116
117/**
Andrew Walbranca808b12020-05-15 17:22:28 +0100118 * Buffer for retrieving memory region information from the TEE for when a
119 * region is reclaimed by a VM. Access to this buffer must be guarded by the VM
120 * lock of the TEE VM.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000121 */
Andrew Walbranca808b12020-05-15 17:22:28 +0100122alignas(PAGE_SIZE) static uint8_t
123 tee_retrieve_buffer[HF_MAILBOX_SIZE * MAX_FRAGMENTS];
124
125/**
J-Alves917d2f22020-10-30 18:39:30 +0000126 * Extracts the index from a memory handle allocated by Hafnium's current world.
127 */
128uint64_t ffa_memory_handle_get_index(ffa_memory_handle_t handle)
129{
130 return handle & ~FFA_MEMORY_HANDLE_ALLOCATOR_MASK;
131}
132
133/**
Andrew Walbranca808b12020-05-15 17:22:28 +0100134 * Initialises the next available `struct ffa_memory_share_state` and sets
135 * `share_state_ret` to a pointer to it. If `handle` is
136 * `FFA_MEMORY_HANDLE_INVALID` then allocates an appropriate handle, otherwise
137 * uses the provided handle which is assumed to be globally unique.
138 *
139 * Returns true on success or false if none are available.
140 */
141static bool allocate_share_state(
142 struct share_states_locked share_states, uint32_t share_func,
143 struct ffa_memory_region *memory_region, uint32_t fragment_length,
144 ffa_memory_handle_t handle,
145 struct ffa_memory_share_state **share_state_ret)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000146{
Andrew Walbrana65a1322020-04-06 19:32:32 +0100147 uint64_t i;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000148
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000149 assert(share_states.share_states != NULL);
150 assert(memory_region != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000151
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000152 for (i = 0; i < MAX_MEM_SHARES; ++i) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100153 if (share_states.share_states[i].share_func == 0) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000154 uint32_t j;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100155 struct ffa_memory_share_state *allocated_state =
Andrew Walbranca808b12020-05-15 17:22:28 +0100156 &share_states.share_states[i];
157 struct ffa_composite_memory_region *composite =
158 ffa_memory_region_get_composite(memory_region,
159 0);
160
161 if (handle == FFA_MEMORY_HANDLE_INVALID) {
J-Alvesee68c542020-10-29 17:48:20 +0000162 memory_region->handle =
Olivier Deprez55a189e2021-06-09 15:45:27 +0200163 plat_ffa_memory_handle_make(i);
Andrew Walbranca808b12020-05-15 17:22:28 +0100164 } else {
J-Alvesee68c542020-10-29 17:48:20 +0000165 memory_region->handle = handle;
Andrew Walbranca808b12020-05-15 17:22:28 +0100166 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000167 allocated_state->share_func = share_func;
168 allocated_state->memory_region = memory_region;
Andrew Walbranca808b12020-05-15 17:22:28 +0100169 allocated_state->fragment_count = 1;
170 allocated_state->fragments[0] = composite->constituents;
171 allocated_state->fragment_constituent_counts[0] =
172 (fragment_length -
173 ffa_composite_constituent_offset(memory_region,
174 0)) /
175 sizeof(struct ffa_memory_region_constituent);
176 allocated_state->sending_complete = false;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000177 for (j = 0; j < MAX_MEM_SHARE_RECIPIENTS; ++j) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100178 allocated_state->retrieved_fragment_count[j] =
179 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000180 }
Andrew Walbranca808b12020-05-15 17:22:28 +0100181 if (share_state_ret != NULL) {
182 *share_state_ret = allocated_state;
183 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000184 return true;
185 }
186 }
187
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000188 return false;
189}
190
191/** Locks the share states lock. */
192struct share_states_locked share_states_lock(void)
193{
194 sl_lock(&share_states_lock_instance);
195
196 return (struct share_states_locked){.share_states = share_states};
197}
198
199/** Unlocks the share states lock. */
200static void share_states_unlock(struct share_states_locked *share_states)
201{
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000202 assert(share_states->share_states != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000203 share_states->share_states = NULL;
204 sl_unlock(&share_states_lock_instance);
205}
206
207/**
Andrew Walbranca808b12020-05-15 17:22:28 +0100208 * If the given handle is a valid handle for an allocated share state then
209 * initialises `share_state_ret` to point to the share state and returns true.
210 * Otherwise returns false.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000211 */
212static bool get_share_state(struct share_states_locked share_states,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100213 ffa_memory_handle_t handle,
214 struct ffa_memory_share_state **share_state_ret)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000215{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100216 struct ffa_memory_share_state *share_state;
J-Alves917d2f22020-10-30 18:39:30 +0000217 uint64_t index;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000218
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000219 assert(share_states.share_states != NULL);
220 assert(share_state_ret != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100221
222 /*
223 * First look for a share_state allocated by us, in which case the
224 * handle is based on the index.
225 */
Olivier Deprez55a189e2021-06-09 15:45:27 +0200226 if (plat_ffa_memory_handle_allocated_by_current_world(handle)) {
J-Alves917d2f22020-10-30 18:39:30 +0000227 index = ffa_memory_handle_get_index(handle);
Andrew Walbranca808b12020-05-15 17:22:28 +0100228 if (index < MAX_MEM_SHARES) {
229 share_state = &share_states.share_states[index];
230 if (share_state->share_func != 0) {
231 *share_state_ret = share_state;
232 return true;
233 }
234 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000235 }
236
Andrew Walbranca808b12020-05-15 17:22:28 +0100237 /* Fall back to a linear scan. */
238 for (index = 0; index < MAX_MEM_SHARES; ++index) {
239 share_state = &share_states.share_states[index];
J-Alvesee68c542020-10-29 17:48:20 +0000240 if (share_state->memory_region != NULL &&
241 share_state->memory_region->handle == handle &&
Andrew Walbranca808b12020-05-15 17:22:28 +0100242 share_state->share_func != 0) {
243 *share_state_ret = share_state;
244 return true;
245 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000246 }
247
Andrew Walbranca808b12020-05-15 17:22:28 +0100248 return false;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000249}
250
251/** Marks a share state as unallocated. */
252static void share_state_free(struct share_states_locked share_states,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100253 struct ffa_memory_share_state *share_state,
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000254 struct mpool *page_pool)
255{
Andrew Walbranca808b12020-05-15 17:22:28 +0100256 uint32_t i;
257
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000258 assert(share_states.share_states != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000259 share_state->share_func = 0;
Andrew Walbranca808b12020-05-15 17:22:28 +0100260 share_state->sending_complete = false;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000261 mpool_free(page_pool, share_state->memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +0100262 /*
263 * First fragment is part of the same page as the `memory_region`, so it
264 * doesn't need to be freed separately.
265 */
266 share_state->fragments[0] = NULL;
267 share_state->fragment_constituent_counts[0] = 0;
268 for (i = 1; i < share_state->fragment_count; ++i) {
269 mpool_free(page_pool, share_state->fragments[i]);
270 share_state->fragments[i] = NULL;
271 share_state->fragment_constituent_counts[i] = 0;
272 }
273 share_state->fragment_count = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000274 share_state->memory_region = NULL;
275}
276
Andrew Walbranca808b12020-05-15 17:22:28 +0100277/** Checks whether the given share state has been fully sent. */
278static bool share_state_sending_complete(
279 struct share_states_locked share_states,
280 struct ffa_memory_share_state *share_state)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000281{
Andrew Walbranca808b12020-05-15 17:22:28 +0100282 struct ffa_composite_memory_region *composite;
283 uint32_t expected_constituent_count;
284 uint32_t fragment_constituent_count_total = 0;
285 uint32_t i;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000286
Andrew Walbranca808b12020-05-15 17:22:28 +0100287 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000288 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100289
290 /*
291 * Share state must already be valid, or it's not possible to get hold
292 * of it.
293 */
294 CHECK(share_state->memory_region != NULL &&
295 share_state->share_func != 0);
296
297 composite =
298 ffa_memory_region_get_composite(share_state->memory_region, 0);
299 expected_constituent_count = composite->constituent_count;
300 for (i = 0; i < share_state->fragment_count; ++i) {
301 fragment_constituent_count_total +=
302 share_state->fragment_constituent_counts[i];
303 }
304 dlog_verbose(
305 "Checking completion: constituent count %d/%d from %d "
306 "fragments.\n",
307 fragment_constituent_count_total, expected_constituent_count,
308 share_state->fragment_count);
309
310 return fragment_constituent_count_total == expected_constituent_count;
311}
312
313/**
314 * Calculates the offset of the next fragment expected for the given share
315 * state.
316 */
317static uint32_t share_state_next_fragment_offset(
318 struct share_states_locked share_states,
319 struct ffa_memory_share_state *share_state)
320{
321 uint32_t next_fragment_offset;
322 uint32_t i;
323
324 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000325 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100326
327 next_fragment_offset =
328 ffa_composite_constituent_offset(share_state->memory_region, 0);
329 for (i = 0; i < share_state->fragment_count; ++i) {
330 next_fragment_offset +=
331 share_state->fragment_constituent_counts[i] *
332 sizeof(struct ffa_memory_region_constituent);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000333 }
334
Andrew Walbranca808b12020-05-15 17:22:28 +0100335 return next_fragment_offset;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000336}
337
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100338static void dump_memory_region(struct ffa_memory_region *memory_region)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000339{
340 uint32_t i;
341
342 if (LOG_LEVEL < LOG_LEVEL_VERBOSE) {
343 return;
344 }
345
Olivier Deprez935e1b12020-12-22 18:01:29 +0100346 dlog("from VM %#x, attributes %#x, flags %#x, tag %u, to "
Olivier Deprezf92e5d42020-11-13 16:00:54 +0100347 "%u "
Andrew Walbrana65a1322020-04-06 19:32:32 +0100348 "recipients [",
349 memory_region->sender, memory_region->attributes,
Olivier Deprez935e1b12020-12-22 18:01:29 +0100350 memory_region->flags, memory_region->tag,
Andrew Walbrana65a1322020-04-06 19:32:32 +0100351 memory_region->receiver_count);
352 for (i = 0; i < memory_region->receiver_count; ++i) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000353 if (i != 0) {
354 dlog(", ");
355 }
Olivier Deprezf92e5d42020-11-13 16:00:54 +0100356 dlog("VM %#x: %#x (offset %u)",
Andrew Walbrana65a1322020-04-06 19:32:32 +0100357 memory_region->receivers[i].receiver_permissions.receiver,
358 memory_region->receivers[i]
359 .receiver_permissions.permissions,
360 memory_region->receivers[i]
361 .composite_memory_region_offset);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000362 }
363 dlog("]");
364}
365
366static void dump_share_states(void)
367{
368 uint32_t i;
369
370 if (LOG_LEVEL < LOG_LEVEL_VERBOSE) {
371 return;
372 }
373
374 dlog("Current share states:\n");
375 sl_lock(&share_states_lock_instance);
376 for (i = 0; i < MAX_MEM_SHARES; ++i) {
377 if (share_states[i].share_func != 0) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000378 switch (share_states[i].share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100379 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000380 dlog("SHARE");
381 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100382 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000383 dlog("LEND");
384 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100385 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000386 dlog("DONATE");
387 break;
388 default:
389 dlog("invalid share_func %#x",
390 share_states[i].share_func);
391 }
Olivier Deprez935e1b12020-12-22 18:01:29 +0100392 dlog(" %#x (", share_states[i].memory_region->handle);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000393 dump_memory_region(share_states[i].memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +0100394 if (share_states[i].sending_complete) {
395 dlog("): fully sent");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000396 } else {
Andrew Walbranca808b12020-05-15 17:22:28 +0100397 dlog("): partially sent");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000398 }
J-Alves2a0d2882020-10-29 14:49:50 +0000399 dlog(" with %d fragments, %d retrieved, "
400 " sender's original mode: %#x\n",
Andrew Walbranca808b12020-05-15 17:22:28 +0100401 share_states[i].fragment_count,
J-Alves2a0d2882020-10-29 14:49:50 +0000402 share_states[i].retrieved_fragment_count[0],
403 share_states[i].sender_orig_mode);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000404 }
405 }
406 sl_unlock(&share_states_lock_instance);
407}
408
Andrew Walbran475c1452020-02-07 13:22:22 +0000409/* TODO: Add device attributes: GRE, cacheability, shareability. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100410static inline uint32_t ffa_memory_permissions_to_mode(
J-Alves7cd5eb32020-10-16 19:06:10 +0100411 ffa_memory_access_permissions_t permissions, uint32_t default_mode)
Andrew Walbran475c1452020-02-07 13:22:22 +0000412{
413 uint32_t mode = 0;
414
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100415 switch (ffa_get_data_access_attr(permissions)) {
416 case FFA_DATA_ACCESS_RO:
Andrew Walbran475c1452020-02-07 13:22:22 +0000417 mode = MM_MODE_R;
418 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100419 case FFA_DATA_ACCESS_RW:
Andrew Walbran475c1452020-02-07 13:22:22 +0000420 mode = MM_MODE_R | MM_MODE_W;
421 break;
J-Alves7cd5eb32020-10-16 19:06:10 +0100422 case FFA_DATA_ACCESS_NOT_SPECIFIED:
423 mode = (default_mode & (MM_MODE_R | MM_MODE_W));
424 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100425 case FFA_DATA_ACCESS_RESERVED:
426 panic("Tried to convert FFA_DATA_ACCESS_RESERVED.");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100427 }
428
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100429 switch (ffa_get_instruction_access_attr(permissions)) {
430 case FFA_INSTRUCTION_ACCESS_NX:
Andrew Walbran475c1452020-02-07 13:22:22 +0000431 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100432 case FFA_INSTRUCTION_ACCESS_X:
Andrew Walbrana65a1322020-04-06 19:32:32 +0100433 mode |= MM_MODE_X;
434 break;
J-Alves7cd5eb32020-10-16 19:06:10 +0100435 case FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED:
436 mode |= (default_mode & MM_MODE_X);
437 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100438 case FFA_INSTRUCTION_ACCESS_RESERVED:
439 panic("Tried to convert FFA_INSTRUCTION_ACCESS_RESVERVED.");
Andrew Walbran475c1452020-02-07 13:22:22 +0000440 }
441
442 return mode;
443}
444
Jose Marinho75509b42019-04-09 09:34:59 +0100445/**
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000446 * Get the current mode in the stage-2 page table of the given vm of all the
447 * pages in the given constituents, if they all have the same mode, or return
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100448 * an appropriate FF-A error if not.
Jose Marinho75509b42019-04-09 09:34:59 +0100449 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100450static struct ffa_value constituents_get_mode(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000451 struct vm_locked vm, uint32_t *orig_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100452 struct ffa_memory_region_constituent **fragments,
453 const uint32_t *fragment_constituent_counts, uint32_t fragment_count)
Jose Marinho75509b42019-04-09 09:34:59 +0100454{
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100455 uint32_t i;
Andrew Walbranca808b12020-05-15 17:22:28 +0100456 uint32_t j;
Jose Marinho75509b42019-04-09 09:34:59 +0100457
Andrew Walbranca808b12020-05-15 17:22:28 +0100458 if (fragment_count == 0 || fragment_constituent_counts[0] == 0) {
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100459 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000460 * Fail if there are no constituents. Otherwise we would get an
461 * uninitialised *orig_mode.
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100462 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100463 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho75509b42019-04-09 09:34:59 +0100464 }
465
Andrew Walbranca808b12020-05-15 17:22:28 +0100466 for (i = 0; i < fragment_count; ++i) {
467 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
468 ipaddr_t begin = ipa_init(fragments[i][j].address);
469 size_t size = fragments[i][j].page_count * PAGE_SIZE;
470 ipaddr_t end = ipa_add(begin, size);
471 uint32_t current_mode;
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100472
Andrew Walbranca808b12020-05-15 17:22:28 +0100473 /* Fail if addresses are not page-aligned. */
474 if (!is_aligned(ipa_addr(begin), PAGE_SIZE) ||
475 !is_aligned(ipa_addr(end), PAGE_SIZE)) {
476 return ffa_error(FFA_INVALID_PARAMETERS);
477 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100478
Andrew Walbranca808b12020-05-15 17:22:28 +0100479 /*
480 * Ensure that this constituent memory range is all
481 * mapped with the same mode.
482 */
Raghu Krishnamurthy785d52f2021-02-13 00:02:40 -0800483 if (!vm_mem_get_mode(vm, begin, end, &current_mode)) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100484 return ffa_error(FFA_DENIED);
485 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100486
Andrew Walbranca808b12020-05-15 17:22:28 +0100487 /*
488 * Ensure that all constituents are mapped with the same
489 * mode.
490 */
491 if (i == 0) {
492 *orig_mode = current_mode;
493 } else if (current_mode != *orig_mode) {
494 dlog_verbose(
495 "Expected mode %#x but was %#x for %d "
496 "pages at %#x.\n",
497 *orig_mode, current_mode,
498 fragments[i][j].page_count,
499 ipa_addr(begin));
500 return ffa_error(FFA_DENIED);
501 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100502 }
Jose Marinho75509b42019-04-09 09:34:59 +0100503 }
504
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100505 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000506}
507
508/**
509 * Verify that all pages have the same mode, that the starting mode
510 * constitutes a valid state and obtain the next mode to apply
511 * to the sending VM.
512 *
513 * Returns:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100514 * 1) FFA_DENIED if a state transition was not found;
515 * 2) FFA_DENIED if the pages being shared do not have the same mode within
Andrew Walbrana65a1322020-04-06 19:32:32 +0100516 * the <from> VM;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100517 * 3) FFA_INVALID_PARAMETERS if the beginning and end IPAs are not page
Andrew Walbrana65a1322020-04-06 19:32:32 +0100518 * aligned;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100519 * 4) FFA_INVALID_PARAMETERS if the requested share type was not handled.
520 * Or FFA_SUCCESS on success.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000521 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100522static struct ffa_value ffa_send_check_transition(
Andrew Walbrana65a1322020-04-06 19:32:32 +0100523 struct vm_locked from, uint32_t share_func,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100524 ffa_memory_access_permissions_t permissions, uint32_t *orig_from_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100525 struct ffa_memory_region_constituent **fragments,
526 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
527 uint32_t *from_mode)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000528{
529 const uint32_t state_mask =
530 MM_MODE_INVALID | MM_MODE_UNOWNED | MM_MODE_SHARED;
J-Alves7cd5eb32020-10-16 19:06:10 +0100531 uint32_t required_from_mode;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100532 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000533
Andrew Walbranca808b12020-05-15 17:22:28 +0100534 ret = constituents_get_mode(from, orig_from_mode, fragments,
535 fragment_constituent_counts,
536 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100537 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100538 dlog_verbose("Inconsistent modes.\n", fragment_count);
Andrew Walbrana65a1322020-04-06 19:32:32 +0100539 return ret;
Andrew Scullb5f49e02019-10-02 13:20:47 +0100540 }
541
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000542 /* Ensure the address range is normal memory and not a device. */
543 if (*orig_from_mode & MM_MODE_D) {
544 dlog_verbose("Can't share device memory (mode is %#x).\n",
545 *orig_from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100546 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000547 }
548
549 /*
550 * Ensure the sender is the owner and has exclusive access to the
551 * memory.
552 */
553 if ((*orig_from_mode & state_mask) != 0) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100554 return ffa_error(FFA_DENIED);
Andrew Walbrana65a1322020-04-06 19:32:32 +0100555 }
556
J-Alves7cd5eb32020-10-16 19:06:10 +0100557 required_from_mode =
558 ffa_memory_permissions_to_mode(permissions, *orig_from_mode);
559
Andrew Walbrana65a1322020-04-06 19:32:32 +0100560 if ((*orig_from_mode & required_from_mode) != required_from_mode) {
561 dlog_verbose(
562 "Sender tried to send memory with permissions which "
563 "required mode %#x but only had %#x itself.\n",
564 required_from_mode, *orig_from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100565 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000566 }
567
568 /* Find the appropriate new mode. */
569 *from_mode = ~state_mask & *orig_from_mode;
Andrew Walbrane7ad3c02019-12-24 17:03:04 +0000570 switch (share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100571 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000572 *from_mode |= MM_MODE_INVALID | MM_MODE_UNOWNED;
Jose Marinho75509b42019-04-09 09:34:59 +0100573 break;
574
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100575 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000576 *from_mode |= MM_MODE_INVALID;
Andrew Walbran648fc3e2019-10-22 16:23:05 +0100577 break;
578
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100579 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000580 *from_mode |= MM_MODE_SHARED;
Jose Marinho56c25732019-05-20 09:48:53 +0100581 break;
582
Jose Marinho75509b42019-04-09 09:34:59 +0100583 default:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100584 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho75509b42019-04-09 09:34:59 +0100585 }
586
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100587 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000588}
589
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100590static struct ffa_value ffa_relinquish_check_transition(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000591 struct vm_locked from, uint32_t *orig_from_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100592 struct ffa_memory_region_constituent **fragments,
593 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
594 uint32_t *from_mode)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000595{
596 const uint32_t state_mask =
597 MM_MODE_INVALID | MM_MODE_UNOWNED | MM_MODE_SHARED;
598 uint32_t orig_from_state;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100599 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000600
Andrew Walbranca808b12020-05-15 17:22:28 +0100601 ret = constituents_get_mode(from, orig_from_mode, fragments,
602 fragment_constituent_counts,
603 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100604 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbrana65a1322020-04-06 19:32:32 +0100605 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000606 }
607
608 /* Ensure the address range is normal memory and not a device. */
609 if (*orig_from_mode & MM_MODE_D) {
610 dlog_verbose("Can't relinquish device memory (mode is %#x).\n",
611 *orig_from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100612 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000613 }
614
615 /*
616 * Ensure the relinquishing VM is not the owner but has access to the
617 * memory.
618 */
619 orig_from_state = *orig_from_mode & state_mask;
620 if ((orig_from_state & ~MM_MODE_SHARED) != MM_MODE_UNOWNED) {
621 dlog_verbose(
622 "Tried to relinquish memory in state %#x (masked %#x "
Andrew Walbranca808b12020-05-15 17:22:28 +0100623 "but should be %#x).\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000624 *orig_from_mode, orig_from_state, MM_MODE_UNOWNED);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100625 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000626 }
627
628 /* Find the appropriate new mode. */
629 *from_mode = (~state_mask & *orig_from_mode) | MM_MODE_UNMAPPED_MASK;
630
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100631 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000632}
633
634/**
635 * Verify that all pages have the same mode, that the starting mode
636 * constitutes a valid state and obtain the next mode to apply
637 * to the retrieving VM.
638 *
639 * Returns:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100640 * 1) FFA_DENIED if a state transition was not found;
641 * 2) FFA_DENIED if the pages being shared do not have the same mode within
Andrew Walbrana65a1322020-04-06 19:32:32 +0100642 * the <to> VM;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100643 * 3) FFA_INVALID_PARAMETERS if the beginning and end IPAs are not page
Andrew Walbrana65a1322020-04-06 19:32:32 +0100644 * aligned;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100645 * 4) FFA_INVALID_PARAMETERS if the requested share type was not handled.
646 * Or FFA_SUCCESS on success.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000647 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100648static struct ffa_value ffa_retrieve_check_transition(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000649 struct vm_locked to, uint32_t share_func,
Andrew Walbranca808b12020-05-15 17:22:28 +0100650 struct ffa_memory_region_constituent **fragments,
651 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
652 uint32_t memory_to_attributes, uint32_t *to_mode)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000653{
654 uint32_t orig_to_mode;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100655 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000656
Andrew Walbranca808b12020-05-15 17:22:28 +0100657 ret = constituents_get_mode(to, &orig_to_mode, fragments,
658 fragment_constituent_counts,
659 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100660 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100661 dlog_verbose("Inconsistent modes.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100662 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000663 }
664
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100665 if (share_func == FFA_MEM_RECLAIM_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000666 const uint32_t state_mask =
667 MM_MODE_INVALID | MM_MODE_UNOWNED | MM_MODE_SHARED;
668 uint32_t orig_to_state = orig_to_mode & state_mask;
669
670 if (orig_to_state != MM_MODE_INVALID &&
671 orig_to_state != MM_MODE_SHARED) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100672 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000673 }
674 } else {
675 /*
676 * Ensure the retriever has the expected state. We don't care
677 * about the MM_MODE_SHARED bit; either with or without it set
678 * are both valid representations of the !O-NA state.
679 */
680 if ((orig_to_mode & MM_MODE_UNMAPPED_MASK) !=
681 MM_MODE_UNMAPPED_MASK) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100682 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000683 }
684 }
685
686 /* Find the appropriate new mode. */
687 *to_mode = memory_to_attributes;
688 switch (share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100689 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000690 *to_mode |= 0;
691 break;
692
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100693 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000694 *to_mode |= MM_MODE_UNOWNED;
695 break;
696
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100697 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000698 *to_mode |= MM_MODE_UNOWNED | MM_MODE_SHARED;
699 break;
700
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100701 case FFA_MEM_RECLAIM_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000702 *to_mode |= 0;
703 break;
704
705 default:
Andrew Walbranca808b12020-05-15 17:22:28 +0100706 dlog_error("Invalid share_func %#x.\n", share_func);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100707 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000708 }
709
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100710 return (struct ffa_value){.func = FFA_SUCCESS_32};
Jose Marinho75509b42019-04-09 09:34:59 +0100711}
Jose Marinho09b1db82019-08-08 09:16:59 +0100712
713/**
714 * Updates a VM's page table such that the given set of physical address ranges
715 * are mapped in the address space at the corresponding address ranges, in the
716 * mode provided.
717 *
718 * If commit is false, the page tables will be allocated from the mpool but no
719 * mappings will actually be updated. This function must always be called first
720 * with commit false to check that it will succeed before calling with commit
721 * true, to avoid leaving the page table in a half-updated state. To make a
722 * series of changes atomically you can call them all with commit false before
723 * calling them all with commit true.
724 *
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -0700725 * vm_ptable_defrag should always be called after a series of page table
726 * updates, whether they succeed or fail.
Jose Marinho09b1db82019-08-08 09:16:59 +0100727 *
728 * Returns true on success, or false if the update failed and no changes were
729 * made to memory mappings.
730 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100731static bool ffa_region_group_identity_map(
Andrew Walbranf4b51af2020-02-03 14:44:54 +0000732 struct vm_locked vm_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +0100733 struct ffa_memory_region_constituent **fragments,
734 const uint32_t *fragment_constituent_counts, uint32_t fragment_count,
Daniel Boulby4dd3f532021-09-21 09:57:08 +0100735 uint32_t mode, struct mpool *ppool, bool commit)
Jose Marinho09b1db82019-08-08 09:16:59 +0100736{
Andrew Walbranca808b12020-05-15 17:22:28 +0100737 uint32_t i;
738 uint32_t j;
Jose Marinho09b1db82019-08-08 09:16:59 +0100739
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -0700740 if (vm_locked.vm->el0_partition) {
741 mode |= MM_MODE_USER | MM_MODE_NG;
742 }
743
Andrew Walbranca808b12020-05-15 17:22:28 +0100744 /* Iterate over the memory region constituents within each fragment. */
745 for (i = 0; i < fragment_count; ++i) {
746 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
747 size_t size = fragments[i][j].page_count * PAGE_SIZE;
748 paddr_t pa_begin =
749 pa_from_ipa(ipa_init(fragments[i][j].address));
750 paddr_t pa_end = pa_add(pa_begin, size);
751
752 if (commit) {
753 vm_identity_commit(vm_locked, pa_begin, pa_end,
754 mode, ppool, NULL);
755 } else if (!vm_identity_prepare(vm_locked, pa_begin,
756 pa_end, mode, ppool)) {
757 return false;
758 }
Jose Marinho09b1db82019-08-08 09:16:59 +0100759 }
760 }
761
762 return true;
763}
764
765/**
766 * Clears a region of physical memory by overwriting it with zeros. The data is
767 * flushed from the cache so the memory has been cleared across the system.
768 */
769static bool clear_memory(paddr_t begin, paddr_t end, struct mpool *ppool)
770{
771 /*
Fuad Tabbaed294af2019-12-20 10:43:01 +0000772 * TODO: change this to a CPU local single page window rather than a
Jose Marinho09b1db82019-08-08 09:16:59 +0100773 * global mapping of the whole range. Such an approach will limit
774 * the changes to stage-1 tables and will allow only local
775 * invalidation.
776 */
777 bool ret;
778 struct mm_stage1_locked stage1_locked = mm_lock_stage1();
779 void *ptr =
780 mm_identity_map(stage1_locked, begin, end, MM_MODE_W, ppool);
781 size_t size = pa_difference(begin, end);
782
783 if (!ptr) {
784 /* TODO: partial defrag of failed range. */
785 /* Recover any memory consumed in failed mapping. */
786 mm_defrag(stage1_locked, ppool);
787 goto fail;
788 }
789
790 memset_s(ptr, size, 0, size);
791 arch_mm_flush_dcache(ptr, size);
792 mm_unmap(stage1_locked, begin, end, ppool);
793
794 ret = true;
795 goto out;
796
797fail:
798 ret = false;
799
800out:
801 mm_unlock_stage1(&stage1_locked);
802
803 return ret;
804}
805
806/**
807 * Clears a region of physical memory by overwriting it with zeros. The data is
808 * flushed from the cache so the memory has been cleared across the system.
809 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100810static bool ffa_clear_memory_constituents(
Andrew Walbranca808b12020-05-15 17:22:28 +0100811 struct ffa_memory_region_constituent **fragments,
812 const uint32_t *fragment_constituent_counts, uint32_t fragment_count,
813 struct mpool *page_pool)
Jose Marinho09b1db82019-08-08 09:16:59 +0100814{
815 struct mpool local_page_pool;
Andrew Walbranca808b12020-05-15 17:22:28 +0100816 uint32_t i;
Jose Marinho09b1db82019-08-08 09:16:59 +0100817 struct mm_stage1_locked stage1_locked;
818 bool ret = false;
819
820 /*
821 * Create a local pool so any freed memory can't be used by another
822 * thread. This is to ensure each constituent that is mapped can be
823 * unmapped again afterwards.
824 */
Andrew Walbran475c1452020-02-07 13:22:22 +0000825 mpool_init_with_fallback(&local_page_pool, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +0100826
Andrew Walbranca808b12020-05-15 17:22:28 +0100827 /* Iterate over the memory region constituents within each fragment. */
828 for (i = 0; i < fragment_count; ++i) {
829 uint32_t j;
Jose Marinho09b1db82019-08-08 09:16:59 +0100830
Andrew Walbranca808b12020-05-15 17:22:28 +0100831 for (j = 0; j < fragment_constituent_counts[j]; ++j) {
832 size_t size = fragments[i][j].page_count * PAGE_SIZE;
833 paddr_t begin =
834 pa_from_ipa(ipa_init(fragments[i][j].address));
835 paddr_t end = pa_add(begin, size);
836
837 if (!clear_memory(begin, end, &local_page_pool)) {
838 /*
839 * api_clear_memory will defrag on failure, so
840 * no need to do it here.
841 */
842 goto out;
843 }
Jose Marinho09b1db82019-08-08 09:16:59 +0100844 }
845 }
846
847 /*
848 * Need to defrag after clearing, as it may have added extra mappings to
849 * the stage 1 page table.
850 */
851 stage1_locked = mm_lock_stage1();
852 mm_defrag(stage1_locked, &local_page_pool);
853 mm_unlock_stage1(&stage1_locked);
854
855 ret = true;
856
857out:
858 mpool_fini(&local_page_pool);
859 return ret;
860}
861
862/**
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000863 * Validates and prepares memory to be sent from the calling VM to another.
Jose Marinho09b1db82019-08-08 09:16:59 +0100864 *
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000865 * This function requires the calling context to hold the <from> VM lock.
Jose Marinho09b1db82019-08-08 09:16:59 +0100866 *
867 * Returns:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000868 * In case of error, one of the following values is returned:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100869 * 1) FFA_INVALID_PARAMETERS - The endpoint provided parameters were
Jose Marinho09b1db82019-08-08 09:16:59 +0100870 * erroneous;
Andrew Walbranf07f04d2020-05-01 18:09:00 +0100871 * 2) FFA_NO_MEMORY - Hafnium did not have sufficient memory to complete the
872 * request.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100873 * 3) FFA_DENIED - The sender doesn't have sufficient access to send the
Andrew Walbrana65a1322020-04-06 19:32:32 +0100874 * memory with the given permissions.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100875 * Success is indicated by FFA_SUCCESS.
Jose Marinho09b1db82019-08-08 09:16:59 +0100876 */
Andrew Walbran996d1d12020-05-27 14:08:43 +0100877static struct ffa_value ffa_send_check_update(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000878 struct vm_locked from_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +0100879 struct ffa_memory_region_constituent **fragments,
880 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
881 uint32_t share_func, ffa_memory_access_permissions_t permissions,
Andrew Walbran37c574e2020-06-03 11:45:46 +0100882 struct mpool *page_pool, bool clear, uint32_t *orig_from_mode_ret)
Jose Marinho09b1db82019-08-08 09:16:59 +0100883{
Andrew Walbranca808b12020-05-15 17:22:28 +0100884 uint32_t i;
Jose Marinho09b1db82019-08-08 09:16:59 +0100885 uint32_t orig_from_mode;
886 uint32_t from_mode;
Jose Marinho09b1db82019-08-08 09:16:59 +0100887 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100888 struct ffa_value ret;
Jose Marinho09b1db82019-08-08 09:16:59 +0100889
890 /*
Andrew Walbrana65a1322020-04-06 19:32:32 +0100891 * Make sure constituents are properly aligned to a 64-bit boundary. If
892 * not we would get alignment faults trying to read (64-bit) values.
Jose Marinho09b1db82019-08-08 09:16:59 +0100893 */
Andrew Walbranca808b12020-05-15 17:22:28 +0100894 for (i = 0; i < fragment_count; ++i) {
895 if (!is_aligned(fragments[i], 8)) {
896 dlog_verbose("Constituents not aligned.\n");
897 return ffa_error(FFA_INVALID_PARAMETERS);
898 }
Jose Marinho09b1db82019-08-08 09:16:59 +0100899 }
900
901 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000902 * Check if the state transition is lawful for the sender, ensure that
903 * all constituents of a memory region being shared are at the same
904 * state.
Jose Marinho09b1db82019-08-08 09:16:59 +0100905 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100906 ret = ffa_send_check_transition(from_locked, share_func, permissions,
Andrew Walbranca808b12020-05-15 17:22:28 +0100907 &orig_from_mode, fragments,
908 fragment_constituent_counts,
909 fragment_count, &from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100910 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100911 dlog_verbose("Invalid transition for send.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100912 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +0100913 }
914
Andrew Walbran37c574e2020-06-03 11:45:46 +0100915 if (orig_from_mode_ret != NULL) {
916 *orig_from_mode_ret = orig_from_mode;
917 }
918
Jose Marinho09b1db82019-08-08 09:16:59 +0100919 /*
920 * Create a local pool so any freed memory can't be used by another
921 * thread. This is to ensure the original mapping can be restored if the
922 * clear fails.
923 */
Andrew Walbran475c1452020-02-07 13:22:22 +0000924 mpool_init_with_fallback(&local_page_pool, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +0100925
926 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000927 * First reserve all required memory for the new page table entries
928 * without committing, to make sure the entire operation will succeed
929 * without exhausting the page pool.
Jose Marinho09b1db82019-08-08 09:16:59 +0100930 */
Andrew Walbranca808b12020-05-15 17:22:28 +0100931 if (!ffa_region_group_identity_map(
932 from_locked, fragments, fragment_constituent_counts,
933 fragment_count, from_mode, page_pool, false)) {
Jose Marinho09b1db82019-08-08 09:16:59 +0100934 /* TODO: partial defrag of failed range. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100935 ret = ffa_error(FFA_NO_MEMORY);
Jose Marinho09b1db82019-08-08 09:16:59 +0100936 goto out;
937 }
938
939 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000940 * Update the mapping for the sender. This won't allocate because the
941 * transaction was already prepared above, but may free pages in the
942 * case that a whole block is being unmapped that was previously
943 * partially mapped.
Jose Marinho09b1db82019-08-08 09:16:59 +0100944 */
Andrew Walbranca808b12020-05-15 17:22:28 +0100945 CHECK(ffa_region_group_identity_map(
946 from_locked, fragments, fragment_constituent_counts,
947 fragment_count, from_mode, &local_page_pool, true));
Jose Marinho09b1db82019-08-08 09:16:59 +0100948
949 /* Clear the memory so no VM or device can see the previous contents. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100950 if (clear && !ffa_clear_memory_constituents(
Andrew Walbranca808b12020-05-15 17:22:28 +0100951 fragments, fragment_constituent_counts,
952 fragment_count, page_pool)) {
Jose Marinho09b1db82019-08-08 09:16:59 +0100953 /*
954 * On failure, roll back by returning memory to the sender. This
955 * may allocate pages which were previously freed into
956 * `local_page_pool` by the call above, but will never allocate
957 * more pages than that so can never fail.
958 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100959 CHECK(ffa_region_group_identity_map(
Andrew Walbranca808b12020-05-15 17:22:28 +0100960 from_locked, fragments, fragment_constituent_counts,
961 fragment_count, orig_from_mode, &local_page_pool,
962 true));
Jose Marinho09b1db82019-08-08 09:16:59 +0100963
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100964 ret = ffa_error(FFA_NO_MEMORY);
Jose Marinho09b1db82019-08-08 09:16:59 +0100965 goto out;
966 }
967
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100968 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000969
970out:
971 mpool_fini(&local_page_pool);
972
973 /*
974 * Tidy up the page table by reclaiming failed mappings (if there was an
975 * error) or merging entries into blocks where possible (on success).
976 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -0700977 vm_ptable_defrag(from_locked, page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000978
979 return ret;
980}
981
982/**
983 * Validates and maps memory shared from one VM to another.
984 *
985 * This function requires the calling context to hold the <to> lock.
986 *
987 * Returns:
988 * In case of error, one of the following values is returned:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100989 * 1) FFA_INVALID_PARAMETERS - The endpoint provided parameters were
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000990 * erroneous;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100991 * 2) FFA_NO_MEMORY - Hafnium did not have sufficient memory to complete
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000992 * the request.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100993 * Success is indicated by FFA_SUCCESS.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000994 */
Andrew Walbran996d1d12020-05-27 14:08:43 +0100995static struct ffa_value ffa_retrieve_check_update(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000996 struct vm_locked to_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +0100997 struct ffa_memory_region_constituent **fragments,
998 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
999 uint32_t memory_to_attributes, uint32_t share_func, bool clear,
1000 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001001{
Andrew Walbranca808b12020-05-15 17:22:28 +01001002 uint32_t i;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001003 uint32_t to_mode;
1004 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001005 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001006
1007 /*
Andrew Walbranca808b12020-05-15 17:22:28 +01001008 * Make sure constituents are properly aligned to a 64-bit boundary. If
1009 * not we would get alignment faults trying to read (64-bit) values.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001010 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001011 for (i = 0; i < fragment_count; ++i) {
1012 if (!is_aligned(fragments[i], 8)) {
1013 return ffa_error(FFA_INVALID_PARAMETERS);
1014 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001015 }
1016
1017 /*
1018 * Check if the state transition is lawful for the recipient, and ensure
1019 * that all constituents of the memory region being retrieved are at the
1020 * same state.
1021 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001022 ret = ffa_retrieve_check_transition(
1023 to_locked, share_func, fragments, fragment_constituent_counts,
1024 fragment_count, memory_to_attributes, &to_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001025 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001026 dlog_verbose("Invalid transition for retrieve.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +01001027 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001028 }
1029
1030 /*
1031 * Create a local pool so any freed memory can't be used by another
1032 * thread. This is to ensure the original mapping can be restored if the
1033 * clear fails.
1034 */
1035 mpool_init_with_fallback(&local_page_pool, page_pool);
1036
1037 /*
1038 * First reserve all required memory for the new page table entries in
1039 * the recipient page tables without committing, to make sure the entire
1040 * operation will succeed without exhausting the page pool.
1041 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001042 if (!ffa_region_group_identity_map(
1043 to_locked, fragments, fragment_constituent_counts,
1044 fragment_count, to_mode, page_pool, false)) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001045 /* TODO: partial defrag of failed range. */
1046 dlog_verbose(
1047 "Insufficient memory to update recipient page "
1048 "table.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001049 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001050 goto out;
1051 }
1052
1053 /* Clear the memory so no VM or device can see the previous contents. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001054 if (clear && !ffa_clear_memory_constituents(
Andrew Walbranca808b12020-05-15 17:22:28 +01001055 fragments, fragment_constituent_counts,
1056 fragment_count, page_pool)) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001057 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001058 goto out;
1059 }
1060
Jose Marinho09b1db82019-08-08 09:16:59 +01001061 /*
1062 * Complete the transfer by mapping the memory into the recipient. This
1063 * won't allocate because the transaction was already prepared above, so
1064 * it doesn't need to use the `local_page_pool`.
1065 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001066 CHECK(ffa_region_group_identity_map(
1067 to_locked, fragments, fragment_constituent_counts,
1068 fragment_count, to_mode, page_pool, true));
Jose Marinho09b1db82019-08-08 09:16:59 +01001069
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001070 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Jose Marinho09b1db82019-08-08 09:16:59 +01001071
1072out:
1073 mpool_fini(&local_page_pool);
1074
1075 /*
Andrew Walbranf07f04d2020-05-01 18:09:00 +01001076 * Tidy up the page table by reclaiming failed mappings (if there was an
1077 * error) or merging entries into blocks where possible (on success).
Jose Marinho09b1db82019-08-08 09:16:59 +01001078 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001079 vm_ptable_defrag(to_locked, page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001080
1081 return ret;
1082}
1083
Andrew Walbran290b0c92020-02-03 16:37:14 +00001084/**
1085 * Reclaims the given memory from the TEE. To do this space is first reserved in
1086 * the <to> VM's page table, then the reclaim request is sent on to the TEE,
1087 * then (if that is successful) the memory is mapped back into the <to> VM's
1088 * page table.
1089 *
1090 * This function requires the calling context to hold the <to> lock.
1091 *
1092 * Returns:
1093 * In case of error, one of the following values is returned:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001094 * 1) FFA_INVALID_PARAMETERS - The endpoint provided parameters were
Andrew Walbran290b0c92020-02-03 16:37:14 +00001095 * erroneous;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001096 * 2) FFA_NO_MEMORY - Hafnium did not have sufficient memory to complete
Andrew Walbran290b0c92020-02-03 16:37:14 +00001097 * the request.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001098 * Success is indicated by FFA_SUCCESS.
Andrew Walbran290b0c92020-02-03 16:37:14 +00001099 */
Andrew Walbran996d1d12020-05-27 14:08:43 +01001100static struct ffa_value ffa_tee_reclaim_check_update(
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001101 struct vm_locked to_locked, ffa_memory_handle_t handle,
1102 struct ffa_memory_region_constituent *constituents,
Andrew Walbran290b0c92020-02-03 16:37:14 +00001103 uint32_t constituent_count, uint32_t memory_to_attributes, bool clear,
1104 struct mpool *page_pool)
1105{
Andrew Walbran290b0c92020-02-03 16:37:14 +00001106 uint32_t to_mode;
1107 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001108 struct ffa_value ret;
1109 ffa_memory_region_flags_t tee_flags;
Andrew Walbran290b0c92020-02-03 16:37:14 +00001110
1111 /*
Andrew Walbranca808b12020-05-15 17:22:28 +01001112 * Make sure constituents are properly aligned to a 64-bit boundary. If
1113 * not we would get alignment faults trying to read (64-bit) values.
Andrew Walbran290b0c92020-02-03 16:37:14 +00001114 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001115 if (!is_aligned(constituents, 8)) {
Andrew Walbran290b0c92020-02-03 16:37:14 +00001116 dlog_verbose("Constituents not aligned.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001117 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran290b0c92020-02-03 16:37:14 +00001118 }
1119
1120 /*
1121 * Check if the state transition is lawful for the recipient, and ensure
1122 * that all constituents of the memory region being retrieved are at the
1123 * same state.
1124 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001125 ret = ffa_retrieve_check_transition(to_locked, FFA_MEM_RECLAIM_32,
Andrew Walbranca808b12020-05-15 17:22:28 +01001126 &constituents, &constituent_count,
1127 1, memory_to_attributes, &to_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001128 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbran290b0c92020-02-03 16:37:14 +00001129 dlog_verbose("Invalid transition.\n");
1130 return ret;
1131 }
1132
1133 /*
1134 * Create a local pool so any freed memory can't be used by another
1135 * thread. This is to ensure the original mapping can be restored if the
1136 * clear fails.
1137 */
1138 mpool_init_with_fallback(&local_page_pool, page_pool);
1139
1140 /*
1141 * First reserve all required memory for the new page table entries in
1142 * the recipient page tables without committing, to make sure the entire
1143 * operation will succeed without exhausting the page pool.
1144 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001145 if (!ffa_region_group_identity_map(to_locked, &constituents,
1146 &constituent_count, 1, to_mode,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001147 page_pool, false)) {
Andrew Walbran290b0c92020-02-03 16:37:14 +00001148 /* TODO: partial defrag of failed range. */
1149 dlog_verbose(
1150 "Insufficient memory to update recipient page "
1151 "table.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001152 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran290b0c92020-02-03 16:37:14 +00001153 goto out;
1154 }
1155
1156 /*
1157 * Forward the request to the TEE and see what happens.
1158 */
1159 tee_flags = 0;
1160 if (clear) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001161 tee_flags |= FFA_MEMORY_REGION_FLAG_CLEAR;
Andrew Walbran290b0c92020-02-03 16:37:14 +00001162 }
Olivier Deprez112d2b52020-09-30 07:39:23 +02001163 ret = arch_other_world_call(
1164 (struct ffa_value){.func = FFA_MEM_RECLAIM_32,
1165 .arg1 = (uint32_t)handle,
1166 .arg2 = (uint32_t)(handle >> 32),
1167 .arg3 = tee_flags});
Andrew Walbran290b0c92020-02-03 16:37:14 +00001168
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001169 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbran290b0c92020-02-03 16:37:14 +00001170 dlog_verbose(
Andrew Walbranca808b12020-05-15 17:22:28 +01001171 "Got %#x (%d) from TEE in response to FFA_MEM_RECLAIM, "
1172 "expected FFA_SUCCESS.\n",
Andrew Walbran290b0c92020-02-03 16:37:14 +00001173 ret.func, ret.arg2);
1174 goto out;
1175 }
1176
1177 /*
1178 * The TEE was happy with it, so complete the reclaim by mapping the
1179 * memory into the recipient. This won't allocate because the
1180 * transaction was already prepared above, so it doesn't need to use the
1181 * `local_page_pool`.
1182 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001183 CHECK(ffa_region_group_identity_map(to_locked, &constituents,
1184 &constituent_count, 1, to_mode,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001185 page_pool, true));
Andrew Walbran290b0c92020-02-03 16:37:14 +00001186
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001187 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran290b0c92020-02-03 16:37:14 +00001188
1189out:
1190 mpool_fini(&local_page_pool);
1191
1192 /*
Andrew Walbranf07f04d2020-05-01 18:09:00 +01001193 * Tidy up the page table by reclaiming failed mappings (if there was an
1194 * error) or merging entries into blocks where possible (on success).
Andrew Walbran290b0c92020-02-03 16:37:14 +00001195 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001196 vm_ptable_defrag(to_locked, page_pool);
Andrew Walbran290b0c92020-02-03 16:37:14 +00001197
1198 return ret;
1199}
1200
Andrew Walbran996d1d12020-05-27 14:08:43 +01001201static struct ffa_value ffa_relinquish_check_update(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001202 struct vm_locked from_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01001203 struct ffa_memory_region_constituent **fragments,
1204 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
1205 struct mpool *page_pool, bool clear)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001206{
1207 uint32_t orig_from_mode;
1208 uint32_t from_mode;
1209 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001210 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001211
Andrew Walbranca808b12020-05-15 17:22:28 +01001212 ret = ffa_relinquish_check_transition(
1213 from_locked, &orig_from_mode, fragments,
1214 fragment_constituent_counts, fragment_count, &from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001215 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001216 dlog_verbose("Invalid transition for relinquish.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +01001217 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001218 }
1219
1220 /*
1221 * Create a local pool so any freed memory can't be used by another
1222 * thread. This is to ensure the original mapping can be restored if the
1223 * clear fails.
1224 */
1225 mpool_init_with_fallback(&local_page_pool, page_pool);
1226
1227 /*
1228 * First reserve all required memory for the new page table entries
1229 * without committing, to make sure the entire operation will succeed
1230 * without exhausting the page pool.
1231 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001232 if (!ffa_region_group_identity_map(
1233 from_locked, fragments, fragment_constituent_counts,
1234 fragment_count, from_mode, page_pool, false)) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001235 /* TODO: partial defrag of failed range. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001236 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001237 goto out;
1238 }
1239
1240 /*
1241 * Update the mapping for the sender. This won't allocate because the
1242 * transaction was already prepared above, but may free pages in the
1243 * case that a whole block is being unmapped that was previously
1244 * partially mapped.
1245 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001246 CHECK(ffa_region_group_identity_map(
1247 from_locked, fragments, fragment_constituent_counts,
1248 fragment_count, from_mode, &local_page_pool, true));
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001249
1250 /* Clear the memory so no VM or device can see the previous contents. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001251 if (clear && !ffa_clear_memory_constituents(
Andrew Walbranca808b12020-05-15 17:22:28 +01001252 fragments, fragment_constituent_counts,
1253 fragment_count, page_pool)) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001254 /*
1255 * On failure, roll back by returning memory to the sender. This
1256 * may allocate pages which were previously freed into
1257 * `local_page_pool` by the call above, but will never allocate
1258 * more pages than that so can never fail.
1259 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001260 CHECK(ffa_region_group_identity_map(
Andrew Walbranca808b12020-05-15 17:22:28 +01001261 from_locked, fragments, fragment_constituent_counts,
1262 fragment_count, orig_from_mode, &local_page_pool,
1263 true));
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001264
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001265 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001266 goto out;
1267 }
1268
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001269 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001270
1271out:
1272 mpool_fini(&local_page_pool);
1273
1274 /*
1275 * Tidy up the page table by reclaiming failed mappings (if there was an
1276 * error) or merging entries into blocks where possible (on success).
1277 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001278 vm_ptable_defrag(from_locked, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +01001279
1280 return ret;
1281}
1282
1283/**
Andrew Walbranca808b12020-05-15 17:22:28 +01001284 * Complete a memory sending operation by checking that it is valid, updating
1285 * the sender page table, and then either marking the share state as having
1286 * completed sending (on success) or freeing it (on failure).
1287 *
1288 * Returns FFA_SUCCESS with the handle encoded, or the relevant FFA_ERROR.
1289 */
1290static struct ffa_value ffa_memory_send_complete(
1291 struct vm_locked from_locked, struct share_states_locked share_states,
Andrew Walbran37c574e2020-06-03 11:45:46 +01001292 struct ffa_memory_share_state *share_state, struct mpool *page_pool,
1293 uint32_t *orig_from_mode_ret)
Andrew Walbranca808b12020-05-15 17:22:28 +01001294{
1295 struct ffa_memory_region *memory_region = share_state->memory_region;
1296 struct ffa_value ret;
1297
1298 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +00001299 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +01001300
1301 /* Check that state is valid in sender page table and update. */
1302 ret = ffa_send_check_update(
1303 from_locked, share_state->fragments,
1304 share_state->fragment_constituent_counts,
1305 share_state->fragment_count, share_state->share_func,
1306 memory_region->receivers[0].receiver_permissions.permissions,
Andrew Walbran37c574e2020-06-03 11:45:46 +01001307 page_pool, memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR,
1308 orig_from_mode_ret);
Andrew Walbranca808b12020-05-15 17:22:28 +01001309 if (ret.func != FFA_SUCCESS_32) {
1310 /*
1311 * Free share state, it failed to send so it can't be retrieved.
1312 */
1313 dlog_verbose("Complete failed, freeing share state.\n");
1314 share_state_free(share_states, share_state, page_pool);
1315 return ret;
1316 }
1317
1318 share_state->sending_complete = true;
1319 dlog_verbose("Marked sending complete.\n");
1320
J-Alvesee68c542020-10-29 17:48:20 +00001321 return ffa_mem_success(share_state->memory_region->handle);
Andrew Walbranca808b12020-05-15 17:22:28 +01001322}
1323
1324/**
Andrew Walbrana65a1322020-04-06 19:32:32 +01001325 * Check that the given `memory_region` represents a valid memory send request
1326 * of the given `share_func` type, return the clear flag and permissions via the
1327 * respective output parameters, and update the permissions if necessary.
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001328 *
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001329 * Returns FFA_SUCCESS if the request was valid, or the relevant FFA_ERROR if
Andrew Walbrana65a1322020-04-06 19:32:32 +01001330 * not.
1331 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001332static struct ffa_value ffa_memory_send_validate(
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001333 struct vm_locked from_locked, struct ffa_memory_region *memory_region,
1334 uint32_t memory_share_length, uint32_t fragment_length,
1335 uint32_t share_func, ffa_memory_access_permissions_t *permissions)
Andrew Walbrana65a1322020-04-06 19:32:32 +01001336{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001337 struct ffa_composite_memory_region *composite;
Andrew Walbran130a8ae2020-05-15 16:27:15 +01001338 uint32_t receivers_length;
Andrew Walbran352aa3d2020-05-01 17:51:33 +01001339 uint32_t constituents_offset;
Andrew Walbran130a8ae2020-05-15 16:27:15 +01001340 uint32_t constituents_length;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001341 enum ffa_data_access data_access;
1342 enum ffa_instruction_access instruction_access;
Andrew Walbrana65a1322020-04-06 19:32:32 +01001343
Daniel Boulbya2f8c662021-11-26 17:52:53 +00001344 assert(permissions != NULL);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001345
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001346 /*
1347 * This should already be checked by the caller, just making the
1348 * assumption clear here.
1349 */
Daniel Boulbya2f8c662021-11-26 17:52:53 +00001350 assert(memory_region->receiver_count == 1);
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001351
Andrew Walbrana65a1322020-04-06 19:32:32 +01001352 /* The sender must match the message sender. */
1353 if (memory_region->sender != from_locked.vm->id) {
1354 dlog_verbose("Invalid sender %d.\n", memory_region->sender);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001355 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001356 }
1357
Andrew Walbrana65a1322020-04-06 19:32:32 +01001358 /*
1359 * Ensure that the composite header is within the memory bounds and
1360 * doesn't overlap the first part of the message.
1361 */
Andrew Walbran130a8ae2020-05-15 16:27:15 +01001362 receivers_length = sizeof(struct ffa_memory_access) *
1363 memory_region->receiver_count;
Andrew Walbran352aa3d2020-05-01 17:51:33 +01001364 constituents_offset =
1365 ffa_composite_constituent_offset(memory_region, 0);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001366 if (memory_region->receivers[0].composite_memory_region_offset <
Andrew Walbran130a8ae2020-05-15 16:27:15 +01001367 sizeof(struct ffa_memory_region) + receivers_length ||
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001368 constituents_offset > fragment_length) {
Andrew Walbrana65a1322020-04-06 19:32:32 +01001369 dlog_verbose(
Andrew Walbran352aa3d2020-05-01 17:51:33 +01001370 "Invalid composite memory region descriptor offset "
1371 "%d.\n",
1372 memory_region->receivers[0]
1373 .composite_memory_region_offset);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001374 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001375 }
1376
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001377 composite = ffa_memory_region_get_composite(memory_region, 0);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001378
1379 /*
Andrew Walbranf07f04d2020-05-01 18:09:00 +01001380 * Ensure the number of constituents are within the memory bounds.
Andrew Walbrana65a1322020-04-06 19:32:32 +01001381 */
Andrew Walbran130a8ae2020-05-15 16:27:15 +01001382 constituents_length = sizeof(struct ffa_memory_region_constituent) *
1383 composite->constituent_count;
Andrew Walbran352aa3d2020-05-01 17:51:33 +01001384 if (memory_share_length != constituents_offset + constituents_length) {
1385 dlog_verbose("Invalid length %d or composite offset %d.\n",
Andrew Walbran130a8ae2020-05-15 16:27:15 +01001386 memory_share_length,
Andrew Walbrana65a1322020-04-06 19:32:32 +01001387 memory_region->receivers[0]
1388 .composite_memory_region_offset);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001389 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001390 }
Andrew Walbranca808b12020-05-15 17:22:28 +01001391 if (fragment_length < memory_share_length &&
1392 fragment_length < HF_MAILBOX_SIZE) {
1393 dlog_warning(
1394 "Initial fragment length %d smaller than mailbox "
1395 "size.\n",
1396 fragment_length);
1397 }
Andrew Walbrana65a1322020-04-06 19:32:32 +01001398
Andrew Walbrana65a1322020-04-06 19:32:32 +01001399 /*
1400 * Clear is not allowed for memory sharing, as the sender still has
1401 * access to the memory.
1402 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001403 if ((memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR) &&
1404 share_func == FFA_MEM_SHARE_32) {
Andrew Walbrana65a1322020-04-06 19:32:32 +01001405 dlog_verbose("Memory can't be cleared while being shared.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001406 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001407 }
1408
1409 /* No other flags are allowed/supported here. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001410 if (memory_region->flags & ~FFA_MEMORY_REGION_FLAG_CLEAR) {
Andrew Walbrana65a1322020-04-06 19:32:32 +01001411 dlog_verbose("Invalid flags %#x.\n", memory_region->flags);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001412 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001413 }
1414
1415 /* Check that the permissions are valid. */
1416 *permissions =
1417 memory_region->receivers[0].receiver_permissions.permissions;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001418 data_access = ffa_get_data_access_attr(*permissions);
1419 instruction_access = ffa_get_instruction_access_attr(*permissions);
1420 if (data_access == FFA_DATA_ACCESS_RESERVED ||
1421 instruction_access == FFA_INSTRUCTION_ACCESS_RESERVED) {
Andrew Walbrana65a1322020-04-06 19:32:32 +01001422 dlog_verbose("Reserved value for receiver permissions %#x.\n",
1423 *permissions);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001424 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001425 }
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001426 if (instruction_access != FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
Andrew Walbrana65a1322020-04-06 19:32:32 +01001427 dlog_verbose(
1428 "Invalid instruction access permissions %#x for "
1429 "sending memory.\n",
1430 *permissions);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001431 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001432 }
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001433 if (share_func == FFA_MEM_SHARE_32) {
1434 if (data_access == FFA_DATA_ACCESS_NOT_SPECIFIED) {
Andrew Walbrana65a1322020-04-06 19:32:32 +01001435 dlog_verbose(
1436 "Invalid data access permissions %#x for "
1437 "sharing memory.\n",
1438 *permissions);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001439 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001440 }
1441 /*
Andrew Walbrandd8248f2020-06-22 13:39:30 +01001442 * According to section 5.11.3 of the FF-A 1.0 spec NX is
1443 * required for share operations (but must not be specified by
1444 * the sender) so set it in the copy that we store, ready to be
Andrew Walbrana65a1322020-04-06 19:32:32 +01001445 * returned to the retriever.
1446 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001447 ffa_set_instruction_access_attr(permissions,
1448 FFA_INSTRUCTION_ACCESS_NX);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001449 memory_region->receivers[0].receiver_permissions.permissions =
1450 *permissions;
1451 }
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001452 if (share_func == FFA_MEM_LEND_32 &&
1453 data_access == FFA_DATA_ACCESS_NOT_SPECIFIED) {
Andrew Walbrana65a1322020-04-06 19:32:32 +01001454 dlog_verbose(
1455 "Invalid data access permissions %#x for lending "
1456 "memory.\n",
1457 *permissions);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001458 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001459 }
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001460 if (share_func == FFA_MEM_DONATE_32 &&
1461 data_access != FFA_DATA_ACCESS_NOT_SPECIFIED) {
Andrew Walbrana65a1322020-04-06 19:32:32 +01001462 dlog_verbose(
1463 "Invalid data access permissions %#x for donating "
1464 "memory.\n",
1465 *permissions);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001466 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001467 }
1468
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001469 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbrana65a1322020-04-06 19:32:32 +01001470}
1471
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001472/** Forwards a memory send message on to the TEE. */
1473static struct ffa_value memory_send_tee_forward(
1474 struct vm_locked tee_locked, ffa_vm_id_t sender_vm_id,
1475 uint32_t share_func, struct ffa_memory_region *memory_region,
1476 uint32_t memory_share_length, uint32_t fragment_length)
1477{
1478 struct ffa_value ret;
1479
1480 memcpy_s(tee_locked.vm->mailbox.recv, FFA_MSG_PAYLOAD_MAX,
1481 memory_region, fragment_length);
1482 tee_locked.vm->mailbox.recv_size = fragment_length;
1483 tee_locked.vm->mailbox.recv_sender = sender_vm_id;
1484 tee_locked.vm->mailbox.recv_func = share_func;
1485 tee_locked.vm->mailbox.state = MAILBOX_STATE_RECEIVED;
Olivier Deprez112d2b52020-09-30 07:39:23 +02001486 ret = arch_other_world_call(
1487 (struct ffa_value){.func = share_func,
1488 .arg1 = memory_share_length,
1489 .arg2 = fragment_length});
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001490 /*
1491 * After the call to the TEE completes it must have finished reading its
1492 * RX buffer, so it is ready for another message.
1493 */
1494 tee_locked.vm->mailbox.state = MAILBOX_STATE_EMPTY;
1495
1496 return ret;
1497}
1498
Andrew Walbrana65a1322020-04-06 19:32:32 +01001499/**
Andrew Walbranca808b12020-05-15 17:22:28 +01001500 * Gets the share state for continuing an operation to donate, lend or share
1501 * memory, and checks that it is a valid request.
1502 *
1503 * Returns FFA_SUCCESS if the request was valid, or the relevant FFA_ERROR if
1504 * not.
1505 */
1506static struct ffa_value ffa_memory_send_continue_validate(
1507 struct share_states_locked share_states, ffa_memory_handle_t handle,
1508 struct ffa_memory_share_state **share_state_ret, ffa_vm_id_t from_vm_id,
1509 struct mpool *page_pool)
1510{
1511 struct ffa_memory_share_state *share_state;
1512 struct ffa_memory_region *memory_region;
1513
Daniel Boulbya2f8c662021-11-26 17:52:53 +00001514 assert(share_state_ret != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +01001515
1516 /*
1517 * Look up the share state by handle and make sure that the VM ID
1518 * matches.
1519 */
1520 if (!get_share_state(share_states, handle, &share_state)) {
1521 dlog_verbose(
1522 "Invalid handle %#x for memory send continuation.\n",
1523 handle);
1524 return ffa_error(FFA_INVALID_PARAMETERS);
1525 }
1526 memory_region = share_state->memory_region;
1527
1528 if (memory_region->sender != from_vm_id) {
1529 dlog_verbose("Invalid sender %d.\n", memory_region->sender);
1530 return ffa_error(FFA_INVALID_PARAMETERS);
1531 }
1532
1533 if (share_state->sending_complete) {
1534 dlog_verbose(
1535 "Sending of memory handle %#x is already complete.\n",
1536 handle);
1537 return ffa_error(FFA_INVALID_PARAMETERS);
1538 }
1539
1540 if (share_state->fragment_count == MAX_FRAGMENTS) {
1541 /*
1542 * Log a warning as this is a sign that MAX_FRAGMENTS should
1543 * probably be increased.
1544 */
1545 dlog_warning(
1546 "Too many fragments for memory share with handle %#x; "
1547 "only %d supported.\n",
1548 handle, MAX_FRAGMENTS);
1549 /* Free share state, as it's not possible to complete it. */
1550 share_state_free(share_states, share_state, page_pool);
1551 return ffa_error(FFA_NO_MEMORY);
1552 }
1553
1554 *share_state_ret = share_state;
1555
1556 return (struct ffa_value){.func = FFA_SUCCESS_32};
1557}
1558
1559/**
1560 * Forwards a memory send continuation message on to the TEE.
1561 */
1562static struct ffa_value memory_send_continue_tee_forward(
1563 struct vm_locked tee_locked, ffa_vm_id_t sender_vm_id, void *fragment,
1564 uint32_t fragment_length, ffa_memory_handle_t handle)
1565{
1566 struct ffa_value ret;
1567
1568 memcpy_s(tee_locked.vm->mailbox.recv, FFA_MSG_PAYLOAD_MAX, fragment,
1569 fragment_length);
1570 tee_locked.vm->mailbox.recv_size = fragment_length;
1571 tee_locked.vm->mailbox.recv_sender = sender_vm_id;
1572 tee_locked.vm->mailbox.recv_func = FFA_MEM_FRAG_TX_32;
1573 tee_locked.vm->mailbox.state = MAILBOX_STATE_RECEIVED;
Olivier Deprez112d2b52020-09-30 07:39:23 +02001574 ret = arch_other_world_call(
Andrew Walbranca808b12020-05-15 17:22:28 +01001575 (struct ffa_value){.func = FFA_MEM_FRAG_TX_32,
1576 .arg1 = (uint32_t)handle,
1577 .arg2 = (uint32_t)(handle >> 32),
1578 .arg3 = fragment_length,
1579 .arg4 = (uint64_t)sender_vm_id << 16});
1580 /*
1581 * After the call to the TEE completes it must have finished reading its
1582 * RX buffer, so it is ready for another message.
1583 */
1584 tee_locked.vm->mailbox.state = MAILBOX_STATE_EMPTY;
1585
1586 return ret;
1587}
1588
1589/**
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001590 * Validates a call to donate, lend or share memory to a non-TEE VM and then
1591 * updates the stage-2 page tables. Specifically, check if the message length
1592 * and number of memory region constituents match, and if the transition is
1593 * valid for the type of memory sending operation.
Andrew Walbran475c1452020-02-07 13:22:22 +00001594 *
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001595 * Assumes that the caller has already found and locked the sender VM and copied
1596 * the memory region descriptor from the sender's TX buffer to a freshly
1597 * allocated page from Hafnium's internal pool. The caller must have also
1598 * validated that the receiver VM ID is valid.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001599 *
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001600 * This function takes ownership of the `memory_region` passed in and will free
1601 * it when necessary; it must not be freed by the caller.
Jose Marinho09b1db82019-08-08 09:16:59 +01001602 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001603struct ffa_value ffa_memory_send(struct vm_locked from_locked,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001604 struct ffa_memory_region *memory_region,
Andrew Walbran130a8ae2020-05-15 16:27:15 +01001605 uint32_t memory_share_length,
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001606 uint32_t fragment_length, uint32_t share_func,
1607 struct mpool *page_pool)
Jose Marinho09b1db82019-08-08 09:16:59 +01001608{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001609 ffa_memory_access_permissions_t permissions;
1610 struct ffa_value ret;
Andrew Walbranca808b12020-05-15 17:22:28 +01001611 struct share_states_locked share_states;
1612 struct ffa_memory_share_state *share_state;
Jose Marinho09b1db82019-08-08 09:16:59 +01001613
1614 /*
Andrew Walbrana65a1322020-04-06 19:32:32 +01001615 * If there is an error validating the `memory_region` then we need to
1616 * free it because we own it but we won't be storing it in a share state
1617 * after all.
Jose Marinho09b1db82019-08-08 09:16:59 +01001618 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001619 ret = ffa_memory_send_validate(from_locked, memory_region,
1620 memory_share_length, fragment_length,
1621 share_func, &permissions);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001622 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001623 mpool_free(page_pool, memory_region);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001624 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01001625 }
1626
Andrew Walbrana65a1322020-04-06 19:32:32 +01001627 /* Set flag for share function, ready to be retrieved later. */
1628 switch (share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001629 case FFA_MEM_SHARE_32:
Andrew Walbrana65a1322020-04-06 19:32:32 +01001630 memory_region->flags |=
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001631 FFA_MEMORY_REGION_TRANSACTION_TYPE_SHARE;
Andrew Walbrana65a1322020-04-06 19:32:32 +01001632 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001633 case FFA_MEM_LEND_32:
1634 memory_region->flags |= FFA_MEMORY_REGION_TRANSACTION_TYPE_LEND;
Andrew Walbrana65a1322020-04-06 19:32:32 +01001635 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001636 case FFA_MEM_DONATE_32:
Andrew Walbrana65a1322020-04-06 19:32:32 +01001637 memory_region->flags |=
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001638 FFA_MEMORY_REGION_TRANSACTION_TYPE_DONATE;
Andrew Walbrana65a1322020-04-06 19:32:32 +01001639 break;
Jose Marinho09b1db82019-08-08 09:16:59 +01001640 }
1641
Andrew Walbranca808b12020-05-15 17:22:28 +01001642 share_states = share_states_lock();
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001643 /*
1644 * Allocate a share state before updating the page table. Otherwise if
1645 * updating the page table succeeded but allocating the share state
1646 * failed then it would leave the memory in a state where nobody could
1647 * get it back.
1648 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001649 if (!allocate_share_state(share_states, share_func, memory_region,
1650 fragment_length, FFA_MEMORY_HANDLE_INVALID,
1651 &share_state)) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001652 dlog_verbose("Failed to allocate share state.\n");
1653 mpool_free(page_pool, memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +01001654 ret = ffa_error(FFA_NO_MEMORY);
1655 goto out;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001656 }
1657
Andrew Walbranca808b12020-05-15 17:22:28 +01001658 if (fragment_length == memory_share_length) {
1659 /* No more fragments to come, everything fit in one message. */
J-Alves2a0d2882020-10-29 14:49:50 +00001660 ret = ffa_memory_send_complete(
1661 from_locked, share_states, share_state, page_pool,
1662 &(share_state->sender_orig_mode));
Andrew Walbranca808b12020-05-15 17:22:28 +01001663 } else {
1664 ret = (struct ffa_value){
1665 .func = FFA_MEM_FRAG_RX_32,
J-Alvesee68c542020-10-29 17:48:20 +00001666 .arg1 = (uint32_t)memory_region->handle,
1667 .arg2 = (uint32_t)(memory_region->handle >> 32),
Andrew Walbranca808b12020-05-15 17:22:28 +01001668 .arg3 = fragment_length};
1669 }
1670
1671out:
1672 share_states_unlock(&share_states);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001673 dump_share_states();
Andrew Walbranca808b12020-05-15 17:22:28 +01001674 return ret;
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001675}
1676
1677/**
1678 * Validates a call to donate, lend or share memory to the TEE and then updates
1679 * the stage-2 page tables. Specifically, check if the message length and number
1680 * of memory region constituents match, and if the transition is valid for the
1681 * type of memory sending operation.
1682 *
1683 * Assumes that the caller has already found and locked the sender VM and the
1684 * TEE VM, and copied the memory region descriptor from the sender's TX buffer
1685 * to a freshly allocated page from Hafnium's internal pool. The caller must
1686 * have also validated that the receiver VM ID is valid.
1687 *
1688 * This function takes ownership of the `memory_region` passed in and will free
1689 * it when necessary; it must not be freed by the caller.
1690 */
1691struct ffa_value ffa_memory_tee_send(
1692 struct vm_locked from_locked, struct vm_locked to_locked,
1693 struct ffa_memory_region *memory_region, uint32_t memory_share_length,
1694 uint32_t fragment_length, uint32_t share_func, struct mpool *page_pool)
1695{
1696 ffa_memory_access_permissions_t permissions;
1697 struct ffa_value ret;
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001698
1699 /*
1700 * If there is an error validating the `memory_region` then we need to
1701 * free it because we own it but we won't be storing it in a share state
1702 * after all.
1703 */
1704 ret = ffa_memory_send_validate(from_locked, memory_region,
1705 memory_share_length, fragment_length,
1706 share_func, &permissions);
1707 if (ret.func != FFA_SUCCESS_32) {
1708 goto out;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001709 }
1710
Andrew Walbranca808b12020-05-15 17:22:28 +01001711 if (fragment_length == memory_share_length) {
1712 /* No more fragments to come, everything fit in one message. */
1713 struct ffa_composite_memory_region *composite =
1714 ffa_memory_region_get_composite(memory_region, 0);
1715 struct ffa_memory_region_constituent *constituents =
1716 composite->constituents;
Andrew Walbran37c574e2020-06-03 11:45:46 +01001717 struct mpool local_page_pool;
1718 uint32_t orig_from_mode;
1719
1720 /*
1721 * Use a local page pool so that we can roll back if necessary.
1722 */
1723 mpool_init_with_fallback(&local_page_pool, page_pool);
Andrew Walbranca808b12020-05-15 17:22:28 +01001724
1725 ret = ffa_send_check_update(
1726 from_locked, &constituents,
1727 &composite->constituent_count, 1, share_func,
Andrew Walbran37c574e2020-06-03 11:45:46 +01001728 permissions, &local_page_pool,
1729 memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR,
1730 &orig_from_mode);
Andrew Walbranca808b12020-05-15 17:22:28 +01001731 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbran37c574e2020-06-03 11:45:46 +01001732 mpool_fini(&local_page_pool);
Andrew Walbranca808b12020-05-15 17:22:28 +01001733 goto out;
1734 }
1735
1736 /* Forward memory send message on to TEE. */
1737 ret = memory_send_tee_forward(
1738 to_locked, from_locked.vm->id, share_func,
1739 memory_region, memory_share_length, fragment_length);
Andrew Walbran37c574e2020-06-03 11:45:46 +01001740
1741 if (ret.func != FFA_SUCCESS_32) {
1742 dlog_verbose(
1743 "TEE didn't successfully complete memory send "
1744 "operation; returned %#x (%d). Rolling back.\n",
1745 ret.func, ret.arg2);
1746
1747 /*
1748 * The TEE failed to complete the send operation, so
1749 * roll back the page table update for the VM. This
1750 * can't fail because it won't try to allocate more
1751 * memory than was freed into the `local_page_pool` by
1752 * `ffa_send_check_update` in the initial update.
1753 */
1754 CHECK(ffa_region_group_identity_map(
1755 from_locked, &constituents,
1756 &composite->constituent_count, 1,
1757 orig_from_mode, &local_page_pool, true));
1758 }
1759
1760 mpool_fini(&local_page_pool);
Andrew Walbranca808b12020-05-15 17:22:28 +01001761 } else {
1762 struct share_states_locked share_states = share_states_lock();
1763 ffa_memory_handle_t handle;
1764
1765 /*
1766 * We need to wait for the rest of the fragments before we can
1767 * check whether the transaction is valid and unmap the memory.
1768 * Call the TEE so it can do its initial validation and assign a
1769 * handle, and allocate a share state to keep what we have so
1770 * far.
1771 */
1772 ret = memory_send_tee_forward(
1773 to_locked, from_locked.vm->id, share_func,
1774 memory_region, memory_share_length, fragment_length);
1775 if (ret.func == FFA_ERROR_32) {
1776 goto out_unlock;
1777 } else if (ret.func != FFA_MEM_FRAG_RX_32) {
1778 dlog_warning(
1779 "Got %#x from TEE in response to %#x for "
1780 "fragment with with %d/%d, expected "
1781 "FFA_MEM_FRAG_RX.\n",
1782 ret.func, share_func, fragment_length,
1783 memory_share_length);
1784 ret = ffa_error(FFA_INVALID_PARAMETERS);
1785 goto out_unlock;
1786 }
1787 handle = ffa_frag_handle(ret);
1788 if (ret.arg3 != fragment_length) {
1789 dlog_warning(
1790 "Got unexpected fragment offset %d for "
1791 "FFA_MEM_FRAG_RX from TEE (expected %d).\n",
1792 ret.arg3, fragment_length);
1793 ret = ffa_error(FFA_INVALID_PARAMETERS);
1794 goto out_unlock;
1795 }
1796 if (ffa_frag_sender(ret) != from_locked.vm->id) {
1797 dlog_warning(
1798 "Got unexpected sender ID %d for "
1799 "FFA_MEM_FRAG_RX from TEE (expected %d).\n",
1800 ffa_frag_sender(ret), from_locked.vm->id);
1801 ret = ffa_error(FFA_INVALID_PARAMETERS);
1802 goto out_unlock;
1803 }
1804
1805 if (!allocate_share_state(share_states, share_func,
1806 memory_region, fragment_length,
1807 handle, NULL)) {
1808 dlog_verbose("Failed to allocate share state.\n");
1809 ret = ffa_error(FFA_NO_MEMORY);
1810 goto out_unlock;
1811 }
1812 /*
1813 * Don't free the memory region fragment, as it has been stored
1814 * in the share state.
1815 */
1816 memory_region = NULL;
1817 out_unlock:
1818 share_states_unlock(&share_states);
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001819 }
1820
Andrew Walbranca808b12020-05-15 17:22:28 +01001821out:
1822 if (memory_region != NULL) {
1823 mpool_free(page_pool, memory_region);
1824 }
1825 dump_share_states();
1826 return ret;
1827}
1828
1829/**
1830 * Continues an operation to donate, lend or share memory to a non-TEE VM. If
1831 * this is the last fragment then checks that the transition is valid for the
1832 * type of memory sending operation and updates the stage-2 page tables of the
1833 * sender.
1834 *
1835 * Assumes that the caller has already found and locked the sender VM and copied
1836 * the memory region descriptor from the sender's TX buffer to a freshly
1837 * allocated page from Hafnium's internal pool.
1838 *
1839 * This function takes ownership of the `fragment` passed in; it must not be
1840 * freed by the caller.
1841 */
1842struct ffa_value ffa_memory_send_continue(struct vm_locked from_locked,
1843 void *fragment,
1844 uint32_t fragment_length,
1845 ffa_memory_handle_t handle,
1846 struct mpool *page_pool)
1847{
1848 struct share_states_locked share_states = share_states_lock();
1849 struct ffa_memory_share_state *share_state;
1850 struct ffa_value ret;
1851 struct ffa_memory_region *memory_region;
1852
1853 ret = ffa_memory_send_continue_validate(share_states, handle,
1854 &share_state,
1855 from_locked.vm->id, page_pool);
1856 if (ret.func != FFA_SUCCESS_32) {
1857 goto out_free_fragment;
1858 }
1859 memory_region = share_state->memory_region;
1860
1861 if (memory_region->receivers[0].receiver_permissions.receiver ==
1862 HF_TEE_VM_ID) {
1863 dlog_error(
1864 "Got hypervisor-allocated handle for memory send to "
1865 "TEE. This should never happen, and indicates a bug in "
1866 "EL3 code.\n");
1867 ret = ffa_error(FFA_INVALID_PARAMETERS);
1868 goto out_free_fragment;
1869 }
1870
1871 /* Add this fragment. */
1872 share_state->fragments[share_state->fragment_count] = fragment;
1873 share_state->fragment_constituent_counts[share_state->fragment_count] =
1874 fragment_length / sizeof(struct ffa_memory_region_constituent);
1875 share_state->fragment_count++;
1876
1877 /* Check whether the memory send operation is now ready to complete. */
1878 if (share_state_sending_complete(share_states, share_state)) {
J-Alves2a0d2882020-10-29 14:49:50 +00001879 ret = ffa_memory_send_complete(
1880 from_locked, share_states, share_state, page_pool,
1881 &(share_state->sender_orig_mode));
Andrew Walbranca808b12020-05-15 17:22:28 +01001882 } else {
1883 ret = (struct ffa_value){
1884 .func = FFA_MEM_FRAG_RX_32,
1885 .arg1 = (uint32_t)handle,
1886 .arg2 = (uint32_t)(handle >> 32),
1887 .arg3 = share_state_next_fragment_offset(share_states,
1888 share_state)};
1889 }
1890 goto out;
1891
1892out_free_fragment:
1893 mpool_free(page_pool, fragment);
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001894
1895out:
Andrew Walbranca808b12020-05-15 17:22:28 +01001896 share_states_unlock(&share_states);
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001897 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001898}
1899
Andrew Walbranca808b12020-05-15 17:22:28 +01001900/**
1901 * Continues an operation to donate, lend or share memory to the TEE VM. If this
1902 * is the last fragment then checks that the transition is valid for the type of
1903 * memory sending operation and updates the stage-2 page tables of the sender.
1904 *
1905 * Assumes that the caller has already found and locked the sender VM and copied
1906 * the memory region descriptor from the sender's TX buffer to a freshly
1907 * allocated page from Hafnium's internal pool.
1908 *
1909 * This function takes ownership of the `memory_region` passed in and will free
1910 * it when necessary; it must not be freed by the caller.
1911 */
1912struct ffa_value ffa_memory_tee_send_continue(struct vm_locked from_locked,
1913 struct vm_locked to_locked,
1914 void *fragment,
1915 uint32_t fragment_length,
1916 ffa_memory_handle_t handle,
1917 struct mpool *page_pool)
1918{
1919 struct share_states_locked share_states = share_states_lock();
1920 struct ffa_memory_share_state *share_state;
1921 struct ffa_value ret;
1922 struct ffa_memory_region *memory_region;
1923
1924 ret = ffa_memory_send_continue_validate(share_states, handle,
1925 &share_state,
1926 from_locked.vm->id, page_pool);
1927 if (ret.func != FFA_SUCCESS_32) {
1928 goto out_free_fragment;
1929 }
1930 memory_region = share_state->memory_region;
1931
1932 if (memory_region->receivers[0].receiver_permissions.receiver !=
1933 HF_TEE_VM_ID) {
1934 dlog_error(
1935 "Got SPM-allocated handle for memory send to non-TEE "
1936 "VM. This should never happen, and indicates a bug.\n");
1937 ret = ffa_error(FFA_INVALID_PARAMETERS);
1938 goto out_free_fragment;
1939 }
1940
1941 if (to_locked.vm->mailbox.state != MAILBOX_STATE_EMPTY ||
1942 to_locked.vm->mailbox.recv == NULL) {
1943 /*
1944 * If the TEE RX buffer is not available, tell the sender to
1945 * retry by returning the current offset again.
1946 */
1947 ret = (struct ffa_value){
1948 .func = FFA_MEM_FRAG_RX_32,
1949 .arg1 = (uint32_t)handle,
1950 .arg2 = (uint32_t)(handle >> 32),
1951 .arg3 = share_state_next_fragment_offset(share_states,
1952 share_state),
1953 };
1954 goto out_free_fragment;
1955 }
1956
1957 /* Add this fragment. */
1958 share_state->fragments[share_state->fragment_count] = fragment;
1959 share_state->fragment_constituent_counts[share_state->fragment_count] =
1960 fragment_length / sizeof(struct ffa_memory_region_constituent);
1961 share_state->fragment_count++;
1962
1963 /* Check whether the memory send operation is now ready to complete. */
1964 if (share_state_sending_complete(share_states, share_state)) {
Andrew Walbran37c574e2020-06-03 11:45:46 +01001965 struct mpool local_page_pool;
1966 uint32_t orig_from_mode;
1967
1968 /*
1969 * Use a local page pool so that we can roll back if necessary.
1970 */
1971 mpool_init_with_fallback(&local_page_pool, page_pool);
1972
Andrew Walbranca808b12020-05-15 17:22:28 +01001973 ret = ffa_memory_send_complete(from_locked, share_states,
Andrew Walbran37c574e2020-06-03 11:45:46 +01001974 share_state, &local_page_pool,
1975 &orig_from_mode);
Andrew Walbranca808b12020-05-15 17:22:28 +01001976
1977 if (ret.func == FFA_SUCCESS_32) {
1978 /*
1979 * Forward final fragment on to the TEE so that
1980 * it can complete the memory sending operation.
1981 */
1982 ret = memory_send_continue_tee_forward(
1983 to_locked, from_locked.vm->id, fragment,
1984 fragment_length, handle);
1985
1986 if (ret.func != FFA_SUCCESS_32) {
1987 /*
1988 * The error will be passed on to the caller,
1989 * but log it here too.
1990 */
1991 dlog_verbose(
1992 "TEE didn't successfully complete "
1993 "memory send operation; returned %#x "
Andrew Walbran37c574e2020-06-03 11:45:46 +01001994 "(%d). Rolling back.\n",
Andrew Walbranca808b12020-05-15 17:22:28 +01001995 ret.func, ret.arg2);
Andrew Walbran37c574e2020-06-03 11:45:46 +01001996
1997 /*
1998 * The TEE failed to complete the send
1999 * operation, so roll back the page table update
2000 * for the VM. This can't fail because it won't
2001 * try to allocate more memory than was freed
2002 * into the `local_page_pool` by
2003 * `ffa_send_check_update` in the initial
2004 * update.
2005 */
2006 CHECK(ffa_region_group_identity_map(
2007 from_locked, share_state->fragments,
2008 share_state
2009 ->fragment_constituent_counts,
2010 share_state->fragment_count,
2011 orig_from_mode, &local_page_pool,
2012 true));
Andrew Walbranca808b12020-05-15 17:22:28 +01002013 }
Andrew Walbran37c574e2020-06-03 11:45:46 +01002014
Andrew Walbranca808b12020-05-15 17:22:28 +01002015 /* Free share state. */
2016 share_state_free(share_states, share_state, page_pool);
2017 } else {
2018 /* Abort sending to TEE. */
2019 struct ffa_value tee_ret =
Olivier Deprez112d2b52020-09-30 07:39:23 +02002020 arch_other_world_call((struct ffa_value){
Andrew Walbranca808b12020-05-15 17:22:28 +01002021 .func = FFA_MEM_RECLAIM_32,
2022 .arg1 = (uint32_t)handle,
2023 .arg2 = (uint32_t)(handle >> 32)});
2024
2025 if (tee_ret.func != FFA_SUCCESS_32) {
2026 /*
2027 * Nothing we can do if TEE doesn't abort
2028 * properly, just log it.
2029 */
2030 dlog_verbose(
2031 "TEE didn't successfully abort failed "
2032 "memory send operation; returned %#x "
2033 "(%d).\n",
2034 tee_ret.func, tee_ret.arg2);
2035 }
2036 /*
2037 * We don't need to free the share state in this case
2038 * because ffa_memory_send_complete does that already.
2039 */
2040 }
Andrew Walbran37c574e2020-06-03 11:45:46 +01002041
2042 mpool_fini(&local_page_pool);
Andrew Walbranca808b12020-05-15 17:22:28 +01002043 } else {
2044 uint32_t next_fragment_offset =
2045 share_state_next_fragment_offset(share_states,
2046 share_state);
2047
2048 ret = memory_send_continue_tee_forward(
2049 to_locked, from_locked.vm->id, fragment,
2050 fragment_length, handle);
2051
2052 if (ret.func != FFA_MEM_FRAG_RX_32 ||
2053 ffa_frag_handle(ret) != handle ||
2054 ret.arg3 != next_fragment_offset ||
2055 ffa_frag_sender(ret) != from_locked.vm->id) {
2056 dlog_verbose(
2057 "Got unexpected result from forwarding "
2058 "FFA_MEM_FRAG_TX to TEE: %#x (handle %#x, "
2059 "offset %d, sender %d); expected "
2060 "FFA_MEM_FRAG_RX (handle %#x, offset %d, "
2061 "sender %d).\n",
2062 ret.func, ffa_frag_handle(ret), ret.arg3,
2063 ffa_frag_sender(ret), handle,
2064 next_fragment_offset, from_locked.vm->id);
2065 /* Free share state. */
2066 share_state_free(share_states, share_state, page_pool);
2067 ret = ffa_error(FFA_INVALID_PARAMETERS);
2068 goto out;
2069 }
2070
2071 ret = (struct ffa_value){.func = FFA_MEM_FRAG_RX_32,
2072 .arg1 = (uint32_t)handle,
2073 .arg2 = (uint32_t)(handle >> 32),
2074 .arg3 = next_fragment_offset};
2075 }
2076 goto out;
2077
2078out_free_fragment:
2079 mpool_free(page_pool, fragment);
2080
2081out:
2082 share_states_unlock(&share_states);
2083 return ret;
2084}
2085
2086/** Clean up after the receiver has finished retrieving a memory region. */
2087static void ffa_memory_retrieve_complete(
2088 struct share_states_locked share_states,
2089 struct ffa_memory_share_state *share_state, struct mpool *page_pool)
2090{
2091 if (share_state->share_func == FFA_MEM_DONATE_32) {
2092 /*
2093 * Memory that has been donated can't be relinquished,
2094 * so no need to keep the share state around.
2095 */
2096 share_state_free(share_states, share_state, page_pool);
2097 dlog_verbose("Freed share state for donate.\n");
2098 }
2099}
2100
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002101struct ffa_value ffa_memory_retrieve(struct vm_locked to_locked,
2102 struct ffa_memory_region *retrieve_request,
Andrew Walbran130a8ae2020-05-15 16:27:15 +01002103 uint32_t retrieve_request_length,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002104 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002105{
Andrew Walbran130a8ae2020-05-15 16:27:15 +01002106 uint32_t expected_retrieve_request_length =
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002107 sizeof(struct ffa_memory_region) +
Andrew Walbrana65a1322020-04-06 19:32:32 +01002108 retrieve_request->receiver_count *
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002109 sizeof(struct ffa_memory_access);
2110 ffa_memory_handle_t handle = retrieve_request->handle;
2111 ffa_memory_region_flags_t transaction_type =
Andrew Walbrana65a1322020-04-06 19:32:32 +01002112 retrieve_request->flags &
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002113 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK;
2114 struct ffa_memory_region *memory_region;
2115 ffa_memory_access_permissions_t sent_permissions;
2116 enum ffa_data_access sent_data_access;
2117 enum ffa_instruction_access sent_instruction_access;
2118 ffa_memory_access_permissions_t requested_permissions;
2119 enum ffa_data_access requested_data_access;
2120 enum ffa_instruction_access requested_instruction_access;
2121 ffa_memory_access_permissions_t permissions;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002122 uint32_t memory_to_attributes;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002123 struct share_states_locked share_states;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002124 struct ffa_memory_share_state *share_state;
2125 struct ffa_value ret;
Andrew Walbranca808b12020-05-15 17:22:28 +01002126 struct ffa_composite_memory_region *composite;
2127 uint32_t total_length;
2128 uint32_t fragment_length;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002129
2130 dump_share_states();
2131
Andrew Walbran130a8ae2020-05-15 16:27:15 +01002132 if (retrieve_request_length != expected_retrieve_request_length) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002133 dlog_verbose(
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002134 "Invalid length for FFA_MEM_RETRIEVE_REQ, expected %d "
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002135 "but was %d.\n",
Andrew Walbran130a8ae2020-05-15 16:27:15 +01002136 expected_retrieve_request_length,
2137 retrieve_request_length);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002138 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002139 }
2140
Andrew Walbrana65a1322020-04-06 19:32:32 +01002141 if (retrieve_request->receiver_count != 1) {
2142 dlog_verbose(
2143 "Multi-way memory sharing not supported (got %d "
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002144 "receivers descriptors on FFA_MEM_RETRIEVE_REQ, "
Andrew Walbrana65a1322020-04-06 19:32:32 +01002145 "expected 1).\n",
2146 retrieve_request->receiver_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002147 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01002148 }
2149
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002150 share_states = share_states_lock();
2151 if (!get_share_state(share_states, handle, &share_state)) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002152 dlog_verbose("Invalid handle %#x for FFA_MEM_RETRIEVE_REQ.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002153 handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002154 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002155 goto out;
2156 }
2157
Andrew Walbrana65a1322020-04-06 19:32:32 +01002158 memory_region = share_state->memory_region;
2159 CHECK(memory_region != NULL);
2160
2161 /*
2162 * Check that the transaction type expected by the receiver is correct,
2163 * if it has been specified.
2164 */
2165 if (transaction_type !=
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002166 FFA_MEMORY_REGION_TRANSACTION_TYPE_UNSPECIFIED &&
Andrew Walbrana65a1322020-04-06 19:32:32 +01002167 transaction_type != (memory_region->flags &
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002168 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK)) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002169 dlog_verbose(
2170 "Incorrect transaction type %#x for "
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002171 "FFA_MEM_RETRIEVE_REQ, expected %#x for handle %#x.\n",
Andrew Walbrana65a1322020-04-06 19:32:32 +01002172 transaction_type,
2173 memory_region->flags &
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002174 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK,
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002175 handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002176 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002177 goto out;
2178 }
2179
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002180 if (retrieve_request->sender != memory_region->sender) {
2181 dlog_verbose(
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002182 "Incorrect sender ID %d for FFA_MEM_RETRIEVE_REQ, "
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002183 "expected %d for handle %#x.\n",
2184 retrieve_request->sender, memory_region->sender,
2185 handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002186 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002187 goto out;
2188 }
2189
2190 if (retrieve_request->tag != memory_region->tag) {
2191 dlog_verbose(
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002192 "Incorrect tag %d for FFA_MEM_RETRIEVE_REQ, expected "
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002193 "%d for handle %#x.\n",
2194 retrieve_request->tag, memory_region->tag, handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002195 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002196 goto out;
2197 }
2198
Andrew Walbrana65a1322020-04-06 19:32:32 +01002199 if (retrieve_request->receivers[0].receiver_permissions.receiver !=
2200 to_locked.vm->id) {
2201 dlog_verbose(
2202 "Retrieve request receiver VM ID %d didn't match "
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002203 "caller of FFA_MEM_RETRIEVE_REQ.\n",
Andrew Walbrana65a1322020-04-06 19:32:32 +01002204 retrieve_request->receivers[0]
2205 .receiver_permissions.receiver);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002206 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01002207 goto out;
2208 }
2209
2210 if (memory_region->receivers[0].receiver_permissions.receiver !=
2211 to_locked.vm->id) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002212 dlog_verbose(
Andrew Walbranf07f04d2020-05-01 18:09:00 +01002213 "Incorrect receiver VM ID %d for FFA_MEM_RETRIEVE_REQ, "
2214 "expected %d for handle %#x.\n",
Andrew Walbrana65a1322020-04-06 19:32:32 +01002215 to_locked.vm->id,
2216 memory_region->receivers[0]
2217 .receiver_permissions.receiver,
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002218 handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002219 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002220 goto out;
2221 }
2222
Andrew Walbranca808b12020-05-15 17:22:28 +01002223 if (!share_state->sending_complete) {
2224 dlog_verbose(
2225 "Memory with handle %#x not fully sent, can't "
2226 "retrieve.\n",
2227 handle);
2228 ret = ffa_error(FFA_INVALID_PARAMETERS);
2229 goto out;
2230 }
2231
2232 if (share_state->retrieved_fragment_count[0] != 0) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002233 dlog_verbose("Memory with handle %#x already retrieved.\n",
2234 handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002235 ret = ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002236 goto out;
2237 }
2238
Andrew Walbrana65a1322020-04-06 19:32:32 +01002239 if (retrieve_request->receivers[0].composite_memory_region_offset !=
2240 0) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002241 dlog_verbose(
2242 "Retriever specified address ranges not supported (got "
Andrew Walbranf07f04d2020-05-01 18:09:00 +01002243 "offset %d).\n",
Andrew Walbrana65a1322020-04-06 19:32:32 +01002244 retrieve_request->receivers[0]
2245 .composite_memory_region_offset);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002246 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002247 goto out;
2248 }
2249
J-Alves84658fc2021-06-17 14:37:32 +01002250 if ((retrieve_request->flags & ~0x7FF) != 0U) {
2251 dlog_verbose(
2252 "Bits 31-10 must be zero in memory region's flags.\n");
2253 ret = ffa_error(FFA_INVALID_PARAMETERS);
2254 goto out;
2255 }
2256
2257 if (share_state->share_func == FFA_MEM_SHARE_32 &&
2258 (retrieve_request->flags &
2259 (FFA_MEMORY_REGION_FLAG_CLEAR |
2260 FFA_MEMORY_REGION_FLAG_CLEAR_RELINQUISH)) != 0U) {
2261 dlog_verbose(
2262 "Memory Share operation can't clean after relinquish "
2263 "memory region.\n");
2264 ret = ffa_error(FFA_INVALID_PARAMETERS);
2265 goto out;
2266 }
2267
Andrew Walbrana65a1322020-04-06 19:32:32 +01002268 /*
2269 * Check permissions from sender against permissions requested by
2270 * receiver.
2271 */
2272 /* TODO: Check attributes too. */
2273 sent_permissions =
2274 memory_region->receivers[0].receiver_permissions.permissions;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002275 sent_data_access = ffa_get_data_access_attr(sent_permissions);
Andrew Walbrana65a1322020-04-06 19:32:32 +01002276 sent_instruction_access =
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002277 ffa_get_instruction_access_attr(sent_permissions);
Andrew Walbrana65a1322020-04-06 19:32:32 +01002278 requested_permissions =
2279 retrieve_request->receivers[0].receiver_permissions.permissions;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002280 requested_data_access = ffa_get_data_access_attr(requested_permissions);
Andrew Walbrana65a1322020-04-06 19:32:32 +01002281 requested_instruction_access =
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002282 ffa_get_instruction_access_attr(requested_permissions);
Andrew Walbrana65a1322020-04-06 19:32:32 +01002283 permissions = 0;
J-Alves84658fc2021-06-17 14:37:32 +01002284
2285 if ((sent_data_access == FFA_DATA_ACCESS_RO ||
2286 requested_permissions == FFA_DATA_ACCESS_RO) &&
2287 (retrieve_request->flags & FFA_MEMORY_REGION_FLAG_CLEAR) != 0U) {
2288 dlog_verbose(
2289 "Receiver has RO permissions can not request clear.\n");
2290 ret = ffa_error(FFA_DENIED);
2291 goto out;
2292 }
2293
Andrew Walbrana65a1322020-04-06 19:32:32 +01002294 switch (sent_data_access) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002295 case FFA_DATA_ACCESS_NOT_SPECIFIED:
2296 case FFA_DATA_ACCESS_RW:
2297 if (requested_data_access == FFA_DATA_ACCESS_NOT_SPECIFIED ||
2298 requested_data_access == FFA_DATA_ACCESS_RW) {
2299 ffa_set_data_access_attr(&permissions,
2300 FFA_DATA_ACCESS_RW);
Andrew Walbrana65a1322020-04-06 19:32:32 +01002301 break;
2302 }
2303 /* Intentional fall-through. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002304 case FFA_DATA_ACCESS_RO:
2305 if (requested_data_access == FFA_DATA_ACCESS_NOT_SPECIFIED ||
2306 requested_data_access == FFA_DATA_ACCESS_RO) {
2307 ffa_set_data_access_attr(&permissions,
2308 FFA_DATA_ACCESS_RO);
Andrew Walbrana65a1322020-04-06 19:32:32 +01002309 break;
2310 }
2311 dlog_verbose(
2312 "Invalid data access requested; sender specified "
2313 "permissions %#x but receiver requested %#x.\n",
2314 sent_permissions, requested_permissions);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002315 ret = ffa_error(FFA_DENIED);
Andrew Walbrana65a1322020-04-06 19:32:32 +01002316 goto out;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002317 case FFA_DATA_ACCESS_RESERVED:
2318 panic("Got unexpected FFA_DATA_ACCESS_RESERVED. Should be "
Andrew Walbrana65a1322020-04-06 19:32:32 +01002319 "checked before this point.");
2320 }
2321 switch (sent_instruction_access) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002322 case FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED:
2323 case FFA_INSTRUCTION_ACCESS_X:
Andrew Walbrana65a1322020-04-06 19:32:32 +01002324 if (requested_instruction_access ==
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002325 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED ||
2326 requested_instruction_access == FFA_INSTRUCTION_ACCESS_X) {
2327 ffa_set_instruction_access_attr(
2328 &permissions, FFA_INSTRUCTION_ACCESS_X);
Andrew Walbrana65a1322020-04-06 19:32:32 +01002329 break;
2330 }
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002331 case FFA_INSTRUCTION_ACCESS_NX:
Andrew Walbrana65a1322020-04-06 19:32:32 +01002332 if (requested_instruction_access ==
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002333 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED ||
2334 requested_instruction_access == FFA_INSTRUCTION_ACCESS_NX) {
2335 ffa_set_instruction_access_attr(
2336 &permissions, FFA_INSTRUCTION_ACCESS_NX);
Andrew Walbrana65a1322020-04-06 19:32:32 +01002337 break;
2338 }
2339 dlog_verbose(
2340 "Invalid instruction access requested; sender "
Andrew Walbranf07f04d2020-05-01 18:09:00 +01002341 "specified permissions %#x but receiver requested "
2342 "%#x.\n",
Andrew Walbrana65a1322020-04-06 19:32:32 +01002343 sent_permissions, requested_permissions);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002344 ret = ffa_error(FFA_DENIED);
Andrew Walbrana65a1322020-04-06 19:32:32 +01002345 goto out;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002346 case FFA_INSTRUCTION_ACCESS_RESERVED:
2347 panic("Got unexpected FFA_INSTRUCTION_ACCESS_RESERVED. Should "
Andrew Walbrana65a1322020-04-06 19:32:32 +01002348 "be checked before this point.");
2349 }
J-Alves7cd5eb32020-10-16 19:06:10 +01002350 memory_to_attributes = ffa_memory_permissions_to_mode(
2351 permissions, share_state->sender_orig_mode);
Andrew Walbran996d1d12020-05-27 14:08:43 +01002352 ret = ffa_retrieve_check_update(
Andrew Walbranca808b12020-05-15 17:22:28 +01002353 to_locked, share_state->fragments,
2354 share_state->fragment_constituent_counts,
2355 share_state->fragment_count, memory_to_attributes,
Andrew Walbran996d1d12020-05-27 14:08:43 +01002356 share_state->share_func, false, page_pool);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002357 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002358 goto out;
2359 }
2360
2361 /*
2362 * Copy response to RX buffer of caller and deliver the message. This
2363 * must be done before the share_state is (possibly) freed.
2364 */
Andrew Walbrana65a1322020-04-06 19:32:32 +01002365 /* TODO: combine attributes from sender and request. */
Andrew Walbranca808b12020-05-15 17:22:28 +01002366 composite = ffa_memory_region_get_composite(memory_region, 0);
2367 /*
2368 * Constituents which we received in the first fragment should always
2369 * fit in the first fragment we are sending, because the header is the
2370 * same size in both cases and we have a fixed message buffer size. So
2371 * `ffa_retrieved_memory_region_init` should never fail.
2372 */
2373 CHECK(ffa_retrieved_memory_region_init(
Andrew Walbrana65a1322020-04-06 19:32:32 +01002374 to_locked.vm->mailbox.recv, HF_MAILBOX_SIZE,
2375 memory_region->sender, memory_region->attributes,
2376 memory_region->flags, handle, to_locked.vm->id, permissions,
Andrew Walbranca808b12020-05-15 17:22:28 +01002377 composite->page_count, composite->constituent_count,
2378 share_state->fragments[0],
2379 share_state->fragment_constituent_counts[0], &total_length,
2380 &fragment_length));
2381 to_locked.vm->mailbox.recv_size = fragment_length;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002382 to_locked.vm->mailbox.recv_sender = HF_HYPERVISOR_VM_ID;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002383 to_locked.vm->mailbox.recv_func = FFA_MEM_RETRIEVE_RESP_32;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002384 to_locked.vm->mailbox.state = MAILBOX_STATE_READ;
2385
Andrew Walbranca808b12020-05-15 17:22:28 +01002386 share_state->retrieved_fragment_count[0] = 1;
2387 if (share_state->retrieved_fragment_count[0] ==
2388 share_state->fragment_count) {
2389 ffa_memory_retrieve_complete(share_states, share_state,
2390 page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002391 }
2392
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002393 ret = (struct ffa_value){.func = FFA_MEM_RETRIEVE_RESP_32,
Andrew Walbranca808b12020-05-15 17:22:28 +01002394 .arg1 = total_length,
2395 .arg2 = fragment_length};
2396
2397out:
2398 share_states_unlock(&share_states);
2399 dump_share_states();
2400 return ret;
2401}
2402
2403struct ffa_value ffa_memory_retrieve_continue(struct vm_locked to_locked,
2404 ffa_memory_handle_t handle,
2405 uint32_t fragment_offset,
2406 struct mpool *page_pool)
2407{
2408 struct ffa_memory_region *memory_region;
2409 struct share_states_locked share_states;
2410 struct ffa_memory_share_state *share_state;
2411 struct ffa_value ret;
2412 uint32_t fragment_index;
2413 uint32_t retrieved_constituents_count;
2414 uint32_t i;
2415 uint32_t expected_fragment_offset;
2416 uint32_t remaining_constituent_count;
2417 uint32_t fragment_length;
2418
2419 dump_share_states();
2420
2421 share_states = share_states_lock();
2422 if (!get_share_state(share_states, handle, &share_state)) {
2423 dlog_verbose("Invalid handle %#x for FFA_MEM_FRAG_RX.\n",
2424 handle);
2425 ret = ffa_error(FFA_INVALID_PARAMETERS);
2426 goto out;
2427 }
2428
2429 memory_region = share_state->memory_region;
2430 CHECK(memory_region != NULL);
2431
2432 if (memory_region->receivers[0].receiver_permissions.receiver !=
2433 to_locked.vm->id) {
2434 dlog_verbose(
2435 "Caller of FFA_MEM_FRAG_RX (%d) is not receiver (%d) "
2436 "of handle %#x.\n",
2437 to_locked.vm->id,
2438 memory_region->receivers[0]
2439 .receiver_permissions.receiver,
2440 handle);
2441 ret = ffa_error(FFA_INVALID_PARAMETERS);
2442 goto out;
2443 }
2444
2445 if (!share_state->sending_complete) {
2446 dlog_verbose(
2447 "Memory with handle %#x not fully sent, can't "
2448 "retrieve.\n",
2449 handle);
2450 ret = ffa_error(FFA_INVALID_PARAMETERS);
2451 goto out;
2452 }
2453
2454 if (share_state->retrieved_fragment_count[0] == 0 ||
2455 share_state->retrieved_fragment_count[0] >=
2456 share_state->fragment_count) {
2457 dlog_verbose(
2458 "Retrieval of memory with handle %#x not yet started "
2459 "or already completed (%d/%d fragments retrieved).\n",
2460 handle, share_state->retrieved_fragment_count[0],
2461 share_state->fragment_count);
2462 ret = ffa_error(FFA_INVALID_PARAMETERS);
2463 goto out;
2464 }
2465
2466 fragment_index = share_state->retrieved_fragment_count[0];
2467
2468 /*
2469 * Check that the given fragment offset is correct by counting how many
2470 * constituents were in the fragments previously sent.
2471 */
2472 retrieved_constituents_count = 0;
2473 for (i = 0; i < fragment_index; ++i) {
2474 retrieved_constituents_count +=
2475 share_state->fragment_constituent_counts[i];
2476 }
2477 expected_fragment_offset =
2478 ffa_composite_constituent_offset(memory_region, 0) +
2479 retrieved_constituents_count *
2480 sizeof(struct ffa_memory_region_constituent);
2481 if (fragment_offset != expected_fragment_offset) {
2482 dlog_verbose("Fragment offset was %d but expected %d.\n",
2483 fragment_offset, expected_fragment_offset);
2484 ret = ffa_error(FFA_INVALID_PARAMETERS);
2485 goto out;
2486 }
2487
2488 remaining_constituent_count = ffa_memory_fragment_init(
2489 to_locked.vm->mailbox.recv, HF_MAILBOX_SIZE,
2490 share_state->fragments[fragment_index],
2491 share_state->fragment_constituent_counts[fragment_index],
2492 &fragment_length);
2493 CHECK(remaining_constituent_count == 0);
2494 to_locked.vm->mailbox.recv_size = fragment_length;
2495 to_locked.vm->mailbox.recv_sender = HF_HYPERVISOR_VM_ID;
2496 to_locked.vm->mailbox.recv_func = FFA_MEM_FRAG_TX_32;
2497 to_locked.vm->mailbox.state = MAILBOX_STATE_READ;
2498 share_state->retrieved_fragment_count[0]++;
2499 if (share_state->retrieved_fragment_count[0] ==
2500 share_state->fragment_count) {
2501 ffa_memory_retrieve_complete(share_states, share_state,
2502 page_pool);
2503 }
2504
2505 ret = (struct ffa_value){.func = FFA_MEM_FRAG_TX_32,
2506 .arg1 = (uint32_t)handle,
2507 .arg2 = (uint32_t)(handle >> 32),
2508 .arg3 = fragment_length};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002509
2510out:
2511 share_states_unlock(&share_states);
2512 dump_share_states();
2513 return ret;
2514}
2515
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002516struct ffa_value ffa_memory_relinquish(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002517 struct vm_locked from_locked,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002518 struct ffa_mem_relinquish *relinquish_request, struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002519{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002520 ffa_memory_handle_t handle = relinquish_request->handle;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002521 struct share_states_locked share_states;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002522 struct ffa_memory_share_state *share_state;
2523 struct ffa_memory_region *memory_region;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002524 bool clear;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002525 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002526
Andrew Walbrana65a1322020-04-06 19:32:32 +01002527 if (relinquish_request->endpoint_count != 1) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002528 dlog_verbose(
Andrew Walbrana65a1322020-04-06 19:32:32 +01002529 "Stream endpoints not supported (got %d endpoints on "
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002530 "FFA_MEM_RELINQUISH, expected 1).\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002531 relinquish_request->endpoint_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002532 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002533 }
2534
Andrew Walbrana65a1322020-04-06 19:32:32 +01002535 if (relinquish_request->endpoints[0] != from_locked.vm->id) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002536 dlog_verbose(
2537 "VM ID %d in relinquish message doesn't match calling "
2538 "VM ID %d.\n",
Andrew Walbrana65a1322020-04-06 19:32:32 +01002539 relinquish_request->endpoints[0], from_locked.vm->id);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002540 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002541 }
2542
2543 dump_share_states();
2544
2545 share_states = share_states_lock();
2546 if (!get_share_state(share_states, handle, &share_state)) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002547 dlog_verbose("Invalid handle %#x for FFA_MEM_RELINQUISH.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002548 handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002549 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002550 goto out;
2551 }
2552
Andrew Walbranca808b12020-05-15 17:22:28 +01002553 if (!share_state->sending_complete) {
2554 dlog_verbose(
2555 "Memory with handle %#x not fully sent, can't "
2556 "relinquish.\n",
2557 handle);
2558 ret = ffa_error(FFA_INVALID_PARAMETERS);
2559 goto out;
2560 }
2561
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002562 memory_region = share_state->memory_region;
2563 CHECK(memory_region != NULL);
2564
Andrew Walbrana65a1322020-04-06 19:32:32 +01002565 if (memory_region->receivers[0].receiver_permissions.receiver !=
2566 from_locked.vm->id) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002567 dlog_verbose(
2568 "VM ID %d tried to relinquish memory region with "
2569 "handle %#x but receiver was %d.\n",
2570 from_locked.vm->id, handle,
Andrew Walbrana65a1322020-04-06 19:32:32 +01002571 memory_region->receivers[0]
2572 .receiver_permissions.receiver);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002573 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002574 goto out;
2575 }
2576
Andrew Walbranca808b12020-05-15 17:22:28 +01002577 if (share_state->retrieved_fragment_count[0] !=
2578 share_state->fragment_count) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002579 dlog_verbose(
Andrew Walbranca808b12020-05-15 17:22:28 +01002580 "Memory with handle %#x not yet fully retrieved, can't "
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002581 "relinquish.\n",
2582 handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002583 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002584 goto out;
2585 }
2586
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002587 clear = relinquish_request->flags & FFA_MEMORY_REGION_FLAG_CLEAR;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002588
2589 /*
2590 * Clear is not allowed for memory that was shared, as the original
2591 * sender still has access to the memory.
2592 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002593 if (clear && share_state->share_func == FFA_MEM_SHARE_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002594 dlog_verbose("Memory which was shared can't be cleared.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002595 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002596 goto out;
2597 }
2598
Andrew Walbranca808b12020-05-15 17:22:28 +01002599 ret = ffa_relinquish_check_update(
2600 from_locked, share_state->fragments,
2601 share_state->fragment_constituent_counts,
2602 share_state->fragment_count, page_pool, clear);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002603
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002604 if (ret.func == FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002605 /*
2606 * Mark memory handle as not retrieved, so it can be reclaimed
2607 * (or retrieved again).
2608 */
Andrew Walbranca808b12020-05-15 17:22:28 +01002609 share_state->retrieved_fragment_count[0] = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002610 }
2611
2612out:
2613 share_states_unlock(&share_states);
2614 dump_share_states();
2615 return ret;
2616}
2617
2618/**
2619 * Validates that the reclaim transition is allowed for the given handle,
2620 * updates the page table of the reclaiming VM, and frees the internal state
2621 * associated with the handle.
2622 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002623struct ffa_value ffa_memory_reclaim(struct vm_locked to_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01002624 ffa_memory_handle_t handle,
2625 ffa_memory_region_flags_t flags,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002626 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002627{
2628 struct share_states_locked share_states;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002629 struct ffa_memory_share_state *share_state;
2630 struct ffa_memory_region *memory_region;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002631 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002632
2633 dump_share_states();
2634
2635 share_states = share_states_lock();
2636 if (!get_share_state(share_states, handle, &share_state)) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002637 dlog_verbose("Invalid handle %#x for FFA_MEM_RECLAIM.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002638 handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002639 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002640 goto out;
2641 }
2642
2643 memory_region = share_state->memory_region;
2644 CHECK(memory_region != NULL);
2645
2646 if (to_locked.vm->id != memory_region->sender) {
2647 dlog_verbose(
Olivier Deprezf92e5d42020-11-13 16:00:54 +01002648 "VM %#x attempted to reclaim memory handle %#x "
2649 "originally sent by VM %#x.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002650 to_locked.vm->id, handle, memory_region->sender);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002651 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002652 goto out;
2653 }
2654
Andrew Walbranca808b12020-05-15 17:22:28 +01002655 if (!share_state->sending_complete) {
2656 dlog_verbose(
2657 "Memory with handle %#x not fully sent, can't "
2658 "reclaim.\n",
2659 handle);
2660 ret = ffa_error(FFA_INVALID_PARAMETERS);
2661 goto out;
2662 }
2663
2664 if (share_state->retrieved_fragment_count[0] != 0) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002665 dlog_verbose(
2666 "Tried to reclaim memory handle %#x that has not been "
2667 "relinquished.\n",
2668 handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002669 ret = ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002670 goto out;
2671 }
2672
Andrew Walbranca808b12020-05-15 17:22:28 +01002673 ret = ffa_retrieve_check_update(
2674 to_locked, share_state->fragments,
2675 share_state->fragment_constituent_counts,
J-Alves2a0d2882020-10-29 14:49:50 +00002676 share_state->fragment_count, share_state->sender_orig_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +01002677 FFA_MEM_RECLAIM_32, flags & FFA_MEM_RECLAIM_CLEAR, page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002678
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002679 if (ret.func == FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002680 share_state_free(share_states, share_state, page_pool);
2681 dlog_verbose("Freed share state after successful reclaim.\n");
2682 }
2683
2684out:
2685 share_states_unlock(&share_states);
2686 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01002687}
Andrew Walbran290b0c92020-02-03 16:37:14 +00002688
2689/**
Andrew Walbranca808b12020-05-15 17:22:28 +01002690 * Validates that the reclaim transition is allowed for the memory region with
2691 * the given handle which was previously shared with the TEE, tells the TEE to
2692 * mark it as reclaimed, and updates the page table of the reclaiming VM.
2693 *
2694 * To do this information about the memory region is first fetched from the TEE.
Andrew Walbran290b0c92020-02-03 16:37:14 +00002695 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002696struct ffa_value ffa_memory_tee_reclaim(struct vm_locked to_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01002697 struct vm_locked from_locked,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002698 ffa_memory_handle_t handle,
Andrew Walbranca808b12020-05-15 17:22:28 +01002699 ffa_memory_region_flags_t flags,
2700 struct mpool *page_pool)
Andrew Walbran290b0c92020-02-03 16:37:14 +00002701{
Andrew Walbranca808b12020-05-15 17:22:28 +01002702 uint32_t request_length = ffa_memory_lender_retrieve_request_init(
2703 from_locked.vm->mailbox.recv, handle, to_locked.vm->id);
2704 struct ffa_value tee_ret;
2705 uint32_t length;
2706 uint32_t fragment_length;
2707 uint32_t fragment_offset;
2708 struct ffa_memory_region *memory_region;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002709 struct ffa_composite_memory_region *composite;
Andrew Walbranca808b12020-05-15 17:22:28 +01002710 uint32_t memory_to_attributes = MM_MODE_R | MM_MODE_W | MM_MODE_X;
2711
2712 CHECK(request_length <= HF_MAILBOX_SIZE);
2713 CHECK(from_locked.vm->id == HF_TEE_VM_ID);
2714
2715 /* Retrieve memory region information from the TEE. */
Olivier Deprez112d2b52020-09-30 07:39:23 +02002716 tee_ret = arch_other_world_call(
Andrew Walbranca808b12020-05-15 17:22:28 +01002717 (struct ffa_value){.func = FFA_MEM_RETRIEVE_REQ_32,
2718 .arg1 = request_length,
2719 .arg2 = request_length});
2720 if (tee_ret.func == FFA_ERROR_32) {
2721 dlog_verbose("Got error %d from EL3.\n", tee_ret.arg2);
2722 return tee_ret;
2723 }
2724 if (tee_ret.func != FFA_MEM_RETRIEVE_RESP_32) {
2725 dlog_verbose(
2726 "Got %#x from EL3, expected FFA_MEM_RETRIEVE_RESP.\n",
2727 tee_ret.func);
2728 return ffa_error(FFA_INVALID_PARAMETERS);
2729 }
2730
2731 length = tee_ret.arg1;
2732 fragment_length = tee_ret.arg2;
2733
2734 if (fragment_length > HF_MAILBOX_SIZE || fragment_length > length ||
2735 length > sizeof(tee_retrieve_buffer)) {
2736 dlog_verbose("Invalid fragment length %d/%d (max %d/%d).\n",
2737 fragment_length, length, HF_MAILBOX_SIZE,
2738 sizeof(tee_retrieve_buffer));
2739 return ffa_error(FFA_INVALID_PARAMETERS);
2740 }
2741
2742 /*
2743 * Copy the first fragment of the memory region descriptor to an
2744 * internal buffer.
2745 */
2746 memcpy_s(tee_retrieve_buffer, sizeof(tee_retrieve_buffer),
2747 from_locked.vm->mailbox.send, fragment_length);
2748
2749 /* Fetch the remaining fragments into the same buffer. */
2750 fragment_offset = fragment_length;
2751 while (fragment_offset < length) {
Olivier Deprez112d2b52020-09-30 07:39:23 +02002752 tee_ret = arch_other_world_call(
Andrew Walbranca808b12020-05-15 17:22:28 +01002753 (struct ffa_value){.func = FFA_MEM_FRAG_RX_32,
2754 .arg1 = (uint32_t)handle,
2755 .arg2 = (uint32_t)(handle >> 32),
2756 .arg3 = fragment_offset});
2757 if (tee_ret.func != FFA_MEM_FRAG_TX_32) {
2758 dlog_verbose(
2759 "Got %#x (%d) from TEE in response to "
2760 "FFA_MEM_FRAG_RX, expected FFA_MEM_FRAG_TX.\n",
2761 tee_ret.func, tee_ret.arg2);
2762 return tee_ret;
2763 }
2764 if (ffa_frag_handle(tee_ret) != handle) {
2765 dlog_verbose(
2766 "Got FFA_MEM_FRAG_TX for unexpected handle %#x "
2767 "in response to FFA_MEM_FRAG_RX for handle "
2768 "%#x.\n",
2769 ffa_frag_handle(tee_ret), handle);
2770 return ffa_error(FFA_INVALID_PARAMETERS);
2771 }
2772 if (ffa_frag_sender(tee_ret) != 0) {
2773 dlog_verbose(
2774 "Got FFA_MEM_FRAG_TX with unexpected sender %d "
2775 "(expected 0).\n",
2776 ffa_frag_sender(tee_ret));
2777 return ffa_error(FFA_INVALID_PARAMETERS);
2778 }
2779 fragment_length = tee_ret.arg3;
2780 if (fragment_length > HF_MAILBOX_SIZE ||
2781 fragment_offset + fragment_length > length) {
2782 dlog_verbose(
2783 "Invalid fragment length %d at offset %d (max "
2784 "%d).\n",
2785 fragment_length, fragment_offset,
2786 HF_MAILBOX_SIZE);
2787 return ffa_error(FFA_INVALID_PARAMETERS);
2788 }
2789 memcpy_s(tee_retrieve_buffer + fragment_offset,
2790 sizeof(tee_retrieve_buffer) - fragment_offset,
2791 from_locked.vm->mailbox.send, fragment_length);
2792
2793 fragment_offset += fragment_length;
2794 }
2795
2796 memory_region = (struct ffa_memory_region *)tee_retrieve_buffer;
Andrew Walbran290b0c92020-02-03 16:37:14 +00002797
2798 if (memory_region->receiver_count != 1) {
2799 /* Only one receiver supported by Hafnium for now. */
2800 dlog_verbose(
2801 "Multiple recipients not supported (got %d, expected "
2802 "1).\n",
2803 memory_region->receiver_count);
Andrew Walbranca808b12020-05-15 17:22:28 +01002804 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran290b0c92020-02-03 16:37:14 +00002805 }
2806
2807 if (memory_region->handle != handle) {
2808 dlog_verbose(
2809 "Got memory region handle %#x from TEE but requested "
2810 "handle %#x.\n",
2811 memory_region->handle, handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002812 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran290b0c92020-02-03 16:37:14 +00002813 }
2814
2815 /* The original sender must match the caller. */
2816 if (to_locked.vm->id != memory_region->sender) {
2817 dlog_verbose(
Olivier Deprezf92e5d42020-11-13 16:00:54 +01002818 "VM %#x attempted to reclaim memory handle %#x "
2819 "originally sent by VM %#x.\n",
Andrew Walbran290b0c92020-02-03 16:37:14 +00002820 to_locked.vm->id, handle, memory_region->sender);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002821 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran290b0c92020-02-03 16:37:14 +00002822 }
2823
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002824 composite = ffa_memory_region_get_composite(memory_region, 0);
Andrew Walbran290b0c92020-02-03 16:37:14 +00002825
2826 /*
Andrew Walbranca808b12020-05-15 17:22:28 +01002827 * Validate that the reclaim transition is allowed for the given memory
2828 * region, forward the request to the TEE and then map the memory back
2829 * into the caller's stage-2 page table.
Andrew Walbran290b0c92020-02-03 16:37:14 +00002830 */
Andrew Walbran996d1d12020-05-27 14:08:43 +01002831 return ffa_tee_reclaim_check_update(
2832 to_locked, handle, composite->constituents,
Andrew Walbranca808b12020-05-15 17:22:28 +01002833 composite->constituent_count, memory_to_attributes,
2834 flags & FFA_MEM_RECLAIM_CLEAR, page_pool);
Andrew Walbran290b0c92020-02-03 16:37:14 +00002835}