blob: c95379cb3fbdf2a6c95831dfb63894e896a986b4 [file] [log] [blame]
Jose Marinho75509b42019-04-09 09:34:59 +01001/*
2 * Copyright 2019 The Hafnium Authors.
3 *
Andrew Walbrane959ec12020-06-17 15:01:09 +01004 * Use of this source code is governed by a BSD-style
5 * license that can be found in the LICENSE file or at
6 * https://opensource.org/licenses/BSD-3-Clause.
Jose Marinho75509b42019-04-09 09:34:59 +01007 */
8
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01009#include "hf/ffa_memory.h"
Andrew Walbran475c1452020-02-07 13:22:22 +000010
Federico Recanati4fd065d2021-12-13 20:06:23 +010011#include "hf/arch/mm.h"
Olivier Deprez112d2b52020-09-30 07:39:23 +020012#include "hf/arch/other_world.h"
Olivier Deprez55a189e2021-06-09 15:45:27 +020013#include "hf/arch/plat/ffa.h"
Andrew Walbran290b0c92020-02-03 16:37:14 +000014
Jose Marinho75509b42019-04-09 09:34:59 +010015#include "hf/api.h"
Daniel Boulbya2f8c662021-11-26 17:52:53 +000016#include "hf/assert.h"
Jose Marinho09b1db82019-08-08 09:16:59 +010017#include "hf/check.h"
Jose Marinho75509b42019-04-09 09:34:59 +010018#include "hf/dlog.h"
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010019#include "hf/ffa_internal.h"
J-Alves66652252022-07-06 09:49:51 +010020#include "hf/ffa_memory_internal.h"
Andrew Walbran475c1452020-02-07 13:22:22 +000021#include "hf/mpool.h"
Jose Marinho75509b42019-04-09 09:34:59 +010022#include "hf/std.h"
Andrew Scull3c257452019-11-26 13:32:50 +000023#include "hf/vm.h"
Jose Marinho75509b42019-04-09 09:34:59 +010024
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000025/**
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010026 * All access to members of a `struct ffa_memory_share_state` must be guarded
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000027 * by this lock.
28 */
29static struct spinlock share_states_lock_instance = SPINLOCK_INIT;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010030static struct ffa_memory_share_state share_states[MAX_MEM_SHARES];
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000031
32/**
J-Alves917d2f22020-10-30 18:39:30 +000033 * Extracts the index from a memory handle allocated by Hafnium's current world.
34 */
35uint64_t ffa_memory_handle_get_index(ffa_memory_handle_t handle)
36{
37 return handle & ~FFA_MEMORY_HANDLE_ALLOCATOR_MASK;
38}
39
40/**
Andrew Walbranca808b12020-05-15 17:22:28 +010041 * Initialises the next available `struct ffa_memory_share_state` and sets
42 * `share_state_ret` to a pointer to it. If `handle` is
43 * `FFA_MEMORY_HANDLE_INVALID` then allocates an appropriate handle, otherwise
44 * uses the provided handle which is assumed to be globally unique.
45 *
46 * Returns true on success or false if none are available.
47 */
J-Alves66652252022-07-06 09:49:51 +010048bool allocate_share_state(struct share_states_locked share_states,
49 uint32_t share_func,
50 struct ffa_memory_region *memory_region,
51 uint32_t fragment_length, ffa_memory_handle_t handle,
52 struct ffa_memory_share_state **share_state_ret)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000053{
Andrew Walbrana65a1322020-04-06 19:32:32 +010054 uint64_t i;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000055
Daniel Boulbya2f8c662021-11-26 17:52:53 +000056 assert(share_states.share_states != NULL);
57 assert(memory_region != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000058
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000059 for (i = 0; i < MAX_MEM_SHARES; ++i) {
Andrew Walbranca808b12020-05-15 17:22:28 +010060 if (share_states.share_states[i].share_func == 0) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000061 uint32_t j;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010062 struct ffa_memory_share_state *allocated_state =
Andrew Walbranca808b12020-05-15 17:22:28 +010063 &share_states.share_states[i];
64 struct ffa_composite_memory_region *composite =
65 ffa_memory_region_get_composite(memory_region,
66 0);
67
68 if (handle == FFA_MEMORY_HANDLE_INVALID) {
J-Alvesee68c542020-10-29 17:48:20 +000069 memory_region->handle =
Olivier Deprez55a189e2021-06-09 15:45:27 +020070 plat_ffa_memory_handle_make(i);
Andrew Walbranca808b12020-05-15 17:22:28 +010071 } else {
J-Alvesee68c542020-10-29 17:48:20 +000072 memory_region->handle = handle;
Andrew Walbranca808b12020-05-15 17:22:28 +010073 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000074 allocated_state->share_func = share_func;
75 allocated_state->memory_region = memory_region;
Andrew Walbranca808b12020-05-15 17:22:28 +010076 allocated_state->fragment_count = 1;
77 allocated_state->fragments[0] = composite->constituents;
78 allocated_state->fragment_constituent_counts[0] =
79 (fragment_length -
80 ffa_composite_constituent_offset(memory_region,
81 0)) /
82 sizeof(struct ffa_memory_region_constituent);
83 allocated_state->sending_complete = false;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000084 for (j = 0; j < MAX_MEM_SHARE_RECIPIENTS; ++j) {
Andrew Walbranca808b12020-05-15 17:22:28 +010085 allocated_state->retrieved_fragment_count[j] =
86 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000087 }
Andrew Walbranca808b12020-05-15 17:22:28 +010088 if (share_state_ret != NULL) {
89 *share_state_ret = allocated_state;
90 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000091 return true;
92 }
93 }
94
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000095 return false;
96}
97
98/** Locks the share states lock. */
99struct share_states_locked share_states_lock(void)
100{
101 sl_lock(&share_states_lock_instance);
102
103 return (struct share_states_locked){.share_states = share_states};
104}
105
106/** Unlocks the share states lock. */
J-Alves66652252022-07-06 09:49:51 +0100107void share_states_unlock(struct share_states_locked *share_states)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000108{
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000109 assert(share_states->share_states != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000110 share_states->share_states = NULL;
111 sl_unlock(&share_states_lock_instance);
112}
113
114/**
Andrew Walbranca808b12020-05-15 17:22:28 +0100115 * If the given handle is a valid handle for an allocated share state then
116 * initialises `share_state_ret` to point to the share state and returns true.
117 * Otherwise returns false.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000118 */
J-Alvesfdd29272022-07-19 13:16:31 +0100119bool get_share_state(struct share_states_locked share_states,
120 ffa_memory_handle_t handle,
121 struct ffa_memory_share_state **share_state_ret)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000122{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100123 struct ffa_memory_share_state *share_state;
J-Alves917d2f22020-10-30 18:39:30 +0000124 uint64_t index;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000125
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000126 assert(share_states.share_states != NULL);
127 assert(share_state_ret != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100128
129 /*
130 * First look for a share_state allocated by us, in which case the
131 * handle is based on the index.
132 */
Olivier Deprez55a189e2021-06-09 15:45:27 +0200133 if (plat_ffa_memory_handle_allocated_by_current_world(handle)) {
J-Alves917d2f22020-10-30 18:39:30 +0000134 index = ffa_memory_handle_get_index(handle);
Andrew Walbranca808b12020-05-15 17:22:28 +0100135 if (index < MAX_MEM_SHARES) {
136 share_state = &share_states.share_states[index];
137 if (share_state->share_func != 0) {
138 *share_state_ret = share_state;
139 return true;
140 }
141 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000142 }
143
Andrew Walbranca808b12020-05-15 17:22:28 +0100144 /* Fall back to a linear scan. */
145 for (index = 0; index < MAX_MEM_SHARES; ++index) {
146 share_state = &share_states.share_states[index];
J-Alvesee68c542020-10-29 17:48:20 +0000147 if (share_state->memory_region != NULL &&
148 share_state->memory_region->handle == handle &&
Andrew Walbranca808b12020-05-15 17:22:28 +0100149 share_state->share_func != 0) {
150 *share_state_ret = share_state;
151 return true;
152 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000153 }
154
Andrew Walbranca808b12020-05-15 17:22:28 +0100155 return false;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000156}
157
158/** Marks a share state as unallocated. */
J-Alvesfdd29272022-07-19 13:16:31 +0100159void share_state_free(struct share_states_locked share_states,
160 struct ffa_memory_share_state *share_state,
161 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000162{
Andrew Walbranca808b12020-05-15 17:22:28 +0100163 uint32_t i;
164
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000165 assert(share_states.share_states != NULL);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000166 share_state->share_func = 0;
Andrew Walbranca808b12020-05-15 17:22:28 +0100167 share_state->sending_complete = false;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000168 mpool_free(page_pool, share_state->memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +0100169 /*
170 * First fragment is part of the same page as the `memory_region`, so it
171 * doesn't need to be freed separately.
172 */
173 share_state->fragments[0] = NULL;
174 share_state->fragment_constituent_counts[0] = 0;
175 for (i = 1; i < share_state->fragment_count; ++i) {
176 mpool_free(page_pool, share_state->fragments[i]);
177 share_state->fragments[i] = NULL;
178 share_state->fragment_constituent_counts[i] = 0;
179 }
180 share_state->fragment_count = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000181 share_state->memory_region = NULL;
J-Alvesa9cd7e32022-07-01 13:49:33 +0100182 share_state->hypervisor_fragment_count = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000183}
184
Andrew Walbranca808b12020-05-15 17:22:28 +0100185/** Checks whether the given share state has been fully sent. */
J-Alvesfdd29272022-07-19 13:16:31 +0100186bool share_state_sending_complete(struct share_states_locked share_states,
187 struct ffa_memory_share_state *share_state)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000188{
Andrew Walbranca808b12020-05-15 17:22:28 +0100189 struct ffa_composite_memory_region *composite;
190 uint32_t expected_constituent_count;
191 uint32_t fragment_constituent_count_total = 0;
192 uint32_t i;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000193
Andrew Walbranca808b12020-05-15 17:22:28 +0100194 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000195 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100196
197 /*
198 * Share state must already be valid, or it's not possible to get hold
199 * of it.
200 */
201 CHECK(share_state->memory_region != NULL &&
202 share_state->share_func != 0);
203
204 composite =
205 ffa_memory_region_get_composite(share_state->memory_region, 0);
206 expected_constituent_count = composite->constituent_count;
207 for (i = 0; i < share_state->fragment_count; ++i) {
208 fragment_constituent_count_total +=
209 share_state->fragment_constituent_counts[i];
210 }
211 dlog_verbose(
212 "Checking completion: constituent count %d/%d from %d "
213 "fragments.\n",
214 fragment_constituent_count_total, expected_constituent_count,
215 share_state->fragment_count);
216
217 return fragment_constituent_count_total == expected_constituent_count;
218}
219
220/**
221 * Calculates the offset of the next fragment expected for the given share
222 * state.
223 */
J-Alvesfdd29272022-07-19 13:16:31 +0100224uint32_t share_state_next_fragment_offset(
Andrew Walbranca808b12020-05-15 17:22:28 +0100225 struct share_states_locked share_states,
226 struct ffa_memory_share_state *share_state)
227{
228 uint32_t next_fragment_offset;
229 uint32_t i;
230
231 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +0000232 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +0100233
234 next_fragment_offset =
235 ffa_composite_constituent_offset(share_state->memory_region, 0);
236 for (i = 0; i < share_state->fragment_count; ++i) {
237 next_fragment_offset +=
238 share_state->fragment_constituent_counts[i] *
239 sizeof(struct ffa_memory_region_constituent);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000240 }
241
Andrew Walbranca808b12020-05-15 17:22:28 +0100242 return next_fragment_offset;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000243}
244
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100245static void dump_memory_region(struct ffa_memory_region *memory_region)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000246{
247 uint32_t i;
248
249 if (LOG_LEVEL < LOG_LEVEL_VERBOSE) {
250 return;
251 }
252
Olivier Deprez935e1b12020-12-22 18:01:29 +0100253 dlog("from VM %#x, attributes %#x, flags %#x, tag %u, to "
Olivier Deprezf92e5d42020-11-13 16:00:54 +0100254 "%u "
Andrew Walbrana65a1322020-04-06 19:32:32 +0100255 "recipients [",
256 memory_region->sender, memory_region->attributes,
Olivier Deprez935e1b12020-12-22 18:01:29 +0100257 memory_region->flags, memory_region->tag,
Andrew Walbrana65a1322020-04-06 19:32:32 +0100258 memory_region->receiver_count);
259 for (i = 0; i < memory_region->receiver_count; ++i) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000260 if (i != 0) {
261 dlog(", ");
262 }
Olivier Deprezf92e5d42020-11-13 16:00:54 +0100263 dlog("VM %#x: %#x (offset %u)",
Andrew Walbrana65a1322020-04-06 19:32:32 +0100264 memory_region->receivers[i].receiver_permissions.receiver,
265 memory_region->receivers[i]
266 .receiver_permissions.permissions,
267 memory_region->receivers[i]
268 .composite_memory_region_offset);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000269 }
270 dlog("]");
271}
272
J-Alves66652252022-07-06 09:49:51 +0100273void dump_share_states(void)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000274{
275 uint32_t i;
276
277 if (LOG_LEVEL < LOG_LEVEL_VERBOSE) {
278 return;
279 }
280
281 dlog("Current share states:\n");
282 sl_lock(&share_states_lock_instance);
283 for (i = 0; i < MAX_MEM_SHARES; ++i) {
284 if (share_states[i].share_func != 0) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000285 switch (share_states[i].share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100286 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000287 dlog("SHARE");
288 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100289 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000290 dlog("LEND");
291 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100292 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000293 dlog("DONATE");
294 break;
295 default:
296 dlog("invalid share_func %#x",
297 share_states[i].share_func);
298 }
Olivier Deprez935e1b12020-12-22 18:01:29 +0100299 dlog(" %#x (", share_states[i].memory_region->handle);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000300 dump_memory_region(share_states[i].memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +0100301 if (share_states[i].sending_complete) {
302 dlog("): fully sent");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000303 } else {
Andrew Walbranca808b12020-05-15 17:22:28 +0100304 dlog("): partially sent");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000305 }
J-Alves2a0d2882020-10-29 14:49:50 +0000306 dlog(" with %d fragments, %d retrieved, "
307 " sender's original mode: %#x\n",
Andrew Walbranca808b12020-05-15 17:22:28 +0100308 share_states[i].fragment_count,
J-Alves2a0d2882020-10-29 14:49:50 +0000309 share_states[i].retrieved_fragment_count[0],
310 share_states[i].sender_orig_mode);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000311 }
312 }
313 sl_unlock(&share_states_lock_instance);
314}
315
Andrew Walbran475c1452020-02-07 13:22:22 +0000316/* TODO: Add device attributes: GRE, cacheability, shareability. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100317static inline uint32_t ffa_memory_permissions_to_mode(
J-Alves7cd5eb32020-10-16 19:06:10 +0100318 ffa_memory_access_permissions_t permissions, uint32_t default_mode)
Andrew Walbran475c1452020-02-07 13:22:22 +0000319{
320 uint32_t mode = 0;
321
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100322 switch (ffa_get_data_access_attr(permissions)) {
323 case FFA_DATA_ACCESS_RO:
Andrew Walbran475c1452020-02-07 13:22:22 +0000324 mode = MM_MODE_R;
325 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100326 case FFA_DATA_ACCESS_RW:
Andrew Walbran475c1452020-02-07 13:22:22 +0000327 mode = MM_MODE_R | MM_MODE_W;
328 break;
J-Alves7cd5eb32020-10-16 19:06:10 +0100329 case FFA_DATA_ACCESS_NOT_SPECIFIED:
330 mode = (default_mode & (MM_MODE_R | MM_MODE_W));
331 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100332 case FFA_DATA_ACCESS_RESERVED:
333 panic("Tried to convert FFA_DATA_ACCESS_RESERVED.");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100334 }
335
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100336 switch (ffa_get_instruction_access_attr(permissions)) {
337 case FFA_INSTRUCTION_ACCESS_NX:
Andrew Walbran475c1452020-02-07 13:22:22 +0000338 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100339 case FFA_INSTRUCTION_ACCESS_X:
Andrew Walbrana65a1322020-04-06 19:32:32 +0100340 mode |= MM_MODE_X;
341 break;
J-Alves7cd5eb32020-10-16 19:06:10 +0100342 case FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED:
343 mode |= (default_mode & MM_MODE_X);
344 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100345 case FFA_INSTRUCTION_ACCESS_RESERVED:
346 panic("Tried to convert FFA_INSTRUCTION_ACCESS_RESVERVED.");
Andrew Walbran475c1452020-02-07 13:22:22 +0000347 }
348
349 return mode;
350}
351
Jose Marinho75509b42019-04-09 09:34:59 +0100352/**
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000353 * Get the current mode in the stage-2 page table of the given vm of all the
354 * pages in the given constituents, if they all have the same mode, or return
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100355 * an appropriate FF-A error if not.
Jose Marinho75509b42019-04-09 09:34:59 +0100356 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100357static struct ffa_value constituents_get_mode(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000358 struct vm_locked vm, uint32_t *orig_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100359 struct ffa_memory_region_constituent **fragments,
360 const uint32_t *fragment_constituent_counts, uint32_t fragment_count)
Jose Marinho75509b42019-04-09 09:34:59 +0100361{
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100362 uint32_t i;
Andrew Walbranca808b12020-05-15 17:22:28 +0100363 uint32_t j;
Jose Marinho75509b42019-04-09 09:34:59 +0100364
Andrew Walbranca808b12020-05-15 17:22:28 +0100365 if (fragment_count == 0 || fragment_constituent_counts[0] == 0) {
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100366 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000367 * Fail if there are no constituents. Otherwise we would get an
368 * uninitialised *orig_mode.
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100369 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100370 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho75509b42019-04-09 09:34:59 +0100371 }
372
Andrew Walbranca808b12020-05-15 17:22:28 +0100373 for (i = 0; i < fragment_count; ++i) {
374 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
375 ipaddr_t begin = ipa_init(fragments[i][j].address);
376 size_t size = fragments[i][j].page_count * PAGE_SIZE;
377 ipaddr_t end = ipa_add(begin, size);
378 uint32_t current_mode;
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100379
Andrew Walbranca808b12020-05-15 17:22:28 +0100380 /* Fail if addresses are not page-aligned. */
381 if (!is_aligned(ipa_addr(begin), PAGE_SIZE) ||
382 !is_aligned(ipa_addr(end), PAGE_SIZE)) {
383 return ffa_error(FFA_INVALID_PARAMETERS);
384 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100385
Andrew Walbranca808b12020-05-15 17:22:28 +0100386 /*
387 * Ensure that this constituent memory range is all
388 * mapped with the same mode.
389 */
Raghu Krishnamurthy785d52f2021-02-13 00:02:40 -0800390 if (!vm_mem_get_mode(vm, begin, end, &current_mode)) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100391 return ffa_error(FFA_DENIED);
392 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100393
Andrew Walbranca808b12020-05-15 17:22:28 +0100394 /*
395 * Ensure that all constituents are mapped with the same
396 * mode.
397 */
398 if (i == 0) {
399 *orig_mode = current_mode;
400 } else if (current_mode != *orig_mode) {
401 dlog_verbose(
402 "Expected mode %#x but was %#x for %d "
403 "pages at %#x.\n",
404 *orig_mode, current_mode,
405 fragments[i][j].page_count,
406 ipa_addr(begin));
407 return ffa_error(FFA_DENIED);
408 }
Jose Marinho7fbbf2e2019-08-05 13:19:58 +0100409 }
Jose Marinho75509b42019-04-09 09:34:59 +0100410 }
411
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100412 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000413}
414
415/**
416 * Verify that all pages have the same mode, that the starting mode
417 * constitutes a valid state and obtain the next mode to apply
418 * to the sending VM.
419 *
420 * Returns:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100421 * 1) FFA_DENIED if a state transition was not found;
422 * 2) FFA_DENIED if the pages being shared do not have the same mode within
Andrew Walbrana65a1322020-04-06 19:32:32 +0100423 * the <from> VM;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100424 * 3) FFA_INVALID_PARAMETERS if the beginning and end IPAs are not page
Andrew Walbrana65a1322020-04-06 19:32:32 +0100425 * aligned;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100426 * 4) FFA_INVALID_PARAMETERS if the requested share type was not handled.
427 * Or FFA_SUCCESS on success.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000428 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100429static struct ffa_value ffa_send_check_transition(
Andrew Walbrana65a1322020-04-06 19:32:32 +0100430 struct vm_locked from, uint32_t share_func,
J-Alves363f5722022-04-25 17:37:37 +0100431 struct ffa_memory_access *receivers, uint32_t receivers_count,
432 uint32_t *orig_from_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100433 struct ffa_memory_region_constituent **fragments,
434 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
435 uint32_t *from_mode)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000436{
437 const uint32_t state_mask =
438 MM_MODE_INVALID | MM_MODE_UNOWNED | MM_MODE_SHARED;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100439 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000440
Andrew Walbranca808b12020-05-15 17:22:28 +0100441 ret = constituents_get_mode(from, orig_from_mode, fragments,
442 fragment_constituent_counts,
443 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100444 if (ret.func != FFA_SUCCESS_32) {
Olivier Depreze7eb1682022-03-16 17:09:03 +0100445 dlog_verbose("Inconsistent modes.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100446 return ret;
Andrew Scullb5f49e02019-10-02 13:20:47 +0100447 }
448
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000449 /* Ensure the address range is normal memory and not a device. */
450 if (*orig_from_mode & MM_MODE_D) {
451 dlog_verbose("Can't share device memory (mode is %#x).\n",
452 *orig_from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100453 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000454 }
455
456 /*
457 * Ensure the sender is the owner and has exclusive access to the
458 * memory.
459 */
460 if ((*orig_from_mode & state_mask) != 0) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100461 return ffa_error(FFA_DENIED);
Andrew Walbrana65a1322020-04-06 19:32:32 +0100462 }
463
J-Alves363f5722022-04-25 17:37:37 +0100464 assert(receivers != NULL && receivers_count > 0U);
J-Alves7cd5eb32020-10-16 19:06:10 +0100465
J-Alves363f5722022-04-25 17:37:37 +0100466 for (uint32_t i = 0U; i < receivers_count; i++) {
467 ffa_memory_access_permissions_t permissions =
468 receivers[i].receiver_permissions.permissions;
469 uint32_t required_from_mode = ffa_memory_permissions_to_mode(
470 permissions, *orig_from_mode);
471
472 if ((*orig_from_mode & required_from_mode) !=
473 required_from_mode) {
474 dlog_verbose(
475 "Sender tried to send memory with permissions "
476 "which "
477 "required mode %#x but only had %#x itself.\n",
478 required_from_mode, *orig_from_mode);
479 return ffa_error(FFA_DENIED);
480 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000481 }
482
483 /* Find the appropriate new mode. */
484 *from_mode = ~state_mask & *orig_from_mode;
Andrew Walbrane7ad3c02019-12-24 17:03:04 +0000485 switch (share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100486 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000487 *from_mode |= MM_MODE_INVALID | MM_MODE_UNOWNED;
Jose Marinho75509b42019-04-09 09:34:59 +0100488 break;
489
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100490 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000491 *from_mode |= MM_MODE_INVALID;
Andrew Walbran648fc3e2019-10-22 16:23:05 +0100492 break;
493
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100494 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000495 *from_mode |= MM_MODE_SHARED;
Jose Marinho56c25732019-05-20 09:48:53 +0100496 break;
497
Jose Marinho75509b42019-04-09 09:34:59 +0100498 default:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100499 return ffa_error(FFA_INVALID_PARAMETERS);
Jose Marinho75509b42019-04-09 09:34:59 +0100500 }
501
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100502 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000503}
504
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100505static struct ffa_value ffa_relinquish_check_transition(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000506 struct vm_locked from, uint32_t *orig_from_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100507 struct ffa_memory_region_constituent **fragments,
508 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
509 uint32_t *from_mode)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000510{
511 const uint32_t state_mask =
512 MM_MODE_INVALID | MM_MODE_UNOWNED | MM_MODE_SHARED;
513 uint32_t orig_from_state;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100514 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000515
Andrew Walbranca808b12020-05-15 17:22:28 +0100516 ret = constituents_get_mode(from, orig_from_mode, fragments,
517 fragment_constituent_counts,
518 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100519 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbrana65a1322020-04-06 19:32:32 +0100520 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000521 }
522
523 /* Ensure the address range is normal memory and not a device. */
524 if (*orig_from_mode & MM_MODE_D) {
525 dlog_verbose("Can't relinquish device memory (mode is %#x).\n",
526 *orig_from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100527 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000528 }
529
530 /*
531 * Ensure the relinquishing VM is not the owner but has access to the
532 * memory.
533 */
534 orig_from_state = *orig_from_mode & state_mask;
535 if ((orig_from_state & ~MM_MODE_SHARED) != MM_MODE_UNOWNED) {
536 dlog_verbose(
537 "Tried to relinquish memory in state %#x (masked %#x "
Andrew Walbranca808b12020-05-15 17:22:28 +0100538 "but should be %#x).\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000539 *orig_from_mode, orig_from_state, MM_MODE_UNOWNED);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100540 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000541 }
542
543 /* Find the appropriate new mode. */
544 *from_mode = (~state_mask & *orig_from_mode) | MM_MODE_UNMAPPED_MASK;
545
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100546 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000547}
548
549/**
550 * Verify that all pages have the same mode, that the starting mode
551 * constitutes a valid state and obtain the next mode to apply
552 * to the retrieving VM.
553 *
554 * Returns:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100555 * 1) FFA_DENIED if a state transition was not found;
556 * 2) FFA_DENIED if the pages being shared do not have the same mode within
Andrew Walbrana65a1322020-04-06 19:32:32 +0100557 * the <to> VM;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100558 * 3) FFA_INVALID_PARAMETERS if the beginning and end IPAs are not page
Andrew Walbrana65a1322020-04-06 19:32:32 +0100559 * aligned;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100560 * 4) FFA_INVALID_PARAMETERS if the requested share type was not handled.
561 * Or FFA_SUCCESS on success.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000562 */
J-Alvesfc19b372022-07-06 12:17:35 +0100563struct ffa_value ffa_retrieve_check_transition(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000564 struct vm_locked to, uint32_t share_func,
Andrew Walbranca808b12020-05-15 17:22:28 +0100565 struct ffa_memory_region_constituent **fragments,
566 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
567 uint32_t memory_to_attributes, uint32_t *to_mode)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000568{
569 uint32_t orig_to_mode;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100570 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000571
Andrew Walbranca808b12020-05-15 17:22:28 +0100572 ret = constituents_get_mode(to, &orig_to_mode, fragments,
573 fragment_constituent_counts,
574 fragment_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100575 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100576 dlog_verbose("Inconsistent modes.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100577 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000578 }
579
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100580 if (share_func == FFA_MEM_RECLAIM_32) {
J-Alves9256f162021-12-09 13:18:43 +0000581 /*
582 * If the original ffa memory send call has been processed
583 * successfully, it is expected the orig_to_mode would overlay
584 * with `state_mask`, as a result of the function
585 * `ffa_send_check_transition`.
586 */
Daniel Boulby9133dad2022-04-25 14:38:44 +0100587 assert((orig_to_mode & (MM_MODE_INVALID | MM_MODE_UNOWNED |
588 MM_MODE_SHARED)) != 0U);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000589 } else {
590 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +0100591 * If the retriever is from virtual FF-A instance:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000592 * Ensure the retriever has the expected state. We don't care
593 * about the MM_MODE_SHARED bit; either with or without it set
594 * are both valid representations of the !O-NA state.
595 */
J-Alvesa9cd7e32022-07-01 13:49:33 +0100596 if (vm_id_is_current_world(to.vm->id) &&
597 to.vm->id != HF_PRIMARY_VM_ID &&
598 (orig_to_mode & MM_MODE_UNMAPPED_MASK) !=
599 MM_MODE_UNMAPPED_MASK) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100600 return ffa_error(FFA_DENIED);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000601 }
602 }
603
604 /* Find the appropriate new mode. */
605 *to_mode = memory_to_attributes;
606 switch (share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100607 case FFA_MEM_DONATE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000608 *to_mode |= 0;
609 break;
610
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100611 case FFA_MEM_LEND_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000612 *to_mode |= MM_MODE_UNOWNED;
613 break;
614
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100615 case FFA_MEM_SHARE_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000616 *to_mode |= MM_MODE_UNOWNED | MM_MODE_SHARED;
617 break;
618
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100619 case FFA_MEM_RECLAIM_32:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000620 *to_mode |= 0;
621 break;
622
623 default:
Andrew Walbranca808b12020-05-15 17:22:28 +0100624 dlog_error("Invalid share_func %#x.\n", share_func);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100625 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000626 }
627
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100628 return (struct ffa_value){.func = FFA_SUCCESS_32};
Jose Marinho75509b42019-04-09 09:34:59 +0100629}
Jose Marinho09b1db82019-08-08 09:16:59 +0100630
631/**
632 * Updates a VM's page table such that the given set of physical address ranges
633 * are mapped in the address space at the corresponding address ranges, in the
634 * mode provided.
635 *
636 * If commit is false, the page tables will be allocated from the mpool but no
637 * mappings will actually be updated. This function must always be called first
638 * with commit false to check that it will succeed before calling with commit
639 * true, to avoid leaving the page table in a half-updated state. To make a
640 * series of changes atomically you can call them all with commit false before
641 * calling them all with commit true.
642 *
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -0700643 * vm_ptable_defrag should always be called after a series of page table
644 * updates, whether they succeed or fail.
Jose Marinho09b1db82019-08-08 09:16:59 +0100645 *
646 * Returns true on success, or false if the update failed and no changes were
647 * made to memory mappings.
648 */
J-Alves66652252022-07-06 09:49:51 +0100649bool ffa_region_group_identity_map(
Andrew Walbranf4b51af2020-02-03 14:44:54 +0000650 struct vm_locked vm_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +0100651 struct ffa_memory_region_constituent **fragments,
652 const uint32_t *fragment_constituent_counts, uint32_t fragment_count,
Daniel Boulby4dd3f532021-09-21 09:57:08 +0100653 uint32_t mode, struct mpool *ppool, bool commit)
Jose Marinho09b1db82019-08-08 09:16:59 +0100654{
Andrew Walbranca808b12020-05-15 17:22:28 +0100655 uint32_t i;
656 uint32_t j;
Jose Marinho09b1db82019-08-08 09:16:59 +0100657
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -0700658 if (vm_locked.vm->el0_partition) {
659 mode |= MM_MODE_USER | MM_MODE_NG;
660 }
661
Andrew Walbranca808b12020-05-15 17:22:28 +0100662 /* Iterate over the memory region constituents within each fragment. */
663 for (i = 0; i < fragment_count; ++i) {
664 for (j = 0; j < fragment_constituent_counts[i]; ++j) {
665 size_t size = fragments[i][j].page_count * PAGE_SIZE;
666 paddr_t pa_begin =
667 pa_from_ipa(ipa_init(fragments[i][j].address));
668 paddr_t pa_end = pa_add(pa_begin, size);
Jens Wiklander4f1880c2022-10-19 17:00:14 +0200669 uint32_t pa_bits =
670 arch_mm_get_pa_bits(arch_mm_get_pa_range());
Federico Recanati4fd065d2021-12-13 20:06:23 +0100671
672 /*
673 * Ensure the requested region falls into system's PA
674 * range.
675 */
Jens Wiklander4f1880c2022-10-19 17:00:14 +0200676 if (((pa_addr(pa_begin) >> pa_bits) > 0) ||
677 ((pa_addr(pa_end) >> pa_bits) > 0)) {
Federico Recanati4fd065d2021-12-13 20:06:23 +0100678 dlog_error("Region is outside of PA Range\n");
679 return false;
680 }
Andrew Walbranca808b12020-05-15 17:22:28 +0100681
682 if (commit) {
683 vm_identity_commit(vm_locked, pa_begin, pa_end,
684 mode, ppool, NULL);
685 } else if (!vm_identity_prepare(vm_locked, pa_begin,
686 pa_end, mode, ppool)) {
687 return false;
688 }
Jose Marinho09b1db82019-08-08 09:16:59 +0100689 }
690 }
691
692 return true;
693}
694
695/**
696 * Clears a region of physical memory by overwriting it with zeros. The data is
697 * flushed from the cache so the memory has been cleared across the system.
698 */
J-Alves7db32002021-12-14 14:44:50 +0000699static bool clear_memory(paddr_t begin, paddr_t end, struct mpool *ppool,
700 uint32_t extra_mode_attributes)
Jose Marinho09b1db82019-08-08 09:16:59 +0100701{
702 /*
Fuad Tabbaed294af2019-12-20 10:43:01 +0000703 * TODO: change this to a CPU local single page window rather than a
Jose Marinho09b1db82019-08-08 09:16:59 +0100704 * global mapping of the whole range. Such an approach will limit
705 * the changes to stage-1 tables and will allow only local
706 * invalidation.
707 */
708 bool ret;
709 struct mm_stage1_locked stage1_locked = mm_lock_stage1();
J-Alves7db32002021-12-14 14:44:50 +0000710 void *ptr = mm_identity_map(stage1_locked, begin, end,
711 MM_MODE_W | (extra_mode_attributes &
712 plat_ffa_other_world_mode()),
713 ppool);
Jose Marinho09b1db82019-08-08 09:16:59 +0100714 size_t size = pa_difference(begin, end);
715
716 if (!ptr) {
Jose Marinho09b1db82019-08-08 09:16:59 +0100717 goto fail;
718 }
719
720 memset_s(ptr, size, 0, size);
721 arch_mm_flush_dcache(ptr, size);
722 mm_unmap(stage1_locked, begin, end, ppool);
723
724 ret = true;
725 goto out;
726
727fail:
728 ret = false;
729
730out:
731 mm_unlock_stage1(&stage1_locked);
732
733 return ret;
734}
735
736/**
737 * Clears a region of physical memory by overwriting it with zeros. The data is
738 * flushed from the cache so the memory has been cleared across the system.
739 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100740static bool ffa_clear_memory_constituents(
J-Alves7db32002021-12-14 14:44:50 +0000741 uint32_t security_state_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +0100742 struct ffa_memory_region_constituent **fragments,
743 const uint32_t *fragment_constituent_counts, uint32_t fragment_count,
744 struct mpool *page_pool)
Jose Marinho09b1db82019-08-08 09:16:59 +0100745{
746 struct mpool local_page_pool;
Andrew Walbranca808b12020-05-15 17:22:28 +0100747 uint32_t i;
Jose Marinho09b1db82019-08-08 09:16:59 +0100748 bool ret = false;
749
750 /*
751 * Create a local pool so any freed memory can't be used by another
752 * thread. This is to ensure each constituent that is mapped can be
753 * unmapped again afterwards.
754 */
Andrew Walbran475c1452020-02-07 13:22:22 +0000755 mpool_init_with_fallback(&local_page_pool, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +0100756
Andrew Walbranca808b12020-05-15 17:22:28 +0100757 /* Iterate over the memory region constituents within each fragment. */
758 for (i = 0; i < fragment_count; ++i) {
759 uint32_t j;
Jose Marinho09b1db82019-08-08 09:16:59 +0100760
Andrew Walbranca808b12020-05-15 17:22:28 +0100761 for (j = 0; j < fragment_constituent_counts[j]; ++j) {
762 size_t size = fragments[i][j].page_count * PAGE_SIZE;
763 paddr_t begin =
764 pa_from_ipa(ipa_init(fragments[i][j].address));
765 paddr_t end = pa_add(begin, size);
766
J-Alves7db32002021-12-14 14:44:50 +0000767 if (!clear_memory(begin, end, &local_page_pool,
768 security_state_mode)) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100769 /*
770 * api_clear_memory will defrag on failure, so
771 * no need to do it here.
772 */
773 goto out;
774 }
Jose Marinho09b1db82019-08-08 09:16:59 +0100775 }
776 }
777
Jose Marinho09b1db82019-08-08 09:16:59 +0100778 ret = true;
779
780out:
781 mpool_fini(&local_page_pool);
782 return ret;
783}
784
785/**
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000786 * Validates and prepares memory to be sent from the calling VM to another.
Jose Marinho09b1db82019-08-08 09:16:59 +0100787 *
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000788 * This function requires the calling context to hold the <from> VM lock.
Jose Marinho09b1db82019-08-08 09:16:59 +0100789 *
790 * Returns:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000791 * In case of error, one of the following values is returned:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100792 * 1) FFA_INVALID_PARAMETERS - The endpoint provided parameters were
Jose Marinho09b1db82019-08-08 09:16:59 +0100793 * erroneous;
Andrew Walbranf07f04d2020-05-01 18:09:00 +0100794 * 2) FFA_NO_MEMORY - Hafnium did not have sufficient memory to complete the
795 * request.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100796 * 3) FFA_DENIED - The sender doesn't have sufficient access to send the
Andrew Walbrana65a1322020-04-06 19:32:32 +0100797 * memory with the given permissions.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100798 * Success is indicated by FFA_SUCCESS.
Jose Marinho09b1db82019-08-08 09:16:59 +0100799 */
J-Alves66652252022-07-06 09:49:51 +0100800struct ffa_value ffa_send_check_update(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000801 struct vm_locked from_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +0100802 struct ffa_memory_region_constituent **fragments,
803 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
J-Alves363f5722022-04-25 17:37:37 +0100804 uint32_t share_func, struct ffa_memory_access *receivers,
805 uint32_t receivers_count, struct mpool *page_pool, bool clear,
806 uint32_t *orig_from_mode_ret)
Jose Marinho09b1db82019-08-08 09:16:59 +0100807{
Andrew Walbranca808b12020-05-15 17:22:28 +0100808 uint32_t i;
Jose Marinho09b1db82019-08-08 09:16:59 +0100809 uint32_t orig_from_mode;
810 uint32_t from_mode;
Jose Marinho09b1db82019-08-08 09:16:59 +0100811 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100812 struct ffa_value ret;
Jose Marinho09b1db82019-08-08 09:16:59 +0100813
814 /*
Andrew Walbrana65a1322020-04-06 19:32:32 +0100815 * Make sure constituents are properly aligned to a 64-bit boundary. If
816 * not we would get alignment faults trying to read (64-bit) values.
Jose Marinho09b1db82019-08-08 09:16:59 +0100817 */
Andrew Walbranca808b12020-05-15 17:22:28 +0100818 for (i = 0; i < fragment_count; ++i) {
819 if (!is_aligned(fragments[i], 8)) {
820 dlog_verbose("Constituents not aligned.\n");
821 return ffa_error(FFA_INVALID_PARAMETERS);
822 }
Jose Marinho09b1db82019-08-08 09:16:59 +0100823 }
824
825 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000826 * Check if the state transition is lawful for the sender, ensure that
827 * all constituents of a memory region being shared are at the same
828 * state.
Jose Marinho09b1db82019-08-08 09:16:59 +0100829 */
J-Alves363f5722022-04-25 17:37:37 +0100830 ret = ffa_send_check_transition(from_locked, share_func, receivers,
831 receivers_count, &orig_from_mode,
832 fragments, fragment_constituent_counts,
Andrew Walbranca808b12020-05-15 17:22:28 +0100833 fragment_count, &from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100834 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100835 dlog_verbose("Invalid transition for send.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100836 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +0100837 }
838
Andrew Walbran37c574e2020-06-03 11:45:46 +0100839 if (orig_from_mode_ret != NULL) {
840 *orig_from_mode_ret = orig_from_mode;
841 }
842
Jose Marinho09b1db82019-08-08 09:16:59 +0100843 /*
844 * Create a local pool so any freed memory can't be used by another
845 * thread. This is to ensure the original mapping can be restored if the
846 * clear fails.
847 */
Andrew Walbran475c1452020-02-07 13:22:22 +0000848 mpool_init_with_fallback(&local_page_pool, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +0100849
850 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000851 * First reserve all required memory for the new page table entries
852 * without committing, to make sure the entire operation will succeed
853 * without exhausting the page pool.
Jose Marinho09b1db82019-08-08 09:16:59 +0100854 */
Andrew Walbranca808b12020-05-15 17:22:28 +0100855 if (!ffa_region_group_identity_map(
856 from_locked, fragments, fragment_constituent_counts,
857 fragment_count, from_mode, page_pool, false)) {
Jose Marinho09b1db82019-08-08 09:16:59 +0100858 /* TODO: partial defrag of failed range. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100859 ret = ffa_error(FFA_NO_MEMORY);
Jose Marinho09b1db82019-08-08 09:16:59 +0100860 goto out;
861 }
862
863 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000864 * Update the mapping for the sender. This won't allocate because the
865 * transaction was already prepared above, but may free pages in the
866 * case that a whole block is being unmapped that was previously
867 * partially mapped.
Jose Marinho09b1db82019-08-08 09:16:59 +0100868 */
Andrew Walbranca808b12020-05-15 17:22:28 +0100869 CHECK(ffa_region_group_identity_map(
870 from_locked, fragments, fragment_constituent_counts,
871 fragment_count, from_mode, &local_page_pool, true));
Jose Marinho09b1db82019-08-08 09:16:59 +0100872
873 /* Clear the memory so no VM or device can see the previous contents. */
J-Alves7db32002021-12-14 14:44:50 +0000874 if (clear &&
875 !ffa_clear_memory_constituents(
876 plat_ffa_owner_world_mode(from_locked.vm->id), fragments,
877 fragment_constituent_counts, fragment_count, page_pool)) {
Jose Marinho09b1db82019-08-08 09:16:59 +0100878 /*
879 * On failure, roll back by returning memory to the sender. This
880 * may allocate pages which were previously freed into
881 * `local_page_pool` by the call above, but will never allocate
882 * more pages than that so can never fail.
883 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100884 CHECK(ffa_region_group_identity_map(
Andrew Walbranca808b12020-05-15 17:22:28 +0100885 from_locked, fragments, fragment_constituent_counts,
886 fragment_count, orig_from_mode, &local_page_pool,
887 true));
Jose Marinho09b1db82019-08-08 09:16:59 +0100888
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100889 ret = ffa_error(FFA_NO_MEMORY);
Jose Marinho09b1db82019-08-08 09:16:59 +0100890 goto out;
891 }
892
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100893 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000894
895out:
896 mpool_fini(&local_page_pool);
897
898 /*
899 * Tidy up the page table by reclaiming failed mappings (if there was an
900 * error) or merging entries into blocks where possible (on success).
901 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -0700902 vm_ptable_defrag(from_locked, page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000903
904 return ret;
905}
906
907/**
908 * Validates and maps memory shared from one VM to another.
909 *
910 * This function requires the calling context to hold the <to> lock.
911 *
912 * Returns:
913 * In case of error, one of the following values is returned:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100914 * 1) FFA_INVALID_PARAMETERS - The endpoint provided parameters were
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000915 * erroneous;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100916 * 2) FFA_NO_MEMORY - Hafnium did not have sufficient memory to complete
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000917 * the request.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100918 * Success is indicated by FFA_SUCCESS.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000919 */
J-Alvesb5084cf2022-07-06 14:20:12 +0100920struct ffa_value ffa_retrieve_check_update(
J-Alves7db32002021-12-14 14:44:50 +0000921 struct vm_locked to_locked, ffa_vm_id_t from_id,
Andrew Walbranca808b12020-05-15 17:22:28 +0100922 struct ffa_memory_region_constituent **fragments,
923 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
924 uint32_t memory_to_attributes, uint32_t share_func, bool clear,
925 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000926{
Andrew Walbranca808b12020-05-15 17:22:28 +0100927 uint32_t i;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000928 uint32_t to_mode;
929 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100930 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000931
932 /*
Andrew Walbranca808b12020-05-15 17:22:28 +0100933 * Make sure constituents are properly aligned to a 64-bit boundary. If
934 * not we would get alignment faults trying to read (64-bit) values.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000935 */
Andrew Walbranca808b12020-05-15 17:22:28 +0100936 for (i = 0; i < fragment_count; ++i) {
937 if (!is_aligned(fragments[i], 8)) {
J-Alvesb5084cf2022-07-06 14:20:12 +0100938 dlog_verbose("Fragment not properly aligned.\n");
Andrew Walbranca808b12020-05-15 17:22:28 +0100939 return ffa_error(FFA_INVALID_PARAMETERS);
940 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000941 }
942
943 /*
944 * Check if the state transition is lawful for the recipient, and ensure
945 * that all constituents of the memory region being retrieved are at the
946 * same state.
947 */
Andrew Walbranca808b12020-05-15 17:22:28 +0100948 ret = ffa_retrieve_check_transition(
949 to_locked, share_func, fragments, fragment_constituent_counts,
950 fragment_count, memory_to_attributes, &to_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100951 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +0100952 dlog_verbose("Invalid transition for retrieve.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +0100953 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000954 }
955
956 /*
957 * Create a local pool so any freed memory can't be used by another
958 * thread. This is to ensure the original mapping can be restored if the
959 * clear fails.
960 */
961 mpool_init_with_fallback(&local_page_pool, page_pool);
962
963 /*
964 * First reserve all required memory for the new page table entries in
965 * the recipient page tables without committing, to make sure the entire
966 * operation will succeed without exhausting the page pool.
967 */
Andrew Walbranca808b12020-05-15 17:22:28 +0100968 if (!ffa_region_group_identity_map(
969 to_locked, fragments, fragment_constituent_counts,
970 fragment_count, to_mode, page_pool, false)) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000971 /* TODO: partial defrag of failed range. */
972 dlog_verbose(
973 "Insufficient memory to update recipient page "
974 "table.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100975 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000976 goto out;
977 }
978
979 /* Clear the memory so no VM or device can see the previous contents. */
J-Alves7db32002021-12-14 14:44:50 +0000980 if (clear &&
981 !ffa_clear_memory_constituents(
982 plat_ffa_owner_world_mode(from_id), fragments,
983 fragment_constituent_counts, fragment_count, page_pool)) {
J-Alvesb5084cf2022-07-06 14:20:12 +0100984 dlog_verbose("Couldn't clear constituents.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100985 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000986 goto out;
987 }
988
Jose Marinho09b1db82019-08-08 09:16:59 +0100989 /*
990 * Complete the transfer by mapping the memory into the recipient. This
991 * won't allocate because the transaction was already prepared above, so
992 * it doesn't need to use the `local_page_pool`.
993 */
Andrew Walbranca808b12020-05-15 17:22:28 +0100994 CHECK(ffa_region_group_identity_map(
995 to_locked, fragments, fragment_constituent_counts,
996 fragment_count, to_mode, page_pool, true));
Jose Marinho09b1db82019-08-08 09:16:59 +0100997
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100998 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Jose Marinho09b1db82019-08-08 09:16:59 +0100999
1000out:
1001 mpool_fini(&local_page_pool);
1002
1003 /*
Andrew Walbranf07f04d2020-05-01 18:09:00 +01001004 * Tidy up the page table by reclaiming failed mappings (if there was an
1005 * error) or merging entries into blocks where possible (on success).
Jose Marinho09b1db82019-08-08 09:16:59 +01001006 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001007 vm_ptable_defrag(to_locked, page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001008
1009 return ret;
1010}
1011
Andrew Walbran996d1d12020-05-27 14:08:43 +01001012static struct ffa_value ffa_relinquish_check_update(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001013 struct vm_locked from_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01001014 struct ffa_memory_region_constituent **fragments,
1015 uint32_t *fragment_constituent_counts, uint32_t fragment_count,
1016 struct mpool *page_pool, bool clear)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001017{
1018 uint32_t orig_from_mode;
1019 uint32_t from_mode;
1020 struct mpool local_page_pool;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001021 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001022
Andrew Walbranca808b12020-05-15 17:22:28 +01001023 ret = ffa_relinquish_check_transition(
1024 from_locked, &orig_from_mode, fragments,
1025 fragment_constituent_counts, fragment_count, &from_mode);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001026 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001027 dlog_verbose("Invalid transition for relinquish.\n");
Andrew Walbrana65a1322020-04-06 19:32:32 +01001028 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001029 }
1030
1031 /*
1032 * Create a local pool so any freed memory can't be used by another
1033 * thread. This is to ensure the original mapping can be restored if the
1034 * clear fails.
1035 */
1036 mpool_init_with_fallback(&local_page_pool, page_pool);
1037
1038 /*
1039 * First reserve all required memory for the new page table entries
1040 * without committing, to make sure the entire operation will succeed
1041 * without exhausting the page pool.
1042 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001043 if (!ffa_region_group_identity_map(
1044 from_locked, fragments, fragment_constituent_counts,
1045 fragment_count, from_mode, page_pool, false)) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001046 /* TODO: partial defrag of failed range. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001047 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001048 goto out;
1049 }
1050
1051 /*
1052 * Update the mapping for the sender. This won't allocate because the
1053 * transaction was already prepared above, but may free pages in the
1054 * case that a whole block is being unmapped that was previously
1055 * partially mapped.
1056 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001057 CHECK(ffa_region_group_identity_map(
1058 from_locked, fragments, fragment_constituent_counts,
1059 fragment_count, from_mode, &local_page_pool, true));
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001060
1061 /* Clear the memory so no VM or device can see the previous contents. */
J-Alves7db32002021-12-14 14:44:50 +00001062 if (clear &&
1063 !ffa_clear_memory_constituents(
1064 plat_ffa_owner_world_mode(from_locked.vm->id), fragments,
1065 fragment_constituent_counts, fragment_count, page_pool)) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001066 /*
1067 * On failure, roll back by returning memory to the sender. This
1068 * may allocate pages which were previously freed into
1069 * `local_page_pool` by the call above, but will never allocate
1070 * more pages than that so can never fail.
1071 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001072 CHECK(ffa_region_group_identity_map(
Andrew Walbranca808b12020-05-15 17:22:28 +01001073 from_locked, fragments, fragment_constituent_counts,
1074 fragment_count, orig_from_mode, &local_page_pool,
1075 true));
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001076
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001077 ret = ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001078 goto out;
1079 }
1080
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001081 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001082
1083out:
1084 mpool_fini(&local_page_pool);
1085
1086 /*
1087 * Tidy up the page table by reclaiming failed mappings (if there was an
1088 * error) or merging entries into blocks where possible (on success).
1089 */
Raghu Krishnamurthy7ad3d142021-03-28 00:47:35 -07001090 vm_ptable_defrag(from_locked, page_pool);
Jose Marinho09b1db82019-08-08 09:16:59 +01001091
1092 return ret;
1093}
1094
1095/**
Andrew Walbranca808b12020-05-15 17:22:28 +01001096 * Complete a memory sending operation by checking that it is valid, updating
1097 * the sender page table, and then either marking the share state as having
1098 * completed sending (on success) or freeing it (on failure).
1099 *
1100 * Returns FFA_SUCCESS with the handle encoded, or the relevant FFA_ERROR.
1101 */
J-Alvesfdd29272022-07-19 13:16:31 +01001102struct ffa_value ffa_memory_send_complete(
Andrew Walbranca808b12020-05-15 17:22:28 +01001103 struct vm_locked from_locked, struct share_states_locked share_states,
Andrew Walbran37c574e2020-06-03 11:45:46 +01001104 struct ffa_memory_share_state *share_state, struct mpool *page_pool,
1105 uint32_t *orig_from_mode_ret)
Andrew Walbranca808b12020-05-15 17:22:28 +01001106{
1107 struct ffa_memory_region *memory_region = share_state->memory_region;
1108 struct ffa_value ret;
1109
1110 /* Lock must be held. */
Daniel Boulbya2f8c662021-11-26 17:52:53 +00001111 assert(share_states.share_states != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +01001112
1113 /* Check that state is valid in sender page table and update. */
1114 ret = ffa_send_check_update(
1115 from_locked, share_state->fragments,
1116 share_state->fragment_constituent_counts,
1117 share_state->fragment_count, share_state->share_func,
J-Alves363f5722022-04-25 17:37:37 +01001118 memory_region->receivers, memory_region->receiver_count,
Andrew Walbran37c574e2020-06-03 11:45:46 +01001119 page_pool, memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR,
1120 orig_from_mode_ret);
Andrew Walbranca808b12020-05-15 17:22:28 +01001121 if (ret.func != FFA_SUCCESS_32) {
1122 /*
1123 * Free share state, it failed to send so it can't be retrieved.
1124 */
1125 dlog_verbose("Complete failed, freeing share state.\n");
1126 share_state_free(share_states, share_state, page_pool);
1127 return ret;
1128 }
1129
1130 share_state->sending_complete = true;
1131 dlog_verbose("Marked sending complete.\n");
1132
J-Alvesee68c542020-10-29 17:48:20 +00001133 return ffa_mem_success(share_state->memory_region->handle);
Andrew Walbranca808b12020-05-15 17:22:28 +01001134}
1135
1136/**
Federico Recanatia98603a2021-12-20 18:04:03 +01001137 * Check that the memory attributes match Hafnium expectations:
1138 * Normal Memory, Inner shareable, Write-Back Read-Allocate
1139 * Write-Allocate Cacheable.
1140 */
1141static struct ffa_value ffa_memory_attributes_validate(
1142 ffa_memory_access_permissions_t attributes)
1143{
1144 enum ffa_memory_type memory_type;
1145 enum ffa_memory_cacheability cacheability;
1146 enum ffa_memory_shareability shareability;
1147
1148 memory_type = ffa_get_memory_type_attr(attributes);
1149 if (memory_type != FFA_MEMORY_NORMAL_MEM) {
1150 dlog_verbose("Invalid memory type %#x, expected %#x.\n",
1151 memory_type, FFA_MEMORY_NORMAL_MEM);
Federico Recanati3d953f32022-02-17 09:31:29 +01001152 return ffa_error(FFA_DENIED);
Federico Recanatia98603a2021-12-20 18:04:03 +01001153 }
1154
1155 cacheability = ffa_get_memory_cacheability_attr(attributes);
1156 if (cacheability != FFA_MEMORY_CACHE_WRITE_BACK) {
1157 dlog_verbose("Invalid cacheability %#x, expected %#x.\n",
1158 cacheability, FFA_MEMORY_CACHE_WRITE_BACK);
Federico Recanati3d953f32022-02-17 09:31:29 +01001159 return ffa_error(FFA_DENIED);
Federico Recanatia98603a2021-12-20 18:04:03 +01001160 }
1161
1162 shareability = ffa_get_memory_shareability_attr(attributes);
1163 if (shareability != FFA_MEMORY_INNER_SHAREABLE) {
1164 dlog_verbose("Invalid shareability %#x, expected #%x.\n",
1165 shareability, FFA_MEMORY_INNER_SHAREABLE);
Federico Recanati3d953f32022-02-17 09:31:29 +01001166 return ffa_error(FFA_DENIED);
Federico Recanatia98603a2021-12-20 18:04:03 +01001167 }
1168
1169 return (struct ffa_value){.func = FFA_SUCCESS_32};
1170}
1171
1172/**
Andrew Walbrana65a1322020-04-06 19:32:32 +01001173 * Check that the given `memory_region` represents a valid memory send request
1174 * of the given `share_func` type, return the clear flag and permissions via the
1175 * respective output parameters, and update the permissions if necessary.
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001176 *
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001177 * Returns FFA_SUCCESS if the request was valid, or the relevant FFA_ERROR if
Andrew Walbrana65a1322020-04-06 19:32:32 +01001178 * not.
1179 */
J-Alves66652252022-07-06 09:49:51 +01001180struct ffa_value ffa_memory_send_validate(
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001181 struct vm_locked from_locked, struct ffa_memory_region *memory_region,
1182 uint32_t memory_share_length, uint32_t fragment_length,
J-Alves363f5722022-04-25 17:37:37 +01001183 uint32_t share_func)
Andrew Walbrana65a1322020-04-06 19:32:32 +01001184{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001185 struct ffa_composite_memory_region *composite;
Andrew Walbran130a8ae2020-05-15 16:27:15 +01001186 uint32_t receivers_length;
Federico Recanati872cd692022-01-05 13:10:10 +01001187 uint32_t composite_memory_region_offset;
Andrew Walbran352aa3d2020-05-01 17:51:33 +01001188 uint32_t constituents_offset;
Andrew Walbran130a8ae2020-05-15 16:27:15 +01001189 uint32_t constituents_length;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001190 enum ffa_data_access data_access;
1191 enum ffa_instruction_access instruction_access;
Federico Recanatia98603a2021-12-20 18:04:03 +01001192 struct ffa_value ret;
Andrew Walbrana65a1322020-04-06 19:32:32 +01001193
J-Alves95df0ef2022-12-07 10:09:48 +00001194 /* The sender must match the caller. */
1195 if ((!vm_id_is_current_world(from_locked.vm->id) &&
1196 vm_id_is_current_world(memory_region->sender)) ||
1197 (vm_id_is_current_world(from_locked.vm->id) &&
1198 memory_region->sender != from_locked.vm->id)) {
1199 dlog_verbose("Invalid memory sender ID.\n");
1200 return ffa_error(FFA_DENIED);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001201 }
1202
Andrew Walbrana65a1322020-04-06 19:32:32 +01001203 /*
1204 * Ensure that the composite header is within the memory bounds and
1205 * doesn't overlap the first part of the message.
1206 */
Andrew Walbran130a8ae2020-05-15 16:27:15 +01001207 receivers_length = sizeof(struct ffa_memory_access) *
1208 memory_region->receiver_count;
Andrew Walbran352aa3d2020-05-01 17:51:33 +01001209 constituents_offset =
1210 ffa_composite_constituent_offset(memory_region, 0);
Federico Recanati872cd692022-01-05 13:10:10 +01001211 composite_memory_region_offset =
1212 memory_region->receivers[0].composite_memory_region_offset;
1213 if ((composite_memory_region_offset == 0) ||
1214 (composite_memory_region_offset <
1215 sizeof(struct ffa_memory_region) + receivers_length) ||
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001216 constituents_offset > fragment_length) {
Andrew Walbrana65a1322020-04-06 19:32:32 +01001217 dlog_verbose(
Andrew Walbran352aa3d2020-05-01 17:51:33 +01001218 "Invalid composite memory region descriptor offset "
1219 "%d.\n",
1220 memory_region->receivers[0]
1221 .composite_memory_region_offset);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001222 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001223 }
1224
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001225 composite = ffa_memory_region_get_composite(memory_region, 0);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001226
1227 /*
Andrew Walbranf07f04d2020-05-01 18:09:00 +01001228 * Ensure the number of constituents are within the memory bounds.
Andrew Walbrana65a1322020-04-06 19:32:32 +01001229 */
Andrew Walbran130a8ae2020-05-15 16:27:15 +01001230 constituents_length = sizeof(struct ffa_memory_region_constituent) *
1231 composite->constituent_count;
Andrew Walbran352aa3d2020-05-01 17:51:33 +01001232 if (memory_share_length != constituents_offset + constituents_length) {
1233 dlog_verbose("Invalid length %d or composite offset %d.\n",
Andrew Walbran130a8ae2020-05-15 16:27:15 +01001234 memory_share_length,
Andrew Walbrana65a1322020-04-06 19:32:32 +01001235 memory_region->receivers[0]
1236 .composite_memory_region_offset);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001237 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001238 }
Andrew Walbranca808b12020-05-15 17:22:28 +01001239 if (fragment_length < memory_share_length &&
1240 fragment_length < HF_MAILBOX_SIZE) {
1241 dlog_warning(
1242 "Initial fragment length %d smaller than mailbox "
1243 "size.\n",
1244 fragment_length);
1245 }
Andrew Walbrana65a1322020-04-06 19:32:32 +01001246
Andrew Walbrana65a1322020-04-06 19:32:32 +01001247 /*
1248 * Clear is not allowed for memory sharing, as the sender still has
1249 * access to the memory.
1250 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001251 if ((memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR) &&
1252 share_func == FFA_MEM_SHARE_32) {
Andrew Walbrana65a1322020-04-06 19:32:32 +01001253 dlog_verbose("Memory can't be cleared while being shared.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001254 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001255 }
1256
1257 /* No other flags are allowed/supported here. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001258 if (memory_region->flags & ~FFA_MEMORY_REGION_FLAG_CLEAR) {
Andrew Walbrana65a1322020-04-06 19:32:32 +01001259 dlog_verbose("Invalid flags %#x.\n", memory_region->flags);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001260 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001261 }
1262
J-Alves363f5722022-04-25 17:37:37 +01001263 /* Check that the permissions are valid, for each specified receiver. */
1264 for (uint32_t i = 0U; i < memory_region->receiver_count; i++) {
1265 ffa_memory_access_permissions_t permissions =
1266 memory_region->receivers[i]
1267 .receiver_permissions.permissions;
1268 ffa_vm_id_t receiver_id =
1269 memory_region->receivers[i]
1270 .receiver_permissions.receiver;
1271
1272 if (memory_region->sender == receiver_id) {
1273 dlog_verbose("Can't share memory with itself.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001274 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001275 }
Federico Recanati85090c42021-12-15 13:17:54 +01001276
J-Alves363f5722022-04-25 17:37:37 +01001277 for (uint32_t j = i + 1; j < memory_region->receiver_count;
1278 j++) {
1279 if (receiver_id ==
1280 memory_region->receivers[j]
1281 .receiver_permissions.receiver) {
1282 dlog_verbose(
1283 "Repeated receiver(%x) in memory send "
1284 "operation.\n",
1285 memory_region->receivers[j]
1286 .receiver_permissions.receiver);
1287 return ffa_error(FFA_INVALID_PARAMETERS);
1288 }
1289 }
1290
1291 if (composite_memory_region_offset !=
1292 memory_region->receivers[i]
1293 .composite_memory_region_offset) {
1294 dlog_verbose(
1295 "All ffa_memory_access should point to the "
1296 "same composite memory region offset.\n");
1297 return ffa_error(FFA_INVALID_PARAMETERS);
1298 }
1299
1300 data_access = ffa_get_data_access_attr(permissions);
1301 instruction_access =
1302 ffa_get_instruction_access_attr(permissions);
1303 if (data_access == FFA_DATA_ACCESS_RESERVED ||
1304 instruction_access == FFA_INSTRUCTION_ACCESS_RESERVED) {
1305 dlog_verbose(
1306 "Reserved value for receiver permissions "
1307 "%#x.\n",
1308 permissions);
1309 return ffa_error(FFA_INVALID_PARAMETERS);
1310 }
1311 if (instruction_access !=
1312 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED) {
1313 dlog_verbose(
1314 "Invalid instruction access permissions %#x "
1315 "for sending memory.\n",
1316 permissions);
1317 return ffa_error(FFA_INVALID_PARAMETERS);
1318 }
1319 if (share_func == FFA_MEM_SHARE_32) {
1320 if (data_access == FFA_DATA_ACCESS_NOT_SPECIFIED) {
1321 dlog_verbose(
1322 "Invalid data access permissions %#x "
1323 "for sharing memory.\n",
1324 permissions);
1325 return ffa_error(FFA_INVALID_PARAMETERS);
1326 }
1327 /*
1328 * According to section 10.10.3 of the FF-A v1.1 EAC0
1329 * spec, NX is required for share operations (but must
1330 * not be specified by the sender) so set it in the
1331 * copy that we store, ready to be returned to the
1332 * retriever.
1333 */
J-Alvesb19731a2022-06-20 17:30:33 +01001334 if (vm_id_is_current_world(receiver_id)) {
1335 ffa_set_instruction_access_attr(
1336 &permissions,
1337 FFA_INSTRUCTION_ACCESS_NX);
1338 memory_region->receivers[i]
1339 .receiver_permissions.permissions =
1340 permissions;
1341 }
J-Alves363f5722022-04-25 17:37:37 +01001342 }
1343 if (share_func == FFA_MEM_LEND_32 &&
1344 data_access == FFA_DATA_ACCESS_NOT_SPECIFIED) {
1345 dlog_verbose(
1346 "Invalid data access permissions %#x for "
1347 "lending memory.\n",
1348 permissions);
1349 return ffa_error(FFA_INVALID_PARAMETERS);
1350 }
1351
1352 if (share_func == FFA_MEM_DONATE_32 &&
1353 data_access != FFA_DATA_ACCESS_NOT_SPECIFIED) {
1354 dlog_verbose(
1355 "Invalid data access permissions %#x for "
1356 "donating memory.\n",
1357 permissions);
1358 return ffa_error(FFA_INVALID_PARAMETERS);
1359 }
Andrew Walbrana65a1322020-04-06 19:32:32 +01001360 }
1361
Federico Recanatid937f5e2021-12-20 17:38:23 +01001362 /*
J-Alves807794e2022-06-16 13:42:47 +01001363 * If a memory donate or lend with single borrower, the memory type
1364 * shall not be specified by the sender.
Federico Recanatid937f5e2021-12-20 17:38:23 +01001365 */
J-Alves807794e2022-06-16 13:42:47 +01001366 if (share_func == FFA_MEM_DONATE_32 ||
1367 (share_func == FFA_MEM_LEND_32 &&
1368 memory_region->receiver_count == 1)) {
1369 if (ffa_get_memory_type_attr(memory_region->attributes) !=
1370 FFA_MEMORY_NOT_SPECIFIED_MEM) {
1371 dlog_verbose(
1372 "Memory type shall not be specified by "
1373 "sender.\n");
1374 return ffa_error(FFA_INVALID_PARAMETERS);
1375 }
1376 } else {
1377 /*
1378 * Check that sender's memory attributes match Hafnium
1379 * expectations: Normal Memory, Inner shareable, Write-Back
1380 * Read-Allocate Write-Allocate Cacheable.
1381 */
1382 ret = ffa_memory_attributes_validate(memory_region->attributes);
1383 if (ret.func != FFA_SUCCESS_32) {
1384 return ret;
1385 }
Federico Recanatid937f5e2021-12-20 17:38:23 +01001386 }
1387
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001388 return (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Walbrana65a1322020-04-06 19:32:32 +01001389}
1390
1391/**
Andrew Walbranca808b12020-05-15 17:22:28 +01001392 * Gets the share state for continuing an operation to donate, lend or share
1393 * memory, and checks that it is a valid request.
1394 *
1395 * Returns FFA_SUCCESS if the request was valid, or the relevant FFA_ERROR if
1396 * not.
1397 */
J-Alvesfdd29272022-07-19 13:16:31 +01001398struct ffa_value ffa_memory_send_continue_validate(
Andrew Walbranca808b12020-05-15 17:22:28 +01001399 struct share_states_locked share_states, ffa_memory_handle_t handle,
1400 struct ffa_memory_share_state **share_state_ret, ffa_vm_id_t from_vm_id,
1401 struct mpool *page_pool)
1402{
1403 struct ffa_memory_share_state *share_state;
1404 struct ffa_memory_region *memory_region;
1405
Daniel Boulbya2f8c662021-11-26 17:52:53 +00001406 assert(share_state_ret != NULL);
Andrew Walbranca808b12020-05-15 17:22:28 +01001407
1408 /*
1409 * Look up the share state by handle and make sure that the VM ID
1410 * matches.
1411 */
1412 if (!get_share_state(share_states, handle, &share_state)) {
1413 dlog_verbose(
1414 "Invalid handle %#x for memory send continuation.\n",
1415 handle);
1416 return ffa_error(FFA_INVALID_PARAMETERS);
1417 }
1418 memory_region = share_state->memory_region;
1419
J-Alvesfdd29272022-07-19 13:16:31 +01001420 if (vm_id_is_current_world(from_vm_id) &&
1421 memory_region->sender != from_vm_id) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001422 dlog_verbose("Invalid sender %d.\n", memory_region->sender);
1423 return ffa_error(FFA_INVALID_PARAMETERS);
1424 }
1425
1426 if (share_state->sending_complete) {
1427 dlog_verbose(
1428 "Sending of memory handle %#x is already complete.\n",
1429 handle);
1430 return ffa_error(FFA_INVALID_PARAMETERS);
1431 }
1432
1433 if (share_state->fragment_count == MAX_FRAGMENTS) {
1434 /*
1435 * Log a warning as this is a sign that MAX_FRAGMENTS should
1436 * probably be increased.
1437 */
1438 dlog_warning(
1439 "Too many fragments for memory share with handle %#x; "
1440 "only %d supported.\n",
1441 handle, MAX_FRAGMENTS);
1442 /* Free share state, as it's not possible to complete it. */
1443 share_state_free(share_states, share_state, page_pool);
1444 return ffa_error(FFA_NO_MEMORY);
1445 }
1446
1447 *share_state_ret = share_state;
1448
1449 return (struct ffa_value){.func = FFA_SUCCESS_32};
1450}
1451
1452/**
J-Alves95df0ef2022-12-07 10:09:48 +00001453 * Checks if there is at least one receiver from the other world.
1454 */
J-Alvesfdd29272022-07-19 13:16:31 +01001455bool memory_region_receivers_from_other_world(
J-Alves95df0ef2022-12-07 10:09:48 +00001456 struct ffa_memory_region *memory_region)
1457{
1458 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
1459 ffa_vm_id_t receiver = memory_region->receivers[i]
1460 .receiver_permissions.receiver;
1461 if (!vm_id_is_current_world(receiver)) {
1462 return true;
1463 }
1464 }
1465 return false;
1466}
1467
1468/**
J-Alves8505a8a2022-06-15 18:10:18 +01001469 * Validates a call to donate, lend or share memory to a non-other world VM and
1470 * then updates the stage-2 page tables. Specifically, check if the message
1471 * length and number of memory region constituents match, and if the transition
1472 * is valid for the type of memory sending operation.
Andrew Walbran475c1452020-02-07 13:22:22 +00001473 *
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001474 * Assumes that the caller has already found and locked the sender VM and copied
1475 * the memory region descriptor from the sender's TX buffer to a freshly
1476 * allocated page from Hafnium's internal pool. The caller must have also
1477 * validated that the receiver VM ID is valid.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001478 *
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001479 * This function takes ownership of the `memory_region` passed in and will free
1480 * it when necessary; it must not be freed by the caller.
Jose Marinho09b1db82019-08-08 09:16:59 +01001481 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001482struct ffa_value ffa_memory_send(struct vm_locked from_locked,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001483 struct ffa_memory_region *memory_region,
Andrew Walbran130a8ae2020-05-15 16:27:15 +01001484 uint32_t memory_share_length,
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001485 uint32_t fragment_length, uint32_t share_func,
1486 struct mpool *page_pool)
Jose Marinho09b1db82019-08-08 09:16:59 +01001487{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001488 struct ffa_value ret;
Andrew Walbranca808b12020-05-15 17:22:28 +01001489 struct share_states_locked share_states;
1490 struct ffa_memory_share_state *share_state;
Jose Marinho09b1db82019-08-08 09:16:59 +01001491
1492 /*
Andrew Walbrana65a1322020-04-06 19:32:32 +01001493 * If there is an error validating the `memory_region` then we need to
1494 * free it because we own it but we won't be storing it in a share state
1495 * after all.
Jose Marinho09b1db82019-08-08 09:16:59 +01001496 */
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001497 ret = ffa_memory_send_validate(from_locked, memory_region,
1498 memory_share_length, fragment_length,
J-Alves363f5722022-04-25 17:37:37 +01001499 share_func);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001500 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001501 mpool_free(page_pool, memory_region);
Andrew Walbrana65a1322020-04-06 19:32:32 +01001502 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01001503 }
1504
Andrew Walbrana65a1322020-04-06 19:32:32 +01001505 /* Set flag for share function, ready to be retrieved later. */
1506 switch (share_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001507 case FFA_MEM_SHARE_32:
Andrew Walbrana65a1322020-04-06 19:32:32 +01001508 memory_region->flags |=
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001509 FFA_MEMORY_REGION_TRANSACTION_TYPE_SHARE;
Andrew Walbrana65a1322020-04-06 19:32:32 +01001510 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001511 case FFA_MEM_LEND_32:
1512 memory_region->flags |= FFA_MEMORY_REGION_TRANSACTION_TYPE_LEND;
Andrew Walbrana65a1322020-04-06 19:32:32 +01001513 break;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001514 case FFA_MEM_DONATE_32:
Andrew Walbrana65a1322020-04-06 19:32:32 +01001515 memory_region->flags |=
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001516 FFA_MEMORY_REGION_TRANSACTION_TYPE_DONATE;
Andrew Walbrana65a1322020-04-06 19:32:32 +01001517 break;
Jose Marinho09b1db82019-08-08 09:16:59 +01001518 }
1519
Andrew Walbranca808b12020-05-15 17:22:28 +01001520 share_states = share_states_lock();
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001521 /*
1522 * Allocate a share state before updating the page table. Otherwise if
1523 * updating the page table succeeded but allocating the share state
1524 * failed then it would leave the memory in a state where nobody could
1525 * get it back.
1526 */
Andrew Walbranca808b12020-05-15 17:22:28 +01001527 if (!allocate_share_state(share_states, share_func, memory_region,
1528 fragment_length, FFA_MEMORY_HANDLE_INVALID,
1529 &share_state)) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001530 dlog_verbose("Failed to allocate share state.\n");
1531 mpool_free(page_pool, memory_region);
Andrew Walbranca808b12020-05-15 17:22:28 +01001532 ret = ffa_error(FFA_NO_MEMORY);
1533 goto out;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001534 }
1535
Andrew Walbranca808b12020-05-15 17:22:28 +01001536 if (fragment_length == memory_share_length) {
1537 /* No more fragments to come, everything fit in one message. */
J-Alves2a0d2882020-10-29 14:49:50 +00001538 ret = ffa_memory_send_complete(
1539 from_locked, share_states, share_state, page_pool,
1540 &(share_state->sender_orig_mode));
Andrew Walbranca808b12020-05-15 17:22:28 +01001541 } else {
J-Alvesfdd29272022-07-19 13:16:31 +01001542 /*
1543 * Use sender ID from 'memory_region' assuming
1544 * that at this point it has been validated:
1545 * - MBZ at virtual FF-A instance.
1546 */
1547 ffa_vm_id_t sender_to_ret =
1548 (from_locked.vm->id == HF_OTHER_WORLD_ID)
1549 ? memory_region->sender
1550 : 0;
Andrew Walbranca808b12020-05-15 17:22:28 +01001551 ret = (struct ffa_value){
1552 .func = FFA_MEM_FRAG_RX_32,
J-Alvesee68c542020-10-29 17:48:20 +00001553 .arg1 = (uint32_t)memory_region->handle,
1554 .arg2 = (uint32_t)(memory_region->handle >> 32),
J-Alvesfdd29272022-07-19 13:16:31 +01001555 .arg3 = fragment_length,
1556 .arg4 = (uint32_t)(sender_to_ret & 0xffff) << 16};
Andrew Walbranca808b12020-05-15 17:22:28 +01001557 }
1558
1559out:
1560 share_states_unlock(&share_states);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001561 dump_share_states();
Andrew Walbranca808b12020-05-15 17:22:28 +01001562 return ret;
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001563}
1564
1565/**
J-Alves8505a8a2022-06-15 18:10:18 +01001566 * Continues an operation to donate, lend or share memory to a VM from current
1567 * world. If this is the last fragment then checks that the transition is valid
1568 * for the type of memory sending operation and updates the stage-2 page tables
1569 * of the sender.
Andrew Walbranca808b12020-05-15 17:22:28 +01001570 *
1571 * Assumes that the caller has already found and locked the sender VM and copied
1572 * the memory region descriptor from the sender's TX buffer to a freshly
1573 * allocated page from Hafnium's internal pool.
1574 *
1575 * This function takes ownership of the `fragment` passed in; it must not be
1576 * freed by the caller.
1577 */
1578struct ffa_value ffa_memory_send_continue(struct vm_locked from_locked,
1579 void *fragment,
1580 uint32_t fragment_length,
1581 ffa_memory_handle_t handle,
1582 struct mpool *page_pool)
1583{
1584 struct share_states_locked share_states = share_states_lock();
1585 struct ffa_memory_share_state *share_state;
1586 struct ffa_value ret;
1587 struct ffa_memory_region *memory_region;
1588
1589 ret = ffa_memory_send_continue_validate(share_states, handle,
1590 &share_state,
1591 from_locked.vm->id, page_pool);
1592 if (ret.func != FFA_SUCCESS_32) {
1593 goto out_free_fragment;
1594 }
1595 memory_region = share_state->memory_region;
1596
J-Alves95df0ef2022-12-07 10:09:48 +00001597 if (memory_region_receivers_from_other_world(memory_region)) {
Andrew Walbranca808b12020-05-15 17:22:28 +01001598 dlog_error(
1599 "Got hypervisor-allocated handle for memory send to "
J-Alves8505a8a2022-06-15 18:10:18 +01001600 "other world. This should never happen, and indicates "
1601 "a bug in "
Andrew Walbranca808b12020-05-15 17:22:28 +01001602 "EL3 code.\n");
1603 ret = ffa_error(FFA_INVALID_PARAMETERS);
1604 goto out_free_fragment;
1605 }
1606
1607 /* Add this fragment. */
1608 share_state->fragments[share_state->fragment_count] = fragment;
1609 share_state->fragment_constituent_counts[share_state->fragment_count] =
1610 fragment_length / sizeof(struct ffa_memory_region_constituent);
1611 share_state->fragment_count++;
1612
1613 /* Check whether the memory send operation is now ready to complete. */
1614 if (share_state_sending_complete(share_states, share_state)) {
J-Alves2a0d2882020-10-29 14:49:50 +00001615 ret = ffa_memory_send_complete(
1616 from_locked, share_states, share_state, page_pool,
1617 &(share_state->sender_orig_mode));
Andrew Walbranca808b12020-05-15 17:22:28 +01001618 } else {
1619 ret = (struct ffa_value){
1620 .func = FFA_MEM_FRAG_RX_32,
1621 .arg1 = (uint32_t)handle,
1622 .arg2 = (uint32_t)(handle >> 32),
1623 .arg3 = share_state_next_fragment_offset(share_states,
1624 share_state)};
1625 }
1626 goto out;
1627
1628out_free_fragment:
1629 mpool_free(page_pool, fragment);
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001630
1631out:
Andrew Walbranca808b12020-05-15 17:22:28 +01001632 share_states_unlock(&share_states);
Andrew Walbran1a86aa92020-05-15 17:22:28 +01001633 return ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00001634}
1635
Andrew Walbranca808b12020-05-15 17:22:28 +01001636/** Clean up after the receiver has finished retrieving a memory region. */
1637static void ffa_memory_retrieve_complete(
1638 struct share_states_locked share_states,
1639 struct ffa_memory_share_state *share_state, struct mpool *page_pool)
1640{
1641 if (share_state->share_func == FFA_MEM_DONATE_32) {
1642 /*
1643 * Memory that has been donated can't be relinquished,
1644 * so no need to keep the share state around.
1645 */
1646 share_state_free(share_states, share_state, page_pool);
1647 dlog_verbose("Freed share state for donate.\n");
1648 }
1649}
1650
J-Alves96de29f2022-04-26 16:05:24 +01001651/*
1652 * Gets the receiver's access permissions from 'struct ffa_memory_region' and
1653 * returns its index in the receiver's array. If receiver's ID doesn't exist
1654 * in the array, return the region's 'receiver_count'.
1655 */
J-Alvesb5084cf2022-07-06 14:20:12 +01001656uint32_t ffa_memory_region_get_receiver(struct ffa_memory_region *memory_region,
1657 ffa_vm_id_t receiver)
J-Alves96de29f2022-04-26 16:05:24 +01001658{
1659 struct ffa_memory_access *receivers;
1660 uint32_t i;
1661
1662 assert(memory_region != NULL);
1663
1664 receivers = memory_region->receivers;
1665
1666 for (i = 0U; i < memory_region->receiver_count; i++) {
1667 if (receivers[i].receiver_permissions.receiver == receiver) {
1668 break;
1669 }
1670 }
1671
1672 return i;
1673}
1674
1675/**
1676 * Validates the retrieved permissions against those specified by the lender
1677 * of memory share operation. Optionally can help set the permissions to be used
1678 * for the S2 mapping, through the `permissions` argument.
1679 * Returns true if permissions are valid, false otherwise.
1680 */
1681static bool ffa_memory_retrieve_is_memory_access_valid(
1682 enum ffa_data_access sent_data_access,
1683 enum ffa_data_access requested_data_access,
1684 enum ffa_instruction_access sent_instruction_access,
1685 enum ffa_instruction_access requested_instruction_access,
1686 ffa_memory_access_permissions_t *permissions)
1687{
1688 switch (sent_data_access) {
1689 case FFA_DATA_ACCESS_NOT_SPECIFIED:
1690 case FFA_DATA_ACCESS_RW:
1691 if (requested_data_access == FFA_DATA_ACCESS_NOT_SPECIFIED ||
1692 requested_data_access == FFA_DATA_ACCESS_RW) {
1693 if (permissions != NULL) {
1694 ffa_set_data_access_attr(permissions,
1695 FFA_DATA_ACCESS_RW);
1696 }
1697 break;
1698 }
1699 /* Intentional fall-through. */
1700 case FFA_DATA_ACCESS_RO:
1701 if (requested_data_access == FFA_DATA_ACCESS_NOT_SPECIFIED ||
1702 requested_data_access == FFA_DATA_ACCESS_RO) {
1703 if (permissions != NULL) {
1704 ffa_set_data_access_attr(permissions,
1705 FFA_DATA_ACCESS_RO);
1706 }
1707 break;
1708 }
1709 dlog_verbose(
1710 "Invalid data access requested; sender specified "
1711 "permissions %#x but receiver requested %#x.\n",
1712 sent_data_access, requested_data_access);
1713 return false;
1714 case FFA_DATA_ACCESS_RESERVED:
1715 panic("Got unexpected FFA_DATA_ACCESS_RESERVED. Should be "
1716 "checked before this point.");
1717 }
1718
1719 switch (sent_instruction_access) {
1720 case FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED:
1721 case FFA_INSTRUCTION_ACCESS_X:
1722 if (requested_instruction_access ==
1723 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED ||
1724 requested_instruction_access == FFA_INSTRUCTION_ACCESS_X) {
1725 if (permissions != NULL) {
1726 ffa_set_instruction_access_attr(
1727 permissions, FFA_INSTRUCTION_ACCESS_X);
1728 }
1729 break;
1730 }
1731 case FFA_INSTRUCTION_ACCESS_NX:
1732 if (requested_instruction_access ==
1733 FFA_INSTRUCTION_ACCESS_NOT_SPECIFIED ||
1734 requested_instruction_access == FFA_INSTRUCTION_ACCESS_NX) {
1735 if (permissions != NULL) {
1736 ffa_set_instruction_access_attr(
1737 permissions, FFA_INSTRUCTION_ACCESS_NX);
1738 }
1739 break;
1740 }
1741 dlog_verbose(
1742 "Invalid instruction access requested; sender "
1743 "specified permissions %#x but receiver requested "
1744 "%#x.\n",
1745 sent_instruction_access, requested_instruction_access);
1746 return false;
1747 case FFA_INSTRUCTION_ACCESS_RESERVED:
1748 panic("Got unexpected FFA_INSTRUCTION_ACCESS_RESERVED. Should "
1749 "be checked before this point.");
1750 }
1751
1752 return true;
1753}
1754
1755/**
1756 * Validate the receivers' permissions in the retrieve request against those
1757 * specified by the lender.
1758 * In the `permissions` argument returns the permissions to set at S2 for the
1759 * caller to the FFA_MEMORY_RETRIEVE_REQ.
1760 * Returns FFA_SUCCESS if all specified permissions are valid.
1761 */
1762static struct ffa_value ffa_memory_retrieve_validate_memory_access_list(
1763 struct ffa_memory_region *memory_region,
1764 struct ffa_memory_region *retrieve_request, ffa_vm_id_t to_vm_id,
1765 ffa_memory_access_permissions_t *permissions)
1766{
1767 uint32_t retrieve_receiver_index;
1768
1769 assert(permissions != NULL);
1770
1771 if (retrieve_request->receiver_count != memory_region->receiver_count) {
1772 dlog_verbose(
1773 "Retrieve request should contain same list of "
1774 "borrowers, as specified by the lender.\n");
1775 return ffa_error(FFA_INVALID_PARAMETERS);
1776 }
1777
1778 retrieve_receiver_index = retrieve_request->receiver_count;
1779
1780 /* Should be populated with the permissions of the retriever. */
1781 *permissions = 0;
1782
1783 for (uint32_t i = 0U; i < retrieve_request->receiver_count; i++) {
1784 ffa_memory_access_permissions_t sent_permissions;
1785 struct ffa_memory_access *current_receiver =
1786 &retrieve_request->receivers[i];
1787 ffa_memory_access_permissions_t requested_permissions =
1788 current_receiver->receiver_permissions.permissions;
1789 ffa_vm_id_t current_receiver_id =
1790 current_receiver->receiver_permissions.receiver;
1791 bool found_to_id = current_receiver_id == to_vm_id;
1792
1793 /*
1794 * Find the current receiver in the transaction descriptor from
1795 * sender.
1796 */
1797 uint32_t mem_region_receiver_index =
1798 ffa_memory_region_get_receiver(memory_region,
1799 current_receiver_id);
1800
1801 if (mem_region_receiver_index ==
1802 memory_region->receiver_count) {
1803 dlog_verbose("%s: receiver %x not found\n", __func__,
1804 current_receiver_id);
1805 return ffa_error(FFA_DENIED);
1806 }
1807
1808 sent_permissions =
1809 memory_region->receivers[mem_region_receiver_index]
1810 .receiver_permissions.permissions;
1811
1812 if (found_to_id) {
1813 retrieve_receiver_index = i;
1814 }
1815
1816 /*
1817 * Since we are traversing the list of receivers, save the index
1818 * of the caller. As it needs to be there.
1819 */
1820
1821 if (current_receiver->composite_memory_region_offset != 0U) {
1822 dlog_verbose(
1823 "Retriever specified address ranges not "
1824 "supported (got offset %d).\n",
1825 current_receiver
1826 ->composite_memory_region_offset);
1827 return ffa_error(FFA_INVALID_PARAMETERS);
1828 }
1829
1830 /*
1831 * Check permissions from sender against permissions requested
1832 * by receiver.
1833 */
1834 if (!ffa_memory_retrieve_is_memory_access_valid(
1835 ffa_get_data_access_attr(sent_permissions),
1836 ffa_get_data_access_attr(requested_permissions),
1837 ffa_get_instruction_access_attr(sent_permissions),
1838 ffa_get_instruction_access_attr(
1839 requested_permissions),
1840 found_to_id ? permissions : NULL)) {
1841 return ffa_error(FFA_DENIED);
1842 }
1843
1844 /*
1845 * Can't request PM to clear memory if only provided with RO
1846 * permissions.
1847 */
1848 if (found_to_id &&
1849 (ffa_get_data_access_attr(*permissions) ==
1850 FFA_DATA_ACCESS_RO) &&
1851 (retrieve_request->flags & FFA_MEMORY_REGION_FLAG_CLEAR) !=
1852 0U) {
1853 dlog_verbose(
1854 "Receiver has RO permissions can not request "
1855 "clear.\n");
1856 return ffa_error(FFA_DENIED);
1857 }
1858 }
1859
1860 if (retrieve_receiver_index == retrieve_request->receiver_count) {
1861 dlog_verbose(
1862 "Retrieve request does not contain caller's (%x) "
1863 "permissions\n",
1864 to_vm_id);
1865 return ffa_error(FFA_INVALID_PARAMETERS);
1866 }
1867
1868 return (struct ffa_value){.func = FFA_SUCCESS_32};
1869}
1870
J-Alvesa9cd7e32022-07-01 13:49:33 +01001871/*
1872 * According to section 16.4.3 of FF-A v1.1 EAC0 specification, the hypervisor
1873 * may issue an FFA_MEM_RETRIEVE_REQ to obtain the memory region description
1874 * of a pending memory sharing operation whose allocator is the SPM, for
1875 * validation purposes before forwarding an FFA_MEM_RECLAIM call. In doing so
1876 * the memory region descriptor of the retrieve request must be zeroed with the
1877 * exception of the sender ID and handle.
1878 */
1879bool is_ffa_memory_retrieve_borrower_request(struct ffa_memory_region *request,
1880 struct vm_locked to_locked)
1881{
1882 return to_locked.vm->id == HF_HYPERVISOR_VM_ID &&
1883 request->attributes == 0U && request->flags == 0U &&
1884 request->tag == 0U && request->receiver_count == 0U &&
1885 plat_ffa_memory_handle_allocated_by_current_world(
1886 request->handle);
1887}
1888
1889/*
1890 * Helper to reset count of fragments retrieved by the hypervisor.
1891 */
1892static void ffa_memory_retrieve_complete_from_hyp(
1893 struct ffa_memory_share_state *share_state)
1894{
1895 if (share_state->hypervisor_fragment_count ==
1896 share_state->fragment_count) {
1897 share_state->hypervisor_fragment_count = 0;
1898 }
1899}
1900
J-Alves089004f2022-07-13 14:25:44 +01001901/**
1902 * Validate that the memory region descriptor provided by the borrower on
1903 * FFA_MEM_RETRIEVE_REQ, against saved memory region provided by lender at the
1904 * memory sharing call.
1905 */
1906static struct ffa_value ffa_memory_retrieve_validate(
1907 ffa_vm_id_t receiver_id, struct ffa_memory_region *retrieve_request,
1908 struct ffa_memory_region *memory_region, uint32_t *receiver_index,
1909 uint32_t share_func)
1910{
1911 ffa_memory_region_flags_t transaction_type =
1912 retrieve_request->flags &
1913 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK;
1914
1915 assert(retrieve_request != NULL);
1916 assert(memory_region != NULL);
1917 assert(receiver_index != NULL);
1918 assert(retrieve_request->sender == memory_region->sender);
1919
1920 /*
1921 * Check that the transaction type expected by the receiver is
1922 * correct, if it has been specified.
1923 */
1924 if (transaction_type !=
1925 FFA_MEMORY_REGION_TRANSACTION_TYPE_UNSPECIFIED &&
1926 transaction_type != (memory_region->flags &
1927 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK)) {
1928 dlog_verbose(
1929 "Incorrect transaction type %#x for "
1930 "FFA_MEM_RETRIEVE_REQ, expected %#x for handle %#x.\n",
1931 transaction_type,
1932 memory_region->flags &
1933 FFA_MEMORY_REGION_TRANSACTION_TYPE_MASK,
1934 retrieve_request->handle);
1935 return ffa_error(FFA_INVALID_PARAMETERS);
1936 }
1937
1938 if (retrieve_request->tag != memory_region->tag) {
1939 dlog_verbose(
1940 "Incorrect tag %d for FFA_MEM_RETRIEVE_REQ, expected "
1941 "%d for handle %#x.\n",
1942 retrieve_request->tag, memory_region->tag,
1943 retrieve_request->handle);
1944 return ffa_error(FFA_INVALID_PARAMETERS);
1945 }
1946
1947 *receiver_index =
1948 ffa_memory_region_get_receiver(memory_region, receiver_id);
1949
1950 if (*receiver_index == memory_region->receiver_count) {
1951 dlog_verbose(
1952 "Incorrect receiver VM ID %d for "
1953 "FFA_MEM_RETRIEVE_REQ, for handle %#x.\n",
1954 receiver_id, handle);
1955 return ffa_error(FFA_INVALID_PARAMETERS);
1956 }
1957
1958 if ((retrieve_request->flags &
1959 FFA_MEMORY_REGION_ADDRESS_RANGE_HINT_VALID) != 0U) {
1960 dlog_verbose(
1961 "Retriever specified 'address range alignment 'hint' "
1962 "not supported.\n");
1963 return ffa_error(FFA_INVALID_PARAMETERS);
1964 }
1965 if ((retrieve_request->flags &
1966 FFA_MEMORY_REGION_ADDRESS_RANGE_HINT_MASK) != 0) {
1967 dlog_verbose(
1968 "Bits 8-5 must be zero in memory region's flags "
1969 "(address range alignment hint not supported).\n");
1970 return ffa_error(FFA_INVALID_PARAMETERS);
1971 }
1972
1973 if ((retrieve_request->flags & ~0x7FF) != 0U) {
1974 dlog_verbose(
1975 "Bits 31-10 must be zero in memory region's flags.\n");
1976 return ffa_error(FFA_INVALID_PARAMETERS);
1977 }
1978
1979 if (share_func == FFA_MEM_SHARE_32 &&
1980 (retrieve_request->flags &
1981 (FFA_MEMORY_REGION_FLAG_CLEAR |
1982 FFA_MEMORY_REGION_FLAG_CLEAR_RELINQUISH)) != 0U) {
1983 dlog_verbose(
1984 "Memory Share operation can't clean after relinquish "
1985 "memory region.\n");
1986 return ffa_error(FFA_INVALID_PARAMETERS);
1987 }
1988
1989 /*
1990 * If the borrower needs the memory to be cleared before mapping
1991 * to its address space, the sender should have set the flag
1992 * when calling FFA_MEM_LEND/FFA_MEM_DONATE, else return
1993 * FFA_DENIED.
1994 */
1995 if ((retrieve_request->flags & FFA_MEMORY_REGION_FLAG_CLEAR) != 0U &&
1996 (memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR) == 0U) {
1997 dlog_verbose(
1998 "Borrower needs memory cleared. Sender needs to set "
1999 "flag for clearing memory.\n");
2000 return ffa_error(FFA_DENIED);
2001 }
2002
2003 /*
2004 * If memory type is not specified, bypass validation of memory
2005 * attributes in the retrieve request. The retriever is expecting to
2006 * obtain this information from the SPMC.
2007 */
2008 if (ffa_get_memory_type_attr(retrieve_request->attributes) ==
2009 FFA_MEMORY_NOT_SPECIFIED_MEM) {
2010 return (struct ffa_value){.func = FFA_SUCCESS_32};
2011 }
2012
2013 /*
2014 * Ensure receiver's attributes are compatible with how
2015 * Hafnium maps memory: Normal Memory, Inner shareable,
2016 * Write-Back Read-Allocate Write-Allocate Cacheable.
2017 */
2018 return ffa_memory_attributes_validate(retrieve_request->attributes);
2019}
2020
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002021struct ffa_value ffa_memory_retrieve(struct vm_locked to_locked,
2022 struct ffa_memory_region *retrieve_request,
Andrew Walbran130a8ae2020-05-15 16:27:15 +01002023 uint32_t retrieve_request_length,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002024 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002025{
Andrew Walbran130a8ae2020-05-15 16:27:15 +01002026 uint32_t expected_retrieve_request_length =
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002027 sizeof(struct ffa_memory_region) +
Andrew Walbrana65a1322020-04-06 19:32:32 +01002028 retrieve_request->receiver_count *
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002029 sizeof(struct ffa_memory_access);
2030 ffa_memory_handle_t handle = retrieve_request->handle;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002031 struct ffa_memory_region *memory_region;
J-Alvesa9cd7e32022-07-01 13:49:33 +01002032 ffa_memory_access_permissions_t permissions = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002033 uint32_t memory_to_attributes;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002034 struct share_states_locked share_states;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002035 struct ffa_memory_share_state *share_state;
2036 struct ffa_value ret;
Andrew Walbranca808b12020-05-15 17:22:28 +01002037 struct ffa_composite_memory_region *composite;
2038 uint32_t total_length;
2039 uint32_t fragment_length;
J-Alves089004f2022-07-13 14:25:44 +01002040 ffa_vm_id_t receiver_id = to_locked.vm->id;
J-Alvesa9cd7e32022-07-01 13:49:33 +01002041 bool is_send_complete = false;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002042
2043 dump_share_states();
2044
Andrew Walbran130a8ae2020-05-15 16:27:15 +01002045 if (retrieve_request_length != expected_retrieve_request_length) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002046 dlog_verbose(
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002047 "Invalid length for FFA_MEM_RETRIEVE_REQ, expected %d "
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002048 "but was %d.\n",
Andrew Walbran130a8ae2020-05-15 16:27:15 +01002049 expected_retrieve_request_length,
2050 retrieve_request_length);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002051 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002052 }
2053
2054 share_states = share_states_lock();
2055 if (!get_share_state(share_states, handle, &share_state)) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002056 dlog_verbose("Invalid handle %#x for FFA_MEM_RETRIEVE_REQ.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002057 handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002058 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002059 goto out;
2060 }
2061
J-Alves96de29f2022-04-26 16:05:24 +01002062 if (!share_state->sending_complete) {
2063 dlog_verbose(
2064 "Memory with handle %#x not fully sent, can't "
2065 "retrieve.\n",
2066 handle);
2067 ret = ffa_error(FFA_INVALID_PARAMETERS);
2068 goto out;
2069 }
2070
Andrew Walbrana65a1322020-04-06 19:32:32 +01002071 memory_region = share_state->memory_region;
J-Alves089004f2022-07-13 14:25:44 +01002072
Andrew Walbrana65a1322020-04-06 19:32:32 +01002073 CHECK(memory_region != NULL);
2074
J-Alves089004f2022-07-13 14:25:44 +01002075 if (retrieve_request->sender != memory_region->sender) {
2076 dlog_verbose(
2077 "Memory with handle %#x not fully sent, can't "
2078 "retrieve.\n",
2079 handle);
2080 ret = ffa_error(FFA_INVALID_PARAMETERS);
2081 goto out;
2082 }
J-Alves96de29f2022-04-26 16:05:24 +01002083
J-Alvesa9cd7e32022-07-01 13:49:33 +01002084 if (!is_ffa_memory_retrieve_borrower_request(retrieve_request,
2085 to_locked)) {
2086 uint32_t receiver_index;
J-Alvesa9cd7e32022-07-01 13:49:33 +01002087
J-Alvesb5084cf2022-07-06 14:20:12 +01002088 /*
2089 * The SPMC can only process retrieve requests to memory share
2090 * operations with one borrower from the other world. It can't
2091 * determine the ID of the NWd VM that invoked the retrieve
2092 * request interface call. It relies on the hypervisor to
2093 * validate the caller's ID against that provided in the
2094 * `receivers` list of the retrieve response.
2095 * In case there is only one borrower from the NWd in the
2096 * transaction descriptor, record that in the `receiver_id` for
2097 * later use, and validate in the retrieve request message.
2098 */
2099 if (to_locked.vm->id == HF_HYPERVISOR_VM_ID) {
2100 uint32_t other_world_count = 0;
2101
2102 for (uint32_t i = 0; i < memory_region->receiver_count;
2103 i++) {
2104 receiver_id =
2105 retrieve_request->receivers[0]
2106 .receiver_permissions.receiver;
2107 if (!vm_id_is_current_world(receiver_id)) {
2108 other_world_count++;
2109 }
2110 }
2111 if (other_world_count > 1) {
2112 dlog_verbose(
2113 "Support one receiver from the other "
2114 "world.\n");
2115 return ffa_error(FFA_NOT_SUPPORTED);
2116 }
2117 }
2118
2119 /*
2120 * Validate retrieve request, according to what was sent by the
2121 * sender. Function will output the `receiver_index` from the
2122 * provided memory region, and will output `permissions` from
2123 * the validated requested permissions.
2124 */
J-Alves089004f2022-07-13 14:25:44 +01002125 ret = ffa_memory_retrieve_validate(
2126 receiver_id, retrieve_request, memory_region,
2127 &receiver_index, share_state->share_func);
2128 if (ret.func != FFA_SUCCESS_32) {
J-Alvesa9cd7e32022-07-01 13:49:33 +01002129 goto out;
2130 }
2131
2132 if (share_state->retrieved_fragment_count[receiver_index] !=
2133 0U) {
2134 dlog_verbose(
2135 "Memory with handle %#x already retrieved.\n",
2136 handle);
2137 ret = ffa_error(FFA_DENIED);
2138 goto out;
2139 }
2140
J-Alvesa9cd7e32022-07-01 13:49:33 +01002141 ret = ffa_memory_retrieve_validate_memory_access_list(
2142 memory_region, retrieve_request, receiver_id,
2143 &permissions);
J-Alves614d9f42022-06-28 14:03:10 +01002144 if (ret.func != FFA_SUCCESS_32) {
2145 goto out;
2146 }
Federico Recanatia98603a2021-12-20 18:04:03 +01002147
J-Alvesa9cd7e32022-07-01 13:49:33 +01002148 memory_to_attributes = ffa_memory_permissions_to_mode(
2149 permissions, share_state->sender_orig_mode);
2150 ret = ffa_retrieve_check_update(
2151 to_locked, memory_region->sender,
2152 share_state->fragments,
2153 share_state->fragment_constituent_counts,
2154 share_state->fragment_count, memory_to_attributes,
2155 share_state->share_func, false, page_pool);
2156
2157 if (ret.func != FFA_SUCCESS_32) {
2158 goto out;
2159 }
2160
2161 share_state->retrieved_fragment_count[receiver_index] = 1;
2162 is_send_complete =
2163 share_state->retrieved_fragment_count[receiver_index] ==
2164 share_state->fragment_count;
2165 } else {
2166 if (share_state->hypervisor_fragment_count != 0U) {
2167 dlog_verbose(
J-Alvesb5084cf2022-07-06 14:20:12 +01002168 "Memory with handle %#x already retrieved by "
J-Alvesa9cd7e32022-07-01 13:49:33 +01002169 "the hypervisor.\n",
2170 handle);
2171 ret = ffa_error(FFA_DENIED);
2172 goto out;
2173 }
2174
2175 share_state->hypervisor_fragment_count = 1;
2176
2177 ffa_memory_retrieve_complete_from_hyp(share_state);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002178 }
2179
J-Alvesb5084cf2022-07-06 14:20:12 +01002180 /* VMs acquire the RX buffer from SPMC. */
2181 CHECK(plat_ffa_acquire_receiver_rx(to_locked, &ret));
2182
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002183 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01002184 * Copy response to RX buffer of caller and deliver the message.
2185 * This must be done before the share_state is (possibly) freed.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002186 */
Andrew Walbrana65a1322020-04-06 19:32:32 +01002187 /* TODO: combine attributes from sender and request. */
Andrew Walbranca808b12020-05-15 17:22:28 +01002188 composite = ffa_memory_region_get_composite(memory_region, 0);
2189 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01002190 * Constituents which we received in the first fragment should
2191 * always fit in the first fragment we are sending, because the
2192 * header is the same size in both cases and we have a fixed
2193 * message buffer size. So `ffa_retrieved_memory_region_init`
2194 * should never fail.
Andrew Walbranca808b12020-05-15 17:22:28 +01002195 */
2196 CHECK(ffa_retrieved_memory_region_init(
Andrew Walbrana65a1322020-04-06 19:32:32 +01002197 to_locked.vm->mailbox.recv, HF_MAILBOX_SIZE,
2198 memory_region->sender, memory_region->attributes,
J-Alvesa9cd7e32022-07-01 13:49:33 +01002199 memory_region->flags, handle, receiver_id, permissions,
Andrew Walbranca808b12020-05-15 17:22:28 +01002200 composite->page_count, composite->constituent_count,
2201 share_state->fragments[0],
2202 share_state->fragment_constituent_counts[0], &total_length,
2203 &fragment_length));
J-Alvesb5084cf2022-07-06 14:20:12 +01002204
Andrew Walbranca808b12020-05-15 17:22:28 +01002205 to_locked.vm->mailbox.recv_size = fragment_length;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002206 to_locked.vm->mailbox.recv_sender = HF_HYPERVISOR_VM_ID;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002207 to_locked.vm->mailbox.recv_func = FFA_MEM_RETRIEVE_RESP_32;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002208 to_locked.vm->mailbox.state = MAILBOX_STATE_READ;
2209
J-Alvesa9cd7e32022-07-01 13:49:33 +01002210 if (is_send_complete) {
Andrew Walbranca808b12020-05-15 17:22:28 +01002211 ffa_memory_retrieve_complete(share_states, share_state,
2212 page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002213 }
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002214 ret = (struct ffa_value){.func = FFA_MEM_RETRIEVE_RESP_32,
Andrew Walbranca808b12020-05-15 17:22:28 +01002215 .arg1 = total_length,
2216 .arg2 = fragment_length};
2217
2218out:
2219 share_states_unlock(&share_states);
2220 dump_share_states();
2221 return ret;
2222}
2223
2224struct ffa_value ffa_memory_retrieve_continue(struct vm_locked to_locked,
2225 ffa_memory_handle_t handle,
2226 uint32_t fragment_offset,
2227 struct mpool *page_pool)
2228{
2229 struct ffa_memory_region *memory_region;
2230 struct share_states_locked share_states;
2231 struct ffa_memory_share_state *share_state;
2232 struct ffa_value ret;
2233 uint32_t fragment_index;
2234 uint32_t retrieved_constituents_count;
2235 uint32_t i;
2236 uint32_t expected_fragment_offset;
2237 uint32_t remaining_constituent_count;
2238 uint32_t fragment_length;
J-Alvesc7484f12022-05-13 12:41:14 +01002239 uint32_t receiver_index;
Andrew Walbranca808b12020-05-15 17:22:28 +01002240
2241 dump_share_states();
2242
2243 share_states = share_states_lock();
2244 if (!get_share_state(share_states, handle, &share_state)) {
2245 dlog_verbose("Invalid handle %#x for FFA_MEM_FRAG_RX.\n",
2246 handle);
2247 ret = ffa_error(FFA_INVALID_PARAMETERS);
2248 goto out;
2249 }
2250
2251 memory_region = share_state->memory_region;
2252 CHECK(memory_region != NULL);
2253
J-Alvesc7484f12022-05-13 12:41:14 +01002254 receiver_index =
2255 ffa_memory_region_get_receiver(memory_region, to_locked.vm->id);
2256
2257 if (receiver_index == memory_region->receiver_count) {
Andrew Walbranca808b12020-05-15 17:22:28 +01002258 dlog_verbose(
J-Alvesc7484f12022-05-13 12:41:14 +01002259 "Caller of FFA_MEM_FRAG_RX (%x) is not a borrower to "
2260 "memory sharing transaction (%x)\n",
2261 to_locked.vm->id, handle);
Andrew Walbranca808b12020-05-15 17:22:28 +01002262 ret = ffa_error(FFA_INVALID_PARAMETERS);
2263 goto out;
2264 }
2265
2266 if (!share_state->sending_complete) {
2267 dlog_verbose(
2268 "Memory with handle %#x not fully sent, can't "
2269 "retrieve.\n",
2270 handle);
2271 ret = ffa_error(FFA_INVALID_PARAMETERS);
2272 goto out;
2273 }
2274
J-Alvesc7484f12022-05-13 12:41:14 +01002275 if (share_state->retrieved_fragment_count[receiver_index] == 0 ||
2276 share_state->retrieved_fragment_count[receiver_index] >=
Andrew Walbranca808b12020-05-15 17:22:28 +01002277 share_state->fragment_count) {
2278 dlog_verbose(
2279 "Retrieval of memory with handle %#x not yet started "
2280 "or already completed (%d/%d fragments retrieved).\n",
J-Alvesc7484f12022-05-13 12:41:14 +01002281 handle,
2282 share_state->retrieved_fragment_count[receiver_index],
Andrew Walbranca808b12020-05-15 17:22:28 +01002283 share_state->fragment_count);
2284 ret = ffa_error(FFA_INVALID_PARAMETERS);
2285 goto out;
2286 }
2287
J-Alvesc7484f12022-05-13 12:41:14 +01002288 fragment_index = share_state->retrieved_fragment_count[receiver_index];
Andrew Walbranca808b12020-05-15 17:22:28 +01002289
2290 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01002291 * Check that the given fragment offset is correct by counting
2292 * how many constituents were in the fragments previously sent.
Andrew Walbranca808b12020-05-15 17:22:28 +01002293 */
2294 retrieved_constituents_count = 0;
2295 for (i = 0; i < fragment_index; ++i) {
2296 retrieved_constituents_count +=
2297 share_state->fragment_constituent_counts[i];
2298 }
J-Alvesc7484f12022-05-13 12:41:14 +01002299
2300 CHECK(memory_region->receiver_count > 0);
2301
Andrew Walbranca808b12020-05-15 17:22:28 +01002302 expected_fragment_offset =
J-Alvesc7484f12022-05-13 12:41:14 +01002303 ffa_composite_constituent_offset(memory_region,
2304 receiver_index) +
Andrew Walbranca808b12020-05-15 17:22:28 +01002305 retrieved_constituents_count *
J-Alvesc7484f12022-05-13 12:41:14 +01002306 sizeof(struct ffa_memory_region_constituent) -
2307 sizeof(struct ffa_memory_access) *
2308 (memory_region->receiver_count - 1);
Andrew Walbranca808b12020-05-15 17:22:28 +01002309 if (fragment_offset != expected_fragment_offset) {
2310 dlog_verbose("Fragment offset was %d but expected %d.\n",
2311 fragment_offset, expected_fragment_offset);
2312 ret = ffa_error(FFA_INVALID_PARAMETERS);
2313 goto out;
2314 }
2315
2316 remaining_constituent_count = ffa_memory_fragment_init(
2317 to_locked.vm->mailbox.recv, HF_MAILBOX_SIZE,
2318 share_state->fragments[fragment_index],
2319 share_state->fragment_constituent_counts[fragment_index],
2320 &fragment_length);
2321 CHECK(remaining_constituent_count == 0);
2322 to_locked.vm->mailbox.recv_size = fragment_length;
2323 to_locked.vm->mailbox.recv_sender = HF_HYPERVISOR_VM_ID;
2324 to_locked.vm->mailbox.recv_func = FFA_MEM_FRAG_TX_32;
2325 to_locked.vm->mailbox.state = MAILBOX_STATE_READ;
J-Alvesc7484f12022-05-13 12:41:14 +01002326 share_state->retrieved_fragment_count[receiver_index]++;
2327 if (share_state->retrieved_fragment_count[receiver_index] ==
Andrew Walbranca808b12020-05-15 17:22:28 +01002328 share_state->fragment_count) {
2329 ffa_memory_retrieve_complete(share_states, share_state,
2330 page_pool);
2331 }
2332
2333 ret = (struct ffa_value){.func = FFA_MEM_FRAG_TX_32,
2334 .arg1 = (uint32_t)handle,
2335 .arg2 = (uint32_t)(handle >> 32),
2336 .arg3 = fragment_length};
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002337
2338out:
2339 share_states_unlock(&share_states);
2340 dump_share_states();
2341 return ret;
2342}
2343
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002344struct ffa_value ffa_memory_relinquish(
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002345 struct vm_locked from_locked,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002346 struct ffa_mem_relinquish *relinquish_request, struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002347{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002348 ffa_memory_handle_t handle = relinquish_request->handle;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002349 struct share_states_locked share_states;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002350 struct ffa_memory_share_state *share_state;
2351 struct ffa_memory_region *memory_region;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002352 bool clear;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002353 struct ffa_value ret;
J-Alves8eb19162022-04-28 10:56:48 +01002354 uint32_t receiver_index;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002355
Andrew Walbrana65a1322020-04-06 19:32:32 +01002356 if (relinquish_request->endpoint_count != 1) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002357 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01002358 "Stream endpoints not supported (got %d "
2359 "endpoints on "
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002360 "FFA_MEM_RELINQUISH, expected 1).\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002361 relinquish_request->endpoint_count);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002362 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002363 }
2364
Andrew Walbrana65a1322020-04-06 19:32:32 +01002365 if (relinquish_request->endpoints[0] != from_locked.vm->id) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002366 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01002367 "VM ID %d in relinquish message doesn't match "
2368 "calling "
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002369 "VM ID %d.\n",
Andrew Walbrana65a1322020-04-06 19:32:32 +01002370 relinquish_request->endpoints[0], from_locked.vm->id);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002371 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002372 }
2373
2374 dump_share_states();
2375
2376 share_states = share_states_lock();
2377 if (!get_share_state(share_states, handle, &share_state)) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002378 dlog_verbose("Invalid handle %#x for FFA_MEM_RELINQUISH.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002379 handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002380 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002381 goto out;
2382 }
2383
Andrew Walbranca808b12020-05-15 17:22:28 +01002384 if (!share_state->sending_complete) {
2385 dlog_verbose(
2386 "Memory with handle %#x not fully sent, can't "
2387 "relinquish.\n",
2388 handle);
2389 ret = ffa_error(FFA_INVALID_PARAMETERS);
2390 goto out;
2391 }
2392
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002393 memory_region = share_state->memory_region;
2394 CHECK(memory_region != NULL);
2395
J-Alves8eb19162022-04-28 10:56:48 +01002396 receiver_index = ffa_memory_region_get_receiver(memory_region,
2397 from_locked.vm->id);
2398
2399 if (receiver_index == memory_region->receiver_count) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002400 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01002401 "VM ID %d tried to relinquish memory region "
2402 "with "
J-Alves8eb19162022-04-28 10:56:48 +01002403 "handle %#x and it is not a valid borrower.\n",
2404 from_locked.vm->id, handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002405 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002406 goto out;
2407 }
2408
J-Alves8eb19162022-04-28 10:56:48 +01002409 if (share_state->retrieved_fragment_count[receiver_index] !=
Andrew Walbranca808b12020-05-15 17:22:28 +01002410 share_state->fragment_count) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002411 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01002412 "Memory with handle %#x not yet fully "
2413 "retrieved, "
J-Alves8eb19162022-04-28 10:56:48 +01002414 "receiver %x can't relinquish.\n",
2415 handle, from_locked.vm->id);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002416 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002417 goto out;
2418 }
2419
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002420 clear = relinquish_request->flags & FFA_MEMORY_REGION_FLAG_CLEAR;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002421
2422 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01002423 * Clear is not allowed for memory that was shared, as the
2424 * original sender still has access to the memory.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002425 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002426 if (clear && share_state->share_func == FFA_MEM_SHARE_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002427 dlog_verbose("Memory which was shared can't be cleared.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002428 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002429 goto out;
2430 }
2431
Andrew Walbranca808b12020-05-15 17:22:28 +01002432 ret = ffa_relinquish_check_update(
2433 from_locked, share_state->fragments,
2434 share_state->fragment_constituent_counts,
2435 share_state->fragment_count, page_pool, clear);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002436
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002437 if (ret.func == FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002438 /*
J-Alvesa9cd7e32022-07-01 13:49:33 +01002439 * Mark memory handle as not retrieved, so it can be
2440 * reclaimed (or retrieved again).
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002441 */
J-Alves8eb19162022-04-28 10:56:48 +01002442 share_state->retrieved_fragment_count[receiver_index] = 0;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002443 }
2444
2445out:
2446 share_states_unlock(&share_states);
2447 dump_share_states();
2448 return ret;
2449}
2450
2451/**
J-Alvesa9cd7e32022-07-01 13:49:33 +01002452 * Validates that the reclaim transition is allowed for the given
2453 * handle, updates the page table of the reclaiming VM, and frees the
2454 * internal state associated with the handle.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002455 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002456struct ffa_value ffa_memory_reclaim(struct vm_locked to_locked,
Andrew Walbranca808b12020-05-15 17:22:28 +01002457 ffa_memory_handle_t handle,
2458 ffa_memory_region_flags_t flags,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002459 struct mpool *page_pool)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002460{
2461 struct share_states_locked share_states;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002462 struct ffa_memory_share_state *share_state;
2463 struct ffa_memory_region *memory_region;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002464 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002465
2466 dump_share_states();
2467
2468 share_states = share_states_lock();
J-Alvesb5084cf2022-07-06 14:20:12 +01002469 if (get_share_state(share_states, handle, &share_state)) {
2470 memory_region = share_state->memory_region;
2471 } else {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002472 dlog_verbose("Invalid handle %#x for FFA_MEM_RECLAIM.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002473 handle);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002474 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002475 goto out;
2476 }
2477
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002478 CHECK(memory_region != NULL);
2479
J-Alvesa9cd7e32022-07-01 13:49:33 +01002480 if (vm_id_is_current_world(to_locked.vm->id) &&
2481 to_locked.vm->id != memory_region->sender) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002482 dlog_verbose(
Olivier Deprezf92e5d42020-11-13 16:00:54 +01002483 "VM %#x attempted to reclaim memory handle %#x "
2484 "originally sent by VM %#x.\n",
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002485 to_locked.vm->id, handle, memory_region->sender);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002486 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002487 goto out;
2488 }
2489
Andrew Walbranca808b12020-05-15 17:22:28 +01002490 if (!share_state->sending_complete) {
2491 dlog_verbose(
2492 "Memory with handle %#x not fully sent, can't "
2493 "reclaim.\n",
2494 handle);
2495 ret = ffa_error(FFA_INVALID_PARAMETERS);
2496 goto out;
2497 }
2498
J-Alves752236c2022-04-28 11:07:47 +01002499 for (uint32_t i = 0; i < memory_region->receiver_count; i++) {
2500 if (share_state->retrieved_fragment_count[i] != 0) {
2501 dlog_verbose(
J-Alvesa9cd7e32022-07-01 13:49:33 +01002502 "Tried to reclaim memory handle %#x "
2503 "that has "
2504 "not been relinquished by all "
2505 "borrowers(%x).\n",
J-Alves752236c2022-04-28 11:07:47 +01002506 handle,
2507 memory_region->receivers[i]
2508 .receiver_permissions.receiver);
2509 ret = ffa_error(FFA_DENIED);
2510 goto out;
2511 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002512 }
2513
Andrew Walbranca808b12020-05-15 17:22:28 +01002514 ret = ffa_retrieve_check_update(
J-Alves7db32002021-12-14 14:44:50 +00002515 to_locked, memory_region->sender, share_state->fragments,
Andrew Walbranca808b12020-05-15 17:22:28 +01002516 share_state->fragment_constituent_counts,
J-Alves2a0d2882020-10-29 14:49:50 +00002517 share_state->fragment_count, share_state->sender_orig_mode,
Andrew Walbranca808b12020-05-15 17:22:28 +01002518 FFA_MEM_RECLAIM_32, flags & FFA_MEM_RECLAIM_CLEAR, page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002519
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002520 if (ret.func == FFA_SUCCESS_32) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002521 share_state_free(share_states, share_state, page_pool);
J-Alvesa9cd7e32022-07-01 13:49:33 +01002522 dlog_verbose(
2523 "Freed share state after successful "
2524 "reclaim.\n");
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00002525 }
2526
2527out:
2528 share_states_unlock(&share_states);
2529 return ret;
Jose Marinho09b1db82019-08-08 09:16:59 +01002530}