blob: 23af32e3058d56ebfab92d40b1914b6a198d9b5a [file] [log] [blame]
Andrew Scull18834872018-10-12 11:48:09 +01001/*
Federico Recanati25053ee2022-03-14 15:01:53 +01002 * Copyright 2022 The Hafnium Authors.
Andrew Scull18834872018-10-12 11:48:09 +01003 *
Andrew Walbrane959ec12020-06-17 15:01:09 +01004 * Use of this source code is governed by a BSD-style
5 * license that can be found in the LICENSE file or at
6 * https://opensource.org/licenses/BSD-3-Clause.
Andrew Scull18834872018-10-12 11:48:09 +01007 */
8
Andrew Scull18c78fc2018-08-20 12:57:41 +01009#include "hf/api.h"
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +010010
Andrew Walbran318f5732018-11-20 16:23:42 +000011#include "hf/arch/cpu.h"
Daniel Boulbyb2fb80e2021-02-03 15:09:23 +000012#include "hf/arch/ffa.h"
J-Alves0a21db12024-03-28 12:43:30 +000013#include "hf/arch/memcpy_trapped.h"
Olivier Deprez96a2a262020-06-11 17:21:38 +020014#include "hf/arch/mm.h"
Olivier Deprez112d2b52020-09-30 07:39:23 +020015#include "hf/arch/other_world.h"
Olivier Deprez55a189e2021-06-09 15:45:27 +020016#include "hf/arch/plat/ffa.h"
Andrew Walbran508e63c2018-12-20 17:02:37 +000017#include "hf/arch/timer.h"
Olivier Deprez764fd2e2020-07-29 15:14:09 +020018#include "hf/arch/vm.h"
Andrew Walbran318f5732018-11-20 16:23:42 +000019
Karl Meakin49ec1e42024-05-10 13:08:24 +010020#include "hf/bits.h"
Andrew Scull877ae4b2019-07-02 12:52:33 +010021#include "hf/check.h"
Andrew Walbran318f5732018-11-20 16:23:42 +000022#include "hf/dlog.h"
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010023#include "hf/ffa_internal.h"
24#include "hf/ffa_memory.h"
J-Alves33c47bf2022-09-29 11:36:20 +010025#include "hf/ffa_v1_0.h"
Andrew Scull6386f252018-12-06 13:29:10 +000026#include "hf/mm.h"
Andrew Walbranc1ad4ce2019-05-09 11:41:39 +010027#include "hf/plat/console.h"
Manish Pandeya5f39fb2020-09-11 09:47:11 +010028#include "hf/plat/interrupts.h"
Andrew Scull6386f252018-12-06 13:29:10 +000029#include "hf/spinlock.h"
Andrew Scull877ae4b2019-07-02 12:52:33 +010030#include "hf/static_assert.h"
Andrew Scull8d9e1212019-04-05 13:52:55 +010031#include "hf/std.h"
Karl Meakinc5cebbc2024-06-17 11:30:27 +010032#include "hf/vcpu.h"
Andrew Scull18c78fc2018-08-20 12:57:41 +010033#include "hf/vm.h"
34
Andrew Scullf35a5c92018-08-07 18:09:46 +010035#include "vmapi/hf/call.h"
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010036#include "vmapi/hf/ffa.h"
J-Alves126ab502022-09-29 11:37:33 +010037#include "vmapi/hf/ffa_v1_0.h"
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +010038
Daniel Boulby1ddb3d72021-12-16 18:16:50 +000039static_assert(sizeof(struct ffa_partition_info_v1_0) == 8,
Fuad Tabbae4efcc32020-07-16 15:37:27 +010040 "Partition information descriptor size doesn't match the one in "
41 "the FF-A 1.0 EAC specification, Table 82.");
Daniel Boulby1ddb3d72021-12-16 18:16:50 +000042static_assert(sizeof(struct ffa_partition_info) == 24,
43 "Partition information descriptor size doesn't match the one in "
44 "the FF-A 1.1 BETA0 EAC specification, Table 13.34.");
Raghu Krishnamurthyf67776f2022-12-26 10:10:05 -080045static_assert((sizeof(struct ffa_partition_info) & 7) == 0,
46 "Partition information descriptor must be a multiple of 8 bytes"
47 " for ffa_partition_info_get_regs to work correctly. Information"
48 " from this structure are returned in 8 byte registers and the"
49 " count of 8 byte registers is returned by the ABI.");
Wedson Almeida Filhoba641ef2018-12-03 04:19:44 +000050/*
51 * To eliminate the risk of deadlocks, we define a partial order for the
52 * acquisition of locks held concurrently by the same physical CPU. Our current
53 * ordering requirements are as follows:
54 *
Andrew Walbranc1ad4ce2019-05-09 11:41:39 +010055 * vm::lock -> vcpu::lock -> mm_stage1_lock -> dlog sl
Andrew Scull6386f252018-12-06 13:29:10 +000056 *
Andrew Scull4caadaf2019-07-03 13:13:47 +010057 * Locks of the same kind require the lock of lowest address to be locked first,
58 * see `sl_lock_both()`.
Wedson Almeida Filhoba641ef2018-12-03 04:19:44 +000059 */
60
Andrew Scullaa039b32018-10-04 15:02:26 +010061static_assert(HF_MAILBOX_SIZE == PAGE_SIZE,
Andrew Scull13652af2018-09-17 14:49:08 +010062 "Currently, a page is mapped for the send and receive buffers so "
63 "the maximum request is the size of a page.");
64
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000065static_assert(MM_PPOOL_ENTRY_SIZE >= HF_MAILBOX_SIZE,
66 "The page pool entry size must be at least as big as the mailbox "
67 "size, so that memory region descriptors can be copied from the "
68 "mailbox for memory sharing.");
69
Raghu Krishnamurthybefdcc82023-02-24 06:41:56 -080070/*
71 * Maximum ffa_partition_info entries that can be returned by an invocation
72 * of FFA_PARTITION_INFO_GET_REGS_64 is size in bytes, of available
73 * registers/args in struct ffa_value divided by size of struct
74 * ffa_partition_info. For this ABI, arg3-arg17 in ffa_value can be used, i.e.
75 * 15 uint64_t fields. For FF-A v1.1, this value should be 5.
76 */
77#define MAX_INFO_REGS_ENTRIES_PER_CALL \
78 ((15 * sizeof(uint64_t)) / sizeof(struct ffa_partition_info))
79static_assert(MAX_INFO_REGS_ENTRIES_PER_CALL == 5,
80 "FF-A v1.1 supports no more than 5 entries"
81 " per FFA_PARTITION_INFO_GET_REGS64 calls");
82
Wedson Almeida Filho9ed8da52018-12-17 16:09:11 +000083static struct mpool api_page_pool;
Wedson Almeida Filho22d5eaa2018-12-16 00:38:49 +000084
85/**
Wedson Almeida Filho81568c42019-01-04 13:33:02 +000086 * Initialises the API page pool by taking ownership of the contents of the
87 * given page pool.
Wedson Almeida Filho22d5eaa2018-12-16 00:38:49 +000088 */
89void api_init(struct mpool *ppool)
90{
Wedson Almeida Filho9ed8da52018-12-17 16:09:11 +000091 mpool_init_from(&api_page_pool, ppool);
Wedson Almeida Filho22d5eaa2018-12-16 00:38:49 +000092}
93
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +010094/**
J-Alvesad6a0432021-04-09 16:06:21 +010095 * Get target VM vCPU:
96 * If VM is UP then return first vCPU.
97 * If VM is MP then return vCPU whose index matches current CPU index.
98 */
Madhukar Pappireddyb11e0d12021-08-02 19:44:35 -050099struct vcpu *api_ffa_get_vm_vcpu(struct vm *vm, struct vcpu *current)
J-Alvesad6a0432021-04-09 16:06:21 +0100100{
101 ffa_vcpu_index_t current_cpu_index = cpu_index(current->cpu);
102 struct vcpu *vcpu = NULL;
103
Madhukar Pappireddyb11e0d12021-08-02 19:44:35 -0500104 CHECK((vm != NULL) && (current != NULL));
105
Karl Meakin82041822024-05-20 11:14:34 +0100106 if (vm_is_up(vm)) {
J-Alvesad6a0432021-04-09 16:06:21 +0100107 vcpu = vm_get_vcpu(vm, 0);
108 } else if (current_cpu_index < vm->vcpu_count) {
109 vcpu = vm_get_vcpu(vm, current_cpu_index);
110 }
111
112 return vcpu;
113}
114
115/**
J-Alvesfe7f7372020-11-09 11:32:12 +0000116 * Switches the physical CPU back to the corresponding vCPU of the VM whose ID
117 * is given as argument of the function.
118 *
119 * Called to change the context between SPs for direct messaging (when Hafnium
120 * is SPMC), and on the context of the remaining 'api_switch_to_*' functions.
121 *
122 * This function works for partitions that are:
J-Alvesad6a0432021-04-09 16:06:21 +0100123 * - UP migratable.
J-Alvesfe7f7372020-11-09 11:32:12 +0000124 * - MP with pinned Execution Contexts.
125 */
Madhukar Pappireddybd10e572023-03-06 16:39:49 -0600126struct vcpu *api_switch_to_vm(struct vcpu_locked current_locked,
127 struct ffa_value to_ret,
J-Alves19e20cf2023-08-02 12:48:55 +0100128 enum vcpu_state vcpu_state, ffa_id_t to_id)
J-Alvesfe7f7372020-11-09 11:32:12 +0000129{
130 struct vm *to_vm = vm_find(to_id);
Madhukar Pappireddybd10e572023-03-06 16:39:49 -0600131 struct vcpu *next = api_ffa_get_vm_vcpu(to_vm, current_locked.vcpu);
J-Alvesfe7f7372020-11-09 11:32:12 +0000132
133 CHECK(next != NULL);
134
135 /* Set the return value for the target VM. */
136 arch_regs_set_retval(&next->regs, to_ret);
137
138 /* Set the current vCPU state. */
Madhukar Pappireddybd10e572023-03-06 16:39:49 -0600139 current_locked.vcpu->state = vcpu_state;
J-Alvesfe7f7372020-11-09 11:32:12 +0000140
141 return next;
142}
143
144/**
Fuad Tabbab0ef2a42019-12-19 11:19:25 +0000145 * Switches the physical CPU back to the corresponding vCPU of the primary VM.
Andrew Scullaa039b32018-10-04 15:02:26 +0100146 *
147 * This triggers the scheduling logic to run. Run in the context of secondary VM
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100148 * to cause FFA_RUN to return and the primary VM to regain control of the CPU.
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100149 */
Madhukar Pappireddybd10e572023-03-06 16:39:49 -0600150struct vcpu *api_switch_to_primary(struct vcpu_locked current_locked,
J-Alves27b71962022-12-12 15:29:58 +0000151 struct ffa_value primary_ret,
152 enum vcpu_state secondary_state)
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100153{
Andrew Walbran508e63c2018-12-20 17:02:37 +0000154 /*
155 * If the secondary is blocked but has a timer running, sleep until the
156 * timer fires rather than indefinitely.
157 */
Andrew Walbran7a1ea0b2019-10-02 18:18:44 +0100158 switch (primary_ret.func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100159 case HF_FFA_RUN_WAIT_FOR_INTERRUPT:
160 case FFA_MSG_WAIT_32: {
Andrew Walbran7a1ea0b2019-10-02 18:18:44 +0100161 if (arch_timer_enabled_current()) {
162 uint64_t remaining_ns =
163 arch_timer_remaining_ns_current();
164
165 if (remaining_ns == 0) {
166 /*
167 * Timer is pending, so the current vCPU should
168 * be run again right away.
169 */
Andrew Walbran7e824602020-10-22 16:51:40 +0100170 primary_ret = (struct ffa_value){
171 .func = FFA_INTERRUPT_32};
172
Andrew Walbran7a1ea0b2019-10-02 18:18:44 +0100173 } else {
174 primary_ret.arg2 = remaining_ns;
175 }
176 } else {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100177 primary_ret.arg2 = FFA_SLEEP_INDEFINITE;
Andrew Walbran7a1ea0b2019-10-02 18:18:44 +0100178 }
Andrew Scullb06d1752019-02-04 10:15:48 +0000179 break;
Andrew Walbran7a1ea0b2019-10-02 18:18:44 +0100180 }
Andrew Scullb06d1752019-02-04 10:15:48 +0000181
182 default:
183 /* Do nothing. */
184 break;
Andrew Walbran508e63c2018-12-20 17:02:37 +0000185 }
186
Madhukar Pappireddybd10e572023-03-06 16:39:49 -0600187 return api_switch_to_vm(current_locked, primary_ret, secondary_state,
J-Alvesfe7f7372020-11-09 11:32:12 +0000188 HF_PRIMARY_VM_ID);
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100189}
190
191/**
Olivier Deprez2ebae3a2020-06-11 16:34:30 +0200192 * Choose next vCPU to run to be the counterpart vCPU in the other
193 * world (run the normal world if currently running in the secure
194 * world). Set current vCPU state to the given vcpu_state parameter.
195 * Set FF-A return values to the target vCPU in the other world.
196 *
197 * Called in context of a direct message response from a secure
198 * partition to a VM.
199 */
Madhukar Pappireddybd10e572023-03-06 16:39:49 -0600200struct vcpu *api_switch_to_other_world(struct vcpu_locked current_locked,
Manish Pandeya5f39fb2020-09-11 09:47:11 +0100201 struct ffa_value other_world_ret,
202 enum vcpu_state vcpu_state)
Olivier Deprez2ebae3a2020-06-11 16:34:30 +0200203{
Madhukar Pappireddybd10e572023-03-06 16:39:49 -0600204 return api_switch_to_vm(current_locked, other_world_ret, vcpu_state,
J-Alvesfe7f7372020-11-09 11:32:12 +0000205 HF_OTHER_WORLD_ID);
Olivier Deprez2ebae3a2020-06-11 16:34:30 +0200206}
207
208/**
Olivier Deprezee9d6a92019-11-26 09:14:11 +0000209 * Returns true if the given vCPU is executing in context of an
210 * FFA_MSG_SEND_DIRECT_REQ invocation.
211 */
J-Alves27b71962022-12-12 15:29:58 +0000212bool is_ffa_direct_msg_request_ongoing(struct vcpu_locked locked)
Olivier Deprezee9d6a92019-11-26 09:14:11 +0000213{
Kathleen Capellae468c112023-12-13 17:56:28 -0500214 return locked.vcpu->direct_request_origin.vm_id != HF_INVALID_VM_ID;
Olivier Deprezee9d6a92019-11-26 09:14:11 +0000215}
216
217/**
Manish Pandeya5f39fb2020-09-11 09:47:11 +0100218 * Returns true if the VM owning the given vCPU is supporting managed exit and
219 * the vCPU is currently processing a managed exit.
220 */
221static bool api_ffa_is_managed_exit_ongoing(struct vcpu_locked vcpu_locked)
222{
Maksims Svecovs9ddf86a2021-05-06 17:17:21 +0100223 return (plat_ffa_vm_managed_exit_supported(vcpu_locked.vcpu->vm) &&
Manish Pandeya5f39fb2020-09-11 09:47:11 +0100224 vcpu_locked.vcpu->processing_managed_exit);
225}
226
227/**
Fuad Tabbab0ef2a42019-12-19 11:19:25 +0000228 * Returns to the primary VM and signals that the vCPU still has work to do so.
Andrew Scull33fecd32019-01-08 14:48:27 +0000229 */
230struct vcpu *api_preempt(struct vcpu *current)
231{
Madhukar Pappireddybd10e572023-03-06 16:39:49 -0600232 struct vcpu_locked current_locked;
233 struct vcpu *next;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100234 struct ffa_value ret = {
235 .func = FFA_INTERRUPT_32,
Olivier Deprez0d725f52022-07-06 14:20:27 +0200236 .arg1 = ffa_vm_vcpu(current->vm->id, vcpu_index(current)),
Andrew Scull33fecd32019-01-08 14:48:27 +0000237 };
238
Madhukar Pappireddybd10e572023-03-06 16:39:49 -0600239 current_locked = vcpu_lock(current);
240 next = api_switch_to_primary(current_locked, ret, VCPU_STATE_PREEMPTED);
241 vcpu_unlock(&current_locked);
242
243 return next;
Andrew Scull33fecd32019-01-08 14:48:27 +0000244}
245
246/**
Fuad Tabbab0ef2a42019-12-19 11:19:25 +0000247 * Puts the current vCPU in wait for interrupt mode, and returns to the primary
Fuad Tabbaed294af2019-12-20 10:43:01 +0000248 * VM.
Andrew Scullaa039b32018-10-04 15:02:26 +0100249 */
Wedson Almeida Filho00df6c72018-10-18 11:19:24 +0100250struct vcpu *api_wait_for_interrupt(struct vcpu *current)
Andrew Scullaa039b32018-10-04 15:02:26 +0100251{
Madhukar Pappireddybd10e572023-03-06 16:39:49 -0600252 struct vcpu_locked current_locked;
253 struct vcpu *next;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100254 struct ffa_value ret = {
255 .func = HF_FFA_RUN_WAIT_FOR_INTERRUPT,
256 .arg1 = ffa_vm_vcpu(current->vm->id, vcpu_index(current)),
Andrew Scull6d2db332018-10-10 15:28:17 +0100257 };
Wedson Almeida Filho81568c42019-01-04 13:33:02 +0000258
Madhukar Pappireddybd10e572023-03-06 16:39:49 -0600259 current_locked = vcpu_lock(current);
260 next = api_switch_to_primary(current_locked, ret,
Andrew Sculld6ee1102019-04-05 22:12:42 +0100261 VCPU_STATE_BLOCKED_INTERRUPT);
Madhukar Pappireddybd10e572023-03-06 16:39:49 -0600262 vcpu_unlock(&current_locked);
263
264 return next;
Andrew Scullaa039b32018-10-04 15:02:26 +0100265}
266
267/**
Andrew Walbran33645652019-04-15 12:29:31 +0100268 * Puts the current vCPU in off mode, and returns to the primary VM.
269 */
270struct vcpu *api_vcpu_off(struct vcpu *current)
271{
Madhukar Pappireddybd10e572023-03-06 16:39:49 -0600272 struct vcpu_locked current_locked;
273 struct vcpu *next;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100274 struct ffa_value ret = {
275 .func = HF_FFA_RUN_WAIT_FOR_INTERRUPT,
276 .arg1 = ffa_vm_vcpu(current->vm->id, vcpu_index(current)),
Andrew Walbran33645652019-04-15 12:29:31 +0100277 };
278
Madhukar Pappireddybd10e572023-03-06 16:39:49 -0600279 current_locked = vcpu_lock(current);
Andrew Walbran33645652019-04-15 12:29:31 +0100280 /*
281 * Disable the timer, so the scheduler doesn't get told to call back
282 * based on it.
283 */
284 arch_timer_disable_current();
285
Madhukar Pappireddybd10e572023-03-06 16:39:49 -0600286 next = api_switch_to_primary(current_locked, ret, VCPU_STATE_OFF);
287 vcpu_unlock(&current_locked);
288
289 return next;
Andrew Walbran33645652019-04-15 12:29:31 +0100290}
291
292/**
Madhukar Pappireddyb11e0d12021-08-02 19:44:35 -0500293 * The current vCPU is blocked on some resource and needs to relinquish
294 * control back to the execution context of the endpoint that originally
295 * allocated cycles to it.
Andrew Scull66d62bf2019-02-01 13:54:10 +0000296 */
Madhukar Pappireddy184501c2023-05-23 17:24:06 -0500297struct ffa_value api_yield(struct vcpu *current, struct vcpu **next,
298 struct ffa_value *args)
Andrew Scull66d62bf2019-02-01 13:54:10 +0000299{
Olivier Deprezee9d6a92019-11-26 09:14:11 +0000300 struct ffa_value ret = (struct ffa_value){.func = FFA_SUCCESS_32};
301 struct vcpu_locked current_locked;
Madhukar Pappireddyc857ac22022-06-21 17:37:53 -0500302 bool transition_allowed;
Madhukar Pappireddyd456ac22023-06-26 15:29:34 -0500303 enum vcpu_state next_state = VCPU_STATE_RUNNING;
Madhukar Pappireddy184501c2023-05-23 17:24:06 -0500304 uint32_t timeout_low = 0;
305 uint32_t timeout_high = 0;
Madhukar Pappireddybd10e572023-03-06 16:39:49 -0600306 struct vcpu_locked next_locked = (struct vcpu_locked){
307 .vcpu = NULL,
308 };
Madhukar Pappireddy184501c2023-05-23 17:24:06 -0500309
310 if (args != NULL) {
311 if (args->arg4 != 0U || args->arg5 != 0U || args->arg6 != 0U ||
312 args->arg7 != 0U) {
313 dlog_error(
314 "Parameters passed through registers X4-X7 "
315 "must be zero\n");
316 return ffa_error(FFA_INVALID_PARAMETERS);
317 }
318 timeout_low = (uint32_t)args->arg2 & 0xFFFFFFFF;
319 timeout_high = (uint32_t)args->arg3 & 0xFFFFFFFF;
320 }
Andrew Scull66d62bf2019-02-01 13:54:10 +0000321
Karl Meakin5e996992024-05-20 11:27:07 +0100322 if (vm_is_primary(current->vm)) {
Fuad Tabbab0ef2a42019-12-19 11:19:25 +0000323 /* NOOP on the primary as it makes the scheduling decisions. */
Olivier Deprezee9d6a92019-11-26 09:14:11 +0000324 return ret;
Andrew Scull66d62bf2019-02-01 13:54:10 +0000325 }
326
Olivier Deprezee9d6a92019-11-26 09:14:11 +0000327 current_locked = vcpu_lock(current);
Madhukar Pappireddyc857ac22022-06-21 17:37:53 -0500328 transition_allowed = plat_ffa_check_runtime_state_transition(
Madhukar Pappireddybd10e572023-03-06 16:39:49 -0600329 current_locked, current->vm->id, HF_INVALID_VM_ID, next_locked,
330 FFA_YIELD_32, &next_state);
Olivier Deprezee9d6a92019-11-26 09:14:11 +0000331
Madhukar Pappireddyc857ac22022-06-21 17:37:53 -0500332 if (!transition_allowed) {
Madhukar Pappireddybd10e572023-03-06 16:39:49 -0600333 ret = ffa_error(FFA_DENIED);
334 goto out;
Olivier Deprezee9d6a92019-11-26 09:14:11 +0000335 }
336
Madhukar Pappireddy65764072023-05-23 17:18:58 -0500337 /*
338 * The current vCPU is expected to move to BLOCKED state. However,
339 * under certain circumstances, it is allowed for the current vCPU
340 * to be resumed immediately without ever moving to BLOCKED state. One
341 * such scenario occurs when an SP's execution context attempts to
342 * yield cycles while handling secure interrupt. Refer to the comments
343 * in the SPMC variant of the plat_ffa_yield_prepare function.
344 */
Madhukar Pappireddyd456ac22023-06-26 15:29:34 -0500345 assert(!vm_id_is_current_world(current->vm->id) ||
346 next_state == VCPU_STATE_BLOCKED);
Madhukar Pappireddyc857ac22022-06-21 17:37:53 -0500347
Madhukar Pappireddybd10e572023-03-06 16:39:49 -0600348 ret = plat_ffa_yield_prepare(current_locked, next, timeout_low,
349 timeout_high);
350out:
351 vcpu_unlock(&current_locked);
352 return ret;
Andrew Scull66d62bf2019-02-01 13:54:10 +0000353}
354
355/**
Andrew Walbran33645652019-04-15 12:29:31 +0100356 * Switches to the primary so that it can switch to the target, or kick it if it
357 * is already running on a different physical CPU.
358 */
Madhukar Pappireddybd10e572023-03-06 16:39:49 -0600359static struct vcpu *api_wake_up_locked(struct vcpu_locked current_locked,
360 struct vcpu *target_vcpu)
Andrew Walbran33645652019-04-15 12:29:31 +0100361{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100362 struct ffa_value ret = {
Andrew Walbran7e824602020-10-22 16:51:40 +0100363 .func = FFA_INTERRUPT_32,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100364 .arg1 = ffa_vm_vcpu(target_vcpu->vm->id,
365 vcpu_index(target_vcpu)),
Andrew Walbran33645652019-04-15 12:29:31 +0100366 };
Madhukar Pappireddybd10e572023-03-06 16:39:49 -0600367
368 return api_switch_to_primary(current_locked, ret, VCPU_STATE_BLOCKED);
369}
370
371struct vcpu *api_wake_up(struct vcpu *current, struct vcpu *target_vcpu)
372{
373 struct vcpu_locked current_locked;
374 struct vcpu *next;
375
376 current_locked = vcpu_lock(current);
377 next = api_wake_up_locked(current_locked, target_vcpu);
378 vcpu_unlock(&current_locked);
379
380 return next;
Andrew Walbran33645652019-04-15 12:29:31 +0100381}
382
383/**
Andrew Scull38772ab2019-01-24 15:16:50 +0000384 * Aborts the vCPU and triggers its VM to abort fully.
Andrew Scull9726c252019-01-23 13:44:19 +0000385 */
386struct vcpu *api_abort(struct vcpu *current)
387{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100388 struct ffa_value ret = ffa_error(FFA_ABORTED);
Madhukar Pappireddybd10e572023-03-06 16:39:49 -0600389 struct vcpu_locked current_locked;
390 struct vcpu *next;
Madhukar Pappireddy9e09d7a2023-08-08 14:53:49 -0500391 struct vm_locked vm_locked;
Andrew Scull9726c252019-01-23 13:44:19 +0000392
Olivier Deprezf92e5d42020-11-13 16:00:54 +0100393 dlog_notice("Aborting VM %#x vCPU %u\n", current->vm->id,
Andrew Walbran17eebf92020-02-05 16:35:49 +0000394 vcpu_index(current));
Andrew Scull9726c252019-01-23 13:44:19 +0000395
Karl Meakin5e996992024-05-20 11:27:07 +0100396 if (vm_is_primary(current->vm)) {
Andrew Scull9726c252019-01-23 13:44:19 +0000397 /* TODO: what to do when the primary aborts? */
398 for (;;) {
399 /* Do nothing. */
400 }
401 }
402
403 atomic_store_explicit(&current->vm->aborting, true,
404 memory_order_relaxed);
405
Madhukar Pappireddy9e09d7a2023-08-08 14:53:49 -0500406 vm_locked = vm_lock(current->vm);
407 plat_ffa_free_vm_resources(vm_locked);
408 vm_unlock(&vm_locked);
Andrew Scull9726c252019-01-23 13:44:19 +0000409
Madhukar Pappireddybd10e572023-03-06 16:39:49 -0600410 current_locked = vcpu_lock(current);
411 next = api_switch_to_primary(current_locked, ret, VCPU_STATE_ABORTED);
412 vcpu_unlock(&current_locked);
413
414 return next;
Andrew Scull9726c252019-01-23 13:44:19 +0000415}
416
Daniel Boulby1ddb3d72021-12-16 18:16:50 +0000417/*
418 * Format the partition info descriptors according to the version supported
419 * by the endpoint and return the size of the array created.
420 */
421static struct ffa_value send_versioned_partition_info_descriptors(
Daniel Boulby191b5f02022-02-17 17:26:14 +0000422 struct vm_locked vm_locked, struct ffa_partition_info *partitions,
Daniel Boulby1ddb3d72021-12-16 18:16:50 +0000423 uint32_t vm_count)
424{
Daniel Boulby191b5f02022-02-17 17:26:14 +0000425 struct vm *vm = vm_locked.vm;
Karl Meakin0e617d92024-04-05 12:55:22 +0100426 enum ffa_version version = vm->ffa_version;
Daniel Boulby835e1592022-05-30 17:38:51 +0100427 uint32_t partition_info_size;
428 uint32_t buffer_size;
Federico Recanati644f0462022-03-17 12:04:00 +0100429 struct ffa_value ret;
Daniel Boulby1ddb3d72021-12-16 18:16:50 +0000430
J-Alvese8c8c2b2022-12-16 15:34:48 +0000431 /* Acquire receiver's RX buffer. */
432 if (!plat_ffa_acquire_receiver_rx(vm_locked, &ret)) {
433 dlog_verbose("Failed to acquire RX buffer for VM %x\n", vm->id);
434 return ret;
435 }
436
J-Alves122f1a12022-12-12 15:55:42 +0000437 if (vm_is_mailbox_busy(vm_locked)) {
Daniel Boulby1ddb3d72021-12-16 18:16:50 +0000438 /*
439 * Can't retrieve memory information if the mailbox is not
440 * available.
441 */
442 dlog_verbose("RX buffer not ready.\n");
Daniel Boulby191b5f02022-02-17 17:26:14 +0000443 return ffa_error(FFA_BUSY);
Daniel Boulby1ddb3d72021-12-16 18:16:50 +0000444 }
445
Karl Meakin0e617d92024-04-05 12:55:22 +0100446 if (version == FFA_VERSION_1_0) {
Daniel Boulby191b5f02022-02-17 17:26:14 +0000447 struct ffa_partition_info_v1_0 *recv_mailbox = vm->mailbox.recv;
448
Daniel Boulby835e1592022-05-30 17:38:51 +0100449 partition_info_size = sizeof(struct ffa_partition_info_v1_0);
450 buffer_size = partition_info_size * vm_count;
451 if (buffer_size > HF_MAILBOX_SIZE) {
Daniel Boulby191b5f02022-02-17 17:26:14 +0000452 dlog_error(
453 "Partition information does not fit in the "
J-Alves0a21db12024-03-28 12:43:30 +0000454 "VM's RX buffer.\n");
Daniel Boulby191b5f02022-02-17 17:26:14 +0000455 return ffa_error(FFA_NO_MEMORY);
456 }
457
458 for (uint32_t i = 0; i < vm_count; i++) {
459 /*
460 * Populate the VM's RX buffer with the partition
Kathleen Capella402fa852022-11-09 16:16:51 -0500461 * information. Clear properties bits that must be zero
462 * according to DEN0077A FF-A v1.0 REL Table 8.25.
Daniel Boulby191b5f02022-02-17 17:26:14 +0000463 */
464 recv_mailbox[i].vm_id = partitions[i].vm_id;
465 recv_mailbox[i].vcpu_count = partitions[i].vcpu_count;
Kathleen Capella402fa852022-11-09 16:16:51 -0500466 recv_mailbox[i].properties =
467 partitions[i].properties &
468 ~FFA_PARTITION_v1_0_RES_MASK;
Daniel Boulby191b5f02022-02-17 17:26:14 +0000469 }
470
471 } else {
Daniel Boulby835e1592022-05-30 17:38:51 +0100472 partition_info_size = sizeof(struct ffa_partition_info);
473 buffer_size = partition_info_size * vm_count;
J-Alves0a21db12024-03-28 12:43:30 +0000474
Daniel Boulby835e1592022-05-30 17:38:51 +0100475 if (buffer_size > HF_MAILBOX_SIZE) {
Daniel Boulby191b5f02022-02-17 17:26:14 +0000476 dlog_error(
477 "Partition information does not fit in the "
J-Alves0a824e92024-04-26 16:20:12 +0100478 "VM's RX buffer.\n");
Daniel Boulby191b5f02022-02-17 17:26:14 +0000479 return ffa_error(FFA_NO_MEMORY);
480 }
481
J-Alves0a21db12024-03-28 12:43:30 +0000482 /*
483 * Populate the VM's RX buffer with the partition information.
Daniel Boulby191b5f02022-02-17 17:26:14 +0000484 */
J-Alves0a21db12024-03-28 12:43:30 +0000485 if (!memcpy_trapped(vm->mailbox.recv, HF_MAILBOX_SIZE,
486 partitions, buffer_size)) {
487 dlog_error(
488 "%s: Failed to copy ffa_partition_info "
489 "descriptor\n",
490 __func__);
491 return ffa_error(FFA_ABORTED);
492 }
Daniel Boulby191b5f02022-02-17 17:26:14 +0000493 }
494
Daniel Boulby835e1592022-05-30 17:38:51 +0100495 vm->mailbox.recv_size = buffer_size;
Daniel Boulby1ddb3d72021-12-16 18:16:50 +0000496
497 /* Sender is Hypervisor in the normal world (TEE in secure world). */
Daniel Boulby191b5f02022-02-17 17:26:14 +0000498 vm->mailbox.recv_sender = HF_VM_ID_BASE;
499 vm->mailbox.recv_func = FFA_PARTITION_INFO_GET_32;
J-Alvese8c8c2b2022-12-16 15:34:48 +0000500 vm->mailbox.state = MAILBOX_STATE_FULL;
Daniel Boulby1ddb3d72021-12-16 18:16:50 +0000501
Daniel Boulby835e1592022-05-30 17:38:51 +0100502 /*
503 * Return the count of partition information descriptors in w2
504 * and the size of the descriptors in w3.
505 */
506 return (struct ffa_value){.func = FFA_SUCCESS_32,
507 .arg2 = vm_count,
508 .arg3 = partition_info_size};
Daniel Boulby1ddb3d72021-12-16 18:16:50 +0000509}
510
Raghu Krishnamurthyf67776f2022-12-26 10:10:05 -0800511static void api_ffa_fill_partitions_info_array(
512 struct ffa_partition_info *partitions, size_t partitions_len,
J-Alves19e20cf2023-08-02 12:48:55 +0100513 const struct ffa_uuid *uuid, bool count_flag, ffa_id_t vm_id,
Raghu Krishnamurthyf67776f2022-12-26 10:10:05 -0800514 ffa_vm_count_t *vm_count_out)
515{
516 ffa_vm_count_t vm_count = 0;
517 bool uuid_is_null = ffa_uuid_is_null(uuid);
518
519 assert(vm_get_count() <= partitions_len);
520
521 /*
522 * Iterate through the VMs to find the ones with a matching
523 * UUID. A Null UUID retrieves information for all VMs.
524 */
525 for (ffa_vm_count_t index = 0; index < vm_get_count(); ++index) {
526 struct vm *vm = vm_find_index(index);
527
Kathleen Capellaa1de3c52023-08-28 20:10:02 -0400528 for (uint32_t i = 0; i < PARTITION_MAX_UUIDS; i++) {
529 /*
530 * Null UUID indicates reaching the end of a
531 * partition's array of UUIDs.
532 */
533 if (ffa_uuid_is_null(&vm->uuids[i])) {
534 break;
Raghu Krishnamurthyf67776f2022-12-26 10:10:05 -0800535 }
Kathleen Capellaa1de3c52023-08-28 20:10:02 -0400536 if (uuid_is_null ||
537 ffa_uuid_equal(uuid, &vm->uuids[i])) {
538 uint16_t array_index = vm_count;
Raghu Krishnamurthyf67776f2022-12-26 10:10:05 -0800539
Kathleen Capellaa1de3c52023-08-28 20:10:02 -0400540 ++vm_count;
541 if (count_flag) {
542 continue;
543 }
544
545 partitions[array_index].vm_id = vm->id;
546 partitions[array_index].vcpu_count =
547 vm->vcpu_count;
548 partitions[array_index].properties =
549 plat_ffa_partition_properties(vm_id,
550 vm);
551 partitions[array_index].properties |=
552 vm_are_notifications_enabled(vm)
553 ? FFA_PARTITION_NOTIFICATION
554 : 0;
555 partitions[array_index].properties |=
556 FFA_PARTITION_AARCH64_EXEC;
557 if (uuid_is_null) {
558 partitions[array_index].uuid =
559 vm->uuids[i];
560 }
Raghu Krishnamurthyf67776f2022-12-26 10:10:05 -0800561 }
562 }
563 }
564
565 *vm_count_out = vm_count;
566}
Raghu Krishnamurthybefdcc82023-02-24 06:41:56 -0800567
568static inline void api_ffa_pack_vmid_count_props(
J-Alves19e20cf2023-08-02 12:48:55 +0100569 uint64_t *xn, ffa_id_t vm_id, ffa_vcpu_count_t vcpu_count,
Raghu Krishnamurthybefdcc82023-02-24 06:41:56 -0800570 ffa_partition_properties_t properties)
571{
572 *xn = (uint64_t)vm_id;
573 *xn |= (uint64_t)vcpu_count << 16;
574 *xn |= (uint64_t)properties << 32;
575}
576
577static inline void api_ffa_pack_uuid(uint64_t *xn_1, uint64_t *xn_2,
578 struct ffa_uuid *uuid)
579{
580 *xn_1 = (uint64_t)uuid->uuid[0];
581 *xn_1 |= (uint64_t)uuid->uuid[1] << 32;
582 *xn_2 = (uint64_t)uuid->uuid[2];
583 *xn_2 |= (uint64_t)uuid->uuid[3] << 32;
584}
585
Raghu Krishnamurthy2d2b0f42023-04-23 09:54:45 -0700586/**
Raghu Krishnamurthye74d6532023-06-07 12:21:54 -0700587 * This function forwards the FFA_PARTITION_INFO_GET_REGS ABI to the other world
588 * when hafnium is the hypervisor to determine the secure partitions. When
589 * hafnium is the SPMC, this function forwards the call to the SPMD to discover
590 * SPMD logical partitions. The function returns true when partition information
591 * is filled in the partitions array and false if there are errors. Note that
592 * the SPMD and SPMC may return an FF-A error code of FFA_NOT_SUPPORTED when
593 * there are no SPMD logical partitions or no secure partitions respectively,
594 * and this is not considered a failure of the forwarded call. A caller is
595 * expected to check the return value before consuming the information in the
596 * partitions array passed in and ret_count.
Raghu Krishnamurthy2d2b0f42023-04-23 09:54:45 -0700597 */
Raghu Krishnamurthye74d6532023-06-07 12:21:54 -0700598static bool api_ffa_partition_info_get_regs_forward(
599 const struct ffa_uuid *uuid, const uint16_t tag,
600 struct ffa_partition_info *partitions, uint16_t partitions_len,
601 ffa_vm_count_t *ret_count)
602{
603 (void)tag;
604 struct ffa_value ret;
605 uint16_t last_index = UINT16_MAX;
606 uint16_t curr_index = 0;
607 uint16_t start_index = 0;
608
609 if (!plat_ffa_partition_info_get_regs_forward_allowed()) {
610 return true;
611 }
612
613 while (start_index <= last_index) {
614 ret = ffa_partition_info_get_regs(uuid, start_index, 0);
615 if (ffa_func_id(ret) != FFA_SUCCESS_64) {
616 /*
617 * If there are no logical partitions, SPMD returns
618 * NOT_SUPPORTED, that is not an error. If there are no
619 * secure partitions the SPMC returns NOT_SUPPORTED.
620 */
621 if ((ffa_func_id(ret) == FFA_ERROR_32) &&
622 (ffa_error_code(ret) == FFA_NOT_SUPPORTED)) {
623 return true;
624 }
625
626 return false;
627 }
628
629 if (!api_ffa_fill_partition_info_from_regs(
630 ret, start_index, partitions, partitions_len,
631 ret_count)) {
632 return false;
633 }
634
635 last_index = ffa_partition_info_regs_get_last_idx(ret);
636 curr_index = ffa_partition_info_regs_get_curr_idx(ret);
637 start_index = curr_index + 1;
638 }
639 return true;
640}
641
Raghu Krishnamurthy2d2b0f42023-04-23 09:54:45 -0700642bool api_ffa_fill_partition_info_from_regs(
643 struct ffa_value ret, uint16_t start_index,
644 struct ffa_partition_info *partitions, uint16_t partitions_len,
645 ffa_vm_count_t *ret_count)
646{
647 uint16_t vm_count = *ret_count;
648 uint16_t curr_index = 0;
649 uint8_t num_entries = 0;
650 uint8_t idx = 0;
651 /* List of pointers to args in return value. */
652 uint64_t *arg_ptrs[] = {
653 &ret.arg3,
654 &ret.arg4,
655 &ret.arg5,
656 &ret.arg6,
657 &ret.arg7,
658 &ret.extended_val.arg8,
659 &ret.extended_val.arg9,
660 &ret.extended_val.arg10,
661 &ret.extended_val.arg11,
662 &ret.extended_val.arg12,
663 &ret.extended_val.arg13,
664 &ret.extended_val.arg14,
665 &ret.extended_val.arg15,
666 &ret.extended_val.arg16,
667 &ret.extended_val.arg17,
668 };
669
670 if (vm_count > partitions_len) {
671 return false;
672 }
673
674 /*
675 * Tags are currently unused in the implementation. Expect it to be
676 * zero since the implementation does not provide a tag when calling
677 * the FFA_PARTITION_INFO_GET_REGS ABI.
678 */
679 assert(ffa_partition_info_regs_get_tag(ret) == 0);
680
681 /*
682 * Hafnium expects the size of the returned descriptor to be equal to
683 * the size of the structure in the FF-A 1.1 specification. When future
684 * enhancements are made, this assert can be relaxed.
685 */
686 assert(ffa_partition_info_regs_get_desc_size(ret) ==
687 sizeof(struct ffa_partition_info));
688
689 curr_index = ffa_partition_info_regs_get_curr_idx(ret);
690
691 /* FF-A 1.2 ALP0, section 14.9.2 Usage rule 7. */
692 assert(start_index <= curr_index);
693
694 num_entries = curr_index - start_index + 1;
695 if (num_entries > (partitions_len - vm_count) ||
696 num_entries > MAX_INFO_REGS_ENTRIES_PER_CALL) {
697 return false;
698 }
699
700 while (num_entries) {
701 uint64_t info = *(arg_ptrs[(ptrdiff_t)(idx++)]);
702 uint64_t uuid_lo = *(arg_ptrs[(ptrdiff_t)(idx++)]);
703 uint64_t uuid_high = *(arg_ptrs[(ptrdiff_t)(idx++)]);
704
705 partitions[vm_count].vm_id = info & 0xFFFF;
706 partitions[vm_count].vcpu_count = (info >> 16) & 0xFFFF;
707 partitions[vm_count].properties = (info >> 32);
708 partitions[vm_count].uuid.uuid[0] = uuid_lo & 0xFFFFFFFF;
709 partitions[vm_count].uuid.uuid[1] =
710 (uuid_lo >> 32) & 0xFFFFFFFF;
711 partitions[vm_count].uuid.uuid[2] = uuid_high & 0xFFFFFFFF;
712 partitions[vm_count].uuid.uuid[3] =
713 (uuid_high >> 32) & 0xFFFFFFFF;
714 vm_count++;
715 num_entries--;
716 }
717
718 *ret_count = vm_count;
719 return true;
720}
721
Raghu Krishnamurthy7592bcb2022-12-25 13:09:00 -0800722struct ffa_value api_ffa_partition_info_get_regs(struct vcpu *current,
723 const struct ffa_uuid *uuid,
724 const uint16_t start_index,
725 const uint16_t tag)
726{
Raghu Krishnamurthyf67776f2022-12-26 10:10:05 -0800727 struct vm *current_vm = current->vm;
Raghu Krishnamurthyef432cb2022-12-29 06:56:32 -0800728 static struct ffa_partition_info partitions[2 * MAX_VMS];
Raghu Krishnamurthyf67776f2022-12-26 10:10:05 -0800729 bool uuid_is_null = ffa_uuid_is_null(uuid);
730 ffa_vm_count_t vm_count = 0;
731 struct ffa_value ret = ffa_error(FFA_INVALID_PARAMETERS);
732 uint16_t max_idx = 0;
733 uint16_t curr_idx = 0;
Raghu Krishnamurthyf67776f2022-12-26 10:10:05 -0800734 uint8_t num_entries_to_ret = 0;
Raghu Krishnamurthybefdcc82023-02-24 06:41:56 -0800735 uint8_t arg_idx = 3;
736
Raghu Krishnamurthyf67776f2022-12-26 10:10:05 -0800737 /* list of pointers to args in return value */
Raghu Krishnamurthybefdcc82023-02-24 06:41:56 -0800738 uint64_t *arg_ptrs[] = {
739 &(ret).func,
740 &(ret).arg1,
741 &(ret).arg2,
742 &(ret).arg3,
743 &(ret).arg4,
744 &(ret).arg5,
745 &(ret).arg6,
746 &(ret).arg7,
747 &(ret).extended_val.arg8,
748 &(ret).extended_val.arg9,
749 &(ret).extended_val.arg10,
750 &(ret).extended_val.arg11,
751 &(ret).extended_val.arg12,
752 &(ret).extended_val.arg13,
753 &(ret).extended_val.arg14,
754 &(ret).extended_val.arg15,
755 &(ret).extended_val.arg16,
756 &(ret).extended_val.arg17,
Raghu Krishnamurthyf67776f2022-12-26 10:10:05 -0800757 };
758
759 /* TODO: Add support for using tags */
760 if (tag != 0) {
Raghu Krishnamurthybefdcc82023-02-24 06:41:56 -0800761 dlog_error("Tag not 0. Unsupported tag. %d\n", tag);
Raghu Krishnamurthyf67776f2022-12-26 10:10:05 -0800762 return ffa_error(FFA_RETRY);
763 }
764
765 memset_s(&partitions, sizeof(partitions), 0, sizeof(partitions));
766
767 api_ffa_fill_partitions_info_array(partitions, ARRAY_SIZE(partitions),
768 uuid, false, current_vm->id,
769 &vm_count);
770
771 /* If UUID is Null vm_count must not be zero at this stage. */
772 CHECK(!uuid_is_null || vm_count != 0);
773
Raghu Krishnamurthyef432cb2022-12-29 06:56:32 -0800774 /*
775 * When running the Hypervisor:
776 * - If UUID is Null the Hypervisor forwards the query to the SPMC for
777 * it to fill with secure partitions information.
Raghu Krishnamurthy2d2b0f42023-04-23 09:54:45 -0700778 * When running the SPMC, the SPMC forwards the call to the SPMD to
779 * discover any EL3 SPMD logical partitions, if the call came from an
780 * SP. Otherwise, the call is not forwarded.
Raghu Krishnamurthyef432cb2022-12-29 06:56:32 -0800781 * TODO: Note that for this ABI, forwarding on every invocation when
782 * uuid is Null is inefficient,and if performance becomes a problem,
783 * this would be a good place to optimize using strategies such as
784 * caching info etc. For now, assuming this inefficiency is not a major
785 * issue.
786 * - If UUID is non-Null vm_count may be zero because the UUID matches
787 * a secure partition and the query is forwarded to the SPMC.
788 * When running the SPMC:
789 * - If UUID is non-Null and vm_count is zero it means there is no such
790 * partition identified in the system.
791 */
Raghu Krishnamurthy2d2b0f42023-04-23 09:54:45 -0700792 if (vm_id_is_current_world(current_vm->id)) {
Raghu Krishnamurthye74d6532023-06-07 12:21:54 -0700793 if (!api_ffa_partition_info_get_regs_forward(
794 uuid, tag, partitions, ARRAY_SIZE(partitions),
795 &vm_count)) {
Raghu Krishnamurthy2d2b0f42023-04-23 09:54:45 -0700796 dlog_error(
797 "Failed to forward "
798 "ffa_partition_info_get_regs.\n");
799 return ffa_error(FFA_DENIED);
800 }
Raghu Krishnamurthyef432cb2022-12-29 06:56:32 -0800801 }
Raghu Krishnamurthyf67776f2022-12-26 10:10:05 -0800802
803 /*
804 * Unrecognized UUID: does not match any of the VMs (or SPs)
805 * and is not Null.
806 */
Raghu Krishnamurthyef432cb2022-12-29 06:56:32 -0800807 if (vm_count == 0 || vm_count > ARRAY_SIZE(partitions)) {
Olivier Deprez89da0a42023-09-01 17:38:33 +0200808 dlog_verbose(
Raghu Krishnamurthybefdcc82023-02-24 06:41:56 -0800809 "Invalid parameters. vm_count = %d (must not be zero "
Karl Meakine8937d92024-03-19 16:04:25 +0000810 "or > %lu)\n",
Raghu Krishnamurthybefdcc82023-02-24 06:41:56 -0800811 vm_count, ARRAY_SIZE(partitions));
Raghu Krishnamurthyf67776f2022-12-26 10:10:05 -0800812 return ffa_error(FFA_INVALID_PARAMETERS);
813 }
814
815 if (start_index >= vm_count) {
Raghu Krishnamurthybefdcc82023-02-24 06:41:56 -0800816 dlog_error(
817 "start index = %d vm_count = %d (start_index must be "
818 "less than vm_count)\n",
819 start_index, vm_count);
Raghu Krishnamurthyf67776f2022-12-26 10:10:05 -0800820 return ffa_error(FFA_INVALID_PARAMETERS);
821 }
822
Raghu Krishnamurthybefdcc82023-02-24 06:41:56 -0800823 max_idx = vm_count - 1;
Raghu Krishnamurthyf67776f2022-12-26 10:10:05 -0800824 num_entries_to_ret = (max_idx - start_index) + 1;
Raghu Krishnamurthybefdcc82023-02-24 06:41:56 -0800825 num_entries_to_ret =
826 MIN(num_entries_to_ret, MAX_INFO_REGS_ENTRIES_PER_CALL);
Raghu Krishnamurthyf67776f2022-12-26 10:10:05 -0800827 curr_idx = start_index + num_entries_to_ret - 1;
828 assert(curr_idx <= max_idx);
829
830 ret.func = FFA_SUCCESS_64;
831 ret.arg2 = (sizeof(struct ffa_partition_info) & 0xFFFF) << 48;
832 ret.arg2 |= curr_idx << 16;
833 ret.arg2 |= max_idx;
834
835 if (num_entries_to_ret > 1) {
836 ret.extended_val.valid = 1;
837 }
838
839 for (uint16_t idx = start_index; idx <= curr_idx; ++idx) {
Raghu Krishnamurthybefdcc82023-02-24 06:41:56 -0800840 uint64_t *xn_0 = arg_ptrs[arg_idx++];
841 uint64_t *xn_1 = arg_ptrs[arg_idx++];
842 uint64_t *xn_2 = arg_ptrs[arg_idx++];
843
844 api_ffa_pack_vmid_count_props(xn_0, partitions[idx].vm_id,
845 partitions[idx].vcpu_count,
846 partitions[idx].properties);
Raghu Krishnamurthyf67776f2022-12-26 10:10:05 -0800847 if (uuid_is_null) {
Raghu Krishnamurthybefdcc82023-02-24 06:41:56 -0800848 api_ffa_pack_uuid(xn_1, xn_2, &partitions[idx].uuid);
Raghu Krishnamurthyf67776f2022-12-26 10:10:05 -0800849 }
Raghu Krishnamurthycded31b2023-04-23 15:07:35 -0700850 assert(arg_idx <= ARRAY_SIZE(arg_ptrs));
Raghu Krishnamurthyf67776f2022-12-26 10:10:05 -0800851 }
852
853 return ret;
Raghu Krishnamurthy7592bcb2022-12-25 13:09:00 -0800854}
855
Fuad Tabbae4efcc32020-07-16 15:37:27 +0100856struct ffa_value api_ffa_partition_info_get(struct vcpu *current,
Daniel Boulbyb46cad12021-12-13 17:47:21 +0000857 const struct ffa_uuid *uuid,
858 const uint32_t flags)
Fuad Tabbae4efcc32020-07-16 15:37:27 +0100859{
860 struct vm *current_vm = current->vm;
Fuad Tabbae4efcc32020-07-16 15:37:27 +0100861 ffa_vm_count_t vm_count = 0;
Olivier Deprez013f4d62022-11-21 15:46:20 +0100862 bool count_flag = (flags & FFA_PARTITION_COUNT_FLAG_MASK) ==
Daniel Boulbyb46cad12021-12-13 17:47:21 +0000863 FFA_PARTITION_COUNT_FLAG;
Fuad Tabbae4efcc32020-07-16 15:37:27 +0100864 bool uuid_is_null = ffa_uuid_is_null(uuid);
Daniel Boulbyce541842022-05-31 15:54:31 +0100865 struct ffa_partition_info partitions[2 * MAX_VMS] = {0};
Daniel Boulby191b5f02022-02-17 17:26:14 +0000866 struct vm_locked vm_locked;
867 struct ffa_value ret;
Fuad Tabbae4efcc32020-07-16 15:37:27 +0100868
Daniel Boulbyb46cad12021-12-13 17:47:21 +0000869 /* Bits 31:1 Must Be Zero */
Olivier Deprez013f4d62022-11-21 15:46:20 +0100870 if ((flags & ~FFA_PARTITION_COUNT_FLAG_MASK) != 0) {
Daniel Boulbyb46cad12021-12-13 17:47:21 +0000871 return ffa_error(FFA_INVALID_PARAMETERS);
872 }
873
Kathleen Capellaa1de3c52023-08-28 20:10:02 -0400874 api_ffa_fill_partitions_info_array(partitions, ARRAY_SIZE(partitions),
875 uuid, count_flag, current_vm->id,
876 &vm_count);
Fuad Tabbae4efcc32020-07-16 15:37:27 +0100877
Olivier Depreze562e542020-06-11 17:31:54 +0200878 /* If UUID is Null vm_count must not be zero at this stage. */
879 CHECK(!uuid_is_null || vm_count != 0);
880
881 /*
882 * When running the Hypervisor:
883 * - If UUID is Null the Hypervisor forwards the query to the SPMC for
884 * it to fill with secure partitions information.
Daniel Boulbyb46cad12021-12-13 17:47:21 +0000885 * - If UUID is non-Null vm_count may be zero because the UUID matches
Olivier Depreze562e542020-06-11 17:31:54 +0200886 * a secure partition and the query is forwarded to the SPMC.
887 * When running the SPMC:
888 * - If UUID is non-Null and vm_count is zero it means there is no such
889 * partition identified in the system.
890 */
Daniel Boulbyb46cad12021-12-13 17:47:21 +0000891 plat_ffa_partition_info_get_forward(uuid, flags, partitions, &vm_count);
Olivier Depreze562e542020-06-11 17:31:54 +0200892
893 /*
894 * Unrecognized UUID: does not match any of the VMs (or SPs)
895 * and is not Null.
896 */
Fuad Tabbae4efcc32020-07-16 15:37:27 +0100897 if (vm_count == 0) {
898 return ffa_error(FFA_INVALID_PARAMETERS);
899 }
900
Daniel Boulbyb46cad12021-12-13 17:47:21 +0000901 /*
902 * If the count flag is set we don't need to return the partition info
903 * descriptors.
904 */
905 if (count_flag) {
906 return (struct ffa_value){.func = FFA_SUCCESS_32,
907 .arg2 = vm_count};
908 }
909
Daniel Boulby191b5f02022-02-17 17:26:14 +0000910 vm_locked = vm_lock(current_vm);
911 ret = send_versioned_partition_info_descriptors(vm_locked, partitions,
912 vm_count);
913 vm_unlock(&vm_locked);
914 return ret;
Fuad Tabbae4efcc32020-07-16 15:37:27 +0100915}
916
Andrew Scull9726c252019-01-23 13:44:19 +0000917/**
Andrew Scull55c4d8b2018-12-18 18:50:18 +0000918 * Returns the ID of the VM.
919 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100920struct ffa_value api_ffa_id_get(const struct vcpu *current)
Andrew Scull55c4d8b2018-12-18 18:50:18 +0000921{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100922 return (struct ffa_value){.func = FFA_SUCCESS_32,
923 .arg2 = current->vm->id};
Andrew Scull55c4d8b2018-12-18 18:50:18 +0000924}
925
926/**
Olivier Deprez421677d2021-06-18 12:18:53 +0200927 * Returns the SPMC FF-A ID at NS virtual/physical and secure virtual
928 * FF-A instances.
929 * DEN0077A FF-A v1.1 Beta0 section 13.9 FFA_SPM_ID_GET.
Daniel Boulbyb2fb80e2021-02-03 15:09:23 +0000930 */
931struct ffa_value api_ffa_spm_id_get(void)
932{
Karl Meakin0e617d92024-04-05 12:55:22 +0100933 if (FFA_VERSION_1_1 <= FFA_VERSION_COMPILED) {
934 /*
935 * Return the SPMC ID that was fetched during FF-A
936 * initialization.
937 */
938 return (struct ffa_value){.func = FFA_SUCCESS_32,
939 .arg2 = arch_ffa_spmc_id_get()};
940 }
941
J-Alves3829fc02021-03-18 12:49:18 +0000942 return ffa_error(FFA_NOT_SUPPORTED);
Daniel Boulbyb2fb80e2021-02-03 15:09:23 +0000943}
944
945/**
Wedson Almeida Filho03306112018-11-26 00:08:03 +0000946 * This function is called by the architecture-specific context switching
Fuad Tabbab0ef2a42019-12-19 11:19:25 +0000947 * function to indicate that register state for the given vCPU has been saved
948 * and can therefore be used by other pCPUs.
Wedson Almeida Filho03306112018-11-26 00:08:03 +0000949 */
950void api_regs_state_saved(struct vcpu *vcpu)
951{
952 sl_lock(&vcpu->lock);
953 vcpu->regs_available = true;
954 sl_unlock(&vcpu->lock);
955}
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +0000956/**
Andrew Walbran508e63c2018-12-20 17:02:37 +0000957 * Assuming that the arguments have already been checked by the caller, injects
958 * a virtual interrupt of the given ID into the given target vCPU. This doesn't
959 * cause the vCPU to actually be run immediately; it will be taken when the vCPU
960 * is next run, which is up to the scheduler.
961 *
962 * Returns:
963 * - 0 on success if no further action is needed.
964 * - 1 if it was called by the primary VM and the primary VM now needs to wake
965 * up or kick the target vCPU.
966 */
Manish Pandeya5f39fb2020-09-11 09:47:11 +0100967int64_t api_interrupt_inject_locked(struct vcpu_locked target_locked,
Madhukar Pappireddybd10e572023-03-06 16:39:49 -0600968 uint32_t intid,
969 struct vcpu_locked current_locked,
Manish Pandeya5f39fb2020-09-11 09:47:11 +0100970 struct vcpu **next)
Andrew Walbran508e63c2018-12-20 17:02:37 +0000971{
Manish Pandey35e452f2021-02-18 21:36:34 +0000972 struct vcpu *target_vcpu = target_locked.vcpu;
Madhukar Pappireddybd10e572023-03-06 16:39:49 -0600973 struct vcpu *current = current_locked.vcpu;
Daniel Boulby4ca50f02022-07-29 18:29:34 +0100974 struct interrupts *interrupts = &target_vcpu->interrupts;
Andrew Walbran508e63c2018-12-20 17:02:37 +0000975 int64_t ret = 0;
976
Andrew Walbran508e63c2018-12-20 17:02:37 +0000977 /*
Manish Pandey35e452f2021-02-18 21:36:34 +0000978 * We only need to change state and (maybe) trigger a virtual interrupt
979 * if it is enabled and was not previously pending. Otherwise we can
980 * skip everything except setting the pending bit.
Andrew Walbran508e63c2018-12-20 17:02:37 +0000981 */
Daniel Boulby4ca50f02022-07-29 18:29:34 +0100982 if (!(vcpu_is_virt_interrupt_enabled(interrupts, intid) &&
983 !vcpu_is_virt_interrupt_pending(interrupts, intid))) {
Andrew Walbran508e63c2018-12-20 17:02:37 +0000984 goto out;
985 }
986
987 /* Increment the count. */
Daniel Boulby4ca50f02022-07-29 18:29:34 +0100988 vcpu_interrupt_count_increment(target_locked, interrupts, intid);
Andrew Walbran508e63c2018-12-20 17:02:37 +0000989
990 /*
991 * Only need to update state if there was not already an
992 * interrupt enabled and pending.
993 */
Manish Pandey35e452f2021-02-18 21:36:34 +0000994 if (vcpu_interrupt_count_get(target_locked) != 1) {
Andrew Walbran508e63c2018-12-20 17:02:37 +0000995 goto out;
996 }
997
Karl Meakin5e996992024-05-20 11:27:07 +0100998 if (vm_is_primary(current->vm)) {
Andrew Walbran508e63c2018-12-20 17:02:37 +0000999 /*
1000 * If the call came from the primary VM, let it know that it
1001 * should run or kick the target vCPU.
1002 */
1003 ret = 1;
Manish Pandey35e452f2021-02-18 21:36:34 +00001004 } else if (current != target_vcpu && next != NULL) {
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06001005 *next = api_wake_up_locked(current_locked, target_vcpu);
Andrew Walbran508e63c2018-12-20 17:02:37 +00001006 }
1007
1008out:
1009 /* Either way, make it pending. */
Daniel Boulby4ca50f02022-07-29 18:29:34 +01001010 vcpu_virt_interrupt_set_pending(interrupts, intid);
Andrew Walbran508e63c2018-12-20 17:02:37 +00001011
Olivier Deprezc19a6ea2020-08-06 11:16:07 +02001012 return ret;
1013}
1014
Andrew Walbran508e63c2018-12-20 17:02:37 +00001015/**
J-Alves2ced1672022-12-12 14:35:38 +00001016 * Constructs the return value from a successful FFA_MSG_WAIT call, when used
1017 * with FFA_MSG_SEND_32.
Andrew Walbrand4d2fa12019-10-01 16:47:25 +01001018 */
J-Alves27b71962022-12-12 15:29:58 +00001019struct ffa_value ffa_msg_recv_return(const struct vm *receiver)
Andrew Walbrand4d2fa12019-10-01 16:47:25 +01001020{
Andrew Walbrane7ad3c02019-12-24 17:03:04 +00001021 switch (receiver->mailbox.recv_func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001022 case FFA_MSG_SEND_32:
1023 return (struct ffa_value){
1024 .func = FFA_MSG_SEND_32,
Andrew Walbrane7ad3c02019-12-24 17:03:04 +00001025 .arg1 = (receiver->mailbox.recv_sender << 16) |
1026 receiver->id,
1027 .arg3 = receiver->mailbox.recv_size};
Federico Recanati25053ee2022-03-14 15:01:53 +01001028 case FFA_MSG_SEND2_32:
1029 return (struct ffa_value){
1030 .func = FFA_RUN_32,
1031 /*
1032 * TODO: FFA_RUN should return vCPU and VM ID in arg1.
1033 * Retrieving vCPU requires a rework of the function,
1034 * while receiver ID must be set because it's checked by
1035 * other APIs (eg: FFA_NOTIFICATION_GET).
1036 */
1037 .arg1 = receiver->id};
Andrew Walbrane7ad3c02019-12-24 17:03:04 +00001038 default:
1039 /* This should never be reached, but return an error in case. */
Andrew Walbran17eebf92020-02-05 16:35:49 +00001040 dlog_error("Tried to return an invalid message function %#x\n",
1041 receiver->mailbox.recv_func);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001042 return ffa_error(FFA_DENIED);
Andrew Walbrane7ad3c02019-12-24 17:03:04 +00001043 }
Andrew Walbrand4d2fa12019-10-01 16:47:25 +01001044}
1045
J-Alvese8c8c2b2022-12-16 15:34:48 +00001046/**
1047 * Change the state of mailbox to empty, such that the ownership is given to the
1048 * Partition manager.
Karl Meakin1064a9c2024-06-04 17:20:43 +01001049 * Returns FFA_SUCCESS if the mailbox was reset successfully, FFA_ERROR
1050 * otherwise.
J-Alvese8c8c2b2022-12-16 15:34:48 +00001051 */
Karl Meakin1064a9c2024-06-04 17:20:43 +01001052static struct ffa_value api_release_mailbox(struct vm_locked vm_locked)
J-Alvese8c8c2b2022-12-16 15:34:48 +00001053{
Karl Meakin1064a9c2024-06-04 17:20:43 +01001054 struct ffa_value ret = {.func = FFA_SUCCESS_32};
J-Alves19e20cf2023-08-02 12:48:55 +01001055 ffa_id_t vm_id = vm_locked.vm->id;
J-Alvese8c8c2b2022-12-16 15:34:48 +00001056
1057 switch (vm_locked.vm->mailbox.state) {
1058 case MAILBOX_STATE_EMPTY:
1059 dlog_verbose("Mailbox of %x is empty.\n", vm_id);
Karl Meakin1064a9c2024-06-04 17:20:43 +01001060 ret = ffa_error(FFA_DENIED);
J-Alves31f8bef2023-07-14 12:46:28 +01001061 break;
J-Alvese8c8c2b2022-12-16 15:34:48 +00001062 case MAILBOX_STATE_FULL:
1063 /* Check it doesn't have pending RX full notifications. */
1064 if (vm_are_fwk_notifications_pending(vm_locked)) {
1065 dlog_verbose(
1066 "Mailbox of endpoint %x has pending "
1067 "messages.\n",
1068 vm_id);
Karl Meakin1064a9c2024-06-04 17:20:43 +01001069 ret = ffa_error(FFA_DENIED);
J-Alvese8c8c2b2022-12-16 15:34:48 +00001070 }
1071 break;
1072 case MAILBOX_STATE_OTHER_WORLD_OWNED:
1073 /*
1074 * The SPMC shouldn't let SP's mailbox get into this state.
1075 * For the Hypervisor, the VM may call FFA_RX_RELEASE, whilst
1076 * the mailbox is in this state. In that case, we should report
1077 * error.
1078 */
1079 if (vm_id_is_current_world(vm_id)) {
1080 dlog_verbose(
Karl Meakin1064a9c2024-06-04 17:20:43 +01001081 "Mailbox of endpoint %x is in an incorrect "
1082 "state.\n",
J-Alvese8c8c2b2022-12-16 15:34:48 +00001083 vm_id);
Karl Meakin1064a9c2024-06-04 17:20:43 +01001084 ret = ffa_error(FFA_ABORTED);
J-Alvese8c8c2b2022-12-16 15:34:48 +00001085 }
1086 break;
1087 }
1088
Karl Meakin1064a9c2024-06-04 17:20:43 +01001089 if (ret.func == FFA_SUCCESS_32) {
1090 vm_locked.vm->mailbox.state = MAILBOX_STATE_EMPTY;
J-Alvese8c8c2b2022-12-16 15:34:48 +00001091 }
1092
Karl Meakin1064a9c2024-06-04 17:20:43 +01001093 return ret;
J-Alvese8c8c2b2022-12-16 15:34:48 +00001094}
1095
Kathleen Capelladb6a08b2023-10-04 13:42:39 -04001096/*
1097 * Helper to check if extended arguments (corresponding to regs x8-x17)
1098 * are zeroed out.
1099 */
Kathleen Capella036cc592023-11-30 18:26:15 -05001100bool api_extended_args_are_zero(struct ffa_value *args)
Kathleen Capelladb6a08b2023-10-04 13:42:39 -04001101{
1102 if (args->extended_val.arg8 != 0U || args->extended_val.arg9 != 0U ||
1103 args->extended_val.arg10 != 0U || args->extended_val.arg11 != 0U ||
1104 args->extended_val.arg12 != 0U || args->extended_val.arg13 != 0U ||
1105 args->extended_val.arg14 != 0U || args->extended_val.arg15 != 0U ||
1106 args->extended_val.arg16 != 0U || args->extended_val.arg17 != 0U) {
1107 return false;
1108 }
1109 return true;
1110}
1111
Madhukar Pappireddy5522c672021-12-17 16:35:51 -06001112struct ffa_value api_ffa_msg_wait(struct vcpu *current, struct vcpu **next,
1113 struct ffa_value *args)
1114{
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06001115 struct vcpu_locked current_locked;
Madhukar Pappireddyd456ac22023-06-26 15:29:34 -05001116 enum vcpu_state next_state = VCPU_STATE_RUNNING;
J-Alvese8c8c2b2022-12-16 15:34:48 +00001117 struct ffa_value ret;
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06001118 struct vcpu_locked next_locked = (struct vcpu_locked){
1119 .vcpu = NULL,
1120 };
Madhukar Pappireddy5522c672021-12-17 16:35:51 -06001121
1122 if (args->arg1 != 0U || args->arg2 != 0U || args->arg3 != 0U ||
1123 args->arg4 != 0U || args->arg5 != 0U || args->arg6 != 0U ||
1124 args->arg7 != 0U) {
1125 return ffa_error(FFA_INVALID_PARAMETERS);
1126 }
1127
Karl Meakin0e617d92024-04-05 12:55:22 +01001128 if (current->vm->ffa_version >= FFA_VERSION_1_2 &&
Kathleen Capelladb6a08b2023-10-04 13:42:39 -04001129 !api_extended_args_are_zero(args)) {
1130 return ffa_error(FFA_INVALID_PARAMETERS);
1131 }
1132
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06001133 current_locked = vcpu_lock(current);
Madhukar Pappireddyc857ac22022-06-21 17:37:53 -05001134 if (!plat_ffa_check_runtime_state_transition(
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06001135 current_locked, current->vm->id, HF_INVALID_VM_ID,
1136 next_locked, FFA_MSG_WAIT_32, &next_state)) {
1137 ret = ffa_error(FFA_DENIED);
1138 goto out;
Madhukar Pappireddyc857ac22022-06-21 17:37:53 -05001139 }
1140
Madhukar Pappireddyd456ac22023-06-26 15:29:34 -05001141 assert(!vm_id_is_current_world(current->vm->id) ||
1142 next_state == VCPU_STATE_WAITING);
Madhukar Pappireddyc857ac22022-06-21 17:37:53 -05001143
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06001144 ret = plat_ffa_msg_wait_prepare(current_locked, next);
1145out:
1146 vcpu_unlock(&current_locked);
J-Alvese8c8c2b2022-12-16 15:34:48 +00001147 return ret;
Madhukar Pappireddy5522c672021-12-17 16:35:51 -06001148}
1149
Andrew Walbrand4d2fa12019-10-01 16:47:25 +01001150/**
Fuad Tabbab0ef2a42019-12-19 11:19:25 +00001151 * Prepares the vCPU to run by updating its state and fetching whether a return
Wedson Almeida Filho03306112018-11-26 00:08:03 +00001152 * value needs to be forced onto the vCPU.
1153 */
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06001154static bool api_vcpu_prepare_run(struct vcpu_locked current_locked,
1155 struct vcpu_locked vcpu_next_locked,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001156 struct ffa_value *run_ret)
Wedson Almeida Filho03306112018-11-26 00:08:03 +00001157{
Max Shvetsov40108e72020-08-27 12:39:50 +01001158 struct vm_locked vm_locked;
Wedson Almeida Filho03306112018-11-26 00:08:03 +00001159 bool ret;
Madhukar Pappireddyb11e0d12021-08-02 19:44:35 -05001160 uint64_t timer_remaining_ns = FFA_SLEEP_INDEFINITE;
Olivier Deprez04168ed2022-09-26 09:20:00 +02001161 bool vcpu_was_init_state = false;
J-Alves6e2abc62021-12-02 14:58:56 +00001162 bool need_vm_lock;
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06001163 struct two_vcpu_locked vcpus_locked;
Wedson Almeida Filho03306112018-11-26 00:08:03 +00001164
Andrew Scullb06d1752019-02-04 10:15:48 +00001165 /*
Andrew Walbrand81c7d82019-11-27 18:34:46 +00001166 * Check that the registers are available so that the vCPU can be run.
Andrew Scullb06d1752019-02-04 10:15:48 +00001167 *
Andrew Scull4caadaf2019-07-03 13:13:47 +01001168 * The VM lock is not needed in the common case so it must only be taken
1169 * when it is going to be needed. This ensures there are no inter-vCPU
1170 * dependencies in the common run case meaning the sensitive context
1171 * switch performance is consistent.
Andrew Scullb06d1752019-02-04 10:15:48 +00001172 */
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06001173 struct vcpu *vcpu = vcpu_next_locked.vcpu;
1174 struct vcpu *current = current_locked.vcpu;
Max Shvetsov40108e72020-08-27 12:39:50 +01001175
Andrew Walbrand81c7d82019-11-27 18:34:46 +00001176 /* The VM needs to be locked to deliver mailbox messages. */
J-Alves6e2abc62021-12-02 14:58:56 +00001177 need_vm_lock = vcpu->state == VCPU_STATE_WAITING ||
1178 (!vcpu->vm->el0_partition &&
1179 (vcpu->state == VCPU_STATE_BLOCKED_INTERRUPT ||
1180 vcpu->state == VCPU_STATE_BLOCKED ||
1181 vcpu->state == VCPU_STATE_PREEMPTED));
1182
Andrew Walbrand81c7d82019-11-27 18:34:46 +00001183 if (need_vm_lock) {
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06001184 vcpu_unlock(&vcpu_next_locked);
1185 vcpu_unlock(&current_locked);
Max Shvetsov40108e72020-08-27 12:39:50 +01001186 vm_locked = vm_lock(vcpu->vm);
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06001187
1188 /* Lock both vCPUs at once to avoid deadlock. */
1189 vcpus_locked = vcpu_lock_both(current, vcpu);
1190 current_locked = vcpus_locked.vcpu1;
1191 vcpu_next_locked = vcpus_locked.vcpu2;
Andrew Walbrand81c7d82019-11-27 18:34:46 +00001192 }
1193
1194 /*
1195 * If the vCPU is already running somewhere then we can't run it here
1196 * simultaneously. While it is actually running then the state should be
1197 * `VCPU_STATE_RUNNING` and `regs_available` should be false. Once it
1198 * stops running but while Hafnium is in the process of switching back
1199 * to the primary there will be a brief period while the state has been
1200 * updated but `regs_available` is still false (until
1201 * `api_regs_state_saved` is called). We can't start running it again
1202 * until this has finished, so count this state as still running for the
1203 * purposes of this check.
1204 */
1205 if (vcpu->state == VCPU_STATE_RUNNING || !vcpu->regs_available) {
1206 /*
1207 * vCPU is running on another pCPU.
1208 *
1209 * It's okay not to return the sleep duration here because the
1210 * other physical CPU that is currently running this vCPU will
1211 * return the sleep duration if needed.
1212 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001213 *run_ret = ffa_error(FFA_BUSY);
Andrew Walbrand81c7d82019-11-27 18:34:46 +00001214 ret = false;
1215 goto out;
Andrew Scullb06d1752019-02-04 10:15:48 +00001216 }
Andrew Scull9726c252019-01-23 13:44:19 +00001217
1218 if (atomic_load_explicit(&vcpu->vm->aborting, memory_order_relaxed)) {
Andrew Sculld6ee1102019-04-05 22:12:42 +01001219 if (vcpu->state != VCPU_STATE_ABORTED) {
Kathleen Capella6e9765b2023-07-11 17:44:58 -04001220 dlog_verbose("VM %#x was aborted, cannot run vCPU %u\n",
1221 vcpu->vm->id, vcpu_index(vcpu));
Andrew Sculld6ee1102019-04-05 22:12:42 +01001222 vcpu->state = VCPU_STATE_ABORTED;
Andrew Scull9726c252019-01-23 13:44:19 +00001223 }
Kathleen Capella6ab05132023-05-10 12:27:35 -04001224 *run_ret = ffa_error(FFA_ABORTED);
Andrew Scull9726c252019-01-23 13:44:19 +00001225 ret = false;
1226 goto out;
1227 }
1228
Andrew Walbran508e63c2018-12-20 17:02:37 +00001229 switch (vcpu->state) {
Andrew Sculld6ee1102019-04-05 22:12:42 +01001230 case VCPU_STATE_RUNNING:
1231 case VCPU_STATE_OFF:
1232 case VCPU_STATE_ABORTED:
Wedson Almeida Filho03306112018-11-26 00:08:03 +00001233 ret = false;
1234 goto out;
Andrew Scullb06d1752019-02-04 10:15:48 +00001235
Madhukar Pappireddyb11e0d12021-08-02 19:44:35 -05001236 case VCPU_STATE_WAITING:
1237 /*
Olivier Deprezb2808332023-02-02 15:25:40 +01001238 * An initial FFA_RUN is necessary for SP's secondary vCPUs to
1239 * reach the message wait loop.
Madhukar Pappireddyb11e0d12021-08-02 19:44:35 -05001240 */
Olivier Deprezb2808332023-02-02 15:25:40 +01001241 if (vcpu->rt_model == RTM_SP_INIT) {
1242 /*
1243 * TODO: this should be removed, but omitting it makes
1244 * normal world arch gicv3 tests failing.
1245 */
1246 vcpu->rt_model = RTM_NONE;
Olivier Deprez04168ed2022-09-26 09:20:00 +02001247
1248 vcpu_was_init_state = true;
Madhukar Pappireddyb11e0d12021-08-02 19:44:35 -05001249 break;
1250 }
1251
J-Alves6e2abc62021-12-02 14:58:56 +00001252 assert(need_vm_lock == true);
1253 if (!vm_locked.vm->el0_partition &&
1254 plat_ffa_inject_notification_pending_interrupt(
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06001255 vcpu_next_locked, current_locked, vm_locked)) {
Federico Recanati9dccc4b2022-04-27 12:52:36 +02001256 /* TODO: setting a return value to override
1257 * the placeholder (FFA_ERROR(INTERRUPTED))
1258 * set by FFA_MSG_WAIT. FF-A v1.1 allows
1259 * FFA_MSG_WAIT to successfully return even if
1260 * it didn't receive a message. TFTF tests are
1261 * still expecting an FFA_ERROR instead,
1262 * should be fixed?
1263 */
1264 arch_regs_set_retval(
1265 &vcpu->regs,
1266 (struct ffa_value){.func = FFA_RUN_32,
1267 // TODO: does it make sense
1268 // to set vCPU/receiver?
1269 .arg1 = 0});
J-Alves6e2abc62021-12-02 14:58:56 +00001270 break;
1271 }
1272
Andrew Scullb06d1752019-02-04 10:15:48 +00001273 /*
1274 * A pending message allows the vCPU to run so the message can
1275 * be delivered directly.
1276 */
J-Alvese8c8c2b2022-12-16 15:34:48 +00001277 if (vcpu->vm->mailbox.state == MAILBOX_STATE_FULL) {
Andrew Walbrand4d2fa12019-10-01 16:47:25 +01001278 arch_regs_set_retval(&vcpu->regs,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001279 ffa_msg_recv_return(vcpu->vm));
Andrew Scullb06d1752019-02-04 10:15:48 +00001280 break;
1281 }
Madhukar Pappireddyb11e0d12021-08-02 19:44:35 -05001282
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06001283 if (vcpu_interrupt_count_get(vcpu_next_locked) > 0) {
Madhukar Pappireddyb11e0d12021-08-02 19:44:35 -05001284 break;
1285 }
1286
1287 if (arch_timer_enabled(&vcpu->regs)) {
1288 timer_remaining_ns =
1289 arch_timer_remaining_ns(&vcpu->regs);
1290 if (timer_remaining_ns == 0) {
1291 break;
1292 }
1293 } else {
1294 dlog_verbose("Timer disabled\n");
1295 }
1296 run_ret->func = FFA_MSG_WAIT_32;
1297 run_ret->arg1 = ffa_vm_vcpu(vcpu->vm->id, vcpu_index(vcpu));
1298 run_ret->arg2 = timer_remaining_ns;
1299 ret = false;
1300 goto out;
Andrew Sculld6ee1102019-04-05 22:12:42 +01001301 case VCPU_STATE_BLOCKED_INTERRUPT:
J-Alves6e2abc62021-12-02 14:58:56 +00001302 if (need_vm_lock &&
1303 plat_ffa_inject_notification_pending_interrupt(
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06001304 vcpu_next_locked, current_locked, vm_locked)) {
1305 assert(vcpu_interrupt_count_get(vcpu_next_locked) > 0);
J-Alves6e2abc62021-12-02 14:58:56 +00001306 break;
1307 }
1308
Andrew Scullb06d1752019-02-04 10:15:48 +00001309 /* Allow virtual interrupts to be delivered. */
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06001310 if (vcpu_interrupt_count_get(vcpu_next_locked) > 0) {
Andrew Scullb06d1752019-02-04 10:15:48 +00001311 break;
1312 }
1313
Andrew Walbran508e63c2018-12-20 17:02:37 +00001314 if (arch_timer_enabled(&vcpu->regs)) {
Andrew Walbran4692a3a2020-08-07 12:42:01 +01001315 timer_remaining_ns =
Andrew Walbran2fc856a2019-11-04 15:17:24 +00001316 arch_timer_remaining_ns(&vcpu->regs);
1317
1318 /*
1319 * The timer expired so allow the interrupt to be
1320 * delivered.
1321 */
1322 if (timer_remaining_ns == 0) {
1323 break;
1324 }
Andrew Walbran4692a3a2020-08-07 12:42:01 +01001325 }
Andrew Walbran2fc856a2019-11-04 15:17:24 +00001326
Andrew Walbran4692a3a2020-08-07 12:42:01 +01001327 /*
1328 * The vCPU is not ready to run, return the appropriate code to
1329 * the primary which called vcpu_run.
1330 */
Madhukar Pappireddyb11e0d12021-08-02 19:44:35 -05001331 run_ret->func = HF_FFA_RUN_WAIT_FOR_INTERRUPT;
Andrew Walbran4692a3a2020-08-07 12:42:01 +01001332 run_ret->arg1 = ffa_vm_vcpu(vcpu->vm->id, vcpu_index(vcpu));
1333 run_ret->arg2 = timer_remaining_ns;
Andrew Walbran508e63c2018-12-20 17:02:37 +00001334
1335 ret = false;
1336 goto out;
Andrew Scullb06d1752019-02-04 10:15:48 +00001337
Madhukar Pappireddyb11e0d12021-08-02 19:44:35 -05001338 case VCPU_STATE_BLOCKED:
1339 /* A blocked vCPU is run unconditionally. Fall through. */
1340 case VCPU_STATE_PREEMPTED:
J-Alves6e2abc62021-12-02 14:58:56 +00001341 /* Check NPI is to be injected here. */
1342 if (need_vm_lock) {
1343 plat_ffa_inject_notification_pending_interrupt(
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06001344 vcpu_next_locked, current_locked, vm_locked);
J-Alves6e2abc62021-12-02 14:58:56 +00001345 }
Andrew Walbran508e63c2018-12-20 17:02:37 +00001346 break;
Madhukar Pappireddyb11e0d12021-08-02 19:44:35 -05001347 default:
1348 /*
1349 * Execution not expected to reach here. Deny the request
1350 * gracefully.
1351 */
1352 *run_ret = ffa_error(FFA_DENIED);
1353 ret = false;
1354 goto out;
Wedson Almeida Filho03306112018-11-26 00:08:03 +00001355 }
1356
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06001357 plat_ffa_init_schedule_mode_ffa_run(current_locked, vcpu_next_locked);
Madhukar Pappireddy1480fce2022-06-21 18:09:25 -05001358
Andrew Scullb06d1752019-02-04 10:15:48 +00001359 /* It has been decided that the vCPU should be run. */
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06001360 vcpu->cpu = current_locked.vcpu->cpu;
Andrew Sculld6ee1102019-04-05 22:12:42 +01001361 vcpu->state = VCPU_STATE_RUNNING;
Wedson Almeida Filho03306112018-11-26 00:08:03 +00001362
Olivier Deprez04168ed2022-09-26 09:20:00 +02001363 if (vcpu_was_init_state) {
1364 vcpu_set_phys_core_idx(vcpu);
1365 vcpu_set_boot_info_gp_reg(vcpu);
J-Alves7ac49052022-02-08 17:20:53 +00001366 }
J-Alves7ac49052022-02-08 17:20:53 +00001367
Wedson Almeida Filho03306112018-11-26 00:08:03 +00001368 /*
Wedson Almeida Filho03306112018-11-26 00:08:03 +00001369 * Mark the registers as unavailable now that we're about to reflect
1370 * them onto the real registers. This will also prevent another physical
1371 * CPU from trying to read these registers.
1372 */
1373 vcpu->regs_available = false;
1374
1375 ret = true;
1376
1377out:
Andrew Scullb06d1752019-02-04 10:15:48 +00001378 if (need_vm_lock) {
Max Shvetsov40108e72020-08-27 12:39:50 +01001379 vm_unlock(&vm_locked);
Andrew Scullb06d1752019-02-04 10:15:48 +00001380 }
Wedson Almeida Filho03306112018-11-26 00:08:03 +00001381 return ret;
1382}
1383
J-Alves19e20cf2023-08-02 12:48:55 +01001384struct ffa_value api_ffa_run(ffa_id_t vm_id, ffa_vcpu_index_t vcpu_idx,
Madhukar Pappireddyb11e0d12021-08-02 19:44:35 -05001385 struct vcpu *current, struct vcpu **next)
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +01001386{
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +01001387 struct vm *vm;
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +01001388 struct vcpu *vcpu;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001389 struct ffa_value ret = ffa_error(FFA_INVALID_PARAMETERS);
Madhukar Pappireddyd456ac22023-06-26 15:29:34 -05001390 enum vcpu_state next_state = VCPU_STATE_RUNNING;
Madhukar Pappireddyc857ac22022-06-21 17:37:53 -05001391 struct vcpu_locked current_locked;
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06001392 struct vcpu_locked vcpu_next_locked;
1393 struct two_vcpu_locked vcpus_locked;
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +01001394
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06001395 current_locked = vcpu_lock(current);
1396 if (!plat_ffa_run_checks(current_locked, vm_id, vcpu_idx, &ret, next)) {
1397 goto out;
Andrew Scull7364a8e2018-07-19 15:39:29 +01001398 }
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +01001399
Raghu Krishnamurthy62f97a72021-07-27 02:14:59 -07001400 if (plat_ffa_run_forward(vm_id, vcpu_idx, &ret)) {
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06001401 goto out;
Raghu Krishnamurthy62f97a72021-07-27 02:14:59 -07001402 }
1403
Andrew Scull19503262018-09-20 14:48:39 +01001404 /* The requested VM must exist. */
Andrew Walbran42347a92019-05-09 13:59:03 +01001405 vm = vm_find(vm_id);
Andrew Scull19503262018-09-20 14:48:39 +01001406 if (vm == NULL) {
Andrew Scull6d2db332018-10-10 15:28:17 +01001407 goto out;
Andrew Scull19503262018-09-20 14:48:39 +01001408 }
1409
Fuad Tabbaed294af2019-12-20 10:43:01 +00001410 /* The requested vCPU must exist. */
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +01001411 if (vcpu_idx >= vm->vcpu_count) {
Andrew Scull6d2db332018-10-10 15:28:17 +01001412 goto out;
Andrew Scull7364a8e2018-07-19 15:39:29 +01001413 }
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +01001414
Madhukar Pappireddyc857ac22022-06-21 17:37:53 -05001415 /*
1416 * Refer Figure 8.13 Scenario 1 of the FF-A v1.1 EAC spec. SPMC
1417 * bypasses the intermediate execution contexts and resumes the
1418 * SP execution context that was originally preempted.
1419 */
1420 if (*next != NULL) {
1421 vcpu = *next;
1422 } else {
1423 vcpu = vm_get_vcpu(vm, vcpu_idx);
1424 }
1425
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06001426 /*
1427 * Unlock current vCPU to allow it to be locked together with next
1428 * vcpu.
1429 */
1430 vcpu_unlock(&current_locked);
1431
1432 /* Lock both vCPUs at once to avoid deadlock. */
1433 vcpus_locked = vcpu_lock_both(current, vcpu);
1434 current_locked = vcpus_locked.vcpu1;
1435 vcpu_next_locked = vcpus_locked.vcpu2;
1436
1437 if (!plat_ffa_check_runtime_state_transition(
1438 current_locked, current->vm->id, HF_INVALID_VM_ID,
1439 vcpu_next_locked, FFA_RUN_32, &next_state)) {
1440 ret = ffa_error(FFA_DENIED);
1441 goto out_vcpu;
Madhukar Pappireddyc857ac22022-06-21 17:37:53 -05001442 }
1443
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06001444 if (!api_vcpu_prepare_run(current_locked, vcpu_next_locked, &ret)) {
1445 goto out_vcpu;
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +01001446 }
Wedson Almeida Filho03306112018-11-26 00:08:03 +00001447
Andrew Walbran508e63c2018-12-20 17:02:37 +00001448 /*
1449 * Inject timer interrupt if timer has expired. It's safe to access
1450 * vcpu->regs here because api_vcpu_prepare_run already made sure that
1451 * regs_available was true (and then set it to false) before returning
1452 * true.
1453 */
1454 if (arch_timer_pending(&vcpu->regs)) {
1455 /* Make virtual timer interrupt pending. */
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06001456 api_interrupt_inject_locked(vcpu_next_locked,
1457 HF_VIRTUAL_TIMER_INTID,
1458 vcpu_next_locked, NULL);
Andrew Walbran508e63c2018-12-20 17:02:37 +00001459
1460 /*
1461 * Set the mask bit so the hardware interrupt doesn't fire
1462 * again. Ideally we wouldn't do this because it affects what
1463 * the secondary vCPU sees, but if we don't then we end up with
1464 * a loop of the interrupt firing each time we try to return to
1465 * the secondary vCPU.
1466 */
1467 arch_timer_mask(&vcpu->regs);
1468 }
1469
Fuad Tabbaed294af2019-12-20 10:43:01 +00001470 /* Switch to the vCPU. */
Wedson Almeida Filho03306112018-11-26 00:08:03 +00001471 *next = vcpu;
Wedson Almeida Filho03306112018-11-26 00:08:03 +00001472
Madhukar Pappireddyd456ac22023-06-26 15:29:34 -05001473 assert(!vm_id_is_current_world(current->vm->id) ||
1474 next_state == VCPU_STATE_BLOCKED);
Madhukar Pappireddyc857ac22022-06-21 17:37:53 -05001475 current->state = VCPU_STATE_BLOCKED;
Madhukar Pappireddyc857ac22022-06-21 17:37:53 -05001476
Andrew Scull33fecd32019-01-08 14:48:27 +00001477 /*
1478 * Set a placeholder return code to the scheduler. This will be
1479 * overwritten when the switch back to the primary occurs.
1480 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001481 ret.func = FFA_INTERRUPT_32;
Andrew Walbran7e824602020-10-22 16:51:40 +01001482 ret.arg1 = 0;
Andrew Walbran7a1ea0b2019-10-02 18:18:44 +01001483 ret.arg2 = 0;
Andrew Scull33fecd32019-01-08 14:48:27 +00001484
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06001485out_vcpu:
1486 vcpu_unlock(&vcpu_next_locked);
1487
Andrew Scull6d2db332018-10-10 15:28:17 +01001488out:
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06001489 vcpu_unlock(&current_locked);
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +01001490 return ret;
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +01001491}
1492
1493/**
Andrew Scull81e85092018-12-12 12:56:20 +00001494 * Check that the mode indicates memory that is valid, owned and exclusive.
1495 */
Andrew Walbran1281ed42019-10-22 17:23:40 +01001496static bool api_mode_valid_owned_and_exclusive(uint32_t mode)
Andrew Scull81e85092018-12-12 12:56:20 +00001497{
Andrew Scullb5f49e02019-10-02 13:20:47 +01001498 return (mode & (MM_MODE_D | MM_MODE_INVALID | MM_MODE_UNOWNED |
1499 MM_MODE_SHARED)) == 0;
Andrew Scull81e85092018-12-12 12:56:20 +00001500}
1501
1502/**
Manish Pandeyd34f8892020-06-19 17:41:07 +01001503 * Configures the hypervisor's stage-1 view of the send and receive pages.
Andrew Sculle1322792019-07-01 17:46:10 +01001504 */
Karl Meakin738bee12024-01-12 15:19:45 +00001505static struct ffa_value api_vm_configure_stage1(
1506 struct mm_stage1_locked mm_stage1_locked, struct vm_locked vm_locked,
1507 paddr_t pa_send_begin, paddr_t pa_send_end, paddr_t pa_recv_begin,
1508 paddr_t pa_recv_end, uint32_t extra_attributes,
1509 struct mpool *local_page_pool)
Andrew Sculle1322792019-07-01 17:46:10 +01001510{
Karl Meakin738bee12024-01-12 15:19:45 +00001511 struct ffa_value ret;
Andrew Sculle1322792019-07-01 17:46:10 +01001512
Karl Meakin738bee12024-01-12 15:19:45 +00001513 /*
1514 * Map the send page as read-only in the SPMC/hypervisor address space.
1515 */
Andrew Sculle1322792019-07-01 17:46:10 +01001516 vm_locked.vm->mailbox.send =
1517 mm_identity_map(mm_stage1_locked, pa_send_begin, pa_send_end,
Olivier Deprez96a2a262020-06-11 17:21:38 +02001518 MM_MODE_R | extra_attributes, local_page_pool);
Andrew Sculle1322792019-07-01 17:46:10 +01001519 if (!vm_locked.vm->mailbox.send) {
Karl Meakin738bee12024-01-12 15:19:45 +00001520 ret = ffa_error(FFA_NO_MEMORY);
1521 goto out;
Andrew Sculle1322792019-07-01 17:46:10 +01001522 }
1523
1524 /*
Karl Meakin738bee12024-01-12 15:19:45 +00001525 * Map the receive page as writable in the SPMC/hypervisor address
1526 * space. On failure, unmap the send page before returning.
Andrew Sculle1322792019-07-01 17:46:10 +01001527 */
1528 vm_locked.vm->mailbox.recv =
1529 mm_identity_map(mm_stage1_locked, pa_recv_begin, pa_recv_end,
Olivier Deprez96a2a262020-06-11 17:21:38 +02001530 MM_MODE_W | extra_attributes, local_page_pool);
Andrew Sculle1322792019-07-01 17:46:10 +01001531 if (!vm_locked.vm->mailbox.recv) {
Karl Meakin738bee12024-01-12 15:19:45 +00001532 ret = ffa_error(FFA_NO_MEMORY);
Andrew Sculle1322792019-07-01 17:46:10 +01001533 goto fail_undo_send;
1534 }
1535
Karl Meakin738bee12024-01-12 15:19:45 +00001536 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Sculle1322792019-07-01 17:46:10 +01001537 goto out;
1538
1539 /*
1540 * The following mappings will not require more memory than is available
1541 * in the local pool.
1542 */
1543fail_undo_send:
1544 vm_locked.vm->mailbox.send = NULL;
Andrew Scull7e8de322019-07-02 13:00:56 +01001545 CHECK(mm_unmap(mm_stage1_locked, pa_send_begin, pa_send_end,
1546 local_page_pool));
Andrew Sculle1322792019-07-01 17:46:10 +01001547
Andrew Sculle1322792019-07-01 17:46:10 +01001548out:
Andrew Sculle1322792019-07-01 17:46:10 +01001549 return ret;
1550}
1551
1552/**
Manish Pandeyd34f8892020-06-19 17:41:07 +01001553 * Sanity checks and configures the send and receive pages in the VM stage-2
1554 * and hypervisor stage-1 page tables.
1555 *
1556 * Returns:
1557 * - FFA_ERROR FFA_INVALID_PARAMETERS if the given addresses are not properly
Daniel Boulby6f8941e2021-06-14 18:27:18 +01001558 * aligned, are the same or have invalid attributes.
Manish Pandeyd34f8892020-06-19 17:41:07 +01001559 * - FFA_ERROR FFA_NO_MEMORY if the hypervisor was unable to map the buffers
1560 * due to insuffient page table memory.
Daniel Boulby6f8941e2021-06-14 18:27:18 +01001561 * - FFA_ERROR FFA_DENIED if the pages are already mapped.
Manish Pandeyd34f8892020-06-19 17:41:07 +01001562 * - FFA_SUCCESS on success if no further action is needed.
Andrew Sculle1322792019-07-01 17:46:10 +01001563 */
Manish Pandeyd34f8892020-06-19 17:41:07 +01001564
1565struct ffa_value api_vm_configure_pages(
1566 struct mm_stage1_locked mm_stage1_locked, struct vm_locked vm_locked,
1567 ipaddr_t send, ipaddr_t recv, uint32_t page_count,
1568 struct mpool *local_page_pool)
Andrew Sculle1322792019-07-01 17:46:10 +01001569{
Manish Pandeyd34f8892020-06-19 17:41:07 +01001570 struct ffa_value ret;
1571 paddr_t pa_send_begin;
1572 paddr_t pa_send_end;
1573 paddr_t pa_recv_begin;
1574 paddr_t pa_recv_end;
Federico Recanati8d8b1cf2022-04-14 13:16:00 +02001575 uint32_t orig_send_mode = 0;
1576 uint32_t orig_recv_mode = 0;
Olivier Deprez96a2a262020-06-11 17:21:38 +02001577 uint32_t extra_attributes;
Manish Pandeyd34f8892020-06-19 17:41:07 +01001578
1579 /* We only allow these to be setup once. */
1580 if (vm_locked.vm->mailbox.send || vm_locked.vm->mailbox.recv) {
Karl Meakin738bee12024-01-12 15:19:45 +00001581 dlog_error("%s: Mailboxes have already been setup for VM %#x\n",
1582 __func__, vm_locked.vm->id);
Manish Pandeyd34f8892020-06-19 17:41:07 +01001583 ret = ffa_error(FFA_DENIED);
1584 goto out;
1585 }
1586
1587 /* Hafnium only supports a fixed size of RX/TX buffers. */
1588 if (page_count != HF_MAILBOX_SIZE / FFA_PAGE_SIZE) {
Karl Meakine8937d92024-03-19 16:04:25 +00001589 dlog_error("%s: Page count must be %zu, it is %d\n", __func__,
Karl Meakin738bee12024-01-12 15:19:45 +00001590 HF_MAILBOX_SIZE / FFA_PAGE_SIZE, page_count);
Manish Pandeyd34f8892020-06-19 17:41:07 +01001591 ret = ffa_error(FFA_INVALID_PARAMETERS);
1592 goto out;
1593 }
1594
1595 /* Fail if addresses are not page-aligned. */
1596 if (!is_aligned(ipa_addr(send), PAGE_SIZE) ||
1597 !is_aligned(ipa_addr(recv), PAGE_SIZE)) {
Karl Meakin738bee12024-01-12 15:19:45 +00001598 dlog_error("%s: Mailbox buffers not page-aligned\n", __func__);
Manish Pandeyd34f8892020-06-19 17:41:07 +01001599 ret = ffa_error(FFA_INVALID_PARAMETERS);
1600 goto out;
1601 }
1602
1603 /* Convert to physical addresses. */
1604 pa_send_begin = pa_from_ipa(send);
1605 pa_send_end = pa_add(pa_send_begin, HF_MAILBOX_SIZE);
1606 pa_recv_begin = pa_from_ipa(recv);
1607 pa_recv_end = pa_add(pa_recv_begin, HF_MAILBOX_SIZE);
1608
1609 /* Fail if the same page is used for the send and receive pages. */
1610 if (pa_addr(pa_send_begin) == pa_addr(pa_recv_begin)) {
Karl Meakin738bee12024-01-12 15:19:45 +00001611 dlog_error("%s: Mailbox buffers overlap\n", __func__);
Manish Pandeyd34f8892020-06-19 17:41:07 +01001612 ret = ffa_error(FFA_INVALID_PARAMETERS);
1613 goto out;
1614 }
Andrew Sculle1322792019-07-01 17:46:10 +01001615
Federico Recanati8d8b1cf2022-04-14 13:16:00 +02001616 /* Set stage 2 translation tables only for virtual FF-A instances. */
1617 if (vm_id_is_current_world(vm_locked.vm->id)) {
1618 /*
1619 * Ensure the pages are valid, owned and exclusive to the VM and
1620 * that the VM has the required access to the memory.
1621 */
1622 if (!vm_mem_get_mode(vm_locked, send, ipa_add(send, PAGE_SIZE),
1623 &orig_send_mode) ||
1624 !api_mode_valid_owned_and_exclusive(orig_send_mode) ||
1625 (orig_send_mode & MM_MODE_R) == 0 ||
1626 (orig_send_mode & MM_MODE_W) == 0) {
1627 dlog_error(
1628 "VM doesn't have required access rights to map "
1629 "TX buffer in stage 2.\n");
1630 ret = ffa_error(FFA_INVALID_PARAMETERS);
1631 goto out;
1632 }
Manish Pandeyd34f8892020-06-19 17:41:07 +01001633
Federico Recanati8d8b1cf2022-04-14 13:16:00 +02001634 if (!vm_mem_get_mode(vm_locked, recv, ipa_add(recv, PAGE_SIZE),
1635 &orig_recv_mode) ||
1636 !api_mode_valid_owned_and_exclusive(orig_recv_mode) ||
1637 (orig_recv_mode & MM_MODE_R) == 0) {
1638 dlog_error(
1639 "VM doesn't have required access rights to map "
1640 "RX buffer in stage 2.\n");
1641 ret = ffa_error(FFA_INVALID_PARAMETERS);
1642 goto out;
1643 }
Andrew Sculle1322792019-07-01 17:46:10 +01001644
Federico Recanati8d8b1cf2022-04-14 13:16:00 +02001645 /* Take memory ownership away from the VM and mark as shared. */
1646 uint32_t mode = MM_MODE_UNOWNED | MM_MODE_SHARED | MM_MODE_R |
1647 MM_MODE_W;
1648 if (vm_locked.vm->el0_partition) {
1649 mode |= MM_MODE_USER | MM_MODE_NG;
1650 }
Raghu Krishnamurthy95c3a272021-02-26 18:09:41 -08001651
Federico Recanati8d8b1cf2022-04-14 13:16:00 +02001652 if (!vm_identity_map(vm_locked, pa_send_begin, pa_send_end,
1653 mode, local_page_pool, NULL)) {
1654 dlog_error(
1655 "Cannot allocate a new entry in stage 2 "
1656 "translation table.\n");
1657 ret = ffa_error(FFA_NO_MEMORY);
1658 goto out;
1659 }
Andrew Sculle1322792019-07-01 17:46:10 +01001660
Federico Recanati8d8b1cf2022-04-14 13:16:00 +02001661 mode = MM_MODE_UNOWNED | MM_MODE_SHARED | MM_MODE_R;
1662 if (vm_locked.vm->el0_partition) {
1663 mode |= MM_MODE_USER | MM_MODE_NG;
1664 }
Raghu Krishnamurthy95c3a272021-02-26 18:09:41 -08001665
Federico Recanati8d8b1cf2022-04-14 13:16:00 +02001666 if (!vm_identity_map(vm_locked, pa_recv_begin, pa_recv_end,
1667 mode, local_page_pool, NULL)) {
1668 /* TODO: partial defrag of failed range. */
1669 /* Recover any memory consumed in failed mapping. */
Karl Meakin738bee12024-01-12 15:19:45 +00001670 dlog_error("%s: cannot map recv page\n", __func__);
Federico Recanati8d8b1cf2022-04-14 13:16:00 +02001671 vm_ptable_defrag(vm_locked, local_page_pool);
Karl Meakin738bee12024-01-12 15:19:45 +00001672 ret = ffa_error(FFA_NO_MEMORY);
Federico Recanati8d8b1cf2022-04-14 13:16:00 +02001673 goto fail_undo_send;
1674 }
Karl Meakin738bee12024-01-12 15:19:45 +00001675 } else {
1676 ret = arch_other_world_vm_configure_rxtx_map(
1677 vm_locked, local_page_pool, pa_send_begin, pa_send_end,
1678 pa_recv_begin, pa_recv_end);
1679 if (ret.func != FFA_SUCCESS_32) {
1680 goto out;
1681 }
Andrew Sculle1322792019-07-01 17:46:10 +01001682 }
1683
Olivier Deprez96a2a262020-06-11 17:21:38 +02001684 /* Get extra send/recv pages mapping attributes for the given VM ID. */
1685 extra_attributes = arch_mm_extra_attributes_from_vm(vm_locked.vm->id);
1686
Raghu Krishnamurthy95c3a272021-02-26 18:09:41 -08001687 /*
1688 * For EL0 partitions, since both the partition and the hypervisor code
1689 * use the EL2&0 translation regime, it is critical to mark the mappings
1690 * of the send and recv buffers as non-global in the TLB. For one, if we
1691 * dont mark it as non-global, it would cause TLB conflicts since there
1692 * would be an identity mapping with non-global attribute in the
1693 * partitions page tables, but another identity mapping in the
1694 * hypervisor page tables with the global attribute. The other issue is
1695 * one of security, we dont want other partitions to be able to access
1696 * other partitions buffers through cached translations.
1697 */
1698 if (vm_locked.vm->el0_partition) {
1699 extra_attributes |= MM_MODE_NG;
1700 }
1701
Karl Meakin738bee12024-01-12 15:19:45 +00001702 ret = api_vm_configure_stage1(
1703 mm_stage1_locked, vm_locked, pa_send_begin, pa_send_end,
1704 pa_recv_begin, pa_recv_end, extra_attributes, local_page_pool);
1705 if (ret.func != FFA_SUCCESS_32) {
Andrew Sculle1322792019-07-01 17:46:10 +01001706 goto fail_undo_send_and_recv;
1707 }
1708
Manish Pandeyd34f8892020-06-19 17:41:07 +01001709 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Sculle1322792019-07-01 17:46:10 +01001710 goto out;
1711
Andrew Sculle1322792019-07-01 17:46:10 +01001712fail_undo_send_and_recv:
Andrew Scull3c257452019-11-26 13:32:50 +00001713 CHECK(vm_identity_map(vm_locked, pa_recv_begin, pa_recv_end,
Federico Recanati8d8b1cf2022-04-14 13:16:00 +02001714 orig_recv_mode, local_page_pool, NULL));
Andrew Sculle1322792019-07-01 17:46:10 +01001715
1716fail_undo_send:
Andrew Scull3c257452019-11-26 13:32:50 +00001717 CHECK(vm_identity_map(vm_locked, pa_send_begin, pa_send_end,
Manish Pandeyd34f8892020-06-19 17:41:07 +01001718 orig_send_mode, local_page_pool, NULL));
Andrew Sculle1322792019-07-01 17:46:10 +01001719
1720out:
Andrew Sculle1322792019-07-01 17:46:10 +01001721 return ret;
1722}
1723
Karl Meakinb9705e22024-04-05 13:58:28 +01001724/**
1725 * Read the buffer addresses and VM ID of an FFA_RXTX_MAP request. Handles
1726 * forwarded messages by reading from the hypervisor's TX buffer.
1727 *
1728 * Returns the VM/SP ID on success.
1729 *
1730 * Returns `HF_INVALID_VM_ID` when the arguments provided via the ABI
1731 * FFA_RXTX_MAP indicate the SPMC should retrieve the RXTX description from the
1732 * Hypervisor RXTX buffers, and the hypervisor hasn't given its own RXTX buffers
1733 * for the SPMC to map.
1734 */
1735static ffa_id_t api_get_rxtx_description(struct vm *current_vm, ipaddr_t *send,
1736 ipaddr_t *recv, uint32_t *page_count)
Federico Recanati8d8b1cf2022-04-14 13:16:00 +02001737{
Karl Meakinb9705e22024-04-05 13:58:28 +01001738 bool forwarded;
1739 struct vm_locked vm_locked;
1740 ffa_id_t owner_vm_id;
1741 struct ffa_endpoint_rx_tx_descriptor *endpoint_desc;
1742 struct ffa_composite_memory_region *rx_region;
1743 struct ffa_composite_memory_region *tx_region;
1744
Federico Recanati8d8b1cf2022-04-14 13:16:00 +02001745 /*
1746 * If the message has been forwarded the effective addresses are in
1747 * hypervisor's TX buffer.
1748 */
Karl Meakinb9705e22024-04-05 13:58:28 +01001749 forwarded = (current_vm->id == HF_OTHER_WORLD_ID) &&
1750 (ipa_addr(*send) == 0) && (ipa_addr(*recv) == 0) &&
1751 (*page_count == 0);
Federico Recanati8d8b1cf2022-04-14 13:16:00 +02001752
1753 if (forwarded) {
Karl Meakinb9705e22024-04-05 13:58:28 +01001754 vm_locked = vm_lock(current_vm);
1755 endpoint_desc = (struct ffa_endpoint_rx_tx_descriptor *)
1756 vm_locked.vm->mailbox.send;
Federico Recanati8d8b1cf2022-04-14 13:16:00 +02001757
Karl Meakinb9705e22024-04-05 13:58:28 +01001758 if (endpoint_desc == NULL) {
1759 dlog_error(
1760 "Trying to access RXTX description, but "
1761 "hypervisor has not provided RXTX buffers\n");
1762 vm_unlock(&vm_locked);
1763 return HF_INVALID_VM_ID;
1764 }
1765
1766 rx_region = ffa_endpoint_get_rx_memory_region(endpoint_desc);
1767 tx_region = ffa_endpoint_get_tx_memory_region(endpoint_desc);
1768
1769 owner_vm_id = endpoint_desc->endpoint_id;
Federico Recanati8d8b1cf2022-04-14 13:16:00 +02001770 *recv = ipa_init(rx_region->constituents[0].address);
1771 *send = ipa_init(tx_region->constituents[0].address);
1772 *page_count = rx_region->constituents[0].page_count;
J-Alves28ad9b42022-12-08 12:19:43 +00001773
1774 vm_unlock(&vm_locked);
Federico Recanati8d8b1cf2022-04-14 13:16:00 +02001775 } else {
Karl Meakinb9705e22024-04-05 13:58:28 +01001776 owner_vm_id = current_vm->id;
Federico Recanati8d8b1cf2022-04-14 13:16:00 +02001777 }
Karl Meakinb9705e22024-04-05 13:58:28 +01001778
1779 return owner_vm_id;
Federico Recanati8d8b1cf2022-04-14 13:16:00 +02001780}
Karl Meakinb9705e22024-04-05 13:58:28 +01001781
Andrew Sculle1322792019-07-01 17:46:10 +01001782/**
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +01001783 * Configures the VM to send/receive data through the specified pages. The pages
Manish Pandeyd34f8892020-06-19 17:41:07 +01001784 * must not be shared. Locking of the page tables combined with a local memory
1785 * pool ensures there will always be enough memory to recover from any errors
1786 * that arise. The stage-1 page tables must be locked so memory cannot be taken
1787 * by another core which could result in this transaction being unable to roll
1788 * back in the case of an error.
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +00001789 *
1790 * Returns:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001791 * - FFA_ERROR FFA_INVALID_PARAMETERS if the given addresses are not properly
Daniel Boulby6f8941e2021-06-14 18:27:18 +01001792 * aligned, are the same or have invalid attributes.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001793 * - FFA_ERROR FFA_NO_MEMORY if the hypervisor was unable to map the buffers
Andrew Walbranbfffb0f2019-11-05 14:02:34 +00001794 * due to insuffient page table memory.
Daniel Boulby6f8941e2021-06-14 18:27:18 +01001795 * - FFA_ERROR FFA_DENIED if the pages are already mapped.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001796 * - FFA_SUCCESS on success if no further action is needed.
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +01001797 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001798struct ffa_value api_ffa_rxtx_map(ipaddr_t send, ipaddr_t recv,
Federico Recanati9f1b6532022-04-14 13:15:28 +02001799 uint32_t page_count, struct vcpu *current)
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +01001800{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001801 struct ffa_value ret;
Federico Recanati8d8b1cf2022-04-14 13:16:00 +02001802 struct vm_locked owner_vm_locked;
Manish Pandeyd34f8892020-06-19 17:41:07 +01001803 struct mm_stage1_locked mm_stage1_locked;
1804 struct mpool local_page_pool;
J-Alves19e20cf2023-08-02 12:48:55 +01001805 ffa_id_t owner_vm_id;
Federico Recanati8d8b1cf2022-04-14 13:16:00 +02001806
Federico Recanati8d8b1cf2022-04-14 13:16:00 +02001807 /*
1808 * Get the original buffer addresses and VM ID in case of forwarded
1809 * message.
1810 */
Karl Meakinb9705e22024-04-05 13:58:28 +01001811 owner_vm_id = api_get_rxtx_description(current->vm, &send, &recv,
1812 &page_count);
1813 if (owner_vm_id == HF_INVALID_VM_ID) {
1814 return ffa_error(FFA_INVALID_PARAMETERS);
1815 }
Federico Recanati8d8b1cf2022-04-14 13:16:00 +02001816
1817 owner_vm_locked = plat_ffa_vm_find_locked_create(owner_vm_id);
1818 if (owner_vm_locked.vm == NULL) {
1819 dlog_error("Cannot map RX/TX for VM ID %#x, not found.\n",
1820 owner_vm_id);
1821 return ffa_error(FFA_DENIED);
1822 }
Andrew Scull220e6212018-12-21 18:09:00 +00001823
Andrew Scull3c0a90a2019-07-01 11:55:53 +01001824 /*
Manish Pandeyd34f8892020-06-19 17:41:07 +01001825 * Create a local pool so any freed memory can't be used by another
1826 * thread. This is to ensure the original mapping can be restored if any
1827 * stage of the process fails.
Andrew Scull3c0a90a2019-07-01 11:55:53 +01001828 */
Manish Pandeyd34f8892020-06-19 17:41:07 +01001829 mpool_init_with_fallback(&local_page_pool, &api_page_pool);
1830
Manish Pandeyd34f8892020-06-19 17:41:07 +01001831 mm_stage1_locked = mm_lock_stage1();
Andrew Scull220e6212018-12-21 18:09:00 +00001832
Federico Recanati8d8b1cf2022-04-14 13:16:00 +02001833 ret = api_vm_configure_pages(mm_stage1_locked, owner_vm_locked, send,
1834 recv, page_count, &local_page_pool);
Manish Pandeyd34f8892020-06-19 17:41:07 +01001835 if (ret.func != FFA_SUCCESS_32) {
Andrew Walbranbfffb0f2019-11-05 14:02:34 +00001836 goto exit;
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +01001837 }
1838
Federico Recanati8d8b1cf2022-04-14 13:16:00 +02001839 /* Forward buffer mapping to SPMC if coming from a VM. */
1840 plat_ffa_rxtx_map_forward(owner_vm_locked);
1841
Federico Recanati9f1b6532022-04-14 13:15:28 +02001842 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Andrew Scull220e6212018-12-21 18:09:00 +00001843
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +01001844exit:
Manish Pandeyd34f8892020-06-19 17:41:07 +01001845 mpool_fini(&local_page_pool);
Manish Pandeyd34f8892020-06-19 17:41:07 +01001846 mm_unlock_stage1(&mm_stage1_locked);
Federico Recanati8d8b1cf2022-04-14 13:16:00 +02001847 vm_unlock(&owner_vm_locked);
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +01001848
1849 return ret;
1850}
1851
1852/**
Daniel Boulby9e420ca2021-07-07 15:03:49 +01001853 * Unmaps the RX/TX buffer pair with a partition or partition manager from the
1854 * translation regime of the caller. Unmap the region for the hypervisor and
1855 * set the memory region to owned and exclusive for the component. Since the
1856 * memory region mapped in the page table, when the buffers were originally
1857 * created we can safely remap it.
1858 *
1859 * Returns:
1860 * - FFA_ERROR FFA_INVALID_PARAMETERS if there is no buffer pair registered on
1861 * behalf of the caller.
1862 * - FFA_SUCCESS on success if no further action is needed.
1863 */
J-Alves19e20cf2023-08-02 12:48:55 +01001864struct ffa_value api_ffa_rxtx_unmap(ffa_id_t allocator_id, struct vcpu *current)
Daniel Boulby9e420ca2021-07-07 15:03:49 +01001865{
1866 struct vm *vm = current->vm;
1867 struct vm_locked vm_locked;
J-Alves19e20cf2023-08-02 12:48:55 +01001868 ffa_id_t owner_vm_id;
Daniel Boulby9e420ca2021-07-07 15:03:49 +01001869 struct mm_stage1_locked mm_stage1_locked;
1870 paddr_t send_pa_begin;
1871 paddr_t send_pa_end;
1872 paddr_t recv_pa_begin;
1873 paddr_t recv_pa_end;
Federico Recanati8da9e332022-02-10 11:00:17 +01001874 struct ffa_value ret = (struct ffa_value){.func = FFA_SUCCESS_32};
Daniel Boulby9e420ca2021-07-07 15:03:49 +01001875
J-Alves661e1b72023-08-02 13:39:40 +01001876 if (vm->id == HF_HYPERVISOR_VM_ID && !ffa_is_vm_id(allocator_id)) {
Olivier Deprez106c8ce2024-02-08 11:49:02 +01001877 dlog_verbose(
J-Alves9bd324a2023-01-12 10:52:52 +00001878 "The Hypervisor must specify a valid VM ID in register "
1879 "W1, if FFA_RXTX_UNMAP call forwarded to SPM.\n");
1880 return ffa_error(FFA_INVALID_PARAMETERS);
1881 }
1882
Federico Recanati8da9e332022-02-10 11:00:17 +01001883 /* Ensure `allocator_id` is set only at Non-Secure Physical instance. */
1884 if (vm_id_is_current_world(vm->id) && (allocator_id != 0)) {
J-Alves9bd324a2023-01-12 10:52:52 +00001885 dlog_error(
1886 "The register W1 (containing ID) must be 0 at virtual "
1887 "instances.\n");
Federico Recanati8da9e332022-02-10 11:00:17 +01001888 return ffa_error(FFA_INVALID_PARAMETERS);
1889 }
1890
1891 /* VM ID of which buffers have to be unmapped. */
1892 owner_vm_id = (allocator_id != 0) ? allocator_id : vm->id;
1893
1894 vm_locked = plat_ffa_vm_find_locked(owner_vm_id);
1895 vm = vm_locked.vm;
1896 if (vm == NULL) {
1897 dlog_error("Cannot unmap RX/TX for VM ID %#x, not found.\n",
1898 owner_vm_id);
Daniel Boulby9e420ca2021-07-07 15:03:49 +01001899 return ffa_error(FFA_INVALID_PARAMETERS);
1900 }
1901
1902 /* Get send and receive buffers. */
1903 if (vm->mailbox.send == NULL || vm->mailbox.recv == NULL) {
Olivier Deprez86d87ae2021-08-19 14:27:46 +02001904 dlog_verbose(
Daniel Boulby9e420ca2021-07-07 15:03:49 +01001905 "No buffer pair registered on behalf of the caller.\n");
Federico Recanati8da9e332022-02-10 11:00:17 +01001906 ret = ffa_error(FFA_INVALID_PARAMETERS);
1907 goto out;
Daniel Boulby9e420ca2021-07-07 15:03:49 +01001908 }
1909
1910 /* Currently a mailbox size of 1 page is assumed. */
1911 send_pa_begin = pa_from_va(va_from_ptr(vm->mailbox.send));
1912 send_pa_end = pa_add(send_pa_begin, HF_MAILBOX_SIZE);
1913 recv_pa_begin = pa_from_va(va_from_ptr(vm->mailbox.recv));
1914 recv_pa_end = pa_add(recv_pa_begin, HF_MAILBOX_SIZE);
1915
Daniel Boulby9e420ca2021-07-07 15:03:49 +01001916 mm_stage1_locked = mm_lock_stage1();
1917
Federico Recanati8da9e332022-02-10 11:00:17 +01001918 /* Reset stage 2 mapping only for virtual FF-A instances. */
1919 if (vm_id_is_current_world(owner_vm_id)) {
1920 /*
1921 * Set the memory region of the buffers back to the default mode
1922 * for the VM. Since this memory region was already mapped for
1923 * the RXTX buffers we can safely remap them.
1924 */
1925 CHECK(vm_identity_map(vm_locked, send_pa_begin, send_pa_end,
1926 MM_MODE_R | MM_MODE_W | MM_MODE_X,
1927 &api_page_pool, NULL));
Daniel Boulby9e420ca2021-07-07 15:03:49 +01001928
Federico Recanati8da9e332022-02-10 11:00:17 +01001929 CHECK(vm_identity_map(vm_locked, recv_pa_begin, recv_pa_end,
1930 MM_MODE_R | MM_MODE_W | MM_MODE_X,
1931 &api_page_pool, NULL));
Karl Meakin738bee12024-01-12 15:19:45 +00001932 } else {
1933 ret = arch_other_world_vm_configure_rxtx_unmap(
1934 vm_locked, &api_page_pool, send_pa_begin, send_pa_end,
1935 recv_pa_begin, recv_pa_end);
1936 if (ret.func != FFA_SUCCESS_32) {
1937 goto out;
1938 }
Federico Recanati8da9e332022-02-10 11:00:17 +01001939 }
Daniel Boulby9e420ca2021-07-07 15:03:49 +01001940
1941 /* Unmap the buffers in the partition manager. */
1942 CHECK(mm_unmap(mm_stage1_locked, send_pa_begin, send_pa_end,
1943 &api_page_pool));
1944 CHECK(mm_unmap(mm_stage1_locked, recv_pa_begin, recv_pa_end,
1945 &api_page_pool));
1946
1947 vm->mailbox.send = NULL;
1948 vm->mailbox.recv = NULL;
Federico Recanati10bd06c2022-02-23 17:32:59 +01001949 plat_ffa_vm_destroy(vm_locked);
Daniel Boulby9e420ca2021-07-07 15:03:49 +01001950
Federico Recanati8da9e332022-02-10 11:00:17 +01001951 /* Forward buffer unmapping to SPMC if coming from a VM. */
J-Alves70079932022-12-07 17:32:20 +00001952 plat_ffa_rxtx_unmap_forward(vm_locked);
Federico Recanati8da9e332022-02-10 11:00:17 +01001953
Daniel Boulby9e420ca2021-07-07 15:03:49 +01001954 mm_unlock_stage1(&mm_stage1_locked);
Federico Recanati8da9e332022-02-10 11:00:17 +01001955
1956out:
Daniel Boulby9e420ca2021-07-07 15:03:49 +01001957 vm_unlock(&vm_locked);
1958
Federico Recanati8da9e332022-02-10 11:00:17 +01001959 return ret;
Daniel Boulby9e420ca2021-07-07 15:03:49 +01001960}
1961
1962/**
Federico Recanati25053ee2022-03-14 15:01:53 +01001963 * Copies data from the sender's send buffer to the recipient's receive buffer
1964 * and notifies the receiver.
1965 */
J-Alves19e20cf2023-08-02 12:48:55 +01001966struct ffa_value api_ffa_msg_send2(ffa_id_t sender_vm_id, uint32_t flags,
Federico Recanati25053ee2022-03-14 15:01:53 +01001967 struct vcpu *current)
1968{
1969 struct vm *from = current->vm;
1970 struct vm *to;
1971 struct vm_locked to_locked;
J-Alves19e20cf2023-08-02 12:48:55 +01001972 ffa_id_t msg_sender_id;
Federico Recanati25053ee2022-03-14 15:01:53 +01001973 struct vm_locked sender_locked;
1974 const void *from_msg;
1975 struct ffa_value ret;
J-Alves19e20cf2023-08-02 12:48:55 +01001976 ffa_id_t sender_id;
1977 ffa_id_t receiver_id;
Federico Recanati25053ee2022-03-14 15:01:53 +01001978 uint32_t msg_size;
1979 ffa_notifications_bitmap_t rx_buffer_full;
1980
Karl Meakina5ea9092024-05-28 15:40:33 +01001981 alignas(8) struct ffa_partition_rxtx_header header;
1982
Federico Recanati25053ee2022-03-14 15:01:53 +01001983 /* Only Hypervisor can set `sender_vm_id` when forwarding messages. */
1984 if (from->id != HF_HYPERVISOR_VM_ID && sender_vm_id != 0) {
1985 dlog_error("Sender VM ID must be zero.\n");
1986 return ffa_error(FFA_INVALID_PARAMETERS);
1987 }
1988
Federico Recanati25053ee2022-03-14 15:01:53 +01001989 /*
1990 * Get message sender's mailbox, which can be different to the `from` vm
1991 * when the message is forwarded.
1992 */
1993 msg_sender_id = (sender_vm_id != 0) ? sender_vm_id : from->id;
1994 sender_locked = plat_ffa_vm_find_locked(msg_sender_id);
1995 if (sender_locked.vm == NULL) {
1996 dlog_error("Cannot send message from VM ID %#x, not found.\n",
1997 msg_sender_id);
1998 return ffa_error(FFA_DENIED);
1999 }
2000
2001 from_msg = sender_locked.vm->mailbox.send;
2002 if (from_msg == NULL) {
2003 dlog_error("Cannot retrieve TX buffer for VM ID %#x.\n",
2004 msg_sender_id);
2005 ret = ffa_error(FFA_DENIED);
2006 goto out_unlock_sender;
2007 }
2008
2009 /*
2010 * Copy message header as safety measure to avoid multiple accesses to
2011 * unsafe memory which could be 'corrupted' between safety checks and
2012 * final buffer copy.
2013 */
J-Alves8a5fd9d2024-04-09 11:22:49 +01002014 if (!memcpy_trapped(&header, FFA_RXTX_HEADER_SIZE, from_msg,
2015 FFA_RXTX_HEADER_SIZE)) {
2016 dlog_error(
2017 "%s: Failed to copy message from sender's(%x) TX "
2018 "buffer.\n",
2019 __func__, sender_locked.vm->id);
2020 ret = ffa_error(FFA_ABORTED);
2021 goto out_unlock_sender;
2022 }
2023
Federico Recanati25053ee2022-03-14 15:01:53 +01002024 sender_id = ffa_rxtx_header_sender(&header);
2025 receiver_id = ffa_rxtx_header_receiver(&header);
2026
2027 /* Ensure Sender IDs from API and from message header match. */
2028 if (msg_sender_id != sender_id) {
2029 dlog_error(
2030 "Message sender VM ID (%#x) doesn't match header's VM "
2031 "ID (%#x).\n",
2032 msg_sender_id, sender_id);
2033 ret = ffa_error(FFA_INVALID_PARAMETERS);
2034 goto out_unlock_sender;
2035 }
2036
2037 /* Disallow reflexive requests as this suggests an error in the VM. */
2038 if (receiver_id == sender_id) {
2039 dlog_error("Sender and receive VM IDs must be different.\n");
2040 ret = ffa_error(FFA_INVALID_PARAMETERS);
2041 goto out_unlock_sender;
2042 }
2043
Federico Recanati7b39ff22022-03-14 15:01:53 +01002044 /* `flags` can be set only at secure virtual FF-A instances. */
J-Alves661e1b72023-08-02 13:39:40 +01002045 if (ffa_is_vm_id(sender_id) && (flags != 0)) {
Federico Recanati7b39ff22022-03-14 15:01:53 +01002046 dlog_error("flags must be zero.\n");
2047 return ffa_error(FFA_INVALID_PARAMETERS);
2048 }
2049
Olivier Deprez37b75112024-03-04 18:27:30 +01002050 if (header.offset != FFA_RXTX_HEADER_SIZE) {
2051 dlog_error("Indirect msg payload must follow the header.\n");
2052 return ffa_error(FFA_INVALID_PARAMETERS);
2053 }
2054
Federico Recanati25053ee2022-03-14 15:01:53 +01002055 /*
2056 * Check if the message has to be forwarded to the SPMC, in
2057 * this case return, the SPMC will handle the buffer copy.
2058 */
2059 if (plat_ffa_msg_send2_forward(receiver_id, sender_id, &ret)) {
2060 goto out_unlock_sender;
2061 }
2062
2063 /* Ensure the receiver VM exists. */
2064 to_locked = plat_ffa_vm_find_locked(receiver_id);
2065 to = to_locked.vm;
2066
2067 if (to == NULL) {
2068 dlog_error("Cannot deliver message to VM %#x, not found.\n",
2069 receiver_id);
2070 ret = ffa_error(FFA_INVALID_PARAMETERS);
2071 goto out_unlock_sender;
2072 }
2073
2074 /*
2075 * Check sender and receiver can use indirect messages.
2076 * Sender is the VM/SP who originally sent the message, not the
2077 * hypervisor possibly relaying it.
2078 */
2079 if (!plat_ffa_is_indirect_msg_supported(sender_locked, to_locked)) {
2080 dlog_verbose("VM %#x doesn't support indirect message\n",
2081 sender_id);
2082 ret = ffa_error(FFA_DENIED);
2083 goto out;
2084 }
2085
J-Alvese8c8c2b2022-12-16 15:34:48 +00002086 if (vm_is_mailbox_busy(to_locked)) {
Federico Recanati25053ee2022-03-14 15:01:53 +01002087 dlog_error(
2088 "Cannot deliver message to VM %#x, RX buffer not "
2089 "ready.\n",
2090 receiver_id);
2091 ret = ffa_error(FFA_BUSY);
2092 goto out;
2093 }
2094
Federico Recanati644f0462022-03-17 12:04:00 +01002095 /* Acquire receiver's RX buffer. */
2096 if (!plat_ffa_acquire_receiver_rx(to_locked, &ret)) {
2097 dlog_error("Failed to acquire RX buffer for VM %#x\n", to->id);
2098 goto out;
2099 }
2100
Federico Recanati25053ee2022-03-14 15:01:53 +01002101 /* Check the size of transfer. */
2102 msg_size = FFA_RXTX_HEADER_SIZE + header.size;
Olivier Deprezf1dd1e12024-03-04 18:21:01 +01002103 if ((msg_size > FFA_MSG_PAYLOAD_MAX) ||
Federico Recanati25053ee2022-03-14 15:01:53 +01002104 (header.size > FFA_PARTITION_MSG_PAYLOAD_MAX)) {
2105 dlog_error("Message is too big.\n");
2106 ret = ffa_error(FFA_INVALID_PARAMETERS);
2107 goto out;
2108 }
2109
2110 /* Copy data. */
J-Alves8a5fd9d2024-04-09 11:22:49 +01002111 if (!memcpy_trapped(to->mailbox.recv, FFA_MSG_PAYLOAD_MAX, from_msg,
2112 msg_size)) {
2113 dlog_error(
2114 "%s: Failed to copy message to receiver's(%x) RX "
2115 "buffer.\n",
2116 __func__, to->id);
2117 ret = ffa_error(FFA_ABORTED);
2118 goto out;
2119 }
2120
Federico Recanati25053ee2022-03-14 15:01:53 +01002121 to->mailbox.recv_size = msg_size;
2122 to->mailbox.recv_sender = sender_id;
2123 to->mailbox.recv_func = FFA_MSG_SEND2_32;
J-Alvese8c8c2b2022-12-16 15:34:48 +00002124 to->mailbox.state = MAILBOX_STATE_FULL;
Federico Recanati25053ee2022-03-14 15:01:53 +01002125
J-Alves661e1b72023-08-02 13:39:40 +01002126 rx_buffer_full = ffa_is_vm_id(sender_id)
Federico Recanati25053ee2022-03-14 15:01:53 +01002127 ? FFA_NOTIFICATION_HYP_BUFFER_FULL_MASK
2128 : FFA_NOTIFICATION_SPM_BUFFER_FULL_MASK;
2129 vm_notifications_framework_set_pending(to_locked, rx_buffer_full);
2130
2131 if ((FFA_NOTIFICATIONS_FLAG_DELAY_SRI & flags) == 0) {
2132 dlog_verbose("SRI was NOT delayed. vcpu: %u!\n",
2133 vcpu_index(current));
2134 plat_ffa_sri_trigger_not_delayed(current->cpu);
2135 } else {
2136 plat_ffa_sri_state_set(DELAYED);
2137 }
2138
2139 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
2140
2141out:
2142 vm_unlock(&to_locked);
2143
2144out_unlock_sender:
2145 vm_unlock(&sender_locked);
2146
2147 return ret;
2148}
2149
2150/**
Andrew Walbran8a0f5ca2019-11-05 13:12:23 +00002151 * Releases the caller's mailbox so that a new message can be received. The
2152 * caller must have copied out all data they wish to preserve as new messages
2153 * will overwrite the old and will arrive asynchronously.
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +00002154 *
2155 * Returns:
Federico Recanati7bef0b92022-03-17 14:56:22 +01002156 * - FFA_ERROR FFA_INVALID_PARAMETERS if message is forwarded to SPMC but
2157 * there's no buffer pair mapped.
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002158 * - FFA_ERROR FFA_DENIED on failure, if the mailbox hasn't been read.
2159 * - FFA_SUCCESS on success if no further action is needed.
2160 * - FFA_RX_RELEASE if it was called by the primary VM and the primary VM now
Andrew Walbran8a0f5ca2019-11-05 13:12:23 +00002161 * needs to wake up or kick waiters. Waiters should be retrieved by calling
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +00002162 * hf_mailbox_waiter_get.
2163 */
J-Alves19e20cf2023-08-02 12:48:55 +01002164struct ffa_value api_ffa_rx_release(ffa_id_t receiver_id, struct vcpu *current)
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +00002165{
Federico Recanati7bef0b92022-03-17 14:56:22 +01002166 struct vm *current_vm = current->vm;
2167 struct vm *vm;
2168 struct vm_locked vm_locked;
J-Alves19e20cf2023-08-02 12:48:55 +01002169 ffa_id_t current_vm_id = current_vm->id;
2170 ffa_id_t release_vm_id;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002171 struct ffa_value ret;
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +00002172
Federico Recanati7bef0b92022-03-17 14:56:22 +01002173 /* `receiver_id` can be set only at Non-Secure Physical interface. */
2174 if (vm_id_is_current_world(current_vm_id) && (receiver_id != 0)) {
2175 dlog_error("Invalid `receiver_id`, must be zero.\n");
2176 return ffa_error(FFA_INVALID_PARAMETERS);
2177 }
2178
2179 /*
2180 * VM ID to be released: `receiver_id` if message has been forwarded by
2181 * Hypervisor to release a VM's buffer, current VM ID otherwise.
2182 */
2183 if (vm_id_is_current_world(current_vm_id) || (receiver_id == 0)) {
2184 release_vm_id = current_vm_id;
2185 } else {
2186 release_vm_id = receiver_id;
2187 }
2188
2189 vm_locked = plat_ffa_vm_find_locked(release_vm_id);
2190 vm = vm_locked.vm;
2191 if (vm == NULL) {
2192 dlog_error("No buffer registered for VM ID %#x.\n",
2193 release_vm_id);
2194 return ffa_error(FFA_INVALID_PARAMETERS);
2195 }
2196
J-Alvese8c8c2b2022-12-16 15:34:48 +00002197 if (plat_ffa_rx_release_forward(vm_locked, &ret)) {
Federico Recanati7bef0b92022-03-17 14:56:22 +01002198 goto out;
2199 }
2200
Karl Meakin1064a9c2024-06-04 17:20:43 +01002201 ret = api_release_mailbox(vm_locked);
Federico Recanati7bef0b92022-03-17 14:56:22 +01002202
2203out:
2204 vm_unlock(&vm_locked);
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +01002205
2206 return ret;
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +01002207}
Andrew Walbran318f5732018-11-20 16:23:42 +00002208
2209/**
Federico Recanati644f0462022-03-17 12:04:00 +01002210 * Acquire ownership of an RX buffer before writing to it. Both
2211 * Hypervisor and SPMC are producers of VM's RX buffer, and they
2212 * could contend for the same buffer. SPMC owns VM's RX buffer after
2213 * it's mapped in its translation regime. This ABI should be
2214 * used by the Hypervisor to get the ownership of a VM's RX buffer
2215 * from the SPMC solving the aforementioned possible contention.
2216 *
2217 * Returns:
2218 * - FFA_DENIED: callee cannot relinquish ownership of RX buffer.
2219 * - FFA_INVALID_PARAMETERS: there is no buffer pair registered for the VM.
2220 * - FFA_NOT_SUPPORTED: function not implemented at the FF-A instance.
2221 */
J-Alves19e20cf2023-08-02 12:48:55 +01002222struct ffa_value api_ffa_rx_acquire(ffa_id_t receiver_id, struct vcpu *current)
Federico Recanati644f0462022-03-17 12:04:00 +01002223{
2224 struct vm_locked receiver_locked;
2225 struct vm *receiver;
2226 struct ffa_value ret;
2227
2228 if ((current->vm->id != HF_HYPERVISOR_VM_ID) ||
J-Alves661e1b72023-08-02 13:39:40 +01002229 !ffa_is_vm_id(receiver_id)) {
Federico Recanati644f0462022-03-17 12:04:00 +01002230 dlog_error(
2231 "FFA_RX_ACQUIRE not supported at this FF-A "
2232 "instance.\n");
2233 return ffa_error(FFA_NOT_SUPPORTED);
2234 }
2235
2236 receiver_locked = plat_ffa_vm_find_locked(receiver_id);
2237 receiver = receiver_locked.vm;
2238
2239 if (receiver == NULL || receiver->mailbox.recv == NULL) {
2240 dlog_error("Cannot retrieve RX buffer for VM ID %#x.\n",
2241 receiver_id);
2242 ret = ffa_error(FFA_INVALID_PARAMETERS);
2243 goto out;
2244 }
2245
2246 if (receiver->mailbox.state != MAILBOX_STATE_EMPTY) {
2247 dlog_error("Mailbox busy for VM ID %#x.\n", receiver_id);
2248 ret = ffa_error(FFA_DENIED);
2249 goto out;
2250 }
2251
J-Alvese8c8c2b2022-12-16 15:34:48 +00002252 receiver->mailbox.state = MAILBOX_STATE_OTHER_WORLD_OWNED;
Federico Recanati644f0462022-03-17 12:04:00 +01002253
2254 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
2255
2256out:
2257 vm_unlock(&receiver_locked);
2258
2259 return ret;
2260}
2261
2262/**
Andrew Walbran318f5732018-11-20 16:23:42 +00002263 * Enables or disables a given interrupt ID for the calling vCPU.
2264 *
2265 * Returns 0 on success, or -1 if the intid is invalid.
2266 */
Manish Pandey35e452f2021-02-18 21:36:34 +00002267int64_t api_interrupt_enable(uint32_t intid, bool enable,
2268 enum interrupt_type type, struct vcpu *current)
Andrew Walbran318f5732018-11-20 16:23:42 +00002269{
Manish Pandey35e452f2021-02-18 21:36:34 +00002270 struct vcpu_locked current_locked;
Daniel Boulby4ca50f02022-07-29 18:29:34 +01002271 struct interrupts *interrupts = &current->interrupts;
Wedson Almeida Filho81568c42019-01-04 13:33:02 +00002272
Andrew Walbran318f5732018-11-20 16:23:42 +00002273 if (intid >= HF_NUM_INTIDS) {
2274 return -1;
2275 }
2276
Manish Pandey35e452f2021-02-18 21:36:34 +00002277 current_locked = vcpu_lock(current);
Andrew Walbran318f5732018-11-20 16:23:42 +00002278 if (enable) {
Andrew Walbran3d84a262018-12-13 14:41:19 +00002279 /*
2280 * If it is pending and was not enabled before, increment the
2281 * count.
2282 */
Daniel Boulby4ca50f02022-07-29 18:29:34 +01002283 if (vcpu_is_virt_interrupt_pending(interrupts, intid) &&
2284 !vcpu_is_virt_interrupt_enabled(interrupts, intid)) {
2285 vcpu_interrupt_count_increment(current_locked,
2286 interrupts, intid);
Andrew Walbran3d84a262018-12-13 14:41:19 +00002287 }
Daniel Boulby4ca50f02022-07-29 18:29:34 +01002288 vcpu_virt_interrupt_set_enabled(interrupts, intid);
2289 vcpu_virt_interrupt_set_type(interrupts, intid, type);
Andrew Walbran318f5732018-11-20 16:23:42 +00002290 } else {
Andrew Walbran3d84a262018-12-13 14:41:19 +00002291 /*
2292 * If it is pending and was enabled before, decrement the count.
2293 */
Daniel Boulby4ca50f02022-07-29 18:29:34 +01002294 if (vcpu_is_virt_interrupt_pending(interrupts, intid) &&
2295 vcpu_is_virt_interrupt_enabled(interrupts, intid)) {
2296 vcpu_interrupt_count_decrement(current_locked,
2297 interrupts, intid);
Andrew Walbran3d84a262018-12-13 14:41:19 +00002298 }
Daniel Boulby4ca50f02022-07-29 18:29:34 +01002299 vcpu_virt_interrupt_clear_enabled(interrupts, intid);
2300 vcpu_virt_interrupt_set_type(interrupts, intid,
2301 INTERRUPT_TYPE_IRQ);
Andrew Walbran318f5732018-11-20 16:23:42 +00002302 }
2303
Manish Pandey35e452f2021-02-18 21:36:34 +00002304 vcpu_unlock(&current_locked);
Andrew Walbran318f5732018-11-20 16:23:42 +00002305 return 0;
2306}
2307
Madhukar Pappireddya1019112022-06-21 18:57:44 -05002308static void api_interrupt_clear_decrement(struct vcpu_locked locked_vcpu,
2309 struct interrupts *interrupts,
2310 uint32_t intid)
2311{
2312 vcpu_virt_interrupt_clear_pending(interrupts, intid);
2313 vcpu_interrupt_count_decrement(locked_vcpu, interrupts, intid);
2314}
2315
Andrew Walbran318f5732018-11-20 16:23:42 +00002316/**
2317 * Returns the ID of the next pending interrupt for the calling vCPU, and
2318 * acknowledges it (i.e. marks it as no longer pending). Returns
2319 * HF_INVALID_INTID if there are no pending interrupts.
2320 */
Wedson Almeida Filhoc559d132019-01-09 19:33:40 +00002321uint32_t api_interrupt_get(struct vcpu *current)
Andrew Walbran318f5732018-11-20 16:23:42 +00002322{
Raghu Krishnamurthy61a025b2022-07-20 08:37:04 -07002323 uint32_t i;
Andrew Walbran318f5732018-11-20 16:23:42 +00002324 uint32_t first_interrupt = HF_INVALID_INTID;
Manish Pandey35e452f2021-02-18 21:36:34 +00002325 struct vcpu_locked current_locked;
Daniel Boulby4ca50f02022-07-29 18:29:34 +01002326 struct interrupts *interrupts = &current->interrupts;
Andrew Walbran318f5732018-11-20 16:23:42 +00002327
2328 /*
2329 * Find the first enabled and pending interrupt ID, return it, and
2330 * deactivate it.
2331 */
Manish Pandey35e452f2021-02-18 21:36:34 +00002332 current_locked = vcpu_lock(current);
Andrew Walbran318f5732018-11-20 16:23:42 +00002333 for (i = 0; i < HF_NUM_INTIDS / INTERRUPT_REGISTER_BITS; ++i) {
2334 uint32_t enabled_and_pending =
Daniel Boulby4ca50f02022-07-29 18:29:34 +01002335 interrupts->interrupt_enabled.bitmap[i] &
2336 interrupts->interrupt_pending.bitmap[i];
Wedson Almeida Filho81568c42019-01-04 13:33:02 +00002337
Andrew Walbran318f5732018-11-20 16:23:42 +00002338 if (enabled_and_pending != 0) {
Andrew Walbran3d84a262018-12-13 14:41:19 +00002339 uint8_t bit_index = ctz(enabled_and_pending);
Daniel Boulby4ca50f02022-07-29 18:29:34 +01002340
2341 first_interrupt =
2342 i * INTERRUPT_REGISTER_BITS + bit_index;
Manish Pandey35e452f2021-02-18 21:36:34 +00002343
Andrew Walbran3d84a262018-12-13 14:41:19 +00002344 /*
2345 * Mark it as no longer pending and decrement the count.
2346 */
Madhukar Pappireddya1019112022-06-21 18:57:44 -05002347 api_interrupt_clear_decrement(
Daniel Boulby4ca50f02022-07-29 18:29:34 +01002348 current_locked, interrupts, first_interrupt);
Andrew Walbran318f5732018-11-20 16:23:42 +00002349 break;
2350 }
2351 }
Andrew Walbran318f5732018-11-20 16:23:42 +00002352
Manish Pandey35e452f2021-02-18 21:36:34 +00002353 vcpu_unlock(&current_locked);
Andrew Walbran318f5732018-11-20 16:23:42 +00002354 return first_interrupt;
2355}
2356
2357/**
Andrew Walbran4cf217a2018-12-14 15:24:50 +00002358 * Returns whether the current vCPU is allowed to inject an interrupt into the
Andrew Walbran318f5732018-11-20 16:23:42 +00002359 * given VM and vCPU.
2360 */
2361static inline bool is_injection_allowed(uint32_t target_vm_id,
2362 struct vcpu *current)
2363{
2364 uint32_t current_vm_id = current->vm->id;
Wedson Almeida Filho81568c42019-01-04 13:33:02 +00002365
Andrew Walbran318f5732018-11-20 16:23:42 +00002366 /*
2367 * The primary VM is allowed to inject interrupts into any VM. Secondary
2368 * VMs are only allowed to inject interrupts into their own vCPUs.
2369 */
2370 return current_vm_id == HF_PRIMARY_VM_ID ||
2371 current_vm_id == target_vm_id;
2372}
2373
2374/**
2375 * Injects a virtual interrupt of the given ID into the given target vCPU.
2376 * This doesn't cause the vCPU to actually be run immediately; it will be taken
2377 * when the vCPU is next run, which is up to the scheduler.
2378 *
Andrew Walbran3d84a262018-12-13 14:41:19 +00002379 * Returns:
2380 * - -1 on failure because the target VM or vCPU doesn't exist, the interrupt
2381 * ID is invalid, or the current VM is not allowed to inject interrupts to
2382 * the target VM.
2383 * - 0 on success if no further action is needed.
2384 * - 1 if it was called by the primary VM and the primary VM now needs to wake
2385 * up or kick the target vCPU.
Andrew Walbran318f5732018-11-20 16:23:42 +00002386 */
J-Alves19e20cf2023-08-02 12:48:55 +01002387int64_t api_interrupt_inject(ffa_id_t target_vm_id,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002388 ffa_vcpu_index_t target_vcpu_idx, uint32_t intid,
Andrew Walbran42347a92019-05-09 13:59:03 +01002389 struct vcpu *current, struct vcpu **next)
Andrew Walbran318f5732018-11-20 16:23:42 +00002390{
Andrew Walbran318f5732018-11-20 16:23:42 +00002391 struct vcpu *target_vcpu;
Andrew Walbran42347a92019-05-09 13:59:03 +01002392 struct vm *target_vm = vm_find(target_vm_id);
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06002393 struct vcpu_locked current_locked;
2394 struct vcpu_locked target_locked;
2395 struct two_vcpu_locked vcpus_locked;
2396 int64_t ret;
Andrew Walbran318f5732018-11-20 16:23:42 +00002397
2398 if (intid >= HF_NUM_INTIDS) {
2399 return -1;
2400 }
Wedson Almeida Filho81568c42019-01-04 13:33:02 +00002401
Andrew Walbran318f5732018-11-20 16:23:42 +00002402 if (target_vm == NULL) {
2403 return -1;
2404 }
Wedson Almeida Filho81568c42019-01-04 13:33:02 +00002405
Andrew Walbran318f5732018-11-20 16:23:42 +00002406 if (target_vcpu_idx >= target_vm->vcpu_count) {
Fuad Tabbab0ef2a42019-12-19 11:19:25 +00002407 /* The requested vCPU must exist. */
Andrew Walbran318f5732018-11-20 16:23:42 +00002408 return -1;
2409 }
Wedson Almeida Filho81568c42019-01-04 13:33:02 +00002410
Andrew Walbran318f5732018-11-20 16:23:42 +00002411 if (!is_injection_allowed(target_vm_id, current)) {
2412 return -1;
2413 }
Wedson Almeida Filho81568c42019-01-04 13:33:02 +00002414
Andrew Walbrane1310df2019-04-29 17:28:28 +01002415 target_vcpu = vm_get_vcpu(target_vm, target_vcpu_idx);
Andrew Walbran318f5732018-11-20 16:23:42 +00002416
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06002417 /* A VM could inject an interrupt for itself. */
2418 if (target_vcpu != current) {
2419 /* Lock both vCPUs at once to avoid deadlock. */
2420 vcpus_locked = vcpu_lock_both(current, target_vcpu);
2421 current_locked = vcpus_locked.vcpu1;
2422 target_locked = vcpus_locked.vcpu2;
2423 } else {
2424 current_locked = vcpu_lock(current);
2425 target_locked = current_locked;
2426 }
2427
Manish Pandey35e452f2021-02-18 21:36:34 +00002428 dlog_verbose(
2429 "Injecting interrupt %u for VM %#x vCPU %u from VM %#x vCPU "
2430 "%u\n",
2431 intid, target_vm_id, target_vcpu_idx, current->vm->id,
2432 vcpu_index(current));
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06002433 ret = api_interrupt_inject_locked(target_locked, intid, current_locked,
2434 next);
2435 if (target_vcpu != current) {
2436 vcpu_unlock(&target_locked);
2437 }
2438
2439 vcpu_unlock(&current_locked);
2440 return ret;
Andrew Walbran318f5732018-11-20 16:23:42 +00002441}
Andrew Scull6386f252018-12-06 13:29:10 +00002442
Karl Meakin6eeec8e2024-03-07 18:07:20 +00002443/**
2444 * Negotiate the FF-A version to be used for this FF-A instance.
2445 * See section 13.2 of the FF-A v1.2 ALP1 spec.
2446 *
2447 * Returns Hafnium's version number (`FFA_VERSION_COMPILED`) on success.
2448 * Returns `FFA_NOT_SUPPORTED` on error:
2449 * - The version is invalid (highest bit set).
2450 * - The requested version is incompatible.
2451 * - The version has already been negotiated and cannot be changed.
2452 */
Daniel Boulbybaeaf2e2021-12-09 11:42:36 +00002453struct ffa_value api_ffa_version(struct vcpu *current,
Karl Meakin0e617d92024-04-05 12:55:22 +01002454 enum ffa_version requested_version)
Jose Marinhofc0b2b62019-06-06 11:18:45 +01002455{
Karl Meakin0e617d92024-04-05 12:55:22 +01002456 static_assert(sizeof(enum ffa_version) == 4,
2457 "enum ffa_version must be 4 bytes wide");
2458
Karl Meakin6eeec8e2024-03-07 18:07:20 +00002459 const struct ffa_value error = {.func = (uint32_t)FFA_NOT_SUPPORTED};
Daniel Boulbybaeaf2e2021-12-09 11:42:36 +00002460 struct vm_locked current_vm_locked;
2461
Karl Meakin0e617d92024-04-05 12:55:22 +01002462 if (!ffa_version_is_valid(requested_version)) {
2463 dlog_error(
2464 "FFA_VERSION: requested version %#x is invalid "
2465 "(highest bit must be zero)\n",
2466 requested_version);
Karl Meakin6eeec8e2024-03-07 18:07:20 +00002467 return error;
Andrew Walbran9fd29072020-04-22 12:12:14 +01002468 }
Jose Marinhofc0b2b62019-06-06 11:18:45 +01002469
Karl Meakin0e617d92024-04-05 12:55:22 +01002470 if (!ffa_versions_are_compatible(requested_version,
2471 FFA_VERSION_COMPILED)) {
Karl Meakin6eeec8e2024-03-07 18:07:20 +00002472 dlog_error(
2473 "FFA_VERSION: requested version v%u.%u is not "
2474 "compatible with v%u.%u\n",
2475 ffa_version_get_major(requested_version),
2476 ffa_version_get_minor(requested_version),
2477 ffa_version_get_major(FFA_VERSION_COMPILED),
2478 ffa_version_get_minor(FFA_VERSION_COMPILED));
2479 return error;
J-Alves3cc9d6f2023-07-20 16:46:15 +01002480 }
2481
Daniel Boulbybaeaf2e2021-12-09 11:42:36 +00002482 current_vm_locked = vm_lock(current->vm);
Karl Meakin6eeec8e2024-03-07 18:07:20 +00002483
2484 if (current_vm_locked.vm->ffa_version_negotiated &&
2485 requested_version != current_vm_locked.vm->ffa_version) {
2486 vm_unlock(&current_vm_locked);
2487 dlog_error(
2488 "FFA_VERSION: Cannot change FF-A version after other "
2489 "FF-A calls have been made\n");
2490 return error;
2491 }
2492
Daniel Boulbybaeaf2e2021-12-09 11:42:36 +00002493 current_vm_locked.vm->ffa_version = requested_version;
2494 vm_unlock(&current_vm_locked);
2495
Karl Meakin6eeec8e2024-03-07 18:07:20 +00002496 return (struct ffa_value){.func = (uint32_t)FFA_VERSION_COMPILED};
Jose Marinhofc0b2b62019-06-06 11:18:45 +01002497}
Andrew Walbranc1ad4ce2019-05-09 11:41:39 +01002498
Jose Marinhoc0f4ff22019-10-09 10:37:42 +01002499/**
Karl Meakin49ec1e42024-05-10 13:08:24 +01002500 * Helper for success return of FFA_FEATURES.
J-Alves6f72ca82021-11-01 12:34:58 +00002501 */
2502struct ffa_value api_ffa_feature_success(uint32_t arg2)
2503{
2504 return (struct ffa_value){
Karl Meakin49ec1e42024-05-10 13:08:24 +01002505 .func = FFA_SUCCESS_32,
2506 .arg1 = 0U,
2507 .arg2 = arg2,
2508 };
J-Alves6f72ca82021-11-01 12:34:58 +00002509}
2510
2511/**
Jose Marinhoc0f4ff22019-10-09 10:37:42 +01002512 * Discovery function returning information about the implementation of optional
Karl Meakin49ec1e42024-05-10 13:08:24 +01002513 * FF-A interfaces. See section 13.3 of the FF-A v1.2 ALP1 spec.
2514 *
2515 * `function_or_feature_id` is interpreted as either a function ID or a feature
2516 * ID, depending on the value of bit 31.
2517 * When it is a feature ID, bits [30:8] MBZ and input_property MBZ.
2518 *
2519 * Returns `FFA_SUCCESS` if the interface is supported.
2520 * Returns `FFA_NOT_SUPPORTED` if the interface is not supported or the
2521 * parameters are invalid.
Jose Marinhoc0f4ff22019-10-09 10:37:42 +01002522 */
Karl Meakin49ec1e42024-05-10 13:08:24 +01002523struct ffa_value api_ffa_features(uint32_t function_or_feature_id,
Karl Meakinf1ed5f12024-02-22 15:57:36 +00002524 uint32_t input_property, struct vcpu *current)
Jose Marinhoc0f4ff22019-10-09 10:37:42 +01002525{
Karl Meakin0e617d92024-04-05 12:55:22 +01002526 const enum ffa_version ffa_version = current->vm->ffa_version;
Karl Meakinf1ed5f12024-02-22 15:57:36 +00002527 const bool el0_partition = current->vm->el0_partition;
Karl Meakin49ec1e42024-05-10 13:08:24 +01002528 const bool is_feature =
2529 IS_BIT_UNSET(function_or_feature_id, FFA_FEATURES_FEATURE_BIT);
Karl Meakinf1ed5f12024-02-22 15:57:36 +00002530
Karl Meakin49ec1e42024-05-10 13:08:24 +01002531 if (is_feature) {
2532 if (ANY_BITS_SET(function_or_feature_id,
2533 FFA_FEATURES_FEATURE_MBZ_HI_BIT,
2534 FFA_FEATURES_FEATURE_MBZ_LO_BIT)) {
Karl Meakin4e8e4792024-07-05 16:28:42 +01002535 dlog_verbose(
Karl Meakin49ec1e42024-05-10 13:08:24 +01002536 "FFA_FEATURES: feature ID %#x is invalid (bits "
2537 "[%u:%u] must be zero)\n",
2538 function_or_feature_id,
2539 FFA_FEATURES_FEATURE_MBZ_HI_BIT,
2540 FFA_FEATURES_FEATURE_MBZ_LO_BIT);
2541 return ffa_error(FFA_NOT_SUPPORTED);
2542 }
Karl Meakin6fd6c1d2024-05-13 12:09:13 +01002543 if (input_property != 0) {
Karl Meakin4e8e4792024-07-05 16:28:42 +01002544 dlog_verbose(
Karl Meakin6fd6c1d2024-05-13 12:09:13 +01002545 "FFA_FEATURES: input_property must be 0 "
2546 "(input_property = %#x)\n",
2547 input_property);
2548 return ffa_error(FFA_NOT_SUPPORTED);
2549 }
Karl Meakin34b8ae92023-01-13 13:33:07 +00002550 }
2551
Karl Meakin49ec1e42024-05-10 13:08:24 +01002552 switch (function_or_feature_id) {
J-Alves6f72ca82021-11-01 12:34:58 +00002553 /* Check support of the given Function ID. */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002554 case FFA_ERROR_32:
2555 case FFA_SUCCESS_32:
2556 case FFA_INTERRUPT_32:
2557 case FFA_VERSION_32:
2558 case FFA_FEATURES_32:
2559 case FFA_RX_RELEASE_32:
Daniel Boulby9e420ca2021-07-07 15:03:49 +01002560 case FFA_RXTX_UNMAP_32:
Fuad Tabbae4efcc32020-07-16 15:37:27 +01002561 case FFA_PARTITION_INFO_GET_32:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002562 case FFA_ID_GET_32:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002563 case FFA_MSG_WAIT_32:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002564 case FFA_RUN_32:
J-Alves95fbb312024-03-20 15:19:16 +00002565 case FFA_MEM_DONATE_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002566 case FFA_MEM_DONATE_32:
2567 case FFA_MEM_LEND_32:
J-Alves95fbb312024-03-20 15:19:16 +00002568 case FFA_MEM_LEND_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002569 case FFA_MEM_SHARE_32:
J-Alves95fbb312024-03-20 15:19:16 +00002570 case FFA_MEM_SHARE_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01002571 case FFA_MEM_RETRIEVE_RESP_32:
2572 case FFA_MEM_RELINQUISH_32:
2573 case FFA_MEM_RECLAIM_32:
J-Alvesff676c12022-05-13 17:25:33 +01002574 case FFA_MEM_FRAG_RX_32:
2575 case FFA_MEM_FRAG_TX_32:
J-Alvesbc3de8b2020-12-07 14:32:04 +00002576 case FFA_MSG_SEND_DIRECT_RESP_64:
Olivier Deprezee9d6a92019-11-26 09:14:11 +00002577 case FFA_MSG_SEND_DIRECT_RESP_32:
J-Alvesbc3de8b2020-12-07 14:32:04 +00002578 case FFA_MSG_SEND_DIRECT_REQ_64:
Olivier Deprezee9d6a92019-11-26 09:14:11 +00002579 case FFA_MSG_SEND_DIRECT_REQ_32:
Karl Meakin0e617d92024-04-05 12:55:22 +01002580
Daniel Boulbyb2fb80e2021-02-03 15:09:23 +00002581 /* FF-A v1.1 features. */
2582 case FFA_SPM_ID_GET_32:
J-Alves6f72ca82021-11-01 12:34:58 +00002583 case FFA_NOTIFICATION_BITMAP_CREATE_32:
2584 case FFA_NOTIFICATION_BITMAP_DESTROY_32:
2585 case FFA_NOTIFICATION_BIND_32:
2586 case FFA_NOTIFICATION_UNBIND_32:
2587 case FFA_NOTIFICATION_SET_32:
2588 case FFA_NOTIFICATION_GET_32:
2589 case FFA_NOTIFICATION_INFO_GET_64:
Federico Recanati25053ee2022-03-14 15:01:53 +01002590 case FFA_MSG_SEND2_32:
Karl Meakin0e617d92024-04-05 12:55:22 +01002591 if (FFA_VERSION_1_1 > FFA_VERSION_COMPILED) {
Karl Meakin4271ff92024-05-13 12:33:15 +01002592 return ffa_error(FFA_NOT_SUPPORTED);
Karl Meakin0e617d92024-04-05 12:55:22 +01002593 }
2594
Kathleen Capellae4fe2962023-09-01 17:08:47 -04002595 /* FF-A v1.2 features. */
2596 case FFA_CONSOLE_LOG_32:
2597 case FFA_CONSOLE_LOG_64:
Raghu Krishnamurthy7592bcb2022-12-25 13:09:00 -08002598 case FFA_PARTITION_INFO_GET_REGS_64:
Kathleen Capella41fea932023-06-23 17:39:28 -04002599 case FFA_MSG_SEND_DIRECT_REQ2_64:
Kathleen Capella087e5022023-09-07 18:04:15 -04002600 case FFA_MSG_SEND_DIRECT_RESP2_64:
Karl Meakin0e617d92024-04-05 12:55:22 +01002601 if (FFA_VERSION_1_2 > FFA_VERSION_COMPILED) {
Karl Meakin4271ff92024-05-13 12:33:15 +01002602 return ffa_error(FFA_NOT_SUPPORTED);
Karl Meakin0e617d92024-04-05 12:55:22 +01002603 }
2604
Karl Meakin49ec1e42024-05-10 13:08:24 +01002605 return api_ffa_feature_success(0);
2606
Karl Meakin5a222642024-06-20 10:23:37 +01002607 /* These functions are only supported on S-EL0 partitions. */
2608 case FFA_MEM_PERM_GET_32:
2609 case FFA_MEM_PERM_SET_32:
2610 case FFA_MEM_PERM_GET_64:
2611 case FFA_MEM_PERM_SET_64:
2612 if (!(vm_id_is_current_world(current->vm->id) &&
2613 el0_partition)) {
2614 dlog_verbose(
2615 "FFA_FEATURE: %s is only supported on S-EL0 "
2616 "partitions\n",
2617 ffa_func_name(function_or_feature_id));
2618 return ffa_error(FFA_NOT_SUPPORTED);
2619 }
2620 return api_ffa_feature_success(0);
2621
Karl Meakin1a8c0cd2024-06-20 10:26:12 +01002622 case FFA_SECONDARY_EP_REGISTER_64:
2623 if (FFA_VERSION_COMPILED < FFA_VERSION_1_1) {
2624 return ffa_error(FFA_NOT_SUPPORTED);
2625 }
2626
2627 if (!(vm_id_is_current_world(current->vm->id) &&
2628 current->vm->vcpu_count > 1)) {
2629 dlog_verbose(
2630 "FFA_FEATURE: %s is only supported on SPs with "
2631 "more than 1 vCPU\n",
2632 ffa_func_name(function_or_feature_id));
2633 return ffa_error(FFA_NOT_SUPPORTED);
2634 }
2635 return api_ffa_feature_success(0);
2636
Karl Meakin0fd67292024-02-06 17:33:05 +00002637 case FFA_RXTX_MAP_64: {
Karl Meakin0e617d92024-04-05 12:55:22 +01002638 if (FFA_VERSION_1_2 > FFA_VERSION_COMPILED) {
Karl Meakin4271ff92024-05-13 12:33:15 +01002639 return ffa_error(FFA_NOT_SUPPORTED);
Karl Meakin0e617d92024-04-05 12:55:22 +01002640 }
2641
Karl Meakin0fd67292024-02-06 17:33:05 +00002642 uint32_t arg2 = 0;
2643 struct ffa_features_rxtx_map_params params = {
2644 .min_buf_size = FFA_RXTX_MAP_MIN_BUF_4K,
2645 .mbz = 0,
2646 .max_buf_size =
Karl Meakin0e617d92024-04-05 12:55:22 +01002647 (ffa_version >= FFA_VERSION_1_2)
Karl Meakin0fd67292024-02-06 17:33:05 +00002648 ? FFA_RXTX_MAP_MAX_BUF_PAGE_COUNT
2649 : 0,
2650 };
2651
2652 memcpy_s(&arg2, sizeof(arg2), &params, sizeof(params));
Karl Meakin49ec1e42024-05-10 13:08:24 +01002653 return api_ffa_feature_success(arg2);
Karl Meakin0fd67292024-02-06 17:33:05 +00002654 }
2655
J-Alves95fbb312024-03-20 15:19:16 +00002656 case FFA_MEM_RETRIEVE_REQ_64:
Karl Meakin49ec1e42024-05-10 13:08:24 +01002657 case FFA_MEM_RETRIEVE_REQ_32: {
Karl Meakin0e617d92024-04-05 12:55:22 +01002658 if (FFA_VERSION_1_2 > FFA_VERSION_COMPILED) {
Karl Meakin4271ff92024-05-13 12:33:15 +01002659 return ffa_error(FFA_NOT_SUPPORTED);
Karl Meakin0e617d92024-04-05 12:55:22 +01002660 }
2661
Karl Meakin49ec1e42024-05-10 13:08:24 +01002662 if (ANY_BITS_SET(input_property,
2663 FFA_FEATURES_MEM_RETRIEVE_REQ_MBZ_HI_BIT,
2664 FFA_FEATURES_MEM_RETRIEVE_REQ_MBZ_LO_BIT) ||
2665 IS_BIT_SET(input_property,
2666 FFA_FEATURES_MEM_RETRIEVE_REQ_MBZ_BIT)) {
2667 dlog_warning(
2668 "FFA_FEATURES: Bits[%u:%u] and Bit[%u] of "
2669 "input_property should be 0 (input_property = "
2670 "%#x)\n",
2671 FFA_FEATURES_MEM_RETRIEVE_REQ_MBZ_HI_BIT,
2672 FFA_FEATURES_MEM_RETRIEVE_REQ_MBZ_LO_BIT,
2673 FFA_FEATURES_MEM_RETRIEVE_REQ_MBZ_BIT,
Karl Meakin34b8ae92023-01-13 13:33:07 +00002674 input_property);
Karl Meakin34b8ae92023-01-13 13:33:07 +00002675 }
2676
Karl Meakin0e617d92024-04-05 12:55:22 +01002677 if (ffa_version >= FFA_VERSION_1_1 &&
2678 (input_property &
2679 FFA_FEATURES_MEM_RETRIEVE_REQ_NS_SUPPORT) == 0U) {
Karl Meakin4e8e4792024-07-05 16:28:42 +01002680 dlog_verbose(
2681 "FFA_FEATURES: NS bit support must be 1\n");
Karl Meakin6fd6c1d2024-05-13 12:09:13 +01002682 return ffa_error(FFA_NOT_SUPPORTED);
Karl Meakin34b8ae92023-01-13 13:33:07 +00002683 }
2684
2685 return api_ffa_feature_success(
2686 FFA_FEATURES_MEM_RETRIEVE_REQ_BUFFER_SUPPORT |
Karl Meakin49ec1e42024-05-10 13:08:24 +01002687 FFA_FEATURES_MEM_RETRIEVE_REQ_NS_SUPPORT |
Karl Meakin34b8ae92023-01-13 13:33:07 +00002688 FFA_FEATURES_MEM_RETRIEVE_REQ_HYPERVISOR_SUPPORT);
Karl Meakin49ec1e42024-05-10 13:08:24 +01002689 }
J-Alves6f72ca82021-11-01 12:34:58 +00002690
J-Alves6f72ca82021-11-01 12:34:58 +00002691 /* Check support of a feature provided respective feature ID. */
Karl Meakin17c9ad32024-04-12 16:41:00 +01002692
2693 /*
2694 * For NPI and MEI, report the IDs as supported only to partitions at
2695 * the virtual FF-A instances.
2696 */
J-Alves6f72ca82021-11-01 12:34:58 +00002697 case FFA_FEATURE_NPI:
Karl Meakin0e617d92024-04-05 12:55:22 +01002698 if (FFA_VERSION_1_2 > FFA_VERSION_COMPILED) {
Karl Meakin4271ff92024-05-13 12:33:15 +01002699 return ffa_error(FFA_NOT_SUPPORTED);
Karl Meakin0e617d92024-04-05 12:55:22 +01002700 }
2701
Karl Meakinf1ed5f12024-02-22 15:57:36 +00002702 if (el0_partition) {
2703 return ffa_error(FFA_NOT_SUPPORTED);
2704 }
Karl Meakin17c9ad32024-04-12 16:41:00 +01002705 if (!vm_id_is_current_world(current->vm->id)) {
2706 return ffa_error(FFA_NOT_SUPPORTED);
2707 }
J-Alves6f72ca82021-11-01 12:34:58 +00002708 return api_ffa_feature_success(HF_NOTIFICATION_PENDING_INTID);
Karl Meakin17c9ad32024-04-12 16:41:00 +01002709
2710 case FFA_FEATURE_MEI:
Karl Meakin0e617d92024-04-05 12:55:22 +01002711 if (FFA_VERSION_1_2 > FFA_VERSION_COMPILED) {
Karl Meakin4271ff92024-05-13 12:33:15 +01002712 return ffa_error(FFA_NOT_SUPPORTED);
Karl Meakin0e617d92024-04-05 12:55:22 +01002713 }
2714
Karl Meakin17c9ad32024-04-12 16:41:00 +01002715 if (!vm_id_is_current_world(current->vm->id)) {
2716 return ffa_error(FFA_NOT_SUPPORTED);
2717 }
2718 return api_ffa_feature_success(HF_MANAGED_EXIT_INTID);
2719
J-Alves6f72ca82021-11-01 12:34:58 +00002720 case FFA_FEATURE_SRI:
Karl Meakin0e617d92024-04-05 12:55:22 +01002721 if (FFA_VERSION_1_2 > FFA_VERSION_COMPILED) {
Karl Meakin4271ff92024-05-13 12:33:15 +01002722 return ffa_error(FFA_NOT_SUPPORTED);
Karl Meakin0e617d92024-04-05 12:55:22 +01002723 }
2724
Karl Meakincb6642c2024-02-27 14:43:45 +00002725 if (!ffa_is_vm_id(current->vm->id)) {
2726 return ffa_error(FFA_NOT_SUPPORTED);
2727 }
J-Alves6f72ca82021-11-01 12:34:58 +00002728 return api_ffa_feature_success(HF_SCHEDULE_RECEIVER_INTID);
Karl Meakin0e617d92024-04-05 12:55:22 +01002729
J-Alves6f72ca82021-11-01 12:34:58 +00002730 /* Platform specific feature support. */
Jose Marinhoc0f4ff22019-10-09 10:37:42 +01002731 default:
Karl Meakin4271ff92024-05-13 12:33:15 +01002732 return ffa_error(FFA_NOT_SUPPORTED);
Jose Marinhoc0f4ff22019-10-09 10:37:42 +01002733 }
2734}
Andrew Walbrane908c4a2019-12-02 17:13:47 +00002735
Olivier Deprezee9d6a92019-11-26 09:14:11 +00002736/**
Kathleen Capella41fea932023-06-23 17:39:28 -04002737 * FF-A specification states that x2/w2 Must Be Zero for FFA_MSG_SEND_DIRECT_REQ
2738 * and FFA_MSG_SEND_DIRECT_RESP interfaces when used for partition messages. See
2739 * FF-A v1.2 Table 16.6: FFA_MSG_SEND_DIRECT_REQ function syntax.
J-Alves645eabe2021-02-22 16:08:27 +00002740 */
2741static inline bool api_ffa_dir_msg_is_arg2_zero(struct ffa_value args)
2742{
2743 return args.arg2 == 0U;
2744}
2745
2746/**
J-Alves76d99af2021-03-10 17:42:11 +00002747 * Limits size of arguments in ffa_value structure to 32-bit.
2748 */
2749static struct ffa_value api_ffa_value_copy32(struct ffa_value args)
2750{
2751 return (struct ffa_value){
2752 .func = (uint32_t)args.func,
2753 .arg1 = (uint32_t)args.arg1,
2754 .arg2 = (uint32_t)0,
2755 .arg3 = (uint32_t)args.arg3,
2756 .arg4 = (uint32_t)args.arg4,
2757 .arg5 = (uint32_t)args.arg5,
2758 .arg6 = (uint32_t)args.arg6,
2759 .arg7 = (uint32_t)args.arg7,
2760 };
2761}
2762
2763/**
Kathleen Capella41fea932023-06-23 17:39:28 -04002764 * Helper to copy direct message payload, depending on SMC used, direct
2765 * messaging interface used, and expected registers size.
J-Alves76d99af2021-03-10 17:42:11 +00002766 */
2767static struct ffa_value api_ffa_dir_msg_value(struct ffa_value args)
2768{
2769 if (args.func == FFA_MSG_SEND_DIRECT_REQ_32 ||
2770 args.func == FFA_MSG_SEND_DIRECT_RESP_32) {
2771 return api_ffa_value_copy32(args);
2772 }
2773
Kathleen Capella41fea932023-06-23 17:39:28 -04002774 if (args.func == FFA_MSG_SEND_DIRECT_REQ_64 ||
2775 args.func == FFA_MSG_SEND_DIRECT_RESP_64) {
2776 args.arg2 = 0;
2777 }
2778
Kathleen Capelladb6a08b2023-10-04 13:42:39 -04002779 if (args.func == FFA_MSG_SEND_DIRECT_REQ2_64) {
2780 args.extended_val.valid = true;
2781 }
2782
Kathleen Capella087e5022023-09-07 18:04:15 -04002783 if (args.func == FFA_MSG_SEND_DIRECT_RESP2_64) {
2784 args.arg2 = 0;
2785 args.arg3 = 0;
2786 }
2787
Kathleen Capella41fea932023-06-23 17:39:28 -04002788 return args;
2789}
2790
2791static bool api_ffa_dir_msg_req2_is_uuid_valid(struct vm *receiver_vm,
2792 struct ffa_value args)
2793{
2794 struct ffa_uuid target_uuid;
Kathleen Capella422b10b2023-06-30 18:28:27 -04002795 uint16_t i;
Kathleen Capella41fea932023-06-23 17:39:28 -04002796
2797 ffa_uuid_unpack_from_uint64(args.arg2, args.arg3, &target_uuid);
2798
Kathleen Capella422b10b2023-06-30 18:28:27 -04002799 /* Allow for use of Nil UUID. */
2800 if (ffa_uuid_is_null(&target_uuid)) {
2801 return true;
2802 }
2803
2804 for (i = 0; i < PARTITION_MAX_UUIDS; i++) {
2805 if (ffa_uuid_is_null(&receiver_vm->uuids[i])) {
2806 break;
2807 }
2808 if (ffa_uuid_equal(&target_uuid, &receiver_vm->uuids[i])) {
2809 return true;
2810 }
2811 }
2812 return false;
J-Alves76d99af2021-03-10 17:42:11 +00002813}
2814
2815/**
Olivier Deprezee9d6a92019-11-26 09:14:11 +00002816 * Send an FF-A direct message request.
Kathleen Capella41fea932023-06-23 17:39:28 -04002817 * This handler covers both FFA_MSG_SEND_DIRECT_REQ_32/64
2818 * and FFA_MSG_SEND_DIRECT_REQ2_64 (introduced in FF-A v1.2) with function-based
2819 * checks to accomodate for the difference between the ABIs.
2820 *
2821 * FFA_MSG_SEND_DIRECT_REQ2_64 works mostly the same as
2822 * FFA_MSG_SEND_DIRECT_REQ_32/64, but adds the ability to send a direct message
2823 * request to a specified UUID within a partition and the usage of an extended
2824 * range of registers (x4-x17, instead of x4-x7) to be used as part of the
2825 * message payload.
Olivier Deprezee9d6a92019-11-26 09:14:11 +00002826 */
J-Alves19e20cf2023-08-02 12:48:55 +01002827struct ffa_value api_ffa_msg_send_direct_req(ffa_id_t sender_vm_id,
2828 ffa_id_t receiver_vm_id,
Olivier Deprezee9d6a92019-11-26 09:14:11 +00002829 struct ffa_value args,
2830 struct vcpu *current,
2831 struct vcpu **next)
2832{
J-Alves17228f72021-04-20 17:13:19 +01002833 struct ffa_value ret;
Olivier Deprezee9d6a92019-11-26 09:14:11 +00002834 struct vm *receiver_vm;
J-Alves6e2abc62021-12-02 14:58:56 +00002835 struct vm_locked receiver_locked;
Olivier Deprezee9d6a92019-11-26 09:14:11 +00002836 struct vcpu *receiver_vcpu;
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06002837 struct vcpu_locked current_locked;
2838 struct vcpu_locked receiver_vcpu_locked;
Olivier Deprezee9d6a92019-11-26 09:14:11 +00002839 struct two_vcpu_locked vcpus_locked;
Madhukar Pappireddyd456ac22023-06-26 15:29:34 -05002840 enum vcpu_state next_state = VCPU_STATE_RUNNING;
Olivier Deprezee9d6a92019-11-26 09:14:11 +00002841
Kathleen Capella41fea932023-06-23 17:39:28 -04002842 if ((args.func == FFA_MSG_SEND_DIRECT_REQ_32 ||
2843 args.func == FFA_MSG_SEND_DIRECT_REQ_64) &&
2844 !api_ffa_dir_msg_is_arg2_zero(args)) {
J-Alves645eabe2021-02-22 16:08:27 +00002845 return ffa_error(FFA_INVALID_PARAMETERS);
2846 }
2847
Olivier Deprez55a189e2021-06-09 15:45:27 +02002848 if (!plat_ffa_is_direct_request_valid(current, sender_vm_id,
2849 receiver_vm_id)) {
J-Alvese0caac32022-12-19 14:37:08 +00002850 dlog_verbose("Invalid direct message request.\n");
J-Alvesaa336102021-03-01 13:02:45 +00002851 return ffa_error(FFA_INVALID_PARAMETERS);
Olivier Deprezee9d6a92019-11-26 09:14:11 +00002852 }
2853
Olivier Deprez55a189e2021-06-09 15:45:27 +02002854 if (plat_ffa_direct_request_forward(receiver_vm_id, args, &ret)) {
J-Alves17228f72021-04-20 17:13:19 +01002855 return ret;
2856 }
2857
2858 ret = (struct ffa_value){.func = FFA_INTERRUPT_32};
2859
Olivier Deprezee9d6a92019-11-26 09:14:11 +00002860 receiver_vm = vm_find(receiver_vm_id);
2861 if (receiver_vm == NULL) {
J-Alves88a13542021-12-14 15:39:52 +00002862 dlog_verbose("Invalid Receiver!\n");
Olivier Deprezee9d6a92019-11-26 09:14:11 +00002863 return ffa_error(FFA_INVALID_PARAMETERS);
2864 }
2865
Kathleen Capella41fea932023-06-23 17:39:28 -04002866 if (args.func == FFA_MSG_SEND_DIRECT_REQ2_64 &&
2867 !api_ffa_dir_msg_req2_is_uuid_valid(receiver_vm, args)) {
2868 dlog_verbose("UUID unrecognized for this VM\n");
2869 return ffa_error(FFA_INVALID_PARAMETERS);
2870 }
2871
Olivier Deprezee9d6a92019-11-26 09:14:11 +00002872 /*
J-Alves439ac972021-11-18 17:32:03 +00002873 * Check if sender supports sending direct message req, and if
2874 * receiver supports receipt of direct message requests.
2875 */
Kathleen Capella41fea932023-06-23 17:39:28 -04002876 if (!plat_ffa_is_direct_request_supported(current->vm, receiver_vm,
2877 args.func)) {
J-Alves439ac972021-11-18 17:32:03 +00002878 return ffa_error(FFA_DENIED);
2879 }
2880
2881 /*
Olivier Deprezc13a8692022-04-08 17:47:14 +02002882 * Per FF-A EAC spec section 4.4.1 the firmware framework supports
Olivier Deprezee9d6a92019-11-26 09:14:11 +00002883 * UP (migratable) or MP partitions with a number of vCPUs matching the
2884 * number of PEs in the system. It further states that MP partitions
2885 * accepting direct request messages cannot migrate.
2886 */
J-Alvesad6a0432021-04-09 16:06:21 +01002887 receiver_vcpu = api_ffa_get_vm_vcpu(receiver_vm, current);
Olivier Deprezee9d6a92019-11-26 09:14:11 +00002888 if (receiver_vcpu == NULL) {
J-Alves88a13542021-12-14 15:39:52 +00002889 dlog_verbose("Invalid vCPU!\n");
Olivier Deprezee9d6a92019-11-26 09:14:11 +00002890 return ffa_error(FFA_INVALID_PARAMETERS);
2891 }
2892
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06002893 /*
2894 * If VM must be locked, it must be done before any of its vCPUs are
2895 * locked.
2896 */
J-Alves6e2abc62021-12-02 14:58:56 +00002897 receiver_locked = vm_lock(receiver_vm);
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06002898
2899 /* Lock both vCPUs at once to avoid deadlock. */
2900 vcpus_locked = vcpu_lock_both(current, receiver_vcpu);
2901 current_locked = vcpus_locked.vcpu1;
2902 receiver_vcpu_locked = vcpus_locked.vcpu2;
2903
Olivier Deprezee9d6a92019-11-26 09:14:11 +00002904 /*
2905 * If destination vCPU is executing or already received an
2906 * FFA_MSG_SEND_DIRECT_REQ then return to caller hinting recipient is
2907 * busy. There is a brief period of time where the vCPU state has
2908 * changed but regs_available is still false thus consider this case as
2909 * the vCPU not yet ready to receive a direct message request.
2910 */
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06002911 if (is_ffa_direct_msg_request_ongoing(receiver_vcpu_locked) ||
Olivier Deprezee9d6a92019-11-26 09:14:11 +00002912 receiver_vcpu->state == VCPU_STATE_RUNNING ||
2913 !receiver_vcpu->regs_available) {
J-Alves88a13542021-12-14 15:39:52 +00002914 dlog_verbose("Receiver is busy with another request.\n");
Olivier Deprezee9d6a92019-11-26 09:14:11 +00002915 ret = ffa_error(FFA_BUSY);
2916 goto out;
2917 }
2918
J-Alves7ea3f7e2024-03-27 11:05:11 +00002919 if (!plat_ffa_check_runtime_state_transition(
2920 current_locked, sender_vm_id, HF_INVALID_VM_ID,
2921 receiver_vcpu_locked, args.func, &next_state)) {
2922 ret = ffa_error(FFA_DENIED);
2923 goto out;
2924 }
2925
Olivier Deprezee9d6a92019-11-26 09:14:11 +00002926 if (atomic_load_explicit(&receiver_vcpu->vm->aborting,
2927 memory_order_relaxed)) {
2928 if (receiver_vcpu->state != VCPU_STATE_ABORTED) {
Kathleen Capella6e9765b2023-07-11 17:44:58 -04002929 dlog_verbose(
2930 "Receiver VM %#x aborted, cannot run vCPU %u\n",
2931 receiver_vcpu->vm->id,
2932 vcpu_index(receiver_vcpu));
Olivier Deprezee9d6a92019-11-26 09:14:11 +00002933 receiver_vcpu->state = VCPU_STATE_ABORTED;
2934 }
2935
2936 ret = ffa_error(FFA_ABORTED);
2937 goto out;
2938 }
2939
2940 switch (receiver_vcpu->state) {
2941 case VCPU_STATE_OFF:
2942 case VCPU_STATE_RUNNING:
2943 case VCPU_STATE_ABORTED:
Olivier Deprezee9d6a92019-11-26 09:14:11 +00002944 case VCPU_STATE_BLOCKED_INTERRUPT:
Madhukar Pappireddyb11e0d12021-08-02 19:44:35 -05002945 case VCPU_STATE_BLOCKED:
2946 case VCPU_STATE_PREEMPTED:
J-Alves88a13542021-12-14 15:39:52 +00002947 dlog_verbose("Receiver's vCPU can't receive request (%u)!\n",
2948 vcpu_index(receiver_vcpu));
Olivier Deprezee9d6a92019-11-26 09:14:11 +00002949 ret = ffa_error(FFA_BUSY);
2950 goto out;
Madhukar Pappireddyb11e0d12021-08-02 19:44:35 -05002951 case VCPU_STATE_WAITING:
Olivier Deprezee9d6a92019-11-26 09:14:11 +00002952 /*
Madhukar Pappireddyb11e0d12021-08-02 19:44:35 -05002953 * We expect target vCPU to be in WAITING state after either
2954 * having called ffa_msg_wait or sent a direct message response.
Olivier Deprezee9d6a92019-11-26 09:14:11 +00002955 */
2956 break;
2957 }
2958
2959 /* Inject timer interrupt if any pending */
2960 if (arch_timer_pending(&receiver_vcpu->regs)) {
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06002961 api_interrupt_inject_locked(receiver_vcpu_locked,
2962 HF_VIRTUAL_TIMER_INTID,
2963 current_locked, NULL);
Olivier Deprezee9d6a92019-11-26 09:14:11 +00002964
2965 arch_timer_mask(&receiver_vcpu->regs);
2966 }
2967
2968 /* The receiver vCPU runs upon direct message invocation */
2969 receiver_vcpu->cpu = current->cpu;
2970 receiver_vcpu->state = VCPU_STATE_RUNNING;
2971 receiver_vcpu->regs_available = false;
Kathleen Capellae468c112023-12-13 17:56:28 -05002972 receiver_vcpu->direct_request_origin.is_ffa_req2 =
2973 (args.func == FFA_MSG_SEND_DIRECT_REQ2_64);
2974 receiver_vcpu->direct_request_origin.vm_id = sender_vm_id;
Olivier Deprezee9d6a92019-11-26 09:14:11 +00002975
J-Alves76d99af2021-03-10 17:42:11 +00002976 arch_regs_set_retval(&receiver_vcpu->regs, api_ffa_dir_msg_value(args));
Olivier Deprezee9d6a92019-11-26 09:14:11 +00002977
Madhukar Pappireddyd456ac22023-06-26 15:29:34 -05002978 assert(!vm_id_is_current_world(current->vm->id) ||
2979 next_state == VCPU_STATE_BLOCKED);
Madhukar Pappireddyb11e0d12021-08-02 19:44:35 -05002980 current->state = VCPU_STATE_BLOCKED;
Olivier Deprezee9d6a92019-11-26 09:14:11 +00002981
Raghu Krishnamurthya9ccf122023-03-27 20:42:01 -07002982 plat_ffa_wind_call_chain_ffa_direct_req(
2983 current_locked, receiver_vcpu_locked, sender_vm_id);
Madhukar Pappireddy49fe6702022-06-21 17:52:23 -05002984
Olivier Deprezee9d6a92019-11-26 09:14:11 +00002985 /* Switch to receiver vCPU targeted to by direct msg request */
2986 *next = receiver_vcpu;
2987
J-Alves6e2abc62021-12-02 14:58:56 +00002988 if (!receiver_locked.vm->el0_partition) {
2989 /*
2990 * If the scheduler in the system is giving CPU cycles to the
2991 * receiver, due to pending notifications, inject the NPI
2992 * interrupt. Following call assumes that '*next' has been set
2993 * to receiver_vcpu.
2994 */
2995 plat_ffa_inject_notification_pending_interrupt(
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06002996 receiver_vcpu_locked, current_locked, receiver_locked);
J-Alves6e2abc62021-12-02 14:58:56 +00002997 }
2998
Olivier Deprezee9d6a92019-11-26 09:14:11 +00002999 /*
3000 * Since this flow will lead to a VM switch, the return value will not
3001 * be applied to current vCPU.
3002 */
3003
3004out:
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06003005 vcpu_unlock(&receiver_vcpu_locked);
J-Alves6e2abc62021-12-02 14:58:56 +00003006 vm_unlock(&receiver_locked);
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06003007 vcpu_unlock(&current_locked);
Olivier Deprezee9d6a92019-11-26 09:14:11 +00003008
3009 return ret;
3010}
3011
3012/**
Madhukar Pappireddy2f76e492022-09-06 15:21:59 -05003013 * Resume the target vCPU after the current vCPU sent a direct response.
3014 * Current vCPU moves to waiting state.
3015 */
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06003016void api_ffa_resume_direct_resp_target(struct vcpu_locked current_locked,
3017 struct vcpu **next,
J-Alves19e20cf2023-08-02 12:48:55 +01003018 ffa_id_t receiver_vm_id,
Madhukar Pappireddy2f76e492022-09-06 15:21:59 -05003019 struct ffa_value to_ret,
3020 bool is_nwd_call_chain)
3021{
Raghu Krishnamurthya9ccf122023-03-27 20:42:01 -07003022 if (plat_ffa_is_spmd_lp_id(receiver_vm_id) ||
3023 !vm_id_is_current_world(receiver_vm_id)) {
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06003024 *next = api_switch_to_other_world(current_locked, to_ret,
Madhukar Pappireddy2f76e492022-09-06 15:21:59 -05003025 VCPU_STATE_WAITING);
3026
3027 /* End of NWd scheduled call chain. */
3028 assert(!is_nwd_call_chain ||
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06003029 (current_locked.vcpu->call_chain.prev_node == NULL));
Madhukar Pappireddy2f76e492022-09-06 15:21:59 -05003030 } else if (receiver_vm_id == HF_PRIMARY_VM_ID) {
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06003031 *next = api_switch_to_primary(current_locked, to_ret,
Madhukar Pappireddy2f76e492022-09-06 15:21:59 -05003032 VCPU_STATE_WAITING);
Madhukar Pappireddy2f76e492022-09-06 15:21:59 -05003033 } else if (vm_id_is_current_world(receiver_vm_id)) {
3034 /*
3035 * It is expected the receiver_vm_id to be from an SP, otherwise
3036 * 'plat_ffa_is_direct_response_valid' should have
3037 * made function return error before getting to this point.
3038 */
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06003039 *next = api_switch_to_vm(current_locked, to_ret,
3040 VCPU_STATE_WAITING, receiver_vm_id);
Madhukar Pappireddy2f76e492022-09-06 15:21:59 -05003041 } else {
3042 panic("Invalid direct message response invocation");
3043 }
3044}
3045
3046/**
Olivier Deprezee9d6a92019-11-26 09:14:11 +00003047 * Send an FF-A direct message response.
Kathleen Capella087e5022023-09-07 18:04:15 -04003048 * This handler covers both FFA_MSG_SEND_DIRECT_RESP_32/64
3049 * and FFA_MSG_SEND_DIRECT_RESP2_64 (introduced in FF-A v1.2) with
3050 * function-based checks to accomodate for the difference between the ABIs.
3051 *
3052 * FFA_MSG_SEND_DIRECT_RESP2_64 is used to respond to requests sent via
3053 * FFA_MSG_SEND_DIRECT_REQ2_64 and adds the usage of an extended range
3054 * of registers (x4-x17, instead of x4-x7) to be used as part of the
3055 * message payload.
Olivier Deprezee9d6a92019-11-26 09:14:11 +00003056 */
J-Alves19e20cf2023-08-02 12:48:55 +01003057struct ffa_value api_ffa_msg_send_direct_resp(ffa_id_t sender_vm_id,
3058 ffa_id_t receiver_vm_id,
Olivier Deprezee9d6a92019-11-26 09:14:11 +00003059 struct ffa_value args,
3060 struct vcpu *current,
3061 struct vcpu **next)
3062{
Olivier Deprez2ebae3a2020-06-11 16:34:30 +02003063 struct vcpu_locked current_locked;
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06003064 struct vcpu_locked next_locked = (struct vcpu_locked){
3065 .vcpu = NULL,
3066 };
Madhukar Pappireddyd456ac22023-06-26 15:29:34 -05003067 enum vcpu_state next_state = VCPU_STATE_RUNNING;
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06003068 struct ffa_value ret = (struct ffa_value){.func = FFA_INTERRUPT_32};
Madhukar Pappireddy2f76e492022-09-06 15:21:59 -05003069 struct ffa_value signal_interrupt =
3070 (struct ffa_value){.func = FFA_INTERRUPT_32};
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06003071 struct ffa_value to_ret = api_ffa_dir_msg_value(args);
3072 struct two_vcpu_locked vcpus_locked;
Kathleen Capellae468c112023-12-13 17:56:28 -05003073 bool received_req2;
J-Alves645eabe2021-02-22 16:08:27 +00003074
Kathleen Capella35238872024-01-12 15:05:52 -05003075 /*
3076 * If using FFA_MSG_SEND_DIRECT_RESP, the caller's
3077 * - x2 MBZ for partition messages
3078 * - x8-x17 SBZ if caller's FF-A version >= FF-A v1.2
3079 */
3080 if (args.func != FFA_MSG_SEND_DIRECT_RESP2_64) {
3081 if (!api_ffa_dir_msg_is_arg2_zero(args)) {
3082 return ffa_error(FFA_INVALID_PARAMETERS);
3083 }
3084
Karl Meakin0e617d92024-04-05 12:55:22 +01003085 if (current->vm->ffa_version >= FFA_VERSION_1_2 &&
Kathleen Capella35238872024-01-12 15:05:52 -05003086 !api_extended_args_are_zero(&args)) {
3087 return ffa_error(FFA_INVALID_PARAMETERS);
3088 }
J-Alves645eabe2021-02-22 16:08:27 +00003089 }
3090
Olivier Deprez55a189e2021-06-09 15:45:27 +02003091 if (!plat_ffa_is_direct_response_valid(current, sender_vm_id,
3092 receiver_vm_id)) {
J-Alvese0caac32022-12-19 14:37:08 +00003093 dlog_verbose("Invalid direct response call.\n");
J-Alvesaa336102021-03-01 13:02:45 +00003094 return ffa_error(FFA_INVALID_PARAMETERS);
Olivier Deprezee9d6a92019-11-26 09:14:11 +00003095 }
3096
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06003097 current_locked = vcpu_lock(current);
3098
3099 if (!plat_ffa_check_runtime_state_transition(
3100 current_locked, sender_vm_id, receiver_vm_id, next_locked,
3101 args.func, &next_state)) {
3102 ret = ffa_error(FFA_DENIED);
3103 goto out;
Madhukar Pappireddyc857ac22022-06-21 17:37:53 -05003104 }
3105
Madhukar Pappireddyd456ac22023-06-26 15:29:34 -05003106 assert(!vm_id_is_current_world(current->vm->id) ||
3107 next_state == VCPU_STATE_WAITING);
Madhukar Pappireddy469fa712022-06-21 18:53:33 -05003108
3109 /*
3110 * Ensure the terminating FFA_MSG_SEND_DIRECT_REQ had a
3111 * defined originator.
3112 */
3113 if (!is_ffa_direct_msg_request_ongoing(current_locked)) {
3114 /*
3115 * Sending direct response but direct request origin
3116 * vCPU is not set.
3117 */
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06003118 ret = ffa_error(FFA_DENIED);
3119 goto out;
Madhukar Pappireddy469fa712022-06-21 18:53:33 -05003120 }
3121
Kathleen Capellae468c112023-12-13 17:56:28 -05003122 received_req2 = current->direct_request_origin.is_ffa_req2;
3123
3124 if (args.func != FFA_MSG_SEND_DIRECT_RESP2_64 && received_req2) {
3125 dlog_verbose(
3126 "%s: FFA_MSG_SEND_DIRECT_RESP must be used with "
3127 "FFA_MSG_SEND_DIRECT_REQ.\n",
3128 __func__);
3129 ret = ffa_error(FFA_DENIED);
3130 goto out;
3131 } else if (args.func == FFA_MSG_SEND_DIRECT_RESP2_64 &&
3132 !received_req2) {
3133 dlog_verbose(
3134 "%s: FFA_MSG_SEND_DIRECT_RESP2 must be used with "
3135 "FFA_MSG_SEND_DIRECT_REQ2.\n",
3136 __func__);
3137 ret = ffa_error(FFA_DENIED);
3138 goto out;
3139 }
3140
Manish Pandeya5f39fb2020-09-11 09:47:11 +01003141 if (api_ffa_is_managed_exit_ongoing(current_locked)) {
Olivier Deprezee9d6a92019-11-26 09:14:11 +00003142 /*
Madhukar Pappireddy469fa712022-06-21 18:53:33 -05003143 * Per FF-A v1.1 EAC0 section 8.3.1.2.1 rule 6, SPMC can signal
Madhukar Pappireddyed4ab942021-08-03 14:22:53 -05003144 * a secure interrupt to a SP that is performing managed exit.
3145 * We have taken a implementation defined choice to not allow
3146 * Managed exit while a SP is processing a secure interrupt.
3147 */
3148 CHECK(!current->processing_secure_interrupt);
3149
Madhukar Pappireddydd6fdfb2021-12-14 12:30:36 -06003150 plat_interrupts_set_priority_mask(current->priority_mask);
Madhukar Pappireddya1019112022-06-21 18:57:44 -05003151 /*
3152 * A SP may be signaled a managed exit but actually not trap
3153 * the virtual interrupt, probably because it has virtual
3154 * interrupts masked, and emit direct resp. In this case the
3155 * managed exit operation is considered completed and it would
3156 * also need to clear the pending managed exit flag for the SP
3157 * vCPU.
3158 */
Manish Pandeya5f39fb2020-09-11 09:47:11 +01003159 current->processing_managed_exit = false;
Madhukar Pappireddya1019112022-06-21 18:57:44 -05003160 struct interrupts *interrupts = &current->interrupts;
3161
3162 if (vcpu_is_virt_interrupt_pending(interrupts,
3163 HF_MANAGED_EXIT_INTID)) {
3164 api_interrupt_clear_decrement(current_locked,
3165 interrupts,
3166 HF_MANAGED_EXIT_INTID);
3167 }
Olivier Deprezee9d6a92019-11-26 09:14:11 +00003168 }
3169
Madhukar Pappireddy2f76e492022-09-06 15:21:59 -05003170 if (plat_ffa_intercept_direct_response(current_locked, next, to_ret,
3171 &signal_interrupt)) {
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06003172 ret = signal_interrupt;
3173 goto out;
Madhukar Pappireddy2f76e492022-09-06 15:21:59 -05003174 }
3175
Kathleen Capellae468c112023-12-13 17:56:28 -05003176 /* Clear direct request origin vm_id and request type for the caller. */
3177 current->direct_request_origin.is_ffa_req2 = false;
3178 current->direct_request_origin.vm_id = HF_INVALID_VM_ID;
Olivier Deprezee9d6a92019-11-26 09:14:11 +00003179
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06003180 api_ffa_resume_direct_resp_target(current_locked, next, receiver_vm_id,
3181 to_ret, false);
3182
3183 /*
3184 * Unlock current vCPU to allow it to be locked together with next
3185 * vcpu.
3186 */
Olivier Deprez2ebae3a2020-06-11 16:34:30 +02003187 vcpu_unlock(&current_locked);
Olivier Deprezee9d6a92019-11-26 09:14:11 +00003188
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06003189 /* Lock both vCPUs at once to avoid deadlock. */
3190 vcpus_locked = vcpu_lock_both(current, *next);
3191 current_locked = vcpus_locked.vcpu1;
3192 next_locked = vcpus_locked.vcpu2;
Olivier Deprezee9d6a92019-11-26 09:14:11 +00003193
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06003194 plat_ffa_unwind_call_chain_ffa_direct_resp(current_locked, next_locked);
3195 vcpu_unlock(&next_locked);
Madhukar Pappireddyc0fb87e2022-06-21 17:59:15 -05003196
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06003197out:
3198 vcpu_unlock(&current_locked);
3199 return ret;
Olivier Deprezee9d6a92019-11-26 09:14:11 +00003200}
3201
J-Alves84658fc2021-06-17 14:37:32 +01003202static bool api_memory_region_check_flags(
3203 struct ffa_memory_region *memory_region, uint32_t share_func)
3204{
3205 switch (share_func) {
J-Alves95fbb312024-03-20 15:19:16 +00003206 case FFA_MEM_SHARE_64:
J-Alves84658fc2021-06-17 14:37:32 +01003207 case FFA_MEM_SHARE_32:
3208 if ((memory_region->flags & FFA_MEMORY_REGION_FLAG_CLEAR) !=
3209 0U) {
3210 return false;
3211 }
3212 /* Intentional fall-through */
J-Alves95fbb312024-03-20 15:19:16 +00003213 case FFA_MEM_LEND_64:
J-Alves84658fc2021-06-17 14:37:32 +01003214 case FFA_MEM_LEND_32:
J-Alves95fbb312024-03-20 15:19:16 +00003215 case FFA_MEM_DONATE_64:
J-Alves84658fc2021-06-17 14:37:32 +01003216 case FFA_MEM_DONATE_32: {
3217 /* Bits 31:2 Must Be Zero. */
3218 ffa_memory_receiver_flags_t to_mask =
3219 ~(FFA_MEMORY_REGION_FLAG_CLEAR |
3220 FFA_MEMORY_REGION_FLAG_TIME_SLICE);
3221
3222 if ((memory_region->flags & to_mask) != 0U) {
3223 return false;
3224 }
3225 break;
3226 }
3227 default:
3228 panic("Check for mem send calls only.\n");
3229 }
3230
3231 /* Last check reserved values are 0 */
3232 return true;
3233}
3234
J-Alves33c47bf2022-09-29 11:36:20 +01003235/*
3236 * Convert memory transaction descriptor from FF-A v1.0 to FF-A v1.1 EAC0.
3237 */
3238static void api_ffa_memory_region_v1_1_from_v1_0(
3239 struct ffa_memory_region_v1_0 *memory_region_v1_0,
3240 struct ffa_memory_region *memory_region_v1_1)
3241{
3242 memory_region_v1_1->sender = memory_region_v1_0->sender;
J-Alves00847062022-10-03 17:08:44 +01003243 memory_region_v1_1->handle = memory_region_v1_0->handle;
Karl Meakin84710f32023-10-12 15:14:49 +01003244 memory_region_v1_1->attributes =
3245 ffa_memory_attributes_extend(memory_region_v1_0->attributes);
J-Alves33c47bf2022-09-29 11:36:20 +01003246 memory_region_v1_1->flags = memory_region_v1_0->flags;
3247 memory_region_v1_1->tag = memory_region_v1_0->tag;
3248 memory_region_v1_1->memory_access_desc_size =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003249 sizeof(struct ffa_memory_access_v1_0);
J-Alves33c47bf2022-09-29 11:36:20 +01003250 memory_region_v1_1->receiver_count = memory_region_v1_0->receiver_count;
Daniel Boulby41ef8ba2023-10-13 17:01:22 +01003251 memory_region_v1_1->receivers_offset = sizeof(struct ffa_memory_region);
J-Alves33c47bf2022-09-29 11:36:20 +01003252
3253 /* Zero reserved fields. */
3254 for (uint32_t i = 0; i < 3U; i++) {
3255 memory_region_v1_1->reserved[i] = 0U;
3256 }
3257}
3258
Daniel Boulby36e0bdb2023-10-09 12:00:41 +01003259/**
3260 * Updates a v1.0 transaction descriptor to v1.1. This gives us the
3261 * memory_access_desc_size field we need for forwards compatability.
3262 * Copy the receivers and composite descriptors to the new struct.
3263 * We also check the fields in the v1.0 transaction descriptor and return:
3264 * - FFA_ERROR FFA_INVALID_PARAMETERS: If any of the fields are not valid
3265 * values, eg the reserved fields are not 0, receiver_count is too large or
3266 * composite offsets are not 0 for retrieve requests or in bounds for send
3267 * requests.
3268 * - FFA ERROR FFA_NOT_SUPPORTED: If an invalid ffa_version is supplied to the
3269 * function. Or the fragment length is more than a single page.
3270 * - FFA_ERROR FFA_NO_MEMORY: If we do not have enough memory for a scratch
3271 * memory transaction descriptor.
3272 * - FFA_SUCCESS: If a successful update has occured.
J-Alves33c47bf2022-09-29 11:36:20 +01003273 */
Daniel Boulby36e0bdb2023-10-09 12:00:41 +01003274static struct ffa_value api_ffa_memory_transaction_descriptor_v1_1_from_v1_0(
3275 void *allocated, uint32_t *fragment_length, uint32_t *total_length,
Karl Meakin0e617d92024-04-05 12:55:22 +01003276 enum ffa_version ffa_version, bool send_transaction)
J-Alves33c47bf2022-09-29 11:36:20 +01003277{
3278 struct ffa_memory_region_v1_0 *memory_region_v1_0;
3279 struct ffa_memory_region *memory_region_v1_1 = NULL;
3280 struct ffa_composite_memory_region *composite_v1_0;
3281 struct ffa_composite_memory_region *composite_v1_1;
3282 size_t receivers_length;
3283 size_t space_left;
Daniel Boulby36e0bdb2023-10-09 12:00:41 +01003284 size_t receivers_end;
J-Alves33c47bf2022-09-29 11:36:20 +01003285 size_t composite_offset_v1_1;
3286 size_t composite_offset_v1_0;
3287 size_t fragment_constituents_size;
3288 size_t fragment_length_v1_1;
3289
J-Alves33c47bf2022-09-29 11:36:20 +01003290 assert(fragment_length != NULL);
3291 assert(total_length != NULL);
3292
Karl Meakin0e617d92024-04-05 12:55:22 +01003293 if (ffa_version >= FFA_VERSION_1_1) {
J-Alves33c47bf2022-09-29 11:36:20 +01003294 return (struct ffa_value){.func = FFA_SUCCESS_32};
3295 }
3296
Karl Meakin0e617d92024-04-05 12:55:22 +01003297 if (ffa_version != FFA_VERSION_1_0) {
J-Alves33c47bf2022-09-29 11:36:20 +01003298 dlog_verbose("%s: Unsupported FF-A version %x\n", __func__,
3299 ffa_version);
3300 return ffa_error(FFA_NOT_SUPPORTED);
3301 }
3302
Daniel Boulby36e0bdb2023-10-09 12:00:41 +01003303 dlog_verbose(
J-Alves0a824e92024-04-26 16:20:12 +01003304 "Updating memory transaction descriptor from v1.0 to v1.1.\n");
J-Alves33c47bf2022-09-29 11:36:20 +01003305
3306 memory_region_v1_0 = (struct ffa_memory_region_v1_0 *)allocated;
3307
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003308 receivers_length = sizeof(struct ffa_memory_access_v1_0) *
J-Alves33c47bf2022-09-29 11:36:20 +01003309 memory_region_v1_0->receiver_count;
Daniel Boulby36e0bdb2023-10-09 12:00:41 +01003310 receivers_end = sizeof(struct ffa_memory_region) + receivers_length;
J-Alves33c47bf2022-09-29 11:36:20 +01003311
3312 /*
3313 * Check the specified composite offset of v1.0 descriptor, and that all
3314 * receivers were configured with the same offset.
3315 */
3316 composite_offset_v1_0 =
3317 memory_region_v1_0->receivers[0].composite_memory_region_offset;
3318
J-Alves33c47bf2022-09-29 11:36:20 +01003319 /* Determine the composite offset for v1.1 descriptor. */
Daniel Boulby36e0bdb2023-10-09 12:00:41 +01003320 if (send_transaction) {
3321 fragment_constituents_size =
3322 *fragment_length - composite_offset_v1_0 -
3323 sizeof(struct ffa_composite_memory_region);
3324 fragment_length_v1_1 =
3325 receivers_end +
3326 sizeof(struct ffa_composite_memory_region) +
3327 fragment_constituents_size;
3328 composite_offset_v1_1 = receivers_end;
3329 } else {
3330 fragment_constituents_size = 0;
3331 fragment_length_v1_1 = receivers_end;
3332 composite_offset_v1_1 = 0;
3333 }
J-Alves33c47bf2022-09-29 11:36:20 +01003334
3335 /*
3336 * Currently only support the simpler cases: memory transaction
3337 * in a single fragment that fits in a MM_PPOOL_ENTRY_SIZE.
3338 * TODO: allocate the entries needed for this fragment_length_v1_1.
3339 * - Check corner when v1.1 descriptor converted size surpasses
3340 * the size of the entry.
3341 */
3342 if (fragment_length_v1_1 > MM_PPOOL_ENTRY_SIZE) {
3343 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003344 "Translation of FF-A v1.0 descriptors for over %lu is "
J-Alves33c47bf2022-09-29 11:36:20 +01003345 "unsupported.",
3346 MM_PPOOL_ENTRY_SIZE);
3347 return ffa_error(FFA_NOT_SUPPORTED);
3348 }
3349
3350 space_left = fragment_length_v1_1;
3351
Daniel Boulby36e0bdb2023-10-09 12:00:41 +01003352 /*
3353 * Allocate a page of memory to construct the v1.1 memory descriptor.
3354 * Earlier we checked that the fragment_length_v1_1 would not be larger
3355 * than a page.
3356 */
J-Alves33c47bf2022-09-29 11:36:20 +01003357 memory_region_v1_1 =
3358 (struct ffa_memory_region *)mpool_alloc(&api_page_pool);
3359 if (memory_region_v1_1 == NULL) {
3360 return ffa_error(FFA_NO_MEMORY);
3361 }
3362
3363 /* Translate header from v1.0 to v1.1. */
3364 api_ffa_memory_region_v1_1_from_v1_0(memory_region_v1_0,
3365 memory_region_v1_1);
3366
3367 space_left -= sizeof(struct ffa_memory_region);
3368
3369 /* Copy memory access information. */
Daniel Boulby41ef8ba2023-10-13 17:01:22 +01003370 memcpy_s((uint8_t *)memory_region_v1_1 +
3371 memory_region_v1_1->receivers_offset,
3372 space_left, memory_region_v1_0->receivers, receivers_length);
J-Alves33c47bf2022-09-29 11:36:20 +01003373
3374 /* Initialize the memory access descriptors with composite offset. */
3375 for (uint32_t i = 0; i < memory_region_v1_1->receiver_count; i++) {
3376 struct ffa_memory_access *receiver =
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003377 ffa_memory_region_get_receiver(memory_region_v1_1, i);
3378 assert(receiver != NULL);
J-Alves33c47bf2022-09-29 11:36:20 +01003379 receiver->composite_memory_region_offset =
3380 composite_offset_v1_1;
3381 }
3382
3383 space_left -= receivers_length;
3384
Daniel Boulby36e0bdb2023-10-09 12:00:41 +01003385 /* Composite memory descriptors to copy. */
3386 if (send_transaction) {
3387 /* Init v1.1 composite. */
3388 composite_v1_1 = (struct ffa_composite_memory_region
3389 *)((uint8_t *)memory_region_v1_1 +
3390 composite_offset_v1_1);
J-Alves33c47bf2022-09-29 11:36:20 +01003391
Daniel Boulby36e0bdb2023-10-09 12:00:41 +01003392 composite_v1_0 = ffa_memory_region_get_composite_v1_0(
3393 memory_region_v1_0, 0);
3394 composite_v1_1->constituent_count =
3395 composite_v1_0->constituent_count;
3396 composite_v1_1->page_count = composite_v1_0->page_count;
J-Alves33c47bf2022-09-29 11:36:20 +01003397
Daniel Boulby36e0bdb2023-10-09 12:00:41 +01003398 space_left -= sizeof(struct ffa_composite_memory_region);
J-Alves33c47bf2022-09-29 11:36:20 +01003399
Daniel Boulby36e0bdb2023-10-09 12:00:41 +01003400 /* Initialize v1.1 constituents. */
3401 memcpy_s(composite_v1_1->constituents, space_left,
3402 composite_v1_0->constituents,
3403 fragment_constituents_size);
J-Alves33c47bf2022-09-29 11:36:20 +01003404
Daniel Boulby36e0bdb2023-10-09 12:00:41 +01003405 space_left -= fragment_constituents_size;
3406 }
3407
J-Alves33c47bf2022-09-29 11:36:20 +01003408 assert(space_left == 0U);
3409
J-Alves33c47bf2022-09-29 11:36:20 +01003410 /*
3411 * Remove the v1.0 fragment size, and resultant size of v1.1 fragment.
3412 */
3413 *total_length = *total_length - *fragment_length + fragment_length_v1_1;
3414 *fragment_length = fragment_length_v1_1;
3415
3416 /*
Daniel Boulby36e0bdb2023-10-09 12:00:41 +01003417 * After successfully updating to v1.1 copy the descriptor to the
3418 * internal buffer given as a parameter (used to prevent TOCTOU attacks)
3419 * and free the scratch memory used to construct it.
J-Alves33c47bf2022-09-29 11:36:20 +01003420 */
Daniel Boulby36e0bdb2023-10-09 12:00:41 +01003421 memcpy_s(allocated, MM_PPOOL_ENTRY_SIZE, memory_region_v1_1,
3422 *fragment_length);
3423 mpool_free(&api_page_pool, memory_region_v1_1);
J-Alves33c47bf2022-09-29 11:36:20 +01003424
3425 return (struct ffa_value){.func = FFA_SUCCESS_32};
3426}
3427
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003428struct ffa_value api_ffa_mem_send(uint32_t share_func, uint32_t length,
3429 uint32_t fragment_length, ipaddr_t address,
Andrew Walbran1a86aa92020-05-15 17:22:28 +01003430 uint32_t page_count, struct vcpu *current)
Andrew Walbrane908c4a2019-12-02 17:13:47 +00003431{
3432 struct vm *from = current->vm;
3433 struct vm *to;
3434 const void *from_msg;
J-Alves33c47bf2022-09-29 11:36:20 +01003435 void *allocated_entry;
Daniel Boulby36e0bdb2023-10-09 12:00:41 +01003436 struct ffa_memory_region *memory_region = NULL;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003437 struct ffa_value ret;
J-Alves363f5722022-04-25 17:37:37 +01003438 bool targets_other_world = false;
Karl Meakin0e617d92024-04-05 12:55:22 +01003439 enum ffa_version ffa_version;
Andrew Walbrane908c4a2019-12-02 17:13:47 +00003440
3441 if (ipa_addr(address) != 0 || page_count != 0) {
3442 /*
3443 * Hafnium only supports passing the descriptor in the TX
3444 * mailbox.
3445 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003446 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrane908c4a2019-12-02 17:13:47 +00003447 }
3448
Andrew Walbranca808b12020-05-15 17:22:28 +01003449 if (fragment_length > length) {
3450 dlog_verbose(
3451 "Fragment length %d greater than total length %d.\n",
3452 fragment_length, length);
3453 return ffa_error(FFA_INVALID_PARAMETERS);
3454 }
J-Alves33c47bf2022-09-29 11:36:20 +01003455
3456 if (fragment_length > HF_MAILBOX_SIZE ||
3457 fragment_length > MM_PPOOL_ENTRY_SIZE) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003458 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrane908c4a2019-12-02 17:13:47 +00003459 }
3460
3461 /*
3462 * Check that the sender has configured its send buffer. If the TX
3463 * mailbox at from_msg is configured (i.e. from_msg != NULL) then it can
3464 * be safely accessed after releasing the lock since the TX mailbox
3465 * address can only be configured once.
3466 */
3467 sl_lock(&from->lock);
3468 from_msg = from->mailbox.send;
J-Alves33c47bf2022-09-29 11:36:20 +01003469 ffa_version = from->ffa_version;
Andrew Walbrane908c4a2019-12-02 17:13:47 +00003470 sl_unlock(&from->lock);
3471
3472 if (from_msg == NULL) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003473 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrane908c4a2019-12-02 17:13:47 +00003474 }
3475
3476 /*
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003477 * Copy the memory region descriptor to a fresh page from the memory
3478 * pool. This prevents the sender from changing it underneath us, and
3479 * also lets us keep it around in the share state table if needed.
Andrew Walbrane908c4a2019-12-02 17:13:47 +00003480 */
J-Alves33c47bf2022-09-29 11:36:20 +01003481 allocated_entry = mpool_alloc(&api_page_pool);
3482 if (allocated_entry == NULL) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003483 dlog_verbose("Failed to allocate memory region copy.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003484 return ffa_error(FFA_NO_MEMORY);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003485 }
J-Alves33c47bf2022-09-29 11:36:20 +01003486
J-Alves8f2150d2024-03-28 16:15:56 +00003487 if (!memcpy_trapped(allocated_entry, MM_PPOOL_ENTRY_SIZE, from_msg,
3488 fragment_length)) {
3489 dlog_error(
3490 "%s: Failed to copy FF-A memory region descriptor.\n",
3491 __func__);
3492 return ffa_error(FFA_ABORTED);
3493 }
J-Alves33c47bf2022-09-29 11:36:20 +01003494
Daniel Boulbyc7dc9322023-10-27 15:12:07 +01003495 if (!ffa_memory_region_sanity_check(allocated_entry, ffa_version,
3496 fragment_length, true)) {
3497 ret = ffa_error(FFA_INVALID_PARAMETERS);
3498 goto out;
3499 }
3500
Daniel Boulby36e0bdb2023-10-09 12:00:41 +01003501 ret = api_ffa_memory_transaction_descriptor_v1_1_from_v1_0(
3502 allocated_entry, &fragment_length, &length, ffa_version, true);
J-Alves33c47bf2022-09-29 11:36:20 +01003503 if (ret.func != FFA_SUCCESS_32) {
3504 goto out;
3505 }
3506
Daniel Boulby36e0bdb2023-10-09 12:00:41 +01003507 memory_region = allocated_entry;
3508
J-Alves33c47bf2022-09-29 11:36:20 +01003509 if (fragment_length < sizeof(struct ffa_memory_region) +
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003510 memory_region->memory_access_desc_size) {
J-Alves33c47bf2022-09-29 11:36:20 +01003511 dlog_verbose(
3512 "Initial fragment length %d smaller than header size "
Karl Meakine8937d92024-03-19 16:04:25 +00003513 "%lu.\n",
J-Alves33c47bf2022-09-29 11:36:20 +01003514 fragment_length,
3515 sizeof(struct ffa_memory_region) +
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003516 memory_region->memory_access_desc_size);
3517 ret = ffa_error(FFA_INVALID_PARAMETERS);
3518 goto out;
J-Alves33c47bf2022-09-29 11:36:20 +01003519 }
Andrew Walbrane908c4a2019-12-02 17:13:47 +00003520
J-Alves84658fc2021-06-17 14:37:32 +01003521 if (!api_memory_region_check_flags(memory_region, share_func)) {
3522 dlog_verbose(
3523 "Memory region reserved arguments must be zero.\n");
3524 ret = ffa_error(FFA_INVALID_PARAMETERS);
3525 goto out;
3526 }
3527
J-Alves363f5722022-04-25 17:37:37 +01003528 if (memory_region->receiver_count == 0U) {
3529 dlog_verbose("Receiver count can't be 0.\n");
3530 ret = ffa_error(FFA_INVALID_PARAMETERS);
3531 goto out;
3532 }
3533
J-Alves95fbb312024-03-20 15:19:16 +00003534 if ((share_func == FFA_MEM_DONATE_32 ||
3535 share_func == FFA_MEM_DONATE_64) &&
J-Alves363f5722022-04-25 17:37:37 +01003536 memory_region->receiver_count != 1U) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003537 dlog_verbose(
J-Alves0a824e92024-04-26 16:20:12 +01003538 "FFA_MEM_DONATE only supports one recipient. Specified "
3539 "%u\n",
J-Alves363f5722022-04-25 17:37:37 +01003540 memory_region->receiver_count);
3541 ret = ffa_error(FFA_INVALID_PARAMETERS);
3542 goto out;
3543 }
3544
Andrew Walbrane908c4a2019-12-02 17:13:47 +00003545 /*
3546 * Ensure that the receiver VM exists and isn't the same as the sender.
J-Alves363f5722022-04-25 17:37:37 +01003547 * If there is a receiver from the other world, track it for later
3548 * forwarding if needed.
Andrew Walbrane908c4a2019-12-02 17:13:47 +00003549 */
J-Alves363f5722022-04-25 17:37:37 +01003550 for (uint32_t i = 0U; i < memory_region->receiver_count; i++) {
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003551 struct ffa_memory_access *receiver =
3552 ffa_memory_region_get_receiver(memory_region, i);
J-Alvesc9227c82024-04-24 21:00:58 +01003553 ffa_id_t receiver_id;
3554
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003555 assert(receiver != NULL);
J-Alvesc9227c82024-04-24 21:00:58 +01003556
3557 receiver_id = receiver->receiver_permissions.receiver;
Daniel Boulbyd5ae44b2023-12-12 12:18:11 +00003558
J-Alves363f5722022-04-25 17:37:37 +01003559 to = vm_find(receiver_id);
3560
J-Alves7de61af2024-03-27 10:29:46 +00003561 if ((vm_id_is_current_world(receiver_id) && to == NULL) ||
3562 to == from) {
J-Alves363f5722022-04-25 17:37:37 +01003563 dlog_verbose("%s: invalid receiver.\n", __func__);
3564 ret = ffa_error(FFA_INVALID_PARAMETERS);
3565 goto out;
3566 }
3567
J-Alvesc9227c82024-04-24 21:00:58 +01003568 if (!plat_ffa_is_memory_send_valid(
3569 receiver_id, from->id, share_func,
3570 memory_region->receiver_count > 1)) {
J-Alves363f5722022-04-25 17:37:37 +01003571 ret = ffa_error(FFA_DENIED);
3572 goto out;
3573 }
3574
3575 /* Capture if any of the receivers is from the other world. */
3576 if (!targets_other_world) {
3577 targets_other_world =
3578 !vm_id_is_current_world(receiver_id);
3579 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003580 }
3581
J-Alves363f5722022-04-25 17:37:37 +01003582 if (targets_other_world) {
J-Alves66652252022-07-06 09:49:51 +01003583 ret = plat_ffa_other_world_mem_send(
3584 from, share_func, &memory_region, length,
3585 fragment_length, &api_page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003586 } else {
3587 struct vm_locked from_locked = vm_lock(from);
3588
Andrew Walbran1a86aa92020-05-15 17:22:28 +01003589 ret = ffa_memory_send(from_locked, memory_region, length,
3590 fragment_length, share_func,
3591 &api_page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003592 /*
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003593 * ffa_memory_send takes ownership of the memory_region, so
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003594 * make sure we don't free it.
3595 */
3596 memory_region = NULL;
3597
3598 vm_unlock(&from_locked);
3599 }
3600
3601out:
3602 if (memory_region != NULL) {
3603 mpool_free(&api_page_pool, memory_region);
3604 }
3605
3606 return ret;
3607}
3608
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003609struct ffa_value api_ffa_mem_retrieve_req(uint32_t length,
3610 uint32_t fragment_length,
3611 ipaddr_t address, uint32_t page_count,
3612 struct vcpu *current)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003613{
3614 struct vm *to = current->vm;
3615 struct vm_locked to_locked;
3616 const void *to_msg;
J-Alves00847062022-10-03 17:08:44 +01003617 void *retrieve_msg;
3618 struct ffa_memory_region *retrieve_request = NULL;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003619 uint32_t message_buffer_size;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003620 struct ffa_value ret;
Karl Meakin0e617d92024-04-05 12:55:22 +01003621 enum ffa_version ffa_version;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003622
3623 if (ipa_addr(address) != 0 || page_count != 0) {
3624 /*
3625 * Hafnium only supports passing the descriptor in the TX
3626 * mailbox.
3627 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003628 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbrane908c4a2019-12-02 17:13:47 +00003629 }
3630
Andrew Walbrana65a1322020-04-06 19:32:32 +01003631 if (fragment_length != length) {
J-Alves33c47bf2022-09-29 11:36:20 +01003632 dlog_verbose("Fragmentation not supported.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003633 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003634 }
Andrew Walbrane908c4a2019-12-02 17:13:47 +00003635
J-Alves00847062022-10-03 17:08:44 +01003636 retrieve_msg = cpu_get_buffer(current->cpu);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003637 message_buffer_size = cpu_get_buffer_size(current->cpu);
3638 if (length > HF_MAILBOX_SIZE || length > message_buffer_size) {
3639 dlog_verbose("Retrieve request too long.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003640 return ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003641 }
3642
3643 to_locked = vm_lock(to);
3644 to_msg = to->mailbox.send;
J-Alves00847062022-10-03 17:08:44 +01003645 ffa_version = to->ffa_version;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003646
3647 if (to_msg == NULL) {
3648 dlog_verbose("TX buffer not setup.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003649 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003650 goto out;
3651 }
3652
3653 /*
3654 * Copy the retrieve request descriptor to an internal buffer, so that
3655 * the caller can't change it underneath us.
3656 */
J-Alves8f2150d2024-03-28 16:15:56 +00003657 if (!memcpy_trapped(retrieve_msg, message_buffer_size, to_msg,
3658 length)) {
3659 dlog_error(
3660 "%s: Failed to copy FF-A retrieve request "
3661 "descriptor.\n",
3662 __func__);
3663 ret = ffa_error(FFA_ABORTED);
3664 goto out;
3665 }
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003666
J-Alvese8c8c2b2022-12-16 15:34:48 +00003667 if ((vm_is_mailbox_other_world_owned(to_locked) &&
3668 !plat_ffa_acquire_receiver_rx(to_locked, &ret)) ||
3669 vm_is_mailbox_busy(to_locked)) {
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003670 /*
J-Alvese8c8c2b2022-12-16 15:34:48 +00003671 * Can't retrieve memory information if the mailbox is
3672 * not available.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003673 */
J-Alves59ed0042022-07-28 18:26:41 +01003674 dlog_verbose("%s: RX buffer not ready.\n", __func__);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003675 ret = ffa_error(FFA_BUSY);
Andrew Walbrane908c4a2019-12-02 17:13:47 +00003676 goto out;
3677 }
3678
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003679 if (!is_ffa_hypervisor_retrieve_request(retrieve_msg, to_locked)) {
3680 if (!ffa_memory_region_sanity_check(retrieve_msg, ffa_version,
3681 fragment_length, false)) {
3682 ret = ffa_error(FFA_INVALID_PARAMETERS);
3683 goto out;
3684 }
3685 /*
3686 * If required, transform the retrieve request to FF-A v1.1.
3687 */
3688 ret = api_ffa_memory_transaction_descriptor_v1_1_from_v1_0(
3689 retrieve_msg, &fragment_length, &length, ffa_version,
3690 false);
Daniel Boulbyc7dc9322023-10-27 15:12:07 +01003691
Daniel Boulby44e9b3b2024-01-17 12:21:44 +00003692 if (ret.func != FFA_SUCCESS_32) {
3693 goto out;
3694 }
J-Alves00847062022-10-03 17:08:44 +01003695 }
3696
Daniel Boulby36e0bdb2023-10-09 12:00:41 +01003697 retrieve_request = retrieve_msg;
Kathleen Capella578784f2023-09-01 18:03:27 -04003698
J-Alvesb5084cf2022-07-06 14:20:12 +01003699 if (plat_ffa_memory_handle_allocated_by_current_world(
3700 retrieve_request->handle)) {
3701 ret = ffa_memory_retrieve(to_locked, retrieve_request, length,
3702 &api_page_pool);
3703 } else {
Daniel Boulbyfdd59f72023-10-19 16:43:19 +01003704 dlog_error("Invalid FF-A memory handle.\n");
3705 ret = ffa_error(FFA_INVALID_PARAMETERS);
J-Alvesb5084cf2022-07-06 14:20:12 +01003706 }
Andrew Walbrane908c4a2019-12-02 17:13:47 +00003707out:
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003708 vm_unlock(&to_locked);
3709 return ret;
3710}
3711
J-Alvesa39a8442024-04-26 15:41:47 +01003712/**
3713 * Copies the memory relinquish descriptor from the partition's TX buffer, to
3714 * the buffer of the local CPU. Do it safely, and return error if:
3715 * - FFA_ABORTED: if the `memcpy_trapped` fails.
3716 * - FFA_INVALID_PARAMETERS: if the size of the full memory relinquish
3717 * descriptor doesn't fit the local CPU buffer.
3718 *
3719 * Returns FFA_SUCCESS if copying goes well, and sets 'out_relinquish'
3720 * to the address of the cpu buffer with the relinquish descriptor.
3721 */
3722static struct ffa_value api_get_ffa_mem_relinquish_descriptor(
3723 struct vcpu *current, const void *from_msg,
3724 struct ffa_mem_relinquish **out_relinquish)
3725{
3726 struct ffa_mem_relinquish *relinquish_request;
3727 uint32_t from_msg_size;
3728 uint32_t total_from_msg_size;
3729 uint32_t dst_size;
3730 vaddr_t dst;
3731 vaddr_t src;
3732
3733 assert(from_msg != NULL);
3734 assert(out_relinquish != NULL);
3735
3736 /*
3737 * Copy the relinquish descriptor to an internal buffer, so that the
3738 * caller can't change it underneath us.
3739 */
3740 relinquish_request =
3741 (struct ffa_mem_relinquish *)cpu_get_buffer(current->cpu);
3742
3743 /* Set the destination for the copy. */
3744 dst = va_from_ptr(relinquish_request);
3745 src = va_from_ptr(from_msg);
3746
3747 dst_size = cpu_get_buffer_size(current->cpu);
3748
3749 /* Only copy the size to start with. */
3750 from_msg_size = sizeof(struct ffa_mem_relinquish);
3751 total_from_msg_size = from_msg_size;
3752
3753 if (!memcpy_trapped(ptr_from_va(dst), dst_size, ptr_from_va(src),
3754 from_msg_size)) {
3755 dlog_error(
3756 "%s: Failed to copy FF-A memory relinquish "
3757 "descriptor.\n",
3758 __func__);
3759 return ffa_error(FFA_ABORTED);
3760 }
3761
3762 if (relinquish_request->endpoint_count != 1) {
3763 dlog_error("%s: relinquish descriptor must have 1 endpoint\n",
3764 __func__);
3765 return ffa_error(FFA_INVALID_PARAMETERS);
3766 }
3767
3768 /*
3769 * Increment the `dst` to the position right after the copied header.
3770 * Increment the `src` to point at the list of endpoints.
3771 *
3772 * Calculate the new `dst_size` which is the size of the allocated cpu
3773 * buffer, minus the size of the copied memory relinquish header.
3774 *
3775 * Only after the above: determine the new `from_msg_size` in accordance
3776 * to the endpoint count.
3777 */
3778 dst = va_add(dst, from_msg_size);
3779 src = va_add(src, from_msg_size);
3780
3781 /*
3782 * Check if it is safe to copy the rest of the message.
3783 * This also serves as a santiy check to 'endpoint_count'.
3784 * The size of what is left in the descriptor, based on endpoint_count,
3785 * shall not be bigger than the size of the mailbox minus the size of
3786 * the header which was previously copied in this function.
3787 */
3788 dst_size -= from_msg_size;
3789 from_msg_size = relinquish_request->endpoint_count * sizeof(ffa_id_t);
3790 total_from_msg_size += from_msg_size;
3791
3792 if (total_from_msg_size > HF_MAILBOX_SIZE ||
3793 total_from_msg_size > dst_size) {
3794 dlog_verbose(
3795 "Relinquish message too long. Endpoint count: %u\n",
3796 relinquish_request->endpoint_count);
3797 return ffa_error(FFA_INVALID_PARAMETERS);
3798 }
3799
3800 /* Copy the remaining fragment. */
3801 if (!memcpy_trapped(ptr_from_va(dst), dst_size, ptr_from_va(src),
3802 from_msg_size)) {
3803 dlog_error("%s: Failed to copy FF-A relinquish request.\n",
3804 __func__);
3805 return ffa_error(FFA_ABORTED);
3806 }
3807
3808 /*
3809 * Set the output address for the relinquish descriptor to the current
3810 * cpu's buffer.
3811 */
3812 *out_relinquish = relinquish_request;
3813
3814 return (struct ffa_value){.func = FFA_SUCCESS_32};
3815}
3816
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003817struct ffa_value api_ffa_mem_relinquish(struct vcpu *current)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003818{
3819 struct vm *from = current->vm;
3820 struct vm_locked from_locked;
3821 const void *from_msg;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003822 struct ffa_value ret;
J-Alvesa39a8442024-04-26 15:41:47 +01003823 struct ffa_mem_relinquish *relinquish_request;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003824
3825 from_locked = vm_lock(from);
3826 from_msg = from->mailbox.send;
3827
3828 if (from_msg == NULL) {
3829 dlog_verbose("TX buffer not setup.\n");
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003830 ret = ffa_error(FFA_INVALID_PARAMETERS);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003831 goto out;
3832 }
3833
J-Alvesa39a8442024-04-26 15:41:47 +01003834 ret = api_get_ffa_mem_relinquish_descriptor(current, from_msg,
3835 &relinquish_request);
3836
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003837 /*
J-Alvesa39a8442024-04-26 15:41:47 +01003838 * If the descriptor was safely copied, continue with the handling of
3839 * the retrieve request.
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003840 */
J-Alvesa39a8442024-04-26 15:41:47 +01003841 if (ret.func == FFA_SUCCESS_32) {
3842 ret = ffa_memory_relinquish(from_locked, relinquish_request,
3843 &api_page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003844 }
J-Alves8f2150d2024-03-28 16:15:56 +00003845
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003846out:
3847 vm_unlock(&from_locked);
3848 return ret;
3849}
3850
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003851struct ffa_value api_ffa_mem_reclaim(ffa_memory_handle_t handle,
3852 ffa_memory_region_flags_t flags,
3853 struct vcpu *current)
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003854{
3855 struct vm *to = current->vm;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003856 struct ffa_value ret;
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003857
Olivier Deprez55a189e2021-06-09 15:45:27 +02003858 if (plat_ffa_memory_handle_allocated_by_current_world(handle)) {
Andrew Walbran290b0c92020-02-03 16:37:14 +00003859 struct vm_locked to_locked = vm_lock(to);
3860
Andrew Walbranca808b12020-05-15 17:22:28 +01003861 ret = ffa_memory_reclaim(to_locked, handle, flags,
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01003862 &api_page_pool);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +00003863
Andrew Walbran290b0c92020-02-03 16:37:14 +00003864 vm_unlock(&to_locked);
3865 } else {
J-Alvesfc19b372022-07-06 12:17:35 +01003866 ret = plat_ffa_other_world_mem_reclaim(to, handle, flags,
3867 &api_page_pool);
Andrew Walbranca808b12020-05-15 17:22:28 +01003868 }
3869
3870 return ret;
3871}
3872
3873struct ffa_value api_ffa_mem_frag_rx(ffa_memory_handle_t handle,
3874 uint32_t fragment_offset,
J-Alves19e20cf2023-08-02 12:48:55 +01003875 ffa_id_t sender_vm_id,
Andrew Walbranca808b12020-05-15 17:22:28 +01003876 struct vcpu *current)
3877{
3878 struct vm *to = current->vm;
3879 struct vm_locked to_locked;
3880 struct ffa_value ret;
3881
3882 /* Sender ID MBZ at virtual instance. */
J-Alves59ed0042022-07-28 18:26:41 +01003883 if (vm_id_is_current_world(to->id)) {
3884 if (sender_vm_id != 0) {
3885 dlog_verbose("%s: Invalid sender.\n", __func__);
3886 return ffa_error(FFA_INVALID_PARAMETERS);
3887 }
Andrew Walbranca808b12020-05-15 17:22:28 +01003888 }
3889
3890 to_locked = vm_lock(to);
3891
J-Alves122f1a12022-12-12 15:55:42 +00003892 if (vm_is_mailbox_busy(to_locked)) {
Andrew Walbranca808b12020-05-15 17:22:28 +01003893 /*
3894 * Can't retrieve memory information if the mailbox is not
3895 * available.
3896 */
J-Alves59ed0042022-07-28 18:26:41 +01003897 dlog_verbose("%s: RX buffer not ready partition %x.\n",
3898 __func__, to_locked.vm->id);
Andrew Walbranca808b12020-05-15 17:22:28 +01003899 ret = ffa_error(FFA_BUSY);
3900 goto out;
3901 }
3902
3903 ret = ffa_memory_retrieve_continue(to_locked, handle, fragment_offset,
J-Alves59ed0042022-07-28 18:26:41 +01003904 sender_vm_id, &api_page_pool);
Andrew Walbranca808b12020-05-15 17:22:28 +01003905out:
3906 vm_unlock(&to_locked);
3907 return ret;
3908}
3909
3910struct ffa_value api_ffa_mem_frag_tx(ffa_memory_handle_t handle,
3911 uint32_t fragment_length,
J-Alves19e20cf2023-08-02 12:48:55 +01003912 ffa_id_t sender_vm_id,
Andrew Walbranca808b12020-05-15 17:22:28 +01003913 struct vcpu *current)
3914{
3915 struct vm *from = current->vm;
3916 const void *from_msg;
3917 void *fragment_copy;
3918 struct ffa_value ret;
3919
3920 /* Sender ID MBZ at virtual instance. */
J-Alvesfdd29272022-07-19 13:16:31 +01003921 if (vm_id_is_current_world(from->id) && sender_vm_id != 0) {
3922 dlog_verbose("Invalid sender.");
Andrew Walbranca808b12020-05-15 17:22:28 +01003923 return ffa_error(FFA_INVALID_PARAMETERS);
3924 }
3925
3926 /*
3927 * Check that the sender has configured its send buffer. If the TX
3928 * mailbox at from_msg is configured (i.e. from_msg != NULL) then it can
3929 * be safely accessed after releasing the lock since the TX mailbox
3930 * address can only be configured once.
3931 */
3932 sl_lock(&from->lock);
3933 from_msg = from->mailbox.send;
3934 sl_unlock(&from->lock);
3935
3936 if (from_msg == NULL) {
J-Alves59ed0042022-07-28 18:26:41 +01003937 dlog_verbose("Mailbox from %x is not set.\n", from->id);
Andrew Walbranca808b12020-05-15 17:22:28 +01003938 return ffa_error(FFA_INVALID_PARAMETERS);
3939 }
3940
3941 /*
3942 * Copy the fragment to a fresh page from the memory pool. This prevents
3943 * the sender from changing it underneath us, and also lets us keep it
3944 * around in the share state table if needed.
3945 */
3946 if (fragment_length > HF_MAILBOX_SIZE ||
3947 fragment_length > MM_PPOOL_ENTRY_SIZE) {
3948 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00003949 "Fragment length %d larger than mailbox size %zu.\n",
Andrew Walbranca808b12020-05-15 17:22:28 +01003950 fragment_length, HF_MAILBOX_SIZE);
3951 return ffa_error(FFA_INVALID_PARAMETERS);
3952 }
3953 if (fragment_length < sizeof(struct ffa_memory_region_constituent) ||
3954 fragment_length % sizeof(struct ffa_memory_region_constituent) !=
3955 0) {
3956 dlog_verbose("Invalid fragment length %d.\n", fragment_length);
3957 return ffa_error(FFA_INVALID_PARAMETERS);
3958 }
3959 fragment_copy = mpool_alloc(&api_page_pool);
3960 if (fragment_copy == NULL) {
3961 dlog_verbose("Failed to allocate fragment copy.\n");
3962 return ffa_error(FFA_NO_MEMORY);
3963 }
J-Alves8f2150d2024-03-28 16:15:56 +00003964
3965 if (!memcpy_trapped(fragment_copy, MM_PPOOL_ENTRY_SIZE, from_msg,
3966 fragment_length)) {
3967 dlog_error("%s: Failed to copy fragment.\n", __func__);
3968 return ffa_error(FFA_ABORTED);
3969 }
Andrew Walbranca808b12020-05-15 17:22:28 +01003970
3971 /*
3972 * Hafnium doesn't support fragmentation of memory retrieve requests
3973 * (because it doesn't support caller-specified mappings, so a request
3974 * will never be larger than a single page), so this must be part of a
3975 * memory send (i.e. donate, lend or share) request.
3976 *
3977 * We can tell from the handle whether the memory transaction is for the
J-Alvesfdd29272022-07-19 13:16:31 +01003978 * other world or not.
Andrew Walbranca808b12020-05-15 17:22:28 +01003979 */
J-Alvesfdd29272022-07-19 13:16:31 +01003980 if (plat_ffa_memory_handle_allocated_by_current_world(handle)) {
Andrew Walbranca808b12020-05-15 17:22:28 +01003981 struct vm_locked from_locked = vm_lock(from);
3982
3983 ret = ffa_memory_send_continue(from_locked, fragment_copy,
3984 fragment_length, handle,
3985 &api_page_pool);
3986 /*
3987 * `ffa_memory_send_continue` takes ownership of the
3988 * fragment_copy, so we don't need to free it here.
3989 */
3990 vm_unlock(&from_locked);
3991 } else {
J-Alvesfdd29272022-07-19 13:16:31 +01003992 ret = plat_ffa_other_world_mem_send_continue(
3993 from, fragment_copy, fragment_length, handle,
3994 &api_page_pool);
Andrew Walbran290b0c92020-02-03 16:37:14 +00003995 }
Andrew Walbrane908c4a2019-12-02 17:13:47 +00003996
3997 return ret;
3998}
Max Shvetsov40108e72020-08-27 12:39:50 +01003999
Olivier Deprezd614d322021-06-18 15:21:00 +02004000/**
4001 * Register an entry point for a vCPU in warm boot cases.
4002 * DEN0077A FF-A v1.1 Beta0 section 18.3.2.1 FFA_SECONDARY_EP_REGISTER.
4003 */
Max Shvetsov40108e72020-08-27 12:39:50 +01004004struct ffa_value api_ffa_secondary_ep_register(ipaddr_t entry_point,
4005 struct vcpu *current)
4006{
4007 struct vm_locked vm_locked;
Olivier Deprezb2808332023-02-02 15:25:40 +01004008 struct vcpu_locked current_locked;
Max Shvetsov40108e72020-08-27 12:39:50 +01004009
Olivier Deprezd614d322021-06-18 15:21:00 +02004010 /*
4011 * Reject if interface is not supported at this FF-A instance
4012 * (DEN0077A FF-A v1.1 Beta0 Table 18.29) or the VM is UP.
4013 */
4014 if (!plat_ffa_is_secondary_ep_register_supported() ||
Karl Meakin82041822024-05-20 11:14:34 +01004015 vm_is_up(current->vm)) {
Olivier Deprezd614d322021-06-18 15:21:00 +02004016 return ffa_error(FFA_NOT_SUPPORTED);
4017 }
4018
4019 /*
4020 * No further check is made on the address validity
4021 * (FF-A v1.1 Beta0 Table 18.29) as the VM boundaries are not known
4022 * from the VM or vCPU structure.
4023 * DEN0077A FF-A v1.1 Beta0 section 18.3.2.1.1:
4024 * For each SP [...] the Framework assumes that the same entry point
4025 * address is used for initializing any execution context during a
4026 * secondary cold boot.
4027 * If this function is invoked multiple times, then the entry point
4028 * address specified in the last valid invocation must be used by the
4029 * callee.
4030 */
Olivier Deprezb2808332023-02-02 15:25:40 +01004031 current_locked = vcpu_lock(current);
4032 if (current->rt_model != RTM_SP_INIT) {
4033 dlog_error(
4034 "FFA_SECONDARY_EP_REGISTER can only be called while "
4035 "vCPU in run-time state for initialization.\n");
4036 vcpu_unlock(&current_locked);
4037 return ffa_error(FFA_DENIED);
Olivier Deprezd614d322021-06-18 15:21:00 +02004038 }
Olivier Deprezb2808332023-02-02 15:25:40 +01004039 vcpu_unlock(&current_locked);
Olivier Deprezd614d322021-06-18 15:21:00 +02004040
Olivier Deprezb2808332023-02-02 15:25:40 +01004041 vm_locked = vm_lock(current->vm);
Max Shvetsov40108e72020-08-27 12:39:50 +01004042 vm_locked.vm->secondary_ep = entry_point;
4043 vm_unlock(&vm_locked);
4044
Olivier Deprezb2808332023-02-02 15:25:40 +01004045 return (struct ffa_value){.func = FFA_SUCCESS_32};
Max Shvetsov40108e72020-08-27 12:39:50 +01004046}
J-Alvesa0f317d2021-06-09 13:31:59 +01004047
J-Alves19e20cf2023-08-02 12:48:55 +01004048struct ffa_value api_ffa_notification_bitmap_create(ffa_id_t vm_id,
J-Alvesa0f317d2021-06-09 13:31:59 +01004049 ffa_vcpu_count_t vcpu_count,
4050 struct vcpu *current)
4051{
J-Alves7ccaccf2024-02-01 14:25:23 +00004052 const struct ffa_value ret =
4053 plat_ffa_is_notifications_bitmap_access_valid(current, vm_id);
4054
4055 if (ffa_func_id(ret) != FFA_SUCCESS_32) {
4056 dlog_verbose(
4057 "FFA_NOTIFICATION_BITMAP_CREATE to be used by "
4058 "hypervisor for valid NWd VM IDs only (%x).\n",
4059 vm_id);
4060 return ret;
J-Alvesa0f317d2021-06-09 13:31:59 +01004061 }
4062
4063 return plat_ffa_notifications_bitmap_create(vm_id, vcpu_count);
4064}
4065
J-Alves19e20cf2023-08-02 12:48:55 +01004066struct ffa_value api_ffa_notification_bitmap_destroy(ffa_id_t vm_id,
J-Alvesa0f317d2021-06-09 13:31:59 +01004067 struct vcpu *current)
4068{
J-Alves7ccaccf2024-02-01 14:25:23 +00004069 const struct ffa_value ret =
4070 plat_ffa_is_notifications_bitmap_access_valid(current, vm_id);
4071
4072 if (ffa_func_id(ret) != FFA_SUCCESS_32) {
4073 dlog_verbose(
4074 "FFA_NOTIFICATION_BITMAP_DESTROY to be used by "
4075 "hypervisor for valid NWd VM IDs only (%x).\n",
4076 vm_id);
4077 return ret;
J-Alvesa0f317d2021-06-09 13:31:59 +01004078 }
4079
4080 return plat_ffa_notifications_bitmap_destroy(vm_id);
4081}
J-Alvesc003a7a2021-03-18 13:06:53 +00004082
4083struct ffa_value api_ffa_notification_update_bindings(
J-Alves19e20cf2023-08-02 12:48:55 +01004084 ffa_id_t sender_vm_id, ffa_id_t receiver_vm_id, uint32_t flags,
J-Alvesc003a7a2021-03-18 13:06:53 +00004085 ffa_notifications_bitmap_t notifications, bool is_bind,
4086 struct vcpu *current)
4087{
4088 struct ffa_value ret = {.func = FFA_SUCCESS_32};
4089 struct vm_locked receiver_locked;
4090 const bool is_per_vcpu = (flags & FFA_NOTIFICATION_FLAG_PER_VCPU) != 0U;
J-Alves19e20cf2023-08-02 12:48:55 +01004091 const ffa_id_t id_to_update = is_bind ? sender_vm_id : HF_INVALID_VM_ID;
4092 const ffa_id_t id_to_validate =
J-Alvesc003a7a2021-03-18 13:06:53 +00004093 is_bind ? HF_INVALID_VM_ID : sender_vm_id;
J-Alves1daeaea2022-09-06 17:41:24 +01004094 const uint32_t flags_mbz =
4095 is_bind ? ~FFA_NOTIFICATIONS_FLAG_PER_VCPU : ~0U;
4096
4097 if ((flags_mbz & flags) != 0U) {
4098 return ffa_error(FFA_INVALID_PARAMETERS);
4099 }
J-Alvesc003a7a2021-03-18 13:06:53 +00004100
4101 if (!plat_ffa_is_notifications_bind_valid(current, sender_vm_id,
4102 receiver_vm_id)) {
4103 dlog_verbose("Invalid use of notifications bind interface.\n");
4104 return ffa_error(FFA_INVALID_PARAMETERS);
4105 }
4106
J-Alvesb15e9402021-09-08 11:44:42 +01004107 if (plat_ffa_notifications_update_bindings_forward(
4108 receiver_vm_id, sender_vm_id, flags, notifications, is_bind,
4109 &ret)) {
J-Alvesb15e9402021-09-08 11:44:42 +01004110 return ret;
4111 }
4112
J-Alvesc003a7a2021-03-18 13:06:53 +00004113 if (notifications == 0U) {
Karl Meakine8937d92024-03-19 16:04:25 +00004114 dlog_verbose("No notifications have been specified %lx.\n",
J-Alves1512acc2024-02-01 16:57:10 +00004115 notifications);
J-Alvesc003a7a2021-03-18 13:06:53 +00004116 return ffa_error(FFA_INVALID_PARAMETERS);
4117 }
4118
4119 /**
4120 * This check assumes receiver is the current VM, and has been enforced
4121 * by 'plat_ffa_is_notifications_bind_valid'.
4122 */
4123 receiver_locked = plat_ffa_vm_find_locked(receiver_vm_id);
4124
4125 if (receiver_locked.vm == NULL) {
4126 dlog_verbose("Receiver doesn't exist!\n");
J-Alves7333dcf2024-02-01 17:25:10 +00004127 return ffa_error(FFA_INVALID_PARAMETERS);
J-Alvesc003a7a2021-03-18 13:06:53 +00004128 }
4129
J-Alves09ff9d82021-11-02 11:55:20 +00004130 if (!vm_locked_are_notifications_enabled(receiver_locked)) {
J-Alvesc003a7a2021-03-18 13:06:53 +00004131 dlog_verbose("Notifications are not enabled.\n");
4132 ret = ffa_error(FFA_NOT_SUPPORTED);
4133 goto out;
4134 }
4135
4136 if (is_bind && vm_id_is_current_world(sender_vm_id) &&
4137 vm_find(sender_vm_id) == NULL) {
4138 dlog_verbose("Sender VM does not exist!\n");
4139 ret = ffa_error(FFA_INVALID_PARAMETERS);
4140 goto out;
4141 }
4142
4143 /*
4144 * Can't bind/unbind notifications if at least one is bound to a
4145 * different sender.
4146 */
4147 if (!vm_notifications_validate_bound_sender(
J-Alves661e1b72023-08-02 13:39:40 +01004148 receiver_locked, ffa_is_vm_id(sender_vm_id), id_to_validate,
4149 notifications)) {
J-Alvesf0208142024-02-01 17:28:58 +00004150 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00004151 "Sender %x not permitted to set notifications %lx to "
J-Alvesf0208142024-02-01 17:28:58 +00004152 "%x.\n",
4153 sender_vm_id, notifications, receiver_vm_id);
J-Alvesc003a7a2021-03-18 13:06:53 +00004154 ret = ffa_error(FFA_DENIED);
4155 goto out;
4156 }
4157
4158 /**
4159 * Check if there is a pending notification within those specified in
4160 * the bitmap.
4161 */
4162 if (vm_are_notifications_pending(receiver_locked,
J-Alves661e1b72023-08-02 13:39:40 +01004163 ffa_is_vm_id(sender_vm_id),
J-Alvesc003a7a2021-03-18 13:06:53 +00004164 notifications)) {
Karl Meakine8937d92024-03-19 16:04:25 +00004165 dlog_verbose("Notifications within '%lx' pending.\n",
J-Alvesc003a7a2021-03-18 13:06:53 +00004166 notifications);
4167 ret = ffa_error(FFA_DENIED);
4168 goto out;
4169 }
4170
4171 vm_notifications_update_bindings(
J-Alves661e1b72023-08-02 13:39:40 +01004172 receiver_locked, ffa_is_vm_id(sender_vm_id), id_to_update,
J-Alvesc003a7a2021-03-18 13:06:53 +00004173 notifications, is_per_vcpu && is_bind);
4174
4175out:
4176 vm_unlock(&receiver_locked);
4177 return ret;
4178}
J-Alvesaa79c012021-07-09 14:29:45 +01004179
4180struct ffa_value api_ffa_notification_set(
J-Alves19e20cf2023-08-02 12:48:55 +01004181 ffa_id_t sender_vm_id, ffa_id_t receiver_vm_id, uint32_t flags,
J-Alvesaa79c012021-07-09 14:29:45 +01004182 ffa_notifications_bitmap_t notifications, struct vcpu *current)
4183{
4184 struct ffa_value ret;
4185 struct vm_locked receiver_locked;
J-Alvesaa79c012021-07-09 14:29:45 +01004186 /*
4187 * Check if is per-vCPU or global, and extracting vCPU ID according
4188 * to table 17.19 of the FF-A v1.1 Beta 0 spec.
4189 */
4190 bool is_per_vcpu = (flags & FFA_NOTIFICATION_FLAG_PER_VCPU) != 0U;
4191 ffa_vcpu_index_t vcpu_id = (uint16_t)(flags >> 16);
J-Alves6b754a12024-02-14 14:26:30 +00004192 const uint32_t flags_mbz =
4193 ~(FFA_NOTIFICATIONS_FLAG_PER_VCPU |
4194 FFA_NOTIFICATIONS_FLAG_DELAY_SRI | (0xFFFFU << 16));
4195
4196 if ((flags_mbz & flags) != 0U) {
4197 dlog_verbose("%s: caller shouldn't set bits that MBZ.\n",
4198 __func__);
4199 return ffa_error(FFA_INVALID_PARAMETERS);
4200 }
J-Alvesaa79c012021-07-09 14:29:45 +01004201
J-Alves30ac91d2024-02-14 15:33:12 +00004202 /* Global notifications must target any vCPU. */
4203 if (!is_per_vcpu && vcpu_id != 0U) {
4204 dlog_verbose(
4205 "For global notifications vCPU ID MBZ in call to set "
4206 "notifications.\n");
4207 return ffa_error(FFA_INVALID_PARAMETERS);
4208 }
4209
J-Alvesaa79c012021-07-09 14:29:45 +01004210 if (!plat_ffa_is_notification_set_valid(current, sender_vm_id,
4211 receiver_vm_id)) {
4212 dlog_verbose("Invalid use of notifications set interface.\n");
4213 return ffa_error(FFA_INVALID_PARAMETERS);
4214 }
4215
4216 if (notifications == 0U) {
4217 dlog_verbose("No notifications have been specified.\n");
4218 return ffa_error(FFA_INVALID_PARAMETERS);
4219 }
4220
J-Alvesde7bd2f2021-09-09 19:54:35 +01004221 if (plat_ffa_notification_set_forward(sender_vm_id, receiver_vm_id,
4222 flags, notifications, &ret)) {
4223 return ret;
4224 }
4225
J-Alvesaa79c012021-07-09 14:29:45 +01004226 /*
4227 * This check assumes receiver is the current VM, and has been enforced
4228 * by 'plat_ffa_is_notification_set_valid'.
4229 */
4230 receiver_locked = plat_ffa_vm_find_locked(receiver_vm_id);
4231
4232 if (receiver_locked.vm == NULL) {
4233 dlog_verbose("Receiver ID is not valid.\n");
4234 return ffa_error(FFA_INVALID_PARAMETERS);
4235 }
4236
J-Alves09ff9d82021-11-02 11:55:20 +00004237 if (!vm_locked_are_notifications_enabled(receiver_locked)) {
J-Alvesaa79c012021-07-09 14:29:45 +01004238 dlog_verbose("Receiver's notifications not enabled.\n");
4239 ret = ffa_error(FFA_DENIED);
4240 goto out;
4241 }
4242
4243 /*
J-Alves89c22132024-02-14 15:44:41 +00004244 * Check the if the notifications are bound as global if per-vCPU flag
4245 * is set, or if they are bound as per-vCPU and caller setting as
4246 * global. In either case, return FFA_INVALID_PARAMETERS.
4247 */
4248 if (vm_notifications_validate_binding(
4249 receiver_locked, ffa_is_vm_id(sender_vm_id), sender_vm_id,
4250 notifications, !is_per_vcpu)) {
Karl Meakine8937d92024-03-19 16:04:25 +00004251 dlog_verbose("Notifications in %lx are %s\n", notifications,
J-Alves89c22132024-02-14 15:44:41 +00004252 !is_per_vcpu ? "global" : "per-vCPU");
4253 ret = ffa_error(FFA_INVALID_PARAMETERS);
4254 goto out;
4255 }
4256
4257 /*
J-Alvesaa79c012021-07-09 14:29:45 +01004258 * If notifications are not bound to the sender, they wouldn't be
4259 * enabled either for the receiver.
4260 */
4261 if (!vm_notifications_validate_binding(
J-Alves661e1b72023-08-02 13:39:40 +01004262 receiver_locked, ffa_is_vm_id(sender_vm_id), sender_vm_id,
4263 notifications, is_per_vcpu)) {
J-Alvesaa79c012021-07-09 14:29:45 +01004264 dlog_verbose("Notifications bindings not valid.\n");
4265 ret = ffa_error(FFA_DENIED);
4266 goto out;
4267 }
4268
4269 if (is_per_vcpu && vcpu_id >= receiver_locked.vm->vcpu_count) {
4270 dlog_verbose("Invalid VCPU ID!\n");
4271 ret = ffa_error(FFA_INVALID_PARAMETERS);
4272 goto out;
4273 }
4274
J-Alves7461ef22021-10-18 17:21:33 +01004275 /* Set notifications pending. */
J-Alves5a16c962022-03-25 12:32:51 +00004276 vm_notifications_partition_set_pending(
J-Alves661e1b72023-08-02 13:39:40 +01004277 receiver_locked, ffa_is_vm_id(sender_vm_id), notifications,
J-Alves5a16c962022-03-25 12:32:51 +00004278 vcpu_id, is_per_vcpu);
4279
Karl Meakine8937d92024-03-19 16:04:25 +00004280 dlog_verbose("Set the notifications: %lx.\n", notifications);
J-Alvesaa79c012021-07-09 14:29:45 +01004281
J-Alves13394022021-06-30 13:48:49 +01004282 if ((FFA_NOTIFICATIONS_FLAG_DELAY_SRI & flags) == 0) {
4283 dlog_verbose("SRI was NOT delayed. vcpu: %u!\n",
4284 vcpu_index(current));
4285 plat_ffa_sri_trigger_not_delayed(current->cpu);
4286 } else {
4287 plat_ffa_sri_state_set(DELAYED);
4288 }
J-Alvesaa79c012021-07-09 14:29:45 +01004289
J-Alves13394022021-06-30 13:48:49 +01004290 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
J-Alvesaa79c012021-07-09 14:29:45 +01004291out:
4292 vm_unlock(&receiver_locked);
4293
4294 return ret;
4295}
4296
4297static struct ffa_value api_ffa_notification_get_success_return(
4298 ffa_notifications_bitmap_t from_sp, ffa_notifications_bitmap_t from_vm,
4299 ffa_notifications_bitmap_t from_framework)
4300{
4301 return (struct ffa_value){
4302 .func = FFA_SUCCESS_32,
4303 .arg1 = 0U,
4304 .arg2 = (uint32_t)from_sp,
4305 .arg3 = (uint32_t)(from_sp >> 32),
4306 .arg4 = (uint32_t)from_vm,
4307 .arg5 = (uint32_t)(from_vm >> 32),
4308 .arg6 = (uint32_t)from_framework,
4309 .arg7 = (uint32_t)(from_framework >> 32),
4310 };
4311}
4312
J-Alves19e20cf2023-08-02 12:48:55 +01004313struct ffa_value api_ffa_notification_get(ffa_id_t receiver_vm_id,
J-Alvesaa79c012021-07-09 14:29:45 +01004314 ffa_vcpu_index_t vcpu_id,
4315 uint32_t flags, struct vcpu *current)
4316{
J-Alves663682a2022-03-25 13:56:51 +00004317 ffa_notifications_bitmap_t framework_notifications = 0;
J-Alvesaa79c012021-07-09 14:29:45 +01004318 ffa_notifications_bitmap_t sp_notifications = 0;
4319 ffa_notifications_bitmap_t vm_notifications = 0;
4320 struct vm_locked receiver_locked;
4321 struct ffa_value ret;
J-Alvesfc95a302022-04-22 14:18:23 +01004322 const uint32_t flags_mbz = ~(FFA_NOTIFICATION_FLAG_BITMAP_HYP |
4323 FFA_NOTIFICATION_FLAG_BITMAP_SPM |
4324 FFA_NOTIFICATION_FLAG_BITMAP_SP |
4325 FFA_NOTIFICATION_FLAG_BITMAP_VM);
4326
4327 /* The FF-A v1.1 EAC0 specification states bits [31:4] Must Be Zero. */
4328 if ((flags & flags_mbz) != 0U) {
4329 dlog_verbose(
4330 "Invalid flags bit(s) set in notifications get. [31:4] "
4331 "MBZ(%x)\n",
4332 flags);
4333 return ffa_error(FFA_INVALID_PARAMETERS);
4334 }
J-Alvesaa79c012021-07-09 14:29:45 +01004335
4336 /*
J-Alvesfc95a302022-04-22 14:18:23 +01004337 * Following check should capture wrong uses of the interface,
4338 * depending on whether Hafnium is SPMC or hypervisor. On the
4339 * rest of the function it is assumed this condition is met.
J-Alvesaa79c012021-07-09 14:29:45 +01004340 */
J-Alvesfc95a302022-04-22 14:18:23 +01004341 if (!plat_ffa_is_notification_get_valid(current, receiver_vm_id,
4342 flags)) {
J-Alvesaa79c012021-07-09 14:29:45 +01004343 dlog_verbose("Invalid use of notifications get interface.\n");
4344 return ffa_error(FFA_INVALID_PARAMETERS);
4345 }
4346
4347 /*
4348 * This check assumes receiver is the current VM, and has been enforced
4349 * by `plat_ffa_is_notifications_get_valid`.
4350 */
4351 receiver_locked = plat_ffa_vm_find_locked(receiver_vm_id);
4352
4353 /*
4354 * `plat_ffa_is_notifications_get_valid` ensures following is never
4355 * true.
4356 */
4357 CHECK(receiver_locked.vm != NULL);
4358
4359 if (receiver_locked.vm->vcpu_count <= vcpu_id ||
Karl Meakin82041822024-05-20 11:14:34 +01004360 (vm_is_mp(current->vm) && cpu_index(current->cpu) != vcpu_id)) {
J-Alves1abb3342022-01-05 11:59:10 +00004361 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +00004362 "Invalid VCPU ID %u. vcpu count %u current core: "
4363 "%zu!\n",
J-Alves1abb3342022-01-05 11:59:10 +00004364 vcpu_id, receiver_locked.vm->vcpu_count,
4365 cpu_index(current->cpu));
J-Alvesaa79c012021-07-09 14:29:45 +01004366 ret = ffa_error(FFA_INVALID_PARAMETERS);
4367 goto out;
4368 }
4369
4370 if ((flags & FFA_NOTIFICATION_FLAG_BITMAP_SP) != 0U) {
J-Alves98ff9562021-09-09 14:39:41 +01004371 if (!plat_ffa_notifications_get_from_sp(
4372 receiver_locked, vcpu_id, &sp_notifications,
4373 &ret)) {
4374 dlog_verbose("Failed to get notifications from sps.");
4375 goto out;
4376 }
J-Alvesaa79c012021-07-09 14:29:45 +01004377 }
4378
4379 if ((flags & FFA_NOTIFICATION_FLAG_BITMAP_VM) != 0U) {
J-Alves5136dda2022-03-25 12:26:38 +00004380 vm_notifications = vm_notifications_partition_get_pending(
J-Alvesaa79c012021-07-09 14:29:45 +01004381 receiver_locked, true, vcpu_id);
4382 }
4383
J-Alvesd605a092022-03-28 14:20:48 +01004384 if ((flags & FFA_NOTIFICATION_FLAG_BITMAP_HYP) != 0U ||
4385 (flags & FFA_NOTIFICATION_FLAG_BITMAP_SPM) != 0U) {
4386 if (!plat_ffa_notifications_get_framework_notifications(
4387 receiver_locked, &framework_notifications, flags,
4388 vcpu_id, &ret)) {
4389 dlog_verbose(
4390 "Failed to get notifications from "
4391 "framework.\n");
4392 goto out;
4393 }
4394 }
4395
J-Alves663682a2022-03-25 13:56:51 +00004396 ret = api_ffa_notification_get_success_return(
4397 sp_notifications, vm_notifications, framework_notifications);
J-Alvesaa79c012021-07-09 14:29:45 +01004398
J-Alvesfe23ebe2021-10-13 16:07:07 +01004399 /*
4400 * If there are no more pending notifications, change `sri_state` to
4401 * handled.
4402 */
4403 if (vm_is_notifications_pending_count_zero()) {
4404 plat_ffa_sri_state_set(HANDLED);
4405 }
4406
J-Alves6e2abc62021-12-02 14:58:56 +00004407 if (!receiver_locked.vm->el0_partition &&
4408 !vm_are_global_notifications_pending(receiver_locked)) {
4409 vm_notifications_set_npi_injected(receiver_locked, false);
4410 }
4411
J-Alvesaa79c012021-07-09 14:29:45 +01004412out:
4413 vm_unlock(&receiver_locked);
4414
4415 return ret;
4416}
J-Alvesc8e8a222021-06-08 17:33:52 +01004417
4418/**
4419 * Prepares successful return for FFA_NOTIFICATION_INFO_GET, as described by
4420 * the section 17.7.1 of the FF-A v1.1 Beta0 specification.
4421 */
4422static struct ffa_value api_ffa_notification_info_get_success_return(
4423 const uint16_t *ids, uint32_t ids_count, const uint32_t *lists_sizes,
J-Alvesfe23ebe2021-10-13 16:07:07 +01004424 uint32_t lists_count)
J-Alvesc8e8a222021-06-08 17:33:52 +01004425{
4426 struct ffa_value ret = (struct ffa_value){.func = FFA_SUCCESS_64};
4427
4428 /*
4429 * Copying content of ids into ret structure. Use 5 registers (x3-x7) to
4430 * hold the list of ids.
4431 */
4432 memcpy_s(&ret.arg3,
4433 sizeof(ret.arg3) * FFA_NOTIFICATIONS_INFO_GET_REGS_RET, ids,
4434 sizeof(ids[0]) * ids_count);
4435
4436 /*
4437 * According to the spec x2 should have:
4438 * - Bit flagging if there are more notifications pending;
4439 * - The total number of elements (i.e. total list size);
4440 * - The number of VCPU IDs within each VM specific list.
4441 */
J-Alvesfe23ebe2021-10-13 16:07:07 +01004442 ret.arg2 = vm_notifications_pending_not_retrieved_by_scheduler()
4443 ? FFA_NOTIFICATIONS_INFO_GET_FLAG_MORE_PENDING
4444 : 0;
J-Alvesc8e8a222021-06-08 17:33:52 +01004445
4446 ret.arg2 |= (lists_count & FFA_NOTIFICATIONS_LISTS_COUNT_MASK)
4447 << FFA_NOTIFICATIONS_LISTS_COUNT_SHIFT;
4448
4449 for (unsigned int i = 0; i < lists_count; i++) {
4450 ret.arg2 |= (lists_sizes[i] & FFA_NOTIFICATIONS_LIST_SIZE_MASK)
4451 << FFA_NOTIFICATIONS_LIST_SHIFT(i + 1);
4452 }
4453
4454 return ret;
4455}
4456
4457struct ffa_value api_ffa_notification_info_get(struct vcpu *current)
4458{
4459 /*
4460 * Following set of variables should be populated with the return info.
4461 * At a successfull handling of this interface, they should be used
4462 * to populate the 'ret' structure in accordance to the table 17.29
4463 * of the FF-A v1.1 Beta0 specification.
4464 */
4465 uint16_t ids[FFA_NOTIFICATIONS_INFO_GET_MAX_IDS];
4466 uint32_t lists_sizes[FFA_NOTIFICATIONS_INFO_GET_MAX_IDS] = {0};
4467 uint32_t lists_count = 0;
4468 uint32_t ids_count = 0;
4469 bool list_is_full = false;
J-Alves13394022021-06-30 13:48:49 +01004470 struct ffa_value result;
J-Alvesc8e8a222021-06-08 17:33:52 +01004471
4472 /*
4473 * This interface can only be called at NS virtual/physical FF-A
4474 * instance by the endpoint implementing the primary scheduler and the
4475 * Hypervisor/OS kernel.
4476 * In the SPM, following check passes if call has been forwarded from
4477 * the hypervisor.
4478 */
Karl Meakin5e996992024-05-20 11:27:07 +01004479
4480 if (!vm_is_primary(current->vm)) {
J-Alvesc8e8a222021-06-08 17:33:52 +01004481 dlog_verbose(
4482 "Only the receiver's scheduler can use this "
4483 "interface\n");
4484 return ffa_error(FFA_NOT_SUPPORTED);
4485 }
4486
J-Alvesca058c22021-09-10 14:02:07 +01004487 /*
4488 * Forward call to the other world, and fill the arrays used to assemble
4489 * return.
4490 */
4491 plat_ffa_notification_info_get_forward(
4492 ids, &ids_count, lists_sizes, &lists_count,
4493 FFA_NOTIFICATIONS_INFO_GET_MAX_IDS);
4494
4495 list_is_full = ids_count == FFA_NOTIFICATIONS_INFO_GET_MAX_IDS;
4496
J-Alvesc8e8a222021-06-08 17:33:52 +01004497 /* Get notifications' info from this world */
4498 for (ffa_vm_count_t index = 0; index < vm_get_count() && !list_is_full;
4499 ++index) {
4500 struct vm_locked vm_locked = vm_lock(vm_find_index(index));
4501
4502 list_is_full = vm_notifications_info_get(
4503 vm_locked, ids, &ids_count, lists_sizes, &lists_count,
4504 FFA_NOTIFICATIONS_INFO_GET_MAX_IDS);
4505
4506 vm_unlock(&vm_locked);
4507 }
4508
4509 if (!list_is_full) {
4510 /* Grab notifications info from other world */
J-Alvesfe23ebe2021-10-13 16:07:07 +01004511 plat_ffa_vm_notifications_info_get(
J-Alvesc8e8a222021-06-08 17:33:52 +01004512 ids, &ids_count, lists_sizes, &lists_count,
4513 FFA_NOTIFICATIONS_INFO_GET_MAX_IDS);
4514 }
4515
4516 if (ids_count == 0) {
J-Alvesca058c22021-09-10 14:02:07 +01004517 dlog_verbose(
4518 "Notification info get has no data to retrieve.\n");
J-Alves13394022021-06-30 13:48:49 +01004519 result = ffa_error(FFA_NO_DATA);
4520 } else {
4521 result = api_ffa_notification_info_get_success_return(
J-Alvesfe23ebe2021-10-13 16:07:07 +01004522 ids, ids_count, lists_sizes, lists_count);
J-Alvesc8e8a222021-06-08 17:33:52 +01004523 }
4524
J-Alvesfe23ebe2021-10-13 16:07:07 +01004525 plat_ffa_sri_state_set(HANDLED);
4526
J-Alves13394022021-06-30 13:48:49 +01004527 return result;
J-Alvesc8e8a222021-06-08 17:33:52 +01004528}
Raghu Krishnamurthyea6d25f2021-09-14 15:27:06 -07004529
4530struct ffa_value api_ffa_mem_perm_get(vaddr_t base_addr, struct vcpu *current)
4531{
4532 struct vm_locked vm_locked;
4533 struct ffa_value ret = ffa_error(FFA_INVALID_PARAMETERS);
4534 bool mode_ret = false;
4535 uint32_t mode = 0;
4536
4537 if (!plat_ffa_is_mem_perm_get_valid(current)) {
J-Alves5a099172024-02-22 14:34:51 +00004538 return ffa_error(FFA_DENIED);
Raghu Krishnamurthyea6d25f2021-09-14 15:27:06 -07004539 }
4540
4541 if (!(current->vm->el0_partition)) {
4542 return ffa_error(FFA_DENIED);
4543 }
4544
4545 vm_locked = vm_lock(current->vm);
4546
4547 /*
4548 * mm_get_mode is used to check if the given base_addr page is already
4549 * mapped. If the page is unmapped, return error. If the page is mapped
4550 * appropriate attributes are returned to the caller. Note that
4551 * mm_get_mode returns true if the address is in the valid VA range as
4552 * supported by the architecture and MMU configurations, as opposed to
4553 * whether a page is mapped or not. For a page to be known as mapped,
4554 * the API must return true AND the returned mode must not have
4555 * MM_MODE_INVALID set.
4556 */
4557 mode_ret = mm_get_mode(&vm_locked.vm->ptable, base_addr,
4558 va_add(base_addr, PAGE_SIZE), &mode);
4559 if (!mode_ret || (mode & MM_MODE_INVALID)) {
4560 ret = ffa_error(FFA_INVALID_PARAMETERS);
4561 goto out;
4562 }
4563
4564 /* No memory should be marked RWX */
4565 CHECK((mode & (MM_MODE_R | MM_MODE_W | MM_MODE_X)) !=
4566 (MM_MODE_R | MM_MODE_W | MM_MODE_X));
4567
4568 /*
4569 * S-EL0 partitions are expected to have all their pages marked as
4570 * non-global.
4571 */
4572 CHECK((mode & (MM_MODE_NG | MM_MODE_USER)) ==
4573 (MM_MODE_NG | MM_MODE_USER));
4574
4575 if (mode & MM_MODE_W) {
4576 /* No memory should be writeable but not readable. */
4577 CHECK(mode & MM_MODE_R);
4578 ret = (struct ffa_value){.func = FFA_SUCCESS_32,
4579 .arg2 = (uint32_t)(FFA_MEM_PERM_RW)};
4580 } else if (mode & MM_MODE_R) {
4581 ret = (struct ffa_value){.func = FFA_SUCCESS_32,
4582 .arg2 = (uint32_t)(FFA_MEM_PERM_RX)};
4583 if (!(mode & MM_MODE_X)) {
4584 ret.arg2 = (uint32_t)(FFA_MEM_PERM_RO);
4585 }
4586 }
4587out:
4588 vm_unlock(&vm_locked);
4589 return ret;
4590}
4591
4592struct ffa_value api_ffa_mem_perm_set(vaddr_t base_addr, uint32_t page_count,
4593 uint32_t mem_perm, struct vcpu *current)
4594{
4595 struct vm_locked vm_locked;
4596 struct ffa_value ret;
4597 bool mode_ret = false;
4598 uint32_t original_mode;
4599 uint32_t new_mode;
4600 struct mpool local_page_pool;
4601
4602 if (!plat_ffa_is_mem_perm_set_valid(current)) {
J-Alves5a099172024-02-22 14:34:51 +00004603 return ffa_error(FFA_DENIED);
Raghu Krishnamurthyea6d25f2021-09-14 15:27:06 -07004604 }
4605
4606 if (!(current->vm->el0_partition)) {
4607 return ffa_error(FFA_DENIED);
4608 }
4609
4610 if (!is_aligned(va_addr(base_addr), PAGE_SIZE)) {
4611 return ffa_error(FFA_INVALID_PARAMETERS);
4612 }
4613
4614 if ((mem_perm != FFA_MEM_PERM_RW) && (mem_perm != FFA_MEM_PERM_RO) &&
4615 (mem_perm != FFA_MEM_PERM_RX)) {
4616 return ffa_error(FFA_INVALID_PARAMETERS);
4617 }
4618
4619 /*
4620 * Create a local pool so any freed memory can't be used by another
4621 * thread. This is to ensure the original mapping can be restored if any
4622 * stage of the process fails.
4623 */
4624 mpool_init_with_fallback(&local_page_pool, &api_page_pool);
4625
4626 vm_locked = vm_lock(current->vm);
4627
4628 /*
4629 * All regions accessible by the partition are mapped during boot. If we
4630 * cannot get a successful translation for the page range, the request
4631 * to change permissions is rejected.
4632 * mm_get_mode is used to check if the given address range is already
4633 * mapped. If the range is unmapped, return error. If the range is
4634 * mapped appropriate attributes are returned to the caller. Note that
4635 * mm_get_mode returns true if the address is in the valid VA range as
4636 * supported by the architecture and MMU configurations, as opposed to
4637 * whether a page is mapped or not. For a page to be known as mapped,
4638 * the API must return true AND the returned mode must not have
4639 * MM_MODE_INVALID set.
4640 */
4641
4642 mode_ret = mm_get_mode(&vm_locked.vm->ptable, base_addr,
4643 va_add(base_addr, page_count * PAGE_SIZE),
4644 &original_mode);
4645 if (!mode_ret || (original_mode & MM_MODE_INVALID)) {
4646 ret = ffa_error(FFA_INVALID_PARAMETERS);
4647 goto out;
4648 }
4649
4650 /* Device memory cannot be marked as executable */
4651 if ((original_mode & MM_MODE_D) && (mem_perm == FFA_MEM_PERM_RX)) {
4652 ret = ffa_error(FFA_INVALID_PARAMETERS);
4653 goto out;
4654 }
4655
4656 new_mode = MM_MODE_USER | MM_MODE_NG;
4657
4658 if (mem_perm == FFA_MEM_PERM_RW) {
4659 new_mode |= MM_MODE_R | MM_MODE_W;
4660 } else if (mem_perm == FFA_MEM_PERM_RX) {
4661 new_mode |= MM_MODE_R | MM_MODE_X;
4662 } else if (mem_perm == FFA_MEM_PERM_RO) {
4663 new_mode |= MM_MODE_R;
4664 }
4665
4666 /*
4667 * Safe to re-map memory, since we know the requested permissions are
4668 * valid, and the memory requested to be re-mapped is also valid.
4669 */
4670 if (!mm_identity_prepare(
4671 &vm_locked.vm->ptable, pa_from_va(base_addr),
4672 pa_from_va(va_add(base_addr, page_count * PAGE_SIZE)),
4673 new_mode, &local_page_pool)) {
4674 /*
4675 * Defrag the table into the local page pool.
4676 * mm_identity_prepare could have allocated or freed pages to
4677 * split blocks or tables etc.
4678 */
4679 mm_stage1_defrag(&vm_locked.vm->ptable, &local_page_pool);
4680
4681 /*
4682 * Guaranteed to succeed mapping with old mode since the mapping
4683 * with old mode already existed and we have a local page pool
4684 * that should have sufficient memory to go back to the original
4685 * state.
4686 */
4687 CHECK(mm_identity_prepare(
4688 &vm_locked.vm->ptable, pa_from_va(base_addr),
4689 pa_from_va(va_add(base_addr, page_count * PAGE_SIZE)),
4690 original_mode, &local_page_pool));
4691 mm_identity_commit(
4692 &vm_locked.vm->ptable, pa_from_va(base_addr),
4693 pa_from_va(va_add(base_addr, page_count * PAGE_SIZE)),
4694 original_mode, &local_page_pool);
4695
4696 mm_stage1_defrag(&vm_locked.vm->ptable, &api_page_pool);
4697 ret = ffa_error(FFA_NO_MEMORY);
4698 goto out;
4699 }
4700
4701 mm_identity_commit(
4702 &vm_locked.vm->ptable, pa_from_va(base_addr),
4703 pa_from_va(va_add(base_addr, page_count * PAGE_SIZE)), new_mode,
4704 &local_page_pool);
4705
4706 ret = (struct ffa_value){.func = FFA_SUCCESS_32};
4707
4708out:
4709 mpool_fini(&local_page_pool);
4710 vm_unlock(&vm_locked);
4711
4712 return ret;
4713}
Maksims Svecovs71b76702022-05-20 15:32:58 +01004714
4715/**
Karl Meakinc5cebbc2024-06-17 11:30:27 +01004716 * Send the contents of the given vCPU's log buffer to the log, preceded
Karl Meakin705b56e2024-05-22 10:04:32 +01004717 * by the VM ID and followed by a newline.
4718 */
Karl Meakin6f1f1212024-07-16 10:18:16 +01004719void api_flush_log_buffer(struct vcpu_locked *vcpu_locked)
Karl Meakin705b56e2024-05-22 10:04:32 +01004720{
Karl Meakin6f1f1212024-07-16 10:18:16 +01004721 /*
4722 * NOTE: This line is parsed by `hftest.py`.
4723 * If you change the format, make sure to update
4724 * `HFTEST_CTRL_JSON_REGEX` as well.
4725 */
4726 struct vcpu *vcpu = vcpu_locked->vcpu;
4727 struct log_buffer *buffer = &vcpu->log_buffer;
4728 ffa_id_t vm_id = vcpu->vm->id;
4729 ffa_id_t vcpu_id = vcpu_index(vcpu);
4730
Karl Meakin7de26952024-06-14 14:50:19 +01004731 buffer->chars[buffer->len] = '\0';
Olivier Deprezbd432092024-07-26 14:01:12 +02004732 dlog("[%x %u] %s\n", vm_id, vcpu_id, buffer->chars);
Karl Meakin7de26952024-06-14 14:50:19 +01004733 buffer->len = 0;
Karl Meakin705b56e2024-05-22 10:04:32 +01004734}
4735
4736/**
Karl Meakin740f74f2024-02-14 18:04:48 +00004737 * Implements FF-A v1.2 FFA_CONSOLE_LOG ABI for buffered logging.
Maksims Svecovs71b76702022-05-20 15:32:58 +01004738 */
4739struct ffa_value api_ffa_console_log(const struct ffa_value args,
4740 struct vcpu *current)
4741{
Karl Meakin740f74f2024-02-14 18:04:48 +00004742 /* Maximum number of characters is 128: 16 registers of 8 bytes each. */
4743 char chars[128] = {0};
Karl Meakin0e617d92024-04-05 12:55:22 +01004744 const bool v1_2 = current->vm->ffa_version >= FFA_VERSION_1_2;
Karl Meakin740f74f2024-02-14 18:04:48 +00004745 const bool log32 = args.func == FFA_CONSOLE_LOG_32;
Maksims Svecovs71b76702022-05-20 15:32:58 +01004746
Karl Meakin740f74f2024-02-14 18:04:48 +00004747 /*
4748 * 32bit: always 6 registers
4749 * 64bit and less than v1.2: 6 registers
4750 * 64bit and v1.2 or greater: 16 registers
4751 */
Karl Meakin66a38bd2024-05-28 16:00:56 +01004752 /* NOLINTNEXTLINE(readability-avoid-nested-conditional-operator) */
Karl Meakin740f74f2024-02-14 18:04:48 +00004753 const size_t registers_max = log32 ? 6 : (v1_2 ? 16 : 6);
4754 const size_t chars_max =
4755 registers_max * (log32 ? sizeof(uint32_t) : sizeof(uint64_t));
4756 const size_t chars_count = args.arg1;
Karl Meakinc5cebbc2024-06-17 11:30:27 +01004757 struct vcpu_locked vcpu_locked;
4758 struct log_buffer *log_buffer;
Karl Meakin740f74f2024-02-14 18:04:48 +00004759
4760 assert(args.func == FFA_CONSOLE_LOG_32 ||
4761 args.func == FFA_CONSOLE_LOG_64);
4762
4763 if (chars_count == 0 || chars_count > chars_max) {
Maksims Svecovs71b76702022-05-20 15:32:58 +01004764 return ffa_error(FFA_INVALID_PARAMETERS);
4765 }
4766
Karl Meakin740f74f2024-02-14 18:04:48 +00004767 if (log32) {
4768 uint32_t *registers = (uint32_t *)chars;
Maksims Svecovs71b76702022-05-20 15:32:58 +01004769
Karl Meakin740f74f2024-02-14 18:04:48 +00004770 registers[0] = args.arg2 & 0xffffffff;
4771 registers[1] = args.arg3 & 0xffffffff;
4772 registers[2] = args.arg4 & 0xffffffff;
4773 registers[3] = args.arg5 & 0xffffffff;
4774 registers[4] = args.arg6 & 0xffffffff;
4775 registers[5] = args.arg7 & 0xffffffff;
4776 } else {
4777 uint64_t *registers = (uint64_t *)chars;
4778
4779 registers[0] = args.arg2;
4780 registers[1] = args.arg3;
4781 registers[2] = args.arg4;
4782 registers[3] = args.arg5;
4783 registers[4] = args.arg6;
4784 registers[5] = args.arg7;
4785 if (v1_2) {
4786 registers[6] = args.extended_val.arg8;
4787 registers[7] = args.extended_val.arg9;
4788 registers[8] = args.extended_val.arg10;
4789 registers[9] = args.extended_val.arg11;
4790 registers[10] = args.extended_val.arg12;
4791 registers[11] = args.extended_val.arg13;
4792 registers[12] = args.extended_val.arg14;
4793 registers[13] = args.extended_val.arg15;
4794 registers[14] = args.extended_val.arg16;
4795 registers[15] = args.extended_val.arg17;
4796 }
4797 }
4798
Karl Meakinc5cebbc2024-06-17 11:30:27 +01004799 vcpu_locked = vcpu_lock(current);
4800 log_buffer = &current->log_buffer;
Karl Meakin7de26952024-06-14 14:50:19 +01004801
Karl Meakin740f74f2024-02-14 18:04:48 +00004802 for (size_t i = 0; i < chars_count; i++) {
4803 bool flush = false;
4804 const char c = chars[i];
4805
4806 if (c == '\n' || c == '\0') {
4807 flush = true;
4808 } else {
Karl Meakin7de26952024-06-14 14:50:19 +01004809 log_buffer->chars[log_buffer->len++] = c;
4810 flush = log_buffer->len == LOG_BUFFER_SIZE;
Karl Meakin740f74f2024-02-14 18:04:48 +00004811 }
4812
4813 if (flush) {
Karl Meakin6f1f1212024-07-16 10:18:16 +01004814 api_flush_log_buffer(&vcpu_locked);
Karl Meakin740f74f2024-02-14 18:04:48 +00004815 }
4816 }
Maksims Svecovs71b76702022-05-20 15:32:58 +01004817
Karl Meakinc5cebbc2024-06-17 11:30:27 +01004818 vcpu_unlock(&vcpu_locked);
Maksims Svecovs71b76702022-05-20 15:32:58 +01004819 return (struct ffa_value){.func = FFA_SUCCESS_32};
4820}