blob: 601d3ed13f2a290a0cad72d959fd1788857aaf28 [file] [log] [blame]
Andrew Scull18834872018-10-12 11:48:09 +01001/*
Andrew Walbran692b3252019-03-07 15:51:31 +00002 * Copyright 2018 The Hafnium Authors.
Andrew Scull18834872018-10-12 11:48:09 +01003 *
Andrew Walbrane959ec12020-06-17 15:01:09 +01004 * Use of this source code is governed by a BSD-style
5 * license that can be found in the LICENSE file or at
6 * https://opensource.org/licenses/BSD-3-Clause.
Andrew Scull18834872018-10-12 11:48:09 +01007 */
8
Andrew Scullc960c032018-10-24 15:13:35 +01009#include <stdnoreturn.h>
10
Andrew Walbran1f32e722019-06-07 17:57:26 +010011#include "hf/arch/barriers.h"
Madhukar Pappireddy77d3bcd2023-03-01 17:26:22 -060012#include "hf/arch/gicv3.h"
Madhukar Pappireddya3787c92024-09-25 14:50:36 -050013#include "hf/arch/host_timer.h"
Andrew Scullc960c032018-10-24 15:13:35 +010014#include "hf/arch/init.h"
J-Alvesa2d1c3b2024-03-28 12:46:58 +000015#include "hf/arch/memcpy_trapped.h"
Olivier Deprez98ad2d22020-05-20 09:52:43 +020016#include "hf/arch/mmu.h"
Maksims Svecovs9ddf86a2021-05-06 17:17:21 +010017#include "hf/arch/plat/ffa.h"
Karl Meakin9724b362024-10-15 14:35:02 +010018#include "hf/arch/plat/ffa/indirect_messaging.h"
Karl Meakin7a664f62024-07-24 17:20:29 +010019#include "hf/arch/plat/ffa/notifications.h"
Karl Meakin5b2da502024-11-07 17:13:51 +000020#include "hf/arch/plat/ffa/vm.h"
Andrew Scull07b6bd32019-12-12 17:19:55 +000021#include "hf/arch/plat/smc.h"
Madhukar Pappireddya3787c92024-09-25 14:50:36 -050022#include "hf/arch/timer.h"
J-Alves03edf402023-07-21 15:13:49 +010023#include "hf/arch/vmid_base.h"
Andrew Scullc960c032018-10-24 15:13:35 +010024
Andrew Scull18c78fc2018-08-20 12:57:41 +010025#include "hf/api.h"
Fuad Tabbac76466d2019-09-06 10:42:12 +010026#include "hf/check.h"
Andrew Scull18c78fc2018-08-20 12:57:41 +010027#include "hf/cpu.h"
28#include "hf/dlog.h"
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010029#include "hf/ffa.h"
J-Alvesb37fd082020-10-22 12:29:21 +010030#include "hf/ffa_internal.h"
Daniel Boulbyf3cf28c2024-08-22 10:46:23 +010031#include "hf/hf_ipi.h"
Andrew Sculla9c172d2019-04-03 14:10:00 +010032#include "hf/panic.h"
Manish Pandeya5f39fb2020-09-11 09:47:11 +010033#include "hf/plat/interrupts.h"
Madhukar Pappireddya3787c92024-09-25 14:50:36 -050034#include "hf/timer_mgmt.h"
Andrew Scull18c78fc2018-08-20 12:57:41 +010035#include "hf/vm.h"
Karl Meakind0356f82024-09-04 13:34:31 +010036#include "hf/vm_ids.h"
Andrew Scull18c78fc2018-08-20 12:57:41 +010037
Andrew Scullf35a5c92018-08-07 18:09:46 +010038#include "vmapi/hf/call.h"
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +010039
Fuad Tabbac76466d2019-09-06 10:42:12 +010040#include "debug_el1.h"
Madhukar Pappireddyf684d192024-09-25 14:35:57 -050041#include "el1_physical_timer.h"
Fuad Tabba77a4b012019-11-15 12:13:08 +000042#include "feature_id.h"
Fuad Tabbaf1d6dc52019-09-18 17:33:14 +010043#include "perfmon.h"
Andrew Scull18c78fc2018-08-20 12:57:41 +010044#include "psci.h"
Andrew Walbran33645652019-04-15 12:29:31 +010045#include "psci_handler.h"
Andrew Scull7fd4bb72018-12-08 23:40:12 +000046#include "smc.h"
Fuad Tabbaba8c44d2019-09-23 14:38:58 +010047#include "sysregs.h"
Karl Meakin5a133552024-05-30 16:06:27 +010048#include "sysregs_defs.h"
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +010049
Fuad Tabbac76466d2019-09-06 10:42:12 +010050/**
Olivier Deprez98ad2d22020-05-20 09:52:43 +020051 * Hypervisor Fault Address Register Non-Secure.
52 */
53#define HPFAR_EL2_NS (UINT64_C(0x1) << 63)
54
55/**
56 * Hypervisor Fault Address Register Faulting IPA.
57 */
58#define HPFAR_EL2_FIPA (UINT64_C(0xFFFFFFFFFF0))
59
60/**
Fuad Tabbac76466d2019-09-06 10:42:12 +010061 * Gets the value to increment for the next PC.
62 * The ESR encodes whether the instruction is 2 bytes or 4 bytes long.
63 */
Fuad Tabba3e9b0222019-11-11 16:47:50 +000064#define GET_NEXT_PC_INC(esr) (GET_ESR_IL(esr) ? 4 : 2)
Fuad Tabbac76466d2019-09-06 10:42:12 +010065
Fuad Tabbac76466d2019-09-06 10:42:12 +010066/**
Andrew Walbran0dd67ff2019-09-12 16:38:50 +010067 * The Client ID field within X7 for an SMC64 call.
68 */
69#define CLIENT_ID_MASK UINT64_C(0xffff)
70
Karl Meakind0356f82024-09-04 13:34:31 +010071/**
72 * Identifies SPMD specific framework messages. See section 18.2 of v1.2 FF-A
73 * specification.
Daniel Boulbyefa381f2022-01-18 14:49:40 +000074 */
Karl Meakind0356f82024-09-04 13:34:31 +010075enum ffa_spmd_framework_msg_func {
76 SPMD_FRAMEWORK_MSG_PSCI_REQ = 0,
77 SPMD_FRAMEWORK_MSG_PSCI_RESP = 2,
78
79 SPMD_FRAMEWORK_MSG_FFA_VERSION_REQ = 8,
80 SPMD_FRAMEWORK_MSG_FFA_VERSION_RESP = 9,
81};
Daniel Boulbyefa381f2022-01-18 14:49:40 +000082
Andrew Walbran0dd67ff2019-09-12 16:38:50 +010083/**
Fuad Tabbac76466d2019-09-06 10:42:12 +010084 * Returns a reference to the currently executing vCPU.
85 */
Andrew Scullc960c032018-10-24 15:13:35 +010086static struct vcpu *current(void)
Andrew Walbran3d84a262018-12-13 14:41:19 +000087{
Daniel Boulby3f784262021-09-27 13:02:54 +010088 // NOLINTNEXTLINE(performance-no-int-to-ptr)
Andrew Walbran3d84a262018-12-13 14:41:19 +000089 return (struct vcpu *)read_msr(tpidr_el2);
90}
91
Andrew Walbran1f8d4872018-12-20 11:21:32 +000092/**
Madhukar Pappireddyd3ac7382024-09-25 14:29:03 -050093 * Saves the state of per-vCPU peripherals, such as the arch timer, and
Andrew Walbran1f8d4872018-12-20 11:21:32 +000094 * informs the arch-independent sections that registers have been saved.
95 */
96void complete_saving_state(struct vcpu *vcpu)
97{
Madhukar Pappireddya3787c92024-09-25 14:50:36 -050098 host_timer_save_arch_timer(&vcpu->regs.arch_timer);
99
100 timer_vcpu_manage(vcpu);
Andrew Walbran1f8d4872018-12-20 11:21:32 +0000101 api_regs_state_saved(vcpu);
Madhukar Pappireddya3787c92024-09-25 14:50:36 -0500102
103 /*
104 * Since switching away from current vCPU, disable the host physical
105 * timer for now. If necessary, the host timer will be reconfigured
106 * at appropriate time to track timer deadline of the vCPU.
107 */
108 host_timer_disable();
Andrew Walbran1f8d4872018-12-20 11:21:32 +0000109}
110
111/**
Madhukar Pappireddyd3ac7382024-09-25 14:29:03 -0500112 * Restores the state of per-vCPU peripherals, such as the arch timer.
Andrew Walbran1f8d4872018-12-20 11:21:32 +0000113 */
114void begin_restoring_state(struct vcpu *vcpu)
115{
Madhukar Pappireddya3787c92024-09-25 14:50:36 -0500116 /*
117 * If a vCPU's timer has expired while it was de-scheduled, SPMC will
118 * inject the virtual timer interrupt before resuming the vCPU.
119 * If not, there is a live state and we need to configure the host timer
120 * to track it again.
121 */
122 if (arch_timer_enabled(&vcpu->regs) &&
123 (arch_timer_remaining_ns(&vcpu->regs) != 0)) {
124 host_timer_track_deadline(&vcpu->regs.arch_timer);
125 }
Andrew Walbran1f8d4872018-12-20 11:21:32 +0000126}
127
Andrew Walbran1f32e722019-06-07 17:57:26 +0100128/**
Andrew Walbran1f32e722019-06-07 17:57:26 +0100129 * Invalidate all stage 1 TLB entries on the current (physical) CPU for the
130 * current VMID.
131 */
132static void invalidate_vm_tlb(void)
133{
Andrew Walbrancff1f682019-07-04 14:52:45 +0100134 /*
135 * Ensure that the last VTTBR write has taken effect so we invalidate
136 * the right set of TLB entries.
137 */
Andrew Walbran1f32e722019-06-07 17:57:26 +0100138 isb();
Andrew Walbrancff1f682019-07-04 14:52:45 +0100139
Olivier Deprez0b0ba8c2023-03-17 11:11:53 +0100140 tlbi(vmalle1);
Andrew Walbrancff1f682019-07-04 14:52:45 +0100141
142 /*
143 * Ensure that no instructions are fetched for the VM until after the
144 * TLB invalidation has taken effect.
145 */
Andrew Walbran1f32e722019-06-07 17:57:26 +0100146 isb();
Andrew Walbrancff1f682019-07-04 14:52:45 +0100147
148 /*
149 * Ensure that no data reads or writes for the VM happen until after the
Fuad Tabba77a4b012019-11-15 12:13:08 +0000150 * TLB invalidation has taken effect. Non-shareable is enough because
151 * the TLB is local to the CPU.
Andrew Walbrancff1f682019-07-04 14:52:45 +0100152 */
David Brazdil851948e2019-08-09 12:02:12 +0100153 dsb(nsh);
Andrew Walbran1f32e722019-06-07 17:57:26 +0100154}
155
156/**
157 * Invalidates the TLB if a different vCPU is being run than the last vCPU of
158 * the same VM which was run on the current pCPU.
159 *
160 * This is necessary because VMs may (contrary to the architecture
161 * specification) use inconsistent ASIDs across vCPUs. c.f. KVM's similar
162 * workaround:
163 * https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=94d0e5980d6791b9
164 */
165void maybe_invalidate_tlb(struct vcpu *vcpu)
166{
167 size_t current_cpu_index = cpu_index(vcpu->cpu);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100168 ffa_vcpu_index_t new_vcpu_index = vcpu_index(vcpu);
Andrew Walbran1f32e722019-06-07 17:57:26 +0100169
170 if (vcpu->vm->arch.last_vcpu_on_cpu[current_cpu_index] !=
171 new_vcpu_index) {
172 /*
173 * The vCPU has changed since the last time this VM was run on
174 * this pCPU, so we need to invalidate the TLB.
175 */
176 invalidate_vm_tlb();
177
178 /* Record the fact that this vCPU is now running on this CPU. */
179 vcpu->vm->arch.last_vcpu_on_cpu[current_cpu_index] =
180 new_vcpu_index;
181 }
182}
183
David Brazdil768f69c2019-12-19 15:46:12 +0000184noreturn void irq_current_exception_noreturn(uintreg_t elr, uintreg_t spsr)
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +0100185{
Wedson Almeida Filho9d5040f2018-10-29 08:41:27 +0000186 (void)elr;
187 (void)spsr;
188
Fuad Tabbad1d67982020-01-08 11:28:29 +0000189 panic("IRQ from current exception level.");
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +0100190}
191
David Brazdil768f69c2019-12-19 15:46:12 +0000192noreturn void fiq_current_exception_noreturn(uintreg_t elr, uintreg_t spsr)
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +0100193{
Wedson Almeida Filho9d5040f2018-10-29 08:41:27 +0000194 (void)elr;
195 (void)spsr;
196
Fuad Tabbad1d67982020-01-08 11:28:29 +0000197 panic("FIQ from current exception level.");
Wedson Almeida Filho9d5040f2018-10-29 08:41:27 +0000198}
199
David Brazdil768f69c2019-12-19 15:46:12 +0000200noreturn void serr_current_exception_noreturn(uintreg_t elr, uintreg_t spsr)
Wedson Almeida Filho9d5040f2018-10-29 08:41:27 +0000201{
202 (void)elr;
203 (void)spsr;
204
Fuad Tabbad1d67982020-01-08 11:28:29 +0000205 panic("SError from current exception level.");
Wedson Almeida Filho9d5040f2018-10-29 08:41:27 +0000206}
207
J-Alvesa2d1c3b2024-03-28 12:46:58 +0000208/**
209 * Returns true if ELR_EL2 is not to be restored from stack.
210 * Currently function doesn't return false, as for all other cases
211 * panics.
212 */
213bool sync_current_exception(uintreg_t elr, uintreg_t spsr)
Wedson Almeida Filho9d5040f2018-10-29 08:41:27 +0000214{
215 uintreg_t esr = read_msr(esr_el2);
Fuad Tabba3e9b0222019-11-11 16:47:50 +0000216 uintreg_t ec = GET_ESR_EC(esr);
Wedson Almeida Filho9d5040f2018-10-29 08:41:27 +0000217 (void)spsr;
218
Fuad Tabbac76466d2019-09-06 10:42:12 +0100219 switch (ec) {
J-Alvesa2d1c3b2024-03-28 12:46:58 +0000220 case EC_DATA_ABORT_SAME_EL: {
221 uint64_t iss = GET_ESR_ISS(esr);
222 uint64_t dfsc = GET_ESR_ISS_DFSC(iss);
223 uint64_t far = read_msr(far_el2);
224
225 /* Handle Granule Protection Fault. */
226 if (is_arch_feat_rme_supported() && dfsc == DFSC_GPF) {
227 dlog_verbose(
Karl Meakine8937d92024-03-19 16:04:25 +0000228 "Granule Protection Fault: esr=%#lx, ec=%#lx, "
229 "far=%#lx, elr=%#lx\n",
J-Alvesa2d1c3b2024-03-28 12:46:58 +0000230 esr, ec, far, elr);
231
232 /*
233 * Change ELR_EL2 only if failed whilst either
234 * reading or writing within 'memcpy_trapped'.
235 */
236 if (elr == (uintptr_t)memcpy_trapped_read ||
237 elr == (uintptr_t)memcpy_trapped_write) {
238 dlog_verbose(
239 "GPF due to data abort on %s.\n",
240 (elr == (uintptr_t)memcpy_trapped_read)
241 ? "read"
242 : "write");
243
244 /*
245 * Update the ELR_EL2 with the return
246 * address, to return error from the
247 * call to 'memcpy_trapped'.
248 */
249 write_msr(ELR_EL2, memcpy_trapped_aborted);
250 return true;
251 }
252 }
253
Kathleen Capellad1c34b52024-04-01 21:27:15 -0400254#if ENABLE_MTE
255 if (dfsc == DFSC_SYNC_TAG_CHECK_FAULT) {
256 dlog_error(
257 "Data abort due to synchronous tag check "
258 "fault: pc=%#lx, esr=%#lx, ec=%#lx, "
259 "far=%#lx, dfsc = %#lx\n",
260 elr, esr, ec, far, dfsc);
261 }
262 break;
263#endif
Karl Meakin5a133552024-05-30 16:06:27 +0100264 if (!GET_ESR_FNV(esr)) {
Andrew Walbran17eebf92020-02-05 16:35:49 +0000265 dlog_error(
Karl Meakine8937d92024-03-19 16:04:25 +0000266 "Data abort: pc=%#lx, esr=%#lx, ec=%#lx, "
267 "far=%#lx\n",
J-Alvesa2d1c3b2024-03-28 12:46:58 +0000268 elr, esr, ec, far);
269
Andrew Scull7364a8e2018-07-19 15:39:29 +0100270 } else {
Andrew Walbran17eebf92020-02-05 16:35:49 +0000271 dlog_error(
Karl Meakine8937d92024-03-19 16:04:25 +0000272 "Data abort: pc=%#lx, esr=%#lx, ec=%#lx, "
Andrew Walbran17eebf92020-02-05 16:35:49 +0000273 "far=invalid\n",
274 elr, esr, ec);
Andrew Scull7364a8e2018-07-19 15:39:29 +0100275 }
J-Alvesa2d1c3b2024-03-28 12:46:58 +0000276 } break;
Wedson Almeida Filhofed69022018-07-11 15:39:12 +0100277 default:
Andrew Walbran17eebf92020-02-05 16:35:49 +0000278 dlog_error(
Karl Meakine8937d92024-03-19 16:04:25 +0000279 "Unknown current sync exception pc=%#lx, esr=%#lx, "
280 "ec=%#lx\n",
Andrew Walbran17eebf92020-02-05 16:35:49 +0000281 elr, esr, ec);
Andrew Scullc960c032018-10-24 15:13:35 +0100282 break;
Wedson Almeida Filhofed69022018-07-11 15:39:12 +0100283 }
Wedson Almeida Filho81568c42019-01-04 13:33:02 +0000284
Andrew Sculla9c172d2019-04-03 14:10:00 +0100285 panic("EL2 exception");
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +0100286}
287
Wedson Almeida Filho03e767a2018-07-30 15:32:03 +0100288/**
Manish Pandey35e452f2021-02-18 21:36:34 +0000289 * Sets or clears the VF bit in the HCR_EL2 register saved in the given
290 * arch_regs.
291 */
292static void set_virtual_fiq(struct arch_regs *r, bool enable)
293{
294 if (enable) {
Olivier Deprez6d408f92022-08-08 19:14:23 +0200295 r->hyp_state.hcr_el2 |= HCR_EL2_VF;
Manish Pandey35e452f2021-02-18 21:36:34 +0000296 } else {
Olivier Deprez6d408f92022-08-08 19:14:23 +0200297 r->hyp_state.hcr_el2 &= ~HCR_EL2_VF;
Manish Pandey35e452f2021-02-18 21:36:34 +0000298 }
299}
300
301/**
J-Alves6f6bf8a2024-07-25 15:17:57 +0100302 * Sets or clears the VI bit in the HCR_EL2 register saved in the given
303 * arch_regs.
Manish Pandey35e452f2021-02-18 21:36:34 +0000304 */
J-Alves6f6bf8a2024-07-25 15:17:57 +0100305static void set_virtual_irq(struct arch_regs *r, bool enable)
Manish Pandey35e452f2021-02-18 21:36:34 +0000306{
Manish Pandey35e452f2021-02-18 21:36:34 +0000307 if (enable) {
J-Alves6f6bf8a2024-07-25 15:17:57 +0100308 r->hyp_state.hcr_el2 |= HCR_EL2_VI;
Manish Pandey35e452f2021-02-18 21:36:34 +0000309 } else {
J-Alves6f6bf8a2024-07-25 15:17:57 +0100310 r->hyp_state.hcr_el2 &= ~HCR_EL2_VI;
Manish Pandey35e452f2021-02-18 21:36:34 +0000311 }
Manish Pandey35e452f2021-02-18 21:36:34 +0000312}
313
J-Alvesb37fd082020-10-22 12:29:21 +0100314#if SECURE_WORLD == 1
Max Shvetsov1ae74f12020-09-18 13:52:20 +0100315/**
Karl Meakind0356f82024-09-04 13:34:31 +0100316 * Handle special direct messages from SPMD to SPMC.
Max Shvetsov1ae74f12020-09-18 13:52:20 +0100317 */
318static bool spmd_handler(struct ffa_value *args, struct vcpu *current)
319{
J-Alves19e20cf2023-08-02 12:48:55 +0100320 ffa_id_t sender = ffa_sender(*args);
321 ffa_id_t receiver = ffa_receiver(*args);
322 ffa_id_t current_vm_id = current->vm->id;
Karl Meakind0356f82024-09-04 13:34:31 +0100323 enum ffa_spmd_framework_msg_func func =
324 (enum ffa_spmd_framework_msg_func)ffa_framework_msg_func(*args);
Max Shvetsov1ae74f12020-09-18 13:52:20 +0100325
326 /*
Daniel Boulbyefa381f2022-01-18 14:49:40 +0000327 * Check if direct message request is originating from the SPMD,
328 * directed to the SPMC and the message is a framework message.
Max Shvetsov1ae74f12020-09-18 13:52:20 +0100329 */
330 if (!(sender == HF_SPMD_VM_ID && receiver == HF_SPMC_VM_ID &&
Karl Meakind0356f82024-09-04 13:34:31 +0100331 current_vm_id == HF_OTHER_WORLD_ID &&
332 ffa_is_framework_msg(*args))) {
Max Shvetsov1ae74f12020-09-18 13:52:20 +0100333 return false;
334 }
335
Olivier Depreza67ab882023-01-10 15:00:54 +0100336 /*
337 * The framework message is conveyed by EL3/SPMD to SPMC so the
338 * current VM id must match to the other world VM id.
339 */
340 CHECK(current->vm->id == HF_HYPERVISOR_VM_ID);
341
Karl Meakind0356f82024-09-04 13:34:31 +0100342 switch (func) {
343 case SPMD_FRAMEWORK_MSG_PSCI_REQ: {
344 enum psci_return_code psci_msg_response =
345 PSCI_ERROR_NOT_SUPPORTED;
Olivier Deprez181074b2023-02-02 14:53:23 +0100346 struct vcpu *boot_vcpu = vcpu_get_boot_vcpu();
347 struct vm *vm = boot_vcpu->vm;
Olivier Deprez98f151e2023-01-10 15:08:54 +0100348 struct vcpu_locked vcpu_locked;
Olivier Deprez181074b2023-02-02 14:53:23 +0100349
Olivier Depreza67ab882023-01-10 15:00:54 +0100350 /*
351 * TODO: the power management event reached the SPMC.
352 * In a later iteration, the power management event can
353 * be passed to the SP by resuming it.
354 */
Daniel Boulbyefa381f2022-01-18 14:49:40 +0000355 switch (args->arg3) {
356 case PSCI_CPU_OFF: {
Olivier Deprez98f151e2023-01-10 15:08:54 +0100357 if (vm_power_management_cpu_off_requested(vm) == true) {
Daniel Boulby5fe882d2023-08-07 10:36:53 +0100358 struct vcpu *vcpu;
359
Olivier Deprez98f151e2023-01-10 15:08:54 +0100360 /* Allow only S-EL1 MP SPs to reach here. */
361 CHECK(vm->el0_partition == false);
362 CHECK(vm->vcpu_count > 1);
363
364 vcpu = vm_get_vcpu(vm, vcpu_index(current));
365 vcpu_locked = vcpu_lock(vcpu);
366 vcpu->state = VCPU_STATE_OFF;
367 vcpu_unlock(&vcpu_locked);
368 cpu_off(vcpu->cpu);
Daniel Boulby5fe882d2023-08-07 10:36:53 +0100369 dlog_verbose("cpu%u off notification!\n",
370 vcpu_index(vcpu));
Olivier Deprez98f151e2023-01-10 15:08:54 +0100371 }
372
Olivier Depreza67ab882023-01-10 15:00:54 +0100373 psci_msg_response = PSCI_RETURN_SUCCESS;
Daniel Boulbyefa381f2022-01-18 14:49:40 +0000374 break;
375 }
376 default:
Olivier Depreza67ab882023-01-10 15:00:54 +0100377 dlog_error(
378 "FF-A PSCI framework message not handled "
Karl Meakine8937d92024-03-19 16:04:25 +0000379 "%#lx %#lx %#lx %#lx\n",
Olivier Depreza67ab882023-01-10 15:00:54 +0100380 args->func, args->arg1, args->arg2, args->arg3);
381 psci_msg_response = PSCI_ERROR_NOT_SUPPORTED;
Daniel Boulbyefa381f2022-01-18 14:49:40 +0000382 }
Olivier Depreza67ab882023-01-10 15:00:54 +0100383
Karl Meakind0356f82024-09-04 13:34:31 +0100384 *args = ffa_framework_msg_resp(HF_SPMC_VM_ID, HF_SPMD_VM_ID,
385 SPMD_FRAMEWORK_MSG_PSCI_RESP,
386 psci_msg_response);
Olivier Depreza67ab882023-01-10 15:00:54 +0100387 return true;
Daniel Boulbyefa381f2022-01-18 14:49:40 +0000388 }
Karl Meakind0356f82024-09-04 13:34:31 +0100389 case SPMD_FRAMEWORK_MSG_FFA_VERSION_REQ: {
Daniel Boulbyefa381f2022-01-18 14:49:40 +0000390 struct ffa_value ret = api_ffa_version(current, args->arg3);
Karl Meakind0356f82024-09-04 13:34:31 +0100391 *args = ffa_framework_msg_resp(
392 HF_SPMC_VM_ID, HF_SPMD_VM_ID,
393 SPMD_FRAMEWORK_MSG_FFA_VERSION_RESP, ret.func);
Olivier Depreza67ab882023-01-10 15:00:54 +0100394 return true;
Max Shvetsov1ae74f12020-09-18 13:52:20 +0100395 }
396 default:
Karl Meakine8937d92024-03-19 16:04:25 +0000397 dlog_error("FF-A framework message not handled %#lx\n",
Olivier Depreza67ab882023-01-10 15:00:54 +0100398 args->arg2);
399
400 /*
401 * TODO: the framework message that was conveyed by a direct
402 * request is not handled although we still want to complete
403 * by a direct response. However, there is no defined error
404 * response to state that the message couldn't be handled.
405 * An alternative would be to return FFA_ERROR.
406 */
Karl Meakind0356f82024-09-04 13:34:31 +0100407 *args = ffa_framework_msg_resp(HF_SPMC_VM_ID, HF_SPMD_VM_ID,
408 func, 0);
Olivier Depreza67ab882023-01-10 15:00:54 +0100409 return true;
Max Shvetsov1ae74f12020-09-18 13:52:20 +0100410 }
Max Shvetsov1ae74f12020-09-18 13:52:20 +0100411}
Madhukar Pappireddycf069a62024-09-25 15:36:32 -0500412
413void spmc_exit_to_nwd(struct vcpu *owd_vcpu)
414{
415 struct vcpu *deadline_vcpu =
416 timer_find_vcpu_nearest_deadline(owd_vcpu->cpu);
417
418 /*
419 * SPMC tracks a vCPU's timer deadline through its host timer such that
420 * it can bring back execution from normal world to signal the timer
421 * virtual interrupt to the SP's vCPU.
422 */
423 if (deadline_vcpu != NULL) {
424 host_timer_track_deadline(&deadline_vcpu->regs.arch_timer);
425 }
426}
J-Alvesb37fd082020-10-22 12:29:21 +0100427#endif
428
Andrew Scullae9962e2019-10-03 16:51:16 +0100429/**
430 * Checks whether to block an SMC being forwarded from a VM.
431 */
432static bool smc_is_blocked(const struct vm *vm, uint32_t func)
Andrew Walbranc1ad4ce2019-05-09 11:41:39 +0100433{
Andrew Scullae9962e2019-10-03 16:51:16 +0100434 bool block_by_default = !vm->smc_whitelist.permissive;
Fuad Tabba8176e3e2019-08-01 10:40:36 +0100435
Andrew Scullae9962e2019-10-03 16:51:16 +0100436 for (size_t i = 0; i < vm->smc_whitelist.smc_count; ++i) {
437 if (func == vm->smc_whitelist.smcs[i]) {
438 return false;
439 }
440 }
Fuad Tabba8176e3e2019-08-01 10:40:36 +0100441
Olivier Deprezf92e5d42020-11-13 16:00:54 +0100442 dlog_notice("SMC %#010x attempted from VM %#x, blocked=%u\n", func,
Andrew Walbran17eebf92020-02-05 16:35:49 +0000443 vm->id, block_by_default);
Andrew Scullae9962e2019-10-03 16:51:16 +0100444
445 /* Access is still allowed in permissive mode. */
446 return block_by_default;
Fuad Tabba8176e3e2019-08-01 10:40:36 +0100447}
448
449/**
Andrew Scullae9962e2019-10-03 16:51:16 +0100450 * Applies SMC access control according to manifest and forwards the call if
451 * access is granted.
Fuad Tabba8176e3e2019-08-01 10:40:36 +0100452 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100453static void smc_forwarder(const struct vm *vm, struct ffa_value *args)
Fuad Tabba8176e3e2019-08-01 10:40:36 +0100454{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100455 struct ffa_value ret;
Andrew Walbran9dadaf22019-12-05 16:50:55 +0000456 uint32_t client_id = vm->id;
457 uintreg_t arg7 = args->arg7;
Andrew Scullae9962e2019-10-03 16:51:16 +0100458
Andrew Walbran9dadaf22019-12-05 16:50:55 +0000459 if (smc_is_blocked(vm, args->func)) {
460 args->func = SMCCC_ERROR_UNKNOWN;
Andrew Scullae9962e2019-10-03 16:51:16 +0100461 return;
462 }
463
Andrew Walbran0dd67ff2019-09-12 16:38:50 +0100464 /*
465 * Set the Client ID but keep the existing Secure OS ID and anything
466 * else (currently unspecified) that the client may have passed in the
467 * upper bits.
468 */
Andrew Walbran9dadaf22019-12-05 16:50:55 +0000469 args->arg7 = client_id | (arg7 & ~CLIENT_ID_MASK);
Andrew Scull07b6bd32019-12-12 17:19:55 +0000470 ret = smc_forward(args->func, args->arg1, args->arg2, args->arg3,
471 args->arg4, args->arg5, args->arg6, args->arg7);
Fuad Tabba8176e3e2019-08-01 10:40:36 +0100472
Andrew Scullae9962e2019-10-03 16:51:16 +0100473 /*
Fuad Tabbab0ef2a42019-12-19 11:19:25 +0000474 * Preserve the value passed by the caller, rather than the generated
475 * client_id. Note that this would also overwrite any return value that
Andrew Scullae9962e2019-10-03 16:51:16 +0100476 * may be in x7, but the SMCs that we are forwarding are legacy calls
477 * from before SMCCC 1.2 so won't have more than 4 return values anyway.
478 */
Andrew Scull07b6bd32019-12-12 17:19:55 +0000479 ret.arg7 = arg7;
480
481 plat_smc_post_forward(*args, &ret);
482
483 *args = ret;
Fuad Tabba8176e3e2019-08-01 10:40:36 +0100484}
485
Olivier Deprezf33a6c72020-06-09 18:28:45 +0200486/**
487 * In the normal world, ffa_handler is always called from the virtual FF-A
Andrew Walbran8e8bf3f2020-10-07 17:58:20 +0100488 * instance (from a VM in EL1). In the secure world, ffa_handler may be called
489 * from the virtual (a secure partition in S-EL1) or physical FF-A instance
490 * (from the normal world via EL3). The function returns true when the call is
491 * handled. The *next pointer is updated to the next vCPU to run, which might be
492 * the 'other world' vCPU if the call originated from the virtual FF-A instance
493 * and has to be forwarded down to EL3, or left as is to resume the current
494 * vCPU.
Olivier Deprezf33a6c72020-06-09 18:28:45 +0200495 */
496static bool ffa_handler(struct ffa_value *args, struct vcpu *current,
497 struct vcpu **next)
Andrew Walbran7d28d9a2019-08-30 16:24:58 +0100498{
J-Alvesbc3de8b2020-12-07 14:32:04 +0000499 uint32_t func = args->func;
Andrew Walbrane7ad3c02019-12-24 17:03:04 +0000500
Jose Marinhoc0f4ff22019-10-09 10:37:42 +0100501 /*
502 * NOTE: When adding new methods to this handler update
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100503 * api_ffa_features accordingly.
Jose Marinhoc0f4ff22019-10-09 10:37:42 +0100504 */
Andrew Walbrane7ad3c02019-12-24 17:03:04 +0000505 switch (func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100506 case FFA_VERSION_32:
Daniel Boulbybaeaf2e2021-12-09 11:42:36 +0000507 *args = api_ffa_version(current, args->arg1);
Andrew Walbran7d28d9a2019-08-30 16:24:58 +0100508 return true;
Fuad Tabbae4efcc32020-07-16 15:37:27 +0100509 case FFA_PARTITION_INFO_GET_32: {
510 struct ffa_uuid uuid;
511
512 ffa_uuid_init(args->arg1, args->arg2, args->arg3, args->arg4,
513 &uuid);
Daniel Boulbyb46cad12021-12-13 17:47:21 +0000514 *args = api_ffa_partition_info_get(current, &uuid, args->arg5);
Fuad Tabbae4efcc32020-07-16 15:37:27 +0100515 return true;
516 }
Raghu Krishnamurthy7592bcb2022-12-25 13:09:00 -0800517 case FFA_PARTITION_INFO_GET_REGS_64: {
518 struct ffa_uuid uuid;
Raghu Krishnamurthy7592bcb2022-12-25 13:09:00 -0800519 uint16_t start_index;
520 uint16_t tag;
521
Karl Meakin9478e322024-09-23 17:47:09 +0100522 ffa_uuid_from_u64x2(args->arg1, args->arg2, &uuid);
Raghu Krishnamurthyd29411a2023-02-17 17:22:04 -0800523 start_index = args->arg3 & 0xFFFF;
524 tag = (args->arg3 >> 16) & 0xFFFF;
Raghu Krishnamurthy7592bcb2022-12-25 13:09:00 -0800525 *args = api_ffa_partition_info_get_regs(current, &uuid,
526 start_index, tag);
527 return true;
528 }
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100529 case FFA_ID_GET_32:
Olivier Deprezf33a6c72020-06-09 18:28:45 +0200530 *args = api_ffa_id_get(current);
Andrew Walbrand230f662019-10-07 18:03:36 +0100531 return true;
Daniel Boulbyb2fb80e2021-02-03 15:09:23 +0000532 case FFA_SPM_ID_GET_32:
533 *args = api_ffa_spm_id_get();
534 return true;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100535 case FFA_FEATURES_32:
Karl Meakinf1ed5f12024-02-22 15:57:36 +0000536 *args = api_ffa_features(args->arg1, args->arg2, current);
Jose Marinhoc0f4ff22019-10-09 10:37:42 +0100537 return true;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100538 case FFA_RX_RELEASE_32:
J-Alvese8c8c2b2022-12-16 15:34:48 +0000539 *args = api_ffa_rx_release(ffa_receiver(*args), current);
Andrew Walbran8a0f5ca2019-11-05 13:12:23 +0000540 return true;
J-Alvesbc3de8b2020-12-07 14:32:04 +0000541 case FFA_RXTX_MAP_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100542 *args = api_ffa_rxtx_map(ipa_init(args->arg1),
543 ipa_init(args->arg2), args->arg3,
Federico Recanati9f1b6532022-04-14 13:15:28 +0200544 current);
Andrew Walbranbfffb0f2019-11-05 14:02:34 +0000545 return true;
Daniel Boulby9e420ca2021-07-07 15:03:49 +0100546 case FFA_RXTX_UNMAP_32:
J-Alves70079932022-12-07 17:32:20 +0000547 *args = api_ffa_rxtx_unmap(ffa_vm_id(*args), current);
Daniel Boulby9e420ca2021-07-07 15:03:49 +0100548 return true;
Federico Recanati644f0462022-03-17 12:04:00 +0100549 case FFA_RX_ACQUIRE_32:
550 *args = api_ffa_rx_acquire(ffa_receiver(*args), current);
551 return true;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100552 case FFA_YIELD_32:
Madhukar Pappireddy184501c2023-05-23 17:24:06 -0500553 *args = api_yield(current, next, args);
Andrew Walbran7d28d9a2019-08-30 16:24:58 +0100554 return true;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100555 case FFA_MSG_SEND_32:
J-Alves27b71962022-12-12 15:29:58 +0000556 *args = plat_ffa_msg_send(
557 ffa_sender(*args), ffa_receiver(*args),
558 ffa_msg_send_size(*args), current, next);
Andrew Walbran7d28d9a2019-08-30 16:24:58 +0100559 return true;
Federico Recanati25053ee2022-03-14 15:01:53 +0100560 case FFA_MSG_SEND2_32:
561 *args = api_ffa_msg_send2(ffa_sender(*args),
562 ffa_msg_send2_flags(*args), current);
563 return true;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100564 case FFA_MSG_WAIT_32:
Madhukar Pappireddy5522c672021-12-17 16:35:51 -0600565 *args = api_ffa_msg_wait(current, next, args);
Andrew Walbran0de4f162019-09-03 16:44:20 +0100566 return true;
J-Alvesbc7ab4f2022-12-13 12:09:25 +0000567#if SECURE_WORLD == 0
Madhukar Pappireddybd10e572023-03-06 16:39:49 -0600568 case FFA_MSG_POLL_32: {
569 struct vcpu_locked current_locked;
570
571 current_locked = vcpu_lock(current);
J-Alves2ced1672022-12-12 14:35:38 +0000572 *args = plat_ffa_msg_recv(false, current_locked, next);
Madhukar Pappireddybd10e572023-03-06 16:39:49 -0600573 vcpu_unlock(&current_locked);
Andrew Walbran7d28d9a2019-08-30 16:24:58 +0100574 return true;
Madhukar Pappireddybd10e572023-03-06 16:39:49 -0600575 }
J-Alvesbc7ab4f2022-12-13 12:09:25 +0000576#endif
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100577 case FFA_RUN_32:
Kathleen Capella036cc592023-11-30 18:26:15 -0500578 /**
579 * Ensure that an FF-A v1.2 endpoint preserves the
580 * runtime state of the calling partition by setting
581 * the extended registers (x8-x17) to zero.
582 */
Karl Meakin0e617d92024-04-05 12:55:22 +0100583 if (current->vm->ffa_version >= FFA_VERSION_1_2 &&
Kathleen Capella036cc592023-11-30 18:26:15 -0500584 !api_extended_args_are_zero(args)) {
585 *args = ffa_error(FFA_INVALID_PARAMETERS);
586 return false;
587 }
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100588 *args = api_ffa_run(ffa_vm_id(*args), ffa_vcpu_index(*args),
Olivier Deprezf33a6c72020-06-09 18:28:45 +0200589 current, next);
Andrew Walbranf0c314d2019-10-02 14:24:26 +0100590 return true;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100591 case FFA_MEM_DONATE_32:
J-Alves95fbb312024-03-20 15:19:16 +0000592 case FFA_MEM_DONATE_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100593 case FFA_MEM_LEND_32:
J-Alves95fbb312024-03-20 15:19:16 +0000594 case FFA_MEM_LEND_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100595 case FFA_MEM_SHARE_32:
J-Alves95fbb312024-03-20 15:19:16 +0000596 case FFA_MEM_SHARE_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100597 *args = api_ffa_mem_send(func, args->arg1, args->arg2,
598 ipa_init(args->arg3), args->arg4,
Olivier Deprezf33a6c72020-06-09 18:28:45 +0200599 current);
Andrew Walbran82d6d152019-12-24 15:02:06 +0000600 return true;
J-Alves95fbb312024-03-20 15:19:16 +0000601 case FFA_MEM_RETRIEVE_REQ_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100602 case FFA_MEM_RETRIEVE_REQ_32:
603 *args = api_ffa_mem_retrieve_req(args->arg1, args->arg2,
604 ipa_init(args->arg3),
Olivier Deprezf33a6c72020-06-09 18:28:45 +0200605 args->arg4, current);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000606 return true;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100607 case FFA_MEM_RELINQUISH_32:
Olivier Deprezf33a6c72020-06-09 18:28:45 +0200608 *args = api_ffa_mem_relinquish(current);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000609 return true;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100610 case FFA_MEM_RECLAIM_32:
611 *args = api_ffa_mem_reclaim(
Andrew Walbran1bbe9402020-04-30 16:47:13 +0100612 ffa_assemble_handle(args->arg1, args->arg2), args->arg3,
Olivier Deprezf33a6c72020-06-09 18:28:45 +0200613 current);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000614 return true;
Andrew Walbranca808b12020-05-15 17:22:28 +0100615 case FFA_MEM_FRAG_RX_32:
616 *args = api_ffa_mem_frag_rx(ffa_frag_handle(*args), args->arg3,
617 (args->arg4 >> 16) & 0xffff,
Olivier Deprezf33a6c72020-06-09 18:28:45 +0200618 current);
Andrew Walbranca808b12020-05-15 17:22:28 +0100619 return true;
620 case FFA_MEM_FRAG_TX_32:
621 *args = api_ffa_mem_frag_tx(ffa_frag_handle(*args), args->arg3,
622 (args->arg4 >> 16) & 0xffff,
Olivier Deprezf33a6c72020-06-09 18:28:45 +0200623 current);
Andrew Walbranca808b12020-05-15 17:22:28 +0100624 return true;
J-Alvesbc3de8b2020-12-07 14:32:04 +0000625 case FFA_MSG_SEND_DIRECT_REQ_64:
Karl Meakind0356f82024-09-04 13:34:31 +0100626 case FFA_MSG_SEND_DIRECT_REQ_32:
Max Shvetsov1ae74f12020-09-18 13:52:20 +0100627#if SECURE_WORLD == 1
628 if (spmd_handler(args, current)) {
629 return true;
630 }
631#endif
Kathleen Capella41fea932023-06-23 17:39:28 -0400632 case FFA_MSG_SEND_DIRECT_REQ2_64:
Karl Meakin13f09812024-10-28 16:33:23 +0000633 *args = api_ffa_msg_send_direct_req(*args, current, next);
Kathleen Capella41fea932023-06-23 17:39:28 -0400634 return true;
J-Alvesbc3de8b2020-12-07 14:32:04 +0000635 case FFA_MSG_SEND_DIRECT_RESP_64:
Olivier Deprezee9d6a92019-11-26 09:14:11 +0000636 case FFA_MSG_SEND_DIRECT_RESP_32:
Kathleen Capella087e5022023-09-07 18:04:15 -0400637 case FFA_MSG_SEND_DIRECT_RESP2_64:
Karl Meakin13f09812024-10-28 16:33:23 +0000638 *args = api_ffa_msg_send_direct_resp(*args, current, next);
Olivier Deprezee9d6a92019-11-26 09:14:11 +0000639 return true;
J-Alvesbc3de8b2020-12-07 14:32:04 +0000640 case FFA_SECONDARY_EP_REGISTER_64:
Olivier Deprezd614d322021-06-18 15:21:00 +0200641 /*
642 * DEN0077A FF-A v1.1 Beta0 section 18.3.2.1.1
643 * The callee must return NOT_SUPPORTED if this function is
644 * invoked by a caller that implements version v1.0 of
645 * the Framework.
646 */
Max Shvetsov40108e72020-08-27 12:39:50 +0100647 *args = api_ffa_secondary_ep_register(ipa_init(args->arg1),
648 current);
649 return true;
J-Alvesa0f317d2021-06-09 13:31:59 +0100650 case FFA_NOTIFICATION_BITMAP_CREATE_32:
651 *args = api_ffa_notification_bitmap_create(
J-Alves19e20cf2023-08-02 12:48:55 +0100652 (ffa_id_t)args->arg1, (ffa_vcpu_count_t)args->arg2,
J-Alvesa0f317d2021-06-09 13:31:59 +0100653 current);
654 return true;
655 case FFA_NOTIFICATION_BITMAP_DESTROY_32:
656 *args = api_ffa_notification_bitmap_destroy(
J-Alves19e20cf2023-08-02 12:48:55 +0100657 (ffa_id_t)args->arg1, current);
J-Alvesa0f317d2021-06-09 13:31:59 +0100658 return true;
J-Alvesc003a7a2021-03-18 13:06:53 +0000659 case FFA_NOTIFICATION_BIND_32:
660 *args = api_ffa_notification_update_bindings(
661 ffa_sender(*args), ffa_receiver(*args), args->arg2,
662 ffa_notifications_bitmap(args->arg3, args->arg4), true,
663 current);
664 return true;
665 case FFA_NOTIFICATION_UNBIND_32:
666 *args = api_ffa_notification_update_bindings(
667 ffa_sender(*args), ffa_receiver(*args), 0,
668 ffa_notifications_bitmap(args->arg3, args->arg4), false,
669 current);
670 return true;
Raghu Krishnamurthyea6d25f2021-09-14 15:27:06 -0700671 case FFA_MEM_PERM_SET_32:
672 case FFA_MEM_PERM_SET_64:
673 *args = api_ffa_mem_perm_set(va_init(args->arg1), args->arg2,
674 args->arg3, current);
675 return true;
676 case FFA_MEM_PERM_GET_32:
677 case FFA_MEM_PERM_GET_64:
678 *args = api_ffa_mem_perm_get(va_init(args->arg1), current);
679 return true;
J-Alvesaa79c012021-07-09 14:29:45 +0100680 case FFA_NOTIFICATION_SET_32:
681 *args = api_ffa_notification_set(
682 ffa_sender(*args), ffa_receiver(*args), args->arg2,
683 ffa_notifications_bitmap(args->arg3, args->arg4),
684 current);
685 return true;
686 case FFA_NOTIFICATION_GET_32:
687 *args = api_ffa_notification_get(
J-Alvesbe6e3032021-11-30 14:54:12 +0000688 ffa_receiver(*args), ffa_notifications_get_vcpu(*args),
689 args->arg2, current);
J-Alvesaa79c012021-07-09 14:29:45 +0100690 return true;
J-Alvesc8e8a222021-06-08 17:33:52 +0100691 case FFA_NOTIFICATION_INFO_GET_64:
692 *args = api_ffa_notification_info_get(current);
693 return true;
Madhukar Pappireddy9e7a11f2021-08-03 13:59:42 -0500694 case FFA_INTERRUPT_32:
J-Alves03edf402023-07-21 15:13:49 +0100695 /*
696 * A malicious SP could invoke a HVC/SMC call with
697 * FFA_INTERRUPT_32 as the function argument. Return error to
698 * avoid DoS.
699 */
700 if (current->vm->id != HF_OTHER_WORLD_ID) {
701 *args = ffa_error(FFA_DENIED);
702 return true;
703 }
J-Alvescf0c4712023-08-04 14:41:50 +0100704
705 plat_ffa_handle_secure_interrupt(current, next);
706
707 /*
708 * If the next vCPU belongs to an SP, the next time the NWd
709 * gets resumed these values will be overwritten by the ABI
710 * that used to handover execution back to the NWd.
711 * If the NWd is to be resumed from here, then it will
712 * receive the FFA_NORMAL_WORLD_RESUME ABI which is to signal
713 * that an interrupt has occured, thought it wasn't handled.
714 * This happens when the target vCPU was in preempted state,
715 * and the SP couldn't not be resumed to handle the interrupt.
716 */
717 *args = (struct ffa_value){.func = FFA_NORMAL_WORLD_RESUME};
Madhukar Pappireddy9e7a11f2021-08-03 13:59:42 -0500718 return true;
Maksims Svecovs71b76702022-05-20 15:32:58 +0100719 case FFA_CONSOLE_LOG_32:
720 case FFA_CONSOLE_LOG_64:
721 *args = api_ffa_console_log(*args, current);
722 return true;
Kathleen Capella6ab05132023-05-10 12:27:35 -0400723 case FFA_ERROR_32:
724 *args = plat_ffa_error_32(current, next, args->arg2);
725 return true;
Andrew Walbran7d28d9a2019-08-30 16:24:58 +0100726
Karl Meakina5ea9092024-05-28 15:40:33 +0100727 default:
Karl Meakina5ea9092024-05-28 15:40:33 +0100728 return false;
729 }
Andrew Walbran7d28d9a2019-08-30 16:24:58 +0100730}
731
732/**
Manish Pandey35e452f2021-02-18 21:36:34 +0000733 * Set or clear VI/VF bits according to pending interrupts.
J-Alves6f6bf8a2024-07-25 15:17:57 +0100734 * If `vcpu` is NULL, the function will set it to the currently running
735 * vCPU.
Andrew Walbran7d28d9a2019-08-30 16:24:58 +0100736 */
J-Alves6f6bf8a2024-07-25 15:17:57 +0100737static void vcpu_update_virtual_interrupts(struct vcpu *vcpu)
Andrew Walbran7d28d9a2019-08-30 16:24:58 +0100738{
Manish Pandey35e452f2021-02-18 21:36:34 +0000739 struct vcpu_locked vcpu_locked;
740
J-Alves6f6bf8a2024-07-25 15:17:57 +0100741 if (vcpu == NULL) {
742 vcpu = current();
Andrew Walbran7d28d9a2019-08-30 16:24:58 +0100743 }
J-Alves6f6bf8a2024-07-25 15:17:57 +0100744
745 /* Only update to those at the virtual instance. */
746 if (vcpu->vm->el0_partition || !vm_id_is_current_world(vcpu->vm->id)) {
747 return;
748 }
749
750 vcpu_locked = vcpu_lock(vcpu);
751 set_virtual_irq(&vcpu->regs,
752 vcpu_interrupt_irq_count_get(vcpu_locked) > 0);
753 set_virtual_fiq(&vcpu->regs,
754 vcpu_interrupt_fiq_count_get(vcpu_locked) > 0);
755 vcpu_unlock(&vcpu_locked);
Andrew Walbran7d28d9a2019-08-30 16:24:58 +0100756}
757
Fuad Tabba8176e3e2019-08-01 10:40:36 +0100758/**
Andrew Walbrand8d3f5d2020-10-07 18:23:01 +0100759 * Handles PSCI and FF-A calls and writes the return value back to the registers
760 * of the vCPU. This is shared between smc_handler and hvc_handler.
761 *
762 * Returns true if the call was handled.
763 */
764static bool hvc_smc_handler(struct ffa_value args, struct vcpu *vcpu,
765 struct vcpu **next)
766{
Karl Meakin6eeec8e2024-03-07 18:07:20 +0000767 const uint32_t func = args.func;
768
Olivier Deprez3caed1c2021-02-05 12:07:36 +0100769 /* Do not expect PSCI calls emitted from within the secure world. */
770#if SECURE_WORLD == 0
Karl Meakin6eeec8e2024-03-07 18:07:20 +0000771 if (psci_handler(vcpu, func, args.arg1, args.arg2, args.arg3,
Andrew Walbrand8d3f5d2020-10-07 18:23:01 +0100772 &vcpu->regs.r[0], next)) {
773 return true;
774 }
Olivier Deprez3caed1c2021-02-05 12:07:36 +0100775#endif
Andrew Walbrand8d3f5d2020-10-07 18:23:01 +0100776
Andrew Walbrand8d3f5d2020-10-07 18:23:01 +0100777 if (ffa_handler(&args, vcpu, next)) {
J-Alves13394022021-06-30 13:48:49 +0100778#if SECURE_WORLD == 1
779 /*
780 * If giving back execution to the NWd, check if the Schedule
Olivier Deprez618c8fc2022-05-30 15:27:49 +0200781 * Receiver Interrupt has been delayed, and trigger it on
782 * current core if so.
J-Alves13394022021-06-30 13:48:49 +0100783 */
784 if ((*next != NULL && (*next)->vm->id == HF_OTHER_WORLD_ID) ||
785 (*next == NULL && vcpu->vm->id == HF_OTHER_WORLD_ID)) {
786 plat_ffa_sri_trigger_if_delayed(vcpu->cpu);
787 }
788#endif
Karl Meakin6eeec8e2024-03-07 18:07:20 +0000789 if (func != FFA_VERSION_32) {
790 struct vm_locked vm_locked = vm_lock(vcpu->vm);
791
792 vm_locked.vm->ffa_version_negotiated = true;
793 vm_unlock(&vm_locked);
794 }
795
Andrew Walbrand8d3f5d2020-10-07 18:23:01 +0100796 arch_regs_set_retval(&vcpu->regs, args);
J-Alves6f6bf8a2024-07-25 15:17:57 +0100797
798 /*
799 * In case there has been an update after handling the last
800 * ff-a call, update the next vCPU directly in the
801 * register.
802 */
Manish Pandey35e452f2021-02-18 21:36:34 +0000803 vcpu_update_virtual_interrupts(*next);
Andrew Walbrand8d3f5d2020-10-07 18:23:01 +0100804 return true;
805 }
806
807 return false;
808}
809
810/**
Fuad Tabba8176e3e2019-08-01 10:40:36 +0100811 * Processes SMC instruction calls.
812 */
Andrew Walbran9dadaf22019-12-05 16:50:55 +0000813static struct vcpu *smc_handler(struct vcpu *vcpu)
Fuad Tabba8176e3e2019-08-01 10:40:36 +0100814{
Andrew Walbrand8d3f5d2020-10-07 18:23:01 +0100815 struct ffa_value args = arch_regs_get_args(&vcpu->regs);
Andrew Walbran9dadaf22019-12-05 16:50:55 +0000816 struct vcpu *next = NULL;
Fuad Tabba8176e3e2019-08-01 10:40:36 +0100817
Olivier Deprez79dbd6f2023-11-29 16:12:36 +0100818 /* Mask out SMCCC SVE hint bit from function id. */
819 args.func &= ~SMCCC_SVE_HINT_MASK;
820
Andrew Walbrand8d3f5d2020-10-07 18:23:01 +0100821 if (hvc_smc_handler(args, vcpu, &next)) {
Andrew Walbran9dadaf22019-12-05 16:50:55 +0000822 return next;
Andrew Walbran4579f7002019-08-30 16:24:58 +0100823 }
824
Andrew Walbran9dadaf22019-12-05 16:50:55 +0000825 smc_forwarder(vcpu->vm, &args);
826 arch_regs_set_retval(&vcpu->regs, args);
Andrew Scull07b6bd32019-12-12 17:19:55 +0000827 return NULL;
Andrew Walbranc1ad4ce2019-05-09 11:41:39 +0100828}
829
Olivier Deprez3caed1c2021-02-05 12:07:36 +0100830#if SECURE_WORLD == 1
831
832/**
833 * Called from other_world_loop return from SMC.
834 * Processes SMC calls originating from the NWd.
835 */
836struct vcpu *smc_handler_from_nwd(struct vcpu *vcpu)
837{
Olivier Deprez79dbd6f2023-11-29 16:12:36 +0100838 struct ffa_value args = arch_regs_get_args(&vcpu->regs);
Olivier Deprez3caed1c2021-02-05 12:07:36 +0100839 struct vcpu *next = NULL;
840
Olivier Deprez5b588332023-09-05 15:08:48 +0200841 plat_save_ns_simd_context(vcpu);
842
Olivier Deprez79dbd6f2023-11-29 16:12:36 +0100843 /* Mask out SMCCC SVE hint bit from function id. */
844 args.func &= ~SMCCC_SVE_HINT_MASK;
845
Olivier Deprez3caed1c2021-02-05 12:07:36 +0100846 if (hvc_smc_handler(args, vcpu, &next)) {
847 return next;
848 }
849
850 /*
851 * If the SMC emitted by the normal world is not handled in the secure
852 * world then return an error stating such ABI is not supported. Only
853 * FF-A calls are supported. We cannot return SMCCC_ERROR_UNKNOWN
854 * directly because the SPMD smc handler would not recognize it as a
855 * standard FF-A call returning from the SPMC.
856 */
857 arch_regs_set_retval(&vcpu->regs, ffa_error(FFA_NOT_SUPPORTED));
858
859 return NULL;
860}
861
862#endif
863
Fuad Tabbaa48d1222019-12-09 15:42:32 +0000864/*
865 * Exception vector offsets.
866 * See Arm Architecture Reference Manual Armv8-A, D1.10.2.
867 */
868
869/**
870 * Offset for synchronous exceptions at current EL with SPx.
871 */
872#define OFFSET_CURRENT_SPX UINT64_C(0x200)
873
874/**
875 * Offset for synchronous exceptions at lower EL using AArch64.
876 */
877#define OFFSET_LOWER_EL_64 UINT64_C(0x400)
878
879/**
880 * Offset for synchronous exceptions at lower EL using AArch32.
881 */
882#define OFFSET_LOWER_EL_32 UINT64_C(0x600)
883
884/**
885 * Returns the address for the exception handler at EL1.
886 */
887static uintreg_t get_el1_exception_handler_addr(const struct vcpu *vcpu)
888{
Raghu Krishnamurthy32626c92021-01-17 09:57:29 -0800889 uintreg_t base_addr = has_vhe_support() ? read_msr(MSR_VBAR_EL12)
890 : read_msr(vbar_el1);
Fuad Tabbaa48d1222019-12-09 15:42:32 +0000891 uintreg_t pe_mode = vcpu->regs.spsr & PSR_PE_MODE_MASK;
892 bool is_arch32 = vcpu->regs.spsr & PSR_ARCH_MODE_32;
893
894 if (pe_mode == PSR_PE_MODE_EL0T) {
895 if (is_arch32) {
896 base_addr += OFFSET_LOWER_EL_32;
897 } else {
898 base_addr += OFFSET_LOWER_EL_64;
899 }
900 } else {
901 CHECK(!is_arch32);
902 base_addr += OFFSET_CURRENT_SPX;
903 }
904
905 return base_addr;
906}
907
908/**
Fuad Tabbab86325a2020-01-10 13:38:15 +0000909 * Injects an exception with the specified Exception Syndrom Register value into
910 * the EL1.
Fuad Tabbaa48d1222019-12-09 15:42:32 +0000911 *
912 * NOTE: This function assumes that the lazy registers haven't been saved, and
913 * writes to the lazy registers of the CPU directly instead of the vCPU.
914 */
Fuad Tabbac3847c72020-08-11 09:32:25 +0100915static void inject_el1_exception(struct vcpu *vcpu, uintreg_t esr_el1_value,
916 uintreg_t far_el1_value)
Fuad Tabbaa48d1222019-12-09 15:42:32 +0000917{
Fuad Tabbaa48d1222019-12-09 15:42:32 +0000918 uintreg_t handler_address = get_el1_exception_handler_addr(vcpu);
Fuad Tabbaa48d1222019-12-09 15:42:32 +0000919
920 /* Update the CPU state to inject the exception. */
Raghu Krishnamurthy32626c92021-01-17 09:57:29 -0800921 if (has_vhe_support()) {
922 write_msr(MSR_ESR_EL12, esr_el1_value);
923 write_msr(MSR_FAR_EL12, far_el1_value);
924 write_msr(MSR_ELR_EL12, vcpu->regs.pc);
925 write_msr(MSR_SPSR_EL12, vcpu->regs.spsr);
926 } else {
927 write_msr(esr_el1, esr_el1_value);
928 write_msr(far_el1, far_el1_value);
929 write_msr(elr_el1, vcpu->regs.pc);
930 write_msr(spsr_el1, vcpu->regs.spsr);
931 }
Fuad Tabbaa48d1222019-12-09 15:42:32 +0000932
933 /*
934 * Mask (disable) interrupts and run in EL1h mode.
935 * EL1h mode is used because by default, taking an exception selects the
936 * stack pointer for the target Exception level. The software can change
937 * that later in the handler if needed.
Fuad Tabbaa48d1222019-12-09 15:42:32 +0000938 */
939 vcpu->regs.spsr = PSR_D | PSR_A | PSR_I | PSR_F | PSR_PE_MODE_EL1H;
940
941 /* Transfer control to the exception hander. */
942 vcpu->regs.pc = handler_address;
Fuad Tabbab86325a2020-01-10 13:38:15 +0000943}
944
945/**
946 * Injects a Data Abort exception (same exception level).
947 */
948static void inject_el1_data_abort_exception(struct vcpu *vcpu,
Fuad Tabbac3847c72020-08-11 09:32:25 +0100949 uintreg_t esr_el2,
950 uintreg_t far_el2)
Fuad Tabbab86325a2020-01-10 13:38:15 +0000951{
952 /*
953 * ISS encoding remains the same, but the EC is changed to reflect
954 * where the exception came from.
955 * See Arm Architecture Reference Manual Armv8-A, pages D13-2943/2982.
956 */
957 uintreg_t esr_el1_value = GET_ESR_ISS(esr_el2) | GET_ESR_IL(esr_el2) |
958 (EC_DATA_ABORT_SAME_EL << ESR_EC_OFFSET);
959
Olivier Deprezf92e5d42020-11-13 16:00:54 +0100960 dlog_notice("Injecting Data Abort exception into VM %#x.\n",
Andrew Walbran17eebf92020-02-05 16:35:49 +0000961 vcpu->vm->id);
Fuad Tabbab86325a2020-01-10 13:38:15 +0000962
Fuad Tabbac3847c72020-08-11 09:32:25 +0100963 inject_el1_exception(vcpu, esr_el1_value, far_el2);
Fuad Tabbab86325a2020-01-10 13:38:15 +0000964}
965
966/**
967 * Injects a Data Abort exception (same exception level).
968 */
969static void inject_el1_instruction_abort_exception(struct vcpu *vcpu,
Fuad Tabbac3847c72020-08-11 09:32:25 +0100970 uintreg_t esr_el2,
971 uintreg_t far_el2)
Fuad Tabbab86325a2020-01-10 13:38:15 +0000972{
973 /*
974 * ISS encoding remains the same, but the EC is changed to reflect
975 * where the exception came from.
976 * See Arm Architecture Reference Manual Armv8-A, pages D13-2941/2980.
977 */
978 uintreg_t esr_el1_value =
979 GET_ESR_ISS(esr_el2) | GET_ESR_IL(esr_el2) |
980 (EC_INSTRUCTION_ABORT_SAME_EL << ESR_EC_OFFSET);
981
Olivier Deprezf92e5d42020-11-13 16:00:54 +0100982 dlog_notice("Injecting Instruction Abort exception into VM %#x.\n",
Andrew Walbran17eebf92020-02-05 16:35:49 +0000983 vcpu->vm->id);
Fuad Tabbab86325a2020-01-10 13:38:15 +0000984
Fuad Tabbac3847c72020-08-11 09:32:25 +0100985 inject_el1_exception(vcpu, esr_el1_value, far_el2);
Fuad Tabbab86325a2020-01-10 13:38:15 +0000986}
987
988/**
989 * Injects an exception with an unknown reason into the EL1.
990 */
991static void inject_el1_unknown_exception(struct vcpu *vcpu, uintreg_t esr_el2)
992{
993 uintreg_t esr_el1_value =
994 GET_ESR_IL(esr_el2) | (EC_UNKNOWN << ESR_EC_OFFSET);
Fuad Tabbac3847c72020-08-11 09:32:25 +0100995
Olivier Deprezda14ddc2022-08-11 14:14:41 +0200996 dlog_notice("Injecting Unknown Reason exception into VM %#x.\n",
997 vcpu->vm->id);
998
Fuad Tabbac3847c72020-08-11 09:32:25 +0100999 /*
1000 * The value of the far_el2 register is UNKNOWN in this case,
1001 * therefore, don't propagate it to avoid leaking sensitive information.
1002 */
Olivier Deprezda14ddc2022-08-11 14:14:41 +02001003 inject_el1_exception(vcpu, esr_el1_value, 0);
1004}
Fuad Tabbaa48d1222019-12-09 15:42:32 +00001005
Olivier Deprezda14ddc2022-08-11 14:14:41 +02001006/**
1007 * Injects an exception because of a system register trap.
1008 */
1009static void inject_el1_sysreg_trap_exception(struct vcpu *vcpu,
1010 uintreg_t esr_el2)
1011{
1012 char *direction_str = ISS_IS_READ(esr_el2) ? "read" : "write";
1013
Andrew Walbran17eebf92020-02-05 16:35:49 +00001014 dlog_notice(
Karl Meakine8937d92024-03-19 16:04:25 +00001015 "Trapped access to system register %s: op0=%lu, op1=%lu, "
1016 "crn=%lu, "
1017 "crm=%lu, op2=%lu, rt=%lu.\n",
Andrew Walbran17eebf92020-02-05 16:35:49 +00001018 direction_str, GET_ISS_OP0(esr_el2), GET_ISS_OP1(esr_el2),
1019 GET_ISS_CRN(esr_el2), GET_ISS_CRM(esr_el2),
1020 GET_ISS_OP2(esr_el2), GET_ISS_RT(esr_el2));
Fuad Tabbaa48d1222019-12-09 15:42:32 +00001021
Olivier Deprezda14ddc2022-08-11 14:14:41 +02001022 inject_el1_unknown_exception(vcpu, esr_el2);
Fuad Tabbaa48d1222019-12-09 15:42:32 +00001023}
1024
Andrew Walbrand8d3f5d2020-10-07 18:23:01 +01001025static struct vcpu *hvc_handler(struct vcpu *vcpu)
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +01001026{
Andrew Walbrand8d3f5d2020-10-07 18:23:01 +01001027 struct ffa_value args = arch_regs_get_args(&vcpu->regs);
Andrew Walbran59182d52019-09-23 17:55:39 +01001028 struct vcpu *next = NULL;
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +01001029
Olivier Deprez79dbd6f2023-11-29 16:12:36 +01001030 /* Mask out SMCCC SVE hint bit from function id. */
1031 args.func &= ~SMCCC_SVE_HINT_MASK;
1032
Andrew Walbrand8d3f5d2020-10-07 18:23:01 +01001033 if (hvc_smc_handler(args, vcpu, &next)) {
Andrew Walbran59182d52019-09-23 17:55:39 +01001034 return next;
Andrew Walbran7d28d9a2019-08-30 16:24:58 +01001035 }
Jose Marinhofc0b2b62019-06-06 11:18:45 +01001036
Andrew Walbran7f920af2019-09-03 17:09:30 +01001037 switch (args.func) {
J-Alves15e30262024-10-14 11:56:07 +01001038#if SECURE_WORLD == 1
Madhukar Pappireddyf675bb62021-08-03 12:57:10 -05001039 case HF_INTERRUPT_DEACTIVATE:
1040 vcpu->regs.r[0] = plat_ffa_interrupt_deactivate(
1041 args.arg1, args.arg2, vcpu);
1042 break;
Madhukar Pappireddy72d23932023-07-24 15:57:28 -05001043
1044 case HF_INTERRUPT_RECONFIGURE:
1045 vcpu->regs.r[0] = plat_ffa_interrupt_reconfigure(
1046 args.arg1, args.arg2, args.arg3, vcpu);
1047 break;
Daniel Boulbyf3cf28c2024-08-22 10:46:23 +01001048
1049 case HF_INTERRUPT_SEND_IPI:
1050 vcpu->regs.r[0] = api_hf_interrupt_send_ipi(args.arg1, vcpu);
1051 break;
Madhukar Pappireddyf675bb62021-08-03 12:57:10 -05001052#endif
Olivier Deprez109c6d42023-11-29 14:58:47 +01001053 case HF_INTERRUPT_ENABLE:
1054 vcpu->regs.r[0] = api_interrupt_enable(args.arg1, args.arg2,
1055 args.arg3, vcpu);
1056 break;
1057
Madhukar Pappireddyc64d0642024-08-07 16:55:46 -05001058 case HF_INTERRUPT_GET: {
1059 struct vcpu_locked current_locked;
Madhukar Pappireddyf675bb62021-08-03 12:57:10 -05001060
Madhukar Pappireddyc64d0642024-08-07 16:55:46 -05001061 current_locked = vcpu_lock(vcpu);
1062 vcpu->regs.r[0] = plat_ffa_interrupt_get(current_locked);
1063 vcpu_unlock(&current_locked);
1064 break;
1065 }
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +01001066 default:
Andrew Walbran59182d52019-09-23 17:55:39 +01001067 vcpu->regs.r[0] = SMCCC_ERROR_UNKNOWN;
J-Alves33172402024-08-15 13:15:34 +01001068 dlog_verbose("Unsupported function %#lx\n", args.func);
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +01001069 }
1070
J-Alves6f6bf8a2024-07-25 15:17:57 +01001071 /*
1072 * In case there has been an update after handling the last
1073 * hypervisor call, update the next vCPU directly in the register.
1074 */
Manish Pandey35e452f2021-02-18 21:36:34 +00001075 vcpu_update_virtual_interrupts(next);
Andrew Walbran3d84a262018-12-13 14:41:19 +00001076
Andrew Walbran59182d52019-09-23 17:55:39 +01001077 return next;
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +01001078}
1079
Wedson Almeida Filho87009642018-07-02 10:20:07 +01001080struct vcpu *irq_lower(void)
1081{
Madhukar Pappireddycbecc962021-08-03 13:11:57 -05001082#if SECURE_WORLD == 1
1083 struct vcpu *next = NULL;
1084
J-Alves03edf402023-07-21 15:13:49 +01001085 plat_ffa_handle_secure_interrupt(current(), &next);
Madhukar Pappireddycbecc962021-08-03 13:11:57 -05001086
1087 /*
1088 * Since we are in interrupt context, set the bit for the
1089 * next vCPU directly in the register.
1090 */
1091 vcpu_update_virtual_interrupts(next);
1092
1093 return next;
1094#else
Andrew Scull9726c252019-01-23 13:44:19 +00001095 /*
1096 * Switch back to primary VM, interrupts will be handled there.
1097 *
1098 * If the VM has aborted, this vCPU will be aborted when the scheduler
1099 * tries to run it again. This means the interrupt will not be delayed
1100 * by the aborted VM.
1101 *
1102 * TODO: Only switch when the interrupt isn't for the current VM.
1103 */
Andrew Scull33fecd32019-01-08 14:48:27 +00001104 return api_preempt(current());
Madhukar Pappireddycbecc962021-08-03 13:11:57 -05001105#endif
Wedson Almeida Filho87009642018-07-02 10:20:07 +01001106}
1107
Madhukar Pappireddy7fc585e2023-03-02 14:31:22 -06001108#if SECURE_WORLD == 1
1109static void spmd_group0_intr_delegate(void)
1110{
1111 struct ffa_value ret;
1112
1113 dlog_verbose("Delegating Group0 interrupt to SPMD\n");
1114
1115 ret = smc_ffa_call((struct ffa_value){.func = FFA_EL3_INTR_HANDLE_32});
1116
1117 /* Check if the Group0 interrupt was handled successfully. */
1118 CHECK(ret.func == FFA_SUCCESS_32);
1119}
1120#endif
1121
Wedson Almeida Filho9d5040f2018-10-29 08:41:27 +00001122struct vcpu *fiq_lower(void)
1123{
Manish Pandeya5f39fb2020-09-11 09:47:11 +01001124#if SECURE_WORLD == 1
1125 struct vcpu_locked current_locked;
1126 struct vcpu *current_vcpu = current();
Daniel Boulby4dd3f532021-09-21 09:57:08 +01001127 int64_t ret;
Madhukar Pappireddy77d3bcd2023-03-01 17:26:22 -06001128 uint32_t intid;
Manish Pandeya5f39fb2020-09-11 09:47:11 +01001129
Madhukar Pappireddy77d3bcd2023-03-01 17:26:22 -06001130 intid = get_highest_pending_g0_interrupt_id();
1131
1132 /* Check for the highest priority pending Group0 interrupt. */
1133 if (intid != SPURIOUS_INTID_OTHER_WORLD) {
Madhukar Pappireddy7fc585e2023-03-02 14:31:22 -06001134 /* Delegate handling of Group0 interrupt to EL3 firmware. */
1135 spmd_group0_intr_delegate();
1136
1137 /* Resume current vCPU. */
1138 return NULL;
Madhukar Pappireddy77d3bcd2023-03-01 17:26:22 -06001139 }
1140
1141 /*
1142 * A special interrupt indicating there is no pending interrupt
1143 * with sufficient priority for current security state. This
1144 * means a non-secure interrupt is pending.
1145 */
Madhukar Pappireddyc40f55f2022-06-22 11:00:41 -05001146 assert(current_vcpu->vm->ns_interrupts_action != NS_ACTION_QUEUED);
1147
Maksims Svecovs9ddf86a2021-05-06 17:17:21 +01001148 if (plat_ffa_vm_managed_exit_supported(current_vcpu->vm)) {
Madhukar Pappireddydd6fdfb2021-12-14 12:30:36 -06001149 uint8_t pmr = plat_interrupts_get_priority_mask();
1150
Madhukar Pappireddy025a4512024-10-14 22:09:19 -05001151 /*
1152 * Mask non-secure interrupt from triggering again till the
1153 * vCPU completes the managed exit sequenece.
1154 */
1155 plat_interrupts_set_priority_mask(SWD_MASK_NS_INT);
Manish Pandeya5f39fb2020-09-11 09:47:11 +01001156
1157 current_locked = vcpu_lock(current_vcpu);
Madhukar Pappireddy025a4512024-10-14 22:09:19 -05001158 current_vcpu->prev_interrupt_priority = pmr;
Manish Pandeya5f39fb2020-09-11 09:47:11 +01001159 ret = api_interrupt_inject_locked(current_locked,
1160 HF_MANAGED_EXIT_INTID,
Madhukar Pappireddybd10e572023-03-06 16:39:49 -06001161 current_locked, NULL);
Manish Pandeya5f39fb2020-09-11 09:47:11 +01001162 if (ret != 0) {
1163 panic("Failed to inject managed exit interrupt\n");
1164 }
1165
1166 /* Entering managed exit sequence. */
1167 current_vcpu->processing_managed_exit = true;
1168
1169 vcpu_unlock(&current_locked);
1170
1171 /*
1172 * Since we are in interrupt context, set the bit for the
1173 * current vCPU directly in the register.
1174 */
1175 vcpu_update_virtual_interrupts(NULL);
1176
1177 /* Resume current vCPU. */
1178 return NULL;
1179 }
Manish Pandeya5f39fb2020-09-11 09:47:11 +01001180
Madhukar Pappireddyd46c06e2022-06-21 18:14:52 -05001181 /*
1182 * Unwind Normal World Scheduled Call chain in response to NS
1183 * Interrupt.
1184 */
1185 return plat_ffa_unwind_nwd_call_chain_interrupt(current_vcpu);
Madhukar Pappireddycbecc962021-08-03 13:11:57 -05001186#else
Wedson Almeida Filho9d5040f2018-10-29 08:41:27 +00001187 return irq_lower();
Madhukar Pappireddycbecc962021-08-03 13:11:57 -05001188#endif
Wedson Almeida Filho9d5040f2018-10-29 08:41:27 +00001189}
1190
Fuad Tabbad1d67982020-01-08 11:28:29 +00001191noreturn struct vcpu *serr_lower(void)
Wedson Almeida Filho9d5040f2018-10-29 08:41:27 +00001192{
Fuad Tabbad1d67982020-01-08 11:28:29 +00001193 /*
1194 * SError exceptions should be isolated and handled by the responsible
1195 * VM/exception level. Getting here indicates a bug, that isolation is
1196 * not working, or a processor that does not support ARMv8.2-IESB, in
1197 * which case Hafnium routes SError exceptions to EL2 (here).
1198 */
1199 panic("SError from a lower exception level.");
Wedson Almeida Filho9d5040f2018-10-29 08:41:27 +00001200}
1201
Wedson Almeida Filho99d2d4c2019-02-14 12:53:46 +00001202/**
1203 * Initialises a fault info structure. It assumes that an FnV bit exists at
1204 * bit offset 10 of the ESR, and that it is only valid when the bottom 6 bits of
1205 * the ESR (the fault status code) are 010000; this is the case for both
1206 * instruction and data aborts, but not necessarily for other exception reasons.
1207 */
1208static struct vcpu_fault_info fault_info_init(uintreg_t esr,
Andrew Walbran1281ed42019-10-22 17:23:40 +01001209 const struct vcpu *vcpu,
1210 uint32_t mode)
Wedson Almeida Filho99d2d4c2019-02-14 12:53:46 +00001211{
1212 uint32_t fsc = esr & 0x3f;
1213 struct vcpu_fault_info r;
Olivier Deprez98ad2d22020-05-20 09:52:43 +02001214 uint64_t hpfar_el2_val;
1215 uint64_t hpfar_el2_fipa;
Wedson Almeida Filho99d2d4c2019-02-14 12:53:46 +00001216
1217 r.mode = mode;
Wedson Almeida Filho99d2d4c2019-02-14 12:53:46 +00001218 r.pc = va_init(vcpu->regs.pc);
1219
Olivier Deprez98ad2d22020-05-20 09:52:43 +02001220 /* Get Hypervisor IPA Fault Address value. */
1221 hpfar_el2_val = read_msr(hpfar_el2);
1222
1223 /* Extract Faulting IPA. */
1224 hpfar_el2_fipa = (hpfar_el2_val & HPFAR_EL2_FIPA) << 8;
1225
1226#if SECURE_WORLD == 1
1227
1228 /**
1229 * Determine if faulting IPA targets NS space.
1230 * At NS-EL2 hpfar_el2 bit 63 is RES0. At S-EL2, this bit determines if
1231 * the faulting Stage-1 address output is a secure or non-secure IPA.
1232 */
1233 if ((hpfar_el2_val & HPFAR_EL2_NS) != 0) {
1234 r.mode |= MM_MODE_NS;
1235 }
1236
1237#endif
1238
Wedson Almeida Filho99d2d4c2019-02-14 12:53:46 +00001239 /*
1240 * Check the FnV bit, which is only valid if dfsc/ifsc is 010000. It
1241 * indicates that we cannot rely on far_el2.
1242 */
Karl Meakin5a133552024-05-30 16:06:27 +01001243 if (fsc == 0x10 && GET_ESR_FNV(esr)) {
Wedson Almeida Filho99d2d4c2019-02-14 12:53:46 +00001244 r.vaddr = va_init(0);
Olivier Deprez98ad2d22020-05-20 09:52:43 +02001245 r.ipaddr = ipa_init(hpfar_el2_fipa);
Wedson Almeida Filho99d2d4c2019-02-14 12:53:46 +00001246 } else {
1247 r.vaddr = va_init(read_msr(far_el2));
Olivier Deprez98ad2d22020-05-20 09:52:43 +02001248 r.ipaddr = ipa_init(hpfar_el2_fipa |
Wedson Almeida Filho99d2d4c2019-02-14 12:53:46 +00001249 (read_msr(far_el2) & (PAGE_SIZE - 1)));
1250 }
1251
1252 return r;
1253}
1254
Fuad Tabbac3847c72020-08-11 09:32:25 +01001255struct vcpu *sync_lower_exception(uintreg_t esr, uintreg_t far)
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +01001256{
Wedson Almeida Filho00df6c72018-10-18 11:19:24 +01001257 struct vcpu *vcpu = current();
Wedson Almeida Filho99d2d4c2019-02-14 12:53:46 +00001258 struct vcpu_fault_info info;
Raghu Krishnamurthyb5775d22021-02-26 18:54:40 -08001259 struct vcpu *new_vcpu = NULL;
Fuad Tabba3e9b0222019-11-11 16:47:50 +00001260 uintreg_t ec = GET_ESR_EC(esr);
Raghu Krishnamurthyb5775d22021-02-26 18:54:40 -08001261 bool is_el0_partition = vcpu->vm->el0_partition;
Raghu Krishnamurthyf16b2ce2021-11-02 07:48:38 -07001262 bool resume = false;
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +01001263
Fuad Tabbac76466d2019-09-06 10:42:12 +01001264 switch (ec) {
Fuad Tabbab86325a2020-01-10 13:38:15 +00001265 case EC_WFI_WFE:
Andrew Walbran48196eb2019-03-04 14:56:24 +00001266 /* Skip the instruction. */
Fuad Tabbac76466d2019-09-06 10:42:12 +01001267 vcpu->regs.pc += GET_NEXT_PC_INC(esr);
Raghu Krishnamurthyb5775d22021-02-26 18:54:40 -08001268
1269 /*
1270 * For EL0 partitions, treat both WFI and WFE the same way so
1271 * that FFA_RUN can be called on the partition to resume it. If
1272 * we treat WFI using api_wait_for_interrupt, the VCPU will be
1273 * in blocked waiting for interrupt but we cannot inject
1274 * interrupts into EL0 partitions.
1275 */
1276 if (is_el0_partition) {
Madhukar Pappireddy184501c2023-05-23 17:24:06 -05001277 api_yield(vcpu, &new_vcpu, NULL);
Raghu Krishnamurthyb5775d22021-02-26 18:54:40 -08001278 return new_vcpu;
1279 }
1280
Wedson Almeida Filho87009642018-07-02 10:20:07 +01001281 /* Check TI bit of ISS, 0 = WFI, 1 = WFE. */
Andrew Scull7364a8e2018-07-19 15:39:29 +01001282 if (esr & 1) {
Andrew Walbran48196eb2019-03-04 14:56:24 +00001283 /* WFE */
1284 /*
1285 * TODO: consider giving the scheduler more context,
1286 * somehow.
1287 */
Madhukar Pappireddy184501c2023-05-23 17:24:06 -05001288 api_yield(vcpu, &new_vcpu, NULL);
Jose Marinho135dff32019-02-28 10:25:57 +00001289 return new_vcpu;
Andrew Scull7364a8e2018-07-19 15:39:29 +01001290 }
Andrew Walbran48196eb2019-03-04 14:56:24 +00001291 /* WFI */
Andrew Scull9726c252019-01-23 13:44:19 +00001292 return api_wait_for_interrupt(vcpu);
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +01001293
Fuad Tabbab86325a2020-01-10 13:38:15 +00001294 case EC_DATA_ABORT_LOWER_EL:
Wedson Almeida Filho99d2d4c2019-02-14 12:53:46 +00001295 info = fault_info_init(
Andrew Walbrane52006c2019-10-22 18:01:28 +01001296 esr, vcpu, (esr & (1U << 6)) ? MM_MODE_W : MM_MODE_R);
Raghu Krishnamurthyb5775d22021-02-26 18:54:40 -08001297
Raghu Krishnamurthyf16b2ce2021-11-02 07:48:38 -07001298 resume = vcpu_handle_page_fault(vcpu, &info);
Raghu Krishnamurthyb5775d22021-02-26 18:54:40 -08001299 if (is_el0_partition) {
1300 dlog_warning("Data abort on EL0 partition\n");
Raghu Krishnamurthyf16b2ce2021-11-02 07:48:38 -07001301 /*
1302 * Abort EL0 context if we should not resume the
1303 * context, or it is an alignment fault.
1304 * vcpu_handle_page_fault() only checks the mode of the
1305 * page in an architecture agnostic way but alignment
1306 * faults on aarch64 can happen on a correctly mapped
1307 * page.
1308 */
1309 if (!resume || ((esr & 0x3f) == 0x21)) {
1310 return api_abort(vcpu);
1311 }
1312 }
1313
1314 if (resume) {
1315 return NULL;
Raghu Krishnamurthyb5775d22021-02-26 18:54:40 -08001316 }
1317
Fuad Tabbab86325a2020-01-10 13:38:15 +00001318 /* Inform the EL1 of the data abort. */
Fuad Tabbac3847c72020-08-11 09:32:25 +01001319 inject_el1_data_abort_exception(vcpu, esr, far);
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +01001320
Fuad Tabbab86325a2020-01-10 13:38:15 +00001321 /* Schedule the same VM to continue running. */
1322 return NULL;
1323
1324 case EC_INSTRUCTION_ABORT_LOWER_EL:
Andrew Sculld3cfaad2019-04-04 11:34:10 +01001325 info = fault_info_init(esr, vcpu, MM_MODE_X);
Raghu Krishnamurthyf16b2ce2021-11-02 07:48:38 -07001326
Wedson Almeida Filho99d2d4c2019-02-14 12:53:46 +00001327 if (vcpu_handle_page_fault(vcpu, &info)) {
1328 return NULL;
1329 }
Raghu Krishnamurthyb5775d22021-02-26 18:54:40 -08001330
1331 if (is_el0_partition) {
1332 dlog_warning("Instruction abort on EL0 partition\n");
1333 return api_abort(vcpu);
1334 }
1335
Fuad Tabbab86325a2020-01-10 13:38:15 +00001336 /* Inform the EL1 of the instruction abort. */
Fuad Tabbac3847c72020-08-11 09:32:25 +01001337 inject_el1_instruction_abort_exception(vcpu, esr, far);
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +01001338
Fuad Tabbab86325a2020-01-10 13:38:15 +00001339 /* Schedule the same VM to continue running. */
1340 return NULL;
Raghu Krishnamurthyb5775d22021-02-26 18:54:40 -08001341 case EC_SVC:
1342 CHECK(is_el0_partition);
1343 return hvc_handler(vcpu);
Fuad Tabbab86325a2020-01-10 13:38:15 +00001344 case EC_HVC:
Raghu Krishnamurthyb5775d22021-02-26 18:54:40 -08001345 if (is_el0_partition) {
1346 dlog_warning("Unexpected HVC Trap on EL0 partition\n");
1347 return api_abort(vcpu);
1348 }
Andrew Walbran59182d52019-09-23 17:55:39 +01001349 return hvc_handler(vcpu);
1350
Fuad Tabbab86325a2020-01-10 13:38:15 +00001351 case EC_SMC: {
Andrew Scullc960c032018-10-24 15:13:35 +01001352 uintreg_t smc_pc = vcpu->regs.pc;
Andrew Walbran9dadaf22019-12-05 16:50:55 +00001353 struct vcpu *next = smc_handler(vcpu);
Wedson Almeida Filho03e767a2018-07-30 15:32:03 +01001354
1355 /* Skip the SMC instruction. */
Fuad Tabbac76466d2019-09-06 10:42:12 +01001356 vcpu->regs.pc = smc_pc + GET_NEXT_PC_INC(esr);
Andrew Walbran9dadaf22019-12-05 16:50:55 +00001357
Andrew Walbran33645652019-04-15 12:29:31 +01001358 return next;
Andrew Scullc960c032018-10-24 15:13:35 +01001359 }
Wedson Almeida Filho03e767a2018-07-30 15:32:03 +01001360
Fuad Tabbab86325a2020-01-10 13:38:15 +00001361 case EC_MSR:
Fuad Tabbac76466d2019-09-06 10:42:12 +01001362 /*
1363 * NOTE: This should never be reached because it goes through a
1364 * separate path handled by handle_system_register_access().
1365 */
1366 panic("Handled by handle_system_register_access().");
1367
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +01001368 default:
Andrew Walbran17eebf92020-02-05 16:35:49 +00001369 dlog_notice(
Karl Meakine8937d92024-03-19 16:04:25 +00001370 "Unknown lower sync exception pc=%#lx, esr=%#lx, "
1371 "ec=%#lx\n",
Andrew Walbran17eebf92020-02-05 16:35:49 +00001372 vcpu->regs.pc, esr, ec);
Andrew Scull9726c252019-01-23 13:44:19 +00001373 break;
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +01001374 }
1375
Raghu Krishnamurthyb5775d22021-02-26 18:54:40 -08001376 if (is_el0_partition) {
1377 return api_abort(vcpu);
1378 }
1379
Fuad Tabba3e9b0222019-11-11 16:47:50 +00001380 /*
Fuad Tabbaa48d1222019-12-09 15:42:32 +00001381 * The exception wasn't handled. Inject to the VM to give it chance to
1382 * handle as an unknown exception.
Fuad Tabba3e9b0222019-11-11 16:47:50 +00001383 */
Fuad Tabbab86325a2020-01-10 13:38:15 +00001384 inject_el1_unknown_exception(vcpu, esr);
1385
1386 /* Schedule the same VM to continue running. */
1387 return NULL;
Fuad Tabba3e9b0222019-11-11 16:47:50 +00001388}
1389
Fuad Tabbac76466d2019-09-06 10:42:12 +01001390/**
Fuad Tabbab0ef2a42019-12-19 11:19:25 +00001391 * Handles EC = 011000, MSR, MRS instruction traps.
Fuad Tabbaed294af2019-12-20 10:43:01 +00001392 * Returns non-null ONLY if the access failed and the vCPU is changing.
Fuad Tabbac76466d2019-09-06 10:42:12 +01001393 */
Fuad Tabbab86325a2020-01-10 13:38:15 +00001394void handle_system_register_access(uintreg_t esr_el2)
Fuad Tabbac76466d2019-09-06 10:42:12 +01001395{
1396 struct vcpu *vcpu = current();
J-Alves19e20cf2023-08-02 12:48:55 +01001397 ffa_id_t vm_id = vcpu->vm->id;
Fuad Tabba3e9b0222019-11-11 16:47:50 +00001398 uintreg_t ec = GET_ESR_EC(esr_el2);
Fuad Tabbac76466d2019-09-06 10:42:12 +01001399
Fuad Tabbab86325a2020-01-10 13:38:15 +00001400 CHECK(ec == EC_MSR);
Fuad Tabbac76466d2019-09-06 10:42:12 +01001401 /*
Fuad Tabbaf1d6dc52019-09-18 17:33:14 +01001402 * Handle accesses to debug and performance monitor registers.
Fuad Tabba3e9b0222019-11-11 16:47:50 +00001403 * Inject an exception for unhandled/unsupported registers.
Fuad Tabbac76466d2019-09-06 10:42:12 +01001404 */
Fuad Tabba3e9b0222019-11-11 16:47:50 +00001405 if (debug_el1_is_register_access(esr_el2)) {
1406 if (!debug_el1_process_access(vcpu, vm_id, esr_el2)) {
Olivier Deprezda14ddc2022-08-11 14:14:41 +02001407 inject_el1_sysreg_trap_exception(vcpu, esr_el2);
Fuad Tabbab86325a2020-01-10 13:38:15 +00001408 return;
Fuad Tabbaf1d6dc52019-09-18 17:33:14 +01001409 }
Fuad Tabba3e9b0222019-11-11 16:47:50 +00001410 } else if (perfmon_is_register_access(esr_el2)) {
1411 if (!perfmon_process_access(vcpu, vm_id, esr_el2)) {
Olivier Deprezda14ddc2022-08-11 14:14:41 +02001412 inject_el1_sysreg_trap_exception(vcpu, esr_el2);
Fuad Tabbab86325a2020-01-10 13:38:15 +00001413 return;
Fuad Tabbaf1d6dc52019-09-18 17:33:14 +01001414 }
Fuad Tabba77a4b012019-11-15 12:13:08 +00001415 } else if (feature_id_is_register_access(esr_el2)) {
1416 if (!feature_id_process_access(vcpu, esr_el2)) {
Olivier Deprezda14ddc2022-08-11 14:14:41 +02001417 inject_el1_sysreg_trap_exception(vcpu, esr_el2);
Fuad Tabbab86325a2020-01-10 13:38:15 +00001418 return;
Fuad Tabba77a4b012019-11-15 12:13:08 +00001419 }
Madhukar Pappireddyf684d192024-09-25 14:35:57 -05001420 } else if (el1_physical_timer_is_register_access(esr_el2)) {
1421 if (!el1_physical_timer_process_access(vcpu, esr_el2)) {
1422 inject_el1_sysreg_trap_exception(vcpu, esr_el2);
1423 return;
1424 }
Fuad Tabbaf1d6dc52019-09-18 17:33:14 +01001425 } else {
Olivier Deprezda14ddc2022-08-11 14:14:41 +02001426 inject_el1_sysreg_trap_exception(vcpu, esr_el2);
Fuad Tabbab86325a2020-01-10 13:38:15 +00001427 return;
Fuad Tabbac76466d2019-09-06 10:42:12 +01001428 }
1429
Fuad Tabbaf1d6dc52019-09-18 17:33:14 +01001430 /* Instruction was fulfilled. Skip it and run the next one. */
Fuad Tabba3e9b0222019-11-11 16:47:50 +00001431 vcpu->regs.pc += GET_NEXT_PC_INC(esr_el2);
Fuad Tabbac76466d2019-09-06 10:42:12 +01001432}