blob: cdc9697dfee1eeeff833b64e3bfc53d48ffdd1a3 [file] [log] [blame]
Andrew Scull18834872018-10-12 11:48:09 +01001/*
Andrew Walbran692b3252019-03-07 15:51:31 +00002 * Copyright 2018 The Hafnium Authors.
Andrew Scull18834872018-10-12 11:48:09 +01003 *
Andrew Walbrane959ec12020-06-17 15:01:09 +01004 * Use of this source code is governed by a BSD-style
5 * license that can be found in the LICENSE file or at
6 * https://opensource.org/licenses/BSD-3-Clause.
Andrew Scull18834872018-10-12 11:48:09 +01007 */
8
Andrew Scullc960c032018-10-24 15:13:35 +01009#include <stdnoreturn.h>
10
Andrew Walbran1f32e722019-06-07 17:57:26 +010011#include "hf/arch/barriers.h"
Madhukar Pappireddy77d3bcd2023-03-01 17:26:22 -060012#include "hf/arch/gicv3.h"
Andrew Scullc960c032018-10-24 15:13:35 +010013#include "hf/arch/init.h"
Olivier Deprez98ad2d22020-05-20 09:52:43 +020014#include "hf/arch/mmu.h"
Maksims Svecovs9ddf86a2021-05-06 17:17:21 +010015#include "hf/arch/plat/ffa.h"
Andrew Scull07b6bd32019-12-12 17:19:55 +000016#include "hf/arch/plat/smc.h"
Andrew Scullc960c032018-10-24 15:13:35 +010017
Andrew Scull18c78fc2018-08-20 12:57:41 +010018#include "hf/api.h"
Fuad Tabbac76466d2019-09-06 10:42:12 +010019#include "hf/check.h"
Andrew Scull18c78fc2018-08-20 12:57:41 +010020#include "hf/cpu.h"
21#include "hf/dlog.h"
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010022#include "hf/ffa.h"
J-Alvesb37fd082020-10-22 12:29:21 +010023#include "hf/ffa_internal.h"
Andrew Sculla9c172d2019-04-03 14:10:00 +010024#include "hf/panic.h"
Manish Pandeya5f39fb2020-09-11 09:47:11 +010025#include "hf/plat/interrupts.h"
Andrew Scull18c78fc2018-08-20 12:57:41 +010026#include "hf/vm.h"
27
Andrew Scullf35a5c92018-08-07 18:09:46 +010028#include "vmapi/hf/call.h"
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +010029
Fuad Tabbac76466d2019-09-06 10:42:12 +010030#include "debug_el1.h"
Fuad Tabba77a4b012019-11-15 12:13:08 +000031#include "feature_id.h"
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +010032#include "msr.h"
Fuad Tabbaf1d6dc52019-09-18 17:33:14 +010033#include "perfmon.h"
Andrew Scull18c78fc2018-08-20 12:57:41 +010034#include "psci.h"
Andrew Walbran33645652019-04-15 12:29:31 +010035#include "psci_handler.h"
Andrew Scull7fd4bb72018-12-08 23:40:12 +000036#include "smc.h"
Fuad Tabbaba8c44d2019-09-23 14:38:58 +010037#include "sysregs.h"
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +010038
Fuad Tabbac76466d2019-09-06 10:42:12 +010039/**
Olivier Deprez98ad2d22020-05-20 09:52:43 +020040 * Hypervisor Fault Address Register Non-Secure.
41 */
42#define HPFAR_EL2_NS (UINT64_C(0x1) << 63)
43
44/**
45 * Hypervisor Fault Address Register Faulting IPA.
46 */
47#define HPFAR_EL2_FIPA (UINT64_C(0xFFFFFFFFFF0))
48
49/**
Fuad Tabbac76466d2019-09-06 10:42:12 +010050 * Gets the value to increment for the next PC.
51 * The ESR encodes whether the instruction is 2 bytes or 4 bytes long.
52 */
Fuad Tabba3e9b0222019-11-11 16:47:50 +000053#define GET_NEXT_PC_INC(esr) (GET_ESR_IL(esr) ? 4 : 2)
Fuad Tabbac76466d2019-09-06 10:42:12 +010054
Fuad Tabbac76466d2019-09-06 10:42:12 +010055/**
Andrew Walbran0dd67ff2019-09-12 16:38:50 +010056 * The Client ID field within X7 for an SMC64 call.
57 */
58#define CLIENT_ID_MASK UINT64_C(0xffff)
59
Daniel Boulbyefa381f2022-01-18 14:49:40 +000060/*
61 * Target function IDs for framework messages from the SPMD.
62 */
Olivier Deprezb76307d2022-06-09 17:17:45 +020063#define SPMD_FWK_MSG_BIT (UINT64_C(1) << 31)
Daniel Boulbyefa381f2022-01-18 14:49:40 +000064#define SPMD_FWK_MSG_FUNC_MASK UINT64_C(0xFF)
Olivier Depreza67ab882023-01-10 15:00:54 +010065#define SPMD_FWK_MSG_PSCI_REQ UINT8_C(0x0)
66#define SPMD_FWK_MSG_PSCI_RESP UINT8_C(0x2)
Daniel Boulbyefa381f2022-01-18 14:49:40 +000067#define SPMD_FWK_MSG_FFA_VERSION_REQ UINT8_C(0x8)
68#define SPMD_FWK_MSG_FFA_VERSION_RESP UINT8_C(0x9)
69
Andrew Walbran0dd67ff2019-09-12 16:38:50 +010070/**
Fuad Tabbac76466d2019-09-06 10:42:12 +010071 * Returns a reference to the currently executing vCPU.
72 */
Andrew Scullc960c032018-10-24 15:13:35 +010073static struct vcpu *current(void)
Andrew Walbran3d84a262018-12-13 14:41:19 +000074{
Daniel Boulby3f784262021-09-27 13:02:54 +010075 // NOLINTNEXTLINE(performance-no-int-to-ptr)
Andrew Walbran3d84a262018-12-13 14:41:19 +000076 return (struct vcpu *)read_msr(tpidr_el2);
77}
78
Andrew Walbran1f8d4872018-12-20 11:21:32 +000079/**
80 * Saves the state of per-vCPU peripherals, such as the virtual timer, and
81 * informs the arch-independent sections that registers have been saved.
82 */
83void complete_saving_state(struct vcpu *vcpu)
84{
Raghu Krishnamurthy32626c92021-01-17 09:57:29 -080085 if (has_vhe_support()) {
86 vcpu->regs.peripherals.cntv_cval_el0 =
87 read_msr(MSR_CNTV_CVAL_EL02);
88 vcpu->regs.peripherals.cntv_ctl_el0 =
89 read_msr(MSR_CNTV_CTL_EL02);
90 } else {
91 vcpu->regs.peripherals.cntv_cval_el0 = read_msr(cntv_cval_el0);
92 vcpu->regs.peripherals.cntv_ctl_el0 = read_msr(cntv_ctl_el0);
93 }
Andrew Walbran1f8d4872018-12-20 11:21:32 +000094
95 api_regs_state_saved(vcpu);
96
97 /*
98 * If switching away from the primary, copy the current EL0 virtual
99 * timer registers to the corresponding EL2 physical timer registers.
100 * This is used to emulate the virtual timer for the primary in case it
101 * should fire while the secondary is running.
102 */
103 if (vcpu->vm->id == HF_PRIMARY_VM_ID) {
104 /*
105 * Clear timer control register before copying compare value, to
106 * avoid a spurious timer interrupt. This could be a problem if
107 * the interrupt is configured as edge-triggered, as it would
108 * then be latched in.
109 */
110 write_msr(cnthp_ctl_el2, 0);
Raghu Krishnamurthy32626c92021-01-17 09:57:29 -0800111
112 if (has_vhe_support()) {
113 write_msr(cnthp_cval_el2, read_msr(MSR_CNTV_CVAL_EL02));
114 write_msr(cnthp_ctl_el2, read_msr(MSR_CNTV_CTL_EL02));
115 } else {
116 write_msr(cnthp_cval_el2, read_msr(cntv_cval_el0));
117 write_msr(cnthp_ctl_el2, read_msr(cntv_ctl_el0));
118 }
Andrew Walbran1f8d4872018-12-20 11:21:32 +0000119 }
120}
121
122/**
123 * Restores the state of per-vCPU peripherals, such as the virtual timer.
124 */
125void begin_restoring_state(struct vcpu *vcpu)
126{
127 /*
128 * Clear timer control register before restoring compare value, to avoid
129 * a spurious timer interrupt. This could be a problem if the interrupt
130 * is configured as edge-triggered, as it would then be latched in.
131 */
Raghu Krishnamurthy32626c92021-01-17 09:57:29 -0800132 if (has_vhe_support()) {
133 write_msr(MSR_CNTV_CTL_EL02, 0);
134 write_msr(MSR_CNTV_CVAL_EL02,
135 vcpu->regs.peripherals.cntv_cval_el0);
136 write_msr(MSR_CNTV_CTL_EL02,
137 vcpu->regs.peripherals.cntv_ctl_el0);
138 } else {
139 write_msr(cntv_ctl_el0, 0);
140 write_msr(cntv_cval_el0, vcpu->regs.peripherals.cntv_cval_el0);
141 write_msr(cntv_ctl_el0, vcpu->regs.peripherals.cntv_ctl_el0);
142 }
Andrew Walbran1f8d4872018-12-20 11:21:32 +0000143
144 /*
145 * If we are switching (back) to the primary, disable the EL2 physical
146 * timer which was being used to emulate the EL0 virtual timer, as the
147 * virtual timer is now running for the primary again.
148 */
149 if (vcpu->vm->id == HF_PRIMARY_VM_ID) {
150 write_msr(cnthp_ctl_el2, 0);
151 write_msr(cnthp_cval_el2, 0);
152 }
153}
154
Andrew Walbran1f32e722019-06-07 17:57:26 +0100155/**
Andrew Walbran1f32e722019-06-07 17:57:26 +0100156 * Invalidate all stage 1 TLB entries on the current (physical) CPU for the
157 * current VMID.
158 */
159static void invalidate_vm_tlb(void)
160{
Andrew Walbrancff1f682019-07-04 14:52:45 +0100161 /*
162 * Ensure that the last VTTBR write has taken effect so we invalidate
163 * the right set of TLB entries.
164 */
Andrew Walbran1f32e722019-06-07 17:57:26 +0100165 isb();
Andrew Walbrancff1f682019-07-04 14:52:45 +0100166
Olivier Deprez0b0ba8c2023-03-17 11:11:53 +0100167 tlbi(vmalle1);
Andrew Walbrancff1f682019-07-04 14:52:45 +0100168
169 /*
170 * Ensure that no instructions are fetched for the VM until after the
171 * TLB invalidation has taken effect.
172 */
Andrew Walbran1f32e722019-06-07 17:57:26 +0100173 isb();
Andrew Walbrancff1f682019-07-04 14:52:45 +0100174
175 /*
176 * Ensure that no data reads or writes for the VM happen until after the
Fuad Tabba77a4b012019-11-15 12:13:08 +0000177 * TLB invalidation has taken effect. Non-shareable is enough because
178 * the TLB is local to the CPU.
Andrew Walbrancff1f682019-07-04 14:52:45 +0100179 */
David Brazdil851948e2019-08-09 12:02:12 +0100180 dsb(nsh);
Andrew Walbran1f32e722019-06-07 17:57:26 +0100181}
182
183/**
184 * Invalidates the TLB if a different vCPU is being run than the last vCPU of
185 * the same VM which was run on the current pCPU.
186 *
187 * This is necessary because VMs may (contrary to the architecture
188 * specification) use inconsistent ASIDs across vCPUs. c.f. KVM's similar
189 * workaround:
190 * https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=94d0e5980d6791b9
191 */
192void maybe_invalidate_tlb(struct vcpu *vcpu)
193{
194 size_t current_cpu_index = cpu_index(vcpu->cpu);
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100195 ffa_vcpu_index_t new_vcpu_index = vcpu_index(vcpu);
Andrew Walbran1f32e722019-06-07 17:57:26 +0100196
197 if (vcpu->vm->arch.last_vcpu_on_cpu[current_cpu_index] !=
198 new_vcpu_index) {
199 /*
200 * The vCPU has changed since the last time this VM was run on
201 * this pCPU, so we need to invalidate the TLB.
202 */
203 invalidate_vm_tlb();
204
205 /* Record the fact that this vCPU is now running on this CPU. */
206 vcpu->vm->arch.last_vcpu_on_cpu[current_cpu_index] =
207 new_vcpu_index;
208 }
209}
210
David Brazdil768f69c2019-12-19 15:46:12 +0000211noreturn void irq_current_exception_noreturn(uintreg_t elr, uintreg_t spsr)
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +0100212{
Wedson Almeida Filho9d5040f2018-10-29 08:41:27 +0000213 (void)elr;
214 (void)spsr;
215
Fuad Tabbad1d67982020-01-08 11:28:29 +0000216 panic("IRQ from current exception level.");
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +0100217}
218
David Brazdil768f69c2019-12-19 15:46:12 +0000219noreturn void fiq_current_exception_noreturn(uintreg_t elr, uintreg_t spsr)
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +0100220{
Wedson Almeida Filho9d5040f2018-10-29 08:41:27 +0000221 (void)elr;
222 (void)spsr;
223
Fuad Tabbad1d67982020-01-08 11:28:29 +0000224 panic("FIQ from current exception level.");
Wedson Almeida Filho9d5040f2018-10-29 08:41:27 +0000225}
226
David Brazdil768f69c2019-12-19 15:46:12 +0000227noreturn void serr_current_exception_noreturn(uintreg_t elr, uintreg_t spsr)
Wedson Almeida Filho9d5040f2018-10-29 08:41:27 +0000228{
229 (void)elr;
230 (void)spsr;
231
Fuad Tabbad1d67982020-01-08 11:28:29 +0000232 panic("SError from current exception level.");
Wedson Almeida Filho9d5040f2018-10-29 08:41:27 +0000233}
234
David Brazdil768f69c2019-12-19 15:46:12 +0000235noreturn void sync_current_exception_noreturn(uintreg_t elr, uintreg_t spsr)
Wedson Almeida Filho9d5040f2018-10-29 08:41:27 +0000236{
237 uintreg_t esr = read_msr(esr_el2);
Fuad Tabba3e9b0222019-11-11 16:47:50 +0000238 uintreg_t ec = GET_ESR_EC(esr);
Wedson Almeida Filho9d5040f2018-10-29 08:41:27 +0000239
240 (void)spsr;
241
Fuad Tabbac76466d2019-09-06 10:42:12 +0100242 switch (ec) {
Fuad Tabbab86325a2020-01-10 13:38:15 +0000243 case EC_DATA_ABORT_SAME_EL:
Andrew Walbrane52006c2019-10-22 18:01:28 +0100244 if (!(esr & (1U << 10))) { /* Check FnV bit. */
Andrew Walbran17eebf92020-02-05 16:35:49 +0000245 dlog_error(
246 "Data abort: pc=%#x, esr=%#x, ec=%#x, "
247 "far=%#x\n",
248 elr, esr, ec, read_msr(far_el2));
Andrew Scull7364a8e2018-07-19 15:39:29 +0100249 } else {
Andrew Walbran17eebf92020-02-05 16:35:49 +0000250 dlog_error(
251 "Data abort: pc=%#x, esr=%#x, ec=%#x, "
252 "far=invalid\n",
253 elr, esr, ec);
Andrew Scull7364a8e2018-07-19 15:39:29 +0100254 }
Wedson Almeida Filhofed69022018-07-11 15:39:12 +0100255
Wedson Almeida Filho81568c42019-01-04 13:33:02 +0000256 break;
Wedson Almeida Filhofed69022018-07-11 15:39:12 +0100257
258 default:
Andrew Walbran17eebf92020-02-05 16:35:49 +0000259 dlog_error(
260 "Unknown current sync exception pc=%#x, esr=%#x, "
261 "ec=%#x\n",
262 elr, esr, ec);
Andrew Scullc960c032018-10-24 15:13:35 +0100263 break;
Wedson Almeida Filhofed69022018-07-11 15:39:12 +0100264 }
Wedson Almeida Filho81568c42019-01-04 13:33:02 +0000265
Andrew Sculla9c172d2019-04-03 14:10:00 +0100266 panic("EL2 exception");
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +0100267}
268
Wedson Almeida Filho03e767a2018-07-30 15:32:03 +0100269/**
Andrew Walbran3d84a262018-12-13 14:41:19 +0000270 * Sets or clears the VI bit in the HCR_EL2 register saved in the given
271 * arch_regs.
272 */
Manish Pandey35e452f2021-02-18 21:36:34 +0000273static void set_virtual_irq(struct arch_regs *r, bool enable)
Andrew Walbran3d84a262018-12-13 14:41:19 +0000274{
275 if (enable) {
Olivier Deprez6d408f92022-08-08 19:14:23 +0200276 r->hyp_state.hcr_el2 |= HCR_EL2_VI;
Andrew Walbran3d84a262018-12-13 14:41:19 +0000277 } else {
Olivier Deprez6d408f92022-08-08 19:14:23 +0200278 r->hyp_state.hcr_el2 &= ~HCR_EL2_VI;
Andrew Walbran3d84a262018-12-13 14:41:19 +0000279 }
280}
281
282/**
283 * Sets or clears the VI bit in the HCR_EL2 register.
284 */
Manish Pandey35e452f2021-02-18 21:36:34 +0000285static void set_virtual_irq_current(bool enable)
Andrew Walbran3d84a262018-12-13 14:41:19 +0000286{
Olivier Deprez6d408f92022-08-08 19:14:23 +0200287 struct vcpu *vcpu = current();
288 uintreg_t hcr_el2 = vcpu->regs.hyp_state.hcr_el2;
Wedson Almeida Filho81568c42019-01-04 13:33:02 +0000289
Andrew Walbran3d84a262018-12-13 14:41:19 +0000290 if (enable) {
291 hcr_el2 |= HCR_EL2_VI;
292 } else {
293 hcr_el2 &= ~HCR_EL2_VI;
294 }
Olivier Deprez6d408f92022-08-08 19:14:23 +0200295 vcpu->regs.hyp_state.hcr_el2 = hcr_el2;
Andrew Walbran3d84a262018-12-13 14:41:19 +0000296}
297
Manish Pandey35e452f2021-02-18 21:36:34 +0000298/**
299 * Sets or clears the VF bit in the HCR_EL2 register saved in the given
300 * arch_regs.
301 */
302static void set_virtual_fiq(struct arch_regs *r, bool enable)
303{
304 if (enable) {
Olivier Deprez6d408f92022-08-08 19:14:23 +0200305 r->hyp_state.hcr_el2 |= HCR_EL2_VF;
Manish Pandey35e452f2021-02-18 21:36:34 +0000306 } else {
Olivier Deprez6d408f92022-08-08 19:14:23 +0200307 r->hyp_state.hcr_el2 &= ~HCR_EL2_VF;
Manish Pandey35e452f2021-02-18 21:36:34 +0000308 }
309}
310
311/**
312 * Sets or clears the VF bit in the HCR_EL2 register.
313 */
314static void set_virtual_fiq_current(bool enable)
315{
Olivier Deprez6d408f92022-08-08 19:14:23 +0200316 struct vcpu *vcpu = current();
317 uintreg_t hcr_el2 = vcpu->regs.hyp_state.hcr_el2;
Manish Pandey35e452f2021-02-18 21:36:34 +0000318
319 if (enable) {
320 hcr_el2 |= HCR_EL2_VF;
321 } else {
322 hcr_el2 &= ~HCR_EL2_VF;
323 }
Olivier Deprez6d408f92022-08-08 19:14:23 +0200324 vcpu->regs.hyp_state.hcr_el2 = hcr_el2;
Manish Pandey35e452f2021-02-18 21:36:34 +0000325}
326
J-Alvesb37fd082020-10-22 12:29:21 +0100327#if SECURE_WORLD == 1
Max Shvetsov1ae74f12020-09-18 13:52:20 +0100328
Max Shvetsov1ae74f12020-09-18 13:52:20 +0100329/**
330 * Handle special direct messages from SPMD to SPMC. For now related to power
331 * management only.
332 */
333static bool spmd_handler(struct ffa_value *args, struct vcpu *current)
334{
J-Alvesd6f4e142021-03-05 13:33:59 +0000335 ffa_vm_id_t sender = ffa_sender(*args);
336 ffa_vm_id_t receiver = ffa_receiver(*args);
Max Shvetsov1ae74f12020-09-18 13:52:20 +0100337 ffa_vm_id_t current_vm_id = current->vm->id;
Daniel Boulbyefa381f2022-01-18 14:49:40 +0000338 uint32_t fwk_msg = ffa_fwk_msg(*args);
339 uint8_t fwk_msg_func_id = fwk_msg & SPMD_FWK_MSG_FUNC_MASK;
Max Shvetsov1ae74f12020-09-18 13:52:20 +0100340
341 /*
Daniel Boulbyefa381f2022-01-18 14:49:40 +0000342 * Check if direct message request is originating from the SPMD,
343 * directed to the SPMC and the message is a framework message.
Max Shvetsov1ae74f12020-09-18 13:52:20 +0100344 */
345 if (!(sender == HF_SPMD_VM_ID && receiver == HF_SPMC_VM_ID &&
Daniel Boulbyefa381f2022-01-18 14:49:40 +0000346 current_vm_id == HF_OTHER_WORLD_ID) ||
347 (fwk_msg & SPMD_FWK_MSG_BIT) == 0) {
Max Shvetsov1ae74f12020-09-18 13:52:20 +0100348 return false;
349 }
350
Olivier Depreza67ab882023-01-10 15:00:54 +0100351 /*
352 * The framework message is conveyed by EL3/SPMD to SPMC so the
353 * current VM id must match to the other world VM id.
354 */
355 CHECK(current->vm->id == HF_HYPERVISOR_VM_ID);
356
Daniel Boulbyefa381f2022-01-18 14:49:40 +0000357 switch (fwk_msg_func_id) {
Olivier Depreza67ab882023-01-10 15:00:54 +0100358 case SPMD_FWK_MSG_PSCI_REQ: {
359 uint32_t psci_msg_response = PSCI_ERROR_NOT_SUPPORTED;
Olivier Deprez181074b2023-02-02 14:53:23 +0100360 struct vcpu *boot_vcpu = vcpu_get_boot_vcpu();
361 struct vm *vm = boot_vcpu->vm;
Olivier Deprez98f151e2023-01-10 15:08:54 +0100362 struct vcpu *vcpu;
363 struct vcpu_locked vcpu_locked;
Olivier Deprez181074b2023-02-02 14:53:23 +0100364
Olivier Depreza67ab882023-01-10 15:00:54 +0100365 /*
366 * TODO: the power management event reached the SPMC.
367 * In a later iteration, the power management event can
368 * be passed to the SP by resuming it.
369 */
Daniel Boulbyefa381f2022-01-18 14:49:40 +0000370 switch (args->arg3) {
371 case PSCI_CPU_OFF: {
Olivier Depreza67ab882023-01-10 15:00:54 +0100372 dlog_verbose("cpu%u off notification!\n",
373 vcpu_index(vcpu));
Daniel Boulbyefa381f2022-01-18 14:49:40 +0000374
Olivier Deprez98f151e2023-01-10 15:08:54 +0100375 if (vm_power_management_cpu_off_requested(vm) == true) {
376 /* Allow only S-EL1 MP SPs to reach here. */
377 CHECK(vm->el0_partition == false);
378 CHECK(vm->vcpu_count > 1);
379
380 vcpu = vm_get_vcpu(vm, vcpu_index(current));
381 vcpu_locked = vcpu_lock(vcpu);
382 vcpu->state = VCPU_STATE_OFF;
383 vcpu_unlock(&vcpu_locked);
384 cpu_off(vcpu->cpu);
385 }
386
Olivier Depreza67ab882023-01-10 15:00:54 +0100387 psci_msg_response = PSCI_RETURN_SUCCESS;
Daniel Boulbyefa381f2022-01-18 14:49:40 +0000388 break;
389 }
390 default:
Olivier Depreza67ab882023-01-10 15:00:54 +0100391 dlog_error(
392 "FF-A PSCI framework message not handled "
393 "%#x %#x %#x %#x\n",
394 args->func, args->arg1, args->arg2, args->arg3);
395 psci_msg_response = PSCI_ERROR_NOT_SUPPORTED;
Daniel Boulbyefa381f2022-01-18 14:49:40 +0000396 }
Olivier Depreza67ab882023-01-10 15:00:54 +0100397
398 *args = (struct ffa_value){
399 .func = FFA_MSG_SEND_DIRECT_RESP_32,
400 .arg1 = ((uint64_t)HF_SPMC_VM_ID << 16) | HF_SPMD_VM_ID,
401 .arg2 = SPMD_FWK_MSG_BIT | SPMD_FWK_MSG_PSCI_RESP,
402 .arg3 = psci_msg_response};
403
404 return true;
Daniel Boulbyefa381f2022-01-18 14:49:40 +0000405 }
406 case SPMD_FWK_MSG_FFA_VERSION_REQ: {
407 struct ffa_value ret = api_ffa_version(current, args->arg3);
Max Shvetsov1ae74f12020-09-18 13:52:20 +0100408 *args = (struct ffa_value){
409 .func = FFA_MSG_SEND_DIRECT_RESP_32,
410 .arg1 = ((uint64_t)HF_SPMC_VM_ID << 16) | HF_SPMD_VM_ID,
Daniel Boulbyefa381f2022-01-18 14:49:40 +0000411 /* Set bit 31 since this is a framework message. */
412 .arg2 = SPMD_FWK_MSG_BIT |
413 SPMD_FWK_MSG_FFA_VERSION_RESP,
414 .arg3 = ret.func};
Olivier Depreza67ab882023-01-10 15:00:54 +0100415 return true;
Max Shvetsov1ae74f12020-09-18 13:52:20 +0100416 }
417 default:
Olivier Depreza67ab882023-01-10 15:00:54 +0100418 dlog_error("FF-A framework message not handled %#x\n",
419 args->arg2);
420
421 /*
422 * TODO: the framework message that was conveyed by a direct
423 * request is not handled although we still want to complete
424 * by a direct response. However, there is no defined error
425 * response to state that the message couldn't be handled.
426 * An alternative would be to return FFA_ERROR.
427 */
Daniel Boulbyefa381f2022-01-18 14:49:40 +0000428 *args = (struct ffa_value){
429 .func = FFA_MSG_SEND_DIRECT_RESP_32,
430 .arg1 = ((uint64_t)HF_SPMC_VM_ID << 16) | HF_SPMD_VM_ID,
431 /* Set bit 31 since this is a framework message. */
432 .arg2 = SPMD_FWK_MSG_BIT | fwk_msg_func_id};
Olivier Depreza67ab882023-01-10 15:00:54 +0100433
434 return true;
Max Shvetsov1ae74f12020-09-18 13:52:20 +0100435 }
436
Olivier Depreza67ab882023-01-10 15:00:54 +0100437 /* Should not reach this point. */
438 assert(false);
439
440 return false;
Max Shvetsov1ae74f12020-09-18 13:52:20 +0100441}
442
J-Alvesb37fd082020-10-22 12:29:21 +0100443#endif
444
Andrew Scullae9962e2019-10-03 16:51:16 +0100445/**
446 * Checks whether to block an SMC being forwarded from a VM.
447 */
448static bool smc_is_blocked(const struct vm *vm, uint32_t func)
Andrew Walbranc1ad4ce2019-05-09 11:41:39 +0100449{
Andrew Scullae9962e2019-10-03 16:51:16 +0100450 bool block_by_default = !vm->smc_whitelist.permissive;
Fuad Tabba8176e3e2019-08-01 10:40:36 +0100451
Andrew Scullae9962e2019-10-03 16:51:16 +0100452 for (size_t i = 0; i < vm->smc_whitelist.smc_count; ++i) {
453 if (func == vm->smc_whitelist.smcs[i]) {
454 return false;
455 }
456 }
Fuad Tabba8176e3e2019-08-01 10:40:36 +0100457
Olivier Deprezf92e5d42020-11-13 16:00:54 +0100458 dlog_notice("SMC %#010x attempted from VM %#x, blocked=%u\n", func,
Andrew Walbran17eebf92020-02-05 16:35:49 +0000459 vm->id, block_by_default);
Andrew Scullae9962e2019-10-03 16:51:16 +0100460
461 /* Access is still allowed in permissive mode. */
462 return block_by_default;
Fuad Tabba8176e3e2019-08-01 10:40:36 +0100463}
464
465/**
Andrew Scullae9962e2019-10-03 16:51:16 +0100466 * Applies SMC access control according to manifest and forwards the call if
467 * access is granted.
Fuad Tabba8176e3e2019-08-01 10:40:36 +0100468 */
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100469static void smc_forwarder(const struct vm *vm, struct ffa_value *args)
Fuad Tabba8176e3e2019-08-01 10:40:36 +0100470{
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100471 struct ffa_value ret;
Andrew Walbran9dadaf22019-12-05 16:50:55 +0000472 uint32_t client_id = vm->id;
473 uintreg_t arg7 = args->arg7;
Andrew Scullae9962e2019-10-03 16:51:16 +0100474
Andrew Walbran9dadaf22019-12-05 16:50:55 +0000475 if (smc_is_blocked(vm, args->func)) {
476 args->func = SMCCC_ERROR_UNKNOWN;
Andrew Scullae9962e2019-10-03 16:51:16 +0100477 return;
478 }
479
Andrew Walbran0dd67ff2019-09-12 16:38:50 +0100480 /*
481 * Set the Client ID but keep the existing Secure OS ID and anything
482 * else (currently unspecified) that the client may have passed in the
483 * upper bits.
484 */
Andrew Walbran9dadaf22019-12-05 16:50:55 +0000485 args->arg7 = client_id | (arg7 & ~CLIENT_ID_MASK);
Andrew Scull07b6bd32019-12-12 17:19:55 +0000486 ret = smc_forward(args->func, args->arg1, args->arg2, args->arg3,
487 args->arg4, args->arg5, args->arg6, args->arg7);
Fuad Tabba8176e3e2019-08-01 10:40:36 +0100488
Andrew Scullae9962e2019-10-03 16:51:16 +0100489 /*
Fuad Tabbab0ef2a42019-12-19 11:19:25 +0000490 * Preserve the value passed by the caller, rather than the generated
491 * client_id. Note that this would also overwrite any return value that
Andrew Scullae9962e2019-10-03 16:51:16 +0100492 * may be in x7, but the SMCs that we are forwarding are legacy calls
493 * from before SMCCC 1.2 so won't have more than 4 return values anyway.
494 */
Andrew Scull07b6bd32019-12-12 17:19:55 +0000495 ret.arg7 = arg7;
496
497 plat_smc_post_forward(*args, &ret);
498
499 *args = ret;
Fuad Tabba8176e3e2019-08-01 10:40:36 +0100500}
501
Olivier Deprezf33a6c72020-06-09 18:28:45 +0200502/**
503 * In the normal world, ffa_handler is always called from the virtual FF-A
Andrew Walbran8e8bf3f2020-10-07 17:58:20 +0100504 * instance (from a VM in EL1). In the secure world, ffa_handler may be called
505 * from the virtual (a secure partition in S-EL1) or physical FF-A instance
506 * (from the normal world via EL3). The function returns true when the call is
507 * handled. The *next pointer is updated to the next vCPU to run, which might be
508 * the 'other world' vCPU if the call originated from the virtual FF-A instance
509 * and has to be forwarded down to EL3, or left as is to resume the current
510 * vCPU.
Olivier Deprezf33a6c72020-06-09 18:28:45 +0200511 */
512static bool ffa_handler(struct ffa_value *args, struct vcpu *current,
513 struct vcpu **next)
Andrew Walbran7d28d9a2019-08-30 16:24:58 +0100514{
J-Alvesbc3de8b2020-12-07 14:32:04 +0000515 uint32_t func = args->func;
Andrew Walbrane7ad3c02019-12-24 17:03:04 +0000516
Jose Marinhoc0f4ff22019-10-09 10:37:42 +0100517 /*
518 * NOTE: When adding new methods to this handler update
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100519 * api_ffa_features accordingly.
Jose Marinhoc0f4ff22019-10-09 10:37:42 +0100520 */
Andrew Walbrane7ad3c02019-12-24 17:03:04 +0000521 switch (func) {
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100522 case FFA_VERSION_32:
Daniel Boulbybaeaf2e2021-12-09 11:42:36 +0000523 *args = api_ffa_version(current, args->arg1);
Andrew Walbran7d28d9a2019-08-30 16:24:58 +0100524 return true;
Fuad Tabbae4efcc32020-07-16 15:37:27 +0100525 case FFA_PARTITION_INFO_GET_32: {
526 struct ffa_uuid uuid;
527
528 ffa_uuid_init(args->arg1, args->arg2, args->arg3, args->arg4,
529 &uuid);
Daniel Boulbyb46cad12021-12-13 17:47:21 +0000530 *args = api_ffa_partition_info_get(current, &uuid, args->arg5);
Fuad Tabbae4efcc32020-07-16 15:37:27 +0100531 return true;
532 }
Raghu Krishnamurthy7592bcb2022-12-25 13:09:00 -0800533 case FFA_PARTITION_INFO_GET_REGS_64: {
534 struct ffa_uuid uuid;
535 uint32_t w0;
536 uint32_t w1;
537 uint32_t w2;
538 uint32_t w3;
539 uint16_t start_index;
540 uint16_t tag;
541
542 w0 = (uint32_t)(args->arg1 & 0xFFFFFFFF);
543 w1 = (uint32_t)(args->arg1 >> 32);
544 w2 = (uint32_t)(args->arg2 & 0xFFFFFFFF);
545 w3 = (uint32_t)(args->arg2 >> 32);
546 ffa_uuid_init(w0, w1, w2, w3, &uuid);
547
Raghu Krishnamurthyd29411a2023-02-17 17:22:04 -0800548 start_index = args->arg3 & 0xFFFF;
549 tag = (args->arg3 >> 16) & 0xFFFF;
Raghu Krishnamurthy7592bcb2022-12-25 13:09:00 -0800550 *args = api_ffa_partition_info_get_regs(current, &uuid,
551 start_index, tag);
552 return true;
553 }
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100554 case FFA_ID_GET_32:
Olivier Deprezf33a6c72020-06-09 18:28:45 +0200555 *args = api_ffa_id_get(current);
Andrew Walbrand230f662019-10-07 18:03:36 +0100556 return true;
Daniel Boulbyb2fb80e2021-02-03 15:09:23 +0000557 case FFA_SPM_ID_GET_32:
558 *args = api_ffa_spm_id_get();
559 return true;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100560 case FFA_FEATURES_32:
Karl Meakin34b8ae92023-01-13 13:33:07 +0000561 *args = api_ffa_features(args->arg1, args->arg2,
562 current->vm->ffa_version);
Jose Marinhoc0f4ff22019-10-09 10:37:42 +0100563 return true;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100564 case FFA_RX_RELEASE_32:
J-Alvese8c8c2b2022-12-16 15:34:48 +0000565 *args = api_ffa_rx_release(ffa_receiver(*args), current);
Andrew Walbran8a0f5ca2019-11-05 13:12:23 +0000566 return true;
J-Alvesbc3de8b2020-12-07 14:32:04 +0000567 case FFA_RXTX_MAP_64:
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100568 *args = api_ffa_rxtx_map(ipa_init(args->arg1),
569 ipa_init(args->arg2), args->arg3,
Federico Recanati9f1b6532022-04-14 13:15:28 +0200570 current);
Andrew Walbranbfffb0f2019-11-05 14:02:34 +0000571 return true;
Daniel Boulby9e420ca2021-07-07 15:03:49 +0100572 case FFA_RXTX_UNMAP_32:
J-Alves70079932022-12-07 17:32:20 +0000573 *args = api_ffa_rxtx_unmap(ffa_vm_id(*args), current);
Daniel Boulby9e420ca2021-07-07 15:03:49 +0100574 return true;
Federico Recanati644f0462022-03-17 12:04:00 +0100575 case FFA_RX_ACQUIRE_32:
576 *args = api_ffa_rx_acquire(ffa_receiver(*args), current);
577 return true;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100578 case FFA_YIELD_32:
Olivier Deprezf33a6c72020-06-09 18:28:45 +0200579 *args = api_yield(current, next);
Andrew Walbran7d28d9a2019-08-30 16:24:58 +0100580 return true;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100581 case FFA_MSG_SEND_32:
J-Alves27b71962022-12-12 15:29:58 +0000582 *args = plat_ffa_msg_send(
583 ffa_sender(*args), ffa_receiver(*args),
584 ffa_msg_send_size(*args), current, next);
Andrew Walbran7d28d9a2019-08-30 16:24:58 +0100585 return true;
Federico Recanati25053ee2022-03-14 15:01:53 +0100586 case FFA_MSG_SEND2_32:
587 *args = api_ffa_msg_send2(ffa_sender(*args),
588 ffa_msg_send2_flags(*args), current);
589 return true;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100590 case FFA_MSG_WAIT_32:
Madhukar Pappireddy5522c672021-12-17 16:35:51 -0600591 *args = api_ffa_msg_wait(current, next, args);
Andrew Walbran0de4f162019-09-03 16:44:20 +0100592 return true;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100593 case FFA_MSG_POLL_32:
Olivier Deprezf33a6c72020-06-09 18:28:45 +0200594 *args = api_ffa_msg_recv(false, current, next);
Andrew Walbran7d28d9a2019-08-30 16:24:58 +0100595 return true;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100596 case FFA_RUN_32:
597 *args = api_ffa_run(ffa_vm_id(*args), ffa_vcpu_index(*args),
Olivier Deprezf33a6c72020-06-09 18:28:45 +0200598 current, next);
Andrew Walbranf0c314d2019-10-02 14:24:26 +0100599 return true;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100600 case FFA_MEM_DONATE_32:
601 case FFA_MEM_LEND_32:
602 case FFA_MEM_SHARE_32:
603 *args = api_ffa_mem_send(func, args->arg1, args->arg2,
604 ipa_init(args->arg3), args->arg4,
Olivier Deprezf33a6c72020-06-09 18:28:45 +0200605 current);
Andrew Walbran82d6d152019-12-24 15:02:06 +0000606 return true;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100607 case FFA_MEM_RETRIEVE_REQ_32:
608 *args = api_ffa_mem_retrieve_req(args->arg1, args->arg2,
609 ipa_init(args->arg3),
Olivier Deprezf33a6c72020-06-09 18:28:45 +0200610 args->arg4, current);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000611 return true;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100612 case FFA_MEM_RELINQUISH_32:
Olivier Deprezf33a6c72020-06-09 18:28:45 +0200613 *args = api_ffa_mem_relinquish(current);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000614 return true;
Andrew Walbranb5ab43c2020-04-30 11:32:54 +0100615 case FFA_MEM_RECLAIM_32:
616 *args = api_ffa_mem_reclaim(
Andrew Walbran1bbe9402020-04-30 16:47:13 +0100617 ffa_assemble_handle(args->arg1, args->arg2), args->arg3,
Olivier Deprezf33a6c72020-06-09 18:28:45 +0200618 current);
Andrew Walbran5de9c3d2020-02-10 13:35:29 +0000619 return true;
Andrew Walbranca808b12020-05-15 17:22:28 +0100620 case FFA_MEM_FRAG_RX_32:
621 *args = api_ffa_mem_frag_rx(ffa_frag_handle(*args), args->arg3,
622 (args->arg4 >> 16) & 0xffff,
Olivier Deprezf33a6c72020-06-09 18:28:45 +0200623 current);
Andrew Walbranca808b12020-05-15 17:22:28 +0100624 return true;
625 case FFA_MEM_FRAG_TX_32:
626 *args = api_ffa_mem_frag_tx(ffa_frag_handle(*args), args->arg3,
627 (args->arg4 >> 16) & 0xffff,
Olivier Deprezf33a6c72020-06-09 18:28:45 +0200628 current);
Andrew Walbranca808b12020-05-15 17:22:28 +0100629 return true;
J-Alvesbc3de8b2020-12-07 14:32:04 +0000630 case FFA_MSG_SEND_DIRECT_REQ_64:
Max Shvetsov1ae74f12020-09-18 13:52:20 +0100631 case FFA_MSG_SEND_DIRECT_REQ_32: {
632#if SECURE_WORLD == 1
633 if (spmd_handler(args, current)) {
634 return true;
635 }
636#endif
J-Alvesd6f4e142021-03-05 13:33:59 +0000637 *args = api_ffa_msg_send_direct_req(ffa_sender(*args),
638 ffa_receiver(*args), *args,
639 current, next);
Olivier Deprezee9d6a92019-11-26 09:14:11 +0000640 return true;
Max Shvetsov1ae74f12020-09-18 13:52:20 +0100641 }
J-Alvesbc3de8b2020-12-07 14:32:04 +0000642 case FFA_MSG_SEND_DIRECT_RESP_64:
Olivier Deprezee9d6a92019-11-26 09:14:11 +0000643 case FFA_MSG_SEND_DIRECT_RESP_32:
J-Alvesd6f4e142021-03-05 13:33:59 +0000644 *args = api_ffa_msg_send_direct_resp(ffa_sender(*args),
645 ffa_receiver(*args), *args,
646 current, next);
Olivier Deprezee9d6a92019-11-26 09:14:11 +0000647 return true;
J-Alvesbc3de8b2020-12-07 14:32:04 +0000648 case FFA_SECONDARY_EP_REGISTER_64:
Olivier Deprezd614d322021-06-18 15:21:00 +0200649 /*
650 * DEN0077A FF-A v1.1 Beta0 section 18.3.2.1.1
651 * The callee must return NOT_SUPPORTED if this function is
652 * invoked by a caller that implements version v1.0 of
653 * the Framework.
654 */
Max Shvetsov40108e72020-08-27 12:39:50 +0100655 *args = api_ffa_secondary_ep_register(ipa_init(args->arg1),
656 current);
657 return true;
J-Alvesa0f317d2021-06-09 13:31:59 +0100658 case FFA_NOTIFICATION_BITMAP_CREATE_32:
659 *args = api_ffa_notification_bitmap_create(
660 (ffa_vm_id_t)args->arg1, (ffa_vcpu_count_t)args->arg2,
661 current);
662 return true;
663 case FFA_NOTIFICATION_BITMAP_DESTROY_32:
664 *args = api_ffa_notification_bitmap_destroy(
665 (ffa_vm_id_t)args->arg1, current);
666 return true;
J-Alvesc003a7a2021-03-18 13:06:53 +0000667 case FFA_NOTIFICATION_BIND_32:
668 *args = api_ffa_notification_update_bindings(
669 ffa_sender(*args), ffa_receiver(*args), args->arg2,
670 ffa_notifications_bitmap(args->arg3, args->arg4), true,
671 current);
672 return true;
673 case FFA_NOTIFICATION_UNBIND_32:
674 *args = api_ffa_notification_update_bindings(
675 ffa_sender(*args), ffa_receiver(*args), 0,
676 ffa_notifications_bitmap(args->arg3, args->arg4), false,
677 current);
678 return true;
Raghu Krishnamurthyea6d25f2021-09-14 15:27:06 -0700679 case FFA_MEM_PERM_SET_32:
680 case FFA_MEM_PERM_SET_64:
681 *args = api_ffa_mem_perm_set(va_init(args->arg1), args->arg2,
682 args->arg3, current);
683 return true;
684 case FFA_MEM_PERM_GET_32:
685 case FFA_MEM_PERM_GET_64:
686 *args = api_ffa_mem_perm_get(va_init(args->arg1), current);
687 return true;
J-Alvesaa79c012021-07-09 14:29:45 +0100688 case FFA_NOTIFICATION_SET_32:
689 *args = api_ffa_notification_set(
690 ffa_sender(*args), ffa_receiver(*args), args->arg2,
691 ffa_notifications_bitmap(args->arg3, args->arg4),
692 current);
693 return true;
694 case FFA_NOTIFICATION_GET_32:
695 *args = api_ffa_notification_get(
J-Alvesbe6e3032021-11-30 14:54:12 +0000696 ffa_receiver(*args), ffa_notifications_get_vcpu(*args),
697 args->arg2, current);
J-Alvesaa79c012021-07-09 14:29:45 +0100698 return true;
J-Alvesc8e8a222021-06-08 17:33:52 +0100699 case FFA_NOTIFICATION_INFO_GET_64:
700 *args = api_ffa_notification_info_get(current);
701 return true;
Madhukar Pappireddy9e7a11f2021-08-03 13:59:42 -0500702 case FFA_INTERRUPT_32:
Madhukar Pappireddydc0c8012022-06-21 15:23:14 -0500703 *args = plat_ffa_handle_secure_interrupt(current, next, true);
Madhukar Pappireddy9e7a11f2021-08-03 13:59:42 -0500704 return true;
Maksims Svecovs71b76702022-05-20 15:32:58 +0100705 case FFA_CONSOLE_LOG_32:
706 case FFA_CONSOLE_LOG_64:
707 *args = api_ffa_console_log(*args, current);
708 return true;
Andrew Walbranf0c314d2019-10-02 14:24:26 +0100709 }
Andrew Walbran7d28d9a2019-08-30 16:24:58 +0100710
711 return false;
712}
713
714/**
Manish Pandey35e452f2021-02-18 21:36:34 +0000715 * Set or clear VI/VF bits according to pending interrupts.
Andrew Walbran7d28d9a2019-08-30 16:24:58 +0100716 */
Manish Pandey35e452f2021-02-18 21:36:34 +0000717static void vcpu_update_virtual_interrupts(struct vcpu *next)
Andrew Walbran7d28d9a2019-08-30 16:24:58 +0100718{
Manish Pandey35e452f2021-02-18 21:36:34 +0000719 struct vcpu_locked vcpu_locked;
720
Andrew Walbran7d28d9a2019-08-30 16:24:58 +0100721 if (next == NULL) {
Raghu Krishnamurthydce438c2021-02-28 15:01:03 -0800722 if (current()->vm->el0_partition) {
723 return;
724 }
725
Andrew Walbran7d28d9a2019-08-30 16:24:58 +0100726 /*
727 * Not switching vCPUs, set the bit for the current vCPU
728 * directly in the register.
729 */
Manish Pandey35e452f2021-02-18 21:36:34 +0000730 vcpu_locked = vcpu_lock(current());
731 set_virtual_irq_current(
732 vcpu_interrupt_irq_count_get(vcpu_locked) > 0);
733 set_virtual_fiq_current(
734 vcpu_interrupt_fiq_count_get(vcpu_locked) > 0);
735 vcpu_unlock(&vcpu_locked);
Olivier Deprez3caed1c2021-02-05 12:07:36 +0100736 } else if (vm_id_is_current_world(next->vm->id)) {
Raghu Krishnamurthydce438c2021-02-28 15:01:03 -0800737 if (next->vm->el0_partition) {
738 return;
739 }
Andrew Walbran7d28d9a2019-08-30 16:24:58 +0100740 /*
741 * About to switch vCPUs, set the bit for the vCPU to which we
742 * are switching in the saved copy of the register.
743 */
Manish Pandey35e452f2021-02-18 21:36:34 +0000744
745 vcpu_locked = vcpu_lock(next);
746 set_virtual_irq(&next->regs,
747 vcpu_interrupt_irq_count_get(vcpu_locked) > 0);
748 set_virtual_fiq(&next->regs,
749 vcpu_interrupt_fiq_count_get(vcpu_locked) > 0);
750 vcpu_unlock(&vcpu_locked);
Andrew Walbran7d28d9a2019-08-30 16:24:58 +0100751 }
752}
753
Fuad Tabba8176e3e2019-08-01 10:40:36 +0100754/**
Andrew Walbrand8d3f5d2020-10-07 18:23:01 +0100755 * Handles PSCI and FF-A calls and writes the return value back to the registers
756 * of the vCPU. This is shared between smc_handler and hvc_handler.
757 *
758 * Returns true if the call was handled.
759 */
760static bool hvc_smc_handler(struct ffa_value args, struct vcpu *vcpu,
761 struct vcpu **next)
762{
Olivier Deprez3caed1c2021-02-05 12:07:36 +0100763 /* Do not expect PSCI calls emitted from within the secure world. */
764#if SECURE_WORLD == 0
Andrew Walbrand8d3f5d2020-10-07 18:23:01 +0100765 if (psci_handler(vcpu, args.func, args.arg1, args.arg2, args.arg3,
766 &vcpu->regs.r[0], next)) {
767 return true;
768 }
Olivier Deprez3caed1c2021-02-05 12:07:36 +0100769#endif
Andrew Walbrand8d3f5d2020-10-07 18:23:01 +0100770
Andrew Walbrand8d3f5d2020-10-07 18:23:01 +0100771 if (ffa_handler(&args, vcpu, next)) {
J-Alves13394022021-06-30 13:48:49 +0100772#if SECURE_WORLD == 1
773 /*
774 * If giving back execution to the NWd, check if the Schedule
Olivier Deprez618c8fc2022-05-30 15:27:49 +0200775 * Receiver Interrupt has been delayed, and trigger it on
776 * current core if so.
J-Alves13394022021-06-30 13:48:49 +0100777 */
778 if ((*next != NULL && (*next)->vm->id == HF_OTHER_WORLD_ID) ||
779 (*next == NULL && vcpu->vm->id == HF_OTHER_WORLD_ID)) {
780 plat_ffa_sri_trigger_if_delayed(vcpu->cpu);
781 }
782#endif
Andrew Walbrand8d3f5d2020-10-07 18:23:01 +0100783 arch_regs_set_retval(&vcpu->regs, args);
Manish Pandey35e452f2021-02-18 21:36:34 +0000784 vcpu_update_virtual_interrupts(*next);
Andrew Walbrand8d3f5d2020-10-07 18:23:01 +0100785 return true;
786 }
787
788 return false;
789}
790
791/**
Fuad Tabba8176e3e2019-08-01 10:40:36 +0100792 * Processes SMC instruction calls.
793 */
Andrew Walbran9dadaf22019-12-05 16:50:55 +0000794static struct vcpu *smc_handler(struct vcpu *vcpu)
Fuad Tabba8176e3e2019-08-01 10:40:36 +0100795{
Andrew Walbrand8d3f5d2020-10-07 18:23:01 +0100796 struct ffa_value args = arch_regs_get_args(&vcpu->regs);
Andrew Walbran9dadaf22019-12-05 16:50:55 +0000797 struct vcpu *next = NULL;
Fuad Tabba8176e3e2019-08-01 10:40:36 +0100798
Andrew Walbrand8d3f5d2020-10-07 18:23:01 +0100799 if (hvc_smc_handler(args, vcpu, &next)) {
Andrew Walbran9dadaf22019-12-05 16:50:55 +0000800 return next;
Andrew Walbran4579f7002019-08-30 16:24:58 +0100801 }
802
Andrew Walbran9dadaf22019-12-05 16:50:55 +0000803 smc_forwarder(vcpu->vm, &args);
804 arch_regs_set_retval(&vcpu->regs, args);
Andrew Scull07b6bd32019-12-12 17:19:55 +0000805 return NULL;
Andrew Walbranc1ad4ce2019-05-09 11:41:39 +0100806}
807
Olivier Deprez3caed1c2021-02-05 12:07:36 +0100808#if SECURE_WORLD == 1
809
810/**
811 * Called from other_world_loop return from SMC.
812 * Processes SMC calls originating from the NWd.
813 */
814struct vcpu *smc_handler_from_nwd(struct vcpu *vcpu)
815{
816 struct ffa_value args = arch_regs_get_args(&vcpu->regs);
817 struct vcpu *next = NULL;
818
819 if (hvc_smc_handler(args, vcpu, &next)) {
820 return next;
821 }
822
823 /*
824 * If the SMC emitted by the normal world is not handled in the secure
825 * world then return an error stating such ABI is not supported. Only
826 * FF-A calls are supported. We cannot return SMCCC_ERROR_UNKNOWN
827 * directly because the SPMD smc handler would not recognize it as a
828 * standard FF-A call returning from the SPMC.
829 */
830 arch_regs_set_retval(&vcpu->regs, ffa_error(FFA_NOT_SUPPORTED));
831
832 return NULL;
833}
834
835#endif
836
Fuad Tabbaa48d1222019-12-09 15:42:32 +0000837/*
838 * Exception vector offsets.
839 * See Arm Architecture Reference Manual Armv8-A, D1.10.2.
840 */
841
842/**
843 * Offset for synchronous exceptions at current EL with SPx.
844 */
845#define OFFSET_CURRENT_SPX UINT64_C(0x200)
846
847/**
848 * Offset for synchronous exceptions at lower EL using AArch64.
849 */
850#define OFFSET_LOWER_EL_64 UINT64_C(0x400)
851
852/**
853 * Offset for synchronous exceptions at lower EL using AArch32.
854 */
855#define OFFSET_LOWER_EL_32 UINT64_C(0x600)
856
857/**
858 * Returns the address for the exception handler at EL1.
859 */
860static uintreg_t get_el1_exception_handler_addr(const struct vcpu *vcpu)
861{
Raghu Krishnamurthy32626c92021-01-17 09:57:29 -0800862 uintreg_t base_addr = has_vhe_support() ? read_msr(MSR_VBAR_EL12)
863 : read_msr(vbar_el1);
Fuad Tabbaa48d1222019-12-09 15:42:32 +0000864 uintreg_t pe_mode = vcpu->regs.spsr & PSR_PE_MODE_MASK;
865 bool is_arch32 = vcpu->regs.spsr & PSR_ARCH_MODE_32;
866
867 if (pe_mode == PSR_PE_MODE_EL0T) {
868 if (is_arch32) {
869 base_addr += OFFSET_LOWER_EL_32;
870 } else {
871 base_addr += OFFSET_LOWER_EL_64;
872 }
873 } else {
874 CHECK(!is_arch32);
875 base_addr += OFFSET_CURRENT_SPX;
876 }
877
878 return base_addr;
879}
880
881/**
Fuad Tabbab86325a2020-01-10 13:38:15 +0000882 * Injects an exception with the specified Exception Syndrom Register value into
883 * the EL1.
Fuad Tabbaa48d1222019-12-09 15:42:32 +0000884 *
885 * NOTE: This function assumes that the lazy registers haven't been saved, and
886 * writes to the lazy registers of the CPU directly instead of the vCPU.
887 */
Fuad Tabbac3847c72020-08-11 09:32:25 +0100888static void inject_el1_exception(struct vcpu *vcpu, uintreg_t esr_el1_value,
889 uintreg_t far_el1_value)
Fuad Tabbaa48d1222019-12-09 15:42:32 +0000890{
Fuad Tabbaa48d1222019-12-09 15:42:32 +0000891 uintreg_t handler_address = get_el1_exception_handler_addr(vcpu);
Fuad Tabbaa48d1222019-12-09 15:42:32 +0000892
893 /* Update the CPU state to inject the exception. */
Raghu Krishnamurthy32626c92021-01-17 09:57:29 -0800894 if (has_vhe_support()) {
895 write_msr(MSR_ESR_EL12, esr_el1_value);
896 write_msr(MSR_FAR_EL12, far_el1_value);
897 write_msr(MSR_ELR_EL12, vcpu->regs.pc);
898 write_msr(MSR_SPSR_EL12, vcpu->regs.spsr);
899 } else {
900 write_msr(esr_el1, esr_el1_value);
901 write_msr(far_el1, far_el1_value);
902 write_msr(elr_el1, vcpu->regs.pc);
903 write_msr(spsr_el1, vcpu->regs.spsr);
904 }
Fuad Tabbaa48d1222019-12-09 15:42:32 +0000905
906 /*
907 * Mask (disable) interrupts and run in EL1h mode.
908 * EL1h mode is used because by default, taking an exception selects the
909 * stack pointer for the target Exception level. The software can change
910 * that later in the handler if needed.
Fuad Tabbaa48d1222019-12-09 15:42:32 +0000911 */
912 vcpu->regs.spsr = PSR_D | PSR_A | PSR_I | PSR_F | PSR_PE_MODE_EL1H;
913
914 /* Transfer control to the exception hander. */
915 vcpu->regs.pc = handler_address;
Fuad Tabbab86325a2020-01-10 13:38:15 +0000916}
917
918/**
919 * Injects a Data Abort exception (same exception level).
920 */
921static void inject_el1_data_abort_exception(struct vcpu *vcpu,
Fuad Tabbac3847c72020-08-11 09:32:25 +0100922 uintreg_t esr_el2,
923 uintreg_t far_el2)
Fuad Tabbab86325a2020-01-10 13:38:15 +0000924{
925 /*
926 * ISS encoding remains the same, but the EC is changed to reflect
927 * where the exception came from.
928 * See Arm Architecture Reference Manual Armv8-A, pages D13-2943/2982.
929 */
930 uintreg_t esr_el1_value = GET_ESR_ISS(esr_el2) | GET_ESR_IL(esr_el2) |
931 (EC_DATA_ABORT_SAME_EL << ESR_EC_OFFSET);
932
Olivier Deprezf92e5d42020-11-13 16:00:54 +0100933 dlog_notice("Injecting Data Abort exception into VM %#x.\n",
Andrew Walbran17eebf92020-02-05 16:35:49 +0000934 vcpu->vm->id);
Fuad Tabbab86325a2020-01-10 13:38:15 +0000935
Fuad Tabbac3847c72020-08-11 09:32:25 +0100936 inject_el1_exception(vcpu, esr_el1_value, far_el2);
Fuad Tabbab86325a2020-01-10 13:38:15 +0000937}
938
939/**
940 * Injects a Data Abort exception (same exception level).
941 */
942static void inject_el1_instruction_abort_exception(struct vcpu *vcpu,
Fuad Tabbac3847c72020-08-11 09:32:25 +0100943 uintreg_t esr_el2,
944 uintreg_t far_el2)
Fuad Tabbab86325a2020-01-10 13:38:15 +0000945{
946 /*
947 * ISS encoding remains the same, but the EC is changed to reflect
948 * where the exception came from.
949 * See Arm Architecture Reference Manual Armv8-A, pages D13-2941/2980.
950 */
951 uintreg_t esr_el1_value =
952 GET_ESR_ISS(esr_el2) | GET_ESR_IL(esr_el2) |
953 (EC_INSTRUCTION_ABORT_SAME_EL << ESR_EC_OFFSET);
954
Olivier Deprezf92e5d42020-11-13 16:00:54 +0100955 dlog_notice("Injecting Instruction Abort exception into VM %#x.\n",
Andrew Walbran17eebf92020-02-05 16:35:49 +0000956 vcpu->vm->id);
Fuad Tabbab86325a2020-01-10 13:38:15 +0000957
Fuad Tabbac3847c72020-08-11 09:32:25 +0100958 inject_el1_exception(vcpu, esr_el1_value, far_el2);
Fuad Tabbab86325a2020-01-10 13:38:15 +0000959}
960
961/**
962 * Injects an exception with an unknown reason into the EL1.
963 */
964static void inject_el1_unknown_exception(struct vcpu *vcpu, uintreg_t esr_el2)
965{
966 uintreg_t esr_el1_value =
967 GET_ESR_IL(esr_el2) | (EC_UNKNOWN << ESR_EC_OFFSET);
Fuad Tabbac3847c72020-08-11 09:32:25 +0100968
Olivier Deprezda14ddc2022-08-11 14:14:41 +0200969 dlog_notice("Injecting Unknown Reason exception into VM %#x.\n",
970 vcpu->vm->id);
971
Fuad Tabbac3847c72020-08-11 09:32:25 +0100972 /*
973 * The value of the far_el2 register is UNKNOWN in this case,
974 * therefore, don't propagate it to avoid leaking sensitive information.
975 */
Olivier Deprezda14ddc2022-08-11 14:14:41 +0200976 inject_el1_exception(vcpu, esr_el1_value, 0);
977}
Fuad Tabbaa48d1222019-12-09 15:42:32 +0000978
Olivier Deprezda14ddc2022-08-11 14:14:41 +0200979/**
980 * Injects an exception because of a system register trap.
981 */
982static void inject_el1_sysreg_trap_exception(struct vcpu *vcpu,
983 uintreg_t esr_el2)
984{
985 char *direction_str = ISS_IS_READ(esr_el2) ? "read" : "write";
986
Andrew Walbran17eebf92020-02-05 16:35:49 +0000987 dlog_notice(
988 "Trapped access to system register %s: op0=%d, op1=%d, crn=%d, "
989 "crm=%d, op2=%d, rt=%d.\n",
990 direction_str, GET_ISS_OP0(esr_el2), GET_ISS_OP1(esr_el2),
991 GET_ISS_CRN(esr_el2), GET_ISS_CRM(esr_el2),
992 GET_ISS_OP2(esr_el2), GET_ISS_RT(esr_el2));
Fuad Tabbaa48d1222019-12-09 15:42:32 +0000993
Olivier Deprezda14ddc2022-08-11 14:14:41 +0200994 inject_el1_unknown_exception(vcpu, esr_el2);
Fuad Tabbaa48d1222019-12-09 15:42:32 +0000995}
996
Andrew Walbrand8d3f5d2020-10-07 18:23:01 +0100997static struct vcpu *hvc_handler(struct vcpu *vcpu)
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +0100998{
Andrew Walbrand8d3f5d2020-10-07 18:23:01 +0100999 struct ffa_value args = arch_regs_get_args(&vcpu->regs);
Andrew Walbran59182d52019-09-23 17:55:39 +01001000 struct vcpu *next = NULL;
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +01001001
Andrew Walbrand8d3f5d2020-10-07 18:23:01 +01001002 if (hvc_smc_handler(args, vcpu, &next)) {
Andrew Walbran59182d52019-09-23 17:55:39 +01001003 return next;
Andrew Walbran7d28d9a2019-08-30 16:24:58 +01001004 }
Jose Marinhofc0b2b62019-06-06 11:18:45 +01001005
Andrew Walbran7f920af2019-09-03 17:09:30 +01001006 switch (args.func) {
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +00001007 case HF_MAILBOX_WRITABLE_GET:
Andrew Walbran59182d52019-09-23 17:55:39 +01001008 vcpu->regs.r[0] = api_mailbox_writable_get(vcpu);
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +00001009 break;
1010
1011 case HF_MAILBOX_WAITER_GET:
Andrew Walbran7f920af2019-09-03 17:09:30 +01001012 vcpu->regs.r[0] = api_mailbox_waiter_get(args.arg1, vcpu);
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +01001013 break;
1014
Wedson Almeida Filhoc559d132019-01-09 19:33:40 +00001015 case HF_INTERRUPT_ENABLE:
Manish Pandey35e452f2021-02-18 21:36:34 +00001016 vcpu->regs.r[0] = api_interrupt_enable(args.arg1, args.arg2,
1017 args.arg3, vcpu);
Andrew Walbran318f5732018-11-20 16:23:42 +00001018 break;
1019
Wedson Almeida Filhoc559d132019-01-09 19:33:40 +00001020 case HF_INTERRUPT_GET:
Andrew Walbran59182d52019-09-23 17:55:39 +01001021 vcpu->regs.r[0] = api_interrupt_get(vcpu);
Andrew Walbran318f5732018-11-20 16:23:42 +00001022 break;
1023
Wedson Almeida Filhoc559d132019-01-09 19:33:40 +00001024 case HF_INTERRUPT_INJECT:
Andrew Walbran7f920af2019-09-03 17:09:30 +01001025 vcpu->regs.r[0] = api_interrupt_inject(args.arg1, args.arg2,
1026 args.arg3, vcpu, &next);
Andrew Walbran318f5732018-11-20 16:23:42 +00001027 break;
1028
Madhukar Pappireddyf675bb62021-08-03 12:57:10 -05001029#if SECURE_WORLD == 1
1030 case HF_INTERRUPT_DEACTIVATE:
1031 vcpu->regs.r[0] = plat_ffa_interrupt_deactivate(
1032 args.arg1, args.arg2, vcpu);
1033 break;
1034#endif
1035
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +01001036 default:
Andrew Walbran59182d52019-09-23 17:55:39 +01001037 vcpu->regs.r[0] = SMCCC_ERROR_UNKNOWN;
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +01001038 }
1039
Manish Pandey35e452f2021-02-18 21:36:34 +00001040 vcpu_update_virtual_interrupts(next);
Andrew Walbran3d84a262018-12-13 14:41:19 +00001041
Andrew Walbran59182d52019-09-23 17:55:39 +01001042 return next;
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +01001043}
1044
Wedson Almeida Filho87009642018-07-02 10:20:07 +01001045struct vcpu *irq_lower(void)
1046{
Madhukar Pappireddycbecc962021-08-03 13:11:57 -05001047#if SECURE_WORLD == 1
1048 struct vcpu *next = NULL;
1049
Madhukar Pappireddydc0c8012022-06-21 15:23:14 -05001050 plat_ffa_handle_secure_interrupt(current(), &next, false);
Madhukar Pappireddycbecc962021-08-03 13:11:57 -05001051
1052 /*
1053 * Since we are in interrupt context, set the bit for the
1054 * next vCPU directly in the register.
1055 */
1056 vcpu_update_virtual_interrupts(next);
1057
1058 return next;
1059#else
Andrew Scull9726c252019-01-23 13:44:19 +00001060 /*
1061 * Switch back to primary VM, interrupts will be handled there.
1062 *
1063 * If the VM has aborted, this vCPU will be aborted when the scheduler
1064 * tries to run it again. This means the interrupt will not be delayed
1065 * by the aborted VM.
1066 *
1067 * TODO: Only switch when the interrupt isn't for the current VM.
1068 */
Andrew Scull33fecd32019-01-08 14:48:27 +00001069 return api_preempt(current());
Madhukar Pappireddycbecc962021-08-03 13:11:57 -05001070#endif
Wedson Almeida Filho87009642018-07-02 10:20:07 +01001071}
1072
Madhukar Pappireddy7fc585e2023-03-02 14:31:22 -06001073#if SECURE_WORLD == 1
1074static void spmd_group0_intr_delegate(void)
1075{
1076 struct ffa_value ret;
1077
1078 dlog_verbose("Delegating Group0 interrupt to SPMD\n");
1079
1080 ret = smc_ffa_call((struct ffa_value){.func = FFA_EL3_INTR_HANDLE_32});
1081
1082 /* Check if the Group0 interrupt was handled successfully. */
1083 CHECK(ret.func == FFA_SUCCESS_32);
1084}
1085#endif
1086
Wedson Almeida Filho9d5040f2018-10-29 08:41:27 +00001087struct vcpu *fiq_lower(void)
1088{
Manish Pandeya5f39fb2020-09-11 09:47:11 +01001089#if SECURE_WORLD == 1
1090 struct vcpu_locked current_locked;
1091 struct vcpu *current_vcpu = current();
Daniel Boulby4dd3f532021-09-21 09:57:08 +01001092 int64_t ret;
Madhukar Pappireddy77d3bcd2023-03-01 17:26:22 -06001093 uint32_t intid;
Manish Pandeya5f39fb2020-09-11 09:47:11 +01001094
Madhukar Pappireddy77d3bcd2023-03-01 17:26:22 -06001095 intid = get_highest_pending_g0_interrupt_id();
1096
1097 /* Check for the highest priority pending Group0 interrupt. */
1098 if (intid != SPURIOUS_INTID_OTHER_WORLD) {
Madhukar Pappireddy7fc585e2023-03-02 14:31:22 -06001099 /* Delegate handling of Group0 interrupt to EL3 firmware. */
1100 spmd_group0_intr_delegate();
1101
1102 /* Resume current vCPU. */
1103 return NULL;
Madhukar Pappireddy77d3bcd2023-03-01 17:26:22 -06001104 }
1105
1106 /*
1107 * A special interrupt indicating there is no pending interrupt
1108 * with sufficient priority for current security state. This
1109 * means a non-secure interrupt is pending.
1110 */
Madhukar Pappireddyc40f55f2022-06-22 11:00:41 -05001111 assert(current_vcpu->vm->ns_interrupts_action != NS_ACTION_QUEUED);
1112
Maksims Svecovs9ddf86a2021-05-06 17:17:21 +01001113 if (plat_ffa_vm_managed_exit_supported(current_vcpu->vm)) {
Madhukar Pappireddydd6fdfb2021-12-14 12:30:36 -06001114 uint8_t pmr = plat_interrupts_get_priority_mask();
1115
Manish Pandeya5f39fb2020-09-11 09:47:11 +01001116 /* Mask all interrupts */
1117 plat_interrupts_set_priority_mask(0x0);
1118
1119 current_locked = vcpu_lock(current_vcpu);
Madhukar Pappireddydd6fdfb2021-12-14 12:30:36 -06001120 current_vcpu->priority_mask = pmr;
Manish Pandeya5f39fb2020-09-11 09:47:11 +01001121 ret = api_interrupt_inject_locked(current_locked,
1122 HF_MANAGED_EXIT_INTID,
1123 current_vcpu, NULL);
1124 if (ret != 0) {
1125 panic("Failed to inject managed exit interrupt\n");
1126 }
1127
1128 /* Entering managed exit sequence. */
1129 current_vcpu->processing_managed_exit = true;
1130
1131 vcpu_unlock(&current_locked);
1132
1133 /*
1134 * Since we are in interrupt context, set the bit for the
1135 * current vCPU directly in the register.
1136 */
1137 vcpu_update_virtual_interrupts(NULL);
1138
1139 /* Resume current vCPU. */
1140 return NULL;
1141 }
Manish Pandeya5f39fb2020-09-11 09:47:11 +01001142
Madhukar Pappireddyd46c06e2022-06-21 18:14:52 -05001143 /*
1144 * Unwind Normal World Scheduled Call chain in response to NS
1145 * Interrupt.
1146 */
1147 return plat_ffa_unwind_nwd_call_chain_interrupt(current_vcpu);
Madhukar Pappireddycbecc962021-08-03 13:11:57 -05001148#else
Wedson Almeida Filho9d5040f2018-10-29 08:41:27 +00001149 return irq_lower();
Madhukar Pappireddycbecc962021-08-03 13:11:57 -05001150#endif
Wedson Almeida Filho9d5040f2018-10-29 08:41:27 +00001151}
1152
Fuad Tabbad1d67982020-01-08 11:28:29 +00001153noreturn struct vcpu *serr_lower(void)
Wedson Almeida Filho9d5040f2018-10-29 08:41:27 +00001154{
Fuad Tabbad1d67982020-01-08 11:28:29 +00001155 /*
1156 * SError exceptions should be isolated and handled by the responsible
1157 * VM/exception level. Getting here indicates a bug, that isolation is
1158 * not working, or a processor that does not support ARMv8.2-IESB, in
1159 * which case Hafnium routes SError exceptions to EL2 (here).
1160 */
1161 panic("SError from a lower exception level.");
Wedson Almeida Filho9d5040f2018-10-29 08:41:27 +00001162}
1163
Wedson Almeida Filho99d2d4c2019-02-14 12:53:46 +00001164/**
1165 * Initialises a fault info structure. It assumes that an FnV bit exists at
1166 * bit offset 10 of the ESR, and that it is only valid when the bottom 6 bits of
1167 * the ESR (the fault status code) are 010000; this is the case for both
1168 * instruction and data aborts, but not necessarily for other exception reasons.
1169 */
1170static struct vcpu_fault_info fault_info_init(uintreg_t esr,
Andrew Walbran1281ed42019-10-22 17:23:40 +01001171 const struct vcpu *vcpu,
1172 uint32_t mode)
Wedson Almeida Filho99d2d4c2019-02-14 12:53:46 +00001173{
1174 uint32_t fsc = esr & 0x3f;
1175 struct vcpu_fault_info r;
Olivier Deprez98ad2d22020-05-20 09:52:43 +02001176 uint64_t hpfar_el2_val;
1177 uint64_t hpfar_el2_fipa;
Wedson Almeida Filho99d2d4c2019-02-14 12:53:46 +00001178
1179 r.mode = mode;
Wedson Almeida Filho99d2d4c2019-02-14 12:53:46 +00001180 r.pc = va_init(vcpu->regs.pc);
1181
Olivier Deprez98ad2d22020-05-20 09:52:43 +02001182 /* Get Hypervisor IPA Fault Address value. */
1183 hpfar_el2_val = read_msr(hpfar_el2);
1184
1185 /* Extract Faulting IPA. */
1186 hpfar_el2_fipa = (hpfar_el2_val & HPFAR_EL2_FIPA) << 8;
1187
1188#if SECURE_WORLD == 1
1189
1190 /**
1191 * Determine if faulting IPA targets NS space.
1192 * At NS-EL2 hpfar_el2 bit 63 is RES0. At S-EL2, this bit determines if
1193 * the faulting Stage-1 address output is a secure or non-secure IPA.
1194 */
1195 if ((hpfar_el2_val & HPFAR_EL2_NS) != 0) {
1196 r.mode |= MM_MODE_NS;
1197 }
1198
1199#endif
1200
Wedson Almeida Filho99d2d4c2019-02-14 12:53:46 +00001201 /*
1202 * Check the FnV bit, which is only valid if dfsc/ifsc is 010000. It
1203 * indicates that we cannot rely on far_el2.
1204 */
Andrew Walbrane52006c2019-10-22 18:01:28 +01001205 if (fsc == 0x10 && esr & (1U << 10)) {
Wedson Almeida Filho99d2d4c2019-02-14 12:53:46 +00001206 r.vaddr = va_init(0);
Olivier Deprez98ad2d22020-05-20 09:52:43 +02001207 r.ipaddr = ipa_init(hpfar_el2_fipa);
Wedson Almeida Filho99d2d4c2019-02-14 12:53:46 +00001208 } else {
1209 r.vaddr = va_init(read_msr(far_el2));
Olivier Deprez98ad2d22020-05-20 09:52:43 +02001210 r.ipaddr = ipa_init(hpfar_el2_fipa |
Wedson Almeida Filho99d2d4c2019-02-14 12:53:46 +00001211 (read_msr(far_el2) & (PAGE_SIZE - 1)));
1212 }
1213
1214 return r;
1215}
1216
Fuad Tabbac3847c72020-08-11 09:32:25 +01001217struct vcpu *sync_lower_exception(uintreg_t esr, uintreg_t far)
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +01001218{
Wedson Almeida Filho00df6c72018-10-18 11:19:24 +01001219 struct vcpu *vcpu = current();
Wedson Almeida Filho99d2d4c2019-02-14 12:53:46 +00001220 struct vcpu_fault_info info;
Raghu Krishnamurthyb5775d22021-02-26 18:54:40 -08001221 struct vcpu *new_vcpu = NULL;
Fuad Tabba3e9b0222019-11-11 16:47:50 +00001222 uintreg_t ec = GET_ESR_EC(esr);
Raghu Krishnamurthyb5775d22021-02-26 18:54:40 -08001223 bool is_el0_partition = vcpu->vm->el0_partition;
Raghu Krishnamurthyf16b2ce2021-11-02 07:48:38 -07001224 bool resume = false;
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +01001225
Fuad Tabbac76466d2019-09-06 10:42:12 +01001226 switch (ec) {
Fuad Tabbab86325a2020-01-10 13:38:15 +00001227 case EC_WFI_WFE:
Andrew Walbran48196eb2019-03-04 14:56:24 +00001228 /* Skip the instruction. */
Fuad Tabbac76466d2019-09-06 10:42:12 +01001229 vcpu->regs.pc += GET_NEXT_PC_INC(esr);
Raghu Krishnamurthyb5775d22021-02-26 18:54:40 -08001230
1231 /*
1232 * For EL0 partitions, treat both WFI and WFE the same way so
1233 * that FFA_RUN can be called on the partition to resume it. If
1234 * we treat WFI using api_wait_for_interrupt, the VCPU will be
1235 * in blocked waiting for interrupt but we cannot inject
1236 * interrupts into EL0 partitions.
1237 */
1238 if (is_el0_partition) {
1239 api_yield(vcpu, &new_vcpu);
1240 return new_vcpu;
1241 }
1242
Wedson Almeida Filho87009642018-07-02 10:20:07 +01001243 /* Check TI bit of ISS, 0 = WFI, 1 = WFE. */
Andrew Scull7364a8e2018-07-19 15:39:29 +01001244 if (esr & 1) {
Andrew Walbran48196eb2019-03-04 14:56:24 +00001245 /* WFE */
1246 /*
1247 * TODO: consider giving the scheduler more context,
1248 * somehow.
1249 */
Andrew Walbran16075b62019-09-03 17:11:07 +01001250 api_yield(vcpu, &new_vcpu);
Jose Marinho135dff32019-02-28 10:25:57 +00001251 return new_vcpu;
Andrew Scull7364a8e2018-07-19 15:39:29 +01001252 }
Andrew Walbran48196eb2019-03-04 14:56:24 +00001253 /* WFI */
Andrew Scull9726c252019-01-23 13:44:19 +00001254 return api_wait_for_interrupt(vcpu);
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +01001255
Fuad Tabbab86325a2020-01-10 13:38:15 +00001256 case EC_DATA_ABORT_LOWER_EL:
Wedson Almeida Filho99d2d4c2019-02-14 12:53:46 +00001257 info = fault_info_init(
Andrew Walbrane52006c2019-10-22 18:01:28 +01001258 esr, vcpu, (esr & (1U << 6)) ? MM_MODE_W : MM_MODE_R);
Raghu Krishnamurthyb5775d22021-02-26 18:54:40 -08001259
Raghu Krishnamurthyf16b2ce2021-11-02 07:48:38 -07001260 resume = vcpu_handle_page_fault(vcpu, &info);
Raghu Krishnamurthyb5775d22021-02-26 18:54:40 -08001261 if (is_el0_partition) {
1262 dlog_warning("Data abort on EL0 partition\n");
Raghu Krishnamurthyf16b2ce2021-11-02 07:48:38 -07001263 /*
1264 * Abort EL0 context if we should not resume the
1265 * context, or it is an alignment fault.
1266 * vcpu_handle_page_fault() only checks the mode of the
1267 * page in an architecture agnostic way but alignment
1268 * faults on aarch64 can happen on a correctly mapped
1269 * page.
1270 */
1271 if (!resume || ((esr & 0x3f) == 0x21)) {
1272 return api_abort(vcpu);
1273 }
1274 }
1275
1276 if (resume) {
1277 return NULL;
Raghu Krishnamurthyb5775d22021-02-26 18:54:40 -08001278 }
1279
Fuad Tabbab86325a2020-01-10 13:38:15 +00001280 /* Inform the EL1 of the data abort. */
Fuad Tabbac3847c72020-08-11 09:32:25 +01001281 inject_el1_data_abort_exception(vcpu, esr, far);
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +01001282
Fuad Tabbab86325a2020-01-10 13:38:15 +00001283 /* Schedule the same VM to continue running. */
1284 return NULL;
1285
1286 case EC_INSTRUCTION_ABORT_LOWER_EL:
Andrew Sculld3cfaad2019-04-04 11:34:10 +01001287 info = fault_info_init(esr, vcpu, MM_MODE_X);
Raghu Krishnamurthyf16b2ce2021-11-02 07:48:38 -07001288
Wedson Almeida Filho99d2d4c2019-02-14 12:53:46 +00001289 if (vcpu_handle_page_fault(vcpu, &info)) {
1290 return NULL;
1291 }
Raghu Krishnamurthyb5775d22021-02-26 18:54:40 -08001292
1293 if (is_el0_partition) {
1294 dlog_warning("Instruction abort on EL0 partition\n");
1295 return api_abort(vcpu);
1296 }
1297
Fuad Tabbab86325a2020-01-10 13:38:15 +00001298 /* Inform the EL1 of the instruction abort. */
Fuad Tabbac3847c72020-08-11 09:32:25 +01001299 inject_el1_instruction_abort_exception(vcpu, esr, far);
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +01001300
Fuad Tabbab86325a2020-01-10 13:38:15 +00001301 /* Schedule the same VM to continue running. */
1302 return NULL;
Raghu Krishnamurthyb5775d22021-02-26 18:54:40 -08001303 case EC_SVC:
1304 CHECK(is_el0_partition);
1305 return hvc_handler(vcpu);
Fuad Tabbab86325a2020-01-10 13:38:15 +00001306 case EC_HVC:
Raghu Krishnamurthyb5775d22021-02-26 18:54:40 -08001307 if (is_el0_partition) {
1308 dlog_warning("Unexpected HVC Trap on EL0 partition\n");
1309 return api_abort(vcpu);
1310 }
Andrew Walbran59182d52019-09-23 17:55:39 +01001311 return hvc_handler(vcpu);
1312
Fuad Tabbab86325a2020-01-10 13:38:15 +00001313 case EC_SMC: {
Andrew Scullc960c032018-10-24 15:13:35 +01001314 uintreg_t smc_pc = vcpu->regs.pc;
Andrew Walbran9dadaf22019-12-05 16:50:55 +00001315 struct vcpu *next = smc_handler(vcpu);
Wedson Almeida Filho03e767a2018-07-30 15:32:03 +01001316
1317 /* Skip the SMC instruction. */
Fuad Tabbac76466d2019-09-06 10:42:12 +01001318 vcpu->regs.pc = smc_pc + GET_NEXT_PC_INC(esr);
Andrew Walbran9dadaf22019-12-05 16:50:55 +00001319
Andrew Walbran33645652019-04-15 12:29:31 +01001320 return next;
Andrew Scullc960c032018-10-24 15:13:35 +01001321 }
Wedson Almeida Filho03e767a2018-07-30 15:32:03 +01001322
Fuad Tabbab86325a2020-01-10 13:38:15 +00001323 case EC_MSR:
Fuad Tabbac76466d2019-09-06 10:42:12 +01001324 /*
1325 * NOTE: This should never be reached because it goes through a
1326 * separate path handled by handle_system_register_access().
1327 */
1328 panic("Handled by handle_system_register_access().");
1329
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +01001330 default:
Andrew Walbran17eebf92020-02-05 16:35:49 +00001331 dlog_notice(
1332 "Unknown lower sync exception pc=%#x, esr=%#x, "
1333 "ec=%#x\n",
1334 vcpu->regs.pc, esr, ec);
Andrew Scull9726c252019-01-23 13:44:19 +00001335 break;
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +01001336 }
1337
Raghu Krishnamurthyb5775d22021-02-26 18:54:40 -08001338 if (is_el0_partition) {
1339 return api_abort(vcpu);
1340 }
1341
Fuad Tabba3e9b0222019-11-11 16:47:50 +00001342 /*
Fuad Tabbaa48d1222019-12-09 15:42:32 +00001343 * The exception wasn't handled. Inject to the VM to give it chance to
1344 * handle as an unknown exception.
Fuad Tabba3e9b0222019-11-11 16:47:50 +00001345 */
Fuad Tabbab86325a2020-01-10 13:38:15 +00001346 inject_el1_unknown_exception(vcpu, esr);
1347
1348 /* Schedule the same VM to continue running. */
1349 return NULL;
Fuad Tabba3e9b0222019-11-11 16:47:50 +00001350}
1351
Fuad Tabbac76466d2019-09-06 10:42:12 +01001352/**
Fuad Tabbab0ef2a42019-12-19 11:19:25 +00001353 * Handles EC = 011000, MSR, MRS instruction traps.
Fuad Tabbaed294af2019-12-20 10:43:01 +00001354 * Returns non-null ONLY if the access failed and the vCPU is changing.
Fuad Tabbac76466d2019-09-06 10:42:12 +01001355 */
Fuad Tabbab86325a2020-01-10 13:38:15 +00001356void handle_system_register_access(uintreg_t esr_el2)
Fuad Tabbac76466d2019-09-06 10:42:12 +01001357{
1358 struct vcpu *vcpu = current();
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001359 ffa_vm_id_t vm_id = vcpu->vm->id;
Fuad Tabba3e9b0222019-11-11 16:47:50 +00001360 uintreg_t ec = GET_ESR_EC(esr_el2);
Fuad Tabbac76466d2019-09-06 10:42:12 +01001361
Fuad Tabbab86325a2020-01-10 13:38:15 +00001362 CHECK(ec == EC_MSR);
Fuad Tabbac76466d2019-09-06 10:42:12 +01001363 /*
Fuad Tabbaf1d6dc52019-09-18 17:33:14 +01001364 * Handle accesses to debug and performance monitor registers.
Fuad Tabba3e9b0222019-11-11 16:47:50 +00001365 * Inject an exception for unhandled/unsupported registers.
Fuad Tabbac76466d2019-09-06 10:42:12 +01001366 */
Fuad Tabba3e9b0222019-11-11 16:47:50 +00001367 if (debug_el1_is_register_access(esr_el2)) {
1368 if (!debug_el1_process_access(vcpu, vm_id, esr_el2)) {
Olivier Deprezda14ddc2022-08-11 14:14:41 +02001369 inject_el1_sysreg_trap_exception(vcpu, esr_el2);
Fuad Tabbab86325a2020-01-10 13:38:15 +00001370 return;
Fuad Tabbaf1d6dc52019-09-18 17:33:14 +01001371 }
Fuad Tabba3e9b0222019-11-11 16:47:50 +00001372 } else if (perfmon_is_register_access(esr_el2)) {
1373 if (!perfmon_process_access(vcpu, vm_id, esr_el2)) {
Olivier Deprezda14ddc2022-08-11 14:14:41 +02001374 inject_el1_sysreg_trap_exception(vcpu, esr_el2);
Fuad Tabbab86325a2020-01-10 13:38:15 +00001375 return;
Fuad Tabbaf1d6dc52019-09-18 17:33:14 +01001376 }
Fuad Tabba77a4b012019-11-15 12:13:08 +00001377 } else if (feature_id_is_register_access(esr_el2)) {
1378 if (!feature_id_process_access(vcpu, esr_el2)) {
Olivier Deprezda14ddc2022-08-11 14:14:41 +02001379 inject_el1_sysreg_trap_exception(vcpu, esr_el2);
Fuad Tabbab86325a2020-01-10 13:38:15 +00001380 return;
Fuad Tabba77a4b012019-11-15 12:13:08 +00001381 }
Fuad Tabbaf1d6dc52019-09-18 17:33:14 +01001382 } else {
Olivier Deprezda14ddc2022-08-11 14:14:41 +02001383 inject_el1_sysreg_trap_exception(vcpu, esr_el2);
Fuad Tabbab86325a2020-01-10 13:38:15 +00001384 return;
Fuad Tabbac76466d2019-09-06 10:42:12 +01001385 }
1386
Fuad Tabbaf1d6dc52019-09-18 17:33:14 +01001387 /* Instruction was fulfilled. Skip it and run the next one. */
Fuad Tabba3e9b0222019-11-11 16:47:50 +00001388 vcpu->regs.pc += GET_NEXT_PC_INC(esr_el2);
Fuad Tabbac76466d2019-09-06 10:42:12 +01001389}