Andrew Scull | 1883487 | 2018-10-12 11:48:09 +0100 | [diff] [blame] | 1 | /* |
Andrew Walbran | 692b325 | 2019-03-07 15:51:31 +0000 | [diff] [blame] | 2 | * Copyright 2018 The Hafnium Authors. |
Andrew Scull | 1883487 | 2018-10-12 11:48:09 +0100 | [diff] [blame] | 3 | * |
Andrew Walbran | e959ec1 | 2020-06-17 15:01:09 +0100 | [diff] [blame] | 4 | * Use of this source code is governed by a BSD-style |
| 5 | * license that can be found in the LICENSE file or at |
| 6 | * https://opensource.org/licenses/BSD-3-Clause. |
Andrew Scull | 1883487 | 2018-10-12 11:48:09 +0100 | [diff] [blame] | 7 | */ |
| 8 | |
Andrew Scull | c960c03 | 2018-10-24 15:13:35 +0100 | [diff] [blame] | 9 | #include <stdnoreturn.h> |
| 10 | |
Andrew Walbran | 1f32e72 | 2019-06-07 17:57:26 +0100 | [diff] [blame] | 11 | #include "hf/arch/barriers.h" |
Madhukar Pappireddy | 77d3bcd | 2023-03-01 17:26:22 -0600 | [diff] [blame] | 12 | #include "hf/arch/gicv3.h" |
Madhukar Pappireddy | a3787c9 | 2024-09-25 14:50:36 -0500 | [diff] [blame] | 13 | #include "hf/arch/host_timer.h" |
J-Alves | a2d1c3b | 2024-03-28 12:46:58 +0000 | [diff] [blame] | 14 | #include "hf/arch/memcpy_trapped.h" |
Olivier Deprez | 98ad2d2 | 2020-05-20 09:52:43 +0200 | [diff] [blame] | 15 | #include "hf/arch/mmu.h" |
Andrew Scull | 07b6bd3 | 2019-12-12 17:19:55 +0000 | [diff] [blame] | 16 | #include "hf/arch/plat/smc.h" |
Madhukar Pappireddy | a3787c9 | 2024-09-25 14:50:36 -0500 | [diff] [blame] | 17 | #include "hf/arch/timer.h" |
J-Alves | 03edf40 | 2023-07-21 15:13:49 +0100 | [diff] [blame] | 18 | #include "hf/arch/vmid_base.h" |
Andrew Scull | c960c03 | 2018-10-24 15:13:35 +0100 | [diff] [blame] | 19 | |
Andrew Scull | 18c78fc | 2018-08-20 12:57:41 +0100 | [diff] [blame] | 20 | #include "hf/api.h" |
Fuad Tabba | c76466d | 2019-09-06 10:42:12 +0100 | [diff] [blame] | 21 | #include "hf/check.h" |
Andrew Scull | 18c78fc | 2018-08-20 12:57:41 +0100 | [diff] [blame] | 22 | #include "hf/cpu.h" |
| 23 | #include "hf/dlog.h" |
Andrew Walbran | b5ab43c | 2020-04-30 11:32:54 +0100 | [diff] [blame] | 24 | #include "hf/ffa.h" |
Karl Meakin | 936ec1e | 2025-01-31 13:17:11 +0000 | [diff] [blame] | 25 | #include "hf/ffa/cpu_cycles.h" |
Karl Meakin | 902af08 | 2024-11-28 14:58:38 +0000 | [diff] [blame] | 26 | #include "hf/ffa/indirect_messaging.h" |
| 27 | #include "hf/ffa/interrupts.h" |
| 28 | #include "hf/ffa/notifications.h" |
| 29 | #include "hf/ffa/vm.h" |
J-Alves | b37fd08 | 2020-10-22 12:29:21 +0100 | [diff] [blame] | 30 | #include "hf/ffa_internal.h" |
Andrew Scull | a9c172d | 2019-04-03 14:10:00 +0100 | [diff] [blame] | 31 | #include "hf/panic.h" |
Manish Pandey | a5f39fb | 2020-09-11 09:47:11 +0100 | [diff] [blame] | 32 | #include "hf/plat/interrupts.h" |
Madhukar Pappireddy | a3787c9 | 2024-09-25 14:50:36 -0500 | [diff] [blame] | 33 | #include "hf/timer_mgmt.h" |
Andrew Scull | 18c78fc | 2018-08-20 12:57:41 +0100 | [diff] [blame] | 34 | #include "hf/vm.h" |
Wedson Almeida Filho | 987c0ff | 2018-06-20 16:34:38 +0100 | [diff] [blame] | 35 | |
Fuad Tabba | c76466d | 2019-09-06 10:42:12 +0100 | [diff] [blame] | 36 | #include "debug_el1.h" |
Madhukar Pappireddy | f684d19 | 2024-09-25 14:35:57 -0500 | [diff] [blame] | 37 | #include "el1_physical_timer.h" |
Fuad Tabba | 77a4b01 | 2019-11-15 12:13:08 +0000 | [diff] [blame] | 38 | #include "feature_id.h" |
Fuad Tabba | f1d6dc5 | 2019-09-18 17:33:14 +0100 | [diff] [blame] | 39 | #include "perfmon.h" |
Andrew Scull | 18c78fc | 2018-08-20 12:57:41 +0100 | [diff] [blame] | 40 | #include "psci.h" |
Andrew Walbran | 3364565 | 2019-04-15 12:29:31 +0100 | [diff] [blame] | 41 | #include "psci_handler.h" |
Andrew Scull | 7fd4bb7 | 2018-12-08 23:40:12 +0000 | [diff] [blame] | 42 | #include "smc.h" |
Fuad Tabba | ba8c44d | 2019-09-23 14:38:58 +0100 | [diff] [blame] | 43 | #include "sysregs.h" |
Karl Meakin | 5a13355 | 2024-05-30 16:06:27 +0100 | [diff] [blame] | 44 | #include "sysregs_defs.h" |
Wedson Almeida Filho | 987c0ff | 2018-06-20 16:34:38 +0100 | [diff] [blame] | 45 | |
Fuad Tabba | c76466d | 2019-09-06 10:42:12 +0100 | [diff] [blame] | 46 | /** |
Olivier Deprez | 98ad2d2 | 2020-05-20 09:52:43 +0200 | [diff] [blame] | 47 | * Hypervisor Fault Address Register Non-Secure. |
| 48 | */ |
| 49 | #define HPFAR_EL2_NS (UINT64_C(0x1) << 63) |
| 50 | |
| 51 | /** |
| 52 | * Hypervisor Fault Address Register Faulting IPA. |
| 53 | */ |
| 54 | #define HPFAR_EL2_FIPA (UINT64_C(0xFFFFFFFFFF0)) |
| 55 | |
| 56 | /** |
Fuad Tabba | c76466d | 2019-09-06 10:42:12 +0100 | [diff] [blame] | 57 | * Gets the value to increment for the next PC. |
| 58 | * The ESR encodes whether the instruction is 2 bytes or 4 bytes long. |
| 59 | */ |
Fuad Tabba | 3e9b022 | 2019-11-11 16:47:50 +0000 | [diff] [blame] | 60 | #define GET_NEXT_PC_INC(esr) (GET_ESR_IL(esr) ? 4 : 2) |
Fuad Tabba | c76466d | 2019-09-06 10:42:12 +0100 | [diff] [blame] | 61 | |
Fuad Tabba | c76466d | 2019-09-06 10:42:12 +0100 | [diff] [blame] | 62 | /** |
Andrew Walbran | 0dd67ff | 2019-09-12 16:38:50 +0100 | [diff] [blame] | 63 | * The Client ID field within X7 for an SMC64 call. |
| 64 | */ |
| 65 | #define CLIENT_ID_MASK UINT64_C(0xffff) |
| 66 | |
Karl Meakin | d0356f8 | 2024-09-04 13:34:31 +0100 | [diff] [blame] | 67 | /** |
| 68 | * Identifies SPMD specific framework messages. See section 18.2 of v1.2 FF-A |
| 69 | * specification. |
Daniel Boulby | efa381f | 2022-01-18 14:49:40 +0000 | [diff] [blame] | 70 | */ |
Karl Meakin | d0356f8 | 2024-09-04 13:34:31 +0100 | [diff] [blame] | 71 | enum ffa_spmd_framework_msg_func { |
| 72 | SPMD_FRAMEWORK_MSG_PSCI_REQ = 0, |
| 73 | SPMD_FRAMEWORK_MSG_PSCI_RESP = 2, |
| 74 | |
| 75 | SPMD_FRAMEWORK_MSG_FFA_VERSION_REQ = 8, |
| 76 | SPMD_FRAMEWORK_MSG_FFA_VERSION_RESP = 9, |
| 77 | }; |
Daniel Boulby | efa381f | 2022-01-18 14:49:40 +0000 | [diff] [blame] | 78 | |
Andrew Walbran | 0dd67ff | 2019-09-12 16:38:50 +0100 | [diff] [blame] | 79 | /** |
Fuad Tabba | c76466d | 2019-09-06 10:42:12 +0100 | [diff] [blame] | 80 | * Returns a reference to the currently executing vCPU. |
| 81 | */ |
Andrew Scull | c960c03 | 2018-10-24 15:13:35 +0100 | [diff] [blame] | 82 | static struct vcpu *current(void) |
Andrew Walbran | 3d84a26 | 2018-12-13 14:41:19 +0000 | [diff] [blame] | 83 | { |
Daniel Boulby | 3f78426 | 2021-09-27 13:02:54 +0100 | [diff] [blame] | 84 | // NOLINTNEXTLINE(performance-no-int-to-ptr) |
Andrew Walbran | 3d84a26 | 2018-12-13 14:41:19 +0000 | [diff] [blame] | 85 | return (struct vcpu *)read_msr(tpidr_el2); |
| 86 | } |
| 87 | |
Andrew Walbran | 1f8d487 | 2018-12-20 11:21:32 +0000 | [diff] [blame] | 88 | /** |
Madhukar Pappireddy | d3ac738 | 2024-09-25 14:29:03 -0500 | [diff] [blame] | 89 | * Saves the state of per-vCPU peripherals, such as the arch timer, and |
Andrew Walbran | 1f8d487 | 2018-12-20 11:21:32 +0000 | [diff] [blame] | 90 | * informs the arch-independent sections that registers have been saved. |
| 91 | */ |
| 92 | void complete_saving_state(struct vcpu *vcpu) |
| 93 | { |
Madhukar Pappireddy | a3787c9 | 2024-09-25 14:50:36 -0500 | [diff] [blame] | 94 | host_timer_save_arch_timer(&vcpu->regs.arch_timer); |
| 95 | |
| 96 | timer_vcpu_manage(vcpu); |
Andrew Walbran | 1f8d487 | 2018-12-20 11:21:32 +0000 | [diff] [blame] | 97 | api_regs_state_saved(vcpu); |
Madhukar Pappireddy | a3787c9 | 2024-09-25 14:50:36 -0500 | [diff] [blame] | 98 | |
| 99 | /* |
| 100 | * Since switching away from current vCPU, disable the host physical |
| 101 | * timer for now. If necessary, the host timer will be reconfigured |
| 102 | * at appropriate time to track timer deadline of the vCPU. |
| 103 | */ |
| 104 | host_timer_disable(); |
Andrew Walbran | 1f8d487 | 2018-12-20 11:21:32 +0000 | [diff] [blame] | 105 | } |
| 106 | |
| 107 | /** |
Madhukar Pappireddy | d3ac738 | 2024-09-25 14:29:03 -0500 | [diff] [blame] | 108 | * Restores the state of per-vCPU peripherals, such as the arch timer. |
Andrew Walbran | 1f8d487 | 2018-12-20 11:21:32 +0000 | [diff] [blame] | 109 | */ |
| 110 | void begin_restoring_state(struct vcpu *vcpu) |
| 111 | { |
Madhukar Pappireddy | a3787c9 | 2024-09-25 14:50:36 -0500 | [diff] [blame] | 112 | /* |
| 113 | * If a vCPU's timer has expired while it was de-scheduled, SPMC will |
| 114 | * inject the virtual timer interrupt before resuming the vCPU. |
| 115 | * If not, there is a live state and we need to configure the host timer |
| 116 | * to track it again. |
| 117 | */ |
| 118 | if (arch_timer_enabled(&vcpu->regs) && |
| 119 | (arch_timer_remaining_ns(&vcpu->regs) != 0)) { |
| 120 | host_timer_track_deadline(&vcpu->regs.arch_timer); |
| 121 | } |
Andrew Walbran | 1f8d487 | 2018-12-20 11:21:32 +0000 | [diff] [blame] | 122 | } |
| 123 | |
Andrew Walbran | 1f32e72 | 2019-06-07 17:57:26 +0100 | [diff] [blame] | 124 | /** |
Andrew Walbran | 1f32e72 | 2019-06-07 17:57:26 +0100 | [diff] [blame] | 125 | * Invalidate all stage 1 TLB entries on the current (physical) CPU for the |
| 126 | * current VMID. |
| 127 | */ |
| 128 | static void invalidate_vm_tlb(void) |
| 129 | { |
Andrew Walbran | cff1f68 | 2019-07-04 14:52:45 +0100 | [diff] [blame] | 130 | /* |
| 131 | * Ensure that the last VTTBR write has taken effect so we invalidate |
| 132 | * the right set of TLB entries. |
| 133 | */ |
Andrew Walbran | 1f32e72 | 2019-06-07 17:57:26 +0100 | [diff] [blame] | 134 | isb(); |
Andrew Walbran | cff1f68 | 2019-07-04 14:52:45 +0100 | [diff] [blame] | 135 | |
Olivier Deprez | 0b0ba8c | 2023-03-17 11:11:53 +0100 | [diff] [blame] | 136 | tlbi(vmalle1); |
Andrew Walbran | cff1f68 | 2019-07-04 14:52:45 +0100 | [diff] [blame] | 137 | |
| 138 | /* |
| 139 | * Ensure that no instructions are fetched for the VM until after the |
| 140 | * TLB invalidation has taken effect. |
| 141 | */ |
Andrew Walbran | 1f32e72 | 2019-06-07 17:57:26 +0100 | [diff] [blame] | 142 | isb(); |
Andrew Walbran | cff1f68 | 2019-07-04 14:52:45 +0100 | [diff] [blame] | 143 | |
| 144 | /* |
| 145 | * Ensure that no data reads or writes for the VM happen until after the |
Fuad Tabba | 77a4b01 | 2019-11-15 12:13:08 +0000 | [diff] [blame] | 146 | * TLB invalidation has taken effect. Non-shareable is enough because |
| 147 | * the TLB is local to the CPU. |
Andrew Walbran | cff1f68 | 2019-07-04 14:52:45 +0100 | [diff] [blame] | 148 | */ |
David Brazdil | 851948e | 2019-08-09 12:02:12 +0100 | [diff] [blame] | 149 | dsb(nsh); |
Andrew Walbran | 1f32e72 | 2019-06-07 17:57:26 +0100 | [diff] [blame] | 150 | } |
| 151 | |
| 152 | /** |
| 153 | * Invalidates the TLB if a different vCPU is being run than the last vCPU of |
| 154 | * the same VM which was run on the current pCPU. |
| 155 | * |
| 156 | * This is necessary because VMs may (contrary to the architecture |
| 157 | * specification) use inconsistent ASIDs across vCPUs. c.f. KVM's similar |
| 158 | * workaround: |
| 159 | * https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=94d0e5980d6791b9 |
| 160 | */ |
| 161 | void maybe_invalidate_tlb(struct vcpu *vcpu) |
| 162 | { |
| 163 | size_t current_cpu_index = cpu_index(vcpu->cpu); |
Andrew Walbran | b5ab43c | 2020-04-30 11:32:54 +0100 | [diff] [blame] | 164 | ffa_vcpu_index_t new_vcpu_index = vcpu_index(vcpu); |
Andrew Walbran | 1f32e72 | 2019-06-07 17:57:26 +0100 | [diff] [blame] | 165 | |
| 166 | if (vcpu->vm->arch.last_vcpu_on_cpu[current_cpu_index] != |
| 167 | new_vcpu_index) { |
| 168 | /* |
| 169 | * The vCPU has changed since the last time this VM was run on |
| 170 | * this pCPU, so we need to invalidate the TLB. |
| 171 | */ |
| 172 | invalidate_vm_tlb(); |
| 173 | |
| 174 | /* Record the fact that this vCPU is now running on this CPU. */ |
| 175 | vcpu->vm->arch.last_vcpu_on_cpu[current_cpu_index] = |
| 176 | new_vcpu_index; |
| 177 | } |
| 178 | } |
| 179 | |
David Brazdil | 768f69c | 2019-12-19 15:46:12 +0000 | [diff] [blame] | 180 | noreturn void irq_current_exception_noreturn(uintreg_t elr, uintreg_t spsr) |
Wedson Almeida Filho | 987c0ff | 2018-06-20 16:34:38 +0100 | [diff] [blame] | 181 | { |
Wedson Almeida Filho | 9d5040f | 2018-10-29 08:41:27 +0000 | [diff] [blame] | 182 | (void)elr; |
| 183 | (void)spsr; |
| 184 | |
Fuad Tabba | d1d6798 | 2020-01-08 11:28:29 +0000 | [diff] [blame] | 185 | panic("IRQ from current exception level."); |
Wedson Almeida Filho | 987c0ff | 2018-06-20 16:34:38 +0100 | [diff] [blame] | 186 | } |
| 187 | |
David Brazdil | 768f69c | 2019-12-19 15:46:12 +0000 | [diff] [blame] | 188 | noreturn void fiq_current_exception_noreturn(uintreg_t elr, uintreg_t spsr) |
Wedson Almeida Filho | 987c0ff | 2018-06-20 16:34:38 +0100 | [diff] [blame] | 189 | { |
Wedson Almeida Filho | 9d5040f | 2018-10-29 08:41:27 +0000 | [diff] [blame] | 190 | (void)elr; |
| 191 | (void)spsr; |
| 192 | |
Fuad Tabba | d1d6798 | 2020-01-08 11:28:29 +0000 | [diff] [blame] | 193 | panic("FIQ from current exception level."); |
Wedson Almeida Filho | 9d5040f | 2018-10-29 08:41:27 +0000 | [diff] [blame] | 194 | } |
| 195 | |
David Brazdil | 768f69c | 2019-12-19 15:46:12 +0000 | [diff] [blame] | 196 | noreturn void serr_current_exception_noreturn(uintreg_t elr, uintreg_t spsr) |
Wedson Almeida Filho | 9d5040f | 2018-10-29 08:41:27 +0000 | [diff] [blame] | 197 | { |
| 198 | (void)elr; |
| 199 | (void)spsr; |
| 200 | |
Fuad Tabba | d1d6798 | 2020-01-08 11:28:29 +0000 | [diff] [blame] | 201 | panic("SError from current exception level."); |
Wedson Almeida Filho | 9d5040f | 2018-10-29 08:41:27 +0000 | [diff] [blame] | 202 | } |
| 203 | |
J-Alves | a2d1c3b | 2024-03-28 12:46:58 +0000 | [diff] [blame] | 204 | /** |
| 205 | * Returns true if ELR_EL2 is not to be restored from stack. |
| 206 | * Currently function doesn't return false, as for all other cases |
| 207 | * panics. |
| 208 | */ |
| 209 | bool sync_current_exception(uintreg_t elr, uintreg_t spsr) |
Wedson Almeida Filho | 9d5040f | 2018-10-29 08:41:27 +0000 | [diff] [blame] | 210 | { |
| 211 | uintreg_t esr = read_msr(esr_el2); |
Fuad Tabba | 3e9b022 | 2019-11-11 16:47:50 +0000 | [diff] [blame] | 212 | uintreg_t ec = GET_ESR_EC(esr); |
Wedson Almeida Filho | 9d5040f | 2018-10-29 08:41:27 +0000 | [diff] [blame] | 213 | (void)spsr; |
| 214 | |
Fuad Tabba | c76466d | 2019-09-06 10:42:12 +0100 | [diff] [blame] | 215 | switch (ec) { |
J-Alves | a2d1c3b | 2024-03-28 12:46:58 +0000 | [diff] [blame] | 216 | case EC_DATA_ABORT_SAME_EL: { |
| 217 | uint64_t iss = GET_ESR_ISS(esr); |
| 218 | uint64_t dfsc = GET_ESR_ISS_DFSC(iss); |
| 219 | uint64_t far = read_msr(far_el2); |
| 220 | |
| 221 | /* Handle Granule Protection Fault. */ |
| 222 | if (is_arch_feat_rme_supported() && dfsc == DFSC_GPF) { |
| 223 | dlog_verbose( |
Karl Meakin | e8937d9 | 2024-03-19 16:04:25 +0000 | [diff] [blame] | 224 | "Granule Protection Fault: esr=%#lx, ec=%#lx, " |
| 225 | "far=%#lx, elr=%#lx\n", |
J-Alves | a2d1c3b | 2024-03-28 12:46:58 +0000 | [diff] [blame] | 226 | esr, ec, far, elr); |
| 227 | |
| 228 | /* |
| 229 | * Change ELR_EL2 only if failed whilst either |
| 230 | * reading or writing within 'memcpy_trapped'. |
| 231 | */ |
| 232 | if (elr == (uintptr_t)memcpy_trapped_read || |
| 233 | elr == (uintptr_t)memcpy_trapped_write) { |
| 234 | dlog_verbose( |
| 235 | "GPF due to data abort on %s.\n", |
| 236 | (elr == (uintptr_t)memcpy_trapped_read) |
| 237 | ? "read" |
| 238 | : "write"); |
| 239 | |
| 240 | /* |
| 241 | * Update the ELR_EL2 with the return |
| 242 | * address, to return error from the |
| 243 | * call to 'memcpy_trapped'. |
| 244 | */ |
| 245 | write_msr(ELR_EL2, memcpy_trapped_aborted); |
| 246 | return true; |
| 247 | } |
| 248 | } |
| 249 | |
Kathleen Capella | d1c34b5 | 2024-04-01 21:27:15 -0400 | [diff] [blame] | 250 | #if ENABLE_MTE |
| 251 | if (dfsc == DFSC_SYNC_TAG_CHECK_FAULT) { |
| 252 | dlog_error( |
| 253 | "Data abort due to synchronous tag check " |
| 254 | "fault: pc=%#lx, esr=%#lx, ec=%#lx, " |
| 255 | "far=%#lx, dfsc = %#lx\n", |
| 256 | elr, esr, ec, far, dfsc); |
| 257 | } |
Kathleen Capella | d1c34b5 | 2024-04-01 21:27:15 -0400 | [diff] [blame] | 258 | #endif |
Karl Meakin | 5a13355 | 2024-05-30 16:06:27 +0100 | [diff] [blame] | 259 | if (!GET_ESR_FNV(esr)) { |
Andrew Walbran | 17eebf9 | 2020-02-05 16:35:49 +0000 | [diff] [blame] | 260 | dlog_error( |
Karl Meakin | e8937d9 | 2024-03-19 16:04:25 +0000 | [diff] [blame] | 261 | "Data abort: pc=%#lx, esr=%#lx, ec=%#lx, " |
| 262 | "far=%#lx\n", |
J-Alves | a2d1c3b | 2024-03-28 12:46:58 +0000 | [diff] [blame] | 263 | elr, esr, ec, far); |
| 264 | |
Andrew Scull | 7364a8e | 2018-07-19 15:39:29 +0100 | [diff] [blame] | 265 | } else { |
Andrew Walbran | 17eebf9 | 2020-02-05 16:35:49 +0000 | [diff] [blame] | 266 | dlog_error( |
Karl Meakin | e8937d9 | 2024-03-19 16:04:25 +0000 | [diff] [blame] | 267 | "Data abort: pc=%#lx, esr=%#lx, ec=%#lx, " |
Andrew Walbran | 17eebf9 | 2020-02-05 16:35:49 +0000 | [diff] [blame] | 268 | "far=invalid\n", |
| 269 | elr, esr, ec); |
Andrew Scull | 7364a8e | 2018-07-19 15:39:29 +0100 | [diff] [blame] | 270 | } |
J-Alves | a2d1c3b | 2024-03-28 12:46:58 +0000 | [diff] [blame] | 271 | } break; |
Wedson Almeida Filho | fed6902 | 2018-07-11 15:39:12 +0100 | [diff] [blame] | 272 | default: |
Andrew Walbran | 17eebf9 | 2020-02-05 16:35:49 +0000 | [diff] [blame] | 273 | dlog_error( |
Karl Meakin | e8937d9 | 2024-03-19 16:04:25 +0000 | [diff] [blame] | 274 | "Unknown current sync exception pc=%#lx, esr=%#lx, " |
| 275 | "ec=%#lx\n", |
Andrew Walbran | 17eebf9 | 2020-02-05 16:35:49 +0000 | [diff] [blame] | 276 | elr, esr, ec); |
Andrew Scull | c960c03 | 2018-10-24 15:13:35 +0100 | [diff] [blame] | 277 | break; |
Wedson Almeida Filho | fed6902 | 2018-07-11 15:39:12 +0100 | [diff] [blame] | 278 | } |
Wedson Almeida Filho | 81568c4 | 2019-01-04 13:33:02 +0000 | [diff] [blame] | 279 | |
Andrew Scull | a9c172d | 2019-04-03 14:10:00 +0100 | [diff] [blame] | 280 | panic("EL2 exception"); |
Wedson Almeida Filho | 987c0ff | 2018-06-20 16:34:38 +0100 | [diff] [blame] | 281 | } |
| 282 | |
Wedson Almeida Filho | 03e767a | 2018-07-30 15:32:03 +0100 | [diff] [blame] | 283 | /** |
Manish Pandey | 35e452f | 2021-02-18 21:36:34 +0000 | [diff] [blame] | 284 | * Sets or clears the VF bit in the HCR_EL2 register saved in the given |
| 285 | * arch_regs. |
| 286 | */ |
| 287 | static void set_virtual_fiq(struct arch_regs *r, bool enable) |
| 288 | { |
| 289 | if (enable) { |
Olivier Deprez | 6d408f9 | 2022-08-08 19:14:23 +0200 | [diff] [blame] | 290 | r->hyp_state.hcr_el2 |= HCR_EL2_VF; |
Manish Pandey | 35e452f | 2021-02-18 21:36:34 +0000 | [diff] [blame] | 291 | } else { |
Olivier Deprez | 6d408f9 | 2022-08-08 19:14:23 +0200 | [diff] [blame] | 292 | r->hyp_state.hcr_el2 &= ~HCR_EL2_VF; |
Manish Pandey | 35e452f | 2021-02-18 21:36:34 +0000 | [diff] [blame] | 293 | } |
| 294 | } |
| 295 | |
| 296 | /** |
J-Alves | 6f6bf8a | 2024-07-25 15:17:57 +0100 | [diff] [blame] | 297 | * Sets or clears the VI bit in the HCR_EL2 register saved in the given |
| 298 | * arch_regs. |
Manish Pandey | 35e452f | 2021-02-18 21:36:34 +0000 | [diff] [blame] | 299 | */ |
J-Alves | 6f6bf8a | 2024-07-25 15:17:57 +0100 | [diff] [blame] | 300 | static void set_virtual_irq(struct arch_regs *r, bool enable) |
Manish Pandey | 35e452f | 2021-02-18 21:36:34 +0000 | [diff] [blame] | 301 | { |
Manish Pandey | 35e452f | 2021-02-18 21:36:34 +0000 | [diff] [blame] | 302 | if (enable) { |
J-Alves | 6f6bf8a | 2024-07-25 15:17:57 +0100 | [diff] [blame] | 303 | r->hyp_state.hcr_el2 |= HCR_EL2_VI; |
Manish Pandey | 35e452f | 2021-02-18 21:36:34 +0000 | [diff] [blame] | 304 | } else { |
J-Alves | 6f6bf8a | 2024-07-25 15:17:57 +0100 | [diff] [blame] | 305 | r->hyp_state.hcr_el2 &= ~HCR_EL2_VI; |
Manish Pandey | 35e452f | 2021-02-18 21:36:34 +0000 | [diff] [blame] | 306 | } |
Manish Pandey | 35e452f | 2021-02-18 21:36:34 +0000 | [diff] [blame] | 307 | } |
| 308 | |
J-Alves | b37fd08 | 2020-10-22 12:29:21 +0100 | [diff] [blame] | 309 | #if SECURE_WORLD == 1 |
Madhukar Pappireddy | 44b85ff | 2024-11-25 10:29:22 -0600 | [diff] [blame] | 310 | /* |
| 311 | * TODO: the power management event reached the SPMC. In a later iteration, the |
| 312 | * power management event can be passed to the SP by resuming it. |
| 313 | */ |
| 314 | static struct ffa_value handle_psci_framework_msg(struct ffa_value *args, |
| 315 | struct vcpu *current) |
| 316 | { |
| 317 | enum psci_return_code psci_msg_response; |
| 318 | uint64_t psci_func = args->arg3; |
| 319 | |
| 320 | switch (psci_func) { |
| 321 | case PSCI_CPU_OFF: { |
| 322 | /* |
| 323 | * Mark all the vCPUs pinned on this CPU as OFF. Note that the |
| 324 | * vCPU of an UP SP is not turned off since SPMC can migrate it |
| 325 | * to an online CPU when needed. |
| 326 | */ |
| 327 | for (ffa_vm_count_t index = 0; index < vm_get_count(); |
| 328 | ++index) { |
| 329 | struct vm *vm = vm_find_index(index); |
| 330 | |
| 331 | if (vm->vcpu_count > 1) { |
| 332 | struct vcpu *vcpu; |
| 333 | struct vcpu_locked vcpu_locked; |
| 334 | |
| 335 | vcpu = vm_get_vcpu(vm, cpu_index(current->cpu)); |
| 336 | vcpu_locked = vcpu_lock(vcpu); |
| 337 | vcpu->state = VCPU_STATE_OFF; |
| 338 | vcpu_unlock(&vcpu_locked); |
| 339 | dlog_verbose("SP%u turned OFF on CPU%zu\n", |
| 340 | vm->id, cpu_index(current->cpu)); |
| 341 | } |
| 342 | } |
| 343 | |
| 344 | /* |
| 345 | * Mark the CPU as turned off and reset the field tracking if |
| 346 | * all the pinned vCPUs have been booted on this CPU. |
| 347 | */ |
| 348 | cpu_off(current->cpu); |
| 349 | current->cpu->last_sp_initialized = false; |
| 350 | psci_msg_response = PSCI_RETURN_SUCCESS; |
| 351 | |
| 352 | break; |
| 353 | } |
| 354 | default: |
| 355 | dlog_error( |
| 356 | "FF-A PSCI framework message not handled " |
| 357 | "%#lx %#lx %#lx %#lx\n", |
| 358 | args->func, args->arg1, args->arg2, args->arg3); |
| 359 | psci_msg_response = PSCI_ERROR_NOT_SUPPORTED; |
| 360 | } |
| 361 | |
| 362 | return ffa_framework_msg_resp(HF_SPMC_VM_ID, HF_SPMD_VM_ID, |
| 363 | SPMD_FRAMEWORK_MSG_PSCI_RESP, |
| 364 | psci_msg_response); |
| 365 | } |
| 366 | |
Max Shvetsov | 1ae74f1 | 2020-09-18 13:52:20 +0100 | [diff] [blame] | 367 | /** |
Karl Meakin | d0356f8 | 2024-09-04 13:34:31 +0100 | [diff] [blame] | 368 | * Handle special direct messages from SPMD to SPMC. |
Max Shvetsov | 1ae74f1 | 2020-09-18 13:52:20 +0100 | [diff] [blame] | 369 | */ |
| 370 | static bool spmd_handler(struct ffa_value *args, struct vcpu *current) |
| 371 | { |
J-Alves | 19e20cf | 2023-08-02 12:48:55 +0100 | [diff] [blame] | 372 | ffa_id_t sender = ffa_sender(*args); |
| 373 | ffa_id_t receiver = ffa_receiver(*args); |
| 374 | ffa_id_t current_vm_id = current->vm->id; |
Karl Meakin | d0356f8 | 2024-09-04 13:34:31 +0100 | [diff] [blame] | 375 | enum ffa_spmd_framework_msg_func func = |
| 376 | (enum ffa_spmd_framework_msg_func)ffa_framework_msg_func(*args); |
Max Shvetsov | 1ae74f1 | 2020-09-18 13:52:20 +0100 | [diff] [blame] | 377 | |
| 378 | /* |
Daniel Boulby | efa381f | 2022-01-18 14:49:40 +0000 | [diff] [blame] | 379 | * Check if direct message request is originating from the SPMD, |
| 380 | * directed to the SPMC and the message is a framework message. |
Max Shvetsov | 1ae74f1 | 2020-09-18 13:52:20 +0100 | [diff] [blame] | 381 | */ |
| 382 | if (!(sender == HF_SPMD_VM_ID && receiver == HF_SPMC_VM_ID && |
Karl Meakin | d0356f8 | 2024-09-04 13:34:31 +0100 | [diff] [blame] | 383 | current_vm_id == HF_OTHER_WORLD_ID && |
| 384 | ffa_is_framework_msg(*args))) { |
Max Shvetsov | 1ae74f1 | 2020-09-18 13:52:20 +0100 | [diff] [blame] | 385 | return false; |
| 386 | } |
| 387 | |
Olivier Deprez | a67ab88 | 2023-01-10 15:00:54 +0100 | [diff] [blame] | 388 | /* |
| 389 | * The framework message is conveyed by EL3/SPMD to SPMC so the |
| 390 | * current VM id must match to the other world VM id. |
| 391 | */ |
| 392 | CHECK(current->vm->id == HF_HYPERVISOR_VM_ID); |
| 393 | |
Karl Meakin | d0356f8 | 2024-09-04 13:34:31 +0100 | [diff] [blame] | 394 | switch (func) { |
| 395 | case SPMD_FRAMEWORK_MSG_PSCI_REQ: { |
Madhukar Pappireddy | 44b85ff | 2024-11-25 10:29:22 -0600 | [diff] [blame] | 396 | *args = handle_psci_framework_msg(args, current); |
Olivier Deprez | a67ab88 | 2023-01-10 15:00:54 +0100 | [diff] [blame] | 397 | return true; |
Daniel Boulby | efa381f | 2022-01-18 14:49:40 +0000 | [diff] [blame] | 398 | } |
Karl Meakin | d0356f8 | 2024-09-04 13:34:31 +0100 | [diff] [blame] | 399 | case SPMD_FRAMEWORK_MSG_FFA_VERSION_REQ: { |
Daniel Boulby | efa381f | 2022-01-18 14:49:40 +0000 | [diff] [blame] | 400 | struct ffa_value ret = api_ffa_version(current, args->arg3); |
Karl Meakin | d0356f8 | 2024-09-04 13:34:31 +0100 | [diff] [blame] | 401 | *args = ffa_framework_msg_resp( |
| 402 | HF_SPMC_VM_ID, HF_SPMD_VM_ID, |
| 403 | SPMD_FRAMEWORK_MSG_FFA_VERSION_RESP, ret.func); |
Olivier Deprez | a67ab88 | 2023-01-10 15:00:54 +0100 | [diff] [blame] | 404 | return true; |
Max Shvetsov | 1ae74f1 | 2020-09-18 13:52:20 +0100 | [diff] [blame] | 405 | } |
| 406 | default: |
Karl Meakin | e8937d9 | 2024-03-19 16:04:25 +0000 | [diff] [blame] | 407 | dlog_error("FF-A framework message not handled %#lx\n", |
Olivier Deprez | a67ab88 | 2023-01-10 15:00:54 +0100 | [diff] [blame] | 408 | args->arg2); |
| 409 | |
| 410 | /* |
| 411 | * TODO: the framework message that was conveyed by a direct |
| 412 | * request is not handled although we still want to complete |
| 413 | * by a direct response. However, there is no defined error |
| 414 | * response to state that the message couldn't be handled. |
| 415 | * An alternative would be to return FFA_ERROR. |
| 416 | */ |
Karl Meakin | d0356f8 | 2024-09-04 13:34:31 +0100 | [diff] [blame] | 417 | *args = ffa_framework_msg_resp(HF_SPMC_VM_ID, HF_SPMD_VM_ID, |
| 418 | func, 0); |
Olivier Deprez | a67ab88 | 2023-01-10 15:00:54 +0100 | [diff] [blame] | 419 | return true; |
Max Shvetsov | 1ae74f1 | 2020-09-18 13:52:20 +0100 | [diff] [blame] | 420 | } |
Max Shvetsov | 1ae74f1 | 2020-09-18 13:52:20 +0100 | [diff] [blame] | 421 | } |
Madhukar Pappireddy | cf069a6 | 2024-09-25 15:36:32 -0500 | [diff] [blame] | 422 | |
| 423 | void spmc_exit_to_nwd(struct vcpu *owd_vcpu) |
| 424 | { |
| 425 | struct vcpu *deadline_vcpu = |
| 426 | timer_find_vcpu_nearest_deadline(owd_vcpu->cpu); |
| 427 | |
| 428 | /* |
| 429 | * SPMC tracks a vCPU's timer deadline through its host timer such that |
| 430 | * it can bring back execution from normal world to signal the timer |
| 431 | * virtual interrupt to the SP's vCPU. |
| 432 | */ |
| 433 | if (deadline_vcpu != NULL) { |
| 434 | host_timer_track_deadline(&deadline_vcpu->regs.arch_timer); |
| 435 | } |
| 436 | } |
J-Alves | b37fd08 | 2020-10-22 12:29:21 +0100 | [diff] [blame] | 437 | #endif |
| 438 | |
Andrew Scull | ae9962e | 2019-10-03 16:51:16 +0100 | [diff] [blame] | 439 | /** |
| 440 | * Checks whether to block an SMC being forwarded from a VM. |
| 441 | */ |
| 442 | static bool smc_is_blocked(const struct vm *vm, uint32_t func) |
Andrew Walbran | c1ad4ce | 2019-05-09 11:41:39 +0100 | [diff] [blame] | 443 | { |
Andrew Scull | ae9962e | 2019-10-03 16:51:16 +0100 | [diff] [blame] | 444 | bool block_by_default = !vm->smc_whitelist.permissive; |
Fuad Tabba | 8176e3e | 2019-08-01 10:40:36 +0100 | [diff] [blame] | 445 | |
Andrew Scull | ae9962e | 2019-10-03 16:51:16 +0100 | [diff] [blame] | 446 | for (size_t i = 0; i < vm->smc_whitelist.smc_count; ++i) { |
| 447 | if (func == vm->smc_whitelist.smcs[i]) { |
| 448 | return false; |
| 449 | } |
| 450 | } |
Fuad Tabba | 8176e3e | 2019-08-01 10:40:36 +0100 | [diff] [blame] | 451 | |
Olivier Deprez | f92e5d4 | 2020-11-13 16:00:54 +0100 | [diff] [blame] | 452 | dlog_notice("SMC %#010x attempted from VM %#x, blocked=%u\n", func, |
Andrew Walbran | 17eebf9 | 2020-02-05 16:35:49 +0000 | [diff] [blame] | 453 | vm->id, block_by_default); |
Andrew Scull | ae9962e | 2019-10-03 16:51:16 +0100 | [diff] [blame] | 454 | |
| 455 | /* Access is still allowed in permissive mode. */ |
| 456 | return block_by_default; |
Fuad Tabba | 8176e3e | 2019-08-01 10:40:36 +0100 | [diff] [blame] | 457 | } |
| 458 | |
| 459 | /** |
Andrew Scull | ae9962e | 2019-10-03 16:51:16 +0100 | [diff] [blame] | 460 | * Applies SMC access control according to manifest and forwards the call if |
| 461 | * access is granted. |
Fuad Tabba | 8176e3e | 2019-08-01 10:40:36 +0100 | [diff] [blame] | 462 | */ |
Andrew Walbran | b5ab43c | 2020-04-30 11:32:54 +0100 | [diff] [blame] | 463 | static void smc_forwarder(const struct vm *vm, struct ffa_value *args) |
Fuad Tabba | 8176e3e | 2019-08-01 10:40:36 +0100 | [diff] [blame] | 464 | { |
Andrew Walbran | b5ab43c | 2020-04-30 11:32:54 +0100 | [diff] [blame] | 465 | struct ffa_value ret; |
Andrew Walbran | 9dadaf2 | 2019-12-05 16:50:55 +0000 | [diff] [blame] | 466 | uint32_t client_id = vm->id; |
| 467 | uintreg_t arg7 = args->arg7; |
Andrew Scull | ae9962e | 2019-10-03 16:51:16 +0100 | [diff] [blame] | 468 | |
Andrew Walbran | 9dadaf2 | 2019-12-05 16:50:55 +0000 | [diff] [blame] | 469 | if (smc_is_blocked(vm, args->func)) { |
| 470 | args->func = SMCCC_ERROR_UNKNOWN; |
Andrew Scull | ae9962e | 2019-10-03 16:51:16 +0100 | [diff] [blame] | 471 | return; |
| 472 | } |
| 473 | |
Andrew Walbran | 0dd67ff | 2019-09-12 16:38:50 +0100 | [diff] [blame] | 474 | /* |
| 475 | * Set the Client ID but keep the existing Secure OS ID and anything |
| 476 | * else (currently unspecified) that the client may have passed in the |
| 477 | * upper bits. |
| 478 | */ |
Andrew Walbran | 9dadaf2 | 2019-12-05 16:50:55 +0000 | [diff] [blame] | 479 | args->arg7 = client_id | (arg7 & ~CLIENT_ID_MASK); |
Andrew Scull | 07b6bd3 | 2019-12-12 17:19:55 +0000 | [diff] [blame] | 480 | ret = smc_forward(args->func, args->arg1, args->arg2, args->arg3, |
| 481 | args->arg4, args->arg5, args->arg6, args->arg7); |
Fuad Tabba | 8176e3e | 2019-08-01 10:40:36 +0100 | [diff] [blame] | 482 | |
Andrew Scull | ae9962e | 2019-10-03 16:51:16 +0100 | [diff] [blame] | 483 | /* |
Fuad Tabba | b0ef2a4 | 2019-12-19 11:19:25 +0000 | [diff] [blame] | 484 | * Preserve the value passed by the caller, rather than the generated |
| 485 | * client_id. Note that this would also overwrite any return value that |
Andrew Scull | ae9962e | 2019-10-03 16:51:16 +0100 | [diff] [blame] | 486 | * may be in x7, but the SMCs that we are forwarding are legacy calls |
| 487 | * from before SMCCC 1.2 so won't have more than 4 return values anyway. |
| 488 | */ |
Andrew Scull | 07b6bd3 | 2019-12-12 17:19:55 +0000 | [diff] [blame] | 489 | ret.arg7 = arg7; |
| 490 | |
| 491 | plat_smc_post_forward(*args, &ret); |
| 492 | |
| 493 | *args = ret; |
Fuad Tabba | 8176e3e | 2019-08-01 10:40:36 +0100 | [diff] [blame] | 494 | } |
| 495 | |
Olivier Deprez | f33a6c7 | 2020-06-09 18:28:45 +0200 | [diff] [blame] | 496 | /** |
| 497 | * In the normal world, ffa_handler is always called from the virtual FF-A |
Andrew Walbran | 8e8bf3f | 2020-10-07 17:58:20 +0100 | [diff] [blame] | 498 | * instance (from a VM in EL1). In the secure world, ffa_handler may be called |
| 499 | * from the virtual (a secure partition in S-EL1) or physical FF-A instance |
| 500 | * (from the normal world via EL3). The function returns true when the call is |
| 501 | * handled. The *next pointer is updated to the next vCPU to run, which might be |
| 502 | * the 'other world' vCPU if the call originated from the virtual FF-A instance |
| 503 | * and has to be forwarded down to EL3, or left as is to resume the current |
| 504 | * vCPU. |
Olivier Deprez | f33a6c7 | 2020-06-09 18:28:45 +0200 | [diff] [blame] | 505 | */ |
| 506 | static bool ffa_handler(struct ffa_value *args, struct vcpu *current, |
| 507 | struct vcpu **next) |
Andrew Walbran | 7d28d9a | 2019-08-30 16:24:58 +0100 | [diff] [blame] | 508 | { |
J-Alves | bc3de8b | 2020-12-07 14:32:04 +0000 | [diff] [blame] | 509 | uint32_t func = args->func; |
Andrew Walbran | e7ad3c0 | 2019-12-24 17:03:04 +0000 | [diff] [blame] | 510 | |
Jose Marinho | c0f4ff2 | 2019-10-09 10:37:42 +0100 | [diff] [blame] | 511 | /* |
| 512 | * NOTE: When adding new methods to this handler update |
Andrew Walbran | b5ab43c | 2020-04-30 11:32:54 +0100 | [diff] [blame] | 513 | * api_ffa_features accordingly. |
Jose Marinho | c0f4ff2 | 2019-10-09 10:37:42 +0100 | [diff] [blame] | 514 | */ |
Andrew Walbran | e7ad3c0 | 2019-12-24 17:03:04 +0000 | [diff] [blame] | 515 | switch (func) { |
Andrew Walbran | b5ab43c | 2020-04-30 11:32:54 +0100 | [diff] [blame] | 516 | case FFA_VERSION_32: |
Daniel Boulby | baeaf2e | 2021-12-09 11:42:36 +0000 | [diff] [blame] | 517 | *args = api_ffa_version(current, args->arg1); |
Andrew Walbran | 7d28d9a | 2019-08-30 16:24:58 +0100 | [diff] [blame] | 518 | return true; |
Fuad Tabba | e4efcc3 | 2020-07-16 15:37:27 +0100 | [diff] [blame] | 519 | case FFA_PARTITION_INFO_GET_32: { |
| 520 | struct ffa_uuid uuid; |
| 521 | |
| 522 | ffa_uuid_init(args->arg1, args->arg2, args->arg3, args->arg4, |
| 523 | &uuid); |
Daniel Boulby | b46cad1 | 2021-12-13 17:47:21 +0000 | [diff] [blame] | 524 | *args = api_ffa_partition_info_get(current, &uuid, args->arg5); |
Fuad Tabba | e4efcc3 | 2020-07-16 15:37:27 +0100 | [diff] [blame] | 525 | return true; |
| 526 | } |
Raghu Krishnamurthy | 7592bcb | 2022-12-25 13:09:00 -0800 | [diff] [blame] | 527 | case FFA_PARTITION_INFO_GET_REGS_64: { |
| 528 | struct ffa_uuid uuid; |
Raghu Krishnamurthy | 7592bcb | 2022-12-25 13:09:00 -0800 | [diff] [blame] | 529 | uint16_t start_index; |
| 530 | uint16_t tag; |
| 531 | |
Karl Meakin | 9478e32 | 2024-09-23 17:47:09 +0100 | [diff] [blame] | 532 | ffa_uuid_from_u64x2(args->arg1, args->arg2, &uuid); |
Raghu Krishnamurthy | d29411a | 2023-02-17 17:22:04 -0800 | [diff] [blame] | 533 | start_index = args->arg3 & 0xFFFF; |
| 534 | tag = (args->arg3 >> 16) & 0xFFFF; |
Raghu Krishnamurthy | 7592bcb | 2022-12-25 13:09:00 -0800 | [diff] [blame] | 535 | *args = api_ffa_partition_info_get_regs(current, &uuid, |
| 536 | start_index, tag); |
| 537 | return true; |
| 538 | } |
Andrew Walbran | b5ab43c | 2020-04-30 11:32:54 +0100 | [diff] [blame] | 539 | case FFA_ID_GET_32: |
Olivier Deprez | f33a6c7 | 2020-06-09 18:28:45 +0200 | [diff] [blame] | 540 | *args = api_ffa_id_get(current); |
Andrew Walbran | d230f66 | 2019-10-07 18:03:36 +0100 | [diff] [blame] | 541 | return true; |
Daniel Boulby | b2fb80e | 2021-02-03 15:09:23 +0000 | [diff] [blame] | 542 | case FFA_SPM_ID_GET_32: |
| 543 | *args = api_ffa_spm_id_get(); |
| 544 | return true; |
Andrew Walbran | b5ab43c | 2020-04-30 11:32:54 +0100 | [diff] [blame] | 545 | case FFA_FEATURES_32: |
Karl Meakin | f1ed5f1 | 2024-02-22 15:57:36 +0000 | [diff] [blame] | 546 | *args = api_ffa_features(args->arg1, args->arg2, current); |
Jose Marinho | c0f4ff2 | 2019-10-09 10:37:42 +0100 | [diff] [blame] | 547 | return true; |
Andrew Walbran | b5ab43c | 2020-04-30 11:32:54 +0100 | [diff] [blame] | 548 | case FFA_RX_RELEASE_32: |
J-Alves | e8c8c2b | 2022-12-16 15:34:48 +0000 | [diff] [blame] | 549 | *args = api_ffa_rx_release(ffa_receiver(*args), current); |
Andrew Walbran | 8a0f5ca | 2019-11-05 13:12:23 +0000 | [diff] [blame] | 550 | return true; |
J-Alves | bc3de8b | 2020-12-07 14:32:04 +0000 | [diff] [blame] | 551 | case FFA_RXTX_MAP_64: |
Andrew Walbran | b5ab43c | 2020-04-30 11:32:54 +0100 | [diff] [blame] | 552 | *args = api_ffa_rxtx_map(ipa_init(args->arg1), |
| 553 | ipa_init(args->arg2), args->arg3, |
Federico Recanati | 9f1b653 | 2022-04-14 13:15:28 +0200 | [diff] [blame] | 554 | current); |
Andrew Walbran | bfffb0f | 2019-11-05 14:02:34 +0000 | [diff] [blame] | 555 | return true; |
Daniel Boulby | 9e420ca | 2021-07-07 15:03:49 +0100 | [diff] [blame] | 556 | case FFA_RXTX_UNMAP_32: |
J-Alves | 7007993 | 2022-12-07 17:32:20 +0000 | [diff] [blame] | 557 | *args = api_ffa_rxtx_unmap(ffa_vm_id(*args), current); |
Daniel Boulby | 9e420ca | 2021-07-07 15:03:49 +0100 | [diff] [blame] | 558 | return true; |
Federico Recanati | 644f046 | 2022-03-17 12:04:00 +0100 | [diff] [blame] | 559 | case FFA_RX_ACQUIRE_32: |
| 560 | *args = api_ffa_rx_acquire(ffa_receiver(*args), current); |
| 561 | return true; |
Andrew Walbran | b5ab43c | 2020-04-30 11:32:54 +0100 | [diff] [blame] | 562 | case FFA_YIELD_32: |
Madhukar Pappireddy | 184501c | 2023-05-23 17:24:06 -0500 | [diff] [blame] | 563 | *args = api_yield(current, next, args); |
Andrew Walbran | 7d28d9a | 2019-08-30 16:24:58 +0100 | [diff] [blame] | 564 | return true; |
Andrew Walbran | b5ab43c | 2020-04-30 11:32:54 +0100 | [diff] [blame] | 565 | case FFA_MSG_SEND_32: |
Karl Meakin | 117c808 | 2024-12-04 16:03:28 +0000 | [diff] [blame] | 566 | *args = ffa_indirect_msg_send( |
J-Alves | 27b7196 | 2022-12-12 15:29:58 +0000 | [diff] [blame] | 567 | ffa_sender(*args), ffa_receiver(*args), |
| 568 | ffa_msg_send_size(*args), current, next); |
Andrew Walbran | 7d28d9a | 2019-08-30 16:24:58 +0100 | [diff] [blame] | 569 | return true; |
Federico Recanati | 25053ee | 2022-03-14 15:01:53 +0100 | [diff] [blame] | 570 | case FFA_MSG_SEND2_32: |
| 571 | *args = api_ffa_msg_send2(ffa_sender(*args), |
| 572 | ffa_msg_send2_flags(*args), current); |
| 573 | return true; |
Andrew Walbran | b5ab43c | 2020-04-30 11:32:54 +0100 | [diff] [blame] | 574 | case FFA_MSG_WAIT_32: |
Madhukar Pappireddy | 5522c67 | 2021-12-17 16:35:51 -0600 | [diff] [blame] | 575 | *args = api_ffa_msg_wait(current, next, args); |
Andrew Walbran | 0de4f16 | 2019-09-03 16:44:20 +0100 | [diff] [blame] | 576 | return true; |
J-Alves | bc7ab4f | 2022-12-13 12:09:25 +0000 | [diff] [blame] | 577 | #if SECURE_WORLD == 0 |
Madhukar Pappireddy | bd10e57 | 2023-03-06 16:39:49 -0600 | [diff] [blame] | 578 | case FFA_MSG_POLL_32: { |
| 579 | struct vcpu_locked current_locked; |
| 580 | |
| 581 | current_locked = vcpu_lock(current); |
Karl Meakin | 117c808 | 2024-12-04 16:03:28 +0000 | [diff] [blame] | 582 | *args = ffa_indirect_msg_recv(false, current_locked, next); |
Madhukar Pappireddy | bd10e57 | 2023-03-06 16:39:49 -0600 | [diff] [blame] | 583 | vcpu_unlock(¤t_locked); |
Andrew Walbran | 7d28d9a | 2019-08-30 16:24:58 +0100 | [diff] [blame] | 584 | return true; |
Madhukar Pappireddy | bd10e57 | 2023-03-06 16:39:49 -0600 | [diff] [blame] | 585 | } |
J-Alves | bc7ab4f | 2022-12-13 12:09:25 +0000 | [diff] [blame] | 586 | #endif |
Andrew Walbran | b5ab43c | 2020-04-30 11:32:54 +0100 | [diff] [blame] | 587 | case FFA_RUN_32: |
Kathleen Capella | 036cc59 | 2023-11-30 18:26:15 -0500 | [diff] [blame] | 588 | /** |
| 589 | * Ensure that an FF-A v1.2 endpoint preserves the |
| 590 | * runtime state of the calling partition by setting |
| 591 | * the extended registers (x8-x17) to zero. |
| 592 | */ |
Karl Meakin | 0e617d9 | 2024-04-05 12:55:22 +0100 | [diff] [blame] | 593 | if (current->vm->ffa_version >= FFA_VERSION_1_2 && |
Kathleen Capella | 036cc59 | 2023-11-30 18:26:15 -0500 | [diff] [blame] | 594 | !api_extended_args_are_zero(args)) { |
| 595 | *args = ffa_error(FFA_INVALID_PARAMETERS); |
| 596 | return false; |
| 597 | } |
Andrew Walbran | b5ab43c | 2020-04-30 11:32:54 +0100 | [diff] [blame] | 598 | *args = api_ffa_run(ffa_vm_id(*args), ffa_vcpu_index(*args), |
Olivier Deprez | f33a6c7 | 2020-06-09 18:28:45 +0200 | [diff] [blame] | 599 | current, next); |
Andrew Walbran | f0c314d | 2019-10-02 14:24:26 +0100 | [diff] [blame] | 600 | return true; |
Andrew Walbran | b5ab43c | 2020-04-30 11:32:54 +0100 | [diff] [blame] | 601 | case FFA_MEM_DONATE_32: |
J-Alves | 95fbb31 | 2024-03-20 15:19:16 +0000 | [diff] [blame] | 602 | case FFA_MEM_DONATE_64: |
Andrew Walbran | b5ab43c | 2020-04-30 11:32:54 +0100 | [diff] [blame] | 603 | case FFA_MEM_LEND_32: |
J-Alves | 95fbb31 | 2024-03-20 15:19:16 +0000 | [diff] [blame] | 604 | case FFA_MEM_LEND_64: |
Andrew Walbran | b5ab43c | 2020-04-30 11:32:54 +0100 | [diff] [blame] | 605 | case FFA_MEM_SHARE_32: |
J-Alves | 95fbb31 | 2024-03-20 15:19:16 +0000 | [diff] [blame] | 606 | case FFA_MEM_SHARE_64: |
Andrew Walbran | b5ab43c | 2020-04-30 11:32:54 +0100 | [diff] [blame] | 607 | *args = api_ffa_mem_send(func, args->arg1, args->arg2, |
| 608 | ipa_init(args->arg3), args->arg4, |
Olivier Deprez | f33a6c7 | 2020-06-09 18:28:45 +0200 | [diff] [blame] | 609 | current); |
Andrew Walbran | 82d6d15 | 2019-12-24 15:02:06 +0000 | [diff] [blame] | 610 | return true; |
J-Alves | 95fbb31 | 2024-03-20 15:19:16 +0000 | [diff] [blame] | 611 | case FFA_MEM_RETRIEVE_REQ_64: |
Andrew Walbran | b5ab43c | 2020-04-30 11:32:54 +0100 | [diff] [blame] | 612 | case FFA_MEM_RETRIEVE_REQ_32: |
| 613 | *args = api_ffa_mem_retrieve_req(args->arg1, args->arg2, |
| 614 | ipa_init(args->arg3), |
Olivier Deprez | f33a6c7 | 2020-06-09 18:28:45 +0200 | [diff] [blame] | 615 | args->arg4, current); |
Andrew Walbran | 5de9c3d | 2020-02-10 13:35:29 +0000 | [diff] [blame] | 616 | return true; |
Andrew Walbran | b5ab43c | 2020-04-30 11:32:54 +0100 | [diff] [blame] | 617 | case FFA_MEM_RELINQUISH_32: |
Olivier Deprez | f33a6c7 | 2020-06-09 18:28:45 +0200 | [diff] [blame] | 618 | *args = api_ffa_mem_relinquish(current); |
Andrew Walbran | 5de9c3d | 2020-02-10 13:35:29 +0000 | [diff] [blame] | 619 | return true; |
Andrew Walbran | b5ab43c | 2020-04-30 11:32:54 +0100 | [diff] [blame] | 620 | case FFA_MEM_RECLAIM_32: |
| 621 | *args = api_ffa_mem_reclaim( |
Andrew Walbran | 1bbe940 | 2020-04-30 16:47:13 +0100 | [diff] [blame] | 622 | ffa_assemble_handle(args->arg1, args->arg2), args->arg3, |
Olivier Deprez | f33a6c7 | 2020-06-09 18:28:45 +0200 | [diff] [blame] | 623 | current); |
Andrew Walbran | 5de9c3d | 2020-02-10 13:35:29 +0000 | [diff] [blame] | 624 | return true; |
Andrew Walbran | ca808b1 | 2020-05-15 17:22:28 +0100 | [diff] [blame] | 625 | case FFA_MEM_FRAG_RX_32: |
| 626 | *args = api_ffa_mem_frag_rx(ffa_frag_handle(*args), args->arg3, |
| 627 | (args->arg4 >> 16) & 0xffff, |
Olivier Deprez | f33a6c7 | 2020-06-09 18:28:45 +0200 | [diff] [blame] | 628 | current); |
Andrew Walbran | ca808b1 | 2020-05-15 17:22:28 +0100 | [diff] [blame] | 629 | return true; |
| 630 | case FFA_MEM_FRAG_TX_32: |
| 631 | *args = api_ffa_mem_frag_tx(ffa_frag_handle(*args), args->arg3, |
| 632 | (args->arg4 >> 16) & 0xffff, |
Olivier Deprez | f33a6c7 | 2020-06-09 18:28:45 +0200 | [diff] [blame] | 633 | current); |
Andrew Walbran | ca808b1 | 2020-05-15 17:22:28 +0100 | [diff] [blame] | 634 | return true; |
J-Alves | bc3de8b | 2020-12-07 14:32:04 +0000 | [diff] [blame] | 635 | case FFA_MSG_SEND_DIRECT_REQ_64: |
Karl Meakin | d0356f8 | 2024-09-04 13:34:31 +0100 | [diff] [blame] | 636 | case FFA_MSG_SEND_DIRECT_REQ_32: |
Max Shvetsov | 1ae74f1 | 2020-09-18 13:52:20 +0100 | [diff] [blame] | 637 | #if SECURE_WORLD == 1 |
| 638 | if (spmd_handler(args, current)) { |
| 639 | return true; |
| 640 | } |
| 641 | #endif |
Kathleen Capella | 41fea93 | 2023-06-23 17:39:28 -0400 | [diff] [blame] | 642 | case FFA_MSG_SEND_DIRECT_REQ2_64: |
Karl Meakin | 13f0981 | 2024-10-28 16:33:23 +0000 | [diff] [blame] | 643 | *args = api_ffa_msg_send_direct_req(*args, current, next); |
Kathleen Capella | 41fea93 | 2023-06-23 17:39:28 -0400 | [diff] [blame] | 644 | return true; |
J-Alves | bc3de8b | 2020-12-07 14:32:04 +0000 | [diff] [blame] | 645 | case FFA_MSG_SEND_DIRECT_RESP_64: |
Olivier Deprez | ee9d6a9 | 2019-11-26 09:14:11 +0000 | [diff] [blame] | 646 | case FFA_MSG_SEND_DIRECT_RESP_32: |
Kathleen Capella | 087e502 | 2023-09-07 18:04:15 -0400 | [diff] [blame] | 647 | case FFA_MSG_SEND_DIRECT_RESP2_64: |
Karl Meakin | 13f0981 | 2024-10-28 16:33:23 +0000 | [diff] [blame] | 648 | *args = api_ffa_msg_send_direct_resp(*args, current, next); |
Olivier Deprez | ee9d6a9 | 2019-11-26 09:14:11 +0000 | [diff] [blame] | 649 | return true; |
J-Alves | bc3de8b | 2020-12-07 14:32:04 +0000 | [diff] [blame] | 650 | case FFA_SECONDARY_EP_REGISTER_64: |
Olivier Deprez | d614d32 | 2021-06-18 15:21:00 +0200 | [diff] [blame] | 651 | /* |
| 652 | * DEN0077A FF-A v1.1 Beta0 section 18.3.2.1.1 |
| 653 | * The callee must return NOT_SUPPORTED if this function is |
| 654 | * invoked by a caller that implements version v1.0 of |
| 655 | * the Framework. |
| 656 | */ |
Max Shvetsov | 40108e7 | 2020-08-27 12:39:50 +0100 | [diff] [blame] | 657 | *args = api_ffa_secondary_ep_register(ipa_init(args->arg1), |
| 658 | current); |
| 659 | return true; |
J-Alves | a0f317d | 2021-06-09 13:31:59 +0100 | [diff] [blame] | 660 | case FFA_NOTIFICATION_BITMAP_CREATE_32: |
| 661 | *args = api_ffa_notification_bitmap_create( |
J-Alves | 19e20cf | 2023-08-02 12:48:55 +0100 | [diff] [blame] | 662 | (ffa_id_t)args->arg1, (ffa_vcpu_count_t)args->arg2, |
J-Alves | a0f317d | 2021-06-09 13:31:59 +0100 | [diff] [blame] | 663 | current); |
| 664 | return true; |
| 665 | case FFA_NOTIFICATION_BITMAP_DESTROY_32: |
| 666 | *args = api_ffa_notification_bitmap_destroy( |
J-Alves | 19e20cf | 2023-08-02 12:48:55 +0100 | [diff] [blame] | 667 | (ffa_id_t)args->arg1, current); |
J-Alves | a0f317d | 2021-06-09 13:31:59 +0100 | [diff] [blame] | 668 | return true; |
J-Alves | c003a7a | 2021-03-18 13:06:53 +0000 | [diff] [blame] | 669 | case FFA_NOTIFICATION_BIND_32: |
| 670 | *args = api_ffa_notification_update_bindings( |
| 671 | ffa_sender(*args), ffa_receiver(*args), args->arg2, |
| 672 | ffa_notifications_bitmap(args->arg3, args->arg4), true, |
| 673 | current); |
| 674 | return true; |
| 675 | case FFA_NOTIFICATION_UNBIND_32: |
| 676 | *args = api_ffa_notification_update_bindings( |
| 677 | ffa_sender(*args), ffa_receiver(*args), 0, |
| 678 | ffa_notifications_bitmap(args->arg3, args->arg4), false, |
| 679 | current); |
| 680 | return true; |
Raghu Krishnamurthy | ea6d25f | 2021-09-14 15:27:06 -0700 | [diff] [blame] | 681 | case FFA_MEM_PERM_SET_32: |
| 682 | case FFA_MEM_PERM_SET_64: |
| 683 | *args = api_ffa_mem_perm_set(va_init(args->arg1), args->arg2, |
| 684 | args->arg3, current); |
| 685 | return true; |
| 686 | case FFA_MEM_PERM_GET_32: |
| 687 | case FFA_MEM_PERM_GET_64: |
| 688 | *args = api_ffa_mem_perm_get(va_init(args->arg1), current); |
| 689 | return true; |
J-Alves | aa79c01 | 2021-07-09 14:29:45 +0100 | [diff] [blame] | 690 | case FFA_NOTIFICATION_SET_32: |
| 691 | *args = api_ffa_notification_set( |
| 692 | ffa_sender(*args), ffa_receiver(*args), args->arg2, |
| 693 | ffa_notifications_bitmap(args->arg3, args->arg4), |
| 694 | current); |
| 695 | return true; |
| 696 | case FFA_NOTIFICATION_GET_32: |
| 697 | *args = api_ffa_notification_get( |
J-Alves | be6e303 | 2021-11-30 14:54:12 +0000 | [diff] [blame] | 698 | ffa_receiver(*args), ffa_notifications_get_vcpu(*args), |
| 699 | args->arg2, current); |
J-Alves | aa79c01 | 2021-07-09 14:29:45 +0100 | [diff] [blame] | 700 | return true; |
J-Alves | c8e8a22 | 2021-06-08 17:33:52 +0100 | [diff] [blame] | 701 | case FFA_NOTIFICATION_INFO_GET_64: |
| 702 | *args = api_ffa_notification_info_get(current); |
| 703 | return true; |
Madhukar Pappireddy | 9e7a11f | 2021-08-03 13:59:42 -0500 | [diff] [blame] | 704 | case FFA_INTERRUPT_32: |
J-Alves | 03edf40 | 2023-07-21 15:13:49 +0100 | [diff] [blame] | 705 | /* |
| 706 | * A malicious SP could invoke a HVC/SMC call with |
| 707 | * FFA_INTERRUPT_32 as the function argument. Return error to |
| 708 | * avoid DoS. |
| 709 | */ |
| 710 | if (current->vm->id != HF_OTHER_WORLD_ID) { |
| 711 | *args = ffa_error(FFA_DENIED); |
| 712 | return true; |
| 713 | } |
J-Alves | cf0c471 | 2023-08-04 14:41:50 +0100 | [diff] [blame] | 714 | |
Karl Meakin | 117c808 | 2024-12-04 16:03:28 +0000 | [diff] [blame] | 715 | ffa_interrupts_handle_secure_interrupt(current, next); |
J-Alves | cf0c471 | 2023-08-04 14:41:50 +0100 | [diff] [blame] | 716 | |
| 717 | /* |
| 718 | * If the next vCPU belongs to an SP, the next time the NWd |
| 719 | * gets resumed these values will be overwritten by the ABI |
| 720 | * that used to handover execution back to the NWd. |
| 721 | * If the NWd is to be resumed from here, then it will |
| 722 | * receive the FFA_NORMAL_WORLD_RESUME ABI which is to signal |
| 723 | * that an interrupt has occured, thought it wasn't handled. |
| 724 | * This happens when the target vCPU was in preempted state, |
| 725 | * and the SP couldn't not be resumed to handle the interrupt. |
| 726 | */ |
| 727 | *args = (struct ffa_value){.func = FFA_NORMAL_WORLD_RESUME}; |
Madhukar Pappireddy | 9e7a11f | 2021-08-03 13:59:42 -0500 | [diff] [blame] | 728 | return true; |
Maksims Svecovs | 71b7670 | 2022-05-20 15:32:58 +0100 | [diff] [blame] | 729 | case FFA_CONSOLE_LOG_32: |
| 730 | case FFA_CONSOLE_LOG_64: |
| 731 | *args = api_ffa_console_log(*args, current); |
| 732 | return true; |
Kathleen Capella | 6ab0513 | 2023-05-10 12:27:35 -0400 | [diff] [blame] | 733 | case FFA_ERROR_32: |
Karl Meakin | fa1dcb8 | 2025-02-10 16:47:50 +0000 | [diff] [blame] | 734 | *args = ffa_cpu_cycles_error_32(current, next, args->arg2); |
Kathleen Capella | 6ab0513 | 2023-05-10 12:27:35 -0400 | [diff] [blame] | 735 | return true; |
Andrew Walbran | 7d28d9a | 2019-08-30 16:24:58 +0100 | [diff] [blame] | 736 | |
Karl Meakin | a5ea909 | 2024-05-28 15:40:33 +0100 | [diff] [blame] | 737 | default: |
Karl Meakin | a5ea909 | 2024-05-28 15:40:33 +0100 | [diff] [blame] | 738 | return false; |
| 739 | } |
Andrew Walbran | 7d28d9a | 2019-08-30 16:24:58 +0100 | [diff] [blame] | 740 | } |
| 741 | |
| 742 | /** |
Manish Pandey | 35e452f | 2021-02-18 21:36:34 +0000 | [diff] [blame] | 743 | * Set or clear VI/VF bits according to pending interrupts. |
J-Alves | 6f6bf8a | 2024-07-25 15:17:57 +0100 | [diff] [blame] | 744 | * If `vcpu` is NULL, the function will set it to the currently running |
| 745 | * vCPU. |
Andrew Walbran | 7d28d9a | 2019-08-30 16:24:58 +0100 | [diff] [blame] | 746 | */ |
J-Alves | 6f6bf8a | 2024-07-25 15:17:57 +0100 | [diff] [blame] | 747 | static void vcpu_update_virtual_interrupts(struct vcpu *vcpu) |
Andrew Walbran | 7d28d9a | 2019-08-30 16:24:58 +0100 | [diff] [blame] | 748 | { |
Manish Pandey | 35e452f | 2021-02-18 21:36:34 +0000 | [diff] [blame] | 749 | struct vcpu_locked vcpu_locked; |
| 750 | |
J-Alves | 6f6bf8a | 2024-07-25 15:17:57 +0100 | [diff] [blame] | 751 | if (vcpu == NULL) { |
| 752 | vcpu = current(); |
Andrew Walbran | 7d28d9a | 2019-08-30 16:24:58 +0100 | [diff] [blame] | 753 | } |
J-Alves | 6f6bf8a | 2024-07-25 15:17:57 +0100 | [diff] [blame] | 754 | |
| 755 | /* Only update to those at the virtual instance. */ |
| 756 | if (vcpu->vm->el0_partition || !vm_id_is_current_world(vcpu->vm->id)) { |
| 757 | return; |
| 758 | } |
| 759 | |
| 760 | vcpu_locked = vcpu_lock(vcpu); |
| 761 | set_virtual_irq(&vcpu->regs, |
| 762 | vcpu_interrupt_irq_count_get(vcpu_locked) > 0); |
| 763 | set_virtual_fiq(&vcpu->regs, |
| 764 | vcpu_interrupt_fiq_count_get(vcpu_locked) > 0); |
| 765 | vcpu_unlock(&vcpu_locked); |
Andrew Walbran | 7d28d9a | 2019-08-30 16:24:58 +0100 | [diff] [blame] | 766 | } |
| 767 | |
Fuad Tabba | 8176e3e | 2019-08-01 10:40:36 +0100 | [diff] [blame] | 768 | /** |
Andrew Walbran | d8d3f5d | 2020-10-07 18:23:01 +0100 | [diff] [blame] | 769 | * Handles PSCI and FF-A calls and writes the return value back to the registers |
| 770 | * of the vCPU. This is shared between smc_handler and hvc_handler. |
| 771 | * |
| 772 | * Returns true if the call was handled. |
| 773 | */ |
| 774 | static bool hvc_smc_handler(struct ffa_value args, struct vcpu *vcpu, |
| 775 | struct vcpu **next) |
| 776 | { |
Karl Meakin | 6eeec8e | 2024-03-07 18:07:20 +0000 | [diff] [blame] | 777 | const uint32_t func = args.func; |
| 778 | |
Olivier Deprez | 3caed1c | 2021-02-05 12:07:36 +0100 | [diff] [blame] | 779 | /* Do not expect PSCI calls emitted from within the secure world. */ |
| 780 | #if SECURE_WORLD == 0 |
Karl Meakin | 6eeec8e | 2024-03-07 18:07:20 +0000 | [diff] [blame] | 781 | if (psci_handler(vcpu, func, args.arg1, args.arg2, args.arg3, |
Andrew Walbran | d8d3f5d | 2020-10-07 18:23:01 +0100 | [diff] [blame] | 782 | &vcpu->regs.r[0], next)) { |
| 783 | return true; |
| 784 | } |
Olivier Deprez | 3caed1c | 2021-02-05 12:07:36 +0100 | [diff] [blame] | 785 | #endif |
Andrew Walbran | d8d3f5d | 2020-10-07 18:23:01 +0100 | [diff] [blame] | 786 | |
Andrew Walbran | d8d3f5d | 2020-10-07 18:23:01 +0100 | [diff] [blame] | 787 | if (ffa_handler(&args, vcpu, next)) { |
J-Alves | 1339402 | 2021-06-30 13:48:49 +0100 | [diff] [blame] | 788 | #if SECURE_WORLD == 1 |
| 789 | /* |
| 790 | * If giving back execution to the NWd, check if the Schedule |
Olivier Deprez | 618c8fc | 2022-05-30 15:27:49 +0200 | [diff] [blame] | 791 | * Receiver Interrupt has been delayed, and trigger it on |
| 792 | * current core if so. |
J-Alves | 1339402 | 2021-06-30 13:48:49 +0100 | [diff] [blame] | 793 | */ |
| 794 | if ((*next != NULL && (*next)->vm->id == HF_OTHER_WORLD_ID) || |
| 795 | (*next == NULL && vcpu->vm->id == HF_OTHER_WORLD_ID)) { |
Karl Meakin | 117c808 | 2024-12-04 16:03:28 +0000 | [diff] [blame] | 796 | ffa_notifications_sri_trigger_if_delayed(vcpu->cpu); |
J-Alves | 1339402 | 2021-06-30 13:48:49 +0100 | [diff] [blame] | 797 | } |
| 798 | #endif |
Karl Meakin | 6eeec8e | 2024-03-07 18:07:20 +0000 | [diff] [blame] | 799 | if (func != FFA_VERSION_32) { |
| 800 | struct vm_locked vm_locked = vm_lock(vcpu->vm); |
| 801 | |
| 802 | vm_locked.vm->ffa_version_negotiated = true; |
| 803 | vm_unlock(&vm_locked); |
| 804 | } |
| 805 | |
Andrew Walbran | d8d3f5d | 2020-10-07 18:23:01 +0100 | [diff] [blame] | 806 | arch_regs_set_retval(&vcpu->regs, args); |
J-Alves | 6f6bf8a | 2024-07-25 15:17:57 +0100 | [diff] [blame] | 807 | |
| 808 | /* |
| 809 | * In case there has been an update after handling the last |
| 810 | * ff-a call, update the next vCPU directly in the |
| 811 | * register. |
| 812 | */ |
Manish Pandey | 35e452f | 2021-02-18 21:36:34 +0000 | [diff] [blame] | 813 | vcpu_update_virtual_interrupts(*next); |
Andrew Walbran | d8d3f5d | 2020-10-07 18:23:01 +0100 | [diff] [blame] | 814 | return true; |
| 815 | } |
| 816 | |
| 817 | return false; |
| 818 | } |
| 819 | |
| 820 | /** |
Fuad Tabba | 8176e3e | 2019-08-01 10:40:36 +0100 | [diff] [blame] | 821 | * Processes SMC instruction calls. |
| 822 | */ |
Andrew Walbran | 9dadaf2 | 2019-12-05 16:50:55 +0000 | [diff] [blame] | 823 | static struct vcpu *smc_handler(struct vcpu *vcpu) |
Fuad Tabba | 8176e3e | 2019-08-01 10:40:36 +0100 | [diff] [blame] | 824 | { |
Andrew Walbran | d8d3f5d | 2020-10-07 18:23:01 +0100 | [diff] [blame] | 825 | struct ffa_value args = arch_regs_get_args(&vcpu->regs); |
Andrew Walbran | 9dadaf2 | 2019-12-05 16:50:55 +0000 | [diff] [blame] | 826 | struct vcpu *next = NULL; |
Fuad Tabba | 8176e3e | 2019-08-01 10:40:36 +0100 | [diff] [blame] | 827 | |
Olivier Deprez | 79dbd6f | 2023-11-29 16:12:36 +0100 | [diff] [blame] | 828 | /* Mask out SMCCC SVE hint bit from function id. */ |
| 829 | args.func &= ~SMCCC_SVE_HINT_MASK; |
| 830 | |
Andrew Walbran | d8d3f5d | 2020-10-07 18:23:01 +0100 | [diff] [blame] | 831 | if (hvc_smc_handler(args, vcpu, &next)) { |
Andrew Walbran | 9dadaf2 | 2019-12-05 16:50:55 +0000 | [diff] [blame] | 832 | return next; |
Andrew Walbran | 4579f700 | 2019-08-30 16:24:58 +0100 | [diff] [blame] | 833 | } |
| 834 | |
Andrew Walbran | 9dadaf2 | 2019-12-05 16:50:55 +0000 | [diff] [blame] | 835 | smc_forwarder(vcpu->vm, &args); |
| 836 | arch_regs_set_retval(&vcpu->regs, args); |
Andrew Scull | 07b6bd3 | 2019-12-12 17:19:55 +0000 | [diff] [blame] | 837 | return NULL; |
Andrew Walbran | c1ad4ce | 2019-05-09 11:41:39 +0100 | [diff] [blame] | 838 | } |
| 839 | |
Olivier Deprez | 3caed1c | 2021-02-05 12:07:36 +0100 | [diff] [blame] | 840 | #if SECURE_WORLD == 1 |
| 841 | |
| 842 | /** |
| 843 | * Called from other_world_loop return from SMC. |
| 844 | * Processes SMC calls originating from the NWd. |
| 845 | */ |
| 846 | struct vcpu *smc_handler_from_nwd(struct vcpu *vcpu) |
| 847 | { |
Olivier Deprez | 79dbd6f | 2023-11-29 16:12:36 +0100 | [diff] [blame] | 848 | struct ffa_value args = arch_regs_get_args(&vcpu->regs); |
Olivier Deprez | 3caed1c | 2021-02-05 12:07:36 +0100 | [diff] [blame] | 849 | struct vcpu *next = NULL; |
| 850 | |
Olivier Deprez | 5b58833 | 2023-09-05 15:08:48 +0200 | [diff] [blame] | 851 | plat_save_ns_simd_context(vcpu); |
| 852 | |
Olivier Deprez | 79dbd6f | 2023-11-29 16:12:36 +0100 | [diff] [blame] | 853 | /* Mask out SMCCC SVE hint bit from function id. */ |
| 854 | args.func &= ~SMCCC_SVE_HINT_MASK; |
| 855 | |
Olivier Deprez | 3caed1c | 2021-02-05 12:07:36 +0100 | [diff] [blame] | 856 | if (hvc_smc_handler(args, vcpu, &next)) { |
| 857 | return next; |
| 858 | } |
| 859 | |
| 860 | /* |
| 861 | * If the SMC emitted by the normal world is not handled in the secure |
| 862 | * world then return an error stating such ABI is not supported. Only |
| 863 | * FF-A calls are supported. We cannot return SMCCC_ERROR_UNKNOWN |
| 864 | * directly because the SPMD smc handler would not recognize it as a |
| 865 | * standard FF-A call returning from the SPMC. |
| 866 | */ |
| 867 | arch_regs_set_retval(&vcpu->regs, ffa_error(FFA_NOT_SUPPORTED)); |
| 868 | |
| 869 | return NULL; |
| 870 | } |
| 871 | |
| 872 | #endif |
| 873 | |
Fuad Tabba | a48d122 | 2019-12-09 15:42:32 +0000 | [diff] [blame] | 874 | /* |
| 875 | * Exception vector offsets. |
| 876 | * See Arm Architecture Reference Manual Armv8-A, D1.10.2. |
| 877 | */ |
| 878 | |
| 879 | /** |
| 880 | * Offset for synchronous exceptions at current EL with SPx. |
| 881 | */ |
| 882 | #define OFFSET_CURRENT_SPX UINT64_C(0x200) |
| 883 | |
| 884 | /** |
| 885 | * Offset for synchronous exceptions at lower EL using AArch64. |
| 886 | */ |
| 887 | #define OFFSET_LOWER_EL_64 UINT64_C(0x400) |
| 888 | |
| 889 | /** |
| 890 | * Offset for synchronous exceptions at lower EL using AArch32. |
| 891 | */ |
| 892 | #define OFFSET_LOWER_EL_32 UINT64_C(0x600) |
| 893 | |
| 894 | /** |
| 895 | * Returns the address for the exception handler at EL1. |
| 896 | */ |
| 897 | static uintreg_t get_el1_exception_handler_addr(const struct vcpu *vcpu) |
| 898 | { |
Raghu Krishnamurthy | 32626c9 | 2021-01-17 09:57:29 -0800 | [diff] [blame] | 899 | uintreg_t base_addr = has_vhe_support() ? read_msr(MSR_VBAR_EL12) |
| 900 | : read_msr(vbar_el1); |
Fuad Tabba | a48d122 | 2019-12-09 15:42:32 +0000 | [diff] [blame] | 901 | uintreg_t pe_mode = vcpu->regs.spsr & PSR_PE_MODE_MASK; |
| 902 | bool is_arch32 = vcpu->regs.spsr & PSR_ARCH_MODE_32; |
| 903 | |
| 904 | if (pe_mode == PSR_PE_MODE_EL0T) { |
| 905 | if (is_arch32) { |
| 906 | base_addr += OFFSET_LOWER_EL_32; |
| 907 | } else { |
| 908 | base_addr += OFFSET_LOWER_EL_64; |
| 909 | } |
| 910 | } else { |
| 911 | CHECK(!is_arch32); |
| 912 | base_addr += OFFSET_CURRENT_SPX; |
| 913 | } |
| 914 | |
| 915 | return base_addr; |
| 916 | } |
| 917 | |
| 918 | /** |
Fuad Tabba | b86325a | 2020-01-10 13:38:15 +0000 | [diff] [blame] | 919 | * Injects an exception with the specified Exception Syndrom Register value into |
| 920 | * the EL1. |
Fuad Tabba | a48d122 | 2019-12-09 15:42:32 +0000 | [diff] [blame] | 921 | * |
| 922 | * NOTE: This function assumes that the lazy registers haven't been saved, and |
| 923 | * writes to the lazy registers of the CPU directly instead of the vCPU. |
| 924 | */ |
Fuad Tabba | c3847c7 | 2020-08-11 09:32:25 +0100 | [diff] [blame] | 925 | static void inject_el1_exception(struct vcpu *vcpu, uintreg_t esr_el1_value, |
| 926 | uintreg_t far_el1_value) |
Fuad Tabba | a48d122 | 2019-12-09 15:42:32 +0000 | [diff] [blame] | 927 | { |
Fuad Tabba | a48d122 | 2019-12-09 15:42:32 +0000 | [diff] [blame] | 928 | uintreg_t handler_address = get_el1_exception_handler_addr(vcpu); |
Fuad Tabba | a48d122 | 2019-12-09 15:42:32 +0000 | [diff] [blame] | 929 | |
| 930 | /* Update the CPU state to inject the exception. */ |
Raghu Krishnamurthy | 32626c9 | 2021-01-17 09:57:29 -0800 | [diff] [blame] | 931 | if (has_vhe_support()) { |
| 932 | write_msr(MSR_ESR_EL12, esr_el1_value); |
| 933 | write_msr(MSR_FAR_EL12, far_el1_value); |
| 934 | write_msr(MSR_ELR_EL12, vcpu->regs.pc); |
| 935 | write_msr(MSR_SPSR_EL12, vcpu->regs.spsr); |
| 936 | } else { |
| 937 | write_msr(esr_el1, esr_el1_value); |
| 938 | write_msr(far_el1, far_el1_value); |
| 939 | write_msr(elr_el1, vcpu->regs.pc); |
| 940 | write_msr(spsr_el1, vcpu->regs.spsr); |
| 941 | } |
Fuad Tabba | a48d122 | 2019-12-09 15:42:32 +0000 | [diff] [blame] | 942 | |
| 943 | /* |
| 944 | * Mask (disable) interrupts and run in EL1h mode. |
| 945 | * EL1h mode is used because by default, taking an exception selects the |
| 946 | * stack pointer for the target Exception level. The software can change |
| 947 | * that later in the handler if needed. |
Fuad Tabba | a48d122 | 2019-12-09 15:42:32 +0000 | [diff] [blame] | 948 | */ |
| 949 | vcpu->regs.spsr = PSR_D | PSR_A | PSR_I | PSR_F | PSR_PE_MODE_EL1H; |
| 950 | |
| 951 | /* Transfer control to the exception hander. */ |
| 952 | vcpu->regs.pc = handler_address; |
Fuad Tabba | b86325a | 2020-01-10 13:38:15 +0000 | [diff] [blame] | 953 | } |
| 954 | |
| 955 | /** |
| 956 | * Injects a Data Abort exception (same exception level). |
| 957 | */ |
| 958 | static void inject_el1_data_abort_exception(struct vcpu *vcpu, |
Fuad Tabba | c3847c7 | 2020-08-11 09:32:25 +0100 | [diff] [blame] | 959 | uintreg_t esr_el2, |
| 960 | uintreg_t far_el2) |
Fuad Tabba | b86325a | 2020-01-10 13:38:15 +0000 | [diff] [blame] | 961 | { |
| 962 | /* |
| 963 | * ISS encoding remains the same, but the EC is changed to reflect |
| 964 | * where the exception came from. |
| 965 | * See Arm Architecture Reference Manual Armv8-A, pages D13-2943/2982. |
| 966 | */ |
| 967 | uintreg_t esr_el1_value = GET_ESR_ISS(esr_el2) | GET_ESR_IL(esr_el2) | |
| 968 | (EC_DATA_ABORT_SAME_EL << ESR_EC_OFFSET); |
| 969 | |
Olivier Deprez | f92e5d4 | 2020-11-13 16:00:54 +0100 | [diff] [blame] | 970 | dlog_notice("Injecting Data Abort exception into VM %#x.\n", |
Andrew Walbran | 17eebf9 | 2020-02-05 16:35:49 +0000 | [diff] [blame] | 971 | vcpu->vm->id); |
Fuad Tabba | b86325a | 2020-01-10 13:38:15 +0000 | [diff] [blame] | 972 | |
Fuad Tabba | c3847c7 | 2020-08-11 09:32:25 +0100 | [diff] [blame] | 973 | inject_el1_exception(vcpu, esr_el1_value, far_el2); |
Fuad Tabba | b86325a | 2020-01-10 13:38:15 +0000 | [diff] [blame] | 974 | } |
| 975 | |
| 976 | /** |
| 977 | * Injects a Data Abort exception (same exception level). |
| 978 | */ |
| 979 | static void inject_el1_instruction_abort_exception(struct vcpu *vcpu, |
Fuad Tabba | c3847c7 | 2020-08-11 09:32:25 +0100 | [diff] [blame] | 980 | uintreg_t esr_el2, |
| 981 | uintreg_t far_el2) |
Fuad Tabba | b86325a | 2020-01-10 13:38:15 +0000 | [diff] [blame] | 982 | { |
| 983 | /* |
| 984 | * ISS encoding remains the same, but the EC is changed to reflect |
| 985 | * where the exception came from. |
| 986 | * See Arm Architecture Reference Manual Armv8-A, pages D13-2941/2980. |
| 987 | */ |
| 988 | uintreg_t esr_el1_value = |
| 989 | GET_ESR_ISS(esr_el2) | GET_ESR_IL(esr_el2) | |
| 990 | (EC_INSTRUCTION_ABORT_SAME_EL << ESR_EC_OFFSET); |
| 991 | |
Olivier Deprez | f92e5d4 | 2020-11-13 16:00:54 +0100 | [diff] [blame] | 992 | dlog_notice("Injecting Instruction Abort exception into VM %#x.\n", |
Andrew Walbran | 17eebf9 | 2020-02-05 16:35:49 +0000 | [diff] [blame] | 993 | vcpu->vm->id); |
Fuad Tabba | b86325a | 2020-01-10 13:38:15 +0000 | [diff] [blame] | 994 | |
Fuad Tabba | c3847c7 | 2020-08-11 09:32:25 +0100 | [diff] [blame] | 995 | inject_el1_exception(vcpu, esr_el1_value, far_el2); |
Fuad Tabba | b86325a | 2020-01-10 13:38:15 +0000 | [diff] [blame] | 996 | } |
| 997 | |
| 998 | /** |
| 999 | * Injects an exception with an unknown reason into the EL1. |
| 1000 | */ |
| 1001 | static void inject_el1_unknown_exception(struct vcpu *vcpu, uintreg_t esr_el2) |
| 1002 | { |
| 1003 | uintreg_t esr_el1_value = |
| 1004 | GET_ESR_IL(esr_el2) | (EC_UNKNOWN << ESR_EC_OFFSET); |
Fuad Tabba | c3847c7 | 2020-08-11 09:32:25 +0100 | [diff] [blame] | 1005 | |
Olivier Deprez | da14ddc | 2022-08-11 14:14:41 +0200 | [diff] [blame] | 1006 | dlog_notice("Injecting Unknown Reason exception into VM %#x.\n", |
| 1007 | vcpu->vm->id); |
| 1008 | |
Fuad Tabba | c3847c7 | 2020-08-11 09:32:25 +0100 | [diff] [blame] | 1009 | /* |
| 1010 | * The value of the far_el2 register is UNKNOWN in this case, |
| 1011 | * therefore, don't propagate it to avoid leaking sensitive information. |
| 1012 | */ |
Olivier Deprez | da14ddc | 2022-08-11 14:14:41 +0200 | [diff] [blame] | 1013 | inject_el1_exception(vcpu, esr_el1_value, 0); |
| 1014 | } |
Fuad Tabba | a48d122 | 2019-12-09 15:42:32 +0000 | [diff] [blame] | 1015 | |
Olivier Deprez | da14ddc | 2022-08-11 14:14:41 +0200 | [diff] [blame] | 1016 | /** |
| 1017 | * Injects an exception because of a system register trap. |
| 1018 | */ |
| 1019 | static void inject_el1_sysreg_trap_exception(struct vcpu *vcpu, |
| 1020 | uintreg_t esr_el2) |
| 1021 | { |
| 1022 | char *direction_str = ISS_IS_READ(esr_el2) ? "read" : "write"; |
| 1023 | |
Andrew Walbran | 17eebf9 | 2020-02-05 16:35:49 +0000 | [diff] [blame] | 1024 | dlog_notice( |
Karl Meakin | e8937d9 | 2024-03-19 16:04:25 +0000 | [diff] [blame] | 1025 | "Trapped access to system register %s: op0=%lu, op1=%lu, " |
| 1026 | "crn=%lu, " |
| 1027 | "crm=%lu, op2=%lu, rt=%lu.\n", |
Andrew Walbran | 17eebf9 | 2020-02-05 16:35:49 +0000 | [diff] [blame] | 1028 | direction_str, GET_ISS_OP0(esr_el2), GET_ISS_OP1(esr_el2), |
| 1029 | GET_ISS_CRN(esr_el2), GET_ISS_CRM(esr_el2), |
| 1030 | GET_ISS_OP2(esr_el2), GET_ISS_RT(esr_el2)); |
Fuad Tabba | a48d122 | 2019-12-09 15:42:32 +0000 | [diff] [blame] | 1031 | |
Olivier Deprez | da14ddc | 2022-08-11 14:14:41 +0200 | [diff] [blame] | 1032 | inject_el1_unknown_exception(vcpu, esr_el2); |
Fuad Tabba | a48d122 | 2019-12-09 15:42:32 +0000 | [diff] [blame] | 1033 | } |
| 1034 | |
Andrew Walbran | d8d3f5d | 2020-10-07 18:23:01 +0100 | [diff] [blame] | 1035 | static struct vcpu *hvc_handler(struct vcpu *vcpu) |
Wedson Almeida Filho | 987c0ff | 2018-06-20 16:34:38 +0100 | [diff] [blame] | 1036 | { |
Andrew Walbran | d8d3f5d | 2020-10-07 18:23:01 +0100 | [diff] [blame] | 1037 | struct ffa_value args = arch_regs_get_args(&vcpu->regs); |
Andrew Walbran | 59182d5 | 2019-09-23 17:55:39 +0100 | [diff] [blame] | 1038 | struct vcpu *next = NULL; |
Wedson Almeida Filho | 987c0ff | 2018-06-20 16:34:38 +0100 | [diff] [blame] | 1039 | |
Olivier Deprez | 79dbd6f | 2023-11-29 16:12:36 +0100 | [diff] [blame] | 1040 | /* Mask out SMCCC SVE hint bit from function id. */ |
| 1041 | args.func &= ~SMCCC_SVE_HINT_MASK; |
| 1042 | |
Andrew Walbran | d8d3f5d | 2020-10-07 18:23:01 +0100 | [diff] [blame] | 1043 | if (hvc_smc_handler(args, vcpu, &next)) { |
Andrew Walbran | 59182d5 | 2019-09-23 17:55:39 +0100 | [diff] [blame] | 1044 | return next; |
Andrew Walbran | 7d28d9a | 2019-08-30 16:24:58 +0100 | [diff] [blame] | 1045 | } |
Jose Marinho | fc0b2b6 | 2019-06-06 11:18:45 +0100 | [diff] [blame] | 1046 | |
Andrew Walbran | 7f920af | 2019-09-03 17:09:30 +0100 | [diff] [blame] | 1047 | switch (args.func) { |
J-Alves | 15e3026 | 2024-10-14 11:56:07 +0100 | [diff] [blame] | 1048 | #if SECURE_WORLD == 1 |
Madhukar Pappireddy | f675bb6 | 2021-08-03 12:57:10 -0500 | [diff] [blame] | 1049 | case HF_INTERRUPT_DEACTIVATE: |
Karl Meakin | 117c808 | 2024-12-04 16:03:28 +0000 | [diff] [blame] | 1050 | vcpu->regs.r[0] = |
| 1051 | ffa_interrupts_deactivate(args.arg1, args.arg2, vcpu); |
Madhukar Pappireddy | f675bb6 | 2021-08-03 12:57:10 -0500 | [diff] [blame] | 1052 | break; |
Madhukar Pappireddy | 72d2393 | 2023-07-24 15:57:28 -0500 | [diff] [blame] | 1053 | |
| 1054 | case HF_INTERRUPT_RECONFIGURE: |
Karl Meakin | 117c808 | 2024-12-04 16:03:28 +0000 | [diff] [blame] | 1055 | vcpu->regs.r[0] = ffa_interrupts_reconfigure( |
Madhukar Pappireddy | 72d2393 | 2023-07-24 15:57:28 -0500 | [diff] [blame] | 1056 | args.arg1, args.arg2, args.arg3, vcpu); |
| 1057 | break; |
Daniel Boulby | f3cf28c | 2024-08-22 10:46:23 +0100 | [diff] [blame] | 1058 | |
| 1059 | case HF_INTERRUPT_SEND_IPI: |
| 1060 | vcpu->regs.r[0] = api_hf_interrupt_send_ipi(args.arg1, vcpu); |
| 1061 | break; |
Madhukar Pappireddy | f675bb6 | 2021-08-03 12:57:10 -0500 | [diff] [blame] | 1062 | #endif |
Olivier Deprez | 109c6d4 | 2023-11-29 14:58:47 +0100 | [diff] [blame] | 1063 | case HF_INTERRUPT_ENABLE: |
| 1064 | vcpu->regs.r[0] = api_interrupt_enable(args.arg1, args.arg2, |
| 1065 | args.arg3, vcpu); |
| 1066 | break; |
| 1067 | |
Madhukar Pappireddy | c64d064 | 2024-08-07 16:55:46 -0500 | [diff] [blame] | 1068 | case HF_INTERRUPT_GET: { |
| 1069 | struct vcpu_locked current_locked; |
Madhukar Pappireddy | f675bb6 | 2021-08-03 12:57:10 -0500 | [diff] [blame] | 1070 | |
Madhukar Pappireddy | c64d064 | 2024-08-07 16:55:46 -0500 | [diff] [blame] | 1071 | current_locked = vcpu_lock(vcpu); |
Karl Meakin | 117c808 | 2024-12-04 16:03:28 +0000 | [diff] [blame] | 1072 | vcpu->regs.r[0] = ffa_interrupts_get(current_locked); |
Madhukar Pappireddy | c64d064 | 2024-08-07 16:55:46 -0500 | [diff] [blame] | 1073 | vcpu_unlock(¤t_locked); |
| 1074 | break; |
| 1075 | } |
Wedson Almeida Filho | 987c0ff | 2018-06-20 16:34:38 +0100 | [diff] [blame] | 1076 | default: |
Andrew Walbran | 59182d5 | 2019-09-23 17:55:39 +0100 | [diff] [blame] | 1077 | vcpu->regs.r[0] = SMCCC_ERROR_UNKNOWN; |
J-Alves | 3317240 | 2024-08-15 13:15:34 +0100 | [diff] [blame] | 1078 | dlog_verbose("Unsupported function %#lx\n", args.func); |
Wedson Almeida Filho | 987c0ff | 2018-06-20 16:34:38 +0100 | [diff] [blame] | 1079 | } |
| 1080 | |
J-Alves | 6f6bf8a | 2024-07-25 15:17:57 +0100 | [diff] [blame] | 1081 | /* |
| 1082 | * In case there has been an update after handling the last |
| 1083 | * hypervisor call, update the next vCPU directly in the register. |
| 1084 | */ |
Manish Pandey | 35e452f | 2021-02-18 21:36:34 +0000 | [diff] [blame] | 1085 | vcpu_update_virtual_interrupts(next); |
Andrew Walbran | 3d84a26 | 2018-12-13 14:41:19 +0000 | [diff] [blame] | 1086 | |
Andrew Walbran | 59182d5 | 2019-09-23 17:55:39 +0100 | [diff] [blame] | 1087 | return next; |
Wedson Almeida Filho | 987c0ff | 2018-06-20 16:34:38 +0100 | [diff] [blame] | 1088 | } |
| 1089 | |
Wedson Almeida Filho | 8700964 | 2018-07-02 10:20:07 +0100 | [diff] [blame] | 1090 | struct vcpu *irq_lower(void) |
| 1091 | { |
Madhukar Pappireddy | cbecc96 | 2021-08-03 13:11:57 -0500 | [diff] [blame] | 1092 | #if SECURE_WORLD == 1 |
| 1093 | struct vcpu *next = NULL; |
| 1094 | |
Karl Meakin | 117c808 | 2024-12-04 16:03:28 +0000 | [diff] [blame] | 1095 | ffa_interrupts_handle_secure_interrupt(current(), &next); |
Madhukar Pappireddy | cbecc96 | 2021-08-03 13:11:57 -0500 | [diff] [blame] | 1096 | |
| 1097 | /* |
| 1098 | * Since we are in interrupt context, set the bit for the |
| 1099 | * next vCPU directly in the register. |
| 1100 | */ |
| 1101 | vcpu_update_virtual_interrupts(next); |
| 1102 | |
| 1103 | return next; |
| 1104 | #else |
Andrew Scull | 9726c25 | 2019-01-23 13:44:19 +0000 | [diff] [blame] | 1105 | /* |
| 1106 | * Switch back to primary VM, interrupts will be handled there. |
| 1107 | * |
| 1108 | * If the VM has aborted, this vCPU will be aborted when the scheduler |
| 1109 | * tries to run it again. This means the interrupt will not be delayed |
| 1110 | * by the aborted VM. |
| 1111 | * |
| 1112 | * TODO: Only switch when the interrupt isn't for the current VM. |
| 1113 | */ |
Andrew Scull | 33fecd3 | 2019-01-08 14:48:27 +0000 | [diff] [blame] | 1114 | return api_preempt(current()); |
Madhukar Pappireddy | cbecc96 | 2021-08-03 13:11:57 -0500 | [diff] [blame] | 1115 | #endif |
Wedson Almeida Filho | 8700964 | 2018-07-02 10:20:07 +0100 | [diff] [blame] | 1116 | } |
| 1117 | |
Madhukar Pappireddy | 7fc585e | 2023-03-02 14:31:22 -0600 | [diff] [blame] | 1118 | #if SECURE_WORLD == 1 |
| 1119 | static void spmd_group0_intr_delegate(void) |
| 1120 | { |
| 1121 | struct ffa_value ret; |
| 1122 | |
| 1123 | dlog_verbose("Delegating Group0 interrupt to SPMD\n"); |
| 1124 | |
| 1125 | ret = smc_ffa_call((struct ffa_value){.func = FFA_EL3_INTR_HANDLE_32}); |
| 1126 | |
| 1127 | /* Check if the Group0 interrupt was handled successfully. */ |
| 1128 | CHECK(ret.func == FFA_SUCCESS_32); |
| 1129 | } |
| 1130 | #endif |
| 1131 | |
Wedson Almeida Filho | 9d5040f | 2018-10-29 08:41:27 +0000 | [diff] [blame] | 1132 | struct vcpu *fiq_lower(void) |
| 1133 | { |
Manish Pandey | a5f39fb | 2020-09-11 09:47:11 +0100 | [diff] [blame] | 1134 | #if SECURE_WORLD == 1 |
| 1135 | struct vcpu_locked current_locked; |
| 1136 | struct vcpu *current_vcpu = current(); |
Madhukar Pappireddy | 77d3bcd | 2023-03-01 17:26:22 -0600 | [diff] [blame] | 1137 | uint32_t intid; |
Manish Pandey | a5f39fb | 2020-09-11 09:47:11 +0100 | [diff] [blame] | 1138 | |
Madhukar Pappireddy | 77d3bcd | 2023-03-01 17:26:22 -0600 | [diff] [blame] | 1139 | intid = get_highest_pending_g0_interrupt_id(); |
| 1140 | |
| 1141 | /* Check for the highest priority pending Group0 interrupt. */ |
| 1142 | if (intid != SPURIOUS_INTID_OTHER_WORLD) { |
Madhukar Pappireddy | 7fc585e | 2023-03-02 14:31:22 -0600 | [diff] [blame] | 1143 | /* Delegate handling of Group0 interrupt to EL3 firmware. */ |
| 1144 | spmd_group0_intr_delegate(); |
| 1145 | |
| 1146 | /* Resume current vCPU. */ |
| 1147 | return NULL; |
Madhukar Pappireddy | 77d3bcd | 2023-03-01 17:26:22 -0600 | [diff] [blame] | 1148 | } |
| 1149 | |
| 1150 | /* |
| 1151 | * A special interrupt indicating there is no pending interrupt |
| 1152 | * with sufficient priority for current security state. This |
| 1153 | * means a non-secure interrupt is pending. |
| 1154 | */ |
Madhukar Pappireddy | c40f55f | 2022-06-22 11:00:41 -0500 | [diff] [blame] | 1155 | assert(current_vcpu->vm->ns_interrupts_action != NS_ACTION_QUEUED); |
| 1156 | |
Karl Meakin | 117c808 | 2024-12-04 16:03:28 +0000 | [diff] [blame] | 1157 | if (ffa_vm_managed_exit_supported(current_vcpu->vm)) { |
Madhukar Pappireddy | dd6fdfb | 2021-12-14 12:30:36 -0600 | [diff] [blame] | 1158 | uint8_t pmr = plat_interrupts_get_priority_mask(); |
| 1159 | |
Madhukar Pappireddy | 025a451 | 2024-10-14 22:09:19 -0500 | [diff] [blame] | 1160 | /* |
| 1161 | * Mask non-secure interrupt from triggering again till the |
| 1162 | * vCPU completes the managed exit sequenece. |
| 1163 | */ |
| 1164 | plat_interrupts_set_priority_mask(SWD_MASK_NS_INT); |
Manish Pandey | a5f39fb | 2020-09-11 09:47:11 +0100 | [diff] [blame] | 1165 | |
| 1166 | current_locked = vcpu_lock(current_vcpu); |
Madhukar Pappireddy | 025a451 | 2024-10-14 22:09:19 -0500 | [diff] [blame] | 1167 | current_vcpu->prev_interrupt_priority = pmr; |
Daniel Boulby | d49d077 | 2025-01-15 11:19:36 +0000 | [diff] [blame^] | 1168 | vcpu_interrupt_inject(current_locked, HF_MANAGED_EXIT_INTID); |
Manish Pandey | a5f39fb | 2020-09-11 09:47:11 +0100 | [diff] [blame] | 1169 | |
| 1170 | /* Entering managed exit sequence. */ |
| 1171 | current_vcpu->processing_managed_exit = true; |
| 1172 | |
| 1173 | vcpu_unlock(¤t_locked); |
| 1174 | |
| 1175 | /* |
| 1176 | * Since we are in interrupt context, set the bit for the |
| 1177 | * current vCPU directly in the register. |
| 1178 | */ |
| 1179 | vcpu_update_virtual_interrupts(NULL); |
| 1180 | |
| 1181 | /* Resume current vCPU. */ |
| 1182 | return NULL; |
| 1183 | } |
Manish Pandey | a5f39fb | 2020-09-11 09:47:11 +0100 | [diff] [blame] | 1184 | |
Madhukar Pappireddy | d46c06e | 2022-06-21 18:14:52 -0500 | [diff] [blame] | 1185 | /* |
| 1186 | * Unwind Normal World Scheduled Call chain in response to NS |
| 1187 | * Interrupt. |
| 1188 | */ |
Karl Meakin | 117c808 | 2024-12-04 16:03:28 +0000 | [diff] [blame] | 1189 | return ffa_interrupts_unwind_nwd_call_chain(current_vcpu); |
Madhukar Pappireddy | cbecc96 | 2021-08-03 13:11:57 -0500 | [diff] [blame] | 1190 | #else |
Wedson Almeida Filho | 9d5040f | 2018-10-29 08:41:27 +0000 | [diff] [blame] | 1191 | return irq_lower(); |
Madhukar Pappireddy | cbecc96 | 2021-08-03 13:11:57 -0500 | [diff] [blame] | 1192 | #endif |
Wedson Almeida Filho | 9d5040f | 2018-10-29 08:41:27 +0000 | [diff] [blame] | 1193 | } |
| 1194 | |
Fuad Tabba | d1d6798 | 2020-01-08 11:28:29 +0000 | [diff] [blame] | 1195 | noreturn struct vcpu *serr_lower(void) |
Wedson Almeida Filho | 9d5040f | 2018-10-29 08:41:27 +0000 | [diff] [blame] | 1196 | { |
Fuad Tabba | d1d6798 | 2020-01-08 11:28:29 +0000 | [diff] [blame] | 1197 | /* |
| 1198 | * SError exceptions should be isolated and handled by the responsible |
| 1199 | * VM/exception level. Getting here indicates a bug, that isolation is |
| 1200 | * not working, or a processor that does not support ARMv8.2-IESB, in |
| 1201 | * which case Hafnium routes SError exceptions to EL2 (here). |
| 1202 | */ |
| 1203 | panic("SError from a lower exception level."); |
Wedson Almeida Filho | 9d5040f | 2018-10-29 08:41:27 +0000 | [diff] [blame] | 1204 | } |
| 1205 | |
Wedson Almeida Filho | 99d2d4c | 2019-02-14 12:53:46 +0000 | [diff] [blame] | 1206 | /** |
| 1207 | * Initialises a fault info structure. It assumes that an FnV bit exists at |
| 1208 | * bit offset 10 of the ESR, and that it is only valid when the bottom 6 bits of |
| 1209 | * the ESR (the fault status code) are 010000; this is the case for both |
| 1210 | * instruction and data aborts, but not necessarily for other exception reasons. |
| 1211 | */ |
| 1212 | static struct vcpu_fault_info fault_info_init(uintreg_t esr, |
Andrew Walbran | 1281ed4 | 2019-10-22 17:23:40 +0100 | [diff] [blame] | 1213 | const struct vcpu *vcpu, |
| 1214 | uint32_t mode) |
Wedson Almeida Filho | 99d2d4c | 2019-02-14 12:53:46 +0000 | [diff] [blame] | 1215 | { |
| 1216 | uint32_t fsc = esr & 0x3f; |
| 1217 | struct vcpu_fault_info r; |
Olivier Deprez | 98ad2d2 | 2020-05-20 09:52:43 +0200 | [diff] [blame] | 1218 | uint64_t hpfar_el2_val; |
| 1219 | uint64_t hpfar_el2_fipa; |
Wedson Almeida Filho | 99d2d4c | 2019-02-14 12:53:46 +0000 | [diff] [blame] | 1220 | |
| 1221 | r.mode = mode; |
Wedson Almeida Filho | 99d2d4c | 2019-02-14 12:53:46 +0000 | [diff] [blame] | 1222 | r.pc = va_init(vcpu->regs.pc); |
| 1223 | |
Olivier Deprez | 98ad2d2 | 2020-05-20 09:52:43 +0200 | [diff] [blame] | 1224 | /* Get Hypervisor IPA Fault Address value. */ |
| 1225 | hpfar_el2_val = read_msr(hpfar_el2); |
| 1226 | |
| 1227 | /* Extract Faulting IPA. */ |
| 1228 | hpfar_el2_fipa = (hpfar_el2_val & HPFAR_EL2_FIPA) << 8; |
| 1229 | |
| 1230 | #if SECURE_WORLD == 1 |
| 1231 | |
| 1232 | /** |
| 1233 | * Determine if faulting IPA targets NS space. |
| 1234 | * At NS-EL2 hpfar_el2 bit 63 is RES0. At S-EL2, this bit determines if |
| 1235 | * the faulting Stage-1 address output is a secure or non-secure IPA. |
| 1236 | */ |
| 1237 | if ((hpfar_el2_val & HPFAR_EL2_NS) != 0) { |
| 1238 | r.mode |= MM_MODE_NS; |
| 1239 | } |
| 1240 | |
| 1241 | #endif |
| 1242 | |
Wedson Almeida Filho | 99d2d4c | 2019-02-14 12:53:46 +0000 | [diff] [blame] | 1243 | /* |
| 1244 | * Check the FnV bit, which is only valid if dfsc/ifsc is 010000. It |
| 1245 | * indicates that we cannot rely on far_el2. |
| 1246 | */ |
Karl Meakin | 5a13355 | 2024-05-30 16:06:27 +0100 | [diff] [blame] | 1247 | if (fsc == 0x10 && GET_ESR_FNV(esr)) { |
Wedson Almeida Filho | 99d2d4c | 2019-02-14 12:53:46 +0000 | [diff] [blame] | 1248 | r.vaddr = va_init(0); |
Olivier Deprez | 98ad2d2 | 2020-05-20 09:52:43 +0200 | [diff] [blame] | 1249 | r.ipaddr = ipa_init(hpfar_el2_fipa); |
Wedson Almeida Filho | 99d2d4c | 2019-02-14 12:53:46 +0000 | [diff] [blame] | 1250 | } else { |
| 1251 | r.vaddr = va_init(read_msr(far_el2)); |
Olivier Deprez | 98ad2d2 | 2020-05-20 09:52:43 +0200 | [diff] [blame] | 1252 | r.ipaddr = ipa_init(hpfar_el2_fipa | |
Wedson Almeida Filho | 99d2d4c | 2019-02-14 12:53:46 +0000 | [diff] [blame] | 1253 | (read_msr(far_el2) & (PAGE_SIZE - 1))); |
| 1254 | } |
| 1255 | |
| 1256 | return r; |
| 1257 | } |
| 1258 | |
Fuad Tabba | c3847c7 | 2020-08-11 09:32:25 +0100 | [diff] [blame] | 1259 | struct vcpu *sync_lower_exception(uintreg_t esr, uintreg_t far) |
Wedson Almeida Filho | 987c0ff | 2018-06-20 16:34:38 +0100 | [diff] [blame] | 1260 | { |
Wedson Almeida Filho | 00df6c7 | 2018-10-18 11:19:24 +0100 | [diff] [blame] | 1261 | struct vcpu *vcpu = current(); |
Wedson Almeida Filho | 99d2d4c | 2019-02-14 12:53:46 +0000 | [diff] [blame] | 1262 | struct vcpu_fault_info info; |
Raghu Krishnamurthy | b5775d2 | 2021-02-26 18:54:40 -0800 | [diff] [blame] | 1263 | struct vcpu *new_vcpu = NULL; |
Fuad Tabba | 3e9b022 | 2019-11-11 16:47:50 +0000 | [diff] [blame] | 1264 | uintreg_t ec = GET_ESR_EC(esr); |
Raghu Krishnamurthy | b5775d2 | 2021-02-26 18:54:40 -0800 | [diff] [blame] | 1265 | bool is_el0_partition = vcpu->vm->el0_partition; |
Raghu Krishnamurthy | f16b2ce | 2021-11-02 07:48:38 -0700 | [diff] [blame] | 1266 | bool resume = false; |
Wedson Almeida Filho | 987c0ff | 2018-06-20 16:34:38 +0100 | [diff] [blame] | 1267 | |
Fuad Tabba | c76466d | 2019-09-06 10:42:12 +0100 | [diff] [blame] | 1268 | switch (ec) { |
Fuad Tabba | b86325a | 2020-01-10 13:38:15 +0000 | [diff] [blame] | 1269 | case EC_WFI_WFE: |
Andrew Walbran | 48196eb | 2019-03-04 14:56:24 +0000 | [diff] [blame] | 1270 | /* Skip the instruction. */ |
Fuad Tabba | c76466d | 2019-09-06 10:42:12 +0100 | [diff] [blame] | 1271 | vcpu->regs.pc += GET_NEXT_PC_INC(esr); |
Raghu Krishnamurthy | b5775d2 | 2021-02-26 18:54:40 -0800 | [diff] [blame] | 1272 | |
| 1273 | /* |
| 1274 | * For EL0 partitions, treat both WFI and WFE the same way so |
| 1275 | * that FFA_RUN can be called on the partition to resume it. If |
| 1276 | * we treat WFI using api_wait_for_interrupt, the VCPU will be |
| 1277 | * in blocked waiting for interrupt but we cannot inject |
| 1278 | * interrupts into EL0 partitions. |
| 1279 | */ |
| 1280 | if (is_el0_partition) { |
Madhukar Pappireddy | 184501c | 2023-05-23 17:24:06 -0500 | [diff] [blame] | 1281 | api_yield(vcpu, &new_vcpu, NULL); |
Raghu Krishnamurthy | b5775d2 | 2021-02-26 18:54:40 -0800 | [diff] [blame] | 1282 | return new_vcpu; |
| 1283 | } |
| 1284 | |
Wedson Almeida Filho | 8700964 | 2018-07-02 10:20:07 +0100 | [diff] [blame] | 1285 | /* Check TI bit of ISS, 0 = WFI, 1 = WFE. */ |
Andrew Scull | 7364a8e | 2018-07-19 15:39:29 +0100 | [diff] [blame] | 1286 | if (esr & 1) { |
Andrew Walbran | 48196eb | 2019-03-04 14:56:24 +0000 | [diff] [blame] | 1287 | /* WFE */ |
| 1288 | /* |
| 1289 | * TODO: consider giving the scheduler more context, |
| 1290 | * somehow. |
| 1291 | */ |
Madhukar Pappireddy | 184501c | 2023-05-23 17:24:06 -0500 | [diff] [blame] | 1292 | api_yield(vcpu, &new_vcpu, NULL); |
Jose Marinho | 135dff3 | 2019-02-28 10:25:57 +0000 | [diff] [blame] | 1293 | return new_vcpu; |
Andrew Scull | 7364a8e | 2018-07-19 15:39:29 +0100 | [diff] [blame] | 1294 | } |
Andrew Walbran | 48196eb | 2019-03-04 14:56:24 +0000 | [diff] [blame] | 1295 | /* WFI */ |
Andrew Scull | 9726c25 | 2019-01-23 13:44:19 +0000 | [diff] [blame] | 1296 | return api_wait_for_interrupt(vcpu); |
Wedson Almeida Filho | 987c0ff | 2018-06-20 16:34:38 +0100 | [diff] [blame] | 1297 | |
Fuad Tabba | b86325a | 2020-01-10 13:38:15 +0000 | [diff] [blame] | 1298 | case EC_DATA_ABORT_LOWER_EL: |
Wedson Almeida Filho | 99d2d4c | 2019-02-14 12:53:46 +0000 | [diff] [blame] | 1299 | info = fault_info_init( |
Andrew Walbran | e52006c | 2019-10-22 18:01:28 +0100 | [diff] [blame] | 1300 | esr, vcpu, (esr & (1U << 6)) ? MM_MODE_W : MM_MODE_R); |
Raghu Krishnamurthy | b5775d2 | 2021-02-26 18:54:40 -0800 | [diff] [blame] | 1301 | |
Raghu Krishnamurthy | f16b2ce | 2021-11-02 07:48:38 -0700 | [diff] [blame] | 1302 | resume = vcpu_handle_page_fault(vcpu, &info); |
Raghu Krishnamurthy | b5775d2 | 2021-02-26 18:54:40 -0800 | [diff] [blame] | 1303 | if (is_el0_partition) { |
| 1304 | dlog_warning("Data abort on EL0 partition\n"); |
Raghu Krishnamurthy | f16b2ce | 2021-11-02 07:48:38 -0700 | [diff] [blame] | 1305 | /* |
| 1306 | * Abort EL0 context if we should not resume the |
| 1307 | * context, or it is an alignment fault. |
| 1308 | * vcpu_handle_page_fault() only checks the mode of the |
| 1309 | * page in an architecture agnostic way but alignment |
| 1310 | * faults on aarch64 can happen on a correctly mapped |
| 1311 | * page. |
| 1312 | */ |
| 1313 | if (!resume || ((esr & 0x3f) == 0x21)) { |
| 1314 | return api_abort(vcpu); |
| 1315 | } |
| 1316 | } |
| 1317 | |
| 1318 | if (resume) { |
| 1319 | return NULL; |
Raghu Krishnamurthy | b5775d2 | 2021-02-26 18:54:40 -0800 | [diff] [blame] | 1320 | } |
| 1321 | |
Fuad Tabba | b86325a | 2020-01-10 13:38:15 +0000 | [diff] [blame] | 1322 | /* Inform the EL1 of the data abort. */ |
Fuad Tabba | c3847c7 | 2020-08-11 09:32:25 +0100 | [diff] [blame] | 1323 | inject_el1_data_abort_exception(vcpu, esr, far); |
Wedson Almeida Filho | 987c0ff | 2018-06-20 16:34:38 +0100 | [diff] [blame] | 1324 | |
Fuad Tabba | b86325a | 2020-01-10 13:38:15 +0000 | [diff] [blame] | 1325 | /* Schedule the same VM to continue running. */ |
| 1326 | return NULL; |
| 1327 | |
| 1328 | case EC_INSTRUCTION_ABORT_LOWER_EL: |
Andrew Scull | d3cfaad | 2019-04-04 11:34:10 +0100 | [diff] [blame] | 1329 | info = fault_info_init(esr, vcpu, MM_MODE_X); |
Raghu Krishnamurthy | f16b2ce | 2021-11-02 07:48:38 -0700 | [diff] [blame] | 1330 | |
Wedson Almeida Filho | 99d2d4c | 2019-02-14 12:53:46 +0000 | [diff] [blame] | 1331 | if (vcpu_handle_page_fault(vcpu, &info)) { |
| 1332 | return NULL; |
| 1333 | } |
Raghu Krishnamurthy | b5775d2 | 2021-02-26 18:54:40 -0800 | [diff] [blame] | 1334 | |
| 1335 | if (is_el0_partition) { |
| 1336 | dlog_warning("Instruction abort on EL0 partition\n"); |
| 1337 | return api_abort(vcpu); |
| 1338 | } |
| 1339 | |
Fuad Tabba | b86325a | 2020-01-10 13:38:15 +0000 | [diff] [blame] | 1340 | /* Inform the EL1 of the instruction abort. */ |
Fuad Tabba | c3847c7 | 2020-08-11 09:32:25 +0100 | [diff] [blame] | 1341 | inject_el1_instruction_abort_exception(vcpu, esr, far); |
Wedson Almeida Filho | 2f94ec1 | 2018-07-26 16:00:48 +0100 | [diff] [blame] | 1342 | |
Fuad Tabba | b86325a | 2020-01-10 13:38:15 +0000 | [diff] [blame] | 1343 | /* Schedule the same VM to continue running. */ |
| 1344 | return NULL; |
Raghu Krishnamurthy | b5775d2 | 2021-02-26 18:54:40 -0800 | [diff] [blame] | 1345 | case EC_SVC: |
| 1346 | CHECK(is_el0_partition); |
| 1347 | return hvc_handler(vcpu); |
Fuad Tabba | b86325a | 2020-01-10 13:38:15 +0000 | [diff] [blame] | 1348 | case EC_HVC: |
Raghu Krishnamurthy | b5775d2 | 2021-02-26 18:54:40 -0800 | [diff] [blame] | 1349 | if (is_el0_partition) { |
| 1350 | dlog_warning("Unexpected HVC Trap on EL0 partition\n"); |
| 1351 | return api_abort(vcpu); |
| 1352 | } |
Andrew Walbran | 59182d5 | 2019-09-23 17:55:39 +0100 | [diff] [blame] | 1353 | return hvc_handler(vcpu); |
| 1354 | |
Fuad Tabba | b86325a | 2020-01-10 13:38:15 +0000 | [diff] [blame] | 1355 | case EC_SMC: { |
Andrew Scull | c960c03 | 2018-10-24 15:13:35 +0100 | [diff] [blame] | 1356 | uintreg_t smc_pc = vcpu->regs.pc; |
Andrew Walbran | 9dadaf2 | 2019-12-05 16:50:55 +0000 | [diff] [blame] | 1357 | struct vcpu *next = smc_handler(vcpu); |
Wedson Almeida Filho | 03e767a | 2018-07-30 15:32:03 +0100 | [diff] [blame] | 1358 | |
| 1359 | /* Skip the SMC instruction. */ |
Fuad Tabba | c76466d | 2019-09-06 10:42:12 +0100 | [diff] [blame] | 1360 | vcpu->regs.pc = smc_pc + GET_NEXT_PC_INC(esr); |
Andrew Walbran | 9dadaf2 | 2019-12-05 16:50:55 +0000 | [diff] [blame] | 1361 | |
Andrew Walbran | 3364565 | 2019-04-15 12:29:31 +0100 | [diff] [blame] | 1362 | return next; |
Andrew Scull | c960c03 | 2018-10-24 15:13:35 +0100 | [diff] [blame] | 1363 | } |
Wedson Almeida Filho | 03e767a | 2018-07-30 15:32:03 +0100 | [diff] [blame] | 1364 | |
Fuad Tabba | b86325a | 2020-01-10 13:38:15 +0000 | [diff] [blame] | 1365 | case EC_MSR: |
Fuad Tabba | c76466d | 2019-09-06 10:42:12 +0100 | [diff] [blame] | 1366 | /* |
| 1367 | * NOTE: This should never be reached because it goes through a |
| 1368 | * separate path handled by handle_system_register_access(). |
| 1369 | */ |
| 1370 | panic("Handled by handle_system_register_access()."); |
| 1371 | |
Wedson Almeida Filho | 987c0ff | 2018-06-20 16:34:38 +0100 | [diff] [blame] | 1372 | default: |
Andrew Walbran | 17eebf9 | 2020-02-05 16:35:49 +0000 | [diff] [blame] | 1373 | dlog_notice( |
Karl Meakin | e8937d9 | 2024-03-19 16:04:25 +0000 | [diff] [blame] | 1374 | "Unknown lower sync exception pc=%#lx, esr=%#lx, " |
| 1375 | "ec=%#lx\n", |
Andrew Walbran | 17eebf9 | 2020-02-05 16:35:49 +0000 | [diff] [blame] | 1376 | vcpu->regs.pc, esr, ec); |
Andrew Scull | 9726c25 | 2019-01-23 13:44:19 +0000 | [diff] [blame] | 1377 | break; |
Wedson Almeida Filho | 987c0ff | 2018-06-20 16:34:38 +0100 | [diff] [blame] | 1378 | } |
| 1379 | |
Raghu Krishnamurthy | b5775d2 | 2021-02-26 18:54:40 -0800 | [diff] [blame] | 1380 | if (is_el0_partition) { |
| 1381 | return api_abort(vcpu); |
| 1382 | } |
| 1383 | |
Fuad Tabba | 3e9b022 | 2019-11-11 16:47:50 +0000 | [diff] [blame] | 1384 | /* |
Fuad Tabba | a48d122 | 2019-12-09 15:42:32 +0000 | [diff] [blame] | 1385 | * The exception wasn't handled. Inject to the VM to give it chance to |
| 1386 | * handle as an unknown exception. |
Fuad Tabba | 3e9b022 | 2019-11-11 16:47:50 +0000 | [diff] [blame] | 1387 | */ |
Fuad Tabba | b86325a | 2020-01-10 13:38:15 +0000 | [diff] [blame] | 1388 | inject_el1_unknown_exception(vcpu, esr); |
| 1389 | |
| 1390 | /* Schedule the same VM to continue running. */ |
| 1391 | return NULL; |
Fuad Tabba | 3e9b022 | 2019-11-11 16:47:50 +0000 | [diff] [blame] | 1392 | } |
| 1393 | |
Fuad Tabba | c76466d | 2019-09-06 10:42:12 +0100 | [diff] [blame] | 1394 | /** |
Fuad Tabba | b0ef2a4 | 2019-12-19 11:19:25 +0000 | [diff] [blame] | 1395 | * Handles EC = 011000, MSR, MRS instruction traps. |
Fuad Tabba | ed294af | 2019-12-20 10:43:01 +0000 | [diff] [blame] | 1396 | * Returns non-null ONLY if the access failed and the vCPU is changing. |
Fuad Tabba | c76466d | 2019-09-06 10:42:12 +0100 | [diff] [blame] | 1397 | */ |
Fuad Tabba | b86325a | 2020-01-10 13:38:15 +0000 | [diff] [blame] | 1398 | void handle_system_register_access(uintreg_t esr_el2) |
Fuad Tabba | c76466d | 2019-09-06 10:42:12 +0100 | [diff] [blame] | 1399 | { |
| 1400 | struct vcpu *vcpu = current(); |
J-Alves | 19e20cf | 2023-08-02 12:48:55 +0100 | [diff] [blame] | 1401 | ffa_id_t vm_id = vcpu->vm->id; |
Fuad Tabba | 3e9b022 | 2019-11-11 16:47:50 +0000 | [diff] [blame] | 1402 | uintreg_t ec = GET_ESR_EC(esr_el2); |
Fuad Tabba | c76466d | 2019-09-06 10:42:12 +0100 | [diff] [blame] | 1403 | |
Fuad Tabba | b86325a | 2020-01-10 13:38:15 +0000 | [diff] [blame] | 1404 | CHECK(ec == EC_MSR); |
Fuad Tabba | c76466d | 2019-09-06 10:42:12 +0100 | [diff] [blame] | 1405 | /* |
Fuad Tabba | f1d6dc5 | 2019-09-18 17:33:14 +0100 | [diff] [blame] | 1406 | * Handle accesses to debug and performance monitor registers. |
Fuad Tabba | 3e9b022 | 2019-11-11 16:47:50 +0000 | [diff] [blame] | 1407 | * Inject an exception for unhandled/unsupported registers. |
Fuad Tabba | c76466d | 2019-09-06 10:42:12 +0100 | [diff] [blame] | 1408 | */ |
Fuad Tabba | 3e9b022 | 2019-11-11 16:47:50 +0000 | [diff] [blame] | 1409 | if (debug_el1_is_register_access(esr_el2)) { |
| 1410 | if (!debug_el1_process_access(vcpu, vm_id, esr_el2)) { |
Olivier Deprez | da14ddc | 2022-08-11 14:14:41 +0200 | [diff] [blame] | 1411 | inject_el1_sysreg_trap_exception(vcpu, esr_el2); |
Fuad Tabba | b86325a | 2020-01-10 13:38:15 +0000 | [diff] [blame] | 1412 | return; |
Fuad Tabba | f1d6dc5 | 2019-09-18 17:33:14 +0100 | [diff] [blame] | 1413 | } |
Fuad Tabba | 3e9b022 | 2019-11-11 16:47:50 +0000 | [diff] [blame] | 1414 | } else if (perfmon_is_register_access(esr_el2)) { |
| 1415 | if (!perfmon_process_access(vcpu, vm_id, esr_el2)) { |
Olivier Deprez | da14ddc | 2022-08-11 14:14:41 +0200 | [diff] [blame] | 1416 | inject_el1_sysreg_trap_exception(vcpu, esr_el2); |
Fuad Tabba | b86325a | 2020-01-10 13:38:15 +0000 | [diff] [blame] | 1417 | return; |
Fuad Tabba | f1d6dc5 | 2019-09-18 17:33:14 +0100 | [diff] [blame] | 1418 | } |
Fuad Tabba | 77a4b01 | 2019-11-15 12:13:08 +0000 | [diff] [blame] | 1419 | } else if (feature_id_is_register_access(esr_el2)) { |
| 1420 | if (!feature_id_process_access(vcpu, esr_el2)) { |
Olivier Deprez | da14ddc | 2022-08-11 14:14:41 +0200 | [diff] [blame] | 1421 | inject_el1_sysreg_trap_exception(vcpu, esr_el2); |
Fuad Tabba | b86325a | 2020-01-10 13:38:15 +0000 | [diff] [blame] | 1422 | return; |
Fuad Tabba | 77a4b01 | 2019-11-15 12:13:08 +0000 | [diff] [blame] | 1423 | } |
Madhukar Pappireddy | f684d19 | 2024-09-25 14:35:57 -0500 | [diff] [blame] | 1424 | } else if (el1_physical_timer_is_register_access(esr_el2)) { |
| 1425 | if (!el1_physical_timer_process_access(vcpu, esr_el2)) { |
| 1426 | inject_el1_sysreg_trap_exception(vcpu, esr_el2); |
| 1427 | return; |
| 1428 | } |
Fuad Tabba | f1d6dc5 | 2019-09-18 17:33:14 +0100 | [diff] [blame] | 1429 | } else { |
Olivier Deprez | da14ddc | 2022-08-11 14:14:41 +0200 | [diff] [blame] | 1430 | inject_el1_sysreg_trap_exception(vcpu, esr_el2); |
Fuad Tabba | b86325a | 2020-01-10 13:38:15 +0000 | [diff] [blame] | 1431 | return; |
Fuad Tabba | c76466d | 2019-09-06 10:42:12 +0100 | [diff] [blame] | 1432 | } |
| 1433 | |
Fuad Tabba | f1d6dc5 | 2019-09-18 17:33:14 +0100 | [diff] [blame] | 1434 | /* Instruction was fulfilled. Skip it and run the next one. */ |
Fuad Tabba | 3e9b022 | 2019-11-11 16:47:50 +0000 | [diff] [blame] | 1435 | vcpu->regs.pc += GET_NEXT_PC_INC(esr_el2); |
Fuad Tabba | c76466d | 2019-09-06 10:42:12 +0100 | [diff] [blame] | 1436 | } |