blob: 231b33825feb48def82e425af83a1d9a621ee9fc [file] [log] [blame]
Karl Meakin8e58ddc2024-11-08 23:19:34 +00001/*
2 * Copyright 2024 The Hafnium Authors.
3 *
4 * Use of this source code is governed by a BSD-style
5 * license that can be found in the LICENSE file or at
6 * https://opensource.org/licenses/BSD-3-Clause.
7 */
8
9#include "hf/plat/interrupts.h"
10
11#include "hf/arch/gicv3.h"
12#include "hf/arch/host_timer.h"
Karl Meakin8e58ddc2024-11-08 23:19:34 +000013
14#include "hf/api.h"
15#include "hf/check.h"
Karl Meakinfa1dcb82025-02-10 16:47:50 +000016#include "hf/ffa/direct_messaging.h"
Karl Meakin902af082024-11-28 14:58:38 +000017#include "hf/ffa/vm.h"
Karl Meakin8e58ddc2024-11-08 23:19:34 +000018#include "hf/hf_ipi.h"
19#include "hf/vm.h"
20
21/**
22 * Drops the current interrupt priority and deactivate the given interrupt ID
23 * for the calling vCPU.
24 *
25 * Returns 0 on success, or -1 otherwise.
26 */
Karl Meakin117c8082024-12-04 16:03:28 +000027int64_t ffa_interrupts_deactivate(uint32_t pint_id, uint32_t vint_id,
28 struct vcpu *current)
Karl Meakin8e58ddc2024-11-08 23:19:34 +000029{
Daniel Boulby3c1506b2025-02-25 10:49:51 +000030 (void)pint_id;
31 (void)vint_id;
Karl Meakin8e58ddc2024-11-08 23:19:34 +000032 int ret = 0;
Daniel Boulby3c1506b2025-02-25 10:49:51 +000033 struct vcpu_locked current_locked;
Karl Meakin8e58ddc2024-11-08 23:19:34 +000034
35 current_locked = vcpu_lock(current);
Karl Meakin8e58ddc2024-11-08 23:19:34 +000036
37 if (current->requires_deactivate_call) {
38 /* There is no preempted vCPU to resume. */
39 assert(current->preempted_vcpu == NULL);
40
41 vcpu_secure_interrupt_complete(current_locked);
42 }
43
Karl Meakin8e58ddc2024-11-08 23:19:34 +000044 vcpu_unlock(&current_locked);
45 return ret;
46}
47
Karl Meakinca38ef92025-02-13 14:20:23 +000048static struct vcpu *ffa_interrupts_find_target_vcpu_secure_interrupt(
Karl Meakin8e58ddc2024-11-08 23:19:34 +000049 struct vcpu *current, uint32_t interrupt_id)
50{
51 /*
52 * Find which VM/SP owns this interrupt. We then find the
53 * corresponding vCPU context for this CPU.
54 */
55 for (ffa_vm_count_t index = 0; index < vm_get_count(); ++index) {
56 struct vm *vm = vm_find_index(index);
57
58 for (uint32_t j = 0; j < HF_NUM_INTIDS; j++) {
59 struct interrupt_descriptor int_desc =
60 vm->interrupt_desc[j];
61
62 /*
63 * Interrupt descriptors are populated
64 * contiguously.
65 */
66 if (!int_desc.valid) {
67 break;
68 }
69 if (int_desc.interrupt_id == interrupt_id) {
70 return api_ffa_get_vm_vcpu(vm, current);
71 }
72 }
73 }
74
75 return NULL;
76}
77
Karl Meakinca38ef92025-02-13 14:20:23 +000078static struct vcpu *ffa_interrupts_find_target_vcpu(struct vcpu *current,
J-Alvesde211782025-02-07 14:44:39 +000079 uint32_t interrupt_id,
80 uint32_t *v_intid)
Karl Meakin8e58ddc2024-11-08 23:19:34 +000081{
82 struct vcpu *target_vcpu;
83
J-Alvesde211782025-02-07 14:44:39 +000084 assert(current != NULL);
85 assert(v_intid != NULL);
86
87 *v_intid = interrupt_id;
88
Karl Meakin8e58ddc2024-11-08 23:19:34 +000089 switch (interrupt_id) {
J-Alvesde211782025-02-07 14:44:39 +000090 case SPURIOUS_INTID_OTHER_WORLD:
91 /*
92 * Spurious interrupt ID indicating that there are no pending
93 * interrupts to acknowledge. For such scenarios, resume the
94 * current vCPU.
95 */
96 target_vcpu = NULL;
97 break;
Karl Meakin8e58ddc2024-11-08 23:19:34 +000098 case HF_IPI_INTID:
Daniel Boulby7011b5a2024-10-15 18:27:26 +010099 /*
100 * Get the next vCPU with a pending IPI. If all vCPUs
101 * have had their IPIs handled this will return NULL.
102 */
103 target_vcpu = hf_ipi_get_pending_target_vcpu(current);
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000104 break;
J-Alvesde211782025-02-07 14:44:39 +0000105 case ARM_SEL2_TIMER_PHYS_INT:
106 /* Disable the S-EL2 physical timer */
107 host_timer_disable();
108 target_vcpu = timer_find_target_vcpu(current);
109
110 if (target_vcpu != NULL) {
111 *v_intid = HF_VIRTUAL_TIMER_INTID;
112 }
113 /*
114 * It is possible for target_vcpu to be NULL in case of spurious
115 * timer interrupt.
116 */
117 break;
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000118 case ARM_EL1_VIRT_TIMER_PHYS_INT:
119 /* Fall through */
120 case ARM_EL1_PHYS_TIMER_PHYS_INT:
121 panic("Timer interrupt not expected to fire: %u\n",
122 interrupt_id);
123 default:
Karl Meakinca38ef92025-02-13 14:20:23 +0000124 target_vcpu = ffa_interrupts_find_target_vcpu_secure_interrupt(
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000125 current, interrupt_id);
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000126
Daniel Boulby7011b5a2024-10-15 18:27:26 +0100127 /* The target vCPU for a secure interrupt cannot be NULL. */
128 CHECK(target_vcpu != NULL);
129 }
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000130
131 return target_vcpu;
132}
133
134/*
Daniel Boulby3c1506b2025-02-25 10:49:51 +0000135 * If the current vCPU is being preempted, record this in the target vCPU
136 * and set the current states to VCPU_STATE_PREEMPTED.
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000137 */
Daniel Boulby3c1506b2025-02-25 10:49:51 +0000138static void ffa_interrupts_set_preempted_vcpu(
139 struct vcpu_locked target_vcpu_locked,
140 struct vcpu_locked current_locked)
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000141{
142 struct vcpu *target_vcpu = target_vcpu_locked.vcpu;
143 struct vcpu *preempted_vcpu = current_locked.vcpu;
144
145 if (preempted_vcpu != NULL) {
146 target_vcpu->preempted_vcpu = preempted_vcpu;
147 preempted_vcpu->state = VCPU_STATE_PREEMPTED;
148 }
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000149}
150
151/**
Karl Meakinfa1dcb82025-02-10 16:47:50 +0000152 * If the interrupts were indeed masked by SPMC before an SP's vCPU was resumed,
153 * restore the priority mask thereby allowing the interrupts to be delivered.
154 */
155void ffa_interrupts_unmask(struct vcpu *current)
156{
157 plat_interrupts_set_priority_mask(current->prev_interrupt_priority);
158}
159
160/**
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000161 * Enforce action of an SP in response to non-secure or other-secure interrupt
162 * by changing the priority mask. Effectively, physical interrupts shall not
163 * trigger which has the same effect as queueing interrupts.
164 */
Karl Meakinfa1dcb82025-02-10 16:47:50 +0000165void ffa_interrupts_mask(struct vcpu_locked receiver_vcpu_locked)
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000166{
167 struct vcpu *receiver_vcpu = receiver_vcpu_locked.vcpu;
168 uint8_t current_priority;
169
170 /* Save current value of priority mask. */
171 current_priority = plat_interrupts_get_priority_mask();
172 receiver_vcpu->prev_interrupt_priority = current_priority;
173
174 if (receiver_vcpu->vm->other_s_interrupts_action ==
175 OTHER_S_INT_ACTION_QUEUED ||
176 receiver_vcpu->scheduling_mode == SPMC_MODE) {
177 /*
178 * If secure interrupts not masked yet, mask them now. We could
179 * enter SPMC scheduled mode when an EL3 SPMD Logical partition
180 * sends a direct request, and we are making the IMPDEF choice
181 * to mask interrupts when such a situation occurs. This keeps
182 * design simple.
183 */
184 if (current_priority > SWD_MASK_ALL_INT) {
185 plat_interrupts_set_priority_mask(SWD_MASK_ALL_INT);
186 }
187 } else if (receiver_vcpu->vm->ns_interrupts_action ==
188 NS_ACTION_QUEUED) {
189 /* If non secure interrupts not masked yet, mask them now. */
190 if (current_priority > SWD_MASK_NS_INT) {
191 plat_interrupts_set_priority_mask(SWD_MASK_NS_INT);
192 }
193 }
194}
195
196/**
197 * Handles the secure interrupt according to the target vCPU's state
198 * in the case the owner of the interrupt is an S-EL0 partition.
199 */
Karl Meakinca38ef92025-02-13 14:20:23 +0000200static struct vcpu *ffa_interrupts_signal_secure_interrupt_sel0(
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000201 struct vcpu_locked current_locked,
202 struct vcpu_locked target_vcpu_locked, uint32_t v_intid)
203{
204 struct vcpu *target_vcpu = target_vcpu_locked.vcpu;
205 struct vcpu *next;
206
207 /* Secure interrupt signaling and queuing for S-EL0 SP. */
208 switch (target_vcpu->state) {
Madhukar Pappireddyfe60d092025-01-24 06:42:54 -0600209 case VCPU_STATE_WAITING: {
210 struct ffa_value ret_interrupt =
211 api_ffa_interrupt_return(v_intid);
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000212
Madhukar Pappireddyfe60d092025-01-24 06:42:54 -0600213 /* FF-A v1.1 EAC0 Table 8.1 case 1 and Table 12.10. */
214 dlog_verbose("S-EL0: Secure interrupt signaled: %x\n",
215 target_vcpu->vm->id);
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000216
Madhukar Pappireddyfe60d092025-01-24 06:42:54 -0600217 vcpu_enter_secure_interrupt_rtm(target_vcpu_locked);
Karl Meakinfa1dcb82025-02-10 16:47:50 +0000218 ffa_interrupts_mask(target_vcpu_locked);
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000219
Madhukar Pappireddyfe60d092025-01-24 06:42:54 -0600220 vcpu_set_running(target_vcpu_locked, &ret_interrupt);
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000221
Madhukar Pappireddyfe60d092025-01-24 06:42:54 -0600222 /*
223 * If the execution was in NWd as well, set the vCPU
224 * in preempted state as well.
225 */
Daniel Boulby3c1506b2025-02-25 10:49:51 +0000226 ffa_interrupts_set_preempted_vcpu(target_vcpu_locked,
227 current_locked);
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000228
Madhukar Pappireddyfe60d092025-01-24 06:42:54 -0600229 /*
230 * The target vcpu could have migrated to a different physical
231 * CPU. SPMC will migrate it to current physical CPU and resume
232 * it.
233 */
234 target_vcpu->cpu = current_locked.vcpu->cpu;
235
236 /* Switch to target vCPU responsible for this interrupt. */
237 next = target_vcpu;
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000238 break;
Madhukar Pappireddyfe60d092025-01-24 06:42:54 -0600239 }
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000240 case VCPU_STATE_BLOCKED:
241 case VCPU_STATE_PREEMPTED:
242 case VCPU_STATE_RUNNING:
243 dlog_verbose("S-EL0: Secure interrupt queued: %x\n",
244 target_vcpu->vm->id);
245 /*
246 * The target vCPU cannot be resumed, SPMC resumes current
247 * vCPU.
248 */
249 next = NULL;
Daniel Boulby3c1506b2025-02-25 10:49:51 +0000250 ffa_interrupts_set_preempted_vcpu(
251 target_vcpu_locked, (struct vcpu_locked){.vcpu = NULL});
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000252 break;
253 default:
254 panic("Secure interrupt cannot be signaled to target SP\n");
255 break;
256 }
257
258 return next;
259}
260
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000261/**
262 * Handles the secure interrupt according to the target vCPU's state
263 * in the case the owner of the interrupt is an S-EL1 partition.
264 */
Karl Meakinca38ef92025-02-13 14:20:23 +0000265static struct vcpu *ffa_interrupts_signal_secure_interrupt_sel1(
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000266 struct vcpu_locked current_locked,
267 struct vcpu_locked target_vcpu_locked, uint32_t v_intid)
268{
269 struct vcpu *target_vcpu = target_vcpu_locked.vcpu;
270 struct vcpu *current = current_locked.vcpu;
271 struct vcpu *next = NULL;
272
J-Alves7e7fce02025-02-07 15:14:56 +0000273 /*
274 * The target vcpu has migrated to a different physical
275 * CPU. Hence, it cannot be resumed on this CPU, SPMC
276 * resumes current vCPU.
277 */
278 if (target_vcpu->cpu != current_locked.vcpu->cpu) {
279 assert(target_vcpu->vm->vcpu_count == 1);
280 }
281
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000282 /* Secure interrupt signaling and queuing for S-EL1 SP. */
283 switch (target_vcpu->state) {
Madhukar Pappireddyfe60d092025-01-24 06:42:54 -0600284 case VCPU_STATE_WAITING: {
285 struct ffa_value ret_interrupt =
286 api_ffa_interrupt_return(v_intid);
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000287
Madhukar Pappireddyfe60d092025-01-24 06:42:54 -0600288 /* FF-A v1.1 EAC0 Table 8.2 case 1 and Table 12.10. */
289 vcpu_enter_secure_interrupt_rtm(target_vcpu_locked);
Karl Meakinfa1dcb82025-02-10 16:47:50 +0000290 ffa_interrupts_mask(target_vcpu_locked);
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000291
Madhukar Pappireddyfe60d092025-01-24 06:42:54 -0600292 /*
293 * Ideally, we have to mask non-secure interrupts here
294 * since the spec mandates that SPMC should make sure
295 * SPMC scheduled call chain cannot be preempted by a
296 * non-secure interrupt. However, our current design
297 * takes care of it implicitly.
298 */
299 vcpu_set_running(target_vcpu_locked, &ret_interrupt);
300
Daniel Boulby3c1506b2025-02-25 10:49:51 +0000301 ffa_interrupts_set_preempted_vcpu(target_vcpu_locked,
302 current_locked);
Madhukar Pappireddyfe60d092025-01-24 06:42:54 -0600303 next = target_vcpu;
304
305 if (target_vcpu->cpu != current_locked.vcpu->cpu) {
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000306 /*
Madhukar Pappireddyfe60d092025-01-24 06:42:54 -0600307 * The target vcpu could have migrated to a different
308 * physical CPU. SPMC will migrate it to current
309 * physical CPU and resume it.
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000310 */
Madhukar Pappireddyfe60d092025-01-24 06:42:54 -0600311 target_vcpu->cpu = current_locked.vcpu->cpu;
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000312 }
313 break;
Madhukar Pappireddyfe60d092025-01-24 06:42:54 -0600314 }
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000315 case VCPU_STATE_BLOCKED:
J-Alves7e7fce02025-02-07 15:14:56 +0000316
317 if (target_vcpu->cpu == current_locked.vcpu->cpu &&
318 ffa_direct_msg_precedes_in_call_chain(current_locked,
319 target_vcpu_locked)) {
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000320 struct ffa_value ret_interrupt =
321 api_ffa_interrupt_return(0);
322
323 /*
324 * If the target vCPU ran earlier in the same call
325 * chain as the current vCPU, SPMC leaves all
326 * intermediate execution contexts in blocked state and
327 * resumes the target vCPU for handling secure
328 * interrupt.
329 * Under the current design, there is only one possible
330 * scenario in which this could happen: both the
331 * preempted (i.e. current) and target vCPU are in the
332 * same NWd scheduled call chain and is described in the
333 * Scenario 1 of Table 8.4 in EAC0 spec.
334 */
335 assert(current_locked.vcpu->scheduling_mode ==
336 NWD_MODE);
337 assert(target_vcpu->scheduling_mode == NWD_MODE);
338
339 /*
340 * The execution preempted the call chain that involved
341 * the targeted and the current SPs.
342 * The targetted SP is set running, whilst the
343 * preempted SP is set PREEMPTED.
344 */
345 vcpu_set_running(target_vcpu_locked, &ret_interrupt);
346
Daniel Boulby3c1506b2025-02-25 10:49:51 +0000347 ffa_interrupts_set_preempted_vcpu(target_vcpu_locked,
348 current_locked);
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000349 next = target_vcpu;
350 } else {
351 /*
J-Alves7e7fce02025-02-07 15:14:56 +0000352 * Either:
353 * - The target vCPU has migrated to a different
354 * physical CPU. Hence, it cannot be resumed on this
355 * CPU, SPMC resumes current vCPU.
356 * - The target vCPU cannot be resumed now because it is
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000357 * in BLOCKED state (it yielded CPU cycles using
358 * FFA_YIELD). SPMC queues the virtual interrupt and
359 * resumes the current vCPU which could belong to either
360 * a VM or a SP.
361 */
362 next = NULL;
Daniel Boulby3c1506b2025-02-25 10:49:51 +0000363 ffa_interrupts_set_preempted_vcpu(
364 target_vcpu_locked,
Karl Meakinca38ef92025-02-13 14:20:23 +0000365 (struct vcpu_locked){.vcpu = NULL});
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000366 }
367 break;
368 case VCPU_STATE_PREEMPTED:
J-Alves7e7fce02025-02-07 15:14:56 +0000369 /*
370 * We do not resume a target vCPU that has been already
371 * pre-empted by an interrupt. Make the vIRQ pending for
372 * target SP(i.e., queue the interrupt) and continue to
373 * resume current vCPU. Refer to section 8.3.2.1 bullet
374 * 3 in the FF-A v1.1 EAC0 spec.
375 */
376 if (target_vcpu->cpu == current_locked.vcpu->cpu &&
377 current->vm->id == HF_OTHER_WORLD_ID) {
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000378 /*
J-Alves7e7fce02025-02-07 15:14:56 +0000379 * The target vCPU must have been preempted by a
380 * non secure interrupt. It could not have been
381 * preempted by a secure interrupt as current
382 * SPMC implementation does not allow secure
383 * interrupt prioritization. Moreover, the
384 * target vCPU should have been in Normal World
385 * scheduled mode as SPMC scheduled mode call
386 * chain cannot be preempted by a non secure
387 * interrupt.
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000388 */
J-Alves7e7fce02025-02-07 15:14:56 +0000389 CHECK(target_vcpu->scheduling_mode == NWD_MODE);
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000390 }
391
392 next = NULL;
Daniel Boulby3c1506b2025-02-25 10:49:51 +0000393 ffa_interrupts_set_preempted_vcpu(
394 target_vcpu_locked, (struct vcpu_locked){.vcpu = NULL});
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000395
396 break;
397 case VCPU_STATE_RUNNING:
398 if (current == target_vcpu) {
399 /*
400 * This is the special scenario where the current
401 * running execution context also happens to be the
402 * target of the secure interrupt. In this case, it
403 * needs to signal completion of secure interrupt
404 * implicitly. Refer to the embedded comment in vcpu.h
405 * file for the description of this variable.
406 */
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000407 current->requires_deactivate_call = true;
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000408 }
J-Alves7e7fce02025-02-07 15:14:56 +0000409
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000410 next = NULL;
Daniel Boulby3c1506b2025-02-25 10:49:51 +0000411 ffa_interrupts_set_preempted_vcpu(
412 target_vcpu_locked, (struct vcpu_locked){.vcpu = NULL});
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000413 break;
414 case VCPU_STATE_BLOCKED_INTERRUPT:
415 /* WFI is no-op for SP. Fall through. */
416 default:
417 /*
418 * vCPU of Target SP cannot be in OFF/ABORTED state if it has
419 * to handle secure interrupt.
420 */
421 panic("Secure interrupt cannot be signaled to target SP\n");
422 break;
423 }
424
425 return next;
426}
427
428/**
429 * Obtain the physical interrupt that triggered from the interrupt controller,
430 * and inject the corresponding virtual interrupt to the target vCPU.
431 * When PEs executing in the Normal World, and secure interrupts trigger,
432 * execution is trapped into EL3. SPMD then routes the interrupt to SPMC
433 * through FFA_INTERRUPT_32 ABI synchronously using eret conduit.
434 */
Karl Meakin117c8082024-12-04 16:03:28 +0000435void ffa_interrupts_handle_secure_interrupt(struct vcpu *current,
436 struct vcpu **next)
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000437{
438 struct vcpu *target_vcpu;
439 struct vcpu_locked target_vcpu_locked =
440 (struct vcpu_locked){.vcpu = NULL};
441 struct vcpu_locked current_locked;
442 uint32_t intid;
443 struct vm_locked target_vm_locked;
444 uint32_t v_intid;
445
446 /* Find pending interrupt id. This also activates the interrupt. */
447 intid = plat_interrupts_get_pending_interrupt_id();
448 v_intid = intid;
449
J-Alvesde211782025-02-07 14:44:39 +0000450 /* Get the target vCPU and get the virtual interrupt ID. */
451 target_vcpu = ffa_interrupts_find_target_vcpu(current, intid, &v_intid);
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000452
453 /*
J-Alvesde211782025-02-07 14:44:39 +0000454 * Spurious interrupt ID indicates there is no pending interrupt to
455 * acknowledge so we do not need to call end of interrupt.
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000456 */
J-Alvesde211782025-02-07 14:44:39 +0000457 if (v_intid != SPURIOUS_INTID_OTHER_WORLD) {
458 /*
459 * End the interrupt to drop the running priority. It also
460 * deactivates the physical interrupt. If not, the interrupt
461 * could trigger again after resuming current vCPU.
462 */
463 plat_interrupts_end_of_interrupt(intid);
464 }
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000465
Daniel Boulby7011b5a2024-10-15 18:27:26 +0100466 if (target_vcpu == NULL) {
467 /* No further handling required. Resume the current vCPU. */
468 *next = NULL;
469 return;
470 }
471
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000472 target_vm_locked = vm_lock(target_vcpu->vm);
473
474 if (target_vcpu == current) {
475 current_locked = vcpu_lock(current);
476 target_vcpu_locked = current_locked;
477 } else {
478 struct two_vcpu_locked vcpus_locked;
479 /* Lock both vCPUs at once to avoid deadlock. */
480 vcpus_locked = vcpu_lock_both(current, target_vcpu);
481 current_locked = vcpus_locked.vcpu1;
482 target_vcpu_locked = vcpus_locked.vcpu2;
483 }
484
485 /*
486 * A race condition can occur with the execution contexts belonging to
487 * an MP SP. An interrupt targeting the execution context on present
488 * core can trigger while the execution context of this SP on a
489 * different core is being aborted. In such scenario, the physical
490 * interrupts beloning to the aborted SP are disabled and the current
491 * execution context is resumed.
492 */
493 if (target_vcpu->state == VCPU_STATE_ABORTED ||
494 atomic_load_explicit(&target_vcpu->vm->aborting,
495 memory_order_relaxed)) {
496 /* Clear fields corresponding to secure interrupt handling. */
497 vcpu_secure_interrupt_complete(target_vcpu_locked);
Karl Meakin117c8082024-12-04 16:03:28 +0000498 ffa_vm_disable_interrupts(target_vm_locked);
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000499
500 /* Resume current vCPU. */
501 *next = NULL;
502 } else {
503 /*
504 * SPMC has started handling a secure interrupt with a clean
505 * slate. This signal should be false unless there was a bug in
506 * source code. Hence, use assert rather than CHECK.
507 */
508 assert(!target_vcpu->requires_deactivate_call);
509
510 /* Set the interrupt pending in the target vCPU. */
Daniel Boulby3c1506b2025-02-25 10:49:51 +0000511 vcpu_virt_interrupt_inject(target_vcpu_locked, v_intid);
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000512
513 switch (intid) {
514 case HF_IPI_INTID:
515 if (hf_ipi_handle(target_vcpu_locked)) {
516 *next = NULL;
517 break;
518 }
519 /*
520 * Fall through in the case handling has not been fully
521 * completed.
522 */
523 default:
524 /*
525 * Either invoke the handler related to partitions from
526 * S-EL0 or from S-EL1.
527 */
528 *next = target_vcpu_locked.vcpu->vm->el0_partition
Karl Meakinca38ef92025-02-13 14:20:23 +0000529 ? ffa_interrupts_signal_secure_interrupt_sel0(
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000530 current_locked,
531 target_vcpu_locked, v_intid)
Karl Meakinca38ef92025-02-13 14:20:23 +0000532 : ffa_interrupts_signal_secure_interrupt_sel1(
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000533 current_locked,
534 target_vcpu_locked, v_intid);
535 }
536 }
537
538 if (target_vcpu_locked.vcpu != NULL) {
539 vcpu_unlock(&target_vcpu_locked);
540 }
541
542 vcpu_unlock(&current_locked);
543 vm_unlock(&target_vm_locked);
544}
545
Karl Meakin117c8082024-12-04 16:03:28 +0000546bool ffa_interrupts_inject_notification_pending_interrupt(
Daniel Boulbyd49d0772025-01-15 11:19:36 +0000547 struct vcpu_locked target_locked, struct vm_locked receiver_locked)
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000548{
549 struct vm *next_vm = target_locked.vcpu->vm;
550 bool ret = false;
551
552 /*
553 * Inject the NPI if:
554 * - The targeted VM ID is from this world (i.e. if it is an SP).
555 * - The partition has global pending notifications and an NPI hasn't
556 * been injected yet.
557 * - There are pending per-vCPU notifications in the next vCPU.
558 */
559 if (vm_id_is_current_world(next_vm->id) &&
560 (vm_are_per_vcpu_notifications_pending(
561 receiver_locked, vcpu_index(target_locked.vcpu)) ||
562 (vm_are_global_notifications_pending(receiver_locked) &&
563 !vm_notifications_is_npi_injected(receiver_locked)))) {
Daniel Boulby3c1506b2025-02-25 10:49:51 +0000564 vcpu_virt_interrupt_inject(target_locked,
565 HF_NOTIFICATION_PENDING_INTID);
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000566 vm_notifications_set_npi_injected(receiver_locked, true);
567 ret = true;
568 }
569
570 return ret;
571}
572
Karl Meakin117c8082024-12-04 16:03:28 +0000573struct vcpu *ffa_interrupts_unwind_nwd_call_chain(struct vcpu *current_vcpu)
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000574{
575 struct vcpu *next;
576 struct two_vcpu_locked both_vcpu_locked;
577
578 /*
579 * The action specified by SP in its manifest is ``Non-secure interrupt
580 * is signaled``. Refer to section 8.2.4 rules and guidelines bullet 4.
581 * Hence, the call chain starts unwinding. The current vCPU must have
582 * been a part of NWd scheduled call chain. Therefore, it is pre-empted
583 * and execution is either handed back to the normal world or to the
584 * previous SP vCPU in the call chain through the FFA_INTERRUPT ABI.
585 * The api_preempt() call is equivalent to calling
586 * api_switch_to_other_world for current vCPU passing FFA_INTERRUPT. The
587 * SP can be resumed later by FFA_RUN.
588 */
589 CHECK(current_vcpu->scheduling_mode == NWD_MODE);
590 assert(current_vcpu->call_chain.next_node == NULL);
591
592 if (current_vcpu->call_chain.prev_node == NULL) {
593 /* End of NWd scheduled call chain */
594 return api_preempt(current_vcpu);
595 }
596
597 next = current_vcpu->call_chain.prev_node;
598 CHECK(next != NULL);
599
600 /*
601 * Lock both vCPUs. Strictly speaking, it may not be necessary since
602 * next is guaranteed to be in BLOCKED state as it is the predecessor of
603 * the current vCPU in the present call chain.
604 */
605 both_vcpu_locked = vcpu_lock_both(current_vcpu, next);
606
607 /* Removing a node from an existing call chain. */
608 current_vcpu->call_chain.prev_node = NULL;
609 current_vcpu->state = VCPU_STATE_PREEMPTED;
610
611 /*
612 * SPMC applies the runtime model till when the vCPU transitions from
613 * running to waiting state. Moreover, the SP continues to remain in
614 * its CPU cycle allocation mode. Hence, rt_model and scheduling_mode
615 * are not changed here.
616 */
617 assert(next->state == VCPU_STATE_BLOCKED);
618 assert(next->call_chain.next_node == current_vcpu);
619
620 next->call_chain.next_node = NULL;
621
622 vcpu_set_running(both_vcpu_locked.vcpu2,
623 &(struct ffa_value){
624 .func = FFA_INTERRUPT_32,
625 .arg1 = ffa_vm_vcpu(current_vcpu->vm->id,
626 vcpu_index(current_vcpu)),
627 });
628
629 sl_unlock(&next->lock);
630 sl_unlock(&current_vcpu->lock);
631
632 return next;
633}
634
Karl Meakinca38ef92025-02-13 14:20:23 +0000635static void ffa_interrupts_enable_virtual_maintenance_interrupts(
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000636 struct vcpu_locked current_locked)
637{
638 struct vcpu *current;
639 struct interrupts *interrupts;
640 struct vm *vm;
641
642 current = current_locked.vcpu;
643 interrupts = &current->interrupts;
644 vm = current->vm;
645
Karl Meakin117c8082024-12-04 16:03:28 +0000646 if (ffa_vm_managed_exit_supported(vm)) {
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000647 vcpu_virt_interrupt_set_enabled(interrupts,
648 HF_MANAGED_EXIT_INTID);
649 /*
650 * SPMC decides the interrupt type for Managed exit signal based
651 * on the partition manifest.
652 */
653 if (vm->me_signal_virq) {
654 vcpu_virt_interrupt_set_type(interrupts,
655 HF_MANAGED_EXIT_INTID,
656 INTERRUPT_TYPE_IRQ);
657 } else {
658 vcpu_virt_interrupt_set_type(interrupts,
659 HF_MANAGED_EXIT_INTID,
660 INTERRUPT_TYPE_FIQ);
661 }
662 }
663
664 if (vm->notifications.enabled) {
665 vcpu_virt_interrupt_set_enabled(interrupts,
666 HF_NOTIFICATION_PENDING_INTID);
667 }
668}
669
670/**
671 * Enable relevant virtual interrupts for Secure Partitions.
672 * For all SPs, any applicable virtual maintenance interrupts are enabled.
673 * Additionally, for S-EL0 partitions, all the interrupts declared in the
674 * partition manifest are enabled at the virtual interrupt controller
675 * interface early during the boot stage as an S-EL0 SP need not call
676 * HF_INTERRUPT_ENABLE hypervisor ABI explicitly.
677 */
Karl Meakin117c8082024-12-04 16:03:28 +0000678void ffa_interrupts_enable_virtual_interrupts(struct vcpu_locked current_locked,
679 struct vm_locked vm_locked)
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000680{
681 struct vcpu *current;
682 struct interrupts *interrupts;
683 struct vm *vm;
684
685 current = current_locked.vcpu;
686 interrupts = &current->interrupts;
687 vm = current->vm;
688 assert(vm == vm_locked.vm);
689
690 if (vm->el0_partition) {
691 for (uint32_t k = 0; k < VM_MANIFEST_MAX_INTERRUPTS; k++) {
692 struct interrupt_descriptor int_desc;
693
694 int_desc = vm_locked.vm->interrupt_desc[k];
695
696 /* Interrupt descriptors are populated contiguously. */
697 if (!int_desc.valid) {
698 break;
699 }
700 vcpu_virt_interrupt_set_enabled(interrupts,
701 int_desc.interrupt_id);
702 }
703 }
704
Karl Meakinca38ef92025-02-13 14:20:23 +0000705 ffa_interrupts_enable_virtual_maintenance_interrupts(current_locked);
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000706}
707
708/**
709 * Reconfigure the interrupt belonging to the current partition at runtime.
710 * At present, this paravirtualized interface only allows the following
711 * commands which signify what change is being requested by the current
712 * partition:
713 * - Change the target CPU of the interrupt.
714 * - Change the security state of the interrupt.
715 * - Enable or disable the physical interrupt.
716 */
Karl Meakin117c8082024-12-04 16:03:28 +0000717int64_t ffa_interrupts_reconfigure(uint32_t int_id, uint32_t command,
718 uint32_t value, struct vcpu *current)
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000719{
720 struct vm *vm = current->vm;
721 struct vm_locked vm_locked;
722 int64_t ret = -1;
723 struct interrupt_descriptor *int_desc = NULL;
724
725 /*
726 * Lock VM to protect interrupt descriptor from being modified
727 * concurrently.
728 */
729 vm_locked = vm_lock(vm);
730
731 switch (command) {
732 case INT_RECONFIGURE_TARGET_PE:
733 /* Here, value represents the target PE index. */
734 if (value >= MAX_CPUS) {
735 dlog_verbose(
736 "Illegal target PE index specified while "
737 "reconfiguring interrupt %x\n",
738 int_id);
739 goto out_unlock;
740 }
741
742 /*
743 * An UP SP cannot reconfigure an interrupt to be targetted to
744 * any other physical CPU except the one it is currently
745 * running on.
746 */
747 if (vm_is_up(vm) && value != cpu_index(current->cpu)) {
748 dlog_verbose(
749 "Illegal target PE index specified by current "
750 "UP SP\n");
751 goto out_unlock;
752 }
753
754 /* Configure the interrupt to be routed to a specific CPU. */
755 int_desc = vm_interrupt_set_target_mpidr(
756 vm_locked, int_id, cpu_find_index(value)->id);
757 break;
758 case INT_RECONFIGURE_SEC_STATE:
759 /* Specify the new security state of the interrupt. */
760 if (value != INT_DESC_SEC_STATE_NS &&
761 value != INT_DESC_SEC_STATE_S) {
762 dlog_verbose(
763 "Illegal value %x specified while "
764 "reconfiguring interrupt %x\n",
765 value, int_id);
766 goto out_unlock;
767 }
768 int_desc = vm_interrupt_set_sec_state(vm_locked, int_id, value);
769 break;
770 case INT_RECONFIGURE_ENABLE:
771 /* Enable or disable the interrupt. */
772 if (value != INT_DISABLE && value != INT_ENABLE) {
773 dlog_verbose(
774 "Illegal value %x specified while "
775 "reconfiguring interrupt %x\n",
776 value, int_id);
777 goto out_unlock;
778 } else {
779 int_desc = vm_interrupt_set_enable(vm_locked, int_id,
780 value == INT_ENABLE);
781 }
782 break;
783 default:
784 dlog_verbose("Interrupt reconfigure: Unsupported command %x\n",
785 command);
786 goto out_unlock;
787 }
788
789 /* Check if the interrupt belongs to the current SP. */
790 if (int_desc == NULL) {
791 dlog_verbose("Interrupt %x does not belong to current SP\n",
792 int_id);
793 goto out_unlock;
794 }
795
796 ret = 0;
797 plat_interrupts_reconfigure_interrupt(*int_desc);
798
799out_unlock:
800 vm_unlock(&vm_locked);
801
802 return ret;
803}
804
805/* Returns the virtual interrupt id to be handled by SP. */
Karl Meakin117c8082024-12-04 16:03:28 +0000806uint32_t ffa_interrupts_get(struct vcpu_locked current_locked)
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000807{
Daniel Boulby3c1506b2025-02-25 10:49:51 +0000808 return vcpu_virt_interrupt_get_pending_and_enabled(current_locked);
Karl Meakin8e58ddc2024-11-08 23:19:34 +0000809}
Karl Meakin8d245542025-01-31 13:19:25 +0000810
811/**
812 * Run the vCPU in SPMC schedule mode under the runtime model for secure
813 * interrupt handling.
814 */
Karl Meakinca38ef92025-02-13 14:20:23 +0000815static void ffa_interrupts_run_in_sec_interrupt_rtm(
Karl Meakin8d245542025-01-31 13:19:25 +0000816 struct vcpu_locked target_vcpu_locked)
817{
818 struct vcpu *target_vcpu;
819
820 target_vcpu = target_vcpu_locked.vcpu;
821
822 /* Mark the registers as unavailable now. */
823 target_vcpu->regs_available = false;
824 target_vcpu->scheduling_mode = SPMC_MODE;
825 target_vcpu->rt_model = RTM_SEC_INTERRUPT;
826 target_vcpu->state = VCPU_STATE_RUNNING;
827 target_vcpu->requires_deactivate_call = false;
828}
829
830bool ffa_interrupts_intercept_call(struct vcpu_locked current_locked,
831 struct vcpu_locked next_locked,
832 struct ffa_value *signal_interrupt)
833{
Daniel Boulby3c1506b2025-02-25 10:49:51 +0000834 uint32_t intid =
835 vcpu_virt_interrupt_peek_pending_and_enabled(current_locked);
Karl Meakin8d245542025-01-31 13:19:25 +0000836
837 /*
838 * Check if there are any pending virtual secure interrupts to be
839 * handled.
840 */
Daniel Boulby3c1506b2025-02-25 10:49:51 +0000841 if (intid != HF_INVALID_INTID) {
Karl Meakin8d245542025-01-31 13:19:25 +0000842 /*
843 * Prepare to signal virtual secure interrupt to S-EL0/S-EL1 SP
844 * in WAITING state. Refer to FF-A v1.2 Table 9.1 and Table 9.2
845 * case 1.
846 */
847 *signal_interrupt = api_ffa_interrupt_return(intid);
848
849 /*
850 * Prepare to resume this partition's vCPU in SPMC
851 * schedule mode to handle virtual secure interrupt.
852 */
Karl Meakinca38ef92025-02-13 14:20:23 +0000853 ffa_interrupts_run_in_sec_interrupt_rtm(current_locked);
Karl Meakin8d245542025-01-31 13:19:25 +0000854
855 current_locked.vcpu->preempted_vcpu = next_locked.vcpu;
856 next_locked.vcpu->state = VCPU_STATE_PREEMPTED;
857
Daniel Boulby3c1506b2025-02-25 10:49:51 +0000858 dlog_verbose(
859 "%s: Pending interrupt %d, intercepting FF-A call.\n",
860 __func__, intid);
Karl Meakin8d245542025-01-31 13:19:25 +0000861
862 return true;
863 }
864
865 return false;
866}