blob: ba52d3b0d24134e2d7a2047c64b71d03712ab20e [file] [log] [blame]
Fuad Tabba5c738432019-12-02 11:02:42 +00001/*
2 * Copyright 2019 The Hafnium Authors.
3 *
Andrew Walbrane959ec12020-06-17 15:01:09 +01004 * Use of this source code is governed by a BSD-style
5 * license that can be found in the LICENSE file or at
6 * https://opensource.org/licenses/BSD-3-Clause.
Fuad Tabba5c738432019-12-02 11:02:42 +00007 */
8
9#include "hf/vcpu.h"
10
Olivier Depreze6f7b9d2021-02-01 11:55:48 +010011#include "hf/arch/cpu.h"
12
Fuad Tabba5c738432019-12-02 11:02:42 +000013#include "hf/check.h"
14#include "hf/dlog.h"
15#include "hf/std.h"
16#include "hf/vm.h"
17
J-Alves7ac49052022-02-08 17:20:53 +000018/** GP register to be used to pass the current vCPU ID, at core bring up. */
19#define PHYS_CORE_IDX_GP_REG 4
20
Fuad Tabba5c738432019-12-02 11:02:42 +000021/**
22 * Locks the given vCPU and updates `locked` to hold the newly locked vCPU.
23 */
24struct vcpu_locked vcpu_lock(struct vcpu *vcpu)
25{
26 struct vcpu_locked locked = {
27 .vcpu = vcpu,
28 };
29
30 sl_lock(&vcpu->lock);
31
32 return locked;
33}
34
35/**
Olivier Deprez0b6f10a2020-08-05 18:21:33 +020036 * Locks two vCPUs ensuring that the locking order is according to the locks'
37 * addresses.
38 */
39struct two_vcpu_locked vcpu_lock_both(struct vcpu *vcpu1, struct vcpu *vcpu2)
40{
41 struct two_vcpu_locked dual_lock;
42
43 sl_lock_both(&vcpu1->lock, &vcpu2->lock);
44 dual_lock.vcpu1.vcpu = vcpu1;
45 dual_lock.vcpu2.vcpu = vcpu2;
46
47 return dual_lock;
48}
49
50/**
Fuad Tabba5c738432019-12-02 11:02:42 +000051 * Unlocks a vCPU previously locked with vpu_lock, and updates `locked` to
52 * reflect the fact that the vCPU is no longer locked.
53 */
54void vcpu_unlock(struct vcpu_locked *locked)
55{
56 sl_unlock(&locked->vcpu->lock);
57 locked->vcpu = NULL;
58}
59
60void vcpu_init(struct vcpu *vcpu, struct vm *vm)
61{
62 memset_s(vcpu, sizeof(*vcpu), 0, sizeof(*vcpu));
63 sl_init(&vcpu->lock);
64 vcpu->regs_available = true;
65 vcpu->vm = vm;
Kathleen Capellae468c112023-12-13 17:56:28 -050066 vcpu->direct_request_origin.is_ffa_req2 = false;
67 vcpu->direct_request_origin.vm_id = HF_INVALID_VM_ID;
Olivier Deprezb2808332023-02-02 15:25:40 +010068 vcpu->rt_model = RTM_SP_INIT;
Madhukar Pappireddyeed861e2024-09-25 13:50:54 -050069 list_init(&vcpu->timer_node);
Daniel Boulby7011b5a2024-10-15 18:27:26 +010070 list_init(&vcpu->ipi_list_node);
Madhukar Pappireddyd6c055d2025-05-08 15:35:46 -050071
72 /*
73 * Though resources have not been allocated to the partition yet, it is
74 * safe to skip the NULL state for vCPU during cold boot and transition
75 * directly to CREATED state.
76 */
77 vcpu->state = VCPU_STATE_CREATED;
Fuad Tabba5c738432019-12-02 11:02:42 +000078}
79
80/**
81 * Initialise the registers for the given vCPU and set the state to
Madhukar Pappireddy98ab38b2025-05-08 15:46:31 -050082 * VCPU_STATE_CREATED. The caller must hold the vCPU lock while calling this.
Fuad Tabba5c738432019-12-02 11:02:42 +000083 */
Madhukar Pappireddy98ab38b2025-05-08 15:46:31 -050084void vcpu_prepare(struct vcpu_locked vcpu, ipaddr_t entry, uintreg_t arg)
Fuad Tabba5c738432019-12-02 11:02:42 +000085{
86 arch_regs_set_pc_arg(&vcpu.vcpu->regs, entry, arg);
Madhukar Pappireddy98ab38b2025-05-08 15:46:31 -050087 vcpu.vcpu->state = VCPU_STATE_CREATED;
Fuad Tabba5c738432019-12-02 11:02:42 +000088}
89
Andrew Walbranb5ab43c2020-04-30 11:32:54 +010090ffa_vcpu_index_t vcpu_index(const struct vcpu *vcpu)
Fuad Tabba5c738432019-12-02 11:02:42 +000091{
92 size_t index = vcpu - vcpu->vm->vcpus;
93
94 CHECK(index < UINT16_MAX);
95 return index;
96}
97
98/**
99 * Check whether the given vcpu_state is an off state, for the purpose of
Madhukar Pappireddyb11e0d12021-08-02 19:44:35 -0500100 * turning vCPUs on and off. Note that Aborted still counts as ON for the
101 * purposes of PSCI, because according to the PSCI specification (section
Olivier Depreze7eb1682022-03-16 17:09:03 +0100102 * 5.7.1) a core is only considered to be off if it has been turned off
103 * with a CPU_OFF call or hasn't yet been turned on with a CPU_ON call.
Madhukar Pappireddy98ab38b2025-05-08 15:46:31 -0500104 * Note that vCPU is considered to be off until it makes the transition
105 * to STARTING state.
Fuad Tabba5c738432019-12-02 11:02:42 +0000106 */
107bool vcpu_is_off(struct vcpu_locked vcpu)
108{
Madhukar Pappireddy98ab38b2025-05-08 15:46:31 -0500109 return (vcpu.vcpu->state == VCPU_STATE_OFF ||
110 vcpu.vcpu->state == VCPU_STATE_CREATED ||
111 vcpu.vcpu->state == VCPU_STATE_NULL);
Fuad Tabba5c738432019-12-02 11:02:42 +0000112}
113
114/**
115 * Starts a vCPU of a secondary VM.
116 *
117 * Returns true if the secondary was reset and started, or false if it was
118 * already on and so nothing was done.
119 */
Max Shvetsov40108e72020-08-27 12:39:50 +0100120bool vcpu_secondary_reset_and_start(struct vcpu_locked vcpu_locked,
121 ipaddr_t entry, uintreg_t arg)
Fuad Tabba5c738432019-12-02 11:02:42 +0000122{
Max Shvetsov40108e72020-08-27 12:39:50 +0100123 struct vm *vm = vcpu_locked.vcpu->vm;
Fuad Tabba5c738432019-12-02 11:02:42 +0000124 bool vcpu_was_off;
125
126 CHECK(vm->id != HF_PRIMARY_VM_ID);
127
Fuad Tabba5c738432019-12-02 11:02:42 +0000128 vcpu_was_off = vcpu_is_off(vcpu_locked);
129 if (vcpu_was_off) {
130 /*
131 * Set vCPU registers to a clean state ready for boot. As this
132 * is a secondary which can migrate between pCPUs, the ID of the
133 * vCPU is defined as the index and does not match the ID of the
134 * pCPU it is running on.
135 */
Max Shvetsov40108e72020-08-27 12:39:50 +0100136 arch_regs_reset(vcpu_locked.vcpu);
Madhukar Pappireddy98ab38b2025-05-08 15:46:31 -0500137 vcpu_prepare(vcpu_locked, entry, arg);
Fuad Tabba5c738432019-12-02 11:02:42 +0000138 }
Fuad Tabba5c738432019-12-02 11:02:42 +0000139
140 return vcpu_was_off;
141}
142
143/**
144 * Handles a page fault. It does so by determining if it's a legitimate or
145 * spurious fault, and recovering from the latter.
146 *
Fuad Tabbaed294af2019-12-20 10:43:01 +0000147 * Returns true if the caller should resume the current vCPU, or false if its VM
Fuad Tabba5c738432019-12-02 11:02:42 +0000148 * should be aborted.
149 */
150bool vcpu_handle_page_fault(const struct vcpu *current,
151 struct vcpu_fault_info *f)
152{
153 struct vm *vm = current->vm;
Karl Meakin07a69ab2025-02-07 14:53:19 +0000154 mm_mode_t mode;
Fuad Tabba5c738432019-12-02 11:02:42 +0000155 uint32_t mask = f->mode | MM_MODE_INVALID;
156 bool resume;
Raghu Krishnamurthy785d52f2021-02-13 00:02:40 -0800157 struct vm_locked locked_vm;
Fuad Tabba5c738432019-12-02 11:02:42 +0000158
Raghu Krishnamurthy785d52f2021-02-13 00:02:40 -0800159 locked_vm = vm_lock(vm);
Fuad Tabba5c738432019-12-02 11:02:42 +0000160 /*
161 * Check if this is a legitimate fault, i.e., if the page table doesn't
162 * allow the access attempted by the VM.
163 *
164 * Otherwise, this is a spurious fault, likely because another CPU is
165 * updating the page table. It is responsible for issuing global TLB
166 * invalidations while holding the VM lock, so we don't need to do
167 * anything else to recover from it. (Acquiring/releasing the lock
168 * ensured that the invalidations have completed.)
169 */
Raghu Krishnamurthyb5775d22021-02-26 18:54:40 -0800170 if (!locked_vm.vm->el0_partition) {
171 resume = vm_mem_get_mode(locked_vm, f->ipaddr,
172 ipa_add(f->ipaddr, 1), &mode) &&
173 (mode & mask) == f->mode;
174 } else {
175 /*
176 * For EL0 partitions we need to get the mode for the faulting
177 * vaddr.
178 */
179 resume =
180 vm_mem_get_mode(locked_vm, ipa_init(va_addr(f->vaddr)),
181 ipa_add(ipa_init(va_addr(f->vaddr)), 1),
182 &mode) &&
183 (mode & mask) == f->mode;
Raghu Krishnamurthyf16b2ce2021-11-02 07:48:38 -0700184
185 /*
186 * For EL0 partitions, if there is an instruction abort and the
187 * mode of the page is RWX, we don't resume since Hafnium does
188 * not allow write and executable pages.
189 */
190 if ((f->mode == MM_MODE_X) &&
191 ((mode & MM_MODE_W) == MM_MODE_W)) {
192 resume = false;
193 }
Raghu Krishnamurthyb5775d22021-02-26 18:54:40 -0800194 }
Fuad Tabba5c738432019-12-02 11:02:42 +0000195
Raghu Krishnamurthy785d52f2021-02-13 00:02:40 -0800196 vm_unlock(&locked_vm);
Fuad Tabba5c738432019-12-02 11:02:42 +0000197
198 if (!resume) {
Andrew Walbran17eebf92020-02-05 16:35:49 +0000199 dlog_warning(
Karl Meakine8937d92024-03-19 16:04:25 +0000200 "Stage-%d page fault: pc=%#lx, vmid=%#x, vcpu=%u, "
201 "vaddr=%#lx, ipaddr=%#lx, mode=%#x %#x\n",
202 current->vm->el0_partition ? 1 : 2, va_addr(f->pc),
203 vm->id, vcpu_index(current), va_addr(f->vaddr),
204 ipa_addr(f->ipaddr), f->mode, mode);
Fuad Tabba5c738432019-12-02 11:02:42 +0000205 }
206
207 return resume;
208}
Olivier Deprez2ebae3a2020-06-11 16:34:30 +0200209
J-Alves7ac49052022-02-08 17:20:53 +0000210void vcpu_set_phys_core_idx(struct vcpu *vcpu)
211{
212 arch_regs_set_gp_reg(&vcpu->regs, cpu_index(vcpu->cpu),
213 PHYS_CORE_IDX_GP_REG);
214}
Olivier Deprez181074b2023-02-02 14:53:23 +0100215
216/**
Olivier Deprez632249e2022-09-26 09:18:31 +0200217 * Sets the designated GP register through which the vCPU expects to receive the
218 * boot info's address.
219 */
220void vcpu_set_boot_info_gp_reg(struct vcpu *vcpu)
221{
222 struct vm *vm = vcpu->vm;
223 uint32_t gp_register_num = vm->boot_info.gp_register_num;
224
225 if (vm->boot_info.blob_addr.ipa != 0U) {
226 arch_regs_set_gp_reg(&vcpu->regs,
227 ipa_addr(vm->boot_info.blob_addr),
228 gp_register_num);
229 }
230}
231
Daniel Boulbyd633a612025-03-07 18:08:04 +0000232static bool vcpu_is_virt_interrupt_enabled(struct interrupts *interrupts,
233 uint32_t intid)
234{
235 return interrupt_bitmap_get_value(&interrupts->interrupt_enabled,
236 intid) == 1U;
237}
238
239static void vcpu_virt_interrupt_set_enabled(struct interrupts *interrupts,
240 uint32_t intid)
241{
242 interrupt_bitmap_set_value(&interrupts->interrupt_enabled, intid);
243}
244
245static void vcpu_virt_interrupt_clear_enabled(struct interrupts *interrupts,
246 uint32_t intid)
247{
248 interrupt_bitmap_clear_value(&interrupts->interrupt_enabled, intid);
249}
250
Daniel Boulbyd21e9b32025-02-13 15:53:21 +0000251static void vcpu_virt_interrupt_set_pending(struct interrupts *interrupts,
252 uint32_t intid)
253{
254 interrupt_bitmap_set_value(&interrupts->interrupt_pending, intid);
255}
256
257static void vcpu_virt_interrupt_clear_pending(struct interrupts *interrupts,
258 uint32_t intid)
259{
260 interrupt_bitmap_clear_value(&interrupts->interrupt_pending, intid);
261}
262
263static void vcpu_irq_count_increment(struct vcpu_locked vcpu_locked)
264{
265 vcpu_locked.vcpu->interrupts.enabled_and_pending_irq_count++;
266}
267
268static void vcpu_irq_count_decrement(struct vcpu_locked vcpu_locked)
269{
270 vcpu_locked.vcpu->interrupts.enabled_and_pending_irq_count--;
271}
272
273static void vcpu_fiq_count_increment(struct vcpu_locked vcpu_locked)
274{
275 vcpu_locked.vcpu->interrupts.enabled_and_pending_fiq_count++;
276}
277
278static void vcpu_fiq_count_decrement(struct vcpu_locked vcpu_locked)
279{
280 vcpu_locked.vcpu->interrupts.enabled_and_pending_fiq_count--;
281}
282
283static void vcpu_interrupt_count_increment(struct vcpu_locked vcpu_locked,
284 uint32_t intid)
285{
286 struct interrupts *interrupts = &vcpu_locked.vcpu->interrupts;
287
288 if (vcpu_virt_interrupt_get_type(interrupts, intid) ==
289 INTERRUPT_TYPE_IRQ) {
290 vcpu_irq_count_increment(vcpu_locked);
291 } else {
292 vcpu_fiq_count_increment(vcpu_locked);
293 }
294}
295
296static void vcpu_interrupt_count_decrement(struct vcpu_locked vcpu_locked,
297 uint32_t intid)
298{
299 struct interrupts *interrupts = &vcpu_locked.vcpu->interrupts;
300
301 if (vcpu_virt_interrupt_get_type(interrupts, intid) ==
302 INTERRUPT_TYPE_IRQ) {
303 vcpu_irq_count_decrement(vcpu_locked);
304 } else {
305 vcpu_fiq_count_decrement(vcpu_locked);
306 }
307}
308
309uint32_t vcpu_virt_interrupt_irq_count_get(struct vcpu_locked vcpu_locked)
310{
311 return vcpu_locked.vcpu->interrupts.enabled_and_pending_irq_count;
312}
313
314uint32_t vcpu_virt_interrupt_fiq_count_get(struct vcpu_locked vcpu_locked)
315{
316 return vcpu_locked.vcpu->interrupts.enabled_and_pending_fiq_count;
317}
318
319uint32_t vcpu_virt_interrupt_count_get(struct vcpu_locked vcpu_locked)
320{
321 return vcpu_virt_interrupt_irq_count_get(vcpu_locked) +
322 vcpu_virt_interrupt_fiq_count_get(vcpu_locked);
323}
324
Daniel Boulbyd7992232025-03-06 17:09:49 +0000325static void vcpu_interrupt_clear_decrement(struct vcpu_locked vcpu_locked,
326 uint32_t intid)
J-Alvesb8730e92024-08-07 18:28:55 +0100327{
328 struct interrupts *interrupts = &(vcpu_locked.vcpu->interrupts);
329
Daniel Boulbyd21e9b32025-02-13 15:53:21 +0000330 /*
331 * Mark the virtual interrupt as no longer pending and decrement
Daniel Boulbyd7992232025-03-06 17:09:49 +0000332 * the interrupt count if it is enabled.
Daniel Boulbyd21e9b32025-02-13 15:53:21 +0000333 */
J-Alvesb8730e92024-08-07 18:28:55 +0100334 vcpu_virt_interrupt_clear_pending(interrupts, intid);
Daniel Boulbyd7992232025-03-06 17:09:49 +0000335 if (vcpu_is_virt_interrupt_enabled(interrupts, intid)) {
336 vcpu_interrupt_count_decrement(vcpu_locked, intid);
337 }
J-Alvesb8730e92024-08-07 18:28:55 +0100338}
339
J-Alves0247fe62024-02-23 10:21:46 +0000340/**
341 * Sets the vcpu in the VCPU_STATE_RUNNING.
J-Alves67a79262024-07-17 12:01:39 +0100342 * With that, its register are set as "not available".
343 * If there are registers to be written to vCPU's context, do so.
344 * However, this action is restricted to WAITING and BLOCKED states,
345 * as such, assert accordingly.
J-Alves0247fe62024-02-23 10:21:46 +0000346 */
J-Alves478faac2024-10-23 10:35:57 +0100347void vcpu_set_running(struct vcpu_locked target_locked,
348 const struct ffa_value *args)
J-Alves12cedae2023-08-04 14:37:37 +0100349{
350 struct vcpu *target_vcpu = target_locked.vcpu;
351
J-Alves67a79262024-07-17 12:01:39 +0100352 if (args != NULL) {
J-Alves0247fe62024-02-23 10:21:46 +0000353 CHECK(target_vcpu->regs_available);
J-Alves67a79262024-07-17 12:01:39 +0100354 assert(target_vcpu->state == VCPU_STATE_WAITING ||
355 target_vcpu->state == VCPU_STATE_BLOCKED);
J-Alves0247fe62024-02-23 10:21:46 +0000356
357 arch_regs_set_retval(&target_vcpu->regs, *args);
358 }
J-Alves12cedae2023-08-04 14:37:37 +0100359
360 /* Mark the registers as unavailable now. */
361 target_vcpu->regs_available = false;
362
363 /* We are about to resume target vCPU. */
364 target_vcpu->state = VCPU_STATE_RUNNING;
365}
366
J-Alves12cedae2023-08-04 14:37:37 +0100367void vcpu_enter_secure_interrupt_rtm(struct vcpu_locked vcpu_locked)
368{
369 struct vcpu *target_vcpu = vcpu_locked.vcpu;
370
371 assert(target_vcpu->scheduling_mode == NONE);
372 assert(target_vcpu->call_chain.prev_node == NULL);
373 assert(target_vcpu->call_chain.next_node == NULL);
374 assert(target_vcpu->rt_model == RTM_NONE);
375
376 target_vcpu->scheduling_mode = SPMC_MODE;
377 target_vcpu->rt_model = RTM_SEC_INTERRUPT;
378}
Madhukar Pappireddy32913cb2024-07-19 13:04:05 -0500379
380static uint16_t queue_increment_index(uint16_t current_idx)
381{
382 /* Look at the next index. Wrap around if necessary. */
383 if (current_idx == VINT_QUEUE_MAX - 1) {
384 return 0;
385 }
386
387 return current_idx + 1;
388}
389
J-Alves23a73032025-03-04 16:20:54 +0000390/**
391 * If tail reaches head of the queue, and the count of queued interrupts
392 * 0, then the queue is empty.
393 */
Madhukar Pappireddy32913cb2024-07-19 13:04:05 -0500394static bool is_queue_empty(struct interrupt_queue *q)
395{
J-Alves23a73032025-03-04 16:20:54 +0000396 return q->head == q->tail && q->queued_vint_count == 0U;
397}
Madhukar Pappireddy32913cb2024-07-19 13:04:05 -0500398
J-Alves23a73032025-03-04 16:20:54 +0000399/**
400 * If tail reaches head of the queue, and the count of queued interrupts
401 * matches the size of the buffer, then the queue is full.
402 */
403static bool is_queue_full(struct interrupt_queue *q)
404{
405 return q->head == q->tail && q->queued_vint_count == VINT_QUEUE_MAX;
Madhukar Pappireddy32913cb2024-07-19 13:04:05 -0500406}
407
408/**
409 * Queue the pending virtual interrupt for target vCPU.
410 *
411 * Returns true if successful in pushing a new entry to the queue, or false
412 * otherwise.
413 */
Daniel Boulby3c1506b2025-02-25 10:49:51 +0000414static bool vcpu_interrupt_queue_push(struct vcpu_locked vcpu_locked,
415 uint32_t vint_id)
Madhukar Pappireddy32913cb2024-07-19 13:04:05 -0500416{
417 struct interrupt_queue *q;
418 uint16_t new_tail;
419
420 assert(vint_id != HF_INVALID_INTID);
421
422 q = &vcpu_locked.vcpu->interrupts.vint_q;
423
424 /*
425 * A new entry is pushed at the tail of the queue. Upon successful
426 * push operation, the tail increments or wraps around.
427 */
428 new_tail = queue_increment_index(q->tail);
429
J-Alves23a73032025-03-04 16:20:54 +0000430 if (is_queue_full(q)) {
Madhukar Pappireddy32913cb2024-07-19 13:04:05 -0500431 return false;
432 }
433
434 /* Add the virtual interrupt to the queue. */
435 q->vint_buffer[q->tail] = vint_id;
436 q->tail = new_tail;
437
J-Alves23a73032025-03-04 16:20:54 +0000438 assert(q->queued_vint_count < VINT_QUEUE_MAX);
439 q->queued_vint_count++;
440
Madhukar Pappireddy32913cb2024-07-19 13:04:05 -0500441 return true;
442}
443
444/**
445 * Remove an entry from the specified vCPU's queue at the head.
Madhukar Pappireddy32913cb2024-07-19 13:04:05 -0500446 * Returns true if successful in removing the entry, or false otherwise.
447 */
Daniel Boulby825ec7e2025-03-06 15:55:09 +0000448static uint32_t vcpu_interrupt_queue_pop(struct vcpu_locked vcpu_locked)
Madhukar Pappireddy32913cb2024-07-19 13:04:05 -0500449{
450 struct interrupt_queue *q;
451 uint16_t new_head;
Daniel Boulby825ec7e2025-03-06 15:55:09 +0000452 uint32_t vint_id;
Madhukar Pappireddy32913cb2024-07-19 13:04:05 -0500453
454 q = &vcpu_locked.vcpu->interrupts.vint_q;
455
456 /* Check if queue is empty. */
457 if (is_queue_empty(q)) {
Daniel Boulby825ec7e2025-03-06 15:55:09 +0000458 return HF_INVALID_INTID;
Madhukar Pappireddy32913cb2024-07-19 13:04:05 -0500459 }
460
461 /*
462 * An entry is removed from the head of the queue. Once successful, the
463 * head is incremented or wrapped around if needed.
464 */
465 new_head = queue_increment_index(q->head);
Daniel Boulby825ec7e2025-03-06 15:55:09 +0000466 vint_id = q->vint_buffer[q->head];
Madhukar Pappireddy32913cb2024-07-19 13:04:05 -0500467 q->head = new_head;
468
J-Alves23a73032025-03-04 16:20:54 +0000469 assert(q->queued_vint_count > 0);
470 q->queued_vint_count--;
471
Daniel Boulby825ec7e2025-03-06 15:55:09 +0000472 return vint_id;
Madhukar Pappireddy32913cb2024-07-19 13:04:05 -0500473}
474
475/**
476 * Look for the first pending virtual interrupt from the vcpu's queue. Note
477 * that the entry is not removed from the queue.
478 *
479 * Returns true if a valid entry exists in the queue, or false otherwise.
480 */
Daniel Boulby825ec7e2025-03-06 15:55:09 +0000481static uint32_t vcpu_interrupt_queue_peek(struct vcpu_locked vcpu_locked)
Madhukar Pappireddy32913cb2024-07-19 13:04:05 -0500482{
483 struct interrupt_queue *q;
484 uint32_t queued_vint;
485
Madhukar Pappireddy32913cb2024-07-19 13:04:05 -0500486 q = &vcpu_locked.vcpu->interrupts.vint_q;
487
488 /* Check if queue is empty. */
489 if (is_queue_empty(q)) {
Daniel Boulby825ec7e2025-03-06 15:55:09 +0000490 return HF_INVALID_INTID;
Madhukar Pappireddy32913cb2024-07-19 13:04:05 -0500491 }
492
493 queued_vint = q->vint_buffer[q->head];
494 assert(queued_vint != HF_INVALID_INTID);
495
Daniel Boulby825ec7e2025-03-06 15:55:09 +0000496 return queued_vint;
Madhukar Pappireddy32913cb2024-07-19 13:04:05 -0500497}
498
499/**
J-Alves3b31f092024-08-07 13:26:29 +0100500 * When interrupt handling is complete the preempted_vcpu field should go back
501 * to NULL.
502 */
503void vcpu_secure_interrupt_complete(struct vcpu_locked vcpu_locked)
504{
505 struct vcpu *vcpu;
506
507 vcpu = vcpu_locked.vcpu;
508 vcpu->preempted_vcpu = NULL;
J-Alves3b31f092024-08-07 13:26:29 +0100509}
Daniel Boulby3c1506b2025-02-25 10:49:51 +0000510
Daniel Boulbyd21e9b32025-02-13 15:53:21 +0000511void vcpu_virt_interrupt_enable(struct vcpu_locked vcpu_locked,
512 uint32_t vint_id, bool enable)
513{
514 struct interrupts *interrupts = &vcpu_locked.vcpu->interrupts;
515
516 if (enable) {
517 /*
518 * If it is pending and was not enabled before, increment the
519 * count.
520 */
521 if (vcpu_is_virt_interrupt_pending(interrupts, vint_id) &&
522 !vcpu_is_virt_interrupt_enabled(interrupts, vint_id)) {
523 vcpu_interrupt_count_increment(vcpu_locked, vint_id);
524 }
525 vcpu_virt_interrupt_set_enabled(interrupts, vint_id);
526 } else {
527 /*
528 * If it is pending and was enabled before, decrement the count.
529 */
530 if (vcpu_is_virt_interrupt_pending(interrupts, vint_id) &&
531 vcpu_is_virt_interrupt_enabled(interrupts, vint_id)) {
532 vcpu_interrupt_count_decrement(vcpu_locked, vint_id);
533 }
534 vcpu_virt_interrupt_clear_enabled(interrupts, vint_id);
535 }
536}
537
Daniel Boulby3c1506b2025-02-25 10:49:51 +0000538/*
539 * Find and return the first intid that is pending and enabled, the interrupt
Daniel Boulby825ec7e2025-03-06 15:55:09 +0000540 * struct for this intid will be at the head of the list so can be popped later.
Daniel Boulby3c1506b2025-02-25 10:49:51 +0000541 */
542uint32_t vcpu_virt_interrupt_peek_pending_and_enabled(
543 struct vcpu_locked vcpu_locked)
544{
Daniel Boulby825ec7e2025-03-06 15:55:09 +0000545 uint32_t vint_id;
Daniel Boulby3c1506b2025-02-25 10:49:51 +0000546 struct interrupts *interrupts = &vcpu_locked.vcpu->interrupts;
547 uint32_t pending_and_enabled_count =
Daniel Boulbyd21e9b32025-02-13 15:53:21 +0000548 vcpu_virt_interrupt_count_get(vcpu_locked);
Daniel Boulby3c1506b2025-02-25 10:49:51 +0000549
550 /* First check there is a pending and enabled interrupt to return. */
551 if (pending_and_enabled_count == 0) {
552 return HF_INVALID_INTID;
553 }
554
555 /*
556 * We know here there is a pending and enabled interrupt in
557 * the queue. So push any interrupts that are not enabled to
558 * the back of the queue until we reach the first enabled one.
559 */
Daniel Boulby825ec7e2025-03-06 15:55:09 +0000560 vint_id = vcpu_interrupt_queue_peek(vcpu_locked);
561 while (!vcpu_is_virt_interrupt_enabled(interrupts, vint_id)) {
562 vcpu_interrupt_queue_pop(vcpu_locked);
Daniel Boulby3c1506b2025-02-25 10:49:51 +0000563 vcpu_interrupt_queue_push(vcpu_locked, vint_id);
Daniel Boulby825ec7e2025-03-06 15:55:09 +0000564 vint_id = vcpu_interrupt_queue_peek(vcpu_locked);
Daniel Boulby3c1506b2025-02-25 10:49:51 +0000565 }
566
567 assert(vint_id != HF_INVALID_INTID);
568
569 return vint_id;
570}
571
572/*
573 * Get the next pending and enabled virtual interrupt ID.
574 * Pops from the queue and clears the bitmap.
575 */
576uint32_t vcpu_virt_interrupt_get_pending_and_enabled(
577 struct vcpu_locked vcpu_locked)
578{
579 uint32_t vint_id =
580 vcpu_virt_interrupt_peek_pending_and_enabled(vcpu_locked);
581
582 if (vint_id != HF_INVALID_INTID) {
Daniel Boulby825ec7e2025-03-06 15:55:09 +0000583 vcpu_interrupt_queue_pop(vcpu_locked);
Daniel Boulby3c1506b2025-02-25 10:49:51 +0000584 vcpu_interrupt_clear_decrement(vcpu_locked, vint_id);
J-Alves0cbd7a32025-02-10 17:29:15 +0000585
586 /*
587 * Resetting the state of the interrupts_info_get_retrieved,
588 * so the interrupts pending in the vCPU can be included in the
589 * FFA_NOTIFICATION_INFO_GET list.
590 * Resetting now as this functions clears the state of the
591 * virtual interrupt.
592 */
593 vcpu_locked.vcpu->interrupts_info_get_retrieved = false;
Daniel Boulby3c1506b2025-02-25 10:49:51 +0000594 }
595
596 return vint_id;
597}
598
599/*
600 * Set a virtual interrupt to pending. Add it to the queue and set the bitmap.
601 */
602void vcpu_virt_interrupt_inject(struct vcpu_locked vcpu_locked,
603 uint32_t vint_id)
604{
605 struct interrupts *interrupts = &vcpu_locked.vcpu->interrupts;
606
607 /*
608 * An interrupt can only be pending once so return if it is
609 * already pending.
610 */
611 if (vcpu_is_virt_interrupt_pending(interrupts, vint_id)) {
612 return;
613 }
614
615 /* Push to the queue and set the bitmap. */
616 if (!vcpu_interrupt_queue_push(vcpu_locked, vint_id)) {
Daniel Boulby9d8fd3e2025-03-06 14:21:39 +0000617 dlog_verbose(
618 "Exhausted interrupt queue for vCPU %u of SP %#x\n",
619 vcpu_index(vcpu_locked.vcpu), vcpu_locked.vcpu->vm->id);
620 assert(false);
621 return;
Daniel Boulby3c1506b2025-02-25 10:49:51 +0000622 }
623 vcpu_virt_interrupt_set_pending(interrupts, vint_id);
624
625 if (vcpu_is_virt_interrupt_enabled(interrupts, vint_id)) {
Daniel Boulbyd21e9b32025-02-13 15:53:21 +0000626 vcpu_interrupt_count_increment(vcpu_locked, vint_id);
Daniel Boulby3c1506b2025-02-25 10:49:51 +0000627 }
628}
Daniel Boulbyd7992232025-03-06 17:09:49 +0000629
630void vcpu_virt_interrupt_clear(struct vcpu_locked vcpu_locked, uint32_t vint_id)
631{
632 struct interrupts *interrupts = &vcpu_locked.vcpu->interrupts;
633 uint32_t queued_vint_count = interrupts->vint_q.queued_vint_count;
634
635 /* See if interrupt is pending and therefore needs to be cleared. */
636 if (!vcpu_is_virt_interrupt_pending(interrupts, vint_id)) {
637 return;
638 }
639
640 for (uint32_t i = 0; i < queued_vint_count; i++) {
Daniel Boulby825ec7e2025-03-06 15:55:09 +0000641 uint32_t intid = vcpu_interrupt_queue_pop(vcpu_locked);
Daniel Boulbyd7992232025-03-06 17:09:49 +0000642
Daniel Boulby825ec7e2025-03-06 15:55:09 +0000643 if (intid == vint_id) {
644 vcpu_interrupt_clear_decrement(vcpu_locked, intid);
645 } else {
646 /*
647 * If the interrupt is not the one we wish to remove,
648 * inject it again. We must pop and push all interrupts
649 * to ensure the FIFO ordering is maintained.
650 */
651 vcpu_interrupt_queue_push(vcpu_locked, intid);
Daniel Boulbyd7992232025-03-06 17:09:49 +0000652 }
653 }
654}
Madhukar Pappireddy5388dd92025-02-10 11:24:19 -0600655
656/**
657 * Prepare the target vCPU to run after receiving direct request ABI.
658 */
659void vcpu_dir_req_set_state(struct vcpu_locked target_locked, bool is_ffa_req2,
660 ffa_id_t sender_vm_id, struct ffa_value args)
661{
662 struct vcpu *target_vcpu = target_locked.vcpu;
663
664 target_vcpu->state = VCPU_STATE_RUNNING;
665 target_vcpu->regs_available = false;
666 target_vcpu->direct_request_origin.is_ffa_req2 = is_ffa_req2;
667 target_vcpu->direct_request_origin.vm_id = sender_vm_id;
668 target_vcpu->direct_request_origin.is_framework =
669 ffa_is_framework_msg(args);
670
671 arch_regs_set_retval(&target_vcpu->regs, args);
672}
673
674/**
675 * Clear direct request origin vm_id and request type for the target vCPU.
676 * Also, the scheduling mode and partition runtime model are reset.
677 */
678void vcpu_dir_req_reset_state(struct vcpu_locked vcpu_locked)
679{
680 struct vcpu *vcpu = vcpu_locked.vcpu;
681
682 /* Clear direct request origin vm_id and request type. */
683 vcpu->direct_request_origin.vm_id = HF_INVALID_VM_ID;
684 vcpu->direct_request_origin.is_framework = false;
685
686 /* Reset runtime model and scheduling mode. */
687 vcpu->scheduling_mode = NONE;
688 vcpu->rt_model = RTM_NONE;
689}