blob: daeb1a0f22f39fb39ff87a8a514547ee63354bf2 [file] [log] [blame]
Andrew Scull18834872018-10-12 11:48:09 +01001/*
Andrew Walbran692b3252019-03-07 15:51:31 +00002 * Copyright 2018 The Hafnium Authors.
Andrew Scull18834872018-10-12 11:48:09 +01003 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * https://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
Andrew Scull18c78fc2018-08-20 12:57:41 +010017#include "hf/cpu.h"
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +010018
Andrew Scull04502e42018-09-03 14:54:52 +010019#include <stdalign.h>
20
Andrew Scull18c78fc2018-08-20 12:57:41 +010021#include "hf/arch/cpu.h"
22
23#include "hf/api.h"
Andrew Scull877ae4b2019-07-02 12:52:33 +010024#include "hf/check.h"
Andrew Scull18c78fc2018-08-20 12:57:41 +010025#include "hf/dlog.h"
Andrew Walbranb037d5b2019-06-25 17:19:41 +010026#include "hf/spci.h"
Andrew Scull8d9e1212019-04-05 13:52:55 +010027#include "hf/std.h"
Andrew Scull18c78fc2018-08-20 12:57:41 +010028#include "hf/vm.h"
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +010029
Andrew Scull19503262018-09-20 14:48:39 +010030#include "vmapi/hf/call.h"
31
Andrew Scull23e93a82018-10-26 14:56:04 +010032#define STACK_SIZE PAGE_SIZE
33
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +010034/* The stack to be used by the CPUs. */
Andrew Scull37402872018-10-24 14:23:06 +010035alignas(2 * sizeof(uintreg_t)) static char callstacks[MAX_CPUS][STACK_SIZE];
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +010036
37/* State of all supported CPUs. The stack of the first one is initialized. */
38struct cpu cpus[MAX_CPUS] = {
39 {
40 .is_on = 1,
Andrew Scullf3d45592018-09-20 14:30:22 +010041 .stack_bottom = &callstacks[0][STACK_SIZE],
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +010042 },
43};
44
Andrew Scullbb3ab6c2018-11-26 20:38:49 +000045static uint32_t cpu_count = 1;
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +010046
Andrew Scullbb3ab6c2018-11-26 20:38:49 +000047static void cpu_init(struct cpu *c)
48{
49 /* TODO: Assumes that c is zeroed out already. */
50 sl_init(&c->lock);
51 c->irq_disable_count = 1;
52}
53
54void cpu_module_init(const uint64_t *cpu_ids, size_t count)
55{
56 uint32_t i;
57 uint32_t j;
58 uint64_t boot_cpu_id = cpus[0].id;
59 bool found_boot_cpu = false;
60
61 cpu_count = count;
62
63 /*
64 * Initialize CPUs with the IDs from the configuration passed in. The
65 * CPUs after the boot CPU are initialized in reverse order. The boot
66 * CPU is initialized when it is found or in place of the last CPU if it
67 * is not found.
68 */
69 j = cpu_count;
70 for (i = 0; i < cpu_count; ++i) {
71 struct cpu *c;
72 uint64_t id = cpu_ids[i];
73
74 if (found_boot_cpu || id != boot_cpu_id) {
75 c = &cpus[--j];
76 } else {
77 found_boot_cpu = true;
78 c = &cpus[0];
79 }
Wedson Almeida Filho81568c42019-01-04 13:33:02 +000080
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +010081 cpu_init(c);
Andrew Scullbb3ab6c2018-11-26 20:38:49 +000082 c->id = id;
Andrew Scullf3d45592018-09-20 14:30:22 +010083 c->stack_bottom = &callstacks[i][STACK_SIZE];
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +010084 }
Andrew Scullbb3ab6c2018-11-26 20:38:49 +000085
86 if (!found_boot_cpu) {
87 /* Boot CPU was initialized but with wrong ID. */
88 dlog("Boot CPU's ID not found in config.");
89 cpus[0].id = boot_cpu_id;
90 }
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +010091}
92
93size_t cpu_index(struct cpu *c)
94{
Wedson Almeida Filho03e767a2018-07-30 15:32:03 +010095 return c - cpus;
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +010096}
97
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +010098void cpu_irq_enable(struct cpu *c)
99{
100 c->irq_disable_count--;
Andrew Scull7364a8e2018-07-19 15:39:29 +0100101 if (!c->irq_disable_count) {
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +0100102 arch_irq_enable();
Andrew Scull7364a8e2018-07-19 15:39:29 +0100103 }
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +0100104}
105
106void cpu_irq_disable(struct cpu *c)
107{
Andrew Scull7364a8e2018-07-19 15:39:29 +0100108 if (!c->irq_disable_count) {
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +0100109 arch_irq_disable();
Andrew Scull7364a8e2018-07-19 15:39:29 +0100110 }
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +0100111 c->irq_disable_count++;
112}
113
Wedson Almeida Filho87009642018-07-02 10:20:07 +0100114/**
115 * Turns CPU on and returns the previous state.
116 */
Andrew Scull37402872018-10-24 14:23:06 +0100117bool cpu_on(struct cpu *c, ipaddr_t entry, uintreg_t arg)
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +0100118{
Wedson Almeida Filho87009642018-07-02 10:20:07 +0100119 bool prev;
120
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +0100121 sl_lock(&c->lock);
Wedson Almeida Filho87009642018-07-02 10:20:07 +0100122 prev = c->is_on;
123 c->is_on = true;
124 sl_unlock(&c->lock);
125
126 if (!prev) {
Andrew Walbran42347a92019-05-09 13:59:03 +0100127 struct vm *vm = vm_find(HF_PRIMARY_VM_ID);
Andrew Walbrane1310df2019-04-29 17:28:28 +0100128 struct vcpu *vcpu = vm_get_vcpu(vm, cpu_index(c));
Andrew Walbranb58f8992019-04-15 12:29:31 +0100129 struct vcpu_locked vcpu_locked;
Wedson Almeida Filho81568c42019-01-04 13:33:02 +0000130
Andrew Walbranb58f8992019-04-15 12:29:31 +0100131 vcpu_locked = vcpu_lock(vcpu);
132 vcpu_on(vcpu_locked, entry, arg);
133 vcpu_unlock(&vcpu_locked);
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +0100134 }
Wedson Almeida Filho87009642018-07-02 10:20:07 +0100135
136 return prev;
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +0100137}
138
Wedson Almeida Filho03e767a2018-07-30 15:32:03 +0100139/**
140 * Prepares the CPU for turning itself off.
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +0100141 */
142void cpu_off(struct cpu *c)
143{
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +0100144 sl_lock(&c->lock);
Wedson Almeida Filho87009642018-07-02 10:20:07 +0100145 c->is_on = false;
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +0100146 sl_unlock(&c->lock);
Wedson Almeida Filho03e767a2018-07-30 15:32:03 +0100147}
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +0100148
Wedson Almeida Filho03e767a2018-07-30 15:32:03 +0100149/**
150 * Searches for a CPU based on its id.
151 */
Andrew Scullbb3ab6c2018-11-26 20:38:49 +0000152struct cpu *cpu_find(uint64_t id)
Wedson Almeida Filho03e767a2018-07-30 15:32:03 +0100153{
154 size_t i;
155
Andrew Scullbb3ab6c2018-11-26 20:38:49 +0000156 for (i = 0; i < cpu_count; i++) {
Wedson Almeida Filho03e767a2018-07-30 15:32:03 +0100157 if (cpus[i].id == id) {
Andrew Scullf3d45592018-09-20 14:30:22 +0100158 return &cpus[i];
Wedson Almeida Filho03e767a2018-07-30 15:32:03 +0100159 }
160 }
161
162 return NULL;
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +0100163}
164
Andrew Walbranb58f8992019-04-15 12:29:31 +0100165/**
166 * Locks the given vCPU and updates `locked` to hold the newly locked vCPU.
167 */
168struct vcpu_locked vcpu_lock(struct vcpu *vcpu)
169{
170 struct vcpu_locked locked = {
171 .vcpu = vcpu,
172 };
173
174 sl_lock(&vcpu->lock);
175
176 return locked;
177}
178
179/**
180 * Unlocks a vCPU previously locked with vpu_lock, and updates `locked` to
181 * reflect the fact that the vCPU is no longer locked.
182 */
183void vcpu_unlock(struct vcpu_locked *locked)
184{
185 sl_unlock(&locked->vcpu->lock);
186 locked->vcpu = NULL;
187}
188
Wedson Almeida Filho87009642018-07-02 10:20:07 +0100189void vcpu_init(struct vcpu *vcpu, struct vm *vm)
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +0100190{
Andrew Scull2b5fbad2019-04-05 13:55:56 +0100191 memset_s(vcpu, sizeof(*vcpu), 0, sizeof(*vcpu));
Wedson Almeida Filho87009642018-07-02 10:20:07 +0100192 sl_init(&vcpu->lock);
Wedson Almeida Filho03306112018-11-26 00:08:03 +0000193 vcpu->regs_available = true;
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +0100194 vcpu->vm = vm;
Andrew Sculld6ee1102019-04-05 22:12:42 +0100195 vcpu->state = VCPU_STATE_OFF;
Wedson Almeida Filho987c0ff2018-06-20 16:34:38 +0100196}
Wedson Almeida Filho87009642018-07-02 10:20:07 +0100197
Andrew Walbranb58f8992019-04-15 12:29:31 +0100198/**
199 * Initialise the registers for the given vCPU and set the state to
200 * VCPU_STATE_READY. The caller must hold the vCPU lock while calling this.
201 */
202void vcpu_on(struct vcpu_locked vcpu, ipaddr_t entry, uintreg_t arg)
Wedson Almeida Filho87009642018-07-02 10:20:07 +0100203{
Andrew Walbranb58f8992019-04-15 12:29:31 +0100204 arch_regs_set_pc_arg(&vcpu.vcpu->regs, entry, arg);
205 vcpu.vcpu->state = VCPU_STATE_READY;
Wedson Almeida Filho87009642018-07-02 10:20:07 +0100206}
207
Andrew Walbranb037d5b2019-06-25 17:19:41 +0100208spci_vcpu_index_t vcpu_index(const struct vcpu *vcpu)
Wedson Almeida Filho9d5040f2018-10-29 08:41:27 +0000209{
Andrew Walbranb037d5b2019-06-25 17:19:41 +0100210 size_t index = vcpu - vcpu->vm->vcpus;
211
Andrew Scull877ae4b2019-07-02 12:52:33 +0100212 CHECK(index < UINT16_MAX);
Andrew Walbranb037d5b2019-06-25 17:19:41 +0100213 return index;
Wedson Almeida Filho9d5040f2018-10-29 08:41:27 +0000214}
Wedson Almeida Filho99d2d4c2019-02-14 12:53:46 +0000215
216/**
Andrew Walbran33645652019-04-15 12:29:31 +0100217 * Check whether the given vcpu_state is an off state, for the purpose of
218 * turning vCPUs on and off. Note that aborted still counts as on in this
219 * context.
Andrew Walbran9a43fee2019-04-18 17:42:32 +0100220 */
Andrew Walbran33645652019-04-15 12:29:31 +0100221bool vcpu_is_off(struct vcpu_locked vcpu)
222{
223 switch (vcpu.vcpu->state) {
224 case VCPU_STATE_OFF:
225 return true;
226 case VCPU_STATE_READY:
227 case VCPU_STATE_RUNNING:
228 case VCPU_STATE_BLOCKED_MAILBOX:
229 case VCPU_STATE_BLOCKED_INTERRUPT:
230 case VCPU_STATE_ABORTED:
231 /*
232 * Aborted still counts as ON for the purposes of PSCI,
233 * because according to the PSCI specification (section
234 * 5.7.1) a core is only considered to be off if it has
235 * been turned off with a CPU_OFF call or hasn't yet
236 * been turned on with a CPU_ON call.
237 */
238 return false;
239 }
240}
241
242/**
243 * Starts a vCPU of a secondary VM.
244 *
245 * Returns true if the secondary was reset and started, or false if it was
246 * already on and so nothing was done.
247 */
248bool vcpu_secondary_reset_and_start(struct vcpu *vcpu, ipaddr_t entry,
Andrew Walbran9a43fee2019-04-18 17:42:32 +0100249 uintreg_t arg)
250{
Andrew Walbranb58f8992019-04-15 12:29:31 +0100251 struct vcpu_locked vcpu_locked;
Andrew Walbran9a43fee2019-04-18 17:42:32 +0100252 struct vm *vm = vcpu->vm;
Andrew Walbran33645652019-04-15 12:29:31 +0100253 bool vcpu_was_off;
Andrew Walbran9a43fee2019-04-18 17:42:32 +0100254
Andrew Scull877ae4b2019-07-02 12:52:33 +0100255 CHECK(vm->id != HF_PRIMARY_VM_ID);
Andrew Walbran9a43fee2019-04-18 17:42:32 +0100256
Andrew Walbranb58f8992019-04-15 12:29:31 +0100257 vcpu_locked = vcpu_lock(vcpu);
Andrew Walbran33645652019-04-15 12:29:31 +0100258 vcpu_was_off = vcpu_is_off(vcpu_locked);
259 if (vcpu_was_off) {
260 /*
261 * Set vCPU registers to a clean state ready for boot. As this
262 * is a secondary which can migrate between pCPUs, the ID of the
263 * vCPU is defined as the index and does not match the ID of the
264 * pCPU it is running on.
265 */
266 arch_regs_reset(&vcpu->regs, false, vm->id, vcpu_index(vcpu),
267 vm->ptable.root);
268 vcpu_on(vcpu_locked, entry, arg);
269 }
Andrew Walbranb58f8992019-04-15 12:29:31 +0100270 vcpu_unlock(&vcpu_locked);
Andrew Walbran33645652019-04-15 12:29:31 +0100271
272 return vcpu_was_off;
Andrew Walbran9a43fee2019-04-18 17:42:32 +0100273}
274
275/**
Wedson Almeida Filho99d2d4c2019-02-14 12:53:46 +0000276 * Handles a page fault. It does so by determining if it's a legitimate or
277 * spurious fault, and recovering from the latter.
278 *
279 * Returns true if the caller should resume the current vcpu, or false if its VM
280 * should be aborted.
281 */
282bool vcpu_handle_page_fault(const struct vcpu *current,
283 struct vcpu_fault_info *f)
284{
285 struct vm *vm = current->vm;
Wedson Almeida Filho99d2d4c2019-02-14 12:53:46 +0000286 int mode;
287 int mask = f->mode | MM_MODE_INVALID;
Andrew Sculld3cfaad2019-04-04 11:34:10 +0100288 bool resume;
Wedson Almeida Filho99d2d4c2019-02-14 12:53:46 +0000289
290 sl_lock(&vm->lock);
291
292 /*
293 * Check if this is a legitimate fault, i.e., if the page table doesn't
294 * allow the access attemped by the VM.
Andrew Sculld3cfaad2019-04-04 11:34:10 +0100295 *
296 * Otherwise, this is a spurious fault, likely because another CPU is
297 * updating the page table. It is responsible for issuing global TLB
298 * invalidations while holding the VM lock, so we don't need to do
299 * anything else to recover from it. (Acquiring/releasing the lock
300 * ensured that the invalidations have completed.)
Wedson Almeida Filho99d2d4c2019-02-14 12:53:46 +0000301 */
Andrew Sculld3cfaad2019-04-04 11:34:10 +0100302 resume = mm_vm_get_mode(&vm->ptable, f->ipaddr, ipa_add(f->ipaddr, 1),
303 &mode) &&
304 (mode & mask) == f->mode;
Wedson Almeida Filho99d2d4c2019-02-14 12:53:46 +0000305
Wedson Almeida Filho99d2d4c2019-02-14 12:53:46 +0000306 sl_unlock(&vm->lock);
Andrew Sculld3cfaad2019-04-04 11:34:10 +0100307
308 if (!resume) {
Wedson Almeida Filho99d2d4c2019-02-14 12:53:46 +0000309 dlog("Stage-2 page fault: pc=0x%x, vmid=%u, vcpu=%u, "
Andrew Sculld3cfaad2019-04-04 11:34:10 +0100310 "vaddr=0x%x, ipaddr=0x%x, mode=0x%x\n",
Wedson Almeida Filho99d2d4c2019-02-14 12:53:46 +0000311 f->pc, vm->id, vcpu_index(current), f->vaddr, f->ipaddr,
Andrew Sculld3cfaad2019-04-04 11:34:10 +0100312 f->mode);
Wedson Almeida Filho99d2d4c2019-02-14 12:53:46 +0000313 }
Andrew Sculld3cfaad2019-04-04 11:34:10 +0100314
315 return resume;
Wedson Almeida Filho99d2d4c2019-02-14 12:53:46 +0000316}