Andrew Scull | 1883487 | 2018-10-12 11:48:09 +0100 | [diff] [blame] | 1 | /* |
Andrew Walbran | 692b325 | 2019-03-07 15:51:31 +0000 | [diff] [blame] | 2 | * Copyright 2018 The Hafnium Authors. |
Andrew Scull | 1883487 | 2018-10-12 11:48:09 +0100 | [diff] [blame] | 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * https://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
Wedson Almeida Filho | 987c0ff | 2018-06-20 16:34:38 +0100 | [diff] [blame] | 17 | #include <stdalign.h> |
Wedson Almeida Filho | 987c0ff | 2018-06-20 16:34:38 +0100 | [diff] [blame] | 18 | #include <stddef.h> |
| 19 | |
Andrew Scull | c960c03 | 2018-10-24 15:13:35 +0100 | [diff] [blame] | 20 | #include "hf/arch/init.h" |
| 21 | |
Andrew Scull | 18c78fc | 2018-08-20 12:57:41 +0100 | [diff] [blame] | 22 | #include "hf/api.h" |
| 23 | #include "hf/boot_params.h" |
| 24 | #include "hf/cpio.h" |
| 25 | #include "hf/cpu.h" |
| 26 | #include "hf/dlog.h" |
| 27 | #include "hf/load.h" |
| 28 | #include "hf/mm.h" |
Wedson Almeida Filho | 22d5eaa | 2018-12-16 00:38:49 +0000 | [diff] [blame] | 29 | #include "hf/mpool.h" |
Andrew Scull | a9c172d | 2019-04-03 14:10:00 +0100 | [diff] [blame] | 30 | #include "hf/panic.h" |
Andrew Walbran | 4869936 | 2019-05-20 14:38:00 +0100 | [diff] [blame] | 31 | #include "hf/plat/console.h" |
Andrew Scull | 8d9e121 | 2019-04-05 13:52:55 +0100 | [diff] [blame] | 32 | #include "hf/std.h" |
Andrew Scull | 18c78fc | 2018-08-20 12:57:41 +0100 | [diff] [blame] | 33 | #include "hf/vm.h" |
Wedson Almeida Filho | 987c0ff | 2018-06-20 16:34:38 +0100 | [diff] [blame] | 34 | |
Andrew Scull | 1950326 | 2018-09-20 14:48:39 +0100 | [diff] [blame] | 35 | #include "vmapi/hf/call.h" |
| 36 | |
Wedson Almeida Filho | 9ed8da5 | 2018-12-17 16:09:11 +0000 | [diff] [blame] | 37 | alignas(alignof( |
Wedson Almeida Filho | 22d5eaa | 2018-12-16 00:38:49 +0000 | [diff] [blame] | 38 | struct mm_page_table)) char ptable_buf[sizeof(struct mm_page_table) * |
| 39 | HEAP_PAGES]; |
Wedson Almeida Filho | 987c0ff | 2018-06-20 16:34:38 +0100 | [diff] [blame] | 40 | |
Wedson Almeida Filho | fed6902 | 2018-07-11 15:39:12 +0100 | [diff] [blame] | 41 | /** |
Wedson Almeida Filho | fed6902 | 2018-07-11 15:39:12 +0100 | [diff] [blame] | 42 | * Performs one-time initialisation of the hypervisor. |
| 43 | */ |
Wedson Almeida Filho | 987c0ff | 2018-06-20 16:34:38 +0100 | [diff] [blame] | 44 | static void one_time_init(void) |
| 45 | { |
Wedson Almeida Filho | fdf4afc | 2018-07-19 15:45:21 +0100 | [diff] [blame] | 46 | struct boot_params params; |
| 47 | struct boot_params_update update; |
Wedson Almeida Filho | fdf4afc | 2018-07-19 15:45:21 +0100 | [diff] [blame] | 48 | struct memiter primary_initrd; |
Wedson Almeida Filho | 9ee60e9 | 2018-07-23 18:56:56 +0100 | [diff] [blame] | 49 | struct memiter cpio; |
Andrew Scull | 8087132 | 2018-08-06 12:04:09 +0100 | [diff] [blame] | 50 | void *initrd; |
Andrew Walbran | 34ce72e | 2018-09-13 16:47:44 +0100 | [diff] [blame] | 51 | size_t i; |
Wedson Almeida Filho | 22d5eaa | 2018-12-16 00:38:49 +0000 | [diff] [blame] | 52 | struct mpool ppool; |
Andrew Scull | 3c0a90a | 2019-07-01 11:55:53 +0100 | [diff] [blame^] | 53 | struct mm_stage1_locked mm_stage1_locked; |
Wedson Almeida Filho | fed6902 | 2018-07-11 15:39:12 +0100 | [diff] [blame] | 54 | |
Andrew Walbran | 4869936 | 2019-05-20 14:38:00 +0100 | [diff] [blame] | 55 | /* Make sure the console is initialised before calling dlog. */ |
| 56 | plat_console_init(); |
| 57 | |
Andrew Scull | fdd716e | 2018-12-20 05:37:31 +0000 | [diff] [blame] | 58 | dlog("Initialising hafnium\n"); |
Wedson Almeida Filho | 987c0ff | 2018-06-20 16:34:38 +0100 | [diff] [blame] | 59 | |
Andrew Scull | c960c03 | 2018-10-24 15:13:35 +0100 | [diff] [blame] | 60 | arch_one_time_init(); |
| 61 | |
Wedson Almeida Filho | 22d5eaa | 2018-12-16 00:38:49 +0000 | [diff] [blame] | 62 | mpool_init(&ppool, sizeof(struct mm_page_table)); |
| 63 | mpool_add_chunk(&ppool, ptable_buf, sizeof(ptable_buf)); |
Wedson Almeida Filho | fed6902 | 2018-07-11 15:39:12 +0100 | [diff] [blame] | 64 | |
Wedson Almeida Filho | 22d5eaa | 2018-12-16 00:38:49 +0000 | [diff] [blame] | 65 | if (!mm_init(&ppool)) { |
Wedson Almeida Filho | fdf4afc | 2018-07-19 15:45:21 +0100 | [diff] [blame] | 66 | panic("mm_init failed"); |
Wedson Almeida Filho | fed6902 | 2018-07-11 15:39:12 +0100 | [diff] [blame] | 67 | } |
| 68 | |
Wedson Almeida Filho | 22d5eaa | 2018-12-16 00:38:49 +0000 | [diff] [blame] | 69 | /* Enable locks now that mm is initialised. */ |
Andrew Scull | fdd716e | 2018-12-20 05:37:31 +0000 | [diff] [blame] | 70 | dlog_enable_lock(); |
Wedson Almeida Filho | 22d5eaa | 2018-12-16 00:38:49 +0000 | [diff] [blame] | 71 | mpool_enable_locks(); |
| 72 | |
Andrew Scull | 3c0a90a | 2019-07-01 11:55:53 +0100 | [diff] [blame^] | 73 | mm_stage1_locked = mm_lock_stage1(); |
| 74 | |
| 75 | if (!plat_get_boot_params(mm_stage1_locked, ¶ms, &ppool)) { |
Wedson Almeida Filho | fdf4afc | 2018-07-19 15:45:21 +0100 | [diff] [blame] | 76 | panic("unable to retrieve boot params"); |
| 77 | } |
Wedson Almeida Filho | fed6902 | 2018-07-11 15:39:12 +0100 | [diff] [blame] | 78 | |
Andrew Scull | bb3ab6c | 2018-11-26 20:38:49 +0000 | [diff] [blame] | 79 | cpu_module_init(params.cpu_ids, params.cpu_count); |
| 80 | |
Andrew Walbran | 34ce72e | 2018-09-13 16:47:44 +0100 | [diff] [blame] | 81 | for (i = 0; i < params.mem_ranges_count; ++i) { |
| 82 | dlog("Memory range: 0x%x - 0x%x\n", |
| 83 | pa_addr(params.mem_ranges[i].begin), |
| 84 | pa_addr(params.mem_ranges[i].end) - 1); |
| 85 | } |
| 86 | |
Andrew Scull | 265ada9 | 2018-07-30 15:19:01 +0100 | [diff] [blame] | 87 | dlog("Ramdisk range: 0x%x - 0x%x\n", pa_addr(params.initrd_begin), |
| 88 | pa_addr(params.initrd_end) - 1); |
Wedson Almeida Filho | fed6902 | 2018-07-11 15:39:12 +0100 | [diff] [blame] | 89 | |
Wedson Almeida Filho | fdf4afc | 2018-07-19 15:45:21 +0100 | [diff] [blame] | 90 | /* Map initrd in, and initialise cpio parser. */ |
Andrew Scull | 3c0a90a | 2019-07-01 11:55:53 +0100 | [diff] [blame^] | 91 | initrd = mm_identity_map(mm_stage1_locked, params.initrd_begin, |
| 92 | params.initrd_end, MM_MODE_R, &ppool); |
Andrew Scull | 8087132 | 2018-08-06 12:04:09 +0100 | [diff] [blame] | 93 | if (!initrd) { |
Wedson Almeida Filho | fdf4afc | 2018-07-19 15:45:21 +0100 | [diff] [blame] | 94 | panic("unable to map initrd in"); |
| 95 | } |
Wedson Almeida Filho | fed6902 | 2018-07-11 15:39:12 +0100 | [diff] [blame] | 96 | |
Andrew Scull | 8087132 | 2018-08-06 12:04:09 +0100 | [diff] [blame] | 97 | memiter_init(&cpio, initrd, |
Andrew Walbran | 2cb4339 | 2019-04-17 12:52:45 +0100 | [diff] [blame] | 98 | pa_difference(params.initrd_begin, params.initrd_end)); |
Wedson Almeida Filho | fed6902 | 2018-07-11 15:39:12 +0100 | [diff] [blame] | 99 | |
Wedson Almeida Filho | fdf4afc | 2018-07-19 15:45:21 +0100 | [diff] [blame] | 100 | /* Load all VMs. */ |
Andrew Scull | 3c0a90a | 2019-07-01 11:55:53 +0100 | [diff] [blame^] | 101 | if (!load_primary(mm_stage1_locked, &cpio, params.kernel_arg, |
| 102 | &primary_initrd, &ppool)) { |
Wedson Almeida Filho | fdf4afc | 2018-07-19 15:45:21 +0100 | [diff] [blame] | 103 | panic("unable to load primary VM"); |
| 104 | } |
Wedson Almeida Filho | fed6902 | 2018-07-11 15:39:12 +0100 | [diff] [blame] | 105 | |
Wedson Almeida Filho | b2c159e | 2018-10-25 13:27:47 +0100 | [diff] [blame] | 106 | /* |
| 107 | * load_secondary will add regions assigned to the secondary VMs from |
| 108 | * mem_ranges to reserved_ranges. |
| 109 | */ |
Andrew Walbran | 34ce72e | 2018-09-13 16:47:44 +0100 | [diff] [blame] | 110 | update.initrd_begin = pa_from_va(va_from_ptr(primary_initrd.next)); |
| 111 | update.initrd_end = pa_from_va(va_from_ptr(primary_initrd.limit)); |
Andrew Walbran | 34ce72e | 2018-09-13 16:47:44 +0100 | [diff] [blame] | 112 | update.reserved_ranges_count = 0; |
Andrew Scull | 3c0a90a | 2019-07-01 11:55:53 +0100 | [diff] [blame^] | 113 | if (!load_secondary(mm_stage1_locked, &cpio, ¶ms, &update, |
| 114 | &ppool)) { |
Wedson Almeida Filho | 84a30a0 | 2018-07-23 20:05:05 +0100 | [diff] [blame] | 115 | panic("unable to load secondary VMs"); |
| 116 | } |
| 117 | |
Andrew Walbran | 34ce72e | 2018-09-13 16:47:44 +0100 | [diff] [blame] | 118 | /* Prepare to run by updating bootparams as seen by primary VM. */ |
Andrew Scull | 3c0a90a | 2019-07-01 11:55:53 +0100 | [diff] [blame^] | 119 | if (!plat_update_boot_params(mm_stage1_locked, &update, &ppool)) { |
Wedson Almeida Filho | fdf4afc | 2018-07-19 15:45:21 +0100 | [diff] [blame] | 120 | panic("plat_update_boot_params failed"); |
| 121 | } |
Wedson Almeida Filho | 987c0ff | 2018-06-20 16:34:38 +0100 | [diff] [blame] | 122 | |
Andrew Scull | 3c0a90a | 2019-07-01 11:55:53 +0100 | [diff] [blame^] | 123 | mm_defrag(mm_stage1_locked, &ppool); |
| 124 | mm_unlock_stage1(&mm_stage1_locked); |
Wedson Almeida Filho | 22d5eaa | 2018-12-16 00:38:49 +0000 | [diff] [blame] | 125 | |
| 126 | /* Initialise the API page pool. ppool will be empty from now on. */ |
| 127 | api_init(&ppool); |
Wedson Almeida Filho | fed6902 | 2018-07-11 15:39:12 +0100 | [diff] [blame] | 128 | |
Andrew Scull | da24197 | 2019-01-05 18:17:48 +0000 | [diff] [blame] | 129 | /* Enable TLB invalidation for VM page table updates. */ |
| 130 | mm_vm_enable_invalidation(); |
| 131 | |
Wedson Almeida Filho | fdf4afc | 2018-07-19 15:45:21 +0100 | [diff] [blame] | 132 | dlog("Hafnium initialisation completed\n"); |
Wedson Almeida Filho | 987c0ff | 2018-06-20 16:34:38 +0100 | [diff] [blame] | 133 | } |
| 134 | |
Wedson Almeida Filho | fed6902 | 2018-07-11 15:39:12 +0100 | [diff] [blame] | 135 | /** |
Wedson Almeida Filho | 987c0ff | 2018-06-20 16:34:38 +0100 | [diff] [blame] | 136 | * The entry point of CPUs when they are turned on. It is supposed to initialise |
Wedson Almeida Filho | 8700964 | 2018-07-02 10:20:07 +0100 | [diff] [blame] | 137 | * all state and return the first vCPU to run. |
Wedson Almeida Filho | 987c0ff | 2018-06-20 16:34:38 +0100 | [diff] [blame] | 138 | */ |
Wedson Almeida Filho | 00df6c7 | 2018-10-18 11:19:24 +0100 | [diff] [blame] | 139 | struct vcpu *cpu_main(struct cpu *c) |
Wedson Almeida Filho | 987c0ff | 2018-06-20 16:34:38 +0100 | [diff] [blame] | 140 | { |
Wedson Almeida Filho | 00df6c7 | 2018-10-18 11:19:24 +0100 | [diff] [blame] | 141 | struct vcpu *vcpu; |
Andrew Scull | c960c03 | 2018-10-24 15:13:35 +0100 | [diff] [blame] | 142 | struct vm *vm; |
Wedson Almeida Filho | 8700964 | 2018-07-02 10:20:07 +0100 | [diff] [blame] | 143 | |
Wedson Almeida Filho | fdf4afc | 2018-07-19 15:45:21 +0100 | [diff] [blame] | 144 | /* |
| 145 | * Do global one-time initialisation just once. We avoid using atomics |
| 146 | * by only touching the variable from cpu 0. |
| 147 | */ |
Wedson Almeida Filho | fed6902 | 2018-07-11 15:39:12 +0100 | [diff] [blame] | 148 | static volatile bool inited = false; |
Wedson Almeida Filho | 81568c4 | 2019-01-04 13:33:02 +0000 | [diff] [blame] | 149 | |
Wedson Almeida Filho | fed6902 | 2018-07-11 15:39:12 +0100 | [diff] [blame] | 150 | if (cpu_index(c) == 0 && !inited) { |
| 151 | inited = true; |
Wedson Almeida Filho | 987c0ff | 2018-06-20 16:34:38 +0100 | [diff] [blame] | 152 | one_time_init(); |
Wedson Almeida Filho | fed6902 | 2018-07-11 15:39:12 +0100 | [diff] [blame] | 153 | } |
Wedson Almeida Filho | 987c0ff | 2018-06-20 16:34:38 +0100 | [diff] [blame] | 154 | |
Wedson Almeida Filho | 03e767a | 2018-07-30 15:32:03 +0100 | [diff] [blame] | 155 | if (!mm_cpu_init()) { |
| 156 | panic("mm_cpu_init failed"); |
| 157 | } |
| 158 | |
Andrew Walbran | 42347a9 | 2019-05-09 13:59:03 +0100 | [diff] [blame] | 159 | vcpu = vm_get_vcpu(vm_find(HF_PRIMARY_VM_ID), cpu_index(c)); |
Andrew Scull | c960c03 | 2018-10-24 15:13:35 +0100 | [diff] [blame] | 160 | vm = vcpu->vm; |
Wedson Almeida Filho | 00df6c7 | 2018-10-18 11:19:24 +0100 | [diff] [blame] | 161 | vcpu->cpu = c; |
Andrew Scull | c960c03 | 2018-10-24 15:13:35 +0100 | [diff] [blame] | 162 | |
| 163 | /* Reset the registers to give a clean start for the primary's vCPU. */ |
Andrew Scull | bb3ab6c | 2018-11-26 20:38:49 +0000 | [diff] [blame] | 164 | arch_regs_reset(&vcpu->regs, true, vm->id, c->id, vm->ptable.root); |
Andrew Scull | c960c03 | 2018-10-24 15:13:35 +0100 | [diff] [blame] | 165 | |
Wedson Almeida Filho | 00df6c7 | 2018-10-18 11:19:24 +0100 | [diff] [blame] | 166 | return vcpu; |
Wedson Almeida Filho | 987c0ff | 2018-06-20 16:34:38 +0100 | [diff] [blame] | 167 | } |