blob: d105f7279e717751175a5e2190f43f98ae46dfbb [file] [log] [blame]
Andrew Scull18834872018-10-12 11:48:09 +01001/*
Andrew Walbran692b3252019-03-07 15:51:31 +00002 * Copyright 2018 The Hafnium Authors.
Andrew Scull18834872018-10-12 11:48:09 +01003 *
Andrew Walbrane959ec12020-06-17 15:01:09 +01004 * Use of this source code is governed by a BSD-style
5 * license that can be found in the LICENSE file or at
6 * https://opensource.org/licenses/BSD-3-Clause.
Andrew Scull18834872018-10-12 11:48:09 +01007 */
8
Andrew Scull18c78fc2018-08-20 12:57:41 +01009#include "hf/load.h"
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +010010
11#include <stdbool.h>
12
Maksims Svecovs134b8f92022-03-04 15:14:09 +000013#include "hf/arch/init.h"
Olivier Deprez112d2b52020-09-30 07:39:23 +020014#include "hf/arch/other_world.h"
J-Alvesa9c7cba2021-08-25 16:26:11 +010015#include "hf/arch/plat/ffa.h"
Fuad Tabba77a4b012019-11-15 12:13:08 +000016#include "hf/arch/vm.h"
17
Andrew Scull18c78fc2018-08-20 12:57:41 +010018#include "hf/api.h"
Andrew Walbran34ce72e2018-09-13 16:47:44 +010019#include "hf/boot_params.h"
Andrew Scull72b43c02019-09-18 13:53:45 +010020#include "hf/check.h"
Andrew Scull18c78fc2018-08-20 12:57:41 +010021#include "hf/dlog.h"
Fuad Tabba50469e02020-06-30 15:14:28 +010022#include "hf/fdt_patch.h"
Andrew Scull5991ec92018-10-08 14:55:02 +010023#include "hf/layout.h"
Andrew Scull18c78fc2018-08-20 12:57:41 +010024#include "hf/memiter.h"
25#include "hf/mm.h"
Andrew Walbran48699362019-05-20 14:38:00 +010026#include "hf/plat/console.h"
Madhukar Pappireddy5fc8be12021-08-03 11:42:53 -050027#include "hf/plat/interrupts.h"
Andrew Scullb1a6d0d2020-01-29 11:25:12 +000028#include "hf/plat/iommu.h"
Andrew Scull877ae4b2019-07-02 12:52:33 +010029#include "hf/static_assert.h"
Andrew Scull8d9e1212019-04-05 13:52:55 +010030#include "hf/std.h"
Andrew Scull18c78fc2018-08-20 12:57:41 +010031#include "hf/vm.h"
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +010032
Andrew Scull19503262018-09-20 14:48:39 +010033#include "vmapi/hf/call.h"
Manish Pandeyd34f8892020-06-19 17:41:07 +010034#include "vmapi/hf/ffa.h"
Andrew Scull19503262018-09-20 14:48:39 +010035
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +010036/**
37 * Copies data to an unmapped location by mapping it for write, copying the
38 * data, then unmapping it.
Andrew Sculld9225b32018-11-19 16:12:41 +000039 *
40 * The data is written so that it is available to all cores with the cache
41 * disabled. When switching to the partitions, the caching is initially disabled
42 * so the data must be available without the cache.
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +010043 */
Andrew Scull3c0a90a2019-07-01 11:55:53 +010044static bool copy_to_unmapped(struct mm_stage1_locked stage1_locked, paddr_t to,
David Brazdil7a462ec2019-08-15 12:27:47 +010045 struct memiter *from_it, struct mpool *ppool)
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +010046{
David Brazdil7a462ec2019-08-15 12:27:47 +010047 const void *from = memiter_base(from_it);
48 size_t size = memiter_size(from_it);
Andrew Scull80871322018-08-06 12:04:09 +010049 paddr_t to_end = pa_add(to, size);
50 void *ptr;
Andrew Scull265ada92018-07-30 15:19:01 +010051
Andrew Scull3c0a90a2019-07-01 11:55:53 +010052 ptr = mm_identity_map(stage1_locked, to, to_end, MM_MODE_W, ppool);
Andrew Scull80871322018-08-06 12:04:09 +010053 if (!ptr) {
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +010054 return false;
55 }
56
Andrew Sculla1aa2ba2019-04-05 11:49:02 +010057 memcpy_s(ptr, size, from, size);
Andrew Scullc059fbe2019-09-12 12:58:40 +010058 arch_mm_flush_dcache(ptr, size);
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +010059
Andrew Scull72b43c02019-09-18 13:53:45 +010060 CHECK(mm_unmap(stage1_locked, to, to_end, ppool));
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +010061
62 return true;
63}
64
Fuad Tabba50469e02020-06-30 15:14:28 +010065/**
66 * Loads the secondary VM's kernel.
67 * Stores the kernel size in kernel_size (if kernel_size is not NULL).
68 * Returns false if it cannot load the kernel.
69 */
Andrew Scull72b43c02019-09-18 13:53:45 +010070static bool load_kernel(struct mm_stage1_locked stage1_locked, paddr_t begin,
71 paddr_t end, const struct manifest_vm *manifest_vm,
Fuad Tabba50469e02020-06-30 15:14:28 +010072 const struct memiter *cpio, struct mpool *ppool,
73 size_t *kernel_size)
Andrew Scull72b43c02019-09-18 13:53:45 +010074{
Andrew Scull72b43c02019-09-18 13:53:45 +010075 struct memiter kernel;
Fuad Tabba50469e02020-06-30 15:14:28 +010076 size_t size;
Andrew Scull72b43c02019-09-18 13:53:45 +010077
David Brazdil136f2942019-09-23 14:11:03 +010078 if (!cpio_get_file(cpio, &manifest_vm->kernel_filename, &kernel)) {
Andrew Walbran17eebf92020-02-05 16:35:49 +000079 dlog_error("Could not find kernel file \"%s\".\n",
80 string_data(&manifest_vm->kernel_filename));
Andrew Scull72b43c02019-09-18 13:53:45 +010081 return false;
82 }
83
Fuad Tabba50469e02020-06-30 15:14:28 +010084 size = memiter_size(&kernel);
85 if (pa_difference(begin, end) < size) {
Andrew Walbran17eebf92020-02-05 16:35:49 +000086 dlog_error("Kernel is larger than available memory.\n");
Andrew Scull72b43c02019-09-18 13:53:45 +010087 return false;
88 }
89
90 if (!copy_to_unmapped(stage1_locked, begin, &kernel, ppool)) {
Andrew Walbran17eebf92020-02-05 16:35:49 +000091 dlog_error("Unable to copy kernel.\n");
Andrew Scull72b43c02019-09-18 13:53:45 +010092 return false;
93 }
94
Fuad Tabba50469e02020-06-30 15:14:28 +010095 if (kernel_size) {
96 *kernel_size = size;
97 }
98
Andrew Scull72b43c02019-09-18 13:53:45 +010099 return true;
100}
101
Manish Pandeyd34f8892020-06-19 17:41:07 +0100102/*
103 * Link RX/TX buffers provided in partition manifest to mailbox
104 */
105static bool link_rxtx_to_mailbox(struct mm_stage1_locked stage1_locked,
106 struct vm_locked vm_locked, struct rx_tx rxtx,
107 struct mpool *ppool)
108{
109 struct ffa_value ret;
110 ipaddr_t send;
111 ipaddr_t recv;
112 uint32_t page_count;
113
114 send = ipa_init(rxtx.tx_buffer->base_address);
115 recv = ipa_init(rxtx.rx_buffer->base_address);
116 page_count = rxtx.tx_buffer->page_count;
117
118 ret = api_vm_configure_pages(stage1_locked, vm_locked, send, recv,
119 page_count, ppool);
120 if (ret.func != FFA_SUCCESS_32) {
121 return false;
122 }
123
124 dlog_verbose(" mailbox: send = %#x, recv = %#x\n",
125 vm_locked.vm->mailbox.send, vm_locked.vm->mailbox.recv);
126
127 return true;
128}
129
Raghu Krishnamurthyad38a9c2022-07-20 07:30:36 -0700130static void infer_interrupt(struct interrupt_info interrupt,
Madhukar Pappireddy5fc8be12021-08-03 11:42:53 -0500131 struct interrupt_descriptor *int_desc)
132{
133 uint32_t attr = interrupt.attributes;
134
135 interrupt_desc_set_id(int_desc, interrupt.id);
136 interrupt_desc_set_priority(int_desc,
137 (attr >> INT_DESC_PRIORITY_SHIFT) & 0xff);
138
139 /* Refer to the comments in interrupt_descriptor struct definition. */
140 interrupt_desc_set_type_config_sec_state(
141 int_desc,
142 (((attr >> INT_DESC_TYPE_SHIFT) & 0x3) << 2) |
143 (((attr >> INT_DESC_CONFIG_SHIFT) & 0x1) << 1) |
144 ((attr >> INT_DESC_SEC_STATE_SHIFT) & 0x1));
145
146 interrupt_desc_set_valid(int_desc, true);
147}
148
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +0100149/**
Andrew Scullae9962e2019-10-03 16:51:16 +0100150 * Performs VM loading activities that are common between the primary and
151 * secondaries.
152 */
Manish Pandeyd34f8892020-06-19 17:41:07 +0100153static bool load_common(struct mm_stage1_locked stage1_locked,
154 struct vm_locked vm_locked,
155 const struct manifest_vm *manifest_vm,
156 struct mpool *ppool)
Andrew Scullae9962e2019-10-03 16:51:16 +0100157{
Madhukar Pappireddy5fc8be12021-08-03 11:42:53 -0500158 struct device_region dev_region;
Raghu Krishnamurthyad38a9c2022-07-20 07:30:36 -0700159 struct interrupt_info interrupt;
Madhukar Pappireddy5fc8be12021-08-03 11:42:53 -0500160 uint32_t k = 0;
161
Manish Pandeyd34f8892020-06-19 17:41:07 +0100162 vm_locked.vm->smc_whitelist = manifest_vm->smc_whitelist;
Raghu Krishnamurthy8c250a92021-07-02 12:16:42 -0700163 vm_locked.vm->uuid = manifest_vm->partition.uuid;
Andrew Scullae9962e2019-10-03 16:51:16 +0100164
Madhukar Pappireddy5fc8be12021-08-03 11:42:53 -0500165 /* Populate the interrupt descriptor for current VM. */
Raghu Krishnamurthy641dcd82022-07-19 23:21:20 -0700166 for (uint16_t i = 0; i < PARTITION_MAX_DEVICE_REGIONS; i++) {
Madhukar Pappireddy5fc8be12021-08-03 11:42:53 -0500167 dev_region = manifest_vm->partition.dev_regions[i];
168
169 CHECK(dev_region.interrupt_count <=
Raghu Krishnamurthy641dcd82022-07-19 23:21:20 -0700170 PARTITION_MAX_INTERRUPTS_PER_DEVICE);
Madhukar Pappireddy5fc8be12021-08-03 11:42:53 -0500171
172 for (uint8_t j = 0; j < dev_region.interrupt_count; j++) {
173 struct interrupt_descriptor int_desc;
174
175 interrupt = dev_region.interrupts[j];
176 infer_interrupt(interrupt, &int_desc);
177 vm_locked.vm->interrupt_desc[k] = int_desc;
178
Madhukar Pappireddy72454a12021-08-03 12:21:46 -0500179 /*
180 * Configure the physical interrupts allocated for this
181 * VM in its partition manifest.
182 */
183 plat_interrupts_configure_interrupt(int_desc);
Madhukar Pappireddy5fc8be12021-08-03 11:42:53 -0500184 k++;
185 CHECK(k <= VM_MANIFEST_MAX_INTERRUPTS);
186 }
187 }
188 dlog_verbose("VM has %d physical interrupts defined in manifest.\n", k);
189
Manish Pandeyd34f8892020-06-19 17:41:07 +0100190 if (manifest_vm->is_ffa_partition) {
Daniel Boulbybaeaf2e2021-12-09 11:42:36 +0000191 vm_locked.vm->ffa_version = manifest_vm->partition.ffa_version;
Manish Pandeyd34f8892020-06-19 17:41:07 +0100192 /* Link rxtx buffers to mailbox */
Raghu Krishnamurthy8c250a92021-07-02 12:16:42 -0700193 if (manifest_vm->partition.rxtx.available) {
Manish Pandeyd34f8892020-06-19 17:41:07 +0100194 if (!link_rxtx_to_mailbox(stage1_locked, vm_locked,
Raghu Krishnamurthy8c250a92021-07-02 12:16:42 -0700195 manifest_vm->partition.rxtx,
Manish Pandeyd34f8892020-06-19 17:41:07 +0100196 ppool)) {
197 dlog_error(
198 "Unable to Link RX/TX buffer with "
199 "mailbox.\n");
200 return false;
201 }
202 }
J-Alvesb37fd082020-10-22 12:29:21 +0100203
Maksims Svecovsb596eab2021-04-27 00:52:27 +0100204 vm_locked.vm->messaging_method =
Raghu Krishnamurthy8c250a92021-07-02 12:16:42 -0700205 manifest_vm->partition.messaging_method;
Manish Pandeyf3be6392020-09-24 17:26:09 +0100206
Madhukar Pappireddy84154052022-06-21 18:30:25 -0500207 vm_locked.vm->ns_interrupts_action =
208 manifest_vm->partition.ns_interrupts_action;
Maksims Svecovs9ddf86a2021-05-06 17:17:21 +0100209
Madhukar Pappireddy046dad02022-06-21 18:43:33 -0500210 vm_locked.vm->me_signal_virq =
211 manifest_vm->partition.me_signal_virq;
212
J-Alvesa4730db2021-11-02 10:31:01 +0000213 vm_locked.vm->notifications.enabled =
214 manifest_vm->partition.notification_support;
215
Raghu Krishnamurthy8c250a92021-07-02 12:16:42 -0700216 vm_locked.vm->boot_order = manifest_vm->partition.boot_order;
217
J-Alves7d38f7b2022-04-13 13:22:30 +0100218 vm_locked.vm->boot_info.gp_register_num =
219 manifest_vm->partition.gp_register_num;
220
221 if (manifest_vm->partition.boot_info) {
222 /*
223 * If the partition expects the boot information blob
224 * per the ff-a v1.1 boot protocol, then its address
225 * shall match the partition's load address.
226 */
227 vm_locked.vm->boot_info.blob_addr =
228 ipa_init(manifest_vm->partition.load_addr);
229 }
230
J-Alvesb37fd082020-10-22 12:29:21 +0100231 /* Updating boot list according to boot_order */
232 vm_update_boot(vm_locked.vm);
J-Alvesa0f317d2021-06-09 13:31:59 +0100233
J-Alves09ff9d82021-11-02 11:55:20 +0000234 if (vm_locked_are_notifications_enabled(vm_locked) &&
J-Alvesa4730db2021-11-02 10:31:01 +0000235 !plat_ffa_notifications_bitmap_create_call(
236 vm_locked.vm->id, vm_locked.vm->vcpu_count)) {
237 return false;
J-Alvesa9c7cba2021-08-25 16:26:11 +0100238 }
Manish Pandeyd34f8892020-06-19 17:41:07 +0100239 }
J-Alvesb37fd082020-10-22 12:29:21 +0100240
Fuad Tabba56970712020-01-10 11:20:09 +0000241 /* Initialize architecture-specific features. */
Manish Pandeyd34f8892020-06-19 17:41:07 +0100242 arch_vm_features_set(vm_locked.vm);
Fuad Tabba77a4b012019-11-15 12:13:08 +0000243
Madhukar Pappireddy54680c72020-10-23 15:02:38 -0500244 if (!plat_iommu_attach_peripheral(stage1_locked, vm_locked, manifest_vm,
245 ppool)) {
246 dlog_error("Unable to attach upstream peripheral device\n");
247 return false;
248 }
249
Andrew Scullae9962e2019-10-03 16:51:16 +0100250 return true;
251}
252
253/**
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +0100254 * Loads the primary VM.
255 */
Andrew Scull72b43c02019-09-18 13:53:45 +0100256static bool load_primary(struct mm_stage1_locked stage1_locked,
Andrew Scullae9962e2019-10-03 16:51:16 +0100257 const struct manifest_vm *manifest_vm,
Andrew Scullb5f49e02019-10-02 13:20:47 +0100258 const struct memiter *cpio,
259 const struct boot_params *params, struct mpool *ppool)
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +0100260{
Olivier Deprez62d99e32020-01-09 15:58:07 +0100261 paddr_t primary_begin;
262 ipaddr_t primary_entry;
David Brazdile6f83222019-09-23 14:47:37 +0100263 struct vm *vm;
Andrew Scull3c257452019-11-26 13:32:50 +0000264 struct vm_locked vm_locked;
David Brazdile6f83222019-09-23 14:47:37 +0100265 struct vcpu_locked vcpu_locked;
Andrew Scullb5f49e02019-10-02 13:20:47 +0100266 size_t i;
Andrew Scull3c257452019-11-26 13:32:50 +0000267 bool ret;
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +0100268
Raghu Krishnamurthyb49549e2021-07-02 08:27:38 -0700269 if (manifest_vm->is_ffa_partition && !manifest_vm->is_hyp_loaded) {
Raghu Krishnamurthy8c250a92021-07-02 12:16:42 -0700270 primary_begin = pa_init(manifest_vm->partition.load_addr);
Olivier Deprez62d99e32020-01-09 15:58:07 +0100271 primary_entry = ipa_add(ipa_from_pa(primary_begin),
Raghu Krishnamurthy8c250a92021-07-02 12:16:42 -0700272 manifest_vm->partition.ep_offset);
Olivier Deprez62d99e32020-01-09 15:58:07 +0100273 } else {
274 primary_begin =
275 (manifest_vm->primary.boot_address ==
276 MANIFEST_INVALID_ADDRESS)
277 ? layout_primary_begin()
278 : pa_init(manifest_vm->primary.boot_address);
279 primary_entry = ipa_from_pa(primary_begin);
280 }
281
David Brazdil080ee312020-02-25 15:30:30 -0800282 paddr_t primary_end = pa_add(primary_begin, RSIZE_MAX);
Andrew Scull72b43c02019-09-18 13:53:45 +0100283
Raghu Krishnamurthycd1eceb2021-01-04 12:20:48 -0800284 /* Primary VM must be a VM */
Raghu Krishnamurthy8c250a92021-07-02 12:16:42 -0700285 CHECK(manifest_vm->partition.run_time_el == EL1);
Raghu Krishnamurthycd1eceb2021-01-04 12:20:48 -0800286
Olivier Deprez62d99e32020-01-09 15:58:07 +0100287 /*
288 * Load the kernel if a filename is specified in the VM manifest.
289 * For an FF-A partition, kernel_filename is undefined indicating
290 * the partition package has already been loaded prior to Hafnium
291 * booting.
292 */
293 if (!string_is_empty(&manifest_vm->kernel_filename)) {
294 if (!load_kernel(stage1_locked, primary_begin, primary_end,
Fuad Tabba50469e02020-06-30 15:14:28 +0100295 manifest_vm, cpio, ppool, NULL)) {
Olivier Deprez62d99e32020-01-09 15:58:07 +0100296 dlog_error("Unable to load primary kernel.\n");
297 return false;
298 }
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +0100299 }
300
Raghu Krishnamurthycd1eceb2021-01-04 12:20:48 -0800301 if (!vm_init_next(MAX_CPUS, ppool, &vm, false)) {
Andrew Walbran7586e042020-02-18 18:19:26 +0000302 dlog_error("Unable to initialise primary VM.\n");
David Brazdile6f83222019-09-23 14:47:37 +0100303 return false;
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +0100304 }
305
David Brazdile6f83222019-09-23 14:47:37 +0100306 if (vm->id != HF_PRIMARY_VM_ID) {
Andrew Walbran7586e042020-02-18 18:19:26 +0000307 dlog_error("Primary VM was not given correct ID.\n");
David Brazdile6f83222019-09-23 14:47:37 +0100308 return false;
309 }
310
Andrew Scull3c257452019-11-26 13:32:50 +0000311 vm_locked = vm_lock(vm);
312
Andrew Scull48929fd2020-01-28 10:39:10 +0000313 if (params->device_mem_ranges_count == 0) {
314 /*
315 * Map 1TB of address space as device memory to, most likely,
316 * make all devices available to the primary VM.
317 *
318 * TODO: remove this once all targets provide valid ranges.
319 */
Andrew Scullf6ab9bc2020-02-26 12:56:37 -0800320 dlog_warning(
321 "Device memory not provided, defaulting to 1 TB.\n");
Andrew Scull48929fd2020-01-28 10:39:10 +0000322
323 if (!vm_identity_map(
324 vm_locked, pa_init(0),
325 pa_init(UINT64_C(1024) * 1024 * 1024 * 1024),
326 MM_MODE_R | MM_MODE_W | MM_MODE_D, ppool, NULL)) {
327 dlog_error(
328 "Unable to initialise address space for "
Andrew Scullf6ab9bc2020-02-26 12:56:37 -0800329 "primary VM.\n");
Andrew Scull48929fd2020-01-28 10:39:10 +0000330 ret = false;
331 goto out;
332 }
David Brazdile6f83222019-09-23 14:47:37 +0100333 }
334
Andrew Scullb5f49e02019-10-02 13:20:47 +0100335 /* Map normal memory as such to permit caching, execution, etc. */
336 for (i = 0; i < params->mem_ranges_count; ++i) {
Andrew Scull3c257452019-11-26 13:32:50 +0000337 if (!vm_identity_map(vm_locked, params->mem_ranges[i].begin,
338 params->mem_ranges[i].end,
339 MM_MODE_R | MM_MODE_W | MM_MODE_X, ppool,
340 NULL)) {
Andrew Walbran17eebf92020-02-05 16:35:49 +0000341 dlog_error(
Andrew Scullf6ab9bc2020-02-26 12:56:37 -0800342 "Unable to initialise memory for primary "
343 "VM.\n");
Andrew Scull3c257452019-11-26 13:32:50 +0000344 ret = false;
345 goto out;
Andrew Scullb5f49e02019-10-02 13:20:47 +0100346 }
347 }
348
Andrew Scull48929fd2020-01-28 10:39:10 +0000349 /* Map device memory as such to prevent execution, speculation etc. */
350 for (i = 0; i < params->device_mem_ranges_count; ++i) {
351 if (!vm_identity_map(
352 vm_locked, params->device_mem_ranges[i].begin,
353 params->device_mem_ranges[i].end,
354 MM_MODE_R | MM_MODE_W | MM_MODE_D, ppool, NULL)) {
355 dlog("Unable to initialise device memory for primary "
Andrew Scullf6ab9bc2020-02-26 12:56:37 -0800356 "VM.\n");
Andrew Scull48929fd2020-01-28 10:39:10 +0000357 ret = false;
358 goto out;
359 }
360 }
361
Manish Pandeyd34f8892020-06-19 17:41:07 +0100362 if (!load_common(stage1_locked, vm_locked, manifest_vm, ppool)) {
363 ret = false;
364 goto out;
365 }
366
Andrew Scull3c257452019-11-26 13:32:50 +0000367 if (!vm_unmap_hypervisor(vm_locked, ppool)) {
Andrew Scullf6ab9bc2020-02-26 12:56:37 -0800368 dlog_error("Unable to unmap hypervisor from primary VM.\n");
Andrew Scull3c257452019-11-26 13:32:50 +0000369 ret = false;
370 goto out;
David Brazdile6f83222019-09-23 14:47:37 +0100371 }
372
Andrew Scullb1a6d0d2020-01-29 11:25:12 +0000373 if (!plat_iommu_unmap_iommus(vm_locked, ppool)) {
Andrew Scullf6ab9bc2020-02-26 12:56:37 -0800374 dlog_error("Unable to unmap IOMMUs from primary VM.\n");
Andrew Scullb1a6d0d2020-01-29 11:25:12 +0000375 ret = false;
376 goto out;
377 }
378
Andrew Walbran7586e042020-02-18 18:19:26 +0000379 dlog_info("Loaded primary VM with %u vCPUs, entry at %#x.\n",
380 vm->vcpu_count, pa_addr(primary_begin));
381
Olivier Deprezb9adff42021-02-01 12:14:05 +0100382 /* Mark the primary to be the first booted VM */
383 vm_update_boot(vm);
384
David Brazdile6f83222019-09-23 14:47:37 +0100385 vcpu_locked = vcpu_lock(vm_get_vcpu(vm, 0));
Olivier Deprez62d99e32020-01-09 15:58:07 +0100386 vcpu_on(vcpu_locked, primary_entry, params->kernel_arg);
David Brazdile6f83222019-09-23 14:47:37 +0100387 vcpu_unlock(&vcpu_locked);
Andrew Scull3c257452019-11-26 13:32:50 +0000388 ret = true;
David Brazdile6f83222019-09-23 14:47:37 +0100389
Andrew Scull3c257452019-11-26 13:32:50 +0000390out:
391 vm_unlock(&vm_locked);
392
393 return ret;
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +0100394}
395
Fuad Tabba50469e02020-06-30 15:14:28 +0100396/**
397 * Loads the secondary VM's FDT.
398 * Stores the total allocated size for the FDT in fdt_allocated_size (if
399 * fdt_allocated_size is not NULL). The allocated size includes additional space
400 * for potential patching.
401 */
402static bool load_secondary_fdt(struct mm_stage1_locked stage1_locked,
403 paddr_t end, size_t fdt_max_size,
404 const struct manifest_vm *manifest_vm,
405 const struct memiter *cpio, struct mpool *ppool,
406 paddr_t *fdt_addr, size_t *fdt_allocated_size)
407{
408 struct memiter fdt;
409 size_t allocated_size;
410
411 CHECK(!string_is_empty(&manifest_vm->secondary.fdt_filename));
412
413 if (!cpio_get_file(cpio, &manifest_vm->secondary.fdt_filename, &fdt)) {
414 dlog_error("Cannot open the secondary VM's FDT.\n");
415 return false;
416 }
417
418 /*
Olivier Deprez701e8bf2022-04-06 18:45:18 +0200419 * Ensure the FDT has one additional page at the end for patching,
Fuad Tabba50469e02020-06-30 15:14:28 +0100420 * and align it to the page boundary.
421 */
422 allocated_size = align_up(memiter_size(&fdt), PAGE_SIZE) + PAGE_SIZE;
423
424 if (allocated_size > fdt_max_size) {
425 dlog_error(
426 "FDT allocated space (%u) is more than the specified "
427 "maximum to use (%u).\n",
428 allocated_size, fdt_max_size);
429 return false;
430 }
431
432 /* Load the FDT to the end of the VM's allocated memory space. */
433 *fdt_addr = pa_init(pa_addr(pa_sub(end, allocated_size)));
434
435 dlog_info("Loading secondary FDT of allocated size %u at 0x%x.\n",
436 allocated_size, pa_addr(*fdt_addr));
437
438 if (!copy_to_unmapped(stage1_locked, *fdt_addr, &fdt, ppool)) {
439 dlog_error("Unable to copy FDT.\n");
440 return false;
441 }
442
443 if (fdt_allocated_size) {
444 *fdt_allocated_size = allocated_size;
445 }
446
447 return true;
448}
449
Olivier Deprez035fa152022-03-14 11:19:10 +0100450/**
451 * Convert the manifest memory region attributes to mode consumed by mm layer.
452 */
453static uint32_t memory_region_attributes_to_mode(uint32_t attributes)
454{
455 uint32_t mode = 0U;
456
457 if ((attributes & MANIFEST_REGION_ATTR_READ) != 0U) {
458 mode |= MM_MODE_R;
459 }
460
461 if ((attributes & MANIFEST_REGION_ATTR_WRITE) != 0U) {
462 mode |= MM_MODE_W;
463 }
464
465 if ((attributes & MANIFEST_REGION_ATTR_EXEC) != 0U) {
466 mode |= MM_MODE_X;
467 }
468
469 assert((mode == (MM_MODE_R | MM_MODE_W)) || (mode == MM_MODE_R) ||
470 (mode == (MM_MODE_R | MM_MODE_X)));
471
472 if ((attributes & MANIFEST_REGION_ATTR_SECURITY) != 0U) {
473 mode |= arch_mm_extra_attributes_from_vm(HF_HYPERVISOR_VM_ID);
474 }
475
476 return mode;
477}
478
479/**
480 * Convert the manifest device region attributes to mode consumed by mm layer.
481 */
482static uint32_t device_region_attributes_to_mode(uint32_t attributes)
483{
484 uint32_t mode = 0U;
485
486 if ((attributes & MANIFEST_REGION_ATTR_READ) != 0U) {
487 mode |= MM_MODE_R;
488 }
489
490 if ((attributes & MANIFEST_REGION_ATTR_WRITE) != 0U) {
491 mode |= MM_MODE_W;
492 }
493
494 assert((mode == (MM_MODE_R | MM_MODE_W)) || (mode == MM_MODE_R));
495
496 if ((attributes & MANIFEST_REGION_ATTR_SECURITY) != 0U) {
497 mode |= arch_mm_extra_attributes_from_vm(HF_HYPERVISOR_VM_ID);
498 }
499
500 return mode | MM_MODE_D;
501}
502
Andrew Scull72b43c02019-09-18 13:53:45 +0100503/*
504 * Loads a secondary VM.
505 */
506static bool load_secondary(struct mm_stage1_locked stage1_locked,
Manish Pandey2145c212020-05-01 16:04:22 +0100507 struct vm_locked primary_vm_locked,
Andrew Scull72b43c02019-09-18 13:53:45 +0100508 paddr_t mem_begin, paddr_t mem_end,
509 const struct manifest_vm *manifest_vm,
510 const struct memiter *cpio, struct mpool *ppool)
511{
Olivier Deprez035fa152022-03-14 11:19:10 +0100512 const char *error_string = " region security state ignored for ";
Andrew Scull72b43c02019-09-18 13:53:45 +0100513 struct vm *vm;
Andrew Scull3c257452019-11-26 13:32:50 +0000514 struct vm_locked vm_locked;
Max Shvetsov40108e72020-08-27 12:39:50 +0100515 struct vcpu_locked vcpu_locked;
Andrew Scull72b43c02019-09-18 13:53:45 +0100516 struct vcpu *vcpu;
517 ipaddr_t secondary_entry;
Andrew Scull3c257452019-11-26 13:32:50 +0000518 bool ret;
Fuad Tabba50469e02020-06-30 15:14:28 +0100519 paddr_t fdt_addr;
520 bool has_fdt;
521 size_t kernel_size = 0;
522 const size_t mem_size = pa_difference(mem_begin, mem_end);
Raghu Krishnamurthyd3ab8c32021-02-10 19:11:30 -0800523 uint32_t map_mode;
Andrew Scull72b43c02019-09-18 13:53:45 +0100524
Olivier Deprez62d99e32020-01-09 15:58:07 +0100525 /*
526 * Load the kernel if a filename is specified in the VM manifest.
527 * For an FF-A partition, kernel_filename is undefined indicating
528 * the partition package has already been loaded prior to Hafnium
529 * booting.
530 */
531 if (!string_is_empty(&manifest_vm->kernel_filename)) {
532 if (!load_kernel(stage1_locked, mem_begin, mem_end, manifest_vm,
Fuad Tabba50469e02020-06-30 15:14:28 +0100533 cpio, ppool, &kernel_size)) {
Olivier Deprez62d99e32020-01-09 15:58:07 +0100534 dlog_error("Unable to load kernel.\n");
535 return false;
536 }
Andrew Scull72b43c02019-09-18 13:53:45 +0100537 }
538
Fuad Tabba50469e02020-06-30 15:14:28 +0100539 has_fdt = !string_is_empty(&manifest_vm->secondary.fdt_filename);
540 if (has_fdt) {
541 /*
542 * Ensure that the FDT does not overwrite the kernel or overlap
543 * its page, for the FDT to start at a page boundary.
544 */
545 const size_t fdt_max_size =
546 mem_size - align_up(kernel_size, PAGE_SIZE);
547
548 size_t fdt_allocated_size;
549
550 if (!load_secondary_fdt(stage1_locked, mem_end, fdt_max_size,
551 manifest_vm, cpio, ppool, &fdt_addr,
552 &fdt_allocated_size)) {
553 dlog_error("Unable to load FDT.\n");
554 return false;
555 }
556
Raghu Krishnamurthyb49549e2021-07-02 08:27:38 -0700557 if (manifest_vm->is_ffa_partition) {
558 plat_ffa_parse_partition_manifest(
559 stage1_locked, fdt_addr, fdt_allocated_size,
560 manifest_vm, ppool);
561 }
562
Fuad Tabba50469e02020-06-30 15:14:28 +0100563 if (!fdt_patch_mem(stage1_locked, fdt_addr, fdt_allocated_size,
564 mem_begin, mem_end, ppool)) {
565 dlog_error("Unable to patch FDT.\n");
566 return false;
567 }
568 }
Raghu Krishnamurthycd1eceb2021-01-04 12:20:48 -0800569 /*
570 * An S-EL0 partition must contain only 1 vCPU (UP migratable) per the
571 * FF-A 1.0 spec.
572 */
Raghu Krishnamurthy8c250a92021-07-02 12:16:42 -0700573 CHECK(manifest_vm->partition.run_time_el != S_EL0 ||
Raghu Krishnamurthycd1eceb2021-01-04 12:20:48 -0800574 manifest_vm->secondary.vcpu_count == 1);
Fuad Tabba50469e02020-06-30 15:14:28 +0100575
Raghu Krishnamurthycd1eceb2021-01-04 12:20:48 -0800576 if (!vm_init_next(manifest_vm->secondary.vcpu_count, ppool, &vm,
Raghu Krishnamurthy8c250a92021-07-02 12:16:42 -0700577 (manifest_vm->partition.run_time_el == S_EL0))) {
Andrew Walbran17eebf92020-02-05 16:35:49 +0000578 dlog_error("Unable to initialise VM.\n");
Andrew Scull72b43c02019-09-18 13:53:45 +0100579 return false;
580 }
581
Andrew Scull3c257452019-11-26 13:32:50 +0000582 vm_locked = vm_lock(vm);
583
Raghu Krishnamurthyd3ab8c32021-02-10 19:11:30 -0800584 /*
Raghu Krishnamurthy2b801692021-07-18 17:46:43 -0700585 * Grant the VM access to the memory. For VM's we mark all memory in
586 * stage-2 tables as RWX and the VM can control permissions using
587 * stage-1 translations. For S-EL0 partitions, hafnium maps the entire
588 * region of memory for the partition as RX. The partition is then
589 * expected to perform its owns relocations and call the FFA_MEM_PERM_*
590 * API's to change permissions on its image layout.
Raghu Krishnamurthyd3ab8c32021-02-10 19:11:30 -0800591 */
Raghu Krishnamurthyd3ab8c32021-02-10 19:11:30 -0800592 if (vm->el0_partition) {
Raghu Krishnamurthy2b801692021-07-18 17:46:43 -0700593 map_mode = MM_MODE_R | MM_MODE_X | MM_MODE_USER | MM_MODE_NG;
594 } else {
595 map_mode = MM_MODE_R | MM_MODE_W | MM_MODE_X;
Raghu Krishnamurthyd3ab8c32021-02-10 19:11:30 -0800596 }
Raghu Krishnamurthy2b801692021-07-18 17:46:43 -0700597
Raghu Krishnamurthyd3ab8c32021-02-10 19:11:30 -0800598 if (!vm_identity_map(vm_locked, mem_begin, mem_end, map_mode, ppool,
Andrew Scull3c257452019-11-26 13:32:50 +0000599 &secondary_entry)) {
Andrew Walbran17eebf92020-02-05 16:35:49 +0000600 dlog_error("Unable to initialise memory.\n");
Andrew Scull3c257452019-11-26 13:32:50 +0000601 ret = false;
602 goto out;
Andrew Scull72b43c02019-09-18 13:53:45 +0100603 }
604
Olivier Deprez62d99e32020-01-09 15:58:07 +0100605 if (manifest_vm->is_ffa_partition) {
Manish Pandey2145c212020-05-01 16:04:22 +0100606 int j = 0;
607 paddr_t region_begin;
608 paddr_t region_end;
609 paddr_t alloc_base = mem_end;
610 size_t size;
611 size_t total_alloc = 0;
Olivier Deprez035fa152022-03-14 11:19:10 +0100612 uint32_t attributes;
Manish Pandey2145c212020-05-01 16:04:22 +0100613
614 /* Map memory-regions */
Raghu Krishnamurthy8c250a92021-07-02 12:16:42 -0700615 while (j < manifest_vm->partition.mem_region_count) {
616 size = manifest_vm->partition.mem_regions[j]
617 .page_count *
Manish Pandey2145c212020-05-01 16:04:22 +0100618 PAGE_SIZE;
619 /*
620 * For memory-regions without base-address, memory
621 * should be allocated inside partition's page table.
622 * Start allocating memory regions in partition's
623 * page table, starting from the end.
624 * TODO: Add mechanism to let partition know of these
625 * memory regions
626 */
Raghu Krishnamurthy8c250a92021-07-02 12:16:42 -0700627 if (manifest_vm->partition.mem_regions[j]
628 .base_address == MANIFEST_INVALID_ADDRESS) {
Manish Pandey2145c212020-05-01 16:04:22 +0100629 total_alloc += size;
630 /* Don't go beyond half the VM's memory space */
631 if (total_alloc >
632 (manifest_vm->secondary.mem_size / 2)) {
633 dlog_error(
634 "Not enough space for memory-"
635 "region allocation");
636 ret = false;
637 goto out;
638 }
639
640 region_end = alloc_base;
641 region_begin = pa_subtract(alloc_base, size);
642 alloc_base = region_begin;
643
Olivier Deprez035fa152022-03-14 11:19:10 +0100644 attributes =
645 manifest_vm->partition.mem_regions[j]
646 .attributes;
647 if ((attributes &
648 MANIFEST_REGION_ATTR_SECURITY) != 0) {
649 if (plat_ffa_is_vm_id(
650 vm_locked.vm->id)) {
651 dlog_warning("Memory%sVMs\n",
652 error_string);
653 attributes &=
654 ~MANIFEST_REGION_ATTR_SECURITY;
655 } else if (!vm->el0_partition) {
656 dlog_warning(
657 "Memory%sS-EL1 "
658 "partitions.\n",
659 error_string);
660 attributes &=
661 ~MANIFEST_REGION_ATTR_SECURITY;
662 }
663 }
664
665 map_mode = memory_region_attributes_to_mode(
666 attributes);
Raghu Krishnamurthyd3ab8c32021-02-10 19:11:30 -0800667 if (vm->el0_partition) {
668 map_mode |= MM_MODE_USER | MM_MODE_NG;
669 }
670
671 if (!vm_identity_map(vm_locked, region_begin,
672 region_end, map_mode,
673 ppool, NULL)) {
Manish Pandey2145c212020-05-01 16:04:22 +0100674 dlog_error(
675 "Unable to map secondary VM "
676 "memory-region.\n");
677 ret = false;
678 goto out;
679 }
680
Olivier Deprez86d87ae2021-08-19 14:27:46 +0200681 dlog_verbose(
Manish Pandey2145c212020-05-01 16:04:22 +0100682 " Memory region %#x - %#x allocated\n",
683 region_begin, region_end);
684 } else {
685 /*
686 * Identity map memory region for both case,
687 * VA(S-EL0) or IPA(S-EL1).
688 */
Raghu Krishnamurthy8c250a92021-07-02 12:16:42 -0700689 region_begin = pa_init(
690 manifest_vm->partition.mem_regions[j]
691 .base_address);
Manish Pandey2145c212020-05-01 16:04:22 +0100692 region_end = pa_add(region_begin, size);
693
Olivier Deprez035fa152022-03-14 11:19:10 +0100694 attributes =
695 manifest_vm->partition.mem_regions[j]
696 .attributes;
697 if ((attributes &
698 MANIFEST_REGION_ATTR_SECURITY) != 0) {
699 if (plat_ffa_is_vm_id(
700 vm_locked.vm->id)) {
701 dlog_warning("Memory%sVMs\n",
702 error_string);
703 attributes &=
704 ~MANIFEST_REGION_ATTR_SECURITY;
705
706 } else if (!vm->el0_partition) {
707 dlog_warning(
708 "Memory%sS-EL1 "
709 "partitions.\n",
710 error_string);
711 attributes &=
712 ~MANIFEST_REGION_ATTR_SECURITY;
713 }
714 }
715
716 map_mode = memory_region_attributes_to_mode(
717 attributes);
Raghu Krishnamurthyd3ab8c32021-02-10 19:11:30 -0800718 if (vm->el0_partition) {
719 map_mode |= MM_MODE_USER | MM_MODE_NG;
720 }
721
722 if (!vm_identity_map(vm_locked, region_begin,
723 region_end, map_mode,
724 ppool, NULL)) {
Manish Pandey2145c212020-05-01 16:04:22 +0100725 dlog_error(
726 "Unable to map secondary VM "
727 "memory-region.\n");
728 ret = false;
729 goto out;
730 }
731 }
732
733 /* Deny the primary VM access to this memory */
734 if (!vm_unmap(primary_vm_locked, region_begin,
735 region_end, ppool)) {
736 dlog_error(
737 "Unable to unmap secondary VM memory-"
738 "region from primary VM.\n");
739 ret = false;
740 goto out;
741 }
742
743 j++;
744 }
745
746 /* Map device-regions */
747 j = 0;
Raghu Krishnamurthy8c250a92021-07-02 12:16:42 -0700748 while (j < manifest_vm->partition.dev_region_count) {
749 region_begin =
750 pa_init(manifest_vm->partition.dev_regions[j]
751 .base_address);
752 size = manifest_vm->partition.dev_regions[j]
753 .page_count *
Manish Pandey2145c212020-05-01 16:04:22 +0100754 PAGE_SIZE;
755 region_end = pa_add(region_begin, size);
756
Olivier Deprez035fa152022-03-14 11:19:10 +0100757 attributes = manifest_vm->partition.dev_regions[j]
758 .attributes;
759 if ((attributes & MANIFEST_REGION_ATTR_SECURITY) != 0) {
760 if (plat_ffa_is_vm_id(vm_locked.vm->id)) {
761 dlog_warning("Device%sVMs\n",
762 error_string);
763 attributes &=
764 ~MANIFEST_REGION_ATTR_SECURITY;
765 } else if (!vm->el0_partition) {
766 dlog_warning(
767 "Device%sS-EL1 "
768 "partitions.\n",
769 error_string);
770 attributes &=
771 ~MANIFEST_REGION_ATTR_SECURITY;
772 }
773 }
774
775 map_mode = device_region_attributes_to_mode(attributes);
Raghu Krishnamurthyd3ab8c32021-02-10 19:11:30 -0800776 if (vm->el0_partition) {
777 map_mode |= MM_MODE_USER | MM_MODE_NG;
778 }
779
780 if (!vm_identity_map(vm_locked, region_begin,
781 region_end, map_mode, ppool,
782 NULL)) {
Manish Pandey2145c212020-05-01 16:04:22 +0100783 dlog_error(
784 "Unable to map secondary VM "
785 "device-region.\n");
786 ret = false;
787 goto out;
788 }
789 /* Deny primary VM access to this region */
790 if (!vm_unmap(primary_vm_locked, region_begin,
791 region_end, ppool)) {
792 dlog_error(
793 "Unable to unmap secondary VM device-"
794 "region from primary VM.\n");
795 ret = false;
796 goto out;
797 }
798 j++;
799 }
800
Raghu Krishnamurthy8c250a92021-07-02 12:16:42 -0700801 secondary_entry = ipa_add(secondary_entry,
802 manifest_vm->partition.ep_offset);
Olivier Deprez62d99e32020-01-09 15:58:07 +0100803 }
804
Raghu Krishnamurthyd3ab8c32021-02-10 19:11:30 -0800805 /*
806 * Map hypervisor into the VM's page table. The hypervisor pages will
807 * not be accessible from EL0 since it will not be marked for user
808 * access.
809 * TODO: Map only the exception vectors and data that exception vectors
810 * require and not the entire hypervisor. This helps with speculative
811 * side-channel attacks.
812 */
813 if (vm->el0_partition) {
814 CHECK(vm_identity_map(vm_locked, layout_text_begin(),
815 layout_text_end(), MM_MODE_X, ppool,
816 NULL));
817
818 CHECK(vm_identity_map(vm_locked, layout_rodata_begin(),
819 layout_rodata_end(), MM_MODE_R, ppool,
820 NULL));
821
822 CHECK(vm_identity_map(vm_locked, layout_data_begin(),
823 layout_data_end(), MM_MODE_R | MM_MODE_W,
824 ppool, NULL));
Maksims Svecovs134b8f92022-03-04 15:14:09 +0000825
826 CHECK(arch_stack_mm_init(mm_lock_ptable_unsafe(&vm->ptable),
827 ppool));
828
Raghu Krishnamurthyd3ab8c32021-02-10 19:11:30 -0800829 plat_console_mm_init(mm_lock_ptable_unsafe(&vm->ptable), ppool);
830 }
831
Manish Pandeyd34f8892020-06-19 17:41:07 +0100832 if (!load_common(stage1_locked, vm_locked, manifest_vm, ppool)) {
833 ret = false;
834 goto out;
835 }
836
Manish Pandey2145c212020-05-01 16:04:22 +0100837 dlog_info("Loaded with %u vCPUs, entry at %#x.\n",
838 manifest_vm->secondary.vcpu_count, pa_addr(mem_begin));
839
Andrew Scull72b43c02019-09-18 13:53:45 +0100840 vcpu = vm_get_vcpu(vm, 0);
Fuad Tabba50469e02020-06-30 15:14:28 +0100841
Max Shvetsov40108e72020-08-27 12:39:50 +0100842 vcpu_locked = vcpu_lock(vcpu);
Madhukar Pappireddy046dad02022-06-21 18:43:33 -0500843
844 /* Enable virtual maintenance interrupts for Secure Partitions. */
845 plat_ffa_enable_virtual_maintenance_interrupts(vcpu_locked);
846
Fuad Tabba50469e02020-06-30 15:14:28 +0100847 if (has_fdt) {
Max Shvetsov40108e72020-08-27 12:39:50 +0100848 vcpu_secondary_reset_and_start(vcpu_locked, secondary_entry,
Fuad Tabba50469e02020-06-30 15:14:28 +0100849 pa_addr(fdt_addr));
850 } else {
851 /*
852 * Without an FDT, secondary VMs expect the memory size to be
853 * passed in register x0, which is what
854 * vcpu_secondary_reset_and_start does in this case.
855 */
Max Shvetsov40108e72020-08-27 12:39:50 +0100856 vcpu_secondary_reset_and_start(vcpu_locked, secondary_entry,
857 mem_size);
Fuad Tabba50469e02020-06-30 15:14:28 +0100858 }
859
Max Shvetsov40108e72020-08-27 12:39:50 +0100860 vcpu_unlock(&vcpu_locked);
861
Andrew Scull3c257452019-11-26 13:32:50 +0000862 ret = true;
Andrew Scull72b43c02019-09-18 13:53:45 +0100863
Andrew Scull3c257452019-11-26 13:32:50 +0000864out:
865 vm_unlock(&vm_locked);
866
867 return ret;
Andrew Scull72b43c02019-09-18 13:53:45 +0100868}
869
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +0100870/**
Andrew Walbran34ce72e2018-09-13 16:47:44 +0100871 * Try to find a memory range of the given size within the given ranges, and
872 * remove it from them. Return true on success, or false if no large enough
873 * contiguous range is found.
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +0100874 */
Hong-Seok Kim09648362019-05-23 15:47:11 +0900875static bool carve_out_mem_range(struct mem_range *mem_ranges,
876 size_t mem_ranges_count, uint64_t size_to_find,
877 paddr_t *found_begin, paddr_t *found_end)
Andrew Walbran34ce72e2018-09-13 16:47:44 +0100878{
879 size_t i;
880
Wedson Almeida Filho81568c42019-01-04 13:33:02 +0000881 /*
882 * TODO(b/116191358): Consider being cleverer about how we pack VMs
883 * together, with a non-greedy algorithm.
884 */
Andrew Walbran34ce72e2018-09-13 16:47:44 +0100885 for (i = 0; i < mem_ranges_count; ++i) {
886 if (size_to_find <=
Andrew Walbran2cb43392019-04-17 12:52:45 +0100887 pa_difference(mem_ranges[i].begin, mem_ranges[i].end)) {
Wedson Almeida Filhob2c159e2018-10-25 13:27:47 +0100888 /*
889 * This range is big enough, take some of it from the
890 * end and reduce its size accordingly.
891 */
Andrew Walbran34ce72e2018-09-13 16:47:44 +0100892 *found_end = mem_ranges[i].end;
893 *found_begin = pa_init(pa_addr(mem_ranges[i].end) -
894 size_to_find);
895 mem_ranges[i].end = *found_begin;
896 return true;
897 }
898 }
899 return false;
900}
901
902/**
903 * Given arrays of memory ranges before and after memory was removed for
904 * secondary VMs, add the difference to the reserved ranges of the given update.
905 * Return true on success, or false if there would be more than MAX_MEM_RANGES
906 * reserved ranges after adding the new ones.
907 * `before` and `after` must be arrays of exactly `mem_ranges_count` elements.
908 */
Hong-Seok Kim09648362019-05-23 15:47:11 +0900909static bool update_reserved_ranges(struct boot_params_update *update,
910 const struct mem_range *before,
911 const struct mem_range *after,
912 size_t mem_ranges_count)
Andrew Walbran34ce72e2018-09-13 16:47:44 +0100913{
914 size_t i;
915
916 for (i = 0; i < mem_ranges_count; ++i) {
917 if (pa_addr(after[i].begin) > pa_addr(before[i].begin)) {
918 if (update->reserved_ranges_count >= MAX_MEM_RANGES) {
Andrew Walbran17eebf92020-02-05 16:35:49 +0000919 dlog_error(
920 "Too many reserved ranges after "
921 "loading secondary VMs.\n");
Andrew Walbran34ce72e2018-09-13 16:47:44 +0100922 return false;
923 }
924 update->reserved_ranges[update->reserved_ranges_count]
925 .begin = before[i].begin;
926 update->reserved_ranges[update->reserved_ranges_count]
927 .end = after[i].begin;
928 update->reserved_ranges_count++;
929 }
930 if (pa_addr(after[i].end) < pa_addr(before[i].end)) {
931 if (update->reserved_ranges_count >= MAX_MEM_RANGES) {
Andrew Walbran17eebf92020-02-05 16:35:49 +0000932 dlog_error(
933 "Too many reserved ranges after "
934 "loading secondary VMs.\n");
Andrew Walbran34ce72e2018-09-13 16:47:44 +0100935 return false;
936 }
937 update->reserved_ranges[update->reserved_ranges_count]
938 .begin = after[i].end;
939 update->reserved_ranges[update->reserved_ranges_count]
940 .end = before[i].end;
941 update->reserved_ranges_count++;
942 }
943 }
944
945 return true;
946}
947
Olivier Deprez112d2b52020-09-30 07:39:23 +0200948static bool init_other_world_vm(struct mpool *ppool)
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +0100949{
Olivier Deprez96a2a262020-06-11 17:21:38 +0200950 struct vm *other_world_vm;
Andrew Walbran34ce72e2018-09-13 16:47:44 +0100951 size_t i;
Andrew Scull72b43c02019-09-18 13:53:45 +0100952
Andrew Walbran9daa57e2019-09-27 13:33:20 +0100953 /*
Olivier Deprez96a2a262020-06-11 17:21:38 +0200954 * Initialise the dummy VM which represents the opposite world:
955 * -TrustZone (or the SPMC) when running the Hypervisor
956 * -the Hypervisor when running TZ/SPMC
Andrew Walbran9daa57e2019-09-27 13:33:20 +0100957 */
Raghu Krishnamurthycd1eceb2021-01-04 12:20:48 -0800958 other_world_vm = vm_init(HF_OTHER_WORLD_ID, MAX_CPUS, ppool, false);
Olivier Deprez96a2a262020-06-11 17:21:38 +0200959 CHECK(other_world_vm != NULL);
960
961 for (i = 0; i < MAX_CPUS; i++) {
962 struct vcpu *vcpu = vm_get_vcpu(other_world_vm, i);
963 struct cpu *cpu = cpu_find_index(i);
964
965 vcpu->cpu = cpu;
966 }
Andrew Walbran9daa57e2019-09-27 13:33:20 +0100967
Olivier Deprez112d2b52020-09-30 07:39:23 +0200968 return arch_other_world_vm_init(other_world_vm, ppool);
969}
970
971/*
972 * Loads alls VMs from the manifest.
973 */
974bool load_vms(struct mm_stage1_locked stage1_locked,
975 const struct manifest *manifest, const struct memiter *cpio,
976 const struct boot_params *params,
977 struct boot_params_update *update, struct mpool *ppool)
978{
979 struct vm *primary;
980 struct mem_range mem_ranges_available[MAX_MEM_RANGES];
981 struct vm_locked primary_vm_locked;
982 size_t i;
983 bool success = true;
984
Andrew Walbranf8716782020-10-29 17:07:07 +0000985 /**
986 * Only try to load the primary VM if it is supposed to be in this
987 * world.
988 */
989 if (vm_id_is_current_world(HF_PRIMARY_VM_ID)) {
990 if (!load_primary(stage1_locked,
991 &manifest->vm[HF_PRIMARY_VM_INDEX], cpio,
992 params, ppool)) {
993 dlog_error("Unable to load primary VM.\n");
994 return false;
995 }
Olivier Deprez112d2b52020-09-30 07:39:23 +0200996 }
997
998 if (!init_other_world_vm(ppool)) {
999 return false;
1000 }
1001
Andrew Walbran34ce72e2018-09-13 16:47:44 +01001002 static_assert(
1003 sizeof(mem_ranges_available) == sizeof(params->mem_ranges),
1004 "mem_range arrays must be the same size for memcpy.");
1005 static_assert(sizeof(mem_ranges_available) < 500,
1006 "This will use too much stack, either make "
1007 "MAX_MEM_RANGES smaller or change this.");
Andrew Sculla1aa2ba2019-04-05 11:49:02 +01001008 memcpy_s(mem_ranges_available, sizeof(mem_ranges_available),
1009 params->mem_ranges, sizeof(params->mem_ranges));
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +01001010
Andrew Walbran34ce72e2018-09-13 16:47:44 +01001011 /* Round the last addresses down to the page size. */
1012 for (i = 0; i < params->mem_ranges_count; ++i) {
Alfredo Mazzinghieb1997c2019-02-07 18:00:01 +00001013 mem_ranges_available[i].end = pa_init(align_down(
1014 pa_addr(mem_ranges_available[i].end), PAGE_SIZE));
Andrew Walbran34ce72e2018-09-13 16:47:44 +01001015 }
Wedson Almeida Filho84a30a02018-07-23 20:05:05 +01001016
Andrew Scull3c257452019-11-26 13:32:50 +00001017 primary = vm_find(HF_PRIMARY_VM_ID);
1018 primary_vm_locked = vm_lock(primary);
1019
David Brazdil0251b942019-09-10 15:59:50 +01001020 for (i = 0; i < manifest->vm_count; ++i) {
David Brazdil0dbb41f2019-09-09 18:03:35 +01001021 const struct manifest_vm *manifest_vm = &manifest->vm[i];
Andrew Walbranb5ab43c2020-04-30 11:32:54 +01001022 ffa_vm_id_t vm_id = HF_VM_ID_OFFSET + i;
David Brazdil7a462ec2019-08-15 12:27:47 +01001023 uint64_t mem_size;
Andrew Scull80871322018-08-06 12:04:09 +01001024 paddr_t secondary_mem_begin;
1025 paddr_t secondary_mem_end;
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +01001026
David Brazdil7a462ec2019-08-15 12:27:47 +01001027 if (vm_id == HF_PRIMARY_VM_ID) {
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +01001028 continue;
1029 }
1030
Olivier Deprez2a8ee342020-08-03 15:10:44 +02001031 dlog_info("Loading VM id %#x: %s.\n", vm_id,
Andrew Walbran17eebf92020-02-05 16:35:49 +00001032 manifest_vm->debug_name);
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +01001033
David Brazdil7a462ec2019-08-15 12:27:47 +01001034 mem_size = align_up(manifest_vm->secondary.mem_size, PAGE_SIZE);
Olivier Deprez62d99e32020-01-09 15:58:07 +01001035
Raghu Krishnamurthyb49549e2021-07-02 08:27:38 -07001036 if (manifest_vm->is_ffa_partition &&
1037 !manifest->vm[i].is_hyp_loaded) {
Olivier Deprez62d99e32020-01-09 15:58:07 +01001038 secondary_mem_begin =
Raghu Krishnamurthy8c250a92021-07-02 12:16:42 -07001039 pa_init(manifest_vm->partition.load_addr);
1040 secondary_mem_end = pa_init(
1041 manifest_vm->partition.load_addr + mem_size);
Olivier Deprez62d99e32020-01-09 15:58:07 +01001042 } else if (!carve_out_mem_range(mem_ranges_available,
1043 params->mem_ranges_count,
1044 mem_size, &secondary_mem_begin,
1045 &secondary_mem_end)) {
Andrew Walbran17eebf92020-02-05 16:35:49 +00001046 dlog_error("Not enough memory (%u bytes).\n", mem_size);
Andrew Walbran34ce72e2018-09-13 16:47:44 +01001047 continue;
1048 }
Andrew Scull80871322018-08-06 12:04:09 +01001049
Manish Pandey2145c212020-05-01 16:04:22 +01001050 if (!load_secondary(stage1_locked, primary_vm_locked,
1051 secondary_mem_begin, secondary_mem_end,
1052 manifest_vm, cpio, ppool)) {
Andrew Walbran17eebf92020-02-05 16:35:49 +00001053 dlog_error("Unable to load VM.\n");
Wedson Almeida Filho84a30a02018-07-23 20:05:05 +01001054 continue;
1055 }
1056
1057 /* Deny the primary VM access to this memory. */
Andrew Scull3c257452019-11-26 13:32:50 +00001058 if (!vm_unmap(primary_vm_locked, secondary_mem_begin,
1059 secondary_mem_end, ppool)) {
Andrew Walbran17eebf92020-02-05 16:35:49 +00001060 dlog_error(
1061 "Unable to unmap secondary VM from primary "
1062 "VM.\n");
Andrew Scull3c257452019-11-26 13:32:50 +00001063 success = false;
1064 break;
Wedson Almeida Filho84a30a02018-07-23 20:05:05 +01001065 }
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +01001066 }
1067
Andrew Scull3c257452019-11-26 13:32:50 +00001068 vm_unlock(&primary_vm_locked);
1069
1070 if (!success) {
1071 return false;
1072 }
1073
Wedson Almeida Filhob2c159e2018-10-25 13:27:47 +01001074 /*
1075 * Add newly reserved areas to update params by looking at the
Andrew Walbran34ce72e2018-09-13 16:47:44 +01001076 * difference between the available ranges from the original params and
1077 * the updated mem_ranges_available. We assume that the number and order
1078 * of available ranges is the same, i.e. we don't remove any ranges
Wedson Almeida Filhob2c159e2018-10-25 13:27:47 +01001079 * above only make them smaller.
1080 */
Andrew Walbran34ce72e2018-09-13 16:47:44 +01001081 return update_reserved_ranges(update, params->mem_ranges,
1082 mem_ranges_available,
1083 params->mem_ranges_count);
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +01001084}