blob: d59bc5d97741d0dd0cd074a904df9495a79f738f [file] [log] [blame]
Andrew Scull18834872018-10-12 11:48:09 +01001/*
Andrew Walbran692b3252019-03-07 15:51:31 +00002 * Copyright 2018 The Hafnium Authors.
Andrew Scull18834872018-10-12 11:48:09 +01003 *
Andrew Walbrane959ec12020-06-17 15:01:09 +01004 * Use of this source code is governed by a BSD-style
5 * license that can be found in the LICENSE file or at
6 * https://opensource.org/licenses/BSD-3-Clause.
Andrew Scull18834872018-10-12 11:48:09 +01007 */
8
Andrew Scull18c78fc2018-08-20 12:57:41 +01009#include "hf/load.h"
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +010010
11#include <stdbool.h>
12
Maksims Svecovs134b8f92022-03-04 15:14:09 +000013#include "hf/arch/init.h"
Olivier Deprez112d2b52020-09-30 07:39:23 +020014#include "hf/arch/other_world.h"
Fuad Tabba77a4b012019-11-15 12:13:08 +000015#include "hf/arch/vm.h"
16
Andrew Scull18c78fc2018-08-20 12:57:41 +010017#include "hf/api.h"
Andrew Walbran34ce72e2018-09-13 16:47:44 +010018#include "hf/boot_params.h"
Andrew Scull72b43c02019-09-18 13:53:45 +010019#include "hf/check.h"
Andrew Scull18c78fc2018-08-20 12:57:41 +010020#include "hf/dlog.h"
Fuad Tabba50469e02020-06-30 15:14:28 +010021#include "hf/fdt_patch.h"
Karl Meakin902af082024-11-28 14:58:38 +000022#include "hf/ffa/interrupts.h"
23#include "hf/ffa/notifications.h"
24#include "hf/ffa/setup_and_discovery.h"
Andrew Scull5991ec92018-10-08 14:55:02 +010025#include "hf/layout.h"
J-Alves77b6f4f2023-03-15 11:34:49 +000026#include "hf/manifest.h"
Andrew Scull18c78fc2018-08-20 12:57:41 +010027#include "hf/memiter.h"
28#include "hf/mm.h"
Andrew Walbran48699362019-05-20 14:38:00 +010029#include "hf/plat/console.h"
Andrew Scullb1a6d0d2020-01-29 11:25:12 +000030#include "hf/plat/iommu.h"
Andrew Scull877ae4b2019-07-02 12:52:33 +010031#include "hf/static_assert.h"
Andrew Scull8d9e1212019-04-05 13:52:55 +010032#include "hf/std.h"
Andrew Scull18c78fc2018-08-20 12:57:41 +010033#include "hf/vm.h"
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +010034
Manish Pandeyd34f8892020-06-19 17:41:07 +010035#include "vmapi/hf/ffa.h"
Andrew Scull19503262018-09-20 14:48:39 +010036
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +010037/**
38 * Copies data to an unmapped location by mapping it for write, copying the
39 * data, then unmapping it.
Andrew Sculld9225b32018-11-19 16:12:41 +000040 *
41 * The data is written so that it is available to all cores with the cache
42 * disabled. When switching to the partitions, the caching is initially disabled
43 * so the data must be available without the cache.
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +010044 */
Andrew Scull3c0a90a2019-07-01 11:55:53 +010045static bool copy_to_unmapped(struct mm_stage1_locked stage1_locked, paddr_t to,
David Brazdil7a462ec2019-08-15 12:27:47 +010046 struct memiter *from_it, struct mpool *ppool)
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +010047{
David Brazdil7a462ec2019-08-15 12:27:47 +010048 const void *from = memiter_base(from_it);
49 size_t size = memiter_size(from_it);
Andrew Scull80871322018-08-06 12:04:09 +010050 paddr_t to_end = pa_add(to, size);
51 void *ptr;
Andrew Scull265ada92018-07-30 15:19:01 +010052
Andrew Scull3c0a90a2019-07-01 11:55:53 +010053 ptr = mm_identity_map(stage1_locked, to, to_end, MM_MODE_W, ppool);
Andrew Scull80871322018-08-06 12:04:09 +010054 if (!ptr) {
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +010055 return false;
56 }
57
Andrew Sculla1aa2ba2019-04-05 11:49:02 +010058 memcpy_s(ptr, size, from, size);
Andrew Scullc059fbe2019-09-12 12:58:40 +010059 arch_mm_flush_dcache(ptr, size);
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +010060
Andrew Scull72b43c02019-09-18 13:53:45 +010061 CHECK(mm_unmap(stage1_locked, to, to_end, ppool));
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +010062
63 return true;
64}
65
Fuad Tabba50469e02020-06-30 15:14:28 +010066/**
67 * Loads the secondary VM's kernel.
68 * Stores the kernel size in kernel_size (if kernel_size is not NULL).
69 * Returns false if it cannot load the kernel.
70 */
Andrew Scull72b43c02019-09-18 13:53:45 +010071static bool load_kernel(struct mm_stage1_locked stage1_locked, paddr_t begin,
72 paddr_t end, const struct manifest_vm *manifest_vm,
Fuad Tabba50469e02020-06-30 15:14:28 +010073 const struct memiter *cpio, struct mpool *ppool,
74 size_t *kernel_size)
Andrew Scull72b43c02019-09-18 13:53:45 +010075{
Andrew Scull72b43c02019-09-18 13:53:45 +010076 struct memiter kernel;
Fuad Tabba50469e02020-06-30 15:14:28 +010077 size_t size;
Andrew Scull72b43c02019-09-18 13:53:45 +010078
David Brazdil136f2942019-09-23 14:11:03 +010079 if (!cpio_get_file(cpio, &manifest_vm->kernel_filename, &kernel)) {
Andrew Walbran17eebf92020-02-05 16:35:49 +000080 dlog_error("Could not find kernel file \"%s\".\n",
81 string_data(&manifest_vm->kernel_filename));
Andrew Scull72b43c02019-09-18 13:53:45 +010082 return false;
83 }
84
Fuad Tabba50469e02020-06-30 15:14:28 +010085 size = memiter_size(&kernel);
86 if (pa_difference(begin, end) < size) {
Andrew Walbran17eebf92020-02-05 16:35:49 +000087 dlog_error("Kernel is larger than available memory.\n");
Andrew Scull72b43c02019-09-18 13:53:45 +010088 return false;
89 }
90
91 if (!copy_to_unmapped(stage1_locked, begin, &kernel, ppool)) {
Andrew Walbran17eebf92020-02-05 16:35:49 +000092 dlog_error("Unable to copy kernel.\n");
Andrew Scull72b43c02019-09-18 13:53:45 +010093 return false;
94 }
95
Fuad Tabba50469e02020-06-30 15:14:28 +010096 if (kernel_size) {
97 *kernel_size = size;
98 }
99
Andrew Scull72b43c02019-09-18 13:53:45 +0100100 return true;
101}
102
Manish Pandeyd34f8892020-06-19 17:41:07 +0100103/*
104 * Link RX/TX buffers provided in partition manifest to mailbox
105 */
106static bool link_rxtx_to_mailbox(struct mm_stage1_locked stage1_locked,
107 struct vm_locked vm_locked, struct rx_tx rxtx,
108 struct mpool *ppool)
109{
110 struct ffa_value ret;
111 ipaddr_t send;
112 ipaddr_t recv;
113 uint32_t page_count;
114
115 send = ipa_init(rxtx.tx_buffer->base_address);
116 recv = ipa_init(rxtx.rx_buffer->base_address);
117 page_count = rxtx.tx_buffer->page_count;
118
119 ret = api_vm_configure_pages(stage1_locked, vm_locked, send, recv,
120 page_count, ppool);
121 if (ret.func != FFA_SUCCESS_32) {
122 return false;
123 }
124
Karl Meakine8937d92024-03-19 16:04:25 +0000125 dlog_verbose(" mailbox: send = %p, recv = %p\n",
Manish Pandeyd34f8892020-06-19 17:41:07 +0100126 vm_locked.vm->mailbox.send, vm_locked.vm->mailbox.recv);
127
128 return true;
129}
130
Raghu Krishnamurthyad38a9c2022-07-20 07:30:36 -0700131static void infer_interrupt(struct interrupt_info interrupt,
Madhukar Pappireddy5fc8be12021-08-03 11:42:53 -0500132 struct interrupt_descriptor *int_desc)
133{
134 uint32_t attr = interrupt.attributes;
135
Daniel Boulby18485942024-10-14 16:23:03 +0100136 int_desc->interrupt_id = interrupt.id;
137 int_desc->priority = (attr >> INT_INFO_ATTR_PRIORITY_SHIFT) & 0xff;
Madhukar Pappireddy5fc8be12021-08-03 11:42:53 -0500138
Daniel Boulby18485942024-10-14 16:23:03 +0100139 int_desc->type = (attr >> INT_INFO_ATTR_TYPE_SHIFT) & 0x3;
140 int_desc->config = (attr >> INT_INFO_ATTR_CONFIG_SHIFT) & 0x1;
141 int_desc->sec_state = (attr >> INT_INFO_ATTR_SEC_STATE_SHIFT) & 0x1;
Madhukar Pappireddy5fc8be12021-08-03 11:42:53 -0500142
Raghu Krishnamurthy98da1ca2022-10-04 08:59:01 -0700143 if (interrupt.mpidr_valid) {
Daniel Boulby18485942024-10-14 16:23:03 +0100144 int_desc->mpidr_valid = true;
145 int_desc->mpidr = interrupt.mpidr;
Raghu Krishnamurthy98da1ca2022-10-04 08:59:01 -0700146 } else {
Daniel Boulby18485942024-10-14 16:23:03 +0100147 int_desc->mpidr_valid = false;
148 int_desc->mpidr = 0;
Raghu Krishnamurthy98da1ca2022-10-04 08:59:01 -0700149 }
150
Daniel Boulby18485942024-10-14 16:23:03 +0100151 int_desc->valid = true;
152 int_desc->enabled = true;
Madhukar Pappireddy5fc8be12021-08-03 11:42:53 -0500153}
154
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +0100155/**
Andrew Scullae9962e2019-10-03 16:51:16 +0100156 * Performs VM loading activities that are common between the primary and
157 * secondaries.
158 */
Manish Pandeyd34f8892020-06-19 17:41:07 +0100159static bool load_common(struct mm_stage1_locked stage1_locked,
160 struct vm_locked vm_locked,
161 const struct manifest_vm *manifest_vm,
162 struct mpool *ppool)
Andrew Scullae9962e2019-10-03 16:51:16 +0100163{
Madhukar Pappireddy5fc8be12021-08-03 11:42:53 -0500164 struct device_region dev_region;
Raghu Krishnamurthyad38a9c2022-07-20 07:30:36 -0700165 struct interrupt_info interrupt;
Madhukar Pappireddy5fc8be12021-08-03 11:42:53 -0500166 uint32_t k = 0;
167
Manish Pandeyd34f8892020-06-19 17:41:07 +0100168 vm_locked.vm->smc_whitelist = manifest_vm->smc_whitelist;
Olivier Depreza15f2352022-09-26 09:17:24 +0200169 vm_locked.vm->power_management =
170 manifest_vm->partition.power_management;
Andrew Scullae9962e2019-10-03 16:51:16 +0100171
Kathleen Capella422b10b2023-06-30 18:28:27 -0400172 /* Populate array of UUIDs. */
173 for (uint16_t i = 0; i < PARTITION_MAX_UUIDS; i++) {
174 struct ffa_uuid current_uuid = manifest_vm->partition.uuids[i];
175
176 if (ffa_uuid_is_null(&current_uuid)) {
177 break;
178 }
179
180 vm_locked.vm->uuids[i] = current_uuid;
181 }
182
J-Alvescc542042024-07-24 19:13:22 +0100183 /*
184 * Populate the interrupt descriptor for current VM.
185 * They can be enabled in runtime using HF_INTERRUPT_ENABLE.
186 */
Raghu Krishnamurthy641dcd82022-07-19 23:21:20 -0700187 for (uint16_t i = 0; i < PARTITION_MAX_DEVICE_REGIONS; i++) {
Madhukar Pappireddy5fc8be12021-08-03 11:42:53 -0500188 dev_region = manifest_vm->partition.dev_regions[i];
189
190 CHECK(dev_region.interrupt_count <=
Raghu Krishnamurthy641dcd82022-07-19 23:21:20 -0700191 PARTITION_MAX_INTERRUPTS_PER_DEVICE);
Madhukar Pappireddy5fc8be12021-08-03 11:42:53 -0500192
193 for (uint8_t j = 0; j < dev_region.interrupt_count; j++) {
Raghu Krishnamurthy98da1ca2022-10-04 08:59:01 -0700194 struct interrupt_descriptor int_desc = {0};
Madhukar Pappireddy5fc8be12021-08-03 11:42:53 -0500195
196 interrupt = dev_region.interrupts[j];
197 infer_interrupt(interrupt, &int_desc);
198 vm_locked.vm->interrupt_desc[k] = int_desc;
Madhukar Pappireddy938faaf2023-07-31 17:56:55 -0500199 assert(int_desc.enabled);
Madhukar Pappireddy5fc8be12021-08-03 11:42:53 -0500200
201 k++;
202 CHECK(k <= VM_MANIFEST_MAX_INTERRUPTS);
203 }
204 }
J-Alvescc542042024-07-24 19:13:22 +0100205
Madhukar Pappireddy5fc8be12021-08-03 11:42:53 -0500206 dlog_verbose("VM has %d physical interrupts defined in manifest.\n", k);
207
Manish Pandeyd34f8892020-06-19 17:41:07 +0100208 if (manifest_vm->is_ffa_partition) {
Daniel Boulbybaeaf2e2021-12-09 11:42:36 +0000209 vm_locked.vm->ffa_version = manifest_vm->partition.ffa_version;
Manish Pandeyd34f8892020-06-19 17:41:07 +0100210 /* Link rxtx buffers to mailbox */
Raghu Krishnamurthy8c250a92021-07-02 12:16:42 -0700211 if (manifest_vm->partition.rxtx.available) {
Manish Pandeyd34f8892020-06-19 17:41:07 +0100212 if (!link_rxtx_to_mailbox(stage1_locked, vm_locked,
Raghu Krishnamurthy8c250a92021-07-02 12:16:42 -0700213 manifest_vm->partition.rxtx,
Manish Pandeyd34f8892020-06-19 17:41:07 +0100214 ppool)) {
215 dlog_error(
216 "Unable to Link RX/TX buffer with "
217 "mailbox.\n");
218 return false;
219 }
220 }
J-Alvesb37fd082020-10-22 12:29:21 +0100221
Maksims Svecovsb596eab2021-04-27 00:52:27 +0100222 vm_locked.vm->messaging_method =
Raghu Krishnamurthy8c250a92021-07-02 12:16:42 -0700223 manifest_vm->partition.messaging_method;
Manish Pandeyf3be6392020-09-24 17:26:09 +0100224
Madhukar Pappireddy84154052022-06-21 18:30:25 -0500225 vm_locked.vm->ns_interrupts_action =
226 manifest_vm->partition.ns_interrupts_action;
Maksims Svecovs9ddf86a2021-05-06 17:17:21 +0100227
Madhukar Pappireddy5c04a382022-12-28 11:29:26 -0600228 vm_locked.vm->other_s_interrupts_action =
229 manifest_vm->partition.other_s_interrupts_action;
230
Madhukar Pappireddy046dad02022-06-21 18:43:33 -0500231 vm_locked.vm->me_signal_virq =
232 manifest_vm->partition.me_signal_virq;
233
J-Alvesa4730db2021-11-02 10:31:01 +0000234 vm_locked.vm->notifications.enabled =
235 manifest_vm->partition.notification_support;
236
Karl Meakina603e082024-08-02 17:57:27 +0100237 vm_locked.vm->vm_availability_messages.vm_created =
238 manifest_vm->partition.vm_availability_messages
239 .vm_created;
240 vm_locked.vm->vm_availability_messages.vm_destroyed =
241 manifest_vm->partition.vm_availability_messages
242 .vm_destroyed;
243
Raghu Krishnamurthy8c250a92021-07-02 12:16:42 -0700244 vm_locked.vm->boot_order = manifest_vm->partition.boot_order;
245
J-Alves7d38f7b2022-04-13 13:22:30 +0100246 vm_locked.vm->boot_info.gp_register_num =
247 manifest_vm->partition.gp_register_num;
248
249 if (manifest_vm->partition.boot_info) {
250 /*
251 * If the partition expects the boot information blob
252 * per the ff-a v1.1 boot protocol, then its address
253 * shall match the partition's load address.
254 */
255 vm_locked.vm->boot_info.blob_addr =
256 ipa_init(manifest_vm->partition.load_addr);
257 }
258
J-Alvesce42f7a2025-02-10 13:57:41 +0000259 vm_locked.vm->sri_policy = manifest_vm->partition.sri_policy;
260
J-Alvesb37fd082020-10-22 12:29:21 +0100261 /* Updating boot list according to boot_order */
Madhukar Pappireddya49ba162024-11-25 09:40:45 -0600262 vm_update_boot(vm_locked.vm);
J-Alvesa0f317d2021-06-09 13:31:59 +0100263
J-Alves09ff9d82021-11-02 11:55:20 +0000264 if (vm_locked_are_notifications_enabled(vm_locked) &&
Karl Meakin117c8082024-12-04 16:03:28 +0000265 !ffa_notifications_bitmap_create_call(
J-Alvesa4730db2021-11-02 10:31:01 +0000266 vm_locked.vm->id, vm_locked.vm->vcpu_count)) {
267 return false;
J-Alvesa9c7cba2021-08-25 16:26:11 +0100268 }
Manish Pandeyd34f8892020-06-19 17:41:07 +0100269 }
J-Alvesb37fd082020-10-22 12:29:21 +0100270
Fuad Tabba56970712020-01-10 11:20:09 +0000271 /* Initialize architecture-specific features. */
Manish Pandeyd34f8892020-06-19 17:41:07 +0100272 arch_vm_features_set(vm_locked.vm);
Fuad Tabba77a4b012019-11-15 12:13:08 +0000273
Madhukar Pappireddy54680c72020-10-23 15:02:38 -0500274 if (!plat_iommu_attach_peripheral(stage1_locked, vm_locked, manifest_vm,
275 ppool)) {
276 dlog_error("Unable to attach upstream peripheral device\n");
277 return false;
278 }
279
Andrew Scullae9962e2019-10-03 16:51:16 +0100280 return true;
281}
282
283/**
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +0100284 * Loads the primary VM.
285 */
Andrew Scull72b43c02019-09-18 13:53:45 +0100286static bool load_primary(struct mm_stage1_locked stage1_locked,
Andrew Scullae9962e2019-10-03 16:51:16 +0100287 const struct manifest_vm *manifest_vm,
Andrew Scullb5f49e02019-10-02 13:20:47 +0100288 const struct memiter *cpio,
289 const struct boot_params *params, struct mpool *ppool)
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +0100290{
Olivier Deprez62d99e32020-01-09 15:58:07 +0100291 paddr_t primary_begin;
292 ipaddr_t primary_entry;
David Brazdile6f83222019-09-23 14:47:37 +0100293 struct vm *vm;
Andrew Scull3c257452019-11-26 13:32:50 +0000294 struct vm_locked vm_locked;
David Brazdile6f83222019-09-23 14:47:37 +0100295 struct vcpu_locked vcpu_locked;
Andrew Scullb5f49e02019-10-02 13:20:47 +0100296 size_t i;
Andrew Scull3c257452019-11-26 13:32:50 +0000297 bool ret;
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +0100298
Raghu Krishnamurthyb49549e2021-07-02 08:27:38 -0700299 if (manifest_vm->is_ffa_partition && !manifest_vm->is_hyp_loaded) {
Raghu Krishnamurthy8c250a92021-07-02 12:16:42 -0700300 primary_begin = pa_init(manifest_vm->partition.load_addr);
Olivier Deprez62d99e32020-01-09 15:58:07 +0100301 primary_entry = ipa_add(ipa_from_pa(primary_begin),
Raghu Krishnamurthy8c250a92021-07-02 12:16:42 -0700302 manifest_vm->partition.ep_offset);
Olivier Deprez62d99e32020-01-09 15:58:07 +0100303 } else {
304 primary_begin =
305 (manifest_vm->primary.boot_address ==
306 MANIFEST_INVALID_ADDRESS)
307 ? layout_primary_begin()
308 : pa_init(manifest_vm->primary.boot_address);
309 primary_entry = ipa_from_pa(primary_begin);
310 }
311
David Brazdil080ee312020-02-25 15:30:30 -0800312 paddr_t primary_end = pa_add(primary_begin, RSIZE_MAX);
Andrew Scull72b43c02019-09-18 13:53:45 +0100313
Raghu Krishnamurthycd1eceb2021-01-04 12:20:48 -0800314 /* Primary VM must be a VM */
Raghu Krishnamurthy8c250a92021-07-02 12:16:42 -0700315 CHECK(manifest_vm->partition.run_time_el == EL1);
Raghu Krishnamurthycd1eceb2021-01-04 12:20:48 -0800316
Olivier Deprez62d99e32020-01-09 15:58:07 +0100317 /*
318 * Load the kernel if a filename is specified in the VM manifest.
319 * For an FF-A partition, kernel_filename is undefined indicating
320 * the partition package has already been loaded prior to Hafnium
321 * booting.
322 */
323 if (!string_is_empty(&manifest_vm->kernel_filename)) {
324 if (!load_kernel(stage1_locked, primary_begin, primary_end,
Fuad Tabba50469e02020-06-30 15:14:28 +0100325 manifest_vm, cpio, ppool, NULL)) {
Olivier Deprez62d99e32020-01-09 15:58:07 +0100326 dlog_error("Unable to load primary kernel.\n");
327 return false;
328 }
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +0100329 }
330
Madhukar Pappireddy070f49e2024-01-12 13:02:27 -0600331 if (!vm_init_next(MAX_CPUS, ppool, &vm, false,
332 manifest_vm->partition.dma_device_count)) {
Andrew Walbran7586e042020-02-18 18:19:26 +0000333 dlog_error("Unable to initialise primary VM.\n");
David Brazdile6f83222019-09-23 14:47:37 +0100334 return false;
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +0100335 }
336
Karl Meakin5e996992024-05-20 11:27:07 +0100337 if (!vm_is_primary(vm)) {
Andrew Walbran7586e042020-02-18 18:19:26 +0000338 dlog_error("Primary VM was not given correct ID.\n");
David Brazdile6f83222019-09-23 14:47:37 +0100339 return false;
340 }
341
Andrew Scull3c257452019-11-26 13:32:50 +0000342 vm_locked = vm_lock(vm);
343
Andrew Scull48929fd2020-01-28 10:39:10 +0000344 if (params->device_mem_ranges_count == 0) {
345 /*
346 * Map 1TB of address space as device memory to, most likely,
347 * make all devices available to the primary VM.
348 *
349 * TODO: remove this once all targets provide valid ranges.
350 */
Andrew Scullf6ab9bc2020-02-26 12:56:37 -0800351 dlog_warning(
352 "Device memory not provided, defaulting to 1 TB.\n");
Andrew Scull48929fd2020-01-28 10:39:10 +0000353
354 if (!vm_identity_map(
355 vm_locked, pa_init(0),
356 pa_init(UINT64_C(1024) * 1024 * 1024 * 1024),
357 MM_MODE_R | MM_MODE_W | MM_MODE_D, ppool, NULL)) {
358 dlog_error(
359 "Unable to initialise address space for "
Andrew Scullf6ab9bc2020-02-26 12:56:37 -0800360 "primary VM.\n");
Andrew Scull48929fd2020-01-28 10:39:10 +0000361 ret = false;
362 goto out;
363 }
David Brazdile6f83222019-09-23 14:47:37 +0100364 }
365
Andrew Scullb5f49e02019-10-02 13:20:47 +0100366 /* Map normal memory as such to permit caching, execution, etc. */
367 for (i = 0; i < params->mem_ranges_count; ++i) {
Andrew Scull3c257452019-11-26 13:32:50 +0000368 if (!vm_identity_map(vm_locked, params->mem_ranges[i].begin,
369 params->mem_ranges[i].end,
370 MM_MODE_R | MM_MODE_W | MM_MODE_X, ppool,
371 NULL)) {
Andrew Walbran17eebf92020-02-05 16:35:49 +0000372 dlog_error(
Andrew Scullf6ab9bc2020-02-26 12:56:37 -0800373 "Unable to initialise memory for primary "
374 "VM.\n");
Andrew Scull3c257452019-11-26 13:32:50 +0000375 ret = false;
376 goto out;
Andrew Scullb5f49e02019-10-02 13:20:47 +0100377 }
378 }
379
Andrew Scull48929fd2020-01-28 10:39:10 +0000380 /* Map device memory as such to prevent execution, speculation etc. */
381 for (i = 0; i < params->device_mem_ranges_count; ++i) {
382 if (!vm_identity_map(
383 vm_locked, params->device_mem_ranges[i].begin,
384 params->device_mem_ranges[i].end,
385 MM_MODE_R | MM_MODE_W | MM_MODE_D, ppool, NULL)) {
386 dlog("Unable to initialise device memory for primary "
Andrew Scullf6ab9bc2020-02-26 12:56:37 -0800387 "VM.\n");
Andrew Scull48929fd2020-01-28 10:39:10 +0000388 ret = false;
389 goto out;
390 }
391 }
392
Manish Pandeyd34f8892020-06-19 17:41:07 +0100393 if (!load_common(stage1_locked, vm_locked, manifest_vm, ppool)) {
394 ret = false;
395 goto out;
396 }
397
Andrew Scull3c257452019-11-26 13:32:50 +0000398 if (!vm_unmap_hypervisor(vm_locked, ppool)) {
Andrew Scullf6ab9bc2020-02-26 12:56:37 -0800399 dlog_error("Unable to unmap hypervisor from primary VM.\n");
Andrew Scull3c257452019-11-26 13:32:50 +0000400 ret = false;
401 goto out;
David Brazdile6f83222019-09-23 14:47:37 +0100402 }
403
Andrew Scullb1a6d0d2020-01-29 11:25:12 +0000404 if (!plat_iommu_unmap_iommus(vm_locked, ppool)) {
Andrew Scullf6ab9bc2020-02-26 12:56:37 -0800405 dlog_error("Unable to unmap IOMMUs from primary VM.\n");
Andrew Scullb1a6d0d2020-01-29 11:25:12 +0000406 ret = false;
407 goto out;
408 }
409
Karl Meakine8937d92024-03-19 16:04:25 +0000410 dlog_info("Loaded primary VM with %u vCPUs, entry at %#lx.\n",
Andrew Walbran7586e042020-02-18 18:19:26 +0000411 vm->vcpu_count, pa_addr(primary_begin));
412
Madhukar Pappireddya49ba162024-11-25 09:40:45 -0600413 /* Mark the first VM to be the first booted VM. */
414 vm_update_boot(vm);
Olivier Deprezb9adff42021-02-01 12:14:05 +0100415
David Brazdile6f83222019-09-23 14:47:37 +0100416 vcpu_locked = vcpu_lock(vm_get_vcpu(vm, 0));
Olivier Deprez62d99e32020-01-09 15:58:07 +0100417 vcpu_on(vcpu_locked, primary_entry, params->kernel_arg);
David Brazdile6f83222019-09-23 14:47:37 +0100418 vcpu_unlock(&vcpu_locked);
Andrew Scull3c257452019-11-26 13:32:50 +0000419 ret = true;
David Brazdile6f83222019-09-23 14:47:37 +0100420
Andrew Scull3c257452019-11-26 13:32:50 +0000421out:
422 vm_unlock(&vm_locked);
423
424 return ret;
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +0100425}
426
Fuad Tabba50469e02020-06-30 15:14:28 +0100427/**
428 * Loads the secondary VM's FDT.
429 * Stores the total allocated size for the FDT in fdt_allocated_size (if
430 * fdt_allocated_size is not NULL). The allocated size includes additional space
431 * for potential patching.
432 */
433static bool load_secondary_fdt(struct mm_stage1_locked stage1_locked,
434 paddr_t end, size_t fdt_max_size,
435 const struct manifest_vm *manifest_vm,
436 const struct memiter *cpio, struct mpool *ppool,
437 paddr_t *fdt_addr, size_t *fdt_allocated_size)
438{
439 struct memiter fdt;
440 size_t allocated_size;
441
442 CHECK(!string_is_empty(&manifest_vm->secondary.fdt_filename));
443
444 if (!cpio_get_file(cpio, &manifest_vm->secondary.fdt_filename, &fdt)) {
445 dlog_error("Cannot open the secondary VM's FDT.\n");
446 return false;
447 }
448
449 /*
Olivier Deprez701e8bf2022-04-06 18:45:18 +0200450 * Ensure the FDT has one additional page at the end for patching,
Fuad Tabba50469e02020-06-30 15:14:28 +0100451 * and align it to the page boundary.
452 */
453 allocated_size = align_up(memiter_size(&fdt), PAGE_SIZE) + PAGE_SIZE;
454
455 if (allocated_size > fdt_max_size) {
456 dlog_error(
Karl Meakine8937d92024-03-19 16:04:25 +0000457 "FDT allocated space (%zu) is more than the specified "
458 "maximum to use (%zu).\n",
Fuad Tabba50469e02020-06-30 15:14:28 +0100459 allocated_size, fdt_max_size);
460 return false;
461 }
462
463 /* Load the FDT to the end of the VM's allocated memory space. */
464 *fdt_addr = pa_init(pa_addr(pa_sub(end, allocated_size)));
465
Karl Meakine8937d92024-03-19 16:04:25 +0000466 dlog_info("Loading secondary FDT of allocated size %zu at 0x%lx.\n",
Fuad Tabba50469e02020-06-30 15:14:28 +0100467 allocated_size, pa_addr(*fdt_addr));
468
469 if (!copy_to_unmapped(stage1_locked, *fdt_addr, &fdt, ppool)) {
470 dlog_error("Unable to copy FDT.\n");
471 return false;
472 }
473
474 if (fdt_allocated_size) {
475 *fdt_allocated_size = allocated_size;
476 }
477
478 return true;
479}
480
Olivier Deprez035fa152022-03-14 11:19:10 +0100481/**
482 * Convert the manifest memory region attributes to mode consumed by mm layer.
483 */
Karl Meakin07a69ab2025-02-07 14:53:19 +0000484static mm_mode_t memory_region_attributes_to_mode(uint32_t attributes)
Olivier Deprez035fa152022-03-14 11:19:10 +0100485{
Karl Meakin07a69ab2025-02-07 14:53:19 +0000486 mm_mode_t mode = 0U;
Olivier Deprez035fa152022-03-14 11:19:10 +0100487
488 if ((attributes & MANIFEST_REGION_ATTR_READ) != 0U) {
489 mode |= MM_MODE_R;
490 }
491
492 if ((attributes & MANIFEST_REGION_ATTR_WRITE) != 0U) {
493 mode |= MM_MODE_W;
494 }
495
496 if ((attributes & MANIFEST_REGION_ATTR_EXEC) != 0U) {
497 mode |= MM_MODE_X;
498 }
499
500 assert((mode == (MM_MODE_R | MM_MODE_W)) || (mode == MM_MODE_R) ||
501 (mode == (MM_MODE_R | MM_MODE_X)));
502
503 if ((attributes & MANIFEST_REGION_ATTR_SECURITY) != 0U) {
Karl Meakin07a69ab2025-02-07 14:53:19 +0000504 mode |= arch_mm_extra_mode_from_vm(HF_HYPERVISOR_VM_ID);
Olivier Deprez035fa152022-03-14 11:19:10 +0100505 }
506
507 return mode;
508}
509
510/**
511 * Convert the manifest device region attributes to mode consumed by mm layer.
512 */
Karl Meakin07a69ab2025-02-07 14:53:19 +0000513static mm_mode_t device_region_attributes_to_mode(uint32_t attributes)
Olivier Deprez035fa152022-03-14 11:19:10 +0100514{
Karl Meakin07a69ab2025-02-07 14:53:19 +0000515 mm_mode_t mode = 0U;
Olivier Deprez035fa152022-03-14 11:19:10 +0100516
517 if ((attributes & MANIFEST_REGION_ATTR_READ) != 0U) {
518 mode |= MM_MODE_R;
519 }
520
521 if ((attributes & MANIFEST_REGION_ATTR_WRITE) != 0U) {
522 mode |= MM_MODE_W;
523 }
524
525 assert((mode == (MM_MODE_R | MM_MODE_W)) || (mode == MM_MODE_R));
526
527 if ((attributes & MANIFEST_REGION_ATTR_SECURITY) != 0U) {
Karl Meakin07a69ab2025-02-07 14:53:19 +0000528 mode |= arch_mm_extra_mode_from_vm(HF_HYPERVISOR_VM_ID);
Olivier Deprez035fa152022-03-14 11:19:10 +0100529 }
530
531 return mode | MM_MODE_D;
532}
533
Daniel Boulbye6df3452022-07-14 18:14:26 +0100534static bool ffa_map_memory_regions(const struct manifest_vm *manifest_vm,
535 const struct vm_locked vm_locked,
536 const struct vm_locked primary_vm_locked,
J-Alvesd8a1d362023-03-08 11:15:28 +0000537 bool is_el0_partition, struct mpool *ppool)
Daniel Boulbye6df3452022-07-14 18:14:26 +0100538{
539#if LOG_LEVEL >= LOG_LEVEL_WARNING
540 const char *error_string = " region security state ignored for ";
541#endif
542 int j = 0;
543 paddr_t region_begin;
544 paddr_t region_end;
Daniel Boulbye6df3452022-07-14 18:14:26 +0100545 size_t size;
Karl Meakin07a69ab2025-02-07 14:53:19 +0000546 mm_mode_t map_mode;
Daniel Boulbye6df3452022-07-14 18:14:26 +0100547 uint32_t attributes;
548
549 /* Map memory-regions */
550 while (j < manifest_vm->partition.mem_region_count) {
Madhukar Pappireddy0e57d3d2023-10-11 15:49:05 -0500551 struct memory_region mem_region;
552
553 mem_region = manifest_vm->partition.mem_regions[j];
554 size = mem_region.page_count * PAGE_SIZE;
Daniel Boulbye6df3452022-07-14 18:14:26 +0100555 /*
J-Alvesd8a1d362023-03-08 11:15:28 +0000556 * Identity map memory region for both case,
557 * VA(S-EL0) or IPA(S-EL1).
Daniel Boulbye6df3452022-07-14 18:14:26 +0100558 */
Madhukar Pappireddy0e57d3d2023-10-11 15:49:05 -0500559 region_begin = pa_init(mem_region.base_address);
J-Alvesd8a1d362023-03-08 11:15:28 +0000560 region_end = pa_add(region_begin, size);
Daniel Boulbye6df3452022-07-14 18:14:26 +0100561
Madhukar Pappireddy0e57d3d2023-10-11 15:49:05 -0500562 attributes = mem_region.attributes;
Daniel Boulbye6df3452022-07-14 18:14:26 +0100563 if ((attributes & MANIFEST_REGION_ATTR_SECURITY) != 0) {
J-Alves661e1b72023-08-02 13:39:40 +0100564 if (ffa_is_vm_id(vm_locked.vm->id)) {
Daniel Boulbye6df3452022-07-14 18:14:26 +0100565 dlog_warning("Memory%sVMs\n", error_string);
566 attributes &= ~MANIFEST_REGION_ATTR_SECURITY;
Daniel Boulbye6df3452022-07-14 18:14:26 +0100567 }
568 }
569
570 map_mode = memory_region_attributes_to_mode(attributes);
571
572 if (is_el0_partition) {
573 map_mode |= MM_MODE_USER | MM_MODE_NG;
574 }
575
576 if (!vm_identity_map(vm_locked, region_begin, region_end,
577 map_mode, ppool, NULL)) {
578 dlog_error(
579 "Unable to map secondary VM "
580 "memory-region.\n");
581 return false;
582 }
583
Madhukar Pappireddy0e57d3d2023-10-11 15:49:05 -0500584 /*
585 * Enforce static DMA isolation through stage 2 address
586 * translation.
587 * Only the DMA device that is specified as part of this memory
588 * region node in the partition manifest will be granted access
589 * to the memory region.
590 */
591 if (mem_region.dma_prop.stream_count > 0 &&
592 !vm_iommu_mm_identity_map(
593 vm_locked, region_begin, region_end, map_mode,
594 ppool, NULL, mem_region.dma_prop.dma_device_id)) {
595 dlog_error(
596 "Unable to map memory-region in the page "
597 "tables of DMA device.\n");
598 return false;
599 }
600
Daniel Boulbye6df3452022-07-14 18:14:26 +0100601 /* Deny the primary VM access to this memory */
602 if (!vm_unmap(primary_vm_locked, region_begin, region_end,
603 ppool)) {
604 dlog_error(
605 "Unable to unmap secondary VM memory-"
606 "region from primary VM.\n");
607 return false;
608 }
609
Karl Meakine8937d92024-03-19 16:04:25 +0000610 dlog_verbose("Memory region %#lx - %#lx allocated.\n",
611 pa_addr(region_begin), pa_addr(region_end));
Daniel Boulbye6df3452022-07-14 18:14:26 +0100612
613 j++;
614 }
615
616 /* Map device-regions */
617 j = 0;
618 while (j < manifest_vm->partition.dev_region_count) {
619 region_begin = pa_init(
620 manifest_vm->partition.dev_regions[j].base_address);
621 size = manifest_vm->partition.dev_regions[j].page_count *
622 PAGE_SIZE;
623 region_end = pa_add(region_begin, size);
624
625 attributes = manifest_vm->partition.dev_regions[j].attributes;
626 if ((attributes & MANIFEST_REGION_ATTR_SECURITY) != 0) {
J-Alves661e1b72023-08-02 13:39:40 +0100627 if (ffa_is_vm_id(vm_locked.vm->id)) {
Daniel Boulbye6df3452022-07-14 18:14:26 +0100628 dlog_warning("Device%sVMs\n", error_string);
629 attributes &= ~MANIFEST_REGION_ATTR_SECURITY;
Daniel Boulbye6df3452022-07-14 18:14:26 +0100630 }
631 }
632
633 map_mode = device_region_attributes_to_mode(attributes);
634 if (is_el0_partition) {
635 map_mode |= MM_MODE_USER | MM_MODE_NG;
636 }
637
638 if (!vm_identity_map(vm_locked, region_begin, region_end,
639 map_mode, ppool, NULL)) {
640 dlog_error(
641 "Unable to map secondary VM "
642 "device-region.\n");
643 return false;
644 }
645 /* Deny primary VM access to this region */
646 if (!vm_unmap(primary_vm_locked, region_begin, region_end,
647 ppool)) {
648 dlog_error(
649 "Unable to unmap secondary VM device-"
650 "region from primary VM.\n");
651 return false;
652 }
653 j++;
654 }
655 return true;
656}
657
Andrew Scull72b43c02019-09-18 13:53:45 +0100658/*
659 * Loads a secondary VM.
660 */
661static bool load_secondary(struct mm_stage1_locked stage1_locked,
Manish Pandey2145c212020-05-01 16:04:22 +0100662 struct vm_locked primary_vm_locked,
Andrew Scull72b43c02019-09-18 13:53:45 +0100663 paddr_t mem_begin, paddr_t mem_end,
664 const struct manifest_vm *manifest_vm,
J-Alves77b6f4f2023-03-15 11:34:49 +0000665 const struct boot_params *boot_params,
Andrew Scull72b43c02019-09-18 13:53:45 +0100666 const struct memiter *cpio, struct mpool *ppool)
667{
668 struct vm *vm;
Andrew Scull3c257452019-11-26 13:32:50 +0000669 struct vm_locked vm_locked;
Max Shvetsov40108e72020-08-27 12:39:50 +0100670 struct vcpu_locked vcpu_locked;
Andrew Scull72b43c02019-09-18 13:53:45 +0100671 struct vcpu *vcpu;
Madhukar Pappireddy958c8412024-11-25 09:54:17 -0600672 ipaddr_t partition_primary_ep;
Andrew Scull3c257452019-11-26 13:32:50 +0000673 bool ret;
Fuad Tabba50469e02020-06-30 15:14:28 +0100674 paddr_t fdt_addr;
675 bool has_fdt;
676 size_t kernel_size = 0;
677 const size_t mem_size = pa_difference(mem_begin, mem_end);
Karl Meakin07a69ab2025-02-07 14:53:19 +0000678 mm_mode_t map_mode;
Daniel Boulby874d5432023-04-27 12:40:24 +0100679 bool is_el0_partition = manifest_vm->partition.run_time_el == S_EL0 ||
680 manifest_vm->partition.run_time_el == EL0;
Jens Wiklanderb697ad02022-11-04 10:15:03 +0100681 size_t n;
Andrew Scull72b43c02019-09-18 13:53:45 +0100682
Olivier Deprez62d99e32020-01-09 15:58:07 +0100683 /*
684 * Load the kernel if a filename is specified in the VM manifest.
685 * For an FF-A partition, kernel_filename is undefined indicating
686 * the partition package has already been loaded prior to Hafnium
687 * booting.
688 */
689 if (!string_is_empty(&manifest_vm->kernel_filename)) {
690 if (!load_kernel(stage1_locked, mem_begin, mem_end, manifest_vm,
Fuad Tabba50469e02020-06-30 15:14:28 +0100691 cpio, ppool, &kernel_size)) {
Olivier Deprez62d99e32020-01-09 15:58:07 +0100692 dlog_error("Unable to load kernel.\n");
693 return false;
694 }
Andrew Scull72b43c02019-09-18 13:53:45 +0100695 }
696
Fuad Tabba50469e02020-06-30 15:14:28 +0100697 has_fdt = !string_is_empty(&manifest_vm->secondary.fdt_filename);
698 if (has_fdt) {
699 /*
700 * Ensure that the FDT does not overwrite the kernel or overlap
701 * its page, for the FDT to start at a page boundary.
702 */
703 const size_t fdt_max_size =
704 mem_size - align_up(kernel_size, PAGE_SIZE);
705
706 size_t fdt_allocated_size;
707
708 if (!load_secondary_fdt(stage1_locked, mem_end, fdt_max_size,
709 manifest_vm, cpio, ppool, &fdt_addr,
710 &fdt_allocated_size)) {
711 dlog_error("Unable to load FDT.\n");
712 return false;
713 }
714
Raghu Krishnamurthyb49549e2021-07-02 08:27:38 -0700715 if (manifest_vm->is_ffa_partition) {
Karl Meakin117c8082024-12-04 16:03:28 +0000716 ffa_setup_parse_partition_manifest(
Raghu Krishnamurthyb49549e2021-07-02 08:27:38 -0700717 stage1_locked, fdt_addr, fdt_allocated_size,
J-Alves77b6f4f2023-03-15 11:34:49 +0000718 manifest_vm, boot_params, ppool);
Raghu Krishnamurthyb49549e2021-07-02 08:27:38 -0700719 }
720
Fuad Tabba50469e02020-06-30 15:14:28 +0100721 if (!fdt_patch_mem(stage1_locked, fdt_addr, fdt_allocated_size,
722 mem_begin, mem_end, ppool)) {
723 dlog_error("Unable to patch FDT.\n");
724 return false;
725 }
726 }
Raghu Krishnamurthycd1eceb2021-01-04 12:20:48 -0800727 /*
728 * An S-EL0 partition must contain only 1 vCPU (UP migratable) per the
729 * FF-A 1.0 spec.
730 */
Daniel Boulbye6df3452022-07-14 18:14:26 +0100731 CHECK(!is_el0_partition || manifest_vm->secondary.vcpu_count == 1);
Fuad Tabba50469e02020-06-30 15:14:28 +0100732
Raghu Krishnamurthycd1eceb2021-01-04 12:20:48 -0800733 if (!vm_init_next(manifest_vm->secondary.vcpu_count, ppool, &vm,
Madhukar Pappireddy070f49e2024-01-12 13:02:27 -0600734 is_el0_partition,
735 manifest_vm->partition.dma_device_count)) {
Andrew Walbran17eebf92020-02-05 16:35:49 +0000736 dlog_error("Unable to initialise VM.\n");
Andrew Scull72b43c02019-09-18 13:53:45 +0100737 return false;
738 }
739
Andrew Scull3c257452019-11-26 13:32:50 +0000740 vm_locked = vm_lock(vm);
741
Raghu Krishnamurthyd3ab8c32021-02-10 19:11:30 -0800742 /*
Raghu Krishnamurthy2b801692021-07-18 17:46:43 -0700743 * Grant the VM access to the memory. For VM's we mark all memory in
744 * stage-2 tables as RWX and the VM can control permissions using
745 * stage-1 translations. For S-EL0 partitions, hafnium maps the entire
746 * region of memory for the partition as RX. The partition is then
747 * expected to perform its owns relocations and call the FFA_MEM_PERM_*
748 * API's to change permissions on its image layout.
Raghu Krishnamurthyd3ab8c32021-02-10 19:11:30 -0800749 */
Daniel Boulbye6df3452022-07-14 18:14:26 +0100750 if (is_el0_partition) {
Raghu Krishnamurthy2b801692021-07-18 17:46:43 -0700751 map_mode = MM_MODE_R | MM_MODE_X | MM_MODE_USER | MM_MODE_NG;
752 } else {
753 map_mode = MM_MODE_R | MM_MODE_W | MM_MODE_X;
Raghu Krishnamurthyd3ab8c32021-02-10 19:11:30 -0800754 }
Raghu Krishnamurthy2b801692021-07-18 17:46:43 -0700755
Raghu Krishnamurthyd3ab8c32021-02-10 19:11:30 -0800756 if (!vm_identity_map(vm_locked, mem_begin, mem_end, map_mode, ppool,
Madhukar Pappireddy958c8412024-11-25 09:54:17 -0600757 &partition_primary_ep)) {
Andrew Walbran17eebf92020-02-05 16:35:49 +0000758 dlog_error("Unable to initialise memory.\n");
Andrew Scull3c257452019-11-26 13:32:50 +0000759 ret = false;
760 goto out;
Andrew Scull72b43c02019-09-18 13:53:45 +0100761 }
762
Olivier Deprez62d99e32020-01-09 15:58:07 +0100763 if (manifest_vm->is_ffa_partition) {
Daniel Boulbye6df3452022-07-14 18:14:26 +0100764 if (!ffa_map_memory_regions(manifest_vm, vm_locked,
J-Alvesd8a1d362023-03-08 11:15:28 +0000765 primary_vm_locked, is_el0_partition,
766 ppool)) {
Daniel Boulbye6df3452022-07-14 18:14:26 +0100767 ret = false;
768 goto out;
Manish Pandey2145c212020-05-01 16:04:22 +0100769 }
Madhukar Pappireddy958c8412024-11-25 09:54:17 -0600770 partition_primary_ep = ipa_add(
771 partition_primary_ep, manifest_vm->partition.ep_offset);
Manish Pandey2145c212020-05-01 16:04:22 +0100772
Madhukar Pappireddy958c8412024-11-25 09:54:17 -0600773 /*
774 * If MP endpoints dont specify the entrypoint for secondary
775 * execution contexts, FF-A spec advises SPMC to reuse the
776 * entrypoint computed for primary execution context.
777 */
778 vm->secondary_ep = partition_primary_ep;
Olivier Deprez62d99e32020-01-09 15:58:07 +0100779 }
780
Raghu Krishnamurthyd3ab8c32021-02-10 19:11:30 -0800781 /*
782 * Map hypervisor into the VM's page table. The hypervisor pages will
783 * not be accessible from EL0 since it will not be marked for user
784 * access.
785 * TODO: Map only the exception vectors and data that exception vectors
786 * require and not the entire hypervisor. This helps with speculative
787 * side-channel attacks.
788 */
Daniel Boulbye6df3452022-07-14 18:14:26 +0100789 if (is_el0_partition) {
Raghu Krishnamurthyd3ab8c32021-02-10 19:11:30 -0800790 CHECK(vm_identity_map(vm_locked, layout_text_begin(),
791 layout_text_end(), MM_MODE_X, ppool,
792 NULL));
793
794 CHECK(vm_identity_map(vm_locked, layout_rodata_begin(),
795 layout_rodata_end(), MM_MODE_R, ppool,
796 NULL));
797
798 CHECK(vm_identity_map(vm_locked, layout_data_begin(),
799 layout_data_end(), MM_MODE_R | MM_MODE_W,
800 ppool, NULL));
Maksims Svecovs134b8f92022-03-04 15:14:09 +0000801
802 CHECK(arch_stack_mm_init(mm_lock_ptable_unsafe(&vm->ptable),
803 ppool));
804
Raghu Krishnamurthyd3ab8c32021-02-10 19:11:30 -0800805 plat_console_mm_init(mm_lock_ptable_unsafe(&vm->ptable), ppool);
806 }
807
Manish Pandeyd34f8892020-06-19 17:41:07 +0100808 if (!load_common(stage1_locked, vm_locked, manifest_vm, ppool)) {
809 ret = false;
810 goto out;
811 }
812
Karl Meakine8937d92024-03-19 16:04:25 +0000813 dlog_info("Loaded with %u vCPUs, entry at %#lx.\n",
Manish Pandey2145c212020-05-01 16:04:22 +0100814 manifest_vm->secondary.vcpu_count, pa_addr(mem_begin));
815
Andrew Scull72b43c02019-09-18 13:53:45 +0100816 vcpu = vm_get_vcpu(vm, 0);
Fuad Tabba50469e02020-06-30 15:14:28 +0100817
Max Shvetsov40108e72020-08-27 12:39:50 +0100818 vcpu_locked = vcpu_lock(vcpu);
Madhukar Pappireddy046dad02022-06-21 18:43:33 -0500819
Fuad Tabba50469e02020-06-30 15:14:28 +0100820 if (has_fdt) {
Madhukar Pappireddy958c8412024-11-25 09:54:17 -0600821 vcpu_secondary_reset_and_start(
822 vcpu_locked, partition_primary_ep, pa_addr(fdt_addr));
Fuad Tabba50469e02020-06-30 15:14:28 +0100823 } else {
824 /*
825 * Without an FDT, secondary VMs expect the memory size to be
826 * passed in register x0, which is what
827 * vcpu_secondary_reset_and_start does in this case.
828 */
Madhukar Pappireddy958c8412024-11-25 09:54:17 -0600829 vcpu_secondary_reset_and_start(vcpu_locked,
830 partition_primary_ep, mem_size);
Fuad Tabba50469e02020-06-30 15:14:28 +0100831 }
832
Max Shvetsov40108e72020-08-27 12:39:50 +0100833 vcpu_unlock(&vcpu_locked);
834
Jens Wiklanderb697ad02022-11-04 10:15:03 +0100835 /*
836 * For all vCPUs,
837 * in a VM: enable the notification pending virtual interrupt if
838 * requested in the manifest.
839 * in a SP: enable the NPI and managed exit virtual interrupts if
840 * requested in the manifest. For a S-EL0 partition, enable
841 * the virtual interrupts IDs matching the secure physical
842 * interrupt IDs declared in device regions.
843 */
844 for (n = 0; n < manifest_vm->secondary.vcpu_count; n++) {
845 vcpu = vm_get_vcpu(vm, n);
846 vcpu_locked = vcpu_lock(vcpu);
Karl Meakin117c8082024-12-04 16:03:28 +0000847 ffa_interrupts_enable_virtual_interrupts(vcpu_locked,
848 vm_locked);
Jens Wiklanderb697ad02022-11-04 10:15:03 +0100849 vcpu_unlock(&vcpu_locked);
850 }
851
Andrew Scull3c257452019-11-26 13:32:50 +0000852 ret = true;
Andrew Scull72b43c02019-09-18 13:53:45 +0100853
Andrew Scull3c257452019-11-26 13:32:50 +0000854out:
855 vm_unlock(&vm_locked);
856
857 return ret;
Andrew Scull72b43c02019-09-18 13:53:45 +0100858}
859
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +0100860/**
Andrew Walbran34ce72e2018-09-13 16:47:44 +0100861 * Try to find a memory range of the given size within the given ranges, and
862 * remove it from them. Return true on success, or false if no large enough
863 * contiguous range is found.
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +0100864 */
Hong-Seok Kim09648362019-05-23 15:47:11 +0900865static bool carve_out_mem_range(struct mem_range *mem_ranges,
866 size_t mem_ranges_count, uint64_t size_to_find,
867 paddr_t *found_begin, paddr_t *found_end)
Andrew Walbran34ce72e2018-09-13 16:47:44 +0100868{
869 size_t i;
870
Wedson Almeida Filho81568c42019-01-04 13:33:02 +0000871 /*
872 * TODO(b/116191358): Consider being cleverer about how we pack VMs
873 * together, with a non-greedy algorithm.
874 */
Andrew Walbran34ce72e2018-09-13 16:47:44 +0100875 for (i = 0; i < mem_ranges_count; ++i) {
876 if (size_to_find <=
Andrew Walbran2cb43392019-04-17 12:52:45 +0100877 pa_difference(mem_ranges[i].begin, mem_ranges[i].end)) {
Wedson Almeida Filhob2c159e2018-10-25 13:27:47 +0100878 /*
879 * This range is big enough, take some of it from the
880 * end and reduce its size accordingly.
881 */
Andrew Walbran34ce72e2018-09-13 16:47:44 +0100882 *found_end = mem_ranges[i].end;
883 *found_begin = pa_init(pa_addr(mem_ranges[i].end) -
884 size_to_find);
885 mem_ranges[i].end = *found_begin;
886 return true;
887 }
888 }
889 return false;
890}
891
892/**
893 * Given arrays of memory ranges before and after memory was removed for
894 * secondary VMs, add the difference to the reserved ranges of the given update.
895 * Return true on success, or false if there would be more than MAX_MEM_RANGES
896 * reserved ranges after adding the new ones.
897 * `before` and `after` must be arrays of exactly `mem_ranges_count` elements.
898 */
Hong-Seok Kim09648362019-05-23 15:47:11 +0900899static bool update_reserved_ranges(struct boot_params_update *update,
900 const struct mem_range *before,
901 const struct mem_range *after,
902 size_t mem_ranges_count)
Andrew Walbran34ce72e2018-09-13 16:47:44 +0100903{
904 size_t i;
905
906 for (i = 0; i < mem_ranges_count; ++i) {
907 if (pa_addr(after[i].begin) > pa_addr(before[i].begin)) {
908 if (update->reserved_ranges_count >= MAX_MEM_RANGES) {
Andrew Walbran17eebf92020-02-05 16:35:49 +0000909 dlog_error(
910 "Too many reserved ranges after "
911 "loading secondary VMs.\n");
Andrew Walbran34ce72e2018-09-13 16:47:44 +0100912 return false;
913 }
914 update->reserved_ranges[update->reserved_ranges_count]
915 .begin = before[i].begin;
916 update->reserved_ranges[update->reserved_ranges_count]
917 .end = after[i].begin;
918 update->reserved_ranges_count++;
919 }
920 if (pa_addr(after[i].end) < pa_addr(before[i].end)) {
921 if (update->reserved_ranges_count >= MAX_MEM_RANGES) {
Andrew Walbran17eebf92020-02-05 16:35:49 +0000922 dlog_error(
923 "Too many reserved ranges after "
924 "loading secondary VMs.\n");
Andrew Walbran34ce72e2018-09-13 16:47:44 +0100925 return false;
926 }
927 update->reserved_ranges[update->reserved_ranges_count]
928 .begin = after[i].end;
929 update->reserved_ranges[update->reserved_ranges_count]
930 .end = before[i].end;
931 update->reserved_ranges_count++;
932 }
933 }
934
935 return true;
936}
937
Olivier Deprez05046922023-03-09 15:48:40 +0100938static bool init_other_world_vm(const struct boot_params *params,
939 struct mpool *ppool)
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +0100940{
Olivier Deprez96a2a262020-06-11 17:21:38 +0200941 struct vm *other_world_vm;
Andrew Walbran34ce72e2018-09-13 16:47:44 +0100942 size_t i;
Andrew Scull72b43c02019-09-18 13:53:45 +0100943
Andrew Walbran9daa57e2019-09-27 13:33:20 +0100944 /*
Olivier Deprez96a2a262020-06-11 17:21:38 +0200945 * Initialise the dummy VM which represents the opposite world:
946 * -TrustZone (or the SPMC) when running the Hypervisor
947 * -the Hypervisor when running TZ/SPMC
Andrew Walbran9daa57e2019-09-27 13:33:20 +0100948 */
Madhukar Pappireddy070f49e2024-01-12 13:02:27 -0600949 other_world_vm = vm_init(HF_OTHER_WORLD_ID, MAX_CPUS, ppool, false, 0);
Olivier Deprez96a2a262020-06-11 17:21:38 +0200950 CHECK(other_world_vm != NULL);
951
952 for (i = 0; i < MAX_CPUS; i++) {
953 struct vcpu *vcpu = vm_get_vcpu(other_world_vm, i);
954 struct cpu *cpu = cpu_find_index(i);
955
956 vcpu->cpu = cpu;
957 }
Andrew Walbran9daa57e2019-09-27 13:33:20 +0100958
Olivier Deprez05046922023-03-09 15:48:40 +0100959 return arch_other_world_vm_init(other_world_vm, params, ppool);
Olivier Deprez112d2b52020-09-30 07:39:23 +0200960}
961
962/*
963 * Loads alls VMs from the manifest.
964 */
965bool load_vms(struct mm_stage1_locked stage1_locked,
966 const struct manifest *manifest, const struct memiter *cpio,
967 const struct boot_params *params,
968 struct boot_params_update *update, struct mpool *ppool)
969{
970 struct vm *primary;
971 struct mem_range mem_ranges_available[MAX_MEM_RANGES];
972 struct vm_locked primary_vm_locked;
973 size_t i;
974 bool success = true;
975
Andrew Walbranf8716782020-10-29 17:07:07 +0000976 /**
977 * Only try to load the primary VM if it is supposed to be in this
978 * world.
979 */
980 if (vm_id_is_current_world(HF_PRIMARY_VM_ID)) {
981 if (!load_primary(stage1_locked,
982 &manifest->vm[HF_PRIMARY_VM_INDEX], cpio,
983 params, ppool)) {
984 dlog_error("Unable to load primary VM.\n");
985 return false;
986 }
Olivier Deprez112d2b52020-09-30 07:39:23 +0200987 }
988
Olivier Deprez05046922023-03-09 15:48:40 +0100989 if (!init_other_world_vm(params, ppool)) {
Olivier Deprez112d2b52020-09-30 07:39:23 +0200990 return false;
991 }
992
Andrew Walbran34ce72e2018-09-13 16:47:44 +0100993 static_assert(
994 sizeof(mem_ranges_available) == sizeof(params->mem_ranges),
995 "mem_range arrays must be the same size for memcpy.");
996 static_assert(sizeof(mem_ranges_available) < 500,
997 "This will use too much stack, either make "
998 "MAX_MEM_RANGES smaller or change this.");
Andrew Sculla1aa2ba2019-04-05 11:49:02 +0100999 memcpy_s(mem_ranges_available, sizeof(mem_ranges_available),
1000 params->mem_ranges, sizeof(params->mem_ranges));
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +01001001
Andrew Walbran34ce72e2018-09-13 16:47:44 +01001002 /* Round the last addresses down to the page size. */
1003 for (i = 0; i < params->mem_ranges_count; ++i) {
Alfredo Mazzinghieb1997c2019-02-07 18:00:01 +00001004 mem_ranges_available[i].end = pa_init(align_down(
1005 pa_addr(mem_ranges_available[i].end), PAGE_SIZE));
Andrew Walbran34ce72e2018-09-13 16:47:44 +01001006 }
Wedson Almeida Filho84a30a02018-07-23 20:05:05 +01001007
Andrew Scull3c257452019-11-26 13:32:50 +00001008 primary = vm_find(HF_PRIMARY_VM_ID);
1009 primary_vm_locked = vm_lock(primary);
1010
David Brazdil0251b942019-09-10 15:59:50 +01001011 for (i = 0; i < manifest->vm_count; ++i) {
David Brazdil0dbb41f2019-09-09 18:03:35 +01001012 const struct manifest_vm *manifest_vm = &manifest->vm[i];
J-Alves19e20cf2023-08-02 12:48:55 +01001013 ffa_id_t vm_id = HF_VM_ID_OFFSET + i;
David Brazdil7a462ec2019-08-15 12:27:47 +01001014 uint64_t mem_size;
Andrew Scull80871322018-08-06 12:04:09 +01001015 paddr_t secondary_mem_begin;
1016 paddr_t secondary_mem_end;
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +01001017
David Brazdil7a462ec2019-08-15 12:27:47 +01001018 if (vm_id == HF_PRIMARY_VM_ID) {
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +01001019 continue;
1020 }
1021
Olivier Deprez2a8ee342020-08-03 15:10:44 +02001022 dlog_info("Loading VM id %#x: %s.\n", vm_id,
Karl Meakine8937d92024-03-19 16:04:25 +00001023 manifest_vm->debug_name.data);
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +01001024
David Brazdil7a462ec2019-08-15 12:27:47 +01001025 mem_size = align_up(manifest_vm->secondary.mem_size, PAGE_SIZE);
Olivier Deprez62d99e32020-01-09 15:58:07 +01001026
Raghu Krishnamurthyb49549e2021-07-02 08:27:38 -07001027 if (manifest_vm->is_ffa_partition &&
1028 !manifest->vm[i].is_hyp_loaded) {
Olivier Deprez62d99e32020-01-09 15:58:07 +01001029 secondary_mem_begin =
Raghu Krishnamurthy8c250a92021-07-02 12:16:42 -07001030 pa_init(manifest_vm->partition.load_addr);
1031 secondary_mem_end = pa_init(
1032 manifest_vm->partition.load_addr + mem_size);
Olivier Deprez62d99e32020-01-09 15:58:07 +01001033 } else if (!carve_out_mem_range(mem_ranges_available,
1034 params->mem_ranges_count,
1035 mem_size, &secondary_mem_begin,
1036 &secondary_mem_end)) {
Karl Meakine8937d92024-03-19 16:04:25 +00001037 dlog_error("Not enough memory (%lu bytes).\n",
1038 mem_size);
Andrew Walbran34ce72e2018-09-13 16:47:44 +01001039 continue;
1040 }
Andrew Scull80871322018-08-06 12:04:09 +01001041
Manish Pandey2145c212020-05-01 16:04:22 +01001042 if (!load_secondary(stage1_locked, primary_vm_locked,
1043 secondary_mem_begin, secondary_mem_end,
J-Alves77b6f4f2023-03-15 11:34:49 +00001044 manifest_vm, params, cpio, ppool)) {
Andrew Walbran17eebf92020-02-05 16:35:49 +00001045 dlog_error("Unable to load VM.\n");
Wedson Almeida Filho84a30a02018-07-23 20:05:05 +01001046 continue;
1047 }
1048
1049 /* Deny the primary VM access to this memory. */
Andrew Scull3c257452019-11-26 13:32:50 +00001050 if (!vm_unmap(primary_vm_locked, secondary_mem_begin,
1051 secondary_mem_end, ppool)) {
Andrew Walbran17eebf92020-02-05 16:35:49 +00001052 dlog_error(
1053 "Unable to unmap secondary VM from primary "
1054 "VM.\n");
Andrew Scull3c257452019-11-26 13:32:50 +00001055 success = false;
1056 break;
Wedson Almeida Filho84a30a02018-07-23 20:05:05 +01001057 }
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +01001058 }
1059
Andrew Scull3c257452019-11-26 13:32:50 +00001060 vm_unlock(&primary_vm_locked);
1061
1062 if (!success) {
1063 return false;
1064 }
1065
Wedson Almeida Filhob2c159e2018-10-25 13:27:47 +01001066 /*
1067 * Add newly reserved areas to update params by looking at the
Andrew Walbran34ce72e2018-09-13 16:47:44 +01001068 * difference between the available ranges from the original params and
1069 * the updated mem_ranges_available. We assume that the number and order
1070 * of available ranges is the same, i.e. we don't remove any ranges
Wedson Almeida Filhob2c159e2018-10-25 13:27:47 +01001071 * above only make them smaller.
1072 */
Andrew Walbran34ce72e2018-09-13 16:47:44 +01001073 return update_reserved_ranges(update, params->mem_ranges,
1074 mem_ranges_available,
1075 params->mem_ranges_count);
Wedson Almeida Filhofdf4afc2018-07-19 15:45:21 +01001076}