refactor(lib/realm): remove 'lib/realm' library
This patch removes the lib/realm library by refactoring its code and
moving its components to related submodules. It also introduces two
new libraries, 'lib/granule' and 'lib/slot_buf' with sources from
'lib/realm'
Change-Id: Id41eb2da20172d8da9a5f066b3e78ec4f1a6d35d
Signed-off-by: Javier Almansa Sobrino <javier.almansasobrino@arm.com>
Signed-off-by: Soby Mathew <soby.mathew@arm.com>
diff --git a/runtime/CMakeLists.txt b/runtime/CMakeLists.txt
index b85c0d1..df6b0d3 100644
--- a/runtime/CMakeLists.txt
+++ b/runtime/CMakeLists.txt
@@ -61,7 +61,6 @@
PRIVATE "core/aarch64/entry.S"
"core/aarch64/head.S"
"core/aarch64/helpers.S"
- "core/aarch64/ns_access.S"
"core/aarch64/run-asm.S"
"core/aarch64/vectors.S")
else()
@@ -75,7 +74,7 @@
"core/init.c"
"core/inject_exp.c"
"core/sysregs.c"
- "core/vmid.c")
+ "core/timers.c")
target_sources(rmm-runtime
PRIVATE "rmi/feature.c"
diff --git a/runtime/core/aarch64/ns_access.S b/runtime/core/aarch64/ns_access.S
deleted file mode 100644
index b39d2ec..0000000
--- a/runtime/core/aarch64/ns_access.S
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * SPDX-License-Identifier: BSD-3-Clause
- * SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
- */
-
-#include <asm_macros.S>
-
-.section ".text"
-
-/*
- * The following addresses are registered with the exception handler:
- */
-.global ns_read
-.global ns_write
-
-.global memcpy_ns_read
-.global memcpy_ns_write
-.global ns_access_ret_0
-
-/*
- * Copy data from NS into Realm memory.
- * The function returns 1 if the copy succeeds.
- * If the access to the NS memory generates a GPF, the exception handler
- * returns to ns_access_ret_0 and 0 is returned to the caller.
- * In case of failure (when 0 is returned), partial data may have been
- * written to the destination buffer
- *
- * x0 - The address of buffer in Realm memory to write into
- * x1 - The address of buffer in NS memory to read from.
- * x2 - The number of bytes to read in bytes.
- * All arguments must be aligned to 8 bytes.
- */
-func memcpy_ns_read
- cbz x2, 2f
- mov x3, #0
-1:
-ns_read:
- ldr x4, [x1], #8
- str x4, [x0], #8
- add x3, x3, #8
- cmp x3, x2
- bne 1b
-2:
- mov x0, #1
- ret
-endfunc memcpy_ns_read
-
-/*
- * Copy data from Realm into NS memory.
- * The function returns 1 if the copy succeeds.
- * If the access to the NS memory generates a GPF, the exception handler
- * returns to ns_access_ret_0 and 0 is returned to the caller.
- * In case of failure (when 0 is returned), partial data may have been
- * written to the destination buffer
- *
- * x0 - The address of buffer in NS memory to write into
- * x1 - The address of buffer in Realm memory to read from.
- * x2 - The number of bytes to write.
- * All arguments must be aligned to 8 bytes.
- */
-func memcpy_ns_write
- cbz x2, 2f
- mov x3, #0
-1:
- ldr x4, [x1], #8
-ns_write:
- str x4, [x0], #8
- add x3, x3, #8
- cmp x3, x2
- bne 1b
-2:
- mov x0, #1
- ret
-endfunc memcpy_ns_write
-
-func ns_access_ret_0
- mov x0, #0
- ret
-endfunc ns_access_ret_0
diff --git a/runtime/core/run.c b/runtime/core/run.c
index 88a0eb3..d189399 100644
--- a/runtime/core/run.c
+++ b/runtime/core/run.c
@@ -274,7 +274,7 @@
rec->ns = ns_state;
/* Map auxiliary granules */
- rec_aux = aux_granules_map(rec->g_aux, rec->num_rec_aux);
+ rec_aux = buffer_aux_granules_map(rec->g_aux, rec->num_rec_aux);
/*
* Associate the attest heap with the current CPU. This heap will be
@@ -377,5 +377,5 @@
assert(ret == 0);
/* Unmap auxiliary granules */
- aux_granules_unmap(rec_aux, rec->num_rec_aux);
+ buffer_aux_unmap(rec_aux, rec->num_rec_aux);
}
diff --git a/runtime/core/timers.c b/runtime/core/timers.c
new file mode 100644
index 0000000..931a80a
--- /dev/null
+++ b/runtime/core/timers.c
@@ -0,0 +1,94 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
+ */
+
+#include <arch_helpers.h>
+#include <rec.h>
+#include <smc-rmi.h>
+#include <timers.h>
+
+/*
+ * Check that timer output is asserted:
+ * Timer enabled: CNTx_CTL_ENABLE = 1
+ * Timer condition is met: CNTx_CTL_ISTATUS = 1
+ * Timer interrupt is not masked: CNTx_CTL_IMASK = 0
+ */
+#define TIMER_ASSERTED(reg) \
+ (((reg) & \
+ (CNTx_CTL_ENABLE | CNTx_CTL_ISTATUS | CNTx_CTL_IMASK)) == \
+ (CNTx_CTL_ENABLE | CNTx_CTL_ISTATUS))
+
+/*
+ * Check the pending state of the timers.
+ *
+ * When a timer output is asserted, its interrupt signal should be masked at
+ * EL2 when running the Realm to prevent the physical interrupt from
+ * continuously exiting the Realm.
+ *
+ * When a timer output is not asserted, the interrupt signal should be
+ * unmasked such that if the timer output becomes asserted again, an exit from
+ * the Realm happens due to a physical IRQ and we can inject a virtual
+ * interrupt again.
+ */
+bool check_pending_timers(struct rec *rec)
+{
+ unsigned long cntv_ctl = read_cntv_ctl_el02();
+ unsigned long cntp_ctl = read_cntp_ctl_el02();
+ unsigned long cnthctl_old = rec->sysregs.cnthctl_el2;
+
+ if (TIMER_ASSERTED(cntv_ctl)) {
+ rec->sysregs.cnthctl_el2 |= CNTHCTL_EL2_CNTVMASK;
+ } else {
+ rec->sysregs.cnthctl_el2 &= ~CNTHCTL_EL2_CNTVMASK;
+ }
+
+ if (TIMER_ASSERTED(cntp_ctl)) {
+ rec->sysregs.cnthctl_el2 |= CNTHCTL_EL2_CNTPMASK;
+ } else {
+ rec->sysregs.cnthctl_el2 &= ~CNTHCTL_EL2_CNTPMASK;
+ }
+
+ if (cnthctl_old != rec->sysregs.cnthctl_el2) {
+ write_cnthctl_el2(rec->sysregs.cnthctl_el2);
+ isb();
+ }
+
+ /*
+ * We don't want to run the Realm just to immediately exit due a
+ * physical interrupt caused by one of the timer interrupts not having
+ * been retired from the CPU interface yet. Check that the interrupts
+ * are retired before entering the Realm.
+ */
+ while (true) {
+ unsigned long hppir = read_icc_hppir1_el1();
+ unsigned int intid =
+ (unsigned int)EXTRACT(ICC_HPPIR1_EL1_INTID, hppir);
+
+ if (!((((rec->sysregs.cnthctl_el2 & CNTHCTL_EL2_CNTVMASK) != 0UL) &&
+ (intid == EL1_VIRT_TIMER_PPI)) ||
+ (((rec->sysregs.cnthctl_el2 & CNTHCTL_EL2_CNTPMASK) != 0UL) &&
+ (intid == EL1_PHYS_TIMER_PPI)))) {
+ break;
+ }
+ }
+
+ /*
+ * Check if the timers changed their output status based on
+ * the previously saved timer state at the last Realm exit.
+ */
+ return (TIMER_ASSERTED(cntv_ctl) !=
+ TIMER_ASSERTED(rec->sysregs.cntv_ctl_el0)) ||
+ (TIMER_ASSERTED(cntp_ctl) !=
+ TIMER_ASSERTED(rec->sysregs.cntp_ctl_el0));
+}
+
+void report_timer_state_to_ns(struct rmi_rec_exit *rec_exit)
+{
+ /* Expose Realm EL1 timer state */
+ rec_exit->cntv_ctl = read_cntv_ctl_el02();
+ rec_exit->cntv_cval = read_cntv_cval_el02() - read_cntvoff_el2();
+
+ rec_exit->cntp_ctl = read_cntp_ctl_el02();
+ rec_exit->cntp_cval = read_cntp_cval_el02() - read_cntpoff_el2();
+}
diff --git a/runtime/core/vmid.c b/runtime/core/vmid.c
deleted file mode 100644
index ba9738a..0000000
--- a/runtime/core/vmid.c
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * SPDX-License-Identifier: BSD-3-Clause
- * SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
- */
-
-#include <arch_features.h>
-#include <assert.h>
-#include <atomics.h>
-#include <sizes.h>
-#include <utils_def.h>
-#include <vmid.h>
-
-#define VMID8_COUNT (U(1) << 8)
-#define VMID16_COUNT (U(1) << 16)
-#define MAX_VMID_COUNT VMID16_COUNT
-#define VMID_ARRAY_LONG_SIZE (MAX_VMID_COUNT / BITS_PER_UL)
-
-/*
- * The bitmap for the reserved/used VMID values.
- */
-IF_NCBMC(static) unsigned long vmids[VMID_ARRAY_LONG_SIZE];
-
-/*
- * Marks the VMID value to be in use. It returns:
- * - True, on success
- * - False, if the vmid is out of range,
- * or if it was already reserved (in use).
- */
-bool vmid_reserve(unsigned int vmid)
-{
- unsigned int offset;
- unsigned int vmid_count;
-
- /* Number of supported VMID values */
- vmid_count = is_feat_vmid16_present() ? VMID16_COUNT : VMID8_COUNT;
-
- /*
- * The input from NS as part of RMI_REALM_CREATE is 'short int' type,
- * so this check will not fail on systems with FEAT_VMID16 implemented.
- */
- if (vmid >= vmid_count) {
- return false;
- }
-
- offset = vmid / BITS_PER_UL;
- vmid %= BITS_PER_UL;
-
- return !atomic_bit_set_acquire_release_64(&vmids[offset], vmid);
-}
-
-/*
- * Marks the VMID value to be not in use.
- */
-void vmid_free(unsigned int vmid)
-{
- unsigned int offset;
- unsigned int __unused vmid_count;
-
- /* Number of supported VMID values */
- vmid_count = is_feat_vmid16_present() ? VMID16_COUNT : VMID8_COUNT;
-
- /* Check the number of supported VMID values */
- assert(vmid < vmid_count);
-
- offset = vmid / BITS_PER_UL;
- vmid %= BITS_PER_UL;
-
- atomic_bit_clear_release_64(&vmids[offset], vmid);
-}
diff --git a/runtime/include/realm.h b/runtime/include/realm.h
new file mode 100644
index 0000000..0b89747
--- /dev/null
+++ b/runtime/include/realm.h
@@ -0,0 +1,233 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
+ */
+
+#ifndef REALM_H
+#define REALM_H
+
+#include <assert.h>
+#include <measurement.h>
+#include <memory.h>
+#include <rec.h>
+#include <s2tt.h>
+
+#define REALM_NEW 0U
+#define REALM_ACTIVE 1U
+#define REALM_SYSTEM_OFF 2U
+
+/* struct rd is protected by the rd granule lock */
+struct rd {
+ /*
+ * 'state' & 'rec_count' are only accessed through dedicated
+ * primitives where the following rules apply:
+ *
+ * (1) To write the value, the RMI handler must hold the rd granule
+ * lock and use a single copy atomic store with release semantics.
+ *
+ * (2) To read the value, the RMI handler must either:
+ * - Hold the rd granule lock and use a 64-bit single copy
+ * atomic load, or
+ * - Hold the rd reference count and use a 64-bit single copy
+ * atomic load with acquire semantics.
+ *
+ * Other members of the structure are accessed with rd granule lock held.
+ */
+ /* 64-bit variable accessed with READ64/WRITE64/ACQUIRE semantic */
+ unsigned long state;
+
+ /* Reference count */
+ unsigned long rec_count;
+
+ /* Realm measurement 8 bytes aligned */
+ unsigned char measurement[MEASUREMENT_SLOT_NR][MAX_MEASUREMENT_SIZE];
+
+ /* Stage 2 configuration of the Realm */
+ struct s2tt_context s2_ctx;
+
+ /* Number of auxiliary REC granules for the Realm */
+ unsigned int num_rec_aux;
+
+ /* Algorithm to use for measurements */
+ enum hash_algo algorithm;
+
+ /* PMU enabled flag */
+ bool pmu_enabled;
+
+ /* Number of PMU counters */
+ unsigned int pmu_num_ctrs;
+
+ /* SIMD configuration */
+ struct simd_config simd_cfg;
+
+ /* Realm Personalization Value */
+ unsigned char rpv[RPV_SIZE];
+};
+COMPILER_ASSERT((U(offsetof(struct rd, measurement)) & 7U) == 0U);
+COMPILER_ASSERT(sizeof(struct rd) <= GRANULE_SIZE);
+
+/*
+ * Sets the rd's state while holding the rd granule lock.
+ */
+static inline void set_rd_state(struct rd *rd, unsigned long state)
+{
+ SCA_WRITE64_RELEASE(&rd->state, state);
+}
+
+/*
+ * Gets the rd's state while holding the rd granule lock.
+ */
+static inline unsigned long get_rd_state_locked(struct rd *rd)
+{
+ return SCA_READ64(&rd->state);
+}
+
+/*
+ * Gets the rd's state while holding the rd's reference count, without
+ * holding the rd granule lock.
+ */
+static inline unsigned long get_rd_state_unlocked(struct rd *rd)
+{
+ return SCA_READ64_ACQUIRE(&rd->state);
+}
+
+/*
+ * Sets the rd's rec_count while holding the rd granule lock.
+ */
+static inline void set_rd_rec_count(struct rd *rd, unsigned long val)
+{
+ SCA_WRITE64_RELEASE(&rd->rec_count, val);
+}
+
+/*
+ * Gets the rd's rec_count while holding the rd granule lock.
+ */
+static inline unsigned long get_rd_rec_count_locked(struct rd *rd)
+{
+ return SCA_READ64(&rd->rec_count);
+}
+
+/*
+ * Gets the rd's rec_count while holding the rd's reference count, without
+ * holding the rd granule lock.
+ */
+static inline unsigned long get_rd_rec_count_unlocked(struct rd *rd)
+{
+ return SCA_READ64_ACQUIRE(&rd->rec_count);
+}
+
+static inline unsigned long realm_ipa_bits(struct rd *rd)
+{
+ return rd->s2_ctx.ipa_bits;
+}
+
+/*
+ * Gets the rd's IPA size.
+ */
+static inline unsigned long realm_ipa_size(struct rd *rd)
+{
+ return (1UL << realm_ipa_bits(rd));
+}
+
+static inline unsigned long realm_par_size(struct rd *rd)
+{
+ return (realm_ipa_size(rd) >> 1UL);
+}
+
+static inline int realm_rtt_starting_level(struct rd *rd)
+{
+ return rd->s2_ctx.s2_starting_level;
+}
+
+/*
+ * Checks that 'address' is within container's parameters.
+ *
+ * 'container_base' is the start address of the container.
+ * 'container_end' is the first address after the container.
+ * The container must not overflow.
+ */
+static inline bool addr_is_contained(unsigned long container_base,
+ unsigned long container_end,
+ unsigned long address)
+{
+ /* Sanity check the container bounds */
+ if (container_base > (container_end - 1UL)) {
+ return false;
+ }
+
+ return (address >= container_base) && (address <= (container_end - 1UL));
+}
+
+/*
+ * Checks that region is within container's parameters.
+ *
+ * 'container_base' is the start address of the container.
+ * 'container_end' is the first address after the container.
+ * The container must not overflow.
+ * 'region_base' is the start address of the region.
+ * 'region_end' is the first address after the region.
+ * The region must not overflow.
+ */
+static inline bool region_is_contained(unsigned long container_base,
+ unsigned long container_end,
+ unsigned long region_base,
+ unsigned long region_end)
+{
+ /* Sanity check the region bounds */
+ if (region_base > (region_end - 1UL)) {
+ return false;
+ }
+
+ return addr_is_contained(container_base, container_end, region_base) &&
+ addr_is_contained(container_base, container_end, region_end - 1UL);
+}
+
+static inline unsigned long rec_ipa_size(struct rec *rec)
+{
+ return (1UL << rec->realm_info.s2_ctx.ipa_bits);
+}
+
+static inline unsigned long rec_par_size(struct rec *rec)
+{
+ return (rec_ipa_size(rec) >> 1U);
+}
+
+static inline bool addr_in_rec_par(struct rec *rec, unsigned long addr)
+{
+ return (addr < rec_par_size(rec));
+}
+
+static inline bool region_in_rec_par(struct rec *rec,
+ unsigned long base, unsigned long end)
+{
+ return region_is_contained(0UL, rec_par_size(rec), base, end);
+}
+
+static inline bool addr_in_par(struct rd *rd, unsigned long addr)
+{
+ return (addr < realm_par_size(rd));
+}
+
+enum s2_walk_status {
+ /* Successful translation */
+ WALK_SUCCESS,
+ /* Parameter 'ipa' is unaligned or is not Protected IPA */
+ WALK_INVALID_PARAMS,
+ /* Mapping is not in the page table */
+ WALK_FAIL
+};
+
+struct s2_walk_result {
+ unsigned long pa;
+ unsigned long rtt_level;
+ enum ripas ripas_val;
+ struct granule *llt;
+};
+
+enum s2_walk_status realm_ipa_to_pa(struct rec *rec,
+ unsigned long ipa,
+ struct s2_walk_result *s2_walk);
+
+enum s2_walk_status realm_ipa_get_ripas(struct rec *rec, unsigned long ipa,
+ enum ripas *ripas_ptr);
+#endif /* REALM_H */
diff --git a/runtime/include/rec.h b/runtime/include/rec.h
new file mode 100644
index 0000000..d65f723
--- /dev/null
+++ b/runtime/include/rec.h
@@ -0,0 +1,279 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
+ */
+
+#ifndef REC_H
+#define REC_H
+
+#ifndef __ASSEMBLER__
+
+#include <arch.h>
+#include <attestation_token.h>
+#include <gic.h>
+#include <memory_alloc.h>
+#include <pauth.h>
+#include <pmu.h>
+#include <ripas.h>
+#include <s2tt.h>
+#include <simd.h>
+#include <sizes.h>
+#include <smc-rmi.h>
+#include <utils_def.h>
+
+#ifndef CBMC
+#define RMM_REC_SAVED_GEN_REG_COUNT U(31)
+#define STRUCT_TYPE struct
+#else /* CBMC */
+#define RMM_REC_SAVED_GEN_REG_COUNT SMC_RESULT_REGS
+/*
+ * Some of the structures inside 'struct rec' don't influence the outcome of
+ * the CBMC tests, so for CBMC build make these a union to get smaller 'rec'
+ * structure.
+ */
+#define STRUCT_TYPE union
+#endif /* CBMC */
+
+struct granule;
+
+/*
+ * System registers whose contents are specific to a REC.
+ */
+STRUCT_TYPE sysreg_state {
+ unsigned long sp_el0;
+ unsigned long sp_el1;
+ unsigned long elr_el1;
+ unsigned long spsr_el1;
+ unsigned long pmcr_el0;
+ unsigned long tpidrro_el0;
+ unsigned long tpidr_el0;
+ unsigned long csselr_el1;
+ unsigned long sctlr_el1;
+ unsigned long actlr_el1;
+ unsigned long cpacr_el1;
+ unsigned long zcr_el1;
+ unsigned long ttbr0_el1;
+ unsigned long ttbr1_el1;
+ unsigned long tcr_el1;
+ unsigned long esr_el1;
+ unsigned long afsr0_el1;
+ unsigned long afsr1_el1;
+ unsigned long far_el1;
+ unsigned long mair_el1;
+ unsigned long vbar_el1;
+ unsigned long contextidr_el1;
+ unsigned long tpidr_el1;
+ unsigned long amair_el1;
+ unsigned long cntkctl_el1;
+ unsigned long par_el1;
+ unsigned long mdscr_el1;
+ unsigned long mdccint_el1;
+ unsigned long disr_el1;
+ unsigned long mpam0_el1;
+
+ /* Timer Registers */
+ unsigned long cnthctl_el2;
+ unsigned long cntvoff_el2;
+ unsigned long cntpoff_el2;
+ unsigned long cntp_ctl_el0;
+ unsigned long cntp_cval_el0;
+ unsigned long cntv_ctl_el0;
+ unsigned long cntv_cval_el0;
+
+ /* GIC Registers */
+ struct gic_cpu_state gicstate;
+
+ /* TODO MPAM */
+ /* TODO Performance Monitor Registers */
+ /* TODO Pointer Authentication Registers */
+
+ unsigned long vmpidr_el2; /* restored only */
+ unsigned long hcr_el2; /* restored only */
+
+ unsigned long cptr_el2; /* restored only */
+};
+
+/*
+ * System registers whose contents are
+ * common across all RECs in a Realm.
+ */
+STRUCT_TYPE common_sysreg_state {
+ unsigned long vttbr_el2;
+ unsigned long vtcr_el2;
+ unsigned long hcr_el2;
+ unsigned long mdcr_el2;
+};
+
+/*
+ * This structure is aligned on cache line size to avoid cache line trashing
+ * when allocated as an array for N CPUs.
+ */
+struct ns_state {
+ STRUCT_TYPE sysreg_state sysregs;
+ unsigned long sp_el0;
+ unsigned long icc_sre_el2;
+ struct pmu_state *pmu;
+} __aligned(CACHE_WRITEBACK_GRANULE);
+
+/*
+ * Data used when handling attestation requests
+ */
+struct rec_attest_data {
+ unsigned char rmm_realm_token_buf[SZ_1K];
+ size_t rmm_realm_token_len;
+
+ struct token_sign_cntxt token_sign_ctx;
+
+ /* Buffer allocation info used for heap init and management */
+ struct buffer_alloc_ctx alloc_ctx;
+};
+COMPILER_ASSERT(sizeof(struct rec_attest_data) <= GRANULE_SIZE);
+
+/*
+ * This structure contains pointers to data that are allocated
+ * in auxilary granules for a REC.
+ */
+struct rec_aux_data {
+ /* Pointer to the heap buffer */
+ uint8_t *attest_heap_buf;
+
+ /* Pointer to PMU state */
+ struct pmu_state *pmu;
+
+ /* SIMD context region */
+ struct simd_context *simd_ctx;
+
+ /* Pointer to attestation-related data */
+ struct rec_attest_data *attest_data;
+
+ /* Address of the attestation token buffer */
+ uintptr_t cca_token_buf;
+};
+
+struct rec {
+ struct granule *g_rec; /* the granule in which this REC lives */
+ unsigned long rec_idx; /* which REC is this */
+ bool runnable;
+
+ unsigned long regs[RMM_REC_SAVED_GEN_REG_COUNT];
+ unsigned long sp_el0;
+
+#ifndef CBMC
+ /*
+ * PAuth state of Realm.
+ * Note that we do not need to save NS state as EL3 will save this as part of world switch.
+ */
+ struct pauth_state pauth;
+#endif /* CBMC*/
+
+ unsigned long pc;
+ unsigned long pstate;
+
+ STRUCT_TYPE sysreg_state sysregs;
+ STRUCT_TYPE common_sysreg_state common_sysregs;
+
+ /* Populated when the REC issues a RIPAS change request */
+ struct {
+ unsigned long base;
+ unsigned long top;
+ unsigned long addr;
+ enum ripas ripas_val;
+ enum ripas_change_destroyed change_destroyed;
+ enum ripas_response response;
+ } set_ripas;
+
+ /*
+ * Common values across all RECs in a Realm.
+ */
+ struct {
+ struct granule *g_rd;
+ bool pmu_enabled;
+ unsigned int pmu_num_ctrs;
+ enum hash_algo algorithm;
+ struct simd_config simd_cfg;
+ struct s2tt_context s2_ctx;
+ } realm_info;
+
+ STRUCT_TYPE {
+ /*
+ * The contents of the *_EL2 system registers at the last time
+ * the REC exited to the host due to a synchronous exception.
+ * These are the unsanitized register values which may differ
+ * from the value returned to the host in rec_exit structure.
+ */
+ unsigned long esr;
+ unsigned long hpfar;
+ unsigned long far;
+ } last_run_info;
+
+ /* Pointer to per-cpu non-secure state */
+ struct ns_state *ns;
+
+ struct {
+ /*
+ * Set to 'true' when there is a pending PSCI
+ * command that must be resolved by the host.
+ * The command is encoded in rec->regs[0].
+ *
+ * A REC with pending PSCI is not schedulable.
+ */
+ bool pending;
+ } psci_info;
+
+ /* Number of auxiliary granules */
+ unsigned int num_rec_aux;
+
+ /* Addresses of auxiliary granules */
+ struct granule *g_aux[MAX_REC_AUX_GRANULES];
+ struct rec_aux_data aux_data;
+ struct {
+ unsigned long vsesr_el2;
+ bool inject;
+ } serror_info;
+
+ /* True if host call is pending */
+ bool host_call;
+
+ /* The active SIMD context that is live in CPU registers */
+ struct simd_context *active_simd_ctx;
+};
+COMPILER_ASSERT(sizeof(struct rec) <= GRANULE_SIZE);
+/*
+ * The `sp_el0` field must immediately follow `regs` field in `struct rec`.
+ * This assumption is used by the assembly code saving and restoring realm
+ * registers.
+ */
+COMPILER_ASSERT(U(offsetof(struct rec, sp_el0)) ==
+ (U(offsetof(struct rec, regs)) + U(sizeof(unsigned long) * RMM_REC_SAVED_GEN_REG_COUNT)));
+
+/*
+ * Check that mpidr has a valid value with all fields except
+ * Aff3[39:32]:Aff2[23:16]:Aff1[15:8]:Aff0[3:0] set to 0.
+ */
+static inline bool mpidr_is_valid(unsigned long mpidr)
+{
+ return (mpidr & ~(MASK(MPIDR_EL2_AFF0) |
+ MASK(MPIDR_EL2_AFF1) |
+ MASK(MPIDR_EL2_AFF2) |
+ MASK(MPIDR_EL2_AFF3))) == 0ULL;
+}
+
+/*
+ * Calculate REC index from mpidr value.
+ * index = Aff3[39:32]:Aff2[23:16]:Aff1[15:8]:Aff0[3:0]
+ */
+static inline unsigned long mpidr_to_rec_idx(unsigned long mpidr)
+{
+ return (MPIDR_EL2_AFF(0, mpidr) +
+ MPIDR_EL2_AFF(1, mpidr) +
+ MPIDR_EL2_AFF(2, mpidr) +
+ MPIDR_EL2_AFF(3, mpidr));
+}
+
+void rec_run_loop(struct rec *rec, struct rmi_rec_exit *rec_exit);
+void inject_serror(struct rec *rec, unsigned long vsesr);
+void emulate_stage2_data_abort(struct rec *rec, struct rmi_rec_exit *rec_exit,
+ unsigned long rtt_level);
+
+#endif /* __ASSEMBLER__ */
+#endif /* REC_H */
diff --git a/runtime/include/timers.h b/runtime/include/timers.h
new file mode 100644
index 0000000..940118b
--- /dev/null
+++ b/runtime/include/timers.h
@@ -0,0 +1,15 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
+ */
+
+#ifndef TIMERS_H
+#define TIMERS_H
+
+struct rec;
+struct rmi_rec_exit;
+
+bool check_pending_timers(struct rec *rec);
+void report_timer_state_to_ns(struct rmi_rec_exit *rec_exit);
+
+#endif /* TIMERS_H */
diff --git a/runtime/rmi/granule.c b/runtime/rmi/granule.c
index c115fd6..68a0ab2 100644
--- a/runtime/rmi/granule.c
+++ b/runtime/rmi/granule.c
@@ -3,6 +3,7 @@
* SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
*/
+#include <buffer.h>
#include <debug.h>
#include <granule.h>
#include <rmm_el3_ifc.h>
@@ -34,7 +35,7 @@
}
granule_set_state(g, GRANULE_STATE_DELEGATED);
- granule_memzero(g, SLOT_DELEGATED);
+ buffer_granule_memzero(g, SLOT_DELEGATED);
granule_unlock(g);
return RMI_SUCCESS;
diff --git a/runtime/rmi/realm.c b/runtime/rmi/realm.c
index 5019b20..64a2aa5 100644
--- a/runtime/rmi/realm.c
+++ b/runtime/rmi/realm.c
@@ -33,7 +33,7 @@
return RMI_ERROR_INPUT;
}
- rd = granule_map(g_rd, SLOT_RD);
+ rd = buffer_granule_map(g_rd, SLOT_RD);
assert(rd != NULL);
if (get_rd_state_locked(rd) == REALM_NEW) {
@@ -199,7 +199,7 @@
num_root_rtts = rd->s2_ctx.num_root_rtts;
for (unsigned int rtt = 0U; rtt < num_root_rtts; rtt++) {
- unsigned long *s2tt = granule_map(g_rtt, SLOT_RTT);
+ unsigned long *s2tt = buffer_granule_map(g_rtt, SLOT_RTT);
assert(s2tt != NULL);
@@ -326,7 +326,7 @@
(i * sizeof(struct granule)));
granule_lock(g, GRANULE_STATE_RTT);
- granule_memzero(g, SLOT_RTT);
+ buffer_granule_memzero(g, SLOT_RTT);
granule_unlock_transition(g, GRANULE_STATE_DELEGATED);
}
}
@@ -424,7 +424,7 @@
return RMI_ERROR_INPUT;
}
- rd = granule_map(g_rd, SLOT_RD);
+ rd = buffer_granule_map(g_rd, SLOT_RD);
assert(rd != NULL);
set_rd_state(rd, REALM_NEW);
@@ -513,7 +513,7 @@
}
}
- rd = granule_map(g_rd, SLOT_RD);
+ rd = buffer_granule_map(g_rd, SLOT_RD);
assert(rd != NULL);
g_rtt = rd->s2_ctx.g_rtt;
@@ -538,7 +538,7 @@
free_sl_rtts(g_rtt, num_rtts);
/* This implicitly destroys the measurement */
- granule_memzero(g_rd, SLOT_RD);
+ buffer_granule_memzero(g_rd, SLOT_RD);
granule_unlock_transition(g_rd, GRANULE_STATE_DELEGATED);
return RMI_SUCCESS;
diff --git a/runtime/rmi/rec.c b/runtime/rmi/rec.c
index 3775670..e396322 100644
--- a/runtime/rmi/rec.c
+++ b/runtime/rmi/rec.c
@@ -163,7 +163,7 @@
granule_lock(g_rec_aux, GRANULE_STATE_REC_AUX);
if (scrub) {
- granule_memzero(g_rec_aux,
+ buffer_granule_memzero(g_rec_aux,
(enum buffer_slot)((unsigned int)SLOT_REC_AUX0 + i));
}
granule_unlock_transition(g_rec_aux, GRANULE_STATE_DELEGATED);
@@ -209,7 +209,7 @@
struct rec_aux_data *aux_data;
/* Map auxiliary granules */
- rec_aux = aux_granules_map(r->g_aux, r->num_rec_aux);
+ rec_aux = buffer_aux_granules_map(r->g_aux, r->num_rec_aux);
assert(rec_aux != NULL);
/*
@@ -241,7 +241,7 @@
rec_simd_state_init(r);
/* Unmap auxiliary granules */
- aux_granules_unmap(rec_aux, r->num_rec_aux);
+ buffer_aux_unmap(rec_aux, r->num_rec_aux);
}
unsigned long smc_rec_create(unsigned long rd_addr,
@@ -302,10 +302,10 @@
goto out_free_aux;
}
- rec = granule_map(g_rec, SLOT_REC);
+ rec = buffer_granule_map(g_rec, SLOT_REC);
assert(rec != NULL);
- rd = granule_map(g_rd, SLOT_RD);
+ rd = buffer_granule_map(g_rd, SLOT_RD);
assert(rd != NULL);
if (get_rd_state_locked(rd) != REALM_NEW) {
@@ -404,7 +404,7 @@
}
}
- rec = granule_map(g_rec, SLOT_REC);
+ rec = buffer_granule_map(g_rec, SLOT_REC);
assert(rec != NULL);
g_rd = rec->realm_info.g_rd;
@@ -442,7 +442,7 @@
return;
}
- rd = granule_map(g_rd, SLOT_RD);
+ rd = buffer_granule_map(g_rd, SLOT_RD);
assert(rd != NULL);
num_rec_aux = rd->num_rec_aux;
@@ -493,10 +493,10 @@
goto out_unlock;
}
- calling_rec = granule_map(g_calling_rec, SLOT_REC);
+ calling_rec = buffer_granule_map(g_calling_rec, SLOT_REC);
assert(calling_rec != NULL);
- target_rec = granule_map(g_target_rec, SLOT_REC2);
+ target_rec = buffer_granule_map(g_target_rec, SLOT_REC2);
assert(target_rec != NULL);
ret = psci_complete_request(calling_rec, target_rec, status);
diff --git a/runtime/rmi/rtt.c b/runtime/rmi/rtt.c
index 84e64bc..c21095a 100644
--- a/runtime/rmi/rtt.c
+++ b/runtime/rmi/rtt.c
@@ -15,6 +15,7 @@
#include <smc-handler.h>
#include <smc-rmi.h>
#include <smc.h>
+#include <status.h>
#include <stddef.h>
#include <string.h>
@@ -97,7 +98,7 @@
return RMI_ERROR_INPUT;
}
- rd = granule_map(g_rd, SLOT_RD);
+ rd = buffer_granule_map(g_rd, SLOT_RD);
assert(rd != NULL);
if (!validate_rtt_structure_cmds(map_addr, level, rd)) {
@@ -126,11 +127,11 @@
goto out_unlock_llt;
}
- parent_s2tt = granule_map(wi.g_llt, SLOT_RTT);
+ parent_s2tt = buffer_granule_map(wi.g_llt, SLOT_RTT);
assert(parent_s2tt != NULL);
parent_s2tte = s2tte_read(&parent_s2tt[wi.index]);
- s2tt = granule_map(g_tbl, SLOT_DELEGATED);
+ s2tt = buffer_granule_map(g_tbl, SLOT_DELEGATED);
assert(s2tt != NULL);
if (s2tte_is_unassigned_empty(&s2_ctx, parent_s2tte)) {
@@ -293,7 +294,7 @@
return;
}
- rd = granule_map(g_rd, SLOT_RD);
+ rd = buffer_granule_map(g_rd, SLOT_RD);
assert(rd != NULL);
if (!validate_rtt_structure_cmds(map_addr, level, rd)) {
@@ -315,7 +316,7 @@
goto out_unlock_parent_table;
}
- parent_s2tt = granule_map(wi.g_llt, SLOT_RTT);
+ parent_s2tt = buffer_granule_map(wi.g_llt, SLOT_RTT);
assert(parent_s2tt != NULL);
parent_s2tte = s2tte_read(&parent_s2tt[wi.index]);
@@ -333,7 +334,7 @@
*/
assert(g_tbl != NULL);
- table = granule_map(g_tbl, SLOT_RTT2);
+ table = buffer_granule_map(g_tbl, SLOT_RTT2);
assert(table != NULL);
/*
@@ -489,7 +490,7 @@
return;
}
- rd = granule_map(g_rd, SLOT_RD);
+ rd = buffer_granule_map(g_rd, SLOT_RD);
assert(rd != NULL);
if (!validate_rtt_structure_cmds(map_addr, level, rd)) {
@@ -508,7 +509,7 @@
s2tt_walk_lock_unlock(&s2_ctx, map_addr, level - 1L, &wi);
- parent_s2tt = granule_map(wi.g_llt, SLOT_RTT);
+ parent_s2tt = buffer_granule_map(wi.g_llt, SLOT_RTT);
assert(parent_s2tt != NULL);
parent_s2tte = s2tte_read(&parent_s2tt[wi.index]);
@@ -547,7 +548,7 @@
res->x[1] = rtt_addr;
skip_non_live = true;
- table = granule_map(g_tbl, SLOT_RTT2);
+ table = buffer_granule_map(g_tbl, SLOT_RTT2);
assert(table != NULL);
if (in_par) {
@@ -624,7 +625,7 @@
return;
}
- rd = granule_map(g_rd, SLOT_RD);
+ rd = buffer_granule_map(g_rd, SLOT_RD);
assert(rd != NULL);
s2_ctx = rd->s2_ctx;
@@ -670,7 +671,7 @@
goto out_unlock_llt;
}
- s2tt = granule_map(wi.g_llt, SLOT_RTT);
+ s2tt = buffer_granule_map(wi.g_llt, SLOT_RTT);
assert(s2tt != NULL);
s2tte = s2tte_read(&s2tt[wi.index]);
@@ -771,7 +772,7 @@
return;
}
- rd = granule_map(g_rd, SLOT_RD);
+ rd = buffer_granule_map(g_rd, SLOT_RD);
assert(rd != NULL);
if (!validate_rtt_entry_cmds(map_addr, level, rd)) {
@@ -788,7 +789,7 @@
granule_unlock(g_rd);
s2tt_walk_lock_unlock(&s2_ctx, map_addr, level, &wi);
- s2tt = granule_map(wi.g_llt, SLOT_RTT);
+ s2tt = buffer_granule_map(wi.g_llt, SLOT_RTT);
assert(s2tt != NULL);
s2tte = s2tte_read(&s2tt[wi.index]);
@@ -894,7 +895,7 @@
return RMI_ERROR_INPUT;
}
- rd = granule_map(g_rd, SLOT_RD);
+ rd = buffer_granule_map(g_rd, SLOT_RD);
assert(rd != NULL);
ret = (g_src != NULL) ?
@@ -915,7 +916,7 @@
goto out_unlock_ll_table;
}
- s2tt = granule_map(wi.g_llt, SLOT_RTT);
+ s2tt = buffer_granule_map(wi.g_llt, SLOT_RTT);
assert(s2tt != NULL);
s2tte = s2tte_read(&s2tt[wi.index]);
@@ -927,7 +928,7 @@
if (g_src != NULL) {
bool ns_access_ok;
- void *data = granule_map(g_data, SLOT_DELEGATED);
+ void *data = buffer_granule_map(g_data, SLOT_DELEGATED);
assert(data != NULL);
@@ -1024,7 +1025,7 @@
return;
}
- rd = granule_map(g_rd, SLOT_RD);
+ rd = buffer_granule_map(g_rd, SLOT_RD);
assert(rd != NULL);
if (!addr_in_par(rd, map_addr) ||
@@ -1043,7 +1044,7 @@
granule_unlock(g_rd);
s2tt_walk_lock_unlock(&s2_ctx, map_addr, S2TT_PAGE_LEVEL, &wi);
- s2tt = granule_map(wi.g_llt, SLOT_RTT);
+ s2tt = buffer_granule_map(wi.g_llt, SLOT_RTT);
assert(s2tt != NULL);
if (wi.last_level != S2TT_PAGE_LEVEL) {
@@ -1084,7 +1085,7 @@
*/
g_data = find_lock_granule(data_addr, GRANULE_STATE_DATA);
assert(g_data != NULL);
- granule_memzero(g_data, SLOT_DELEGATED);
+ buffer_granule_memzero(g_data, SLOT_DELEGATED);
granule_unlock_transition(g_data, GRANULE_STATE_DELEGATED);
res->x[0] = RMI_SUCCESS;
@@ -1203,7 +1204,7 @@
return;
}
- rd = granule_map(g_rd, SLOT_RD);
+ rd = buffer_granule_map(g_rd, SLOT_RD);
assert(rd != NULL);
if (!validate_map_addr(base, S2TT_PAGE_LEVEL, rd) ||
@@ -1227,7 +1228,7 @@
s2tt_walk_lock_unlock(s2_ctx, base, S2TT_PAGE_LEVEL, &wi);
level = wi.last_level;
- s2tt = granule_map(wi.g_llt, SLOT_RTT);
+ s2tt = buffer_granule_map(wi.g_llt, SLOT_RTT);
assert(s2tt != NULL);
map_size = s2tte_map_size(level);
@@ -1381,7 +1382,7 @@
goto out_unlock_rec_rd;
}
- rec = granule_map(g_rec, SLOT_REC);
+ rec = buffer_granule_map(g_rec, SLOT_REC);
assert(rec != NULL);
if (g_rd != rec->realm_info.g_rd) {
@@ -1402,7 +1403,7 @@
goto out_unmap_rec;
}
- rd = granule_map(g_rd, SLOT_RD);
+ rd = buffer_granule_map(g_rd, SLOT_RD);
assert(rd != NULL);
/*
@@ -1427,7 +1428,7 @@
goto out_unlock_llt;
}
- s2tt = granule_map(wi.g_llt, SLOT_RTT);
+ s2tt = buffer_granule_map(wi.g_llt, SLOT_RTT);
assert(s2tt != NULL);
rtt_set_ripas_range(s2_ctx, s2tt, base, top, &wi,
diff --git a/runtime/rmi/run.c b/runtime/rmi/run.c
index 2ba56a8..85a1d84 100644
--- a/runtime/rmi/run.c
+++ b/runtime/rmi/run.c
@@ -4,6 +4,7 @@
*/
#include <arch.h>
+#include <buffer.h>
#include <debug.h>
#include <esr.h>
#include <gic.h>
@@ -213,10 +214,10 @@
return RMI_ERROR_INPUT;
}
- rec = granule_map(g_rec, SLOT_REC);
+ rec = buffer_granule_map(g_rec, SLOT_REC);
assert(rec != NULL);
- rd = granule_map(rec->realm_info.g_rd, SLOT_RD);
+ rd = buffer_granule_map(rec->realm_info.g_rd, SLOT_RD);
assert(rd != NULL);
realm_state = get_rd_state_unlocked(rd);
diff --git a/runtime/rsi/config.c b/runtime/rsi/config.c
index 28a5c71..65bc42b 100644
--- a/runtime/rsi/config.c
+++ b/runtime/rsi/config.c
@@ -3,6 +3,7 @@
* SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
*/
+#include <buffer.h>
#include <granule.h>
#include <realm.h>
#include <rsi-handler.h>
@@ -42,7 +43,7 @@
/* Map Realm data granule to RMM address space */
gr = find_granule(walk_res.pa);
- config = (struct rsi_realm_config *)granule_map(gr, SLOT_RSI_CALL);
+ config = (struct rsi_realm_config *)buffer_granule_map(gr, SLOT_RSI_CALL);
assert(config != NULL);
/* Populate config structure */
diff --git a/runtime/rsi/host_call.c b/runtime/rsi/host_call.c
index 097a8d9..bb704aa 100644
--- a/runtime/rsi/host_call.c
+++ b/runtime/rsi/host_call.c
@@ -67,7 +67,7 @@
/* Map Realm data granule to RMM address space */
gr = find_granule(walk_result.pa);
- data = (uintptr_t)granule_map(gr, SLOT_RSI_CALL);
+ data = (uintptr_t)buffer_granule_map(gr, SLOT_RSI_CALL);
assert(data != 0UL);
host_call = (struct rsi_host_call *)(data + ipa - page_ipa);
diff --git a/runtime/rsi/psci.c b/runtime/rsi/psci.c
index fa2401a..8c8da19 100644
--- a/runtime/rsi/psci.c
+++ b/runtime/rsi/psci.c
@@ -3,6 +3,7 @@
* SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
*/
+#include <buffer.h>
#include <granule.h>
#include <psci.h>
#include <realm.h>
@@ -110,7 +111,7 @@
static unsigned long rd_map_read_rec_count(struct granule *g_rd)
{
unsigned long rec_count;
- struct rd *rd = granule_map(g_rd, SLOT_RD);
+ struct rd *rd = buffer_granule_map(g_rd, SLOT_RD);
assert(rd != NULL);
@@ -228,7 +229,7 @@
* the rd lock here before we set the Realm's new state.
*/
granule_lock(g_rd, GRANULE_STATE_RD);
- rd = granule_map(rec->realm_info.g_rd, SLOT_RD);
+ rd = buffer_granule_map(rec->realm_info.g_rd, SLOT_RD);
assert(rd != NULL);
set_rd_state(rd, REALM_SYSTEM_OFF);
diff --git a/runtime/rsi/realm_attest.c b/runtime/rsi/realm_attest.c
index 2396849..29df074 100644
--- a/runtime/rsi/realm_attest.c
+++ b/runtime/rsi/realm_attest.c
@@ -4,6 +4,7 @@
*/
#include <attestation.h>
+#include <buffer.h>
#include <debug.h>
#include <granule.h>
#include <measurement.h>
@@ -113,7 +114,7 @@
/* Map realm data granule to RMM address space */
gr = find_granule(walk_res.pa);
- realm_att_token = (uintptr_t)granule_map(gr, SLOT_RSI_CALL);
+ realm_att_token = (uintptr_t)buffer_granule_map(gr, SLOT_RSI_CALL);
assert(realm_att_token != 0UL);
if (attest_data->token_sign_ctx.copied_len == 0UL) {
@@ -210,7 +211,7 @@
* simultaneously by another rec
*/
granule_lock(rec->realm_info.g_rd, GRANULE_STATE_RD);
- rd = granule_map(rec->realm_info.g_rd, SLOT_RD);
+ rd = buffer_granule_map(rec->realm_info.g_rd, SLOT_RD);
assert(rd != NULL);
/* Save challenge value in the context */
@@ -328,7 +329,7 @@
assert(g_rd != NULL);
- rd = granule_map(rec->realm_info.g_rd, SLOT_RD);
+ rd = buffer_granule_map(rec->realm_info.g_rd, SLOT_RD);
assert(rd != NULL);
/*
@@ -391,7 +392,7 @@
* simultaneously by another rec
*/
granule_lock(rec->realm_info.g_rd, GRANULE_STATE_RD);
- rd = granule_map(rec->realm_info.g_rd, SLOT_RD);
+ rd = buffer_granule_map(rec->realm_info.g_rd, SLOT_RD);
assert(rd != NULL);
/* Number of 8-bytes words in measurement */
diff --git a/runtime/rsi/realm_ipa_helper.c b/runtime/rsi/realm_ipa_helper.c
index 4a7b3bd..a1395ba 100644
--- a/runtime/rsi/realm_ipa_helper.c
+++ b/runtime/rsi/realm_ipa_helper.c
@@ -54,7 +54,7 @@
s2tt_walk_lock_unlock(s2_ctx, ipa, S2TT_PAGE_LEVEL, &wi);
- ll_table = granule_map(wi.g_llt, SLOT_RTT);
+ ll_table = buffer_granule_map(wi.g_llt, SLOT_RTT);
assert(ll_table != NULL);
s2tte = s2tte_read(&ll_table[wi.index]);
@@ -122,7 +122,7 @@
s2tt_walk_lock_unlock(s2_ctx, ipa, S2TT_PAGE_LEVEL, &wi);
- ll_table = granule_map(wi.g_llt, SLOT_RTT);
+ ll_table = buffer_granule_map(wi.g_llt, SLOT_RTT);
assert(ll_table != NULL);
s2tte = s2tte_read(&ll_table[wi.index]);