TF-RMM Release v0.1.0
This is the first external release of TF-RMM and provides a reference
implementation of Realm Management Monitor (RMM) as specified by the
RMM Beta0 specification[1].
The `docs/readme.rst` has more details about the project and
`docs/getting_started/getting-started.rst` has details on how to get
started with TF-RMM.
[1] https://developer.arm.com/documentation/den0137/1-0bet0/?lang=en
Signed-off-by: Soby Mathew <soby.mathew@arm.com>
Change-Id: I205ef14c015e4a37ae9ae1a64e4cd22eb8da746e
diff --git a/lib/realm/src/aarch64/sve_helpers.S b/lib/realm/src/aarch64/sve_helpers.S
new file mode 100644
index 0000000..f7fc7b1
--- /dev/null
+++ b/lib/realm/src/aarch64/sve_helpers.S
@@ -0,0 +1,235 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
+ */
+
+/*
+ * void save_sve_zcr_fpu_state(unsigned long *data)
+ */
+
+#include <asm_macros.S>
+#include <sve.h>
+
+.globl save_sve_zcr_fpu_state
+.globl save_sve_z_state
+.globl save_sve_p_ffr_state
+.globl restore_sve_zcr_fpu_state
+.globl restore_sve_z_state
+.globl restore_sve_p_ffr_state
+
+func save_sve_zcr_fpu_state
+.arch_extension sve
+ /* Save the FPSR/FPCR */
+ mrs x1, fpsr
+ mrs x2, fpcr
+ stp x1, x2, [x0], #16
+
+ /* Save the ZCR_EL1/EL2 */
+ mrs x1, zcr_el1
+ mrs x2, zcr_el2
+ stp x1, x2, [x0]
+
+ ret
+.arch_extension nosve
+endfunc save_sve_zcr_fpu_state
+
+/*
+ * void save_sve_z_state(unsigned char *data)
+ */
+func save_sve_z_state
+.arch_extension sve
+ /* maximise VL */
+ mrs x1, zcr_el2
+ orr x2, x1, #SVE_VLA_ZCR_LEN_MAX
+ msr zcr_el2, x2
+
+ /* Save the z register bank to memory. */
+ str z0, [x0, #0, MUL VL]
+ str z1, [x0, #1, MUL VL]
+ str z2, [x0, #2, MUL VL]
+ str z3, [x0, #3, MUL VL]
+ str z4, [x0, #4, MUL VL]
+ str z5, [x0, #5, MUL VL]
+ str z6, [x0, #6, MUL VL]
+ str z7, [x0, #7, MUL VL]
+ str z8, [x0, #8, MUL VL]
+ str z9, [x0, #9, MUL VL]
+ str z10, [x0, #10, MUL VL]
+ str z11, [x0, #11, MUL VL]
+ str z12, [x0, #12, MUL VL]
+ str z13, [x0, #13, MUL VL]
+ str z14, [x0, #14, MUL VL]
+ str z15, [x0, #15, MUL VL]
+ str z16, [x0, #16, MUL VL]
+ str z17, [x0, #17, MUL VL]
+ str z18, [x0, #18, MUL VL]
+ str z19, [x0, #19, MUL VL]
+ str z20, [x0, #20, MUL VL]
+ str z21, [x0, #21, MUL VL]
+ str z22, [x0, #22, MUL VL]
+ str z23, [x0, #23, MUL VL]
+ str z24, [x0, #24, MUL VL]
+ str z25, [x0, #25, MUL VL]
+ str z26, [x0, #26, MUL VL]
+ str z27, [x0, #27, MUL VL]
+ str z28, [x0, #28, MUL VL]
+ str z29, [x0, #29, MUL VL]
+ str z30, [x0, #30, MUL VL]
+ str z31, [x0, #31, MUL VL]
+
+ /* restore back zcr */
+ msr zcr_el2, x1
+
+ ret
+.arch_extension nosve
+endfunc save_sve_z_state
+
+/*
+ * void save_sve_p_ffr_state(unsigned char *data)
+ */
+func save_sve_p_ffr_state
+.arch_extension sve
+ /* maximise VL */
+ mrs x1, zcr_el2
+ orr x2, x1, #SVE_VLA_ZCR_LEN_MAX
+ msr zcr_el2, x2
+
+ /* Save the P register bank to memory. */
+ str p0, [x0, #0, MUL VL]
+ str p1, [x0, #1, MUL VL]
+ str p2, [x0, #2, MUL VL]
+ str p3, [x0, #3, MUL VL]
+ str p4, [x0, #4, MUL VL]
+ str p5, [x0, #5, MUL VL]
+ str p6, [x0, #6, MUL VL]
+ str p7, [x0, #7, MUL VL]
+ str p8, [x0, #8, MUL VL]
+ str p9, [x0, #9, MUL VL]
+ str p10, [x0, #10, MUL VL]
+ str p11, [x0, #11, MUL VL]
+ str p12, [x0, #12, MUL VL]
+ str p13, [x0, #13, MUL VL]
+ str p14, [x0, #14, MUL VL]
+ str p15, [x0, #15, MUL VL]
+
+ /* Save the ffr register bank to memory. */
+ rdffr p0.B
+ str p0, [x0, #16, MUL VL]
+
+ /* restore back zcr */
+ msr zcr_el2, x1
+
+ ret
+.arch_extension nosve
+endfunc save_sve_p_ffr_state
+
+/*
+ * void restore_sve_zcr_fpu_state(unsigned long *data)
+ */
+func restore_sve_zcr_fpu_state
+.arch_extension sve
+ /* Load the FPSR/FPCR */
+ ldp x1, x2, [x0], #16
+ msr fpsr, x1
+ msr fpcr, x2
+
+ /* Load the ZCR_EL1/EL2 */
+ ldp x1, x2, [x0]
+ msr zcr_el1, x1
+ msr zcr_el2, x2
+
+ ret
+.arch_extension nosve
+endfunc restore_sve_zcr_fpu_state
+
+/*
+ * void restore_sve_z_state(unsigned char *data)
+ */
+func restore_sve_z_state
+.arch_extension sve
+ /* maximise VL */
+ mrs x1, zcr_el2
+ orr x2, x1, #SVE_VLA_ZCR_LEN_MAX
+ msr zcr_el2, x2
+
+ /* Load the z register bank from memory. */
+ ldr z0, [x0, #0, MUL VL]
+ ldr z1, [x0, #1, MUL VL]
+ ldr z2, [x0, #2, MUL VL]
+ ldr z3, [x0, #3, MUL VL]
+ ldr z4, [x0, #4, MUL VL]
+ ldr z5, [x0, #5, MUL VL]
+ ldr z6, [x0, #6, MUL VL]
+ ldr z7, [x0, #7, MUL VL]
+ ldr z8, [x0, #8, MUL VL]
+ ldr z9, [x0, #9, MUL VL]
+ ldr z10, [x0, #10, MUL VL]
+ ldr z11, [x0, #11, MUL VL]
+ ldr z12, [x0, #12, MUL VL]
+ ldr z13, [x0, #13, MUL VL]
+ ldr z14, [x0, #14, MUL VL]
+ ldr z15, [x0, #15, MUL VL]
+ ldr z16, [x0, #16, MUL VL]
+ ldr z17, [x0, #17, MUL VL]
+ ldr z18, [x0, #18, MUL VL]
+ ldr z19, [x0, #19, MUL VL]
+ ldr z20, [x0, #20, MUL VL]
+ ldr z21, [x0, #21, MUL VL]
+ ldr z22, [x0, #22, MUL VL]
+ ldr z23, [x0, #23, MUL VL]
+ ldr z24, [x0, #24, MUL VL]
+ ldr z25, [x0, #25, MUL VL]
+ ldr z26, [x0, #26, MUL VL]
+ ldr z27, [x0, #27, MUL VL]
+ ldr z28, [x0, #28, MUL VL]
+ ldr z29, [x0, #29, MUL VL]
+ ldr z30, [x0, #30, MUL VL]
+ ldr z31, [x0, #31, MUL VL]
+
+ /* restore back zcr */
+ msr zcr_el2, x1
+
+ ret
+.arch_extension nosve
+endfunc restore_sve_z_state
+
+/*
+ * void restore_sve_p_ffr_state(unsigned char *data)
+ */
+func restore_sve_p_ffr_state
+.arch_extension sve
+ /* maximise VL */
+ mrs x1, zcr_el2
+ orr x2, x1, #SVE_VLA_ZCR_LEN_MAX
+ msr zcr_el2, x2
+
+ /* Load the P register bank from memory. */
+ ldr p1, [x0, #1, MUL VL]
+ ldr p2, [x0, #2, MUL VL]
+ ldr p3, [x0, #3, MUL VL]
+ ldr p4, [x0, #4, MUL VL]
+ ldr p5, [x0, #5, MUL VL]
+ ldr p6, [x0, #6, MUL VL]
+ ldr p7, [x0, #7, MUL VL]
+ ldr p8, [x0, #8, MUL VL]
+ ldr p9, [x0, #9, MUL VL]
+ ldr p10, [x0, #10, MUL VL]
+ ldr p11, [x0, #11, MUL VL]
+ ldr p12, [x0, #12, MUL VL]
+ ldr p13, [x0, #13, MUL VL]
+ ldr p14, [x0, #14, MUL VL]
+ ldr p15, [x0, #15, MUL VL]
+
+ /* Load the ffr register bank from memory. */
+ ldr p0, [x0, #16, MUL VL]
+ wrffr p0.B
+
+ /* restore P0 */
+ ldr p0, [x0]
+
+ /* restore back zcr */
+ msr zcr_el2, x1
+
+ ret
+.arch_extension nosve
+endfunc restore_sve_p_ffr_state
diff --git a/lib/realm/src/buffer.c b/lib/realm/src/buffer.c
new file mode 100644
index 0000000..44823aa
--- /dev/null
+++ b/lib/realm/src/buffer.c
@@ -0,0 +1,390 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
+ */
+
+#include <arch.h>
+#include <arch_helpers.h>
+#include <assert.h>
+#include <attestation_token.h>
+#include <buffer.h>
+#include <cpuid.h>
+#include <debug.h>
+#include <errno.h>
+#include <gic.h>
+#include <granule.h>
+#include <memory_alloc.h>
+#include <sizes.h>
+#include <slot_buf_arch.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <table.h>
+#include <xlat_contexts.h>
+#include <xlat_tables.h>
+
+/*
+ * The VA space size for the high region, which maps the slot buffers,
+ * needs to be a power of two, so round NR_CPU_SLOTS up to the closest
+ * power of two.
+ */
+#define ROUNDED_NR_CPU_SLOTS (1ULL << (64ULL - \
+ __builtin_clzll((NR_CPU_SLOTS) - 1)))
+
+#define RMM_SLOT_BUF_VA_SIZE ((ROUNDED_NR_CPU_SLOTS) * (GRANULE_SIZE))
+
+#define SLOT_VIRT ((ULL(0xffffffffffffffff) - \
+ RMM_SLOT_BUF_VA_SIZE + ULL(1)))
+
+/*
+ * All the slot buffers for a given CPU must be mapped by a single translation
+ * table, which means the max VA size should be <= 4KB * 512
+ */
+COMPILER_ASSERT((RMM_SLOT_BUF_VA_SIZE) <= (GRANULE_SIZE * XLAT_TABLE_ENTRIES));
+
+/*
+ * For all translation stages if FEAT_TTST is implemented, while
+ * the PE is executing in AArch64 state and is using 4KB
+ * translation granules, the min address space size is 64KB
+ */
+COMPILER_ASSERT((RMM_SLOT_BUF_VA_SIZE) >= (1 << 16U));
+
+#define RMM_SLOT_BUF_MMAP MAP_REGION_TRANSIENT( \
+ SLOT_VIRT, \
+ RMM_SLOT_BUF_VA_SIZE, \
+ PAGE_SIZE)
+
+#define SLOT_BUF_MMAP_REGIONS UL(1)
+
+/*
+ * Attributes for a buffer slot page descriptor.
+ * Note that the AF bit on the descriptor is handled by the translation
+ * library (it assumes that access faults are not handled) so it does not
+ * need to be specified here.
+ */
+#define SLOT_DESC_ATTR \
+ (MT_RW_DATA | MT_SHAREABILITY_ISH | MT_NG)
+
+/*
+ * The base tables for all the contexts are manually allocated as a continous
+ * block of memory.
+ */
+static uint64_t transient_base_table[XLAT_TABLE_ENTRIES * MAX_CPUS]
+ __aligned(BASE_XLAT_TABLES_ALIGNMENT)
+ __section("slot_buffer_xlat_tbls");
+
+/* Allocate per-cpu xlat_ctx_tbls */
+static struct xlat_ctx_tbls slot_buf_tbls[MAX_CPUS];
+
+/*
+ * Allocate mmap regions and define common xlat_ctx_cfg shared will
+ * all slot_buf_xlat_ctx
+ */
+XLAT_REGISTER_VA_SPACE(slot_buf, VA_HIGH_REGION,
+ SLOT_BUF_MMAP_REGIONS,
+ RMM_SLOT_BUF_VA_SIZE);
+
+/* context definition */
+static struct xlat_ctx slot_buf_xlat_ctx[MAX_CPUS];
+
+/*
+ * Allocate a cache to store the last level table entry where the slot buffers
+ * are mapped to avoid needing to perform a table walk every time a buffer
+ * slot operation is needed.
+ */
+static struct xlat_table_entry te_cache[MAX_CPUS];
+
+static uintptr_t slot_to_va(enum buffer_slot slot)
+{
+ assert(slot < NR_CPU_SLOTS);
+
+ return (uintptr_t)(SLOT_VIRT + (GRANULE_SIZE * slot));
+}
+
+static inline struct xlat_ctx *get_slot_buf_xlat_ctx(void)
+{
+ return &slot_buf_xlat_ctx[my_cpuid()];
+}
+
+static inline struct xlat_table_entry *get_cache_entry(void)
+{
+ return &te_cache[my_cpuid()];
+}
+
+__unused static uint64_t slot_to_descriptor(enum buffer_slot slot)
+{
+ uint64_t *entry = xlat_get_pte_from_table(get_cache_entry(),
+ slot_to_va(slot));
+
+ return xlat_read_descriptor(entry);
+}
+
+/*
+ * Setup xlat table for slot buffer mechanism for each PE.
+ * Must be called for every PE in the system
+ */
+void slot_buf_setup_xlat(void)
+{
+ unsigned int cpuid = my_cpuid();
+ int ret = xlat_ctx_create_dynamic(get_slot_buf_xlat_ctx(),
+ &slot_buf_xlat_ctx_cfg,
+ &slot_buf_tbls[cpuid],
+ &transient_base_table[
+ XLAT_TABLE_ENTRIES * cpuid],
+ GET_NUM_BASE_LEVEL_ENTRIES(
+ RMM_SLOT_BUF_VA_SIZE),
+ NULL,
+ 0U);
+
+ if (ret == -EINVAL) {
+ /*
+ * If the context was already created, carry on with the
+ * initialization. If it cannot be created, panic.
+ */
+ ERROR("%s (%u): Failed to create the empty context for the slot buffers\n",
+ __func__, __LINE__);
+ panic();
+ }
+
+ if (xlat_ctx_cfg_initialized(get_slot_buf_xlat_ctx()) == false) {
+ /* Add necessary mmap regions during cold boot */
+ struct xlat_mmap_region slot_buf_regions[] = {
+ RMM_SLOT_BUF_MMAP,
+ {0}
+ };
+
+ if (xlat_mmap_add_ctx(get_slot_buf_xlat_ctx(),
+ slot_buf_regions, true) != 0) {
+ ERROR("%s (%u): Failed to map slot buffer memory on high region\n",
+ __func__, __LINE__);
+ panic();
+ }
+
+ }
+
+ if (xlat_ctx_tbls_initialized(get_slot_buf_xlat_ctx()) == false) {
+ /*
+ * Initialize the translation tables for the current context.
+ * This is done on the first boot of each CPU.
+ */
+ int err;
+
+ err = xlat_init_tables_ctx(get_slot_buf_xlat_ctx());
+ if (err != 0) {
+ ERROR("%s (%u): xlat initialization failed with code %i\n",
+ __func__, __LINE__, err);
+ panic();
+ }
+ }
+
+ /*
+ * Confugure MMU registers. This function assumes that all the
+ * contexts of a particular VA region (HIGH or LOW VA) use the same
+ * limits for VA and PA spaces.
+ */
+ if (xlat_arch_setup_mmu_cfg(get_slot_buf_xlat_ctx())) {
+ ERROR("%s (%u): MMU registers failed to initialize\n",
+ __func__, __LINE__);
+ panic();
+ }
+}
+
+/*
+ * Finishes initializing the slot buffer mechanism.
+ * This function must be called after the MMU is enabled.
+ */
+void slot_buf_init(void)
+{
+ if (is_mmu_enabled() == false) {
+ ERROR("%s: MMU must be enabled\n", __func__);
+ panic();
+ }
+
+ /*
+ * Initialize (if not done yet) the internal cache with the last level
+ * translation table that holds the MMU descriptors for the slot
+ * buffers, so we can access them faster when we need to map/unmap.
+ */
+ if ((get_cache_entry())->table == NULL) {
+ if (xlat_get_table_from_va(get_cache_entry(),
+ get_slot_buf_xlat_ctx(),
+ slot_to_va(SLOT_NS)) != 0) {
+ ERROR("%s (%u): Failed to initialize table entry cache for CPU %u\n",
+ __func__, __LINE__, my_cpuid());
+ panic();
+
+ }
+ }
+}
+
+/*
+ * Buffer slots are intended to be transient, and should not be live at
+ * entry/exit of the RMM.
+ */
+void assert_cpu_slots_empty(void)
+{
+ unsigned int i;
+
+ for (i = 0; i < NR_CPU_SLOTS; i++) {
+ assert(slot_to_descriptor(i) == INVALID_DESC);
+ }
+}
+
+static inline bool is_ns_slot(enum buffer_slot slot)
+{
+ return slot == SLOT_NS;
+}
+
+static inline bool is_realm_slot(enum buffer_slot slot)
+{
+ return (slot != SLOT_NS) && (slot < NR_CPU_SLOTS);
+}
+
+static void *ns_granule_map(enum buffer_slot slot, struct granule *granule)
+{
+ unsigned long addr = granule_addr(granule);
+
+ assert(is_ns_slot(slot));
+ return buffer_arch_map(slot, addr, true);
+}
+
+static void ns_buffer_unmap(enum buffer_slot slot)
+{
+ assert(is_ns_slot(slot));
+
+ buffer_arch_unmap((void *)slot_to_va(slot));
+}
+
+/*
+ * Maps a granule @g into the provided @slot, returning
+ * the virtual address.
+ *
+ * The caller must either hold @g::lock or hold a reference.
+ */
+void *granule_map(struct granule *g, enum buffer_slot slot)
+{
+ unsigned long addr = granule_addr(g);
+
+ assert(is_realm_slot(slot));
+
+ return buffer_arch_map(slot, addr, false);
+}
+
+void buffer_unmap(void *buf)
+{
+ buffer_arch_unmap(buf);
+}
+
+bool memcpy_ns_read(void *dest, const void *ns_src, unsigned long size);
+bool memcpy_ns_write(void *ns_dest, const void *src, unsigned long size);
+
+/*
+ * Map a Non secure granule @g into the slot @slot and read data from
+ * this granule to @dest. Unmap the granule once the read is done.
+ *
+ * It returns 'true' on success or `false` if not all data are copied.
+ * Only the least significant bits of @offset are considered, which allows the
+ * full PA of a non-granule aligned buffer to be used for the @offset parameter.
+ */
+bool ns_buffer_read(enum buffer_slot slot,
+ struct granule *ns_gr,
+ unsigned int offset,
+ unsigned int size,
+ void *dest)
+{
+ uintptr_t src;
+ bool retval;
+
+ assert(is_ns_slot(slot));
+ assert(ns_gr != NULL);
+
+ /*
+ * To simplify the trapping mechanism around NS access,
+ * memcpy_ns_read uses a single 8-byte LDR instruction and
+ * all parameters must be aligned accordingly.
+ */
+ assert(ALIGNED(size, 8));
+ assert(ALIGNED(offset, 8));
+ assert(ALIGNED(dest, 8));
+
+ offset &= ~GRANULE_MASK;
+ assert(offset + size <= GRANULE_SIZE);
+
+ src = (uintptr_t)ns_granule_map(slot, ns_gr) + offset;
+ retval = memcpy_ns_read(dest, (void *)src, size);
+ ns_buffer_unmap(slot);
+
+ return retval;
+}
+
+/*
+ * Map a Non secure granule @g into the slot @slot and write data from
+ * this granule to @dest. Unmap the granule once the write is done.
+ *
+ * It returns 'true' on success or `false` if not all data are copied.
+ * Only the least significant bits of @offset are considered, which allows the
+ * full PA of a non-granule aligned buffer to be used for the @offset parameter.
+ */
+bool ns_buffer_write(enum buffer_slot slot,
+ struct granule *ns_gr,
+ unsigned int offset,
+ unsigned int size,
+ void *src)
+{
+ uintptr_t dest;
+ bool retval;
+
+ assert(is_ns_slot(slot));
+ assert(ns_gr != NULL);
+
+ /*
+ * To simplify the trapping mechanism around NS access,
+ * memcpy_ns_write uses a single 8-byte STR instruction and
+ * all parameters must be aligned accordingly.
+ */
+ assert(ALIGNED(size, 8));
+ assert(ALIGNED(offset, 8));
+ assert(ALIGNED(src, 8));
+
+ offset &= ~GRANULE_MASK;
+ assert(offset + size <= GRANULE_SIZE);
+
+ dest = (uintptr_t)ns_granule_map(slot, ns_gr) + offset;
+ retval = memcpy_ns_write((void *)dest, src, size);
+ ns_buffer_unmap(slot);
+
+ return retval;
+}
+
+/******************************************************************************
+ * Internal helpers
+ ******************************************************************************/
+
+void *buffer_map_internal(enum buffer_slot slot, unsigned long addr, bool ns)
+{
+ uint64_t attr = SLOT_DESC_ATTR;
+ uintptr_t va = slot_to_va(slot);
+ struct xlat_table_entry *entry = get_cache_entry();
+
+ assert(GRANULE_ALIGNED(addr));
+
+ attr |= (ns == true ? MT_NS : MT_REALM);
+
+ if (xlat_map_memory_page_with_attrs(entry, va,
+ (uintptr_t)addr, attr) != 0) {
+ /* Error mapping the buffer */
+ return NULL;
+ }
+
+ return (void *)va;
+}
+
+void buffer_unmap_internal(void *buf)
+{
+ /*
+ * Prevent the compiler from moving prior loads/stores to buf after the
+ * update to the translation table. Otherwise, those could fault.
+ */
+ COMPILER_BARRIER();
+
+ xlat_unmap_memory_page(get_cache_entry(), (uintptr_t)buf);
+}
diff --git a/lib/realm/src/granule.c b/lib/realm/src/granule.c
new file mode 100644
index 0000000..3b5ef48
--- /dev/null
+++ b/lib/realm/src/granule.c
@@ -0,0 +1,260 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
+ */
+
+#include <assert.h>
+#include <buffer.h>
+#include <debug.h>
+#include <granule.h>
+#include <mmio.h>
+#include <platform_api.h>
+#include <smc.h>
+#include <status.h>
+#include <stddef.h>
+#include <string.h>
+#include <utils_def.h>
+
+static struct granule granules[RMM_MAX_GRANULES];
+
+/*
+ * Takes a valid pointer to a struct granule, and returns the granule physical
+ * address.
+ *
+ * This is purely a lookup, and provides no guarantees about the attributes of
+ * the granule (i.e. whether it is locked, its state or its reference count).
+ */
+unsigned long granule_addr(struct granule *g)
+{
+ unsigned long idx;
+
+ assert(g != NULL);
+
+ idx = g - &granules[0];
+
+ return plat_granule_idx_to_addr(idx);
+}
+
+/*
+ * Takes a granule index, and returns a pointer to the struct granule.
+ *
+ * This is purely a lookup, and provides no guarantees about the attributes of
+ * the granule (i.e. whether it is locked, its state or its reference count).
+ */
+static struct granule *granule_from_idx(unsigned long idx)
+{
+ assert(idx < RMM_MAX_GRANULES);
+ return &granules[idx];
+}
+
+/*
+ * Takes an aligned granule address, and returns a pointer to the corresponding
+ * struct granule.
+ *
+ * This is purely a lookup, and provides no guarantees about the attributes of
+ * the granule (i.e. whether it is locked, its state or its reference count).
+ */
+struct granule *addr_to_granule(unsigned long addr)
+{
+ unsigned long idx;
+
+ assert(GRANULE_ALIGNED(addr));
+
+ idx = plat_granule_addr_to_idx(addr);
+ return granule_from_idx(idx);
+}
+
+/*
+ * Verifies whether @addr is a valid granule physical address, and returns a
+ * pointer to the corresponding struct granule.
+ *
+ * This is purely a lookup, and provides no guarantees w.r.t the state of the
+ * granule (e.g. locking).
+ *
+ * Returns:
+ * Pointer to the struct granule if @addr is a valid granule physical
+ * address.
+ * NULL if any of:
+ * - @addr is not aligned to the size of a granule.
+ * - @addr is out of range.
+ */
+struct granule *find_granule(unsigned long addr)
+{
+ unsigned long idx;
+
+ if (!GRANULE_ALIGNED(addr)) {
+ return NULL;
+ }
+
+ idx = plat_granule_addr_to_idx(addr);
+
+ if (idx >= RMM_MAX_GRANULES) {
+ return NULL;
+ }
+
+ return granule_from_idx(idx);
+}
+
+/*
+ * Obtain a pointer to a locked granule at @addr if @addr is a valid granule
+ * physical address and the state of the granule at @addr is @expected_state.
+ *
+ * Returns:
+ * A valid granule pointer if @addr is a valid granule physical address.
+ * NULL if any of:
+ * - @addr is not aligned to the size of a granule.
+ * - @addr is out of range.
+ * - if the state of the granule at @addr is not
+ * @expected_state.
+ */
+struct granule *find_lock_granule(unsigned long addr,
+ enum granule_state expected_state)
+{
+ struct granule *g;
+
+ g = find_granule(addr);
+ if (g == NULL) {
+ return NULL;
+ }
+
+ if (!granule_lock_on_state_match(g, expected_state)) {
+ return NULL;
+ }
+
+ return g;
+}
+
+struct granule_set {
+ unsigned int idx;
+ unsigned long addr;
+ enum granule_state state;
+ struct granule *g;
+ struct granule **g_ret;
+};
+
+/*
+ * Sort a set of granules by their address.
+ */
+static void sort_granules(struct granule_set *granules,
+ unsigned long n)
+{
+ unsigned long i;
+
+ for (i = 1UL; i < n; i++) {
+ struct granule_set temp = granules[i];
+ unsigned long j = i;
+
+ while ((j > 0UL) && (granules[j - 1].addr > temp.addr)) {
+ granules[j] = granules[j - 1];
+ j--;
+ }
+ if (i != j) {
+ granules[j] = temp;
+ }
+ }
+}
+
+/*
+ * Find a set of granules and lock them in order of their address.
+ *
+ * @granules: Pointer to array of @n items. Each item must be pre-populated
+ * with ->addr set to the granule's address, and ->state set to
+ * the expected state of the granule, and ->g_ret pointing to
+ * a valid 'struct granule *'.
+ * This function sorts the supplied array in place.
+ * @n: Number of struct granule_set in array pointed to by @granules
+ *
+ * Returns:
+ * True if all granules in @granules were successfully locked.
+ *
+ * False if any two entries in @granules have the same ->addr, or
+ * if, for any entry in @granules, any of the following is true:
+ * - entry->addr is not aligned to the size of a granule
+ * - entry->addr is out of range
+ * - the state of the granule at entry->addr is not entry->state
+ *
+ * Locking only succeeds if the granules are in their expected states as per the
+ * locking rules in granule_types.h.
+ *
+ * If the function succeeds, for all items in @granules, ->g points to a locked
+ * granule in ->state and *->g_ret is set to the pointer value.
+ *
+ * If the function fails, no lock is held and no *->g_ret pointers are
+ * modified.
+ */
+static bool find_lock_granules(struct granule_set *granules,
+ unsigned long n)
+{
+ long i;
+
+ for (i = 0L; i < n; i++) {
+ granules[i].idx = i;
+ }
+
+ sort_granules(granules, n);
+
+ for (i = 0L; i < n; i++) {
+ /* Check for duplicates */
+ if ((i > 0L) && (granules[i].addr == granules[i - 1].addr)) {
+ goto out_err;
+ }
+
+ granules[i].g = find_lock_granule(granules[i].addr,
+ granules[i].state);
+ if (granules[i].g == NULL) {
+ goto out_err;
+ }
+ }
+
+ for (i = 0L; i < n; i++) {
+ *granules[i].g_ret = granules[i].g;
+ }
+
+ return true;
+
+out_err:
+ for (i = i - 1; i >= 0L; i--) {
+ granule_unlock(granules[i].g);
+ }
+
+ return false;
+}
+
+/*
+ * Find two granules and lock them in order of their address.
+ *
+ * See find_lock_granules().
+ */
+bool find_lock_two_granules(
+ unsigned long addr1,
+ enum granule_state expected_state1,
+ struct granule **g1,
+ unsigned long addr2,
+ enum granule_state expected_state2,
+ struct granule **g2)
+{
+ struct granule_set granules[] = {
+ {0U, addr1, expected_state1, NULL, g1},
+ {1U, addr2, expected_state2, NULL, g2}
+ };
+
+ assert((g1 != NULL) && (g2 != NULL));
+
+ return find_lock_granules(granules, ARRAY_SIZE(granules));
+}
+
+void granule_memzero(struct granule *g, enum buffer_slot slot)
+{
+ unsigned long *buf;
+
+ assert(g != NULL);
+
+ buf = granule_map(g, slot);
+ (void)memset(buf, 0, GRANULE_SIZE);
+ buffer_unmap(buf);
+}
+
+void granule_memzero_mapped(void *buf)
+{
+ (void)memset(buf, 0, GRANULE_SIZE);
+}
diff --git a/lib/realm/src/include/aarch64/slot_buf_arch.h b/lib/realm/src/include/aarch64/slot_buf_arch.h
new file mode 100644
index 0000000..15bc95b
--- /dev/null
+++ b/lib/realm/src/include/aarch64/slot_buf_arch.h
@@ -0,0 +1,12 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
+ */
+
+#ifndef SLOT_BUF_ARCH_H
+#define SLOT_BUF_ARCH_H
+
+#define buffer_arch_map buffer_map_internal
+#define buffer_arch_unmap buffer_unmap_internal
+
+#endif /* SLOT_BUF_ARCH_H */
diff --git a/lib/realm/src/include/fake_host/slot_buf_arch.h b/lib/realm/src/include/fake_host/slot_buf_arch.h
new file mode 100644
index 0000000..d71a2d8
--- /dev/null
+++ b/lib/realm/src/include/fake_host/slot_buf_arch.h
@@ -0,0 +1,22 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
+ */
+
+#ifndef SLOT_BUF_ARCH_H
+#define SLOT_BUF_ARCH_H
+
+#include <host_harness.h>
+
+static void *buffer_arch_map(enum buffer_slot slot,
+ unsigned long addr, bool ns)
+{
+ return host_buffer_arch_map(slot, addr, ns);
+}
+
+static void buffer_arch_unmap(void *buf)
+{
+ return host_buffer_arch_unmap(buf);
+}
+
+#endif /* SLOT_BUF_ARCH_H */
diff --git a/lib/realm/src/s2tt.c b/lib/realm/src/s2tt.c
new file mode 100644
index 0000000..7b6f197
--- /dev/null
+++ b/lib/realm/src/s2tt.c
@@ -0,0 +1,886 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
+ */
+
+#include <arch_helpers.h>
+#include <attestation_token.h>
+#include <bitmap.h>
+#include <buffer.h>
+#include <gic.h>
+#include <granule.h>
+#include <memory_alloc.h>
+#include <realm.h>
+#include <ripas.h>
+#include <smc.h>
+#include <status.h>
+#include <stddef.h>
+#include <string.h>
+#include <table.h>
+
+/*
+ * For prototyping we assume 4K pages
+ */
+#define BLOCK_L2_SIZE (GRANULE_SIZE * S2TTES_PER_S2TT)
+
+/*
+ * The maximum number of bits supported by the RMM for a stage 2 translation
+ * output address (including stage 2 table entries).
+ */
+#define S2TTE_OA_BITS 48
+
+#define DESC_TYPE_MASK 0x3UL
+#define S2TTE_L012_TABLE 0x3UL
+#define S2TTE_L012_BLOCK 0x1UL
+#define S2TTE_L3_PAGE 0x3UL
+#define S2TTE_Lx_INVALID 0x0UL
+
+/*
+ * The following constants for the mapping attributes (S2_TTE_MEMATTR_*)
+ * assume that HCR_EL2.FWB is set.
+ */
+#define S2TTE_MEMATTR_SHIFT 2
+#define S2TTE_MEMATTR_MASK (0x7UL << S2TTE_MEMATTR_SHIFT)
+#define S2TTE_MEMATTR_FWB_NORMAL_WB ((1UL << 4) | (2UL << 2))
+#define S2TTE_MEMATTR_FWB_RESERVED ((1UL << 4) | (0UL << 2))
+
+#define S2TTE_AP_SHIFT 6
+#define S2TTE_AP_MASK (3UL << S2TTE_AP_SHIFT)
+#define S2TTE_AP_RW (3UL << S2TTE_AP_SHIFT)
+
+#define S2TTE_SH_SHIFT 8
+#define S2TTE_SH_MASK (3UL << S2TTE_SH_SHIFT)
+#define S2TTE_SH_NS (0UL << S2TTE_SH_SHIFT)
+#define S2TTE_SH_RESERVED (1UL << S2TTE_SH_SHIFT)
+#define S2TTE_SH_OS (2UL << S2TTE_SH_SHIFT)
+#define S2TTE_SH_IS (3UL << S2TTE_SH_SHIFT) /* Inner Shareable */
+
+/*
+ * We set HCR_EL2.FWB So we set bit[4] to 1 and bits[3:2] to 2 and force
+ * everyting to be Normal Write-Back
+ */
+#define S2TTE_MEMATTR_FWB_NORMAL_WB ((1UL << 4) | (2UL << 2))
+#define S2TTE_AF (1UL << 10)
+#define S2TTE_XN (2UL << 53)
+#define S2TTE_NS (1UL << 55)
+
+#define S2TTE_ATTRS (S2TTE_MEMATTR_FWB_NORMAL_WB | S2TTE_AP_RW | \
+ S2TTE_SH_IS | S2TTE_AF)
+
+#define S2TTE_TABLE S2TTE_L012_TABLE
+#define S2TTE_BLOCK (S2TTE_ATTRS | S2TTE_L012_BLOCK)
+#define S2TTE_PAGE (S2TTE_ATTRS | S2TTE_L3_PAGE)
+#define S2TTE_BLOCK_NS (S2TTE_NS | S2TTE_XN | S2TTE_AF | S2TTE_L012_BLOCK)
+#define S2TTE_PAGE_NS (S2TTE_NS | S2TTE_XN | S2TTE_AF | S2TTE_L3_PAGE)
+#define S2TTE_INVALID 0
+
+/*
+ * The type of an S2TTE is one of the following:
+ *
+ * - Invalid
+ * - Valid page
+ * - Valid block
+ * - Table
+ *
+ * Within an invalid S2TTE for a Protected IPA, architecturally RES0 bits are
+ * used to encode the HIPAS and RIPAS.
+ *
+ * A valid S2TTE for a Protected IPA implies HIPAS=ASSIGNED and RIPAS=RAM.
+ *
+ * An invalid S2TTE for an Unprotected IPA implies HIPAS=INVALID_NS.
+ * A valid S2TTE for an Unprotected IPA implies HIPAS=VALID_NS.
+ *
+ * The following table defines the mapping from a (HIPAS, RIPAS) tuple to the
+ * value of the S2TTE.
+ *
+ * ------------------------------------------------------------------------------
+ * IPA HIPAS RIPAS S2TTE value
+ * ==============================================================================
+ * Protected UNASSIGNED EMPTY (S2TTE_INVALID_HIPAS_UNASSIGNED |
+ * S2TTE_INVALID_RIPAS_EMPTY)
+ * Protected UNASSIGNED RAM (S2TTE_INVALID_HIPAS_UNASSIGNED |
+ * S2TTE_INVALID_RIPAS_RAM)
+ * Protected ASSIGNED EMPTY (S2TTE_INVALID_HIPAS_ASSIGNED |
+ * S2TTE_INVALID_RIPAS_EMPTY)
+ * Protected ASSIGNED RAM Valid page / block with NS=0
+ * Protected DESTROYED * S2TTE_INVALID_DESTROYED
+ * Unprotected INVALID_NS N/A S2TTE_INVALID_UNPROTECTED
+ * Unprotected VALID_NS N/A Valid page / block with NS=1
+ * ------------------------------------------------------------------------------
+ */
+
+#define S2TTE_INVALID_HIPAS_SHIFT 2
+#define S2TTE_INVALID_HIPAS_WIDTH 4
+#define S2TTE_INVALID_HIPAS_MASK MASK(S2TTE_INVALID_HIPAS)
+
+#define S2TTE_INVALID_HIPAS_UNASSIGNED (INPLACE(S2TTE_INVALID_HIPAS, 0))
+#define S2TTE_INVALID_HIPAS_ASSIGNED (INPLACE(S2TTE_INVALID_HIPAS, 1))
+#define S2TTE_INVALID_HIPAS_DESTROYED (INPLACE(S2TTE_INVALID_HIPAS, 2))
+
+#define S2TTE_INVALID_RIPAS_SHIFT 6
+#define S2TTE_INVALID_RIPAS_WIDTH 1
+#define S2TTE_INVALID_RIPAS_MASK MASK(S2TTE_INVALID_RIPAS)
+
+#define S2TTE_INVALID_RIPAS_EMPTY (INPLACE(S2TTE_INVALID_RIPAS, 0))
+#define S2TTE_INVALID_RIPAS_RAM (INPLACE(S2TTE_INVALID_RIPAS, 1))
+
+#define S2TTE_INVALID_DESTROYED S2TTE_INVALID_HIPAS_DESTROYED
+#define S2TTE_INVALID_UNPROTECTED 0x0UL
+
+#define NR_RTT_LEVELS 4
+
+/*
+ * Invalidates S2 TLB entries from [ipa, ipa + size] region tagged with `vmid`.
+ */
+static void stage2_tlbi_ipa(const struct realm_s2_context *s2_ctx,
+ unsigned long ipa,
+ unsigned long size)
+{
+ /*
+ * Notes:
+ *
+ * - This follows the description provided in the Arm ARM on
+ * "Invalidation of TLB entries from stage 2 translations".
+ *
+ * - @TODO: Provide additional information to this primitive so that
+ * we can utilize:
+ * - The TTL level hint, see FEAT_TTL,
+ * - Final level lookup only invalidation,
+ * - Address range invalidation.
+ */
+
+ /*
+ * Save the current content of vttb_el2.
+ */
+ unsigned long old_vttbr_el2 = read_vttbr_el2();
+
+ /*
+ * Make 'vmid' the `current vmid`. Note that the tlbi instructions
+ * bellow target the TLB entries that match the `current vmid`.
+ */
+ write_vttbr_el2(INPLACE(VTTBR_EL2_VMID, s2_ctx->vmid));
+ isb();
+
+ /*
+ * Invalidate entries in S2 TLB caches that
+ * match both `ipa` & the `current vmid`.
+ */
+ while (size != 0UL) {
+ tlbiipas2e1is(ipa >> 12);
+ size -= GRANULE_SIZE;
+ ipa += GRANULE_SIZE;
+ }
+ dsb(ish);
+
+ /*
+ * The architecture does not require TLB invalidation by IPA to affect
+ * combined Stage-1 + Stage-2 TLBs. Therefore we must invalidate all of
+ * Stage-1 (tagged with the `current vmid`) after invalidating Stage-2.
+ */
+ tlbivmalle1is();
+ dsb(ish);
+ isb();
+
+ /*
+ * Restore the old content of vttb_el2.
+ */
+ write_vttbr_el2(old_vttbr_el2);
+ isb();
+}
+
+/*
+ * Invalidate S2 TLB entries with "addr" IPA.
+ * Call this function after:
+ * 1. A L3 page desc has been removed.
+ */
+void invalidate_page(const struct realm_s2_context *s2_ctx, unsigned long addr)
+{
+ stage2_tlbi_ipa(s2_ctx, addr, GRANULE_SIZE);
+}
+
+/*
+ * Invalidate S2 TLB entries with "addr" IPA.
+ * Call this function after:
+ * 1. A L2 block desc has been removed, or
+ * 2a. A L2 table desc has been removed, where
+ * 2b. All S2TTEs in L3 table that the L2 table desc was pointed to were invalid.
+ */
+void invalidate_block(const struct realm_s2_context *s2_ctx, unsigned long addr)
+{
+ stage2_tlbi_ipa(s2_ctx, addr, GRANULE_SIZE);
+}
+
+/*
+ * Invalidate S2 TLB entries with "addr" IPA.
+ * Call this function after:
+ * 1a. A L2 table desc has been removed, where
+ * 1b. Some S2TTEs in the table that the L2 table desc was pointed to were valid.
+ */
+void invalidate_pages_in_block(const struct realm_s2_context *s2_ctx, unsigned long addr)
+{
+ stage2_tlbi_ipa(s2_ctx, addr, BLOCK_L2_SIZE);
+}
+
+/*
+ * Return the index of the entry describing @addr in the translation table at
+ * level @level. This only works for non-concatenated page tables, so should
+ * not be called to get the index for the starting level.
+ *
+ * See the library pseudocode
+ * aarch64/translation/vmsa_addrcalc/AArch64.TTEntryAddress on which this is
+ * modeled.
+ */
+static unsigned long s2_addr_to_idx(unsigned long addr, long level)
+{
+ int levels = RTT_PAGE_LEVEL - level;
+ int lsb = levels * S2TTE_STRIDE + GRANULE_SHIFT;
+
+ addr >>= lsb;
+ addr &= (1UL << S2TTE_STRIDE) - 1;
+ return addr;
+}
+
+/*
+ * Return the index of the entry describing @addr in the translation table
+ * starting level. This may return an index >= S2TTES_PER_S2TT when the
+ * combination of @start_level and @ipa_bits implies concatenated
+ * stage 2 tables.
+ *
+ * See the library pseudocode
+ * aarch64/translation/vmsa_addrcalc/AArch64.S2SLTTEntryAddress on which
+ * this is modeled.
+ */
+static unsigned long s2_sl_addr_to_idx(unsigned long addr, int start_level,
+ unsigned long ipa_bits)
+{
+ int levels = RTT_PAGE_LEVEL - start_level;
+ int lsb = levels * S2TTE_STRIDE + GRANULE_SHIFT;
+
+ addr &= (1UL << ipa_bits) - 1UL;
+ addr >>= lsb;
+ return addr;
+}
+
+static unsigned long addr_level_mask(unsigned long addr, long level)
+{
+ int levels = RTT_PAGE_LEVEL - level;
+ unsigned int lsb = levels * S2TTE_STRIDE + GRANULE_SHIFT;
+ unsigned int msb = S2TTE_OA_BITS - 1;
+
+ return addr & BIT_MASK_ULL(msb, lsb);
+}
+
+static inline unsigned long table_entry_to_phys(unsigned long entry)
+{
+ return addr_level_mask(entry, RTT_PAGE_LEVEL);
+}
+
+static inline bool entry_is_table(unsigned long entry)
+{
+ return (entry & DESC_TYPE_MASK) == S2TTE_L012_TABLE;
+}
+
+static unsigned long __table_get_entry(struct granule *g_tbl,
+ unsigned long idx)
+{
+ unsigned long *table, entry;
+
+ table = granule_map(g_tbl, SLOT_RTT);
+ entry = s2tte_read(&table[idx]);
+ buffer_unmap(table);
+
+ return entry;
+}
+
+static struct granule *__find_next_level_idx(struct granule *g_tbl,
+ unsigned long idx)
+{
+ const unsigned long entry = __table_get_entry(g_tbl, idx);
+
+ if (!entry_is_table(entry)) {
+ return NULL;
+ }
+
+ return addr_to_granule(table_entry_to_phys(entry));
+}
+
+static struct granule *__find_lock_next_level(struct granule *g_tbl,
+ unsigned long map_addr,
+ long level)
+{
+ const unsigned long idx = s2_addr_to_idx(map_addr, level);
+ struct granule *g = __find_next_level_idx(g_tbl, idx);
+
+ if (g != NULL) {
+ granule_lock(g, GRANULE_STATE_RTT);
+ }
+
+ return g;
+}
+
+/*
+ * Walk an RTT until level @level using @map_addr.
+ * @g_root is the root (level 0) table and must be locked before the call.
+ * @start_level is the initial lookup level used for the stage 2 translation
+ * tables which may depend on the configuration of the realm, factoring in the
+ * IPA size of the realm and the desired starting level (within the limits
+ * defined by the Armv8 VMSA including options for stage 2 table concatenation).
+ * The function uses hand-over-hand locking to avoid race conditions and allow
+ * concurrent access to RTT tree which is not part of the current walk; when a
+ * next level table is reached it is locked before releasing previously locked
+ * table.
+ * The walk stops when either:
+ * - The entry found is a leaf entry (not an RTT Table entry), or
+ * - Level @level is reached.
+ *
+ * On return:
+ * - rtt_walk::last_level is the last level that has been reached by the walk.
+ * - rtt_walk.g_llt points to the TABLE granule at level @rtt_walk::level.
+ * The granule is locked.
+ * - rtt_walk::index is the entry index at rtt_walk.g_llt for @map_addr.
+ */
+void rtt_walk_lock_unlock(struct granule *g_root,
+ int start_level,
+ unsigned long ipa_bits,
+ unsigned long map_addr,
+ long level,
+ struct rtt_walk *wi)
+{
+ struct granule *g_tbls[NR_RTT_LEVELS] = { NULL };
+ unsigned long sl_idx;
+ int i, last_level;
+
+ assert(start_level >= MIN_STARTING_LEVEL);
+ assert(level >= start_level);
+ assert(map_addr < (1UL << ipa_bits));
+ assert(wi != NULL);
+
+ /* Handle concatenated starting level (SL) tables */
+ sl_idx = s2_sl_addr_to_idx(map_addr, start_level, ipa_bits);
+ if (sl_idx >= S2TTES_PER_S2TT) {
+ unsigned int tt_num = (sl_idx >> S2TTE_STRIDE);
+ struct granule *g_concat_root = g_root + tt_num;
+
+ granule_lock(g_concat_root, GRANULE_STATE_RTT);
+ granule_unlock(g_root);
+ g_root = g_concat_root;
+ }
+
+ g_tbls[start_level] = g_root;
+ for (i = start_level; i < level; i++) {
+ /*
+ * Lock next RTT level. Correct locking order is guaranteed
+ * because reference is obtained from a locked granule
+ * (previous level). Also, hand-over-hand locking/unlocking is
+ * used to avoid race conditions.
+ */
+ g_tbls[i + 1] = __find_lock_next_level(g_tbls[i], map_addr, i);
+ if (g_tbls[i + 1] == NULL) {
+ last_level = i;
+ goto out;
+ }
+ granule_unlock(g_tbls[i]);
+ }
+
+ last_level = level;
+out:
+ wi->last_level = last_level;
+ wi->g_llt = g_tbls[last_level];
+ wi->index = s2_addr_to_idx(map_addr, last_level);
+}
+
+/*
+ * Creates a value which can be OR'd with an s2tte to set RIPAS=@ripas.
+ */
+unsigned long s2tte_create_ripas(enum ripas ripas)
+{
+ if (ripas == RMI_EMPTY) {
+ return S2TTE_INVALID_RIPAS_EMPTY;
+ }
+ return S2TTE_INVALID_RIPAS_RAM;
+}
+
+/*
+ * Creates an invalid s2tte with HIPAS=UNASSIGNED and RIPAS=@ripas.
+ */
+unsigned long s2tte_create_unassigned(enum ripas ripas)
+{
+ return S2TTE_INVALID_HIPAS_UNASSIGNED | s2tte_create_ripas(ripas);
+}
+
+/*
+ * Creates an invalid s2tte with HIPAS=DESTROYED.
+ */
+unsigned long s2tte_create_destroyed(void)
+{
+ return S2TTE_INVALID_DESTROYED;
+}
+
+/*
+ * Creates an invalid s2tte with output address @pa, HIPAS=ASSIGNED and
+ * RIPAS=EMPTY, at level @level.
+ */
+unsigned long s2tte_create_assigned_empty(unsigned long pa, long level)
+{
+ assert(level >= RTT_MIN_BLOCK_LEVEL);
+ assert(addr_is_level_aligned(pa, level));
+ return (pa | S2TTE_INVALID_HIPAS_ASSIGNED | S2TTE_INVALID_RIPAS_EMPTY);
+}
+
+/*
+ * Creates a page or block s2tte for a Protected IPA, with output address @pa.
+ */
+unsigned long s2tte_create_valid(unsigned long pa, long level)
+{
+ assert(level >= RTT_MIN_BLOCK_LEVEL);
+ assert(addr_is_level_aligned(pa, level));
+ if (level == RTT_PAGE_LEVEL) {
+ return (pa | S2TTE_PAGE);
+ }
+ return (pa | S2TTE_BLOCK);
+}
+
+/*
+ * Creates an invalid s2tte with HIPAS=INVALID_NS.
+ */
+unsigned long s2tte_create_invalid_ns(void)
+{
+ return S2TTE_INVALID_UNPROTECTED;
+}
+
+/*
+ * Creates a page or block s2tte for an Unprotected IPA at level @level.
+ *
+ * The following S2 TTE fields are provided through @s2tte argument:
+ * - The physical address
+ * - MemAttr
+ * - S2AP
+ * - Shareability
+ */
+unsigned long s2tte_create_valid_ns(unsigned long s2tte, long level)
+{
+ assert(level >= RTT_MIN_BLOCK_LEVEL);
+ if (level == RTT_PAGE_LEVEL) {
+ return (s2tte | S2TTE_PAGE_NS);
+ }
+ return (s2tte | S2TTE_BLOCK_NS);
+}
+
+/*
+ * Validate the portion of NS S2TTE that is provided by the host.
+ */
+bool host_ns_s2tte_is_valid(unsigned long s2tte, long level)
+{
+ unsigned long mask = addr_level_mask(~0UL, level) |
+ S2TTE_MEMATTR_MASK |
+ S2TTE_AP_MASK |
+ S2TTE_SH_MASK;
+
+ /*
+ * Test that all fields that are not controlled by the host are zero
+ * and that the output address is correctly aligned. Note that
+ * the host is permitted to map any physical address outside PAR.
+ */
+ if ((s2tte & ~mask) != 0UL) {
+ return false;
+ }
+
+ /*
+ * Only one value masked by S2TTE_MEMATTR_MASK is invalid/reserved.
+ */
+ if ((s2tte & S2TTE_MEMATTR_MASK) == S2TTE_MEMATTR_FWB_RESERVED) {
+ return false;
+ }
+
+ /*
+ * Only one value masked by S2TTE_SH_MASK is invalid/reserved.
+ */
+ if ((s2tte & S2TTE_SH_MASK) == S2TTE_SH_RESERVED) {
+ return false;
+ }
+
+ /*
+ * Note that all the values that are masked by S2TTE_AP_MASK are valid.
+ */
+ return true;
+}
+
+/*
+ * Returns the portion of NS S2TTE that is set by the host.
+ */
+unsigned long host_ns_s2tte(unsigned long s2tte, long level)
+{
+ unsigned long mask = addr_level_mask(~0UL, level) |
+ S2TTE_MEMATTR_MASK |
+ S2TTE_AP_MASK |
+ S2TTE_SH_MASK;
+ return (s2tte & mask);
+}
+
+/*
+ * Creates a table s2tte at level @level with output address @pa.
+ */
+unsigned long s2tte_create_table(unsigned long pa, long level)
+{
+ assert(level < RTT_PAGE_LEVEL);
+ assert(GRANULE_ALIGNED(pa));
+
+ return (pa | S2TTE_TABLE);
+}
+
+/*
+ * Returns true if @s2tte has HIPAS=@hipas.
+ */
+static bool s2tte_has_hipas(unsigned long s2tte, unsigned long hipas)
+{
+ unsigned long desc_type = s2tte & DESC_TYPE_MASK;
+ unsigned long invalid_desc_hipas = s2tte & S2TTE_INVALID_HIPAS_MASK;
+
+ if ((desc_type != S2TTE_Lx_INVALID) || (invalid_desc_hipas != hipas)) {
+ return false;
+ }
+ return true;
+}
+
+/*
+ * Returns true if @s2tte has HIPAS=UNASSIGNED or HIPAS=INVALID_NS.
+ */
+bool s2tte_is_unassigned(unsigned long s2tte)
+{
+ return s2tte_has_hipas(s2tte, S2TTE_INVALID_HIPAS_UNASSIGNED);
+}
+
+/*
+ * Returns true if @s2tte has HIPAS=DESTROYED.
+ */
+bool s2tte_is_destroyed(unsigned long s2tte)
+{
+ return s2tte_has_hipas(s2tte, S2TTE_INVALID_HIPAS_DESTROYED);
+}
+
+/*
+ * Returns true if @s2tte has HIPAS=ASSIGNED.
+ */
+bool s2tte_is_assigned(unsigned long s2tte, long level)
+{
+ (void)level;
+
+ return s2tte_has_hipas(s2tte, S2TTE_INVALID_HIPAS_ASSIGNED);
+}
+
+static bool s2tte_check(unsigned long s2tte, long level, unsigned long ns)
+{
+ unsigned long desc_type;
+
+ if ((s2tte & S2TTE_NS) != ns) {
+ return false;
+ }
+
+ desc_type = s2tte & DESC_TYPE_MASK;
+
+ /* Only pages at L3 and valid blocks at L2 allowed */
+ if (((level == RTT_PAGE_LEVEL) && (desc_type == S2TTE_L3_PAGE)) ||
+ ((level == RTT_MIN_BLOCK_LEVEL) && (desc_type == S2TTE_BLOCK))) {
+ return true;
+ }
+
+ return false;
+}
+
+/*
+ * Returns true if @s2tte is a page or block s2tte, and NS=0.
+ */
+bool s2tte_is_valid(unsigned long s2tte, long level)
+{
+ return s2tte_check(s2tte, level, 0UL);
+}
+
+/*
+ * Returns true if @s2tte is a page or block s2tte, and NS=1.
+ */
+bool s2tte_is_valid_ns(unsigned long s2tte, long level)
+{
+ return s2tte_check(s2tte, level, S2TTE_NS);
+}
+
+/*
+ * Returns true if @s2tte is a table at level @level.
+ */
+bool s2tte_is_table(unsigned long s2tte, long level)
+{
+ unsigned long desc_type = s2tte & DESC_TYPE_MASK;
+
+ if ((level < RTT_PAGE_LEVEL) && (desc_type == S2TTE_TABLE)) {
+ return true;
+ }
+
+ return false;
+}
+
+/*
+ * Returns RIPAS of @s2tte.
+ *
+ * Caller should ensure that HIPAS=UNASSIGNED or HIPAS=ASSIGNED.
+ * The s2tte must be not valid/invalid descriptor.
+ */
+enum ripas s2tte_get_ripas(unsigned long s2tte)
+{
+ unsigned long desc_ripas = s2tte & S2TTE_INVALID_RIPAS_MASK;
+
+ /*
+ * If valid s2tte descriptor is passed, then ensure S2AP[0]
+ * bit is 1 (S2AP is set to RW for lower EL), which corresponds
+ * to RIPAS_RAM (bit[6]) on a valid descriptor.
+ */
+ if (((s2tte & DESC_TYPE_MASK) != S2TTE_Lx_INVALID) &&
+ (desc_ripas != S2TTE_INVALID_RIPAS_RAM)) {
+ assert(false);
+ }
+
+ if (desc_ripas == S2TTE_INVALID_RIPAS_EMPTY) {
+ return RMI_EMPTY;
+ }
+
+ return RMI_RAM;
+}
+
+/*
+ * Populates @s2tt with s2ttes which have HIPAS=UNASSIGNED and RIPAS=@ripas.
+ *
+ * The granule is populated before it is made a table,
+ * hence, don't use s2tte_write for access.
+ */
+void s2tt_init_unassigned(unsigned long *s2tt, enum ripas ripas)
+{
+ for (unsigned int i = 0U; i < S2TTES_PER_S2TT; i++) {
+ s2tt[i] = s2tte_create_unassigned(ripas);
+ }
+
+ dsb(ish);
+}
+
+/*
+ * Populates @s2tt with s2ttes which have HIPAS=DESTROYED.
+ *
+ * The granule is populated before it is made a table,
+ * hence, don't use s2tte_write for access.
+ */
+void s2tt_init_destroyed(unsigned long *s2tt)
+{
+ for (unsigned int i = 0U; i < S2TTES_PER_S2TT; i++) {
+ s2tt[i] = s2tte_create_destroyed();
+ }
+
+ dsb(ish);
+}
+
+unsigned long s2tte_map_size(int level)
+{
+ int levels, lsb;
+
+ assert(level <= RTT_PAGE_LEVEL);
+
+ levels = RTT_PAGE_LEVEL - level;
+ lsb = levels * S2TTE_STRIDE + GRANULE_SHIFT;
+ return 1UL << lsb;
+}
+
+/*
+ * Populates @s2tt with HIPAS=ASSIGNED, RIPAS=EMPTY s2ttes that refer to a
+ * contiguous memory block starting at @pa, and mapped at level @level.
+ *
+ * The granule is populated before it is made a table,
+ * hence, don't use s2tte_write for access.
+ */
+void s2tt_init_assigned_empty(unsigned long *s2tt, unsigned long pa, long level)
+{
+ const unsigned long map_size = s2tte_map_size(level);
+ unsigned int i;
+
+ for (i = 0U; i < S2TTES_PER_S2TT; i++) {
+ s2tt[i] = s2tte_create_assigned_empty(pa, level);
+ pa += map_size;
+ }
+ dsb(ish);
+}
+
+/*
+ * Populates @s2tt with HIPAS=VALID, RIPAS=@ripas s2ttes that refer to a
+ * contiguous memory block starting at @pa, and mapped at level @level.
+ *
+ * The granule is populated before it is made a table,
+ * hence, don't use s2tte_write for access.
+ */
+void s2tt_init_valid(unsigned long *s2tt, unsigned long pa, long level)
+{
+ const unsigned long map_size = s2tte_map_size(level);
+ unsigned int i;
+
+ for (i = 0U; i < S2TTES_PER_S2TT; i++) {
+ s2tt[i] = s2tte_create_valid(pa, level);
+ pa += map_size;
+ }
+ dsb(ish);
+}
+
+/*
+ * Populates @s2tt with HIPAS=VALID_NS, RIPAS=@ripas s2ttes that refer to a
+ * contiguous memory block starting at @pa, and mapped at level @level.
+ *
+ * The granule is populated before it is made a table,
+ * hence, don't use s2tte_write for access.
+ */
+void s2tt_init_valid_ns(unsigned long *s2tt, unsigned long pa, long level)
+{
+ const unsigned long map_size = s2tte_map_size(level);
+ unsigned int i;
+
+ for (i = 0U; i < S2TTES_PER_S2TT; i++) {
+ s2tt[i] = s2tte_create_valid_ns(pa, level);
+ pa += map_size;
+ }
+ dsb(ish);
+}
+
+/* Returns physical address of a page entry or block */
+unsigned long s2tte_pa(unsigned long s2tte, long level)
+{
+ if (s2tte_is_unassigned(s2tte) || s2tte_is_destroyed(s2tte) ||
+ s2tte_is_table(s2tte, level)) {
+ assert(false);
+ }
+ return addr_level_mask(s2tte, level);
+}
+
+/* Returns physical address of a table entry */
+unsigned long s2tte_pa_table(unsigned long s2tte, long level)
+{
+ assert(s2tte_is_table(s2tte, level));
+ return addr_level_mask(s2tte, RTT_PAGE_LEVEL);
+}
+
+bool addr_is_level_aligned(unsigned long addr, long level)
+{
+ return (addr == addr_level_mask(addr, level));
+}
+
+typedef bool (*s2tte_type_checker)(unsigned long s2tte);
+
+static bool __table_is_uniform_block(unsigned long *table,
+ s2tte_type_checker s2tte_is_x,
+ enum ripas *ripas_ptr)
+{
+ unsigned long s2tte = s2tte_read(&table[0]);
+ enum ripas ripas;
+ unsigned int i;
+
+ if (!s2tte_is_x(s2tte)) {
+ return false;
+ }
+
+ if (ripas_ptr != NULL) {
+ ripas = s2tte_get_ripas(s2tte);
+ }
+
+ for (i = 1U; i < S2TTES_PER_S2TT; i++) {
+ s2tte = s2tte_read(&table[i]);
+
+ if (!s2tte_is_x(s2tte)) {
+ return false;
+ }
+
+ if ((ripas_ptr != NULL) &&
+ (s2tte_get_ripas(s2tte) != ripas)) {
+ return false;
+ }
+ }
+
+ if (ripas_ptr != NULL) {
+ *ripas_ptr = ripas;
+ }
+
+ return true;
+}
+
+/*
+ * Returns true if all s2ttes in @table have HIPAS=UNASSIGNED and
+ * have the same RIPAS.
+ *
+ * If return value is true, the RIPAS value is returned in @ripas.
+ */
+bool table_is_unassigned_block(unsigned long *table, enum ripas *ripas)
+{
+ return __table_is_uniform_block(table, s2tte_is_unassigned, ripas);
+}
+
+/*
+ * Returns true if all s2ttes in @table have HIPAS=DESTROYED.
+ */
+bool table_is_destroyed_block(unsigned long *table)
+{
+ return __table_is_uniform_block(table, s2tte_is_destroyed, NULL);
+}
+
+typedef bool (*s2tte_type_level_checker)(unsigned long s2tte, long level);
+
+static bool __table_maps_block(unsigned long *table,
+ long level,
+ s2tte_type_level_checker s2tte_is_x)
+{
+ unsigned long base_pa;
+ unsigned long map_size = s2tte_map_size(level);
+ unsigned long s2tte = s2tte_read(&table[0]);
+ unsigned int i;
+
+ if (!s2tte_is_x(s2tte, level)) {
+ return false;
+ }
+
+ base_pa = s2tte_pa(s2tte, level);
+ if (!addr_is_level_aligned(base_pa, level - 1L)) {
+ return false;
+ }
+
+ for (i = 1U; i < S2TTES_PER_S2TT; i++) {
+ unsigned long expected_pa = base_pa + (i * map_size);
+
+ s2tte = s2tte_read(&table[i]);
+
+ if (!s2tte_is_x(s2tte, level)) {
+ return false;
+ }
+
+ if (s2tte_pa(s2tte, level) != expected_pa) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+/*
+ * Returns true if all s2ttes in @table have HIPAS=ASSIGNED
+ * and refer to a contiguous block of granules aligned to @level - 1.
+ */
+bool table_maps_assigned_block(unsigned long *table, long level)
+{
+ return __table_maps_block(table, level, s2tte_is_assigned);
+}
+
+/*
+ * Returns true if all s2ttes in @table have HIPAS=VALID and
+ * refer to a contiguous block of granules aligned to @level - 1.
+ */
+bool table_maps_valid_block(unsigned long *table, long level)
+{
+ return __table_maps_block(table, level, s2tte_is_valid);
+}
+
+/*
+ * Returns true if all s2ttes in @table have HIPAS=VALID_NS and
+ * refer to a contiguous block of granules aligned to @level - 1.
+ */
+bool table_maps_valid_ns_block(unsigned long *table, long level)
+{
+ return __table_maps_block(table, level, s2tte_is_valid_ns);
+}
diff --git a/lib/realm/src/sve.c b/lib/realm/src/sve.c
new file mode 100644
index 0000000..e3b3348
--- /dev/null
+++ b/lib/realm/src/sve.c
@@ -0,0 +1,36 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
+ */
+
+#include <arch_helpers.h>
+#include <assert.h>
+#include <sve.h>
+
+/*
+ * Save the SVE context to memory.
+ * The function saves the maximum implemented
+ * SVE context size.
+ */
+void save_sve_state(struct sve_state *sve)
+{
+ assert(sve != NULL);
+
+ save_sve_zcr_fpu_state(sve->zcr_fpu);
+ save_sve_z_state(sve->z);
+ save_sve_p_ffr_state(sve->p_ffr);
+}
+
+/*
+ * Restore the SVE context from memory.
+ * The function restores the maximum implemented
+ * SVE context size.
+ */
+void restore_sve_state(struct sve_state *sve)
+{
+ assert(sve != NULL);
+
+ restore_sve_z_state(sve->z);
+ restore_sve_p_ffr_state(sve->p_ffr);
+ restore_sve_zcr_fpu_state(sve->zcr_fpu);
+}