TF-RMM Release v0.1.0
This is the first external release of TF-RMM and provides a reference
implementation of Realm Management Monitor (RMM) as specified by the
RMM Beta0 specification[1].
The `docs/readme.rst` has more details about the project and
`docs/getting_started/getting-started.rst` has details on how to get
started with TF-RMM.
[1] https://developer.arm.com/documentation/den0137/1-0bet0/?lang=en
Signed-off-by: Soby Mathew <soby.mathew@arm.com>
Change-Id: I205ef14c015e4a37ae9ae1a64e4cd22eb8da746e
diff --git a/lib/arch/CMakeLists.txt b/lib/arch/CMakeLists.txt
new file mode 100644
index 0000000..bb0367e
--- /dev/null
+++ b/lib/arch/CMakeLists.txt
@@ -0,0 +1,27 @@
+#
+# SPDX-License-Identifier: BSD-3-Clause
+# SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
+#
+
+add_library(rmm-lib-arch)
+
+target_link_libraries(rmm-lib-arch
+ PRIVATE rmm-lib-common)
+
+target_include_directories(rmm-lib-arch
+ PUBLIC "include"
+ "include/${RMM_ARCH}")
+
+target_sources(rmm-lib-arch
+ PRIVATE "src/arch_features.c"
+ "src/fpu_helpers.c")
+
+if(NOT RMM_ARCH STREQUAL fake_host)
+ target_sources(rmm-lib-arch
+ PRIVATE "src/aarch64/cache_helpers.S"
+ "src/aarch64/fpu_helpers.S")
+else()
+ target_sources(rmm-lib-arch
+ PRIVATE "src/fake_host/cache_wrappers.c"
+ "src/fake_host/fpu_helpers_host.c")
+endif()
diff --git a/lib/arch/include/aarch64/asm_macros.S b/lib/arch/include/aarch64/asm_macros.S
new file mode 100644
index 0000000..887ab68
--- /dev/null
+++ b/lib/arch/include/aarch64/asm_macros.S
@@ -0,0 +1,166 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
+ */
+#ifndef ASM_MACROS_S
+#define ASM_MACROS_S
+
+#define TLB_INVALIDATE(_type) \
+ tlbi _type
+
+#define ENTRY(x) .global x; x
+#define ENDPROC(x)
+
+ /*
+ * This macro is used to create a function label and place the
+ * code into a separate text section based on the function name
+ * to enable elimination of unused code during linking. It also adds
+ * basic debug information to enable call stack printing most of the
+ * time. The optional _align parameter can be used to force a
+ * non-standard alignment (indicated in powers of 2). The default is
+ * _align=2 because aarch64 instructions must be word aligned.
+ * Do *not* try to use a raw .align directive. Since func
+ * switches to a new section, this would not have the desired effect.
+ */
+ .macro func _name, _align=2
+ /*
+ * Add Call Frame Information entry in the .debug_frame section for
+ * debugger consumption. This enables callstack printing in debuggers.
+ * This does not use any space in the final loaded binary, only in the
+ * ELF file.
+ * Note that a function manipulating the CFA pointer location (i.e. the
+ * x29 frame pointer on AArch64) should declare it using the
+ * appropriate .cfi* directives, or be prepared to have a degraded
+ * debugging experience.
+ */
+ .cfi_sections .debug_frame
+ .section .text.asm.\_name, "ax"
+ .type \_name, %function
+ /*
+ * .cfi_startproc and .cfi_endproc are needed to output entries in
+ * .debug_frame
+ */
+ .cfi_startproc
+ .align \_align
+ \_name:
+ .endm
+
+ /*
+ * This macro is used to mark the end of a function.
+ */
+ .macro endfunc _name
+ .cfi_endproc
+ .size \_name, . - \_name
+ .endm
+
+
+ .macro dcache_line_size reg, tmp
+ mrs \tmp, ctr_el0
+ ubfx \tmp, \tmp, #16, #4
+ mov \reg, #4
+ lsl \reg, \reg, \tmp
+ .endm
+
+ /*
+ * Declare the exception vector table, enforcing it is aligned on a
+ * 2KB boundary, as required by the ARMv8 architecture.
+ * Use zero bytes as the fill value to be stored in the padding bytes
+ * so that it inserts illegal AArch64 instructions. This increases
+ * security, robustness and potentially facilitates debugging.
+ */
+ .macro vector_base label, section_name=.vectors
+ .section \section_name, "ax"
+ .align 11, 0
+ \label:
+ .endm
+
+ /*
+ * Create an entry in the exception vector table, enforcing it is
+ * aligned on a 128-byte boundary, as required by the ARMv8 architecture.
+ * Use zero bytes as the fill value to be stored in the padding bytes
+ * so that it inserts illegal AArch64 instructions. This increases
+ * security, robustness and potentially facilitates debugging.
+ */
+ .macro vector_entry label, section_name=.vectors
+ .cfi_sections .debug_frame
+ .section \section_name, "ax"
+ .align 7, 0
+ .type \label, %function
+ .cfi_startproc
+ \label:
+ .endm
+
+ /*
+ * Add the bytes until fill the full exception vector, whose size is always
+ * 32 instructions. If there are more than 32 instructions in the
+ * exception vector then an error is emitted.
+ */
+ .macro end_vector_entry label
+ .cfi_endproc
+ .fill \label + (32 * 4) - .
+ .endm
+
+ /*
+ * Helper macro to generate the best mov/movk combinations according
+ * the value to be moved. The 16 bits from '_shift' are tested and
+ * if not zero, they are moved into '_reg' without affecting
+ * other bits.
+ */
+ .macro _mov_imm16 _reg, _val, _shift
+ .if (\_val >> \_shift) & 0xffff
+ .if (\_val & (1 << \_shift - 1))
+ movk \_reg, (\_val >> \_shift) & 0xffff, LSL \_shift
+ .else
+ mov \_reg, \_val & (0xffff << \_shift)
+ .endif
+ .endif
+ .endm
+
+ /*
+ * Helper macro to load arbitrary values into 32 or 64-bit registers
+ * which generates the best mov/movk combinations. Many base addresses
+ * are 64KB aligned the macro will eliminate updating bits 15:0 in
+ * that case
+ */
+ .macro mov_imm _reg, _val
+ .if (\_val) == 0
+ mov \_reg, #0
+ .else
+ _mov_imm16 \_reg, (\_val), 0
+ _mov_imm16 \_reg, (\_val), 16
+ _mov_imm16 \_reg, (\_val), 32
+ _mov_imm16 \_reg, (\_val), 48
+ .endif
+ .endm
+
+ /*
+ * Assembler panic. At the moment there is no support for crash
+ * reporting in assembler without having a stack available, so for
+ * the time being just enter into a busy loop and stay there.
+ */
+ .macro asm_panic
+ b .
+ .endm
+
+ /*
+ * Assembler macro to enable asm_assert. Use this macro wherever
+ * assert is required in assembly. Please note that the macro makes
+ * use of label '300' to provide the logic and the caller
+ * should make sure that this label is not used to branch prior
+ * to calling this macro.
+ */
+ .macro ASM_ASSERT _cc
+ .ifndef .L_assert_filename
+ .pushsection .rodata.str1.1, "aS"
+ .L_assert_filename:
+ .string __FILE__
+ .popsection
+ .endif
+ b.\_cc 300f
+ adr x0, .L_assert_filename
+ mov x1, __LINE__
+ asm_panic
+ 300:
+ .endm
+
+#endif /* ASM_MACROS_S */
diff --git a/lib/arch/include/aarch64/atomics.h b/lib/arch/include/aarch64/atomics.h
new file mode 100644
index 0000000..171692d
--- /dev/null
+++ b/lib/arch/include/aarch64/atomics.h
@@ -0,0 +1,112 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
+ */
+
+#ifndef ATOMICS_H
+#define ATOMICS_H
+
+#include <stdbool.h>
+#include <stdint.h>
+
+/*
+ * Atomically adds @val to the 64-bit value stored at memory location @loc.
+ */
+static inline void atomic_add_64(uint64_t *loc, long val)
+{
+ asm volatile(
+ " stadd %[val], %[loc]\n"
+ : [loc] "+Q" (*loc)
+ : [val] "r" (val)
+ : "memory");
+}
+
+/*
+ * Atomically adds @val to the 64-bit value stored at memory location @loc.
+ * Stores to memory with release semantics.
+ * Returns the old value.
+ */
+static inline unsigned long atomic_load_add_release_64(uint64_t *loc, long val)
+{
+ unsigned long old_val;
+
+ asm volatile(
+ " ldaddl %[val], %[old_val], %[loc]\n"
+ : [loc] "+Q" (*loc),
+ [old_val] "=r" (old_val)
+ : [val] "r" (val)
+ : "memory");
+
+ return old_val;
+}
+
+/*
+ * Atomically set bit @bit in value pointed to by @loc with release semantics.
+ */
+static inline void atomic_bit_set_release_64(uint64_t *loc, int bit)
+{
+ uint64_t mask = (1UL << bit);
+
+ asm volatile(
+ " stsetl %[mask], %[loc]\n"
+ : [loc] "+Q" (*loc)
+ : [mask] "r" (mask)
+ : "memory"
+ );
+}
+
+/*
+ * Atomically clear bit @bit in value pointed to by @loc with release semantics.
+ */
+static inline void atomic_bit_clear_release_64(uint64_t *loc, int bit)
+{
+ uint64_t mask = (1UL << bit);
+
+ asm volatile(
+ " stclrl %[mask], %[loc]\n"
+ : [loc] "+Q" (*loc)
+ : [mask] "r" (mask)
+ : "memory"
+ );
+}
+
+/*
+ * Test bit @bit in value pointed to by @loc with acquire semantics.
+ */
+static inline bool atomic_test_bit_acquire_64(uint64_t *loc, int bit)
+{
+ uint64_t val;
+ uint64_t mask = (1UL << bit);
+
+ asm volatile(
+ " ldar %[val], %[loc]\n"
+ : [val] "=r" (val)
+ : [loc] "Q" (*loc)
+ : "memory"
+ );
+
+ return ((val & mask) != 0UL);
+}
+
+/*
+ * Atomically set bit @bit in value pointed to by @loc
+ * with acquire and release semantics.
+ * Return True if the previous state of @bit was 1, False otherwise.
+ */
+static inline bool atomic_bit_set_acquire_release_64(uint64_t *loc, int bit)
+{
+ uint64_t val;
+ uint64_t mask = (1UL << bit);
+
+ asm volatile(
+ " ldsetal %[mask], %[val], %[loc]\n"
+ : [loc] "+Q" (*loc),
+ [val] "=r" (val)
+ : [mask] "r" (mask)
+ : "memory"
+ );
+
+ return ((val & mask) != 0UL);
+}
+
+#endif /* ATOMICS_H */
diff --git a/lib/arch/include/aarch64/cpuid.h b/lib/arch/include/aarch64/cpuid.h
new file mode 100644
index 0000000..1176a93
--- /dev/null
+++ b/lib/arch/include/aarch64/cpuid.h
@@ -0,0 +1,16 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
+ */
+
+#ifndef CPUID_H
+#define CPUID_H
+
+#include <arch_helpers.h>
+
+static inline unsigned int my_cpuid(void)
+{
+ return (unsigned int)read_tpidr_el2();
+}
+
+#endif /* CPUID_H */
diff --git a/lib/arch/include/aarch64/entropy.h b/lib/arch/include/aarch64/entropy.h
new file mode 100644
index 0000000..c45c159
--- /dev/null
+++ b/lib/arch/include/aarch64/entropy.h
@@ -0,0 +1,34 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
+ */
+
+#ifndef ENTROPY_H
+#define ENTROPY_H
+
+#include <arch.h>
+#include <utils_def.h>
+
+/*
+ * Write 8 bytes of random data in random. Returns true on success, false on
+ * failure.
+ */
+static inline bool arch_collect_entropy(uint64_t *random)
+{
+ unsigned long rc;
+ uint64_t val;
+
+ asm volatile(
+ " mrs %[val], " __XSTRING(RNDR) "\n"
+ " str %[val], %[random_ptr]\n"
+ " cset %[rc], ne\n" /* RNDR sets NZCV to 0b0100 on failure */
+ : [random_ptr] "=m" (*random),
+ [rc] "=r" (rc),
+ [val] "=r" (val)
+ :
+ : "cc"
+ );
+ return (rc == 1);
+}
+
+#endif /* ENTROPY_H */
diff --git a/lib/arch/include/aarch64/instr_helpers.h b/lib/arch/include/aarch64/instr_helpers.h
new file mode 100644
index 0000000..ebedb22
--- /dev/null
+++ b/lib/arch/include/aarch64/instr_helpers.h
@@ -0,0 +1,71 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
+ */
+
+#ifndef INSTR_HELPERS_H
+#define INSTR_HELPERS_H
+
+#include <types.h>
+
+/**********************************************************************
+ * Macros which create inline functions to read or write CPU system
+ * registers
+ *********************************************************************/
+
+#define DEFINE_SYSREG_READ_FUNC_(_name, _reg_name) \
+static inline u_register_t read_ ## _name(void) \
+{ \
+ u_register_t v; \
+ __asm__ volatile ("mrs %0, " #_reg_name : "=r" (v)); \
+ return v; \
+}
+
+#define DEFINE_SYSREG_WRITE_FUNC_(_name, _reg_name) \
+static inline void write_ ## _name(u_register_t v) \
+{ \
+ (void)v; /* To avoid MISRA-C:2012-2.7 warnings */ \
+ __asm__ volatile ("msr " #_reg_name ", %0" : : "r" (v));\
+}
+
+#define SYSREG_WRITE_CONST(reg_name, v) \
+ __asm__ volatile ("msr " #reg_name ", %0" : : "i" (v))
+
+/**********************************************************************
+ * Macros to create inline functions for system instructions
+ *********************************************************************/
+
+/* Define function for simple system instruction */
+#define DEFINE_SYSOP_FUNC(_op) \
+static inline void (_op)(void) \
+{ \
+ __asm__ (#_op); \
+}
+
+/* Define function for system instruction with register parameter */
+#define DEFINE_SYSOP_PARAM_FUNC(_op) \
+static inline void (_op)(uint64_t v) \
+{ \
+ (void)v; /* To avoid MISRA-C:2012-2.7 warnings */ \
+ __asm__ (#_op " %0" : : "r" (v)); \
+}
+
+/* Define function for system instruction with type specifier */
+#define DEFINE_SYSOP_TYPE_FUNC(_op, _type) \
+static inline void (_op ## _type)(void) \
+{ \
+ __asm__ (#_op " " #_type : : : "memory"); \
+}
+
+/* Define function for system instruction with register parameter */
+#define DEFINE_SYSOP_TYPE_PARAM_FUNC(_op, _type) \
+static inline void (_op ## _type)(uint64_t v) \
+{ \
+ (void)v; /* To avoid MISRA-C:2012-2.7 warnings */ \
+ __asm__ (#_op " " #_type ", %0" : : "r" (v)); \
+}
+
+#define dsb(scope) asm volatile("dsb " #scope : : : "memory")
+#define dmb(scope) asm volatile("dmb " #scope : : : "memory")
+
+#endif /* INSTR_HELPERS_H */
diff --git a/lib/arch/include/aarch64/memory.h b/lib/arch/include/aarch64/memory.h
new file mode 100644
index 0000000..34fe271
--- /dev/null
+++ b/lib/arch/include/aarch64/memory.h
@@ -0,0 +1,74 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
+ */
+
+#ifndef MEMORY_H
+#define MEMORY_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+/* Single-Copy Atomic 64-bit write */
+static inline void __sca_write64(uint64_t *ptr, uint64_t val)
+{
+ /* To avoid misra-c2012-2.7 warnings */
+ (void)ptr;
+ (void)val;
+
+ asm volatile(
+ " str %[val], %[ptr]\n"
+ : [ptr] "=m" (*ptr)
+ : [val] "r" (val)
+ );
+}
+#define SCA_WRITE64(_p, _v) __sca_write64((void *)(_p), ((uint64_t)(_v)))
+
+/* Single-Copy Atomic 64-bit write with RELEASE memory ordering semantics*/
+static inline void __sca_write64_release(uint64_t *ptr, uint64_t val)
+{
+ asm volatile(
+ " stlr %[val], %[ptr]\n"
+ : [ptr] "=Q" (*ptr)
+ : [val] "r" (val)
+ );
+}
+#define SCA_WRITE64_RELEASE(_p, _v) __sca_write64_release((void *)(_p), ((uint64_t)(_v)))
+
+/* Single-Copy Atomic 64-bit read */
+static inline uint64_t __sca_read64(uint64_t *ptr)
+{
+ uint64_t val;
+
+ /* To avoid misra-c2012-2.7 warnings */
+ (void)ptr;
+
+ asm volatile(
+ " ldr %[val], %[ptr]\n"
+ : [val] "=r" (val)
+ : [ptr] "m" (*ptr)
+ );
+
+ return val;
+}
+#define SCA_READ64(_p) ((typeof(*(_p)))__sca_read64((void *)(_p)))
+
+/* Single-Copy Atomic 64-bit read with ACQUIRE memory ordering semantics */
+static inline uint64_t __sca_read64_acquire(uint64_t *ptr)
+{
+ uint64_t val;
+
+ /* To avoid misra-c2012-2.7 warnings */
+ (void)ptr;
+
+ asm volatile(
+ " ldar %[val], %[ptr]\n"
+ : [val] "=r" (val)
+ : [ptr] "Q" (*ptr)
+ );
+
+ return val;
+}
+#define SCA_READ64_ACQUIRE(_p) ((typeof(*(_p)))__sca_read64_acquire((void *)(_p)))
+
+#endif /* MEMORY_H */
diff --git a/lib/arch/include/aarch64/mmio.h b/lib/arch/include/aarch64/mmio.h
new file mode 100644
index 0000000..ba8c82c3
--- /dev/null
+++ b/lib/arch/include/aarch64/mmio.h
@@ -0,0 +1,80 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
+ */
+
+#ifndef MMIO_H
+#define MMIO_H
+
+#include <arch_helpers.h>
+#include <stdint.h>
+
+static inline uint8_t read8(volatile void *addr)
+{
+ uint8_t val;
+
+ dsb(ld);
+ asm volatile("ldrb %w0, [%1]" : "=r" (val) : "r" (addr));
+ dsb(ld);
+ return val;
+}
+
+static inline void write8(uint8_t val, volatile void *addr)
+{
+ dsb(st);
+ asm volatile("strb %w0, [%1]" : : "r" (val), "r" (addr));
+ dsb(st);
+}
+
+static inline uint16_t read16(volatile void *addr)
+{
+ uint16_t val;
+
+ dsb(ld);
+ asm volatile("ldrh %w0, [%1]" : "=r" (val) : "r" (addr));
+ dsb(ld);
+ return val;
+}
+
+static inline void write16(uint16_t val, volatile void *addr)
+{
+ dsb(st);
+ asm volatile("strh %w0, [%1]" : : "r" (val), "r" (addr));
+ dsb(st);
+}
+
+static inline uint32_t read32(volatile void *addr)
+{
+ uint32_t val;
+
+ dsb(ld);
+ asm volatile("ldr %w0, [%1]" : "=r" (val) : "r" (addr));
+ dsb(ld);
+ return val;
+}
+
+static inline void write32(uint32_t val, volatile void *addr)
+{
+ dsb(st);
+ asm volatile("str %w0, [%1]" : : "r" (val), "r" (addr));
+ dsb(st);
+}
+
+static inline uint64_t read64(volatile void *addr)
+{
+ uint64_t val;
+
+ dsb(ld);
+ asm volatile("ldr %0, [%1]" : "=r" (val) : "r" (addr));
+ dsb(ld);
+ return val;
+}
+
+static inline void write64(uint64_t val, volatile void *addr)
+{
+ dsb(st);
+ asm volatile("str %0, [%1]" : : "r" (val), "r" (addr));
+ dsb(st);
+}
+
+#endif /* MMIO_H */
diff --git a/lib/arch/include/aarch64/spinlock.h b/lib/arch/include/aarch64/spinlock.h
new file mode 100644
index 0000000..37cab6e
--- /dev/null
+++ b/lib/arch/include/aarch64/spinlock.h
@@ -0,0 +1,47 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
+ */
+
+#ifndef SPINLOCK_H
+#define SPINLOCK_H
+
+/*
+ * A trivial spinlock implementation, per ARM DDI 0487D.a, section K11.3.4.
+ */
+
+typedef struct {
+ unsigned int val;
+} spinlock_t;
+
+static inline void spinlock_acquire(spinlock_t *l)
+{
+ unsigned int tmp;
+
+ asm volatile(
+ " sevl\n"
+ " prfm pstl1keep, %[lock]\n"
+ "1:\n"
+ " wfe\n"
+ " ldaxr %w[tmp], %[lock]\n"
+ " cbnz %w[tmp], 1b\n"
+ " stxr %w[tmp], %w[one], %[lock]\n"
+ " cbnz %w[tmp], 1b\n"
+ : [lock] "+Q" (l->val),
+ [tmp] "=&r" (tmp)
+ : [one] "r" (1)
+ : "memory"
+ );
+}
+
+static inline void spinlock_release(spinlock_t *l)
+{
+ asm volatile(
+ " stlr wzr, %[lock]\n"
+ : [lock] "+Q" (l->val)
+ :
+ : "memory"
+ );
+}
+
+#endif /* SPINLOCK_H */
diff --git a/lib/arch/include/arch.h b/lib/arch/include/arch.h
new file mode 100644
index 0000000..a67baa4
--- /dev/null
+++ b/lib/arch/include/arch.h
@@ -0,0 +1,939 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
+ */
+
+#ifndef ARCH_H
+#define ARCH_H
+
+#include <utils_def.h>
+
+/* Cache line size */
+#define CACHE_WRITEBACK_GRANULE UL(64)
+
+/* Timer interrupt IDs defined by the Server Base System Architecture */
+#define EL1_VIRT_TIMER_PPI UL(27)
+#define EL1_PHYS_TIMER_PPI UL(30)
+
+/* Counter-timer Physical Offset register */
+#define CNTPOFF_EL2 S3_4_C14_C0_6
+
+/* MPAM0 Register */
+#define MPAM0_EL1 S3_0_C10_C5_1
+
+/* Interrupt Controller registers */
+#define ICC_HPPIR1_EL1 S3_0_C12_C12_2
+#define ICC_SRE_EL2 S3_4_C12_C9_5
+
+/* Interrupt Controller Control Register */
+#define ICC_CTLR_EL1 S3_0_C12_C12_4
+
+#define ICC_CTLR_EL1_EXT_RANGE_SHIFT UL(19)
+#define ICC_CTLR_EL1_EXT_RANGE_BIT INPLACE(ICC_CTLR_EL1_EXT_RANGE, UL(1))
+
+/* Virtual GIC registers */
+#define ICH_AP0R0_EL2 S3_4_C12_C8_0
+#define ICH_AP0R1_EL2 S3_4_C12_C8_1
+#define ICH_AP0R2_EL2 S3_4_C12_C8_2
+#define ICH_AP0R3_EL2 S3_4_C12_C8_3
+#define ICH_AP1R0_EL2 S3_4_C12_C9_0
+#define ICH_AP1R1_EL2 S3_4_C12_C9_1
+#define ICH_AP1R2_EL2 S3_4_C12_C9_2
+#define ICH_AP1R3_EL2 S3_4_C12_C9_3
+
+#define ICH_LR0_EL2 S3_4_C12_C12_0
+#define ICH_LR1_EL2 S3_4_C12_C12_1
+#define ICH_LR2_EL2 S3_4_C12_C12_2
+#define ICH_LR3_EL2 S3_4_C12_C12_3
+#define ICH_LR4_EL2 S3_4_C12_C12_4
+#define ICH_LR5_EL2 S3_4_C12_C12_5
+#define ICH_LR6_EL2 S3_4_C12_C12_6
+#define ICH_LR7_EL2 S3_4_C12_C12_7
+#define ICH_LR8_EL2 S3_4_C12_C13_0
+#define ICH_LR9_EL2 S3_4_C12_C13_1
+#define ICH_LR10_EL2 S3_4_C12_C13_2
+#define ICH_LR11_EL2 S3_4_C12_C13_3
+#define ICH_LR12_EL2 S3_4_C12_C13_4
+#define ICH_LR13_EL2 S3_4_C12_C13_5
+#define ICH_LR14_EL2 S3_4_C12_C13_6
+#define ICH_LR15_EL2 S3_4_C12_C13_7
+
+#define ICH_HCR_EL2 S3_4_C12_C11_0
+#define ICH_VTR_EL2 S3_4_C12_C11_1
+#define ICH_MISR_EL2 S3_4_C12_C11_2
+#define ICH_VMCR_EL2 S3_4_C12_C11_7
+
+/* RNDR definition */
+#define RNDR S3_3_C2_C4_0
+
+/* CLIDR definitions */
+#define LOC_SHIFT U(24)
+#define CTYPE_SHIFT(n) U(3 * ((n) - 1))
+#define CLIDR_FIELD_WIDTH U(3)
+
+/* CSSELR definitions */
+#define LEVEL_SHIFT U(1)
+
+/* Data cache set/way op type defines */
+#define DCISW U(0x0)
+#define DCCISW U(0x1)
+#define DCCSW U(0x2)
+
+#define TCR_EL2_T0SZ_SHIFT UL(0)
+#define TCR_EL2_T0SZ_WIDTH UL(6)
+#define TCR_EL2_T0SZ_MASK MASK(TCR_EL2_T0SZ)
+
+#define TCR_EL2_T1SZ_SHIFT UL(16)
+#define TCR_EL2_T1SZ_WIDTH UL(6)
+#define TCR_EL2_T1SZ_MASK MASK(TCR_EL2_T0SZ)
+
+#define TCR_EL2_EPD0_SHIFT UL(7)
+#define TCR_EL2_EPD0_WIDTH UL(1)
+#define TCR_EL2_EPD0_BIT INPLACE(TCR_EL2_EPD0, UL(1))
+
+#define TCR_EL2_IRGN0_SHIFT UL(8)
+#define TCR_EL2_IRGN0_WIDTH UL(2)
+#define TCR_EL2_IRGN0_WBWA INPLACE(TCR_EL2_IRGN0, UL(1))
+
+#define TCR_EL2_ORGN0_SHIFT UL(10)
+#define TCR_EL2_ORGN0_WIDTH UL(2)
+#define TCR_EL2_ORGN0_WBWA INPLACE(TCR_EL2_ORGN0, UL(1))
+
+#define TCR_EL2_IRGN1_SHIFT UL(24)
+#define TCR_EL2_IRGN1_WIDTH UL(2)
+#define TCR_EL2_IRGN1_WBWA INPLACE(TCR_EL2_IRGN1, UL(1))
+
+#define TCR_EL2_ORGN1_SHIFT UL(26)
+#define TCR_EL2_ORGN1_WIDTH UL(2)
+#define TCR_EL2_ORGN1_WBWA INPLACE(TCR_EL2_ORGN1, UL(1))
+
+#define TCR_EL2_SH0_SHIFT UL(12)
+#define TCR_EL2_SH0_WIDTH UL(2)
+#define TCR_EL2_SH0_IS INPLACE(TCR_EL2_SH0, UL(3))
+
+#define TCR_EL2_SH1_SHIFT UL(28)
+#define TCR_EL2_SH1_WIDTH UL(2)
+#define TCR_EL2_SH1_IS INPLACE(TCR_EL2_SH1, UL(3))
+
+#define TCR_EL2_TG0_SHIFT UL(14)
+#define TCR_EL2_TG0_WIDTH UL(2)
+#define TCR_EL2_TG0_4K INPLACE(TCR_EL2_TG0, UL(0))
+
+#define TCR_EL2_TG1_SHIFT UL(30)
+#define TCR_EL2_TG1_WIDTH UL(2)
+#define TCR_EL2_TG1_4K INPLACE(TCR_EL2_TG1, UL(0))
+
+#define TCR_EL2_IPS_SHIFT UL(32)
+#define TCR_EL2_IPS_WIDTH UL(3)
+#define TCR_PS_BITS_4GB INPLACE(TCR_EL2_IPS, UL(0))
+#define TCR_PS_BITS_64GB INPLACE(TCR_EL2_IPS, UL(1))
+#define TCR_PS_BITS_1TB INPLACE(TCR_EL2_IPS, UL(2))
+#define TCR_PS_BITS_4TB INPLACE(TCR_EL2_IPS, UL(3))
+#define TCR_PS_BITS_16TB INPLACE(TCR_EL2_IPS, UL(4))
+#define TCR_PS_BITS_256TB INPLACE(TCR_EL2_IPS, UL(5))
+
+#define ADDR_MASK_48_TO_63 ULL(0xFFFF000000000000)
+#define ADDR_MASK_44_TO_47 ULL(0x0000F00000000000)
+#define ADDR_MASK_42_TO_43 ULL(0x00000C0000000000)
+#define ADDR_MASK_40_TO_41 ULL(0x0000030000000000)
+#define ADDR_MASK_36_TO_39 ULL(0x000000F000000000)
+#define ADDR_MASK_32_TO_35 ULL(0x0000000F00000000)
+
+#define TCR_EL2_AS (UL(1) << 36)
+#define TCR_EL2_HPD0 (UL(1) << 41)
+#define TCR_EL2_HPD1 (UL(1) << 42)
+#define TCR_EL2_E0PD1 (UL(1) << 56) /* TODO: ARMv8.5-E0PD, otherwise RES0 */
+
+#define TCR_TxSZ_MIN UL(16)
+#define TCR_TxSZ_MAX UL(39)
+#define TCR_TxSZ_MAX_TTST UL(48)
+
+/* HCR definitions */
+#define HCR_FWB (UL(1) << 46)
+#define HCR_TEA (UL(1) << 37)
+#define HCR_API (UL(1) << 41)
+#define HCR_APK (UL(1) << 40)
+#define HCR_TERR (UL(1) << 36)
+#define HCR_TLOR (UL(1) << 35)
+#define HCR_E2H (UL(1) << 34)
+#define HCR_RW (UL(1) << 31)
+#define HCR_TGE (UL(1) << 27)
+#define HCR_TSW (UL(1) << 22)
+#define HCR_TACR (UL(1) << 21)
+#define HCR_TIDCP (UL(1) << 20)
+#define HCR_TSC (UL(1) << 19)
+#define HCR_TID3 (UL(1) << 18)
+#define HCR_TWE (UL(1) << 14)
+#define HCR_TWI (UL(1) << 13)
+#define HCR_VSE (UL(1) << 8)
+
+#define HCR_BSU_SHIFT 10
+#define HCR_BSU_WIDTH 2
+#define HCR_BSU_IS INPLACE(HCR_BSU, 1) /* Barriers are promoted to IS */
+
+#define HCR_FB (UL(1) << 9)
+#define HCR_VI (UL(1) << 7)
+#define HCR_AMO (UL(1) << 5)
+#define HCR_IMO (UL(1) << 4)
+#define HCR_FMO (UL(1) << 3)
+#define HCR_PTW (UL(1) << 2)
+#define HCR_SWIO (UL(1) << 1)
+#define HCR_VM (UL(1) << 0)
+
+/* TODO verify that all the traps are enabled */
+#define HCR_FLAGS (HCR_FWB | HCR_E2H | HCR_RW | HCR_TSC | HCR_AMO | \
+ HCR_BSU_IS | HCR_IMO | HCR_FMO | HCR_PTW | HCR_SWIO | HCR_VM | \
+ HCR_TID3 | HCR_TEA)
+
+#define HCR_EL2_INIT (HCR_TGE | HCR_E2H | HCR_TEA)
+
+#define MAIR_ELx_ATTR0_SHIFT 0
+#define MAIR_ELx_ATTR0_WIDTH 8
+#define MAIR_ELx_ATTR0_MASK MASK(MAIR_ELx_ATTR0)
+
+/*******************************************************************************
+ * Definitions of MAIR encodings for device and normal memory
+ ******************************************************************************/
+/*
+ * MAIR encodings for device memory attributes.
+ */
+#define MAIR_DEV_NGNRNE UL(0x0) /* Device nGnRnE */
+#define MAIR_DEV_NGNRNE_IDX 0x1
+
+#define MAIR_DEV_NGNRE UL(0x4)
+
+#define MAIR_NIOWBNTRW 0xff
+#define MAIR_NIOWBNTRW_IDX 0x0
+
+/*
+ * MAIR encodings for normal memory attributes.
+ *
+ * Cache Policy
+ * WT: Write Through
+ * WB: Write Back
+ * NC: Non-Cacheable
+ *
+ * Transient Hint
+ * NTR: Non-Transient
+ * TR: Transient
+ *
+ * Allocation Policy
+ * RA: Read Allocate
+ * WA: Write Allocate
+ * RWA: Read and Write Allocate
+ * NA: No Allocation
+ */
+#define MAIR_NORM_WT_TR_WA UL(0x1)
+#define MAIR_NORM_WT_TR_RA UL(0x2)
+#define MAIR_NORM_WT_TR_RWA UL(0x3)
+#define MAIR_NORM_NC UL(0x4)
+#define MAIR_NORM_WB_TR_WA UL(0x5)
+#define MAIR_NORM_WB_TR_RA UL(0x6)
+#define MAIR_NORM_WB_TR_RWA UL(0x7)
+#define MAIR_NORM_WT_NTR_NA UL(0x8)
+#define MAIR_NORM_WT_NTR_WA UL(0x9)
+#define MAIR_NORM_WT_NTR_RA UL(0xa)
+#define MAIR_NORM_WT_NTR_RWA UL(0xb)
+#define MAIR_NORM_WB_NTR_NA UL(0xc)
+#define MAIR_NORM_WB_NTR_WA UL(0xd)
+#define MAIR_NORM_WB_NTR_RA UL(0xe)
+#define MAIR_NORM_WB_NTR_RWA UL(0xf)
+
+#define MAIR_NORM_OUTER_SHIFT U(4)
+
+#define MAKE_MAIR_NORMAL_MEMORY(inner, outer) \
+ ((inner) | ((outer) << MAIR_NORM_OUTER_SHIFT))
+
+#define MAKE_MAIR_NORMAL_MEMORY_IO(_mair) \
+ MAKE_MAIR_NORMAL_MEMORY(_mair, _mair)
+
+/*
+ * TTBR Definitions
+ */
+#define TTBR_CNP_BIT UL(0x1)
+
+#define TTBRx_EL2_CnP_SHIFT 0
+#define TTBRx_EL2_CnP_WIDTH 1
+
+#define TTBRx_EL2_BADDR_SHIFT 1
+#define TTBRx_EL2_BADDR_WIDTH 47
+
+#define TTBRx_EL2_ASID_SHIFT 48
+#define TTBRx_EL2_ASID_WIDTH 16
+
+/*
+ * VTTBR Definitions
+ */
+#define VTTBR_EL2_VMID_SHIFT 48
+#define VTTBR_EL2_VMID_WIDTH 16
+
+/*
+ * ESR Definitions
+ */
+#define ESR_EL2_EC_SHIFT 26
+#define ESR_EL2_EC_WIDTH 6
+#define ESR_EL2_EC_MASK MASK(ESR_EL2_EC)
+
+#define ESR_EL2_IL_SHIFT 25
+#define ESR_EL2_IL_WIDTH 1
+#define ESR_EL2_IL_MASK MASK(ESR_EL2_IL)
+
+#define ESR_EL2_ISS_SHIFT 0
+#define ESR_EL2_ISS_WIDTH 25
+#define ESR_EL2_ISS_MASK MASK(ESR_EL2_ISS)
+
+#define ESR_EL2_EC_UNKNOWN INPLACE(ESR_EL2_EC, 0)
+#define ESR_EL2_EC_WFX INPLACE(ESR_EL2_EC, 1)
+#define ESR_EL2_EC_FPU INPLACE(ESR_EL2_EC, 7)
+#define ESR_EL2_EC_SVC INPLACE(ESR_EL2_EC, 21)
+#define ESR_EL2_EC_HVC INPLACE(ESR_EL2_EC, 22)
+#define ESR_EL2_EC_SMC INPLACE(ESR_EL2_EC, 23)
+#define ESR_EL2_EC_SYSREG INPLACE(ESR_EL2_EC, 24)
+#define ESR_EL2_EC_SVE INPLACE(ESR_EL2_EC, 25)
+#define ESR_EL2_EC_INST_ABORT INPLACE(ESR_EL2_EC, 32)
+#define ESR_EL2_EC_INST_ABORT_SEL INPLACE(ESR_EL2_EC, 33)
+#define ESR_EL2_EC_DATA_ABORT INPLACE(ESR_EL2_EC, 36)
+#define ESR_EL2_EC_DATA_ABORT_SEL INPLACE(ESR_EL2_EC, 37)
+#define ESR_EL2_EC_SERROR INPLACE(ESR_EL2_EC, 47)
+
+/* Data/Instruction Abort ESR fields */
+#define ESR_EL2_ABORT_ISV_BIT (1UL << 24)
+
+#define ESR_EL2_ABORT_SAS_SHIFT 22
+#define ESR_EL2_ABORT_SAS_WIDTH 2
+#define ESR_EL2_ABORT_SAS_MASK MASK(ESR_EL2_ABORT_SAS)
+
+#define ESR_EL2_ABORT_SAS_BYTE_VAL 0
+#define ESR_EL2_ABORT_SAS_HWORD_VAL 1
+#define ESR_EL2_ABORT_SAS_WORD_VAL 2
+#define ESR_EL2_ABORT_SAS_DWORD_VAL 3
+
+#define ESR_EL2_ABORT_SSE_BIT (1UL << 21)
+
+#define ESR_EL2_ABORT_SRT_SHIFT 16
+#define ESR_EL2_ABORT_SRT_WIDTH 5
+#define ESR_EL2_ABORT_SRT_MASK MASK(ESR_EL2_ABORT_SRT)
+
+#define ESR_EL2_ABORT_SET_SHIFT 11
+#define ESR_EL2_ABORT_SET_WIDTH 2
+#define ESR_EL2_ABORT_SET_MASK MASK(ESR_EL2_ABORT_SET)
+#define ESR_EL2_ABORT_SET_UER INPLACE(ESR_EL2_ABORT_SET, 0)
+#define ESR_EL2_ABORT_SET_UC INPLACE(ESR_EL2_ABORT_SET, 2)
+#define ESR_EL2_ABORT_SET_UEO INPLACE(ESR_EL2_ABORT_SET, 3)
+
+#define ESR_EL2_ABORT_SF_BIT (1UL << 15)
+#define ESR_EL2_ABORT_FNV_BIT (1UL << 10)
+#define ESR_EL2_ABORT_EA_BIT (1UL << 9)
+#define ESR_EL2_ABORT_S1PTW_BIT (1UL << 7)
+#define ESR_EL2_ABORT_WNR_BIT (1UL << 6)
+#define ESR_EL2_ABORT_FSC_SHIFT 0
+#define ESR_EL2_ABORT_FSC_WIDTH 6
+#define ESR_EL2_ABORT_FSC_MASK MASK(ESR_EL2_ABORT_FSC)
+
+#define ESR_EL2_ABORT_FSC_TRANSLATION_FAULT 0x04
+#define ESR_EL2_ABORT_FSC_PERMISSION_FAULT 0x0c
+#define ESR_EL2_ABORT_FSC_TRANSLATION_FAULT_L0 0x04
+#define ESR_EL2_ABORT_FSC_SEA 0x10
+#define ESR_EL2_ABORT_FSC_SEA_TTW_START 0x13
+#define ESR_EL2_ABORT_FSC_SEA_TTW_END 0x17
+#define ESR_EL2_ABORT_FSC_GPF 0x28
+#define ESR_EL2_ABORT_FSC_LEVEL_SHIFT 0
+#define ESR_EL2_ABORT_FSC_LEVEL_WIDTH 2
+#define ESR_EL2_ABORT_FSC_LEVEL_MASK MASK(ESR_EL2_ABORT_FSC_LEVEL)
+
+/* The ESR fields that are reported to the host on Instr./Data Synchronous Abort */
+#define ESR_NONEMULATED_ABORT_MASK ( \
+ ESR_EL2_EC_MASK | \
+ ESR_EL2_ABORT_SET_MASK | \
+ ESR_EL2_ABORT_FNV_BIT | \
+ ESR_EL2_ABORT_EA_BIT | \
+ ESR_EL2_ABORT_FSC_MASK)
+
+#define ESR_EMULATED_ABORT_MASK ( \
+ ESR_NONEMULATED_ABORT_MASK | \
+ ESR_EL2_ABORT_ISV_BIT | \
+ ESR_EL2_ABORT_SAS_MASK | \
+ ESR_EL2_ABORT_SF_BIT | \
+ ESR_EL2_ABORT_WNR_BIT)
+
+#define ESR_EL2_SERROR_DFSC_SHIFT 0
+#define ESR_EL2_SERROR_DFSC_WIDTH 6
+#define ESR_EL2_SERROR_DFSC_MASK MASK(ESR_EL2_SERROR_DFSC)
+#define ESR_EL2_SERROR_DFSC_UNCAT INPLACE(ESR_EL2_SERROR_DFSC, 0)
+#define ESR_EL2_SERROR_DFSC_ASYNC INPLACE(ESR_EL2_SERROR_DFSC, 1)
+
+#define ESR_EL2_SERROR_EA_BIT (1UL << 9)
+
+#define ESR_EL2_SERROR_AET_SHIFT 10
+#define ESR_EL2_SERROR_AET_WIDTH 3
+#define ESR_EL2_SERROR_AET_MASK MASK(ESR_EL2_SERROR_AET)
+#define ESR_EL2_SERROR_AET_UC INPLACE(ESR_EL2_SERROR_AET, 0)
+#define ESR_EL2_SERROR_AET_UEU INPLACE(ESR_EL2_SERROR_AET, 1)
+#define ESR_EL2_SERROR_AET_UEO INPLACE(ESR_EL2_SERROR_AET, 2)
+#define ESR_EL2_SERROR_AET_UER INPLACE(ESR_EL2_SERROR_AET, 3)
+#define ESR_EL2_SERROR_AET_CE INPLACE(ESR_EL2_SERROR_AET, 6)
+
+#define ESR_EL2_SERROR_IESB_BIT (1UL << 13)
+#define ESR_EL2_SERROR_IDS_BIT (1UL << 24)
+
+/* The ESR fields that are reported to the host on SError */
+#define ESR_SERROR_MASK ( \
+ ESR_EL2_SERROR_IDS_BIT | \
+ ESR_EL2_SERROR_AET_MASK | \
+ ESR_EL2_SERROR_EA_BIT | \
+ ESR_EL2_SERROR_DFSC_MASK)
+
+#define ESR_EL2_SYSREG_TRAP_OP0_SHIFT 20
+#define ESR_EL2_SYSREG_TRAP_OP0_WIDTH 2
+#define ESR_EL2_SYSREG_TRAP_OP0_MASK MASK(ESR_EL2_SYSREG_TRAP_OP0)
+
+#define ESR_EL2_SYSREG_TRAP_OP2_SHIFT 17
+#define ESR_EL2_SYSREG_TRAP_OP2_WIDTH 3
+#define ESR_EL2_SYSREG_TRAP_OP2_MASK MASK(ESR_EL2_SYSREG_TRAP_OP2)
+
+#define ESR_EL2_SYSREG_TRAP_OP1_SHIFT 14
+#define ESR_EL2_SYSREG_TRAP_OP1_WIDTH 3
+#define ESR_EL2_SYSREG_TRAP_OP1_MASK MASK(ESR_EL2_SYSREG_TRAP_OP1)
+
+#define ESR_EL2_SYSREG_TRAP_CRN_SHIFT 10
+#define ESR_EL2_SYSREG_TRAP_CRN_WIDTH 4
+#define ESR_EL2_SYSREG_TRAP_CRN_MASK MASK(ESR_EL2_SYSREG_TRAP_CRN)
+
+#define ESR_EL2_SYSREG_TRAP_RT_SHIFT 5
+#define ESR_EL2_SYSREG_TRAP_RT_WIDTH 5
+#define ESR_EL2_SYSREG_TRAP_RT_MASK MASK(ESR_EL2_SYSREG_TRAP_RT)
+
+#define ESR_EL2_SYSREG_TRAP_CRM_SHIFT 1
+#define ESR_EL2_SYSREG_TRAP_CRM_WIDTH 4
+#define ESR_EL2_SYSREG_TRAP_CRM_MASK MASK(ESR_EL2_SYSREG_TRAP_CRM)
+
+/* WFx ESR fields */
+#define ESR_EL2_WFx_TI_BIT (1UL << 0)
+
+/* xVC ESR fields */
+#define ESR_EL2_xVC_IMM_SHIFT 0
+#define ESR_EL2_xVC_IMM_WIDTH 16
+#define ESR_EL2_xVC_IMM_MASK MASK(ESR_EL2_xVC_IMM)
+
+/* ID_AA64PFR0_EL1 definitions */
+#define ID_AA64PFR0_EL1_SVE_SHIFT UL(32)
+#define ID_AA64PFR0_EL1_SVE_WIDTH UL(4)
+#define ID_AA64PFR0_EL1_SVE_MASK UL(0xf)
+
+#define ID_AA64PFR0_EL1_AMU_SHIFT UL(44)
+#define ID_AA64PFR0_EL1_AMU_WIDTH 4
+
+/* ID_AA64MMFR0_EL1 definitions */
+#define ID_AA64MMFR0_EL1_PARANGE_SHIFT U(0)
+#define ID_AA64MMFR0_EL1_PARANGE_MASK ULL(0xf)
+
+#define PARANGE_0000_WIDTH U(32)
+#define PARANGE_0001_WIDTH U(36)
+#define PARANGE_0010_WIDTH U(40)
+#define PARANGE_0011_WIDTH U(42)
+#define PARANGE_0100_WIDTH U(44)
+#define PARANGE_0101_WIDTH U(48)
+#define PARANGE_0110_WIDTH U(52)
+
+#define ID_AA64MMFR0_EL1_ECV_SHIFT U(60)
+#define ID_AA64MMFR0_EL1_ECV_MASK ULL(0xf)
+#define ID_AA64MMFR0_EL1_ECV_NOT_SUPPORTED ULL(0x0)
+#define ID_AA64MMFR0_EL1_ECV_SUPPORTED ULL(0x1)
+#define ID_AA64MMFR0_EL1_ECV_SELF_SYNCH ULL(0x2)
+
+#define ID_AA64MMFR0_EL1_FGT_SHIFT U(56)
+#define ID_AA64MMFR0_EL1_FGT_MASK ULL(0xf)
+#define ID_AA64MMFR0_EL1_FGT_SUPPORTED ULL(0x1)
+#define ID_AA64MMFR0_EL1_FGT_NOT_SUPPORTED ULL(0x0)
+
+#define ID_AA64MMFR0_EL1_TGRAN4_2_SHIFT U(40)
+#define ID_AA64MMFR0_EL1_TGRAN4_2_MASK ULL(0xf)
+#define ID_AA64MMFR0_EL1_TGRAN4_2_TGRAN4 ULL(0x0)
+#define ID_AA64MMFR0_EL1_TGRAN4_2_NOT_SUPPORTED ULL(0x1)
+#define ID_AA64MMFR0_EL1_TGRAN4_2_SUPPORTED ULL(0x2)
+#define ID_AA64MMFR0_EL1_TGRAN4_2_LPA2 ULL(0x3)
+
+#define ID_AA64MMFR0_EL1_TGRAN16_2_SHIFT U(32)
+#define ID_AA64MMFR0_EL1_TGRAN16_2_MASK ULL(0xf)
+#define ID_AA64MMFR0_EL1_TGRAN16_2_TGRAN16 ULL(0x0)
+#define ID_AA64MMFR0_EL1_TGRAN16_2_NOT_SUPPORTED ULL(0x1)
+#define ID_AA64MMFR0_EL1_TGRAN16_2_SUPPORTED ULL(0x2)
+#define ID_AA64MMFR0_EL1_TGRAN16_2_LPA2 ULL(0x3)
+
+#define ID_AA64MMFR0_EL1_TGRAN4_SHIFT U(28)
+#define ID_AA64MMFR0_EL1_TGRAN4_MASK ULL(0xf)
+#define ID_AA64MMFR0_EL1_TGRAN4_SUPPORTED ULL(0x0)
+#define ID_AA64MMFR0_EL1_TGRAN4_LPA2 ULL(0x1)
+#define ID_AA64MMFR0_EL1_TGRAN4_NOT_SUPPORTED ULL(0xf)
+
+#define ID_AA64MMFR0_EL1_TGRAN64_SHIFT UL(24)
+#define ID_AA64MMFR0_EL1_TGRAN64_MASK UL(0xf)
+#define ID_AA64MMFR0_EL1_TGRAN64_SUPPORTED UL(0x0)
+#define ID_AA64MMFR0_EL1_TGRAN64_NOT_SUPPORTED UL(0xf)
+
+#define ID_AA64MMFR0_EL1_TGRAN16_SHIFT UL(20)
+#define ID_AA64MMFR0_EL1_TGRAN16_MASK UL(0xf)
+#define ID_AA64MMFR0_EL1_TGRAN16_NOT_SUPPORTED UL(0x0)
+#define ID_AA64MMFR0_EL1_TGRAN16_SUPPORTED UL(0x1)
+#define ID_AA64MMFR0_EL1_TGRAN16_LPA2 UL(0x2)
+
+/* RNDR definitions */
+#define ID_AA64ISAR0_RNDR_SHIFT UL(60)
+#define ID_AA64ISAR0_RNDR_MASK UL(0xF)
+
+/* ID_AA64MMFR1_EL1 definitions */
+#define ID_AA64MMFR1_EL1_VMIDBits_SHIFT UL(4)
+#define ID_AA64MMFR1_EL1_VMIDBits_MASK UL(0xf)
+#define ID_AA64MMFR1_EL1_VMIDBits_8 UL(0)
+#define ID_AA64MMFR1_EL1_VMIDBits_16 UL(2)
+
+/* HPFAR_EL2 definitions */
+#define HPFAR_EL2_FIPA_SHIFT 4
+#define HPFAR_EL2_FIPA_WIDTH 40
+#define HPFAR_EL2_FIPA_MASK MASK(HPFAR_EL2_FIPA)
+#define HPFAR_EL2_FIPA_OFFSET 8
+
+/* SPSR definitions */
+#define SPSR_EL2_MODE_SHIFT 0
+#define SPSR_EL2_MODE_WIDTH 4
+#define SPSR_EL2_MODE_EL0t INPLACE(SPSR_EL2_MODE, 0)
+
+#define SPSR_EL2_MODE_SHIFT 0
+#define SPSR_EL2_MODE_WIDTH 4
+#define SPSR_EL2_MODE_EL1h INPLACE(SPSR_EL2_MODE, 5)
+#define SPSR_EL2_MODE_EL1t INPLACE(SPSR_EL2_MODE, 4)
+
+/* FIXME: DAIF definitions are redundant here. Might need unification. */
+#define SPSR_EL2_nRW_SHIFT 4
+#define SPSR_EL2_nRW_WIDTH 1
+#define SPSR_EL2_nRW_AARCH64 INPLACE(SPSR_EL2_nRW, 0)
+#define SPSR_EL2_nRW_AARCH32 INPLACE(SPSR_EL2_nRW, 1)
+
+#define SPSR_EL2_DAIF_SHIFT U(6)
+#define SPSR_EL2_DAIF_MASK U(0xf)
+
+#define SPSR_EL2_AIF_SHIFT U(6)
+#define SPSR_EL2_AIF_MASK U(0x7)
+
+#define SPSR_EL2_F_SHIFT 6
+#define SPSR_EL2_F_WIDTH 1
+#define SPSR_EL2_F_BIT INPLACE(SPSR_EL2_F, 1)
+#define DAIF_FIQ_BIT (U(1) << 0)
+
+#define SPSR_EL2_I_SHIFT 7
+#define SPSR_EL2_I_WIDTH 1
+#define SPSR_EL2_I_BIT INPLACE(SPSR_EL2_I, 1)
+#define DAIF_IRQ_BIT (U(1) << 1)
+
+#define SPSR_EL2_A_SHIFT 8
+#define SPSR_EL2_A_WIDTH 1
+#define SPSR_EL2_A_BIT INPLACE(SPSR_EL2_A, 1)
+#define DAIF_ABT_BIT (U(1) << 2)
+
+#define SPSR_EL2_D_SHIFT 9
+#define SPSR_EL2_D_WIDTH 1
+#define SPSR_EL2_D_BIT INPLACE(SPSR_EL2_D, 1)
+#define DAIF_DBG_BIT (U(1) << 3)
+
+#define SPSR_EL2_SSBS_SHIFT 12
+#define SPSR_EL2_SSBS_WIDTH 1
+#define SPSR_EL2_SSBS_BIT INPLACE(SPSR_EL2_SSBS, 1)
+
+#define SPSR_EL2_IL_SHIFT 20
+#define SPSR_EL2_IL_WIDTH 1
+#define SPSR_EL2_IL_BIT INPLACE(SPSR_EL2_IL, 1)
+
+#define SPSR_EL2_SS_SHIFT 21
+#define SPSR_EL2_SS_WIDTH 1
+#define SPSR_EL2_SS_BIT INPLACE(SPSR_EL2_SS, 1)
+
+#define SPSR_EL2_PAN_SHIFT 22
+#define SPSR_EL2_PAN_WIDTH 1
+#define SPSR_EL2_PAN_BIT INPLACE(SPSR_EL2_PAN, 1)
+
+#define SPSR_EL2_UAO_SHIFT 23
+#define SPSR_EL2_UAO_WIDTH 1
+#define SPSR_EL2_UAO_BIT INPLACE(SPSR_EL2_UAO, 1)
+
+#define SPSR_EL2_V_SHIFT 28
+#define SPSR_EL2_V_WIDTH 1
+#define SPSR_EL2_V_BIT INPLACE(SPSR_EL2_V, 1)
+
+#define SPSR_EL2_C_SHIFT 29
+#define SPSR_EL2_C_WIDTH 1
+#define SPSR_EL2_C_BIT INPLACE(SPSR_EL2_C, 1)
+
+#define SPSR_EL2_Z_SHIFT 30
+#define SPSR_EL2_Z_WIDTH 1
+#define SPSR_EL2_Z_BIT INPLACE(SPSR_EL2_Z, 1)
+
+#define SPSR_EL2_N_SHIFT 31
+#define SPSR_EL2_N_WIDTH 1
+#define SPSR_EL2_N_BIT INPLACE(SPSR_EL2_N, 1)
+
+/* VTCR definitions */
+#define VTCR_T0SZ_SHIFT 0
+#define VTCR_T0SZ_WIDTH 6
+
+#define VTCR_SL0_SHIFT 6
+#define VTCR_SL0_WIDTH 2
+
+#define VTCR_SL0_4K_L2 INPLACE(VTCR_SL0, 0)
+#define VTCR_SL0_4K_L1 INPLACE(VTCR_SL0, 1)
+#define VTCR_SL0_4K_L0 INPLACE(VTCR_SL0, 2)
+#define VTCR_SL0_4K_L3 INPLACE(VTCR_SL0, 3)
+
+#define VTCR_IRGN0_SHIFT 8
+#define VTCR_IRGN0_WIDTH 2
+#define VTCR_IRGN0_WBRAWA INPLACE(VTCR_IRGN0, 1)
+
+#define VTCR_ORGN0_SHIFT 10
+#define VTCR_ORGN0_WIDTH 2
+#define VTCR_ORGN0_WBRAWA INPLACE(VTCR_ORGN0, 1)
+
+#define VTCR_SH0_SHIFT 12
+#define VTCR_SH0_WIDTH 2
+#define VTCR_SH0_IS INPLACE(VTCR_SH0, 3)
+
+#define VTCR_TG0_SHIFT 14
+#define VTCR_TG0_WIDTH 2
+#define VTCR_TG0_4K INPLACE(VTCR_TG0, 0)
+
+#define VTCR_PS_SHIFT 16
+#define VTCR_PS_WIDTH 3
+#define VTCR_PS_40 INPLACE(VTCR_PS, 2)
+
+#define VTCR_VS (UL(1) << 19)
+#define VTCR_NSA (UL(1) << 30)
+#define VTCR_RES1 (UL(1) << 31)
+
+#define VTCR_FLAGS ( \
+ VTCR_IRGN0_WBRAWA | /* PTW inner cache attr. is WB RAWA*/ \
+ VTCR_ORGN0_WBRAWA | /* PTW outer cache attr. is WB RAWA*/ \
+ VTCR_SH0_IS | /* PTW shareability attr. is Outer Sharable*/\
+ VTCR_TG0_4K | /* 4K granule size in non-secure PT*/ \
+ VTCR_PS_40 | /* size(PA) = 40 */ \
+ /* VS = 0 size(VMID) = 8 */ \
+ /* NSW = 0 non-secure s2 is made of secure pages*/ \
+ VTCR_NSA | /* non-secure IPA maps to non-secure PA */ \
+ VTCR_RES1 \
+ )
+
+
+/* SCTLR definitions */
+#define SCTLR_EL1_EE (UL(1) << 25)
+#define SCTLR_EL1_SPAN (UL(1) << 23)
+#define SCTLR_EL1_EIS (UL(1) << 22)
+#define SCTLR_EL1_nTWE (UL(1) << 18)
+#define SCTLR_EL1_nTWI (UL(1) << 16)
+#define SCTLR_EL1_EOS (UL(1) << 11)
+#define SCTLR_EL1_nAA (UL(1) << 6)
+#define SCTLR_EL1_CP15BEN (UL(1) << 5)
+#define SCTLR_EL1_SA0 (UL(1) << 4)
+#define SCTLR_EL1_SA (UL(1) << 3)
+
+#define SCTLR_EL1_FLAGS (SCTLR_EL1_SPAN | SCTLR_EL1_EIS | SCTLR_EL1_nTWE | \
+ SCTLR_EL1_nTWI | SCTLR_EL1_EOS | SCTLR_EL1_nAA | SCTLR_EL1_CP15BEN | \
+ SCTLR_EL1_SA0 | SCTLR_EL1_SA)
+
+/* PMCR_EL0 Definitions */
+#define PMCR_EL0_LC_SHIFT 6
+#define PMCR_EL0_LC_WIDTH 1
+#define PMCR_EL0_LC_BIT INPLACE(PMCR_EL0_LC, 1)
+
+#define PMCR_EL0_RES1 PMCR_EL0_LC_BIT
+
+
+/* MDSCR_EL1 Definitions */
+#define MDSCR_EL1_TDCC_SHIFT 12
+#define MDSCR_EL1_TDCC_WIDTH 1
+#define MDSCR_EL1_TDCC_BIT INPLACE(MDSCR_EL1_TDCC, 1)
+
+/* SCTLR register definitions */
+#define SCTLR_EL2_RES1 ((UL(1) << 22) /* TODO: ARMv8.5-CSEH, otherwise RES1 */ | \
+ (1U << 11) /* TODO: ARMv8.5-CSEH, otherwise RES1 */)
+
+#define SCTLR_EL2_M (UL(1) << 0)
+#define SCTLR_EL2_C (UL(1) << 2)
+#define SCTLR_EL2_SA (UL(1) << 3)
+#define SCTLR_EL2_SA0 (UL(1) << 4)
+#define SCTLR_EL2_SED (UL(1) << 8)
+/* TODO: ARMv8.5-CSEH, otherwise RES1 */
+/* #define SCTLR_EL2_EOS (UL(1) << 11) */
+#define SCTLR_EL2_I (UL(1) << 12)
+#define SCTLR_EL2_DZE (UL(1) << 14)
+#define SCTLR_EL2_UCT (UL(1) << 15)
+#define SCTLR_EL2_NTWI (UL(1) << 16)
+#define SCTLR_EL2_NTWE (UL(1) << 18)
+#define SCTLR_EL2_WXN (UL(1) << 19)
+#define SCTLR_EL2_TSCXT (UL(1) << 20)
+/* TODO: ARMv8.5-CSEH, otherwise RES1 */
+/* #define SCTLR_EL2_EIS (UL(1) << 22) */
+#define SCTLR_EL2_SPAN (UL(1) << 23)
+#define SCTLR_EL2_UCI (UL(1) << 26)
+#define SCTLR_EL2_NTLSMD (UL(1) << 28)
+#define SCTLR_EL2_LSMAOE (UL(1) << 29)
+/* HCR_EL2.E2H == 0b1 and HCR_EL2.TGE == 0b1 */
+#define SECURE_SCTLR_EL2_RES1 ((UL(1) << 22) /* TODO: ARMv8.5-CSEH, otherwise RES1 */ | \
+ (UL(1) << 11) /* TODO: ARMv8.5-CSEH, otherwise RES1 */)
+
+#define SCTLR_EL2_INIT (/* SCTLR_EL2_M = 0 (MMU disabled) */ \
+ /* SCTLR_EL2_A = 0 (No alignment checks) */ \
+ SCTLR_EL2_C /* Data accesses are cacheable
+ * as per translation tables */ | \
+ SCTLR_EL2_SA /* SP aligned at EL2 */ | \
+ SCTLR_EL2_SA0 /* SP Alignment check enable for EL0 */ \
+ /* SCTLR_EL2_CP15BEN = 0 (EL0 using AArch32:
+ * EL0 execution of the CP15DMB, CP15DSB, and
+ * CP15ISB instructions is UNDEFINED. */ \
+ /* SCTLR_EL2_NAA = 0 (unaligned MA fault at EL2 and EL0) */ \
+ /* SCTLR_EL2_ITD = 0 (A32 Only) */ | \
+ SCTLR_EL2_SED /* A32 Only, RES1 for non-A32 systems */ \
+ /* SCTLR_EL2_EOS TODO: ARMv8.5-CSEH, otherwise RES1 */ | \
+ SCTLR_EL2_I /* I$ is ON for EL2 and EL0 */ | \
+ SCTLR_EL2_DZE /* Do not trap DC ZVA */ | \
+ SCTLR_EL2_UCT /* Allow EL0 access to CTR_EL0 */ | \
+ SCTLR_EL2_NTWI /* Don't trap WFI from EL0 to EL2 */ | \
+ SCTLR_EL2_NTWE /* Don't trap WFE from EL0 to EL2 */ | \
+ SCTLR_EL2_WXN /* W implies XN */ | \
+ SCTLR_EL2_TSCXT /* Trap EL0 accesss to SCXTNUM_EL0 */ \
+ /* SCTLR_EL2_EIS EL2 exception is context
+ * synchronizing
+ * TODO: ARMv8.5-CSEH, otherwise RES1 */ \
+ /* SCTLR_EL2_SPAN = 0 (Set PSTATE.PAN = 1 on
+ * exceptions to EL2)) */ | \
+ SCTLR_EL2_UCI /* Allow cache maintenance
+ * instructions at EL0 */ | \
+ SCTLR_EL2_NTLSMD /* A32/T32 only */ | \
+ SCTLR_EL2_LSMAOE /* A32/T32 only */ | \
+ SECURE_SCTLR_EL2_RES1)
+
+#define SCTLR_EL2_RUNTIME (SCTLR_EL2_INIT| \
+ SCTLR_EL2_M /* MMU enabled */)
+
+/* CPTR_EL2 definitions */
+#define CPTR_EL2_RES1 ((UL(1) << 13) | (UL(1) << 12) | (UL(1) << 9) | 0xff)
+#define CPTR_EL2_FPEN (UL(1) << 20)
+#define CPTR_EL2_TTA (UL(1) << 28)
+#define CPTR_EL2_TAM (UL(1) << 30)
+#define CPTR_EL2_FPEN_SHIFT 20
+#define CPTR_EL2_FPEN_MASK 0x3
+#define CPTR_EL2_FPEN_TRAP_ALL_00 0x0
+#define CPTR_EL2_FPEN_TRAP_TGE_01 0x1
+#define CPTR_EL2_FPEN_TRAP_ALL_10 0x2
+#define CPTR_EL2_FPEN_NO_TRAP_11 0x3
+#define CPTR_EL2_ZEN_SHIFT UL(16)
+#define CPTR_EL2_ZEN_MASK UL(0x3)
+#define CPTR_EL2_ZEN_TRAP_ALL_00 UL(0x0)
+#define CPTR_EL2_ZEN_NO_TRAP_11 UL(0x3)
+ /* Trap all FPU/SVE accesses */
+#define CPTR_EL2_INIT ((CPTR_EL2_ZEN_TRAP_ALL_00 << \
+ CPTR_EL2_ZEN_SHIFT) | \
+ (CPTR_EL2_FPEN_TRAP_ALL_00 << CPTR_EL2_FPEN_SHIFT) | \
+ CPTR_EL2_TTA /* trap trace access */ | \
+ CPTR_EL2_TAM /* trap AMU access */ | \
+ CPTR_EL2_RES1)
+
+/* MDCR_EL2 definitions */
+#define MDCR_EL2_HLP (U(1) << 26)
+#define MDCR_EL2_HCCD (U(1) << 23)
+#define MDCR_EL2_TTRF (U(1) << 19)
+#define MDCR_EL2_HPMD (U(1) << 17)
+#define MDCR_EL2_TPMS (U(1) << 14)
+#define MDCR_EL2_E2PB(x) ((x) << 12)
+#define MDCR_EL2_E2PB_EL1 U(0x3)
+#define MDCR_EL2_TDRA_BIT (U(1) << 11)
+#define MDCR_EL2_TDOSA_BIT (U(1) << 10)
+#define MDCR_EL2_TDA_BIT (U(1) << 9)
+#define MDCR_EL2_TDE_BIT (U(1) << 8)
+#define MDCR_EL2_HPME_BIT (U(1) << 7)
+#define MDCR_EL2_TPM_BIT (U(1) << 6)
+#define MDCR_EL2_TPMCR_BIT (U(1) << 5)
+#define MDCR_EL2_INIT (MDCR_EL2_TPMCR_BIT \
+ | MDCR_EL2_TPM_BIT \
+ | MDCR_EL2_TDA_BIT)
+
+/* MPIDR definitions */
+#define MPIDR_EL1_AFF_MASK 0xFF
+#define MPIDR_EL1_AFF0_SHIFT 0
+#define MPIDR_EL1_AFF1_SHIFT 8
+#define MPIDR_EL1_AFF2_SHIFT 16
+#define MPIDR_EL1_AFF3_SHIFT 32
+#define MPIDR_EL1_MT_MASK (UL(1) << 24)
+#define MPIDR_EL1_AFFINITY_BITS 8
+
+#define MPIDR_EL1_AFF0 INPLACE(MPIDR_EL1_AFF0, MPIDR_EL1_AFF_MASK)
+#define MPIDR_EL1_AFF1 INPLACE(MPIDR_EL1_AFF1, MPIDR_EL1_AFF_MASK)
+#define MPIDR_EL1_AFF2 INPLACE(MPIDR_EL1_AFF2, MPIDR_EL1_AFF_MASK)
+#define MPIDR_EL1_AFF3 INPLACE(MPIDR_EL1_AFF3, MPIDR_EL1_AFF_MASK)
+
+/*
+ * RmiRecMpidr type definitions.
+ *
+ * 'MPIDR_EL2_AFF<n>_VAL_SHIFT' constants specify the right shift
+ * for affinity field <n> that gives the field's actual value.
+ *
+ * Aff0[3:0] - Affinity level 0
+ * For compatibility with GICv3 only Aff0[3:0] field is used,
+ * and Aff0[7:4] of a REC MPIDR value is RES0.
+ */
+#define MPIDR_EL2_AFF0_SHIFT 0
+#define MPIDR_EL2_AFF0_WIDTH 4
+#define MPIDR_EL2_AFF0_VAL_SHIFT 0
+
+/* Aff1[15:8] - Affinity level 1 */
+#define MPIDR_EL2_AFF1_SHIFT 8
+#define MPIDR_EL2_AFF1_WIDTH 8
+#define MPIDR_EL2_AFF1_VAL_SHIFT 4
+
+/* Aff2[23:16] - Affinity level 2 */
+#define MPIDR_EL2_AFF2_SHIFT 16
+#define MPIDR_EL2_AFF2_WIDTH 8
+#define MPIDR_EL2_AFF2_VAL_SHIFT 4
+
+/* Aff3[39:32] - Affinity level 3 */
+#define MPIDR_EL2_AFF3_SHIFT 32
+#define MPIDR_EL2_AFF3_WIDTH 8
+#define MPIDR_EL2_AFF3_VAL_SHIFT 12
+
+/*
+ * Extract the value of Aff<n> register field shifted right
+ * so it can be evaluated directly.
+ */
+#define MPIDR_EL2_AFF(n, reg) \
+ (((reg) & MASK(MPIDR_EL2_AFF##n)) >> MPIDR_EL2_AFF##n##_VAL_SHIFT)
+
+/* VMPIDR_EL2 bit [31] = RES1 */
+#define VMPIDR_EL2_RES1 (UL(1) << 31)
+
+/* ICC_SRE_EL2 defintions */
+#define ICC_SRE_EL2_ENABLE (UL(1) << 3) /* Enable lower EL access to ICC_SRE_EL1 */
+#define ICC_SRE_EL2_DIB (UL(1) << 2) /* Disable IRQ bypass */
+#define ICC_SRE_EL2_DFB (UL(1) << 1) /* Disable FIQ bypass */
+#define ICC_SRE_EL2_SRE (UL(1) << 0) /* Enable sysreg access */
+
+#define ICC_SRE_EL2_INIT (ICC_SRE_EL2_ENABLE | ICC_SRE_EL2_DIB | \
+ ICC_SRE_EL2_DFB | ICC_SRE_EL2_SRE)
+
+/* MPAM definitions */
+#define MPAM2_EL2_INIT 0x0
+#define MPAMHCR_EL2_INIT 0x0
+
+#define PMSCR_EL2_INIT 0x0
+
+#define SYSREG_ESR(op0, op1, crn, crm, op2) \
+ (((op0) << ESR_EL2_SYSREG_TRAP_OP0_SHIFT) | \
+ ((op1) << ESR_EL2_SYSREG_TRAP_OP1_SHIFT) | \
+ ((crn) << ESR_EL2_SYSREG_TRAP_CRN_SHIFT) | \
+ ((crm) << ESR_EL2_SYSREG_TRAP_CRM_SHIFT) | \
+ ((op2) << ESR_EL2_SYSREG_TRAP_OP2_SHIFT))
+
+#define ESR_EL2_SYSREG_MASK SYSREG_ESR(3, 7, 15, 15, 7)
+
+#define ESR_EL2_SYSREG_ID_MASK SYSREG_ESR(3, 7, 15, 0, 0)
+#define ESR_EL2_SYSREG_ID SYSREG_ESR(3, 0, 0, 0, 0)
+
+#define ESR_EL2_SYSREG_ID_AA64PFR0_EL1 SYSREG_ESR(3, 0, 0, 4, 0)
+#define ESR_EL2_SYSREG_ID_AA64PFR1_EL1 SYSREG_ESR(3, 0, 0, 4, 1)
+#define ESR_EL2_SYSREG_ID_AA64ZFR0_EL1 SYSREG_ESR(3, 0, 0, 4, 4)
+
+#define ESR_EL2_SYSREG_ID_AA64DFR0_EL1 SYSREG_ESR(3, 0, 0, 5, 0)
+#define ESR_EL2_SYSREG_ID_AA64DFR1_EL1 SYSREG_ESR(3, 0, 0, 5, 1)
+
+#define ESR_EL2_SYSREG_ID_AA64AFR0_EL1 SYSREG_ESR(3, 0, 0, 5, 4)
+#define ESR_EL2_SYSREG_ID_AA64AFR1_EL1 SYSREG_ESR(3, 0, 0, 5, 5)
+
+#define ESR_EL2_SYSREG_ID_AA64ISAR0_EL1 SYSREG_ESR(3, 0, 0, 6, 0)
+#define ESR_EL2_SYSREG_ID_AA64ISAR1_EL1 SYSREG_ESR(3, 0, 0, 6, 1)
+
+#define ESR_EL2_SYSREG_ID_AA64MMFR0_EL1 SYSREG_ESR(3, 0, 0, 7, 0)
+#define ESR_EL2_SYSREG_ID_AA64MMFR1_EL1 SYSREG_ESR(3, 0, 0, 7, 1)
+#define ESR_EL2_SYSREG_ID_AA64MMFR2_EL1 SYSREG_ESR(3, 0, 0, 7, 2)
+
+#define ESR_EL2_SYSREG_ID_AA64ISAR1_GPI_SHIFT 28
+#define ESR_EL2_SYSREG_ID_AA64ISAR1_GPA_SHIFT 24
+#define ESR_EL2_SYSREG_ID_AA64ISAR1_API_SHIFT 8
+#define ESR_EL2_SYSREG_ID_AA64ISAR1_APA_SHIFT 4
+
+#define ESR_EL2_SYSREG_TIMERS_MASK SYSREG_ESR(3, 3, 15, 12, 0)
+#define ESR_EL2_SYSREG_TIMERS SYSREG_ESR(3, 3, 14, 0, 0)
+
+#define ESR_EL2_SYSREG_TIMER_CNTP_TVAL_EL0 SYSREG_ESR(3, 3, 14, 2, 0)
+#define ESR_EL2_SYSREG_TIMER_CNTP_CTL_EL0 SYSREG_ESR(3, 3, 14, 2, 1)
+#define ESR_EL2_SYSREG_TIMER_CNTP_CVAL_EL0 SYSREG_ESR(3, 3, 14, 2, 2)
+#define ESR_EL2_SYSREG_TIMER_CNTV_TVAL_EL0 SYSREG_ESR(3, 3, 14, 3, 0)
+#define ESR_EL2_SYSREG_TIMER_CNTV_CTL_EL0 SYSREG_ESR(3, 3, 14, 3, 1)
+#define ESR_EL2_SYSREG_TIMER_CNTV_CVAL_EL0 SYSREG_ESR(3, 3, 14, 3, 2)
+
+#define ESR_EL2_SYSREG_ICC_PMR_EL1 SYSREG_ESR(3, 0, 4, 6, 0)
+
+/*
+ * GIC system registers encoding mask for registers from
+ * ICC_IAR0_EL1(3, 0, 12, 8, 0) to ICC_IGRPEN1_EL1(3, 0, 12, 12, 7).
+ */
+#define ESR_EL2_SYSREG_ICC_EL1_MASK SYSREG_ESR(3, 3, 15, 8, 0)
+#define ESR_EL2_SYSREG_ICC_EL1 SYSREG_ESR(3, 0, 12, 8, 0)
+
+#define ESR_EL2_SYSREG_ICC_DIR SYSREG_ESR(3, 0, 12, 11, 1)
+#define ESR_EL2_SYSREG_ICC_SGI1R_EL1 SYSREG_ESR(3, 0, 12, 11, 5)
+#define ESR_EL2_SYSREG_ICC_SGI0R_EL1 SYSREG_ESR(3, 0, 12, 11, 7)
+
+#define ESR_EL2_SYSREG_DIRECTION (UL(1) << 0)
+#define ESR_EL2_SYSREG_IS_WRITE(esr) (!((esr) & ESR_EL2_SYSREG_DIRECTION))
+
+#define ESR_IL(esr) ((esr) & ESR_EL2_IL_MASK)
+#define ESR_ISS(esr) ((esr) & ESR_EL2_ISS_MASK)
+
+#define ESR_EL2_SYSREG_ISS_RT(esr) \
+ ((ESR_ISS(esr) & ESR_EL2_SYSREG_TRAP_RT_MASK) >> ESR_EL2_SYSREG_TRAP_RT_SHIFT)
+
+#define ICC_HPPIR1_EL1_INTID_SHIFT 0
+#define ICC_HPPIR1_EL1_INTID_WIDTH 24
+#define ICC_HPPIR1_EL1_INTID_MASK MASK(ICC_HPPIR1_EL1_INTID)
+
+#define CNTHCTL_EL2_EL0PCTEN (UL(1) << UL(0))
+#define CNTHCTL_EL2_EL0VCTEN (UL(1) << UL(1))
+#define CNTHCTL_EL2_EL1PCTEN (UL(1) << 10)
+#define CNTHCTL_EL2_EL1PTEN (UL(1) << 11)
+#define CNTHCTL_EL2_EL1TVT (UL(1) << 13)
+#define CNTHCTL_EL2_EL1TVCT (UL(1) << 14)
+#define CNTHCTL_EL2_CNTVMASK (UL(1) << 18)
+#define CNTHCTL_EL2_CNTPMASK (UL(1) << 19)
+
+#define CNTHCTL_EL2_INIT (CNTHCTL_EL2_EL0VCTEN | CNTHCTL_EL2_EL0PCTEN)
+
+#define CNTHCTL_EL2_NO_TRAPS (CNTHCTL_EL2_EL1PCTEN | \
+ CNTHCTL_EL2_EL1PTEN)
+
+#define CNTx_CTL_ENABLE (UL(1) << 0)
+#define CNTx_CTL_IMASK (UL(1) << 1)
+#define CNTx_CTL_ISTATUS (UL(1) << 2)
+
+/*******************************************************************************
+ * Definitions of register offsets, fields and macros for CPU system
+ * instructions.
+ ******************************************************************************/
+
+#define TLBI_ADDR_SHIFT U(12)
+#define TLBI_ADDR_MASK U(0x0FFFFFFFFFFF)
+#define TLBI_ADDR(x) (((x) >> TLBI_ADDR_SHIFT) & TLBI_ADDR_MASK)
+
+/* ID_AA64MMFR2_EL1 definitions */
+#define ID_AA64MMFR2_EL1_ST_SHIFT U(28)
+#define ID_AA64MMFR2_EL1_ST_MASK ULL(0xf)
+
+#define ID_AA64MMFR2_EL1_CNP_SHIFT U(0)
+#define ID_AA64MMFR2_EL1_CNP_MASK ULL(0xf)
+
+/* Custom defined values to indicate the vector offset to exception handlers */
+#define ARM_EXCEPTION_SYNC_LEL 0
+#define ARM_EXCEPTION_IRQ_LEL 1
+#define ARM_EXCEPTION_FIQ_LEL 2
+#define ARM_EXCEPTION_SERROR_LEL 3
+
+#define VBAR_CEL_SP_EL0_OFFSET 0x0
+#define VBAR_CEL_SP_ELx_OFFSET 0x200
+#define VBAR_LEL_AA64_OFFSET 0x400
+#define VBAR_LEL_AA32_OFFSET 0x600
+
+#endif /* ARCH_H */
diff --git a/lib/arch/include/arch_features.h b/lib/arch/include/arch_features.h
new file mode 100644
index 0000000..96aa2e0
--- /dev/null
+++ b/lib/arch/include/arch_features.h
@@ -0,0 +1,69 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
+ */
+
+#ifndef ARCH_FEATURES_H
+#define ARCH_FEATURES_H
+
+#include <arch_helpers.h>
+#include <stdbool.h>
+
+static inline bool is_armv8_4_ttst_present(void)
+{
+ return ((read_id_aa64mmfr2_el1() >> ID_AA64MMFR2_EL1_ST_SHIFT) &
+ ID_AA64MMFR2_EL1_ST_MASK) == 1U;
+}
+
+/*
+ * Check if SVE is enabled
+ * ID_AA64PFR0_EL1.SVE, bits [35:32]:
+ * 0b0000 SVE architectural state and programmers' model are not implemented.
+ * 0b0001 SVE architectural state and programmers' model are implemented.
+ */
+static inline bool is_feat_sve_present(void)
+{
+ return ((read_id_aa64pfr0_el1() >> ID_AA64PFR0_EL1_SVE_SHIFT) &
+ ID_AA64PFR0_EL1_SVE_MASK) != 0UL;
+}
+
+/*
+ * Check if RNDR is available
+ */
+static inline bool is_feat_rng_present(void)
+{
+ return ((read_ID_AA64ISAR0_EL1() >> ID_AA64ISAR0_RNDR_SHIFT) &
+ ID_AA64ISAR0_RNDR_MASK) != 0UL;
+}
+
+/*
+ * Check if FEAT_VMID16 is implemented
+ * ID_AA64MMFR1_EL1.VMIDBits, bits [7:4]:
+ * 0b0000 8 bits.
+ * 0b0010 16 bits.
+ * All other values are reserved.
+ */
+static inline bool is_feat_vmid16_present(void)
+{
+ return (((read_id_aa64mmfr1_el1() >> ID_AA64MMFR1_EL1_VMIDBits_SHIFT) &
+ ID_AA64MMFR1_EL1_VMIDBits_MASK) ==
+ ID_AA64MMFR1_EL1_VMIDBits_16);
+}
+
+/*
+ * Check if FEAT_LPA2 is implemented.
+ * 4KB granule at stage 2 supports 52-bit input and output addresses:
+ * ID_AA64MMFR0_EL1.TGran4_2 bits [43:40]: 0b0011
+ */
+static inline bool is_feat_lpa2_4k_present(void)
+{
+ u_register_t aa64mmfr0 = read_id_aa64mmfr0_el1();
+
+ return ((((aa64mmfr0 >> ID_AA64MMFR0_EL1_TGRAN4_2_SHIFT) &
+ ID_AA64MMFR0_EL1_TGRAN4_2_MASK) ==
+ ID_AA64MMFR0_EL1_TGRAN4_2_LPA2));
+}
+
+unsigned int arch_feat_get_pa_width(void);
+
+#endif /* ARCH_FEATURES_H */
diff --git a/lib/arch/include/arch_helpers.h b/lib/arch/include/arch_helpers.h
new file mode 100644
index 0000000..34c95b3
--- /dev/null
+++ b/lib/arch/include/arch_helpers.h
@@ -0,0 +1,385 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
+ */
+
+/* TODO: This file will need clean up */
+
+#ifndef ARCH_HELPERS_H
+#define ARCH_HELPERS_H
+
+#include <arch.h>
+#include <instr_helpers.h>
+#include <stdbool.h>
+#include <stddef.h>
+
+/* Define read function for system register */
+#define DEFINE_SYSREG_READ_FUNC(_name) \
+ DEFINE_SYSREG_READ_FUNC_(_name, _name)
+
+/* Define read & write function for system register */
+#define DEFINE_SYSREG_RW_FUNCS(_name) \
+ DEFINE_SYSREG_READ_FUNC_(_name, _name) \
+ DEFINE_SYSREG_WRITE_FUNC_(_name, _name)
+
+/* Define read & write function for renamed system register */
+#define DEFINE_RENAME_SYSREG_RW_FUNCS(_name, _reg_name) \
+ DEFINE_SYSREG_READ_FUNC_(_name, _reg_name) \
+ DEFINE_SYSREG_WRITE_FUNC_(_name, _reg_name)
+
+/* Define read function for renamed system register */
+#define DEFINE_RENAME_SYSREG_READ_FUNC(_name, _reg_name) \
+ DEFINE_SYSREG_READ_FUNC_(_name, _reg_name)
+
+/* Define write function for renamed system register */
+#define DEFINE_RENAME_SYSREG_WRITE_FUNC(_name, _reg_name) \
+ DEFINE_SYSREG_WRITE_FUNC_(_name, _reg_name)
+
+/*******************************************************************************
+ * TLB maintenance accessor prototypes
+ ******************************************************************************/
+DEFINE_SYSOP_TYPE_FUNC(tlbi, alle1)
+DEFINE_SYSOP_TYPE_FUNC(tlbi, alle1is)
+DEFINE_SYSOP_TYPE_FUNC(tlbi, alle2)
+DEFINE_SYSOP_TYPE_FUNC(tlbi, alle2is)
+DEFINE_SYSOP_TYPE_FUNC(tlbi, vmalle1)
+DEFINE_SYSOP_TYPE_FUNC(tlbi, vmalle1is)
+DEFINE_SYSOP_TYPE_FUNC(tlbi, vmalls12e1)
+
+DEFINE_SYSOP_TYPE_PARAM_FUNC(tlbi, vaae1is)
+DEFINE_SYSOP_TYPE_PARAM_FUNC(tlbi, vaale1is)
+DEFINE_SYSOP_TYPE_PARAM_FUNC(tlbi, vae2is)
+DEFINE_SYSOP_TYPE_PARAM_FUNC(tlbi, vale2is)
+DEFINE_SYSOP_TYPE_PARAM_FUNC(tlbi, ipas2e1is)
+
+/*******************************************************************************
+ * Cache maintenance accessor prototypes
+ ******************************************************************************/
+DEFINE_SYSOP_TYPE_PARAM_FUNC(dc, isw)
+DEFINE_SYSOP_TYPE_PARAM_FUNC(dc, cisw)
+DEFINE_SYSOP_TYPE_PARAM_FUNC(dc, csw)
+DEFINE_SYSOP_TYPE_PARAM_FUNC(dc, cvac)
+DEFINE_SYSOP_TYPE_PARAM_FUNC(dc, ivac)
+DEFINE_SYSOP_TYPE_PARAM_FUNC(dc, civac)
+DEFINE_SYSOP_TYPE_PARAM_FUNC(dc, cvau)
+DEFINE_SYSOP_TYPE_PARAM_FUNC(dc, zva)
+
+/*******************************************************************************
+ * Address translation accessor prototypes
+ ******************************************************************************/
+DEFINE_SYSOP_TYPE_PARAM_FUNC(at, s12e1r)
+DEFINE_SYSOP_TYPE_PARAM_FUNC(at, s12e1w)
+DEFINE_SYSOP_TYPE_PARAM_FUNC(at, s12e0r)
+DEFINE_SYSOP_TYPE_PARAM_FUNC(at, s12e0w)
+DEFINE_SYSOP_TYPE_PARAM_FUNC(at, s1e1r)
+DEFINE_SYSOP_TYPE_PARAM_FUNC(at, s1e2r)
+
+/*******************************************************************************
+ * Strip Pointer Authentication Code
+ ******************************************************************************/
+DEFINE_SYSOP_PARAM_FUNC(xpaci)
+
+/*******************************************************************************
+ * Cache management
+ ******************************************************************************/
+void flush_dcache_range(uintptr_t addr, size_t size);
+void clean_dcache_range(uintptr_t addr, size_t size);
+void inv_dcache_range(uintptr_t addr, size_t size);
+
+#define is_dcache_enabled() ((read_sctlr_el2() & SCTLR_EL2_C) != 0U)
+
+/*******************************************************************************
+ * MMU management
+ ******************************************************************************/
+#define is_mmu_enabled() ((read_sctlr_el2() & SCTLR_EL2_M) != 0U)
+
+/*******************************************************************************
+ * FPU management
+ ******************************************************************************/
+#define is_fpen_enabled() (((read_cptr_el2() >> CPTR_EL2_FPEN_SHIFT) & \
+ CPTR_EL2_FPEN_MASK) == CPTR_EL2_FPEN_NO_TRAP_11)
+
+/*******************************************************************************
+ * Misc. accessor prototypes
+ ******************************************************************************/
+
+#define write_daifclr(val) SYSREG_WRITE_CONST(daifclr, val)
+#define write_daifset(val) SYSREG_WRITE_CONST(daifset, val)
+
+DEFINE_SYSOP_FUNC(wfi)
+DEFINE_SYSOP_FUNC(wfe)
+DEFINE_SYSOP_FUNC(sev)
+DEFINE_SYSOP_FUNC(isb)
+
+static inline void enable_irq(void)
+{
+ /*
+ * The compiler memory barrier will prevent the compiler from
+ * scheduling non-volatile memory access after the write to the
+ * register.
+ *
+ * This could happen if some initialization code issues non-volatile
+ * accesses to an area used by an interrupt handler, in the assumption
+ * that it is safe as the interrupts are disabled at the time it does
+ * that (according to program order). However, non-volatile accesses
+ * are not necessarily in program order relatively with volatile inline
+ * assembly statements (and volatile accesses).
+ */
+ COMPILER_BARRIER();
+ write_daifclr(DAIF_IRQ_BIT);
+ isb();
+}
+
+static inline void enable_fiq(void)
+{
+ COMPILER_BARRIER();
+ write_daifclr(DAIF_FIQ_BIT);
+ isb();
+}
+
+static inline void enable_serror(void)
+{
+ COMPILER_BARRIER();
+ write_daifclr(DAIF_ABT_BIT);
+ isb();
+}
+
+static inline void enable_debug_exceptions(void)
+{
+ COMPILER_BARRIER();
+ write_daifclr(DAIF_DBG_BIT);
+ isb();
+}
+
+static inline void disable_irq(void)
+{
+ COMPILER_BARRIER();
+ write_daifset(DAIF_IRQ_BIT);
+ isb();
+}
+
+static inline void disable_fiq(void)
+{
+ COMPILER_BARRIER();
+ write_daifset(DAIF_FIQ_BIT);
+ isb();
+}
+
+static inline void disable_serror(void)
+{
+ COMPILER_BARRIER();
+ write_daifset(DAIF_ABT_BIT);
+ isb();
+}
+
+static inline void disable_debug_exceptions(void)
+{
+ COMPILER_BARRIER();
+ write_daifset(DAIF_DBG_BIT);
+ isb();
+}
+
+/*******************************************************************************
+ * System register accessor prototypes
+ ******************************************************************************/
+DEFINE_SYSREG_RW_FUNCS(sp_el0)
+DEFINE_SYSREG_RW_FUNCS(sp_el1)
+DEFINE_SYSREG_RW_FUNCS(elr_el12)
+DEFINE_SYSREG_RW_FUNCS(spsr_el12)
+DEFINE_SYSREG_RW_FUNCS(pmuserenr_el0)
+DEFINE_SYSREG_RW_FUNCS(tpidrro_el0)
+DEFINE_SYSREG_RW_FUNCS(tpidr_el0)
+DEFINE_SYSREG_RW_FUNCS(tpidr_el2)
+DEFINE_SYSREG_RW_FUNCS(csselr_el1)
+DEFINE_SYSREG_RW_FUNCS(sctlr_el12)
+DEFINE_SYSREG_RW_FUNCS(cpacr_el12)
+DEFINE_SYSREG_RW_FUNCS(zcr_el1)
+DEFINE_SYSREG_RW_FUNCS(ttbr0_el12)
+DEFINE_SYSREG_RW_FUNCS(ttbr1_el12)
+DEFINE_SYSREG_RW_FUNCS(tcr_el12)
+DEFINE_SYSREG_RW_FUNCS(esr_el12)
+DEFINE_SYSREG_RW_FUNCS(afsr0_el12)
+DEFINE_SYSREG_RW_FUNCS(afsr1_el12)
+DEFINE_SYSREG_RW_FUNCS(far_el12)
+DEFINE_SYSREG_RW_FUNCS(mair_el12)
+DEFINE_SYSREG_RW_FUNCS(vbar_el12)
+DEFINE_SYSREG_RW_FUNCS(contextidr_el12)
+DEFINE_SYSREG_RW_FUNCS(tpidr_el1)
+DEFINE_SYSREG_RW_FUNCS(amair_el12)
+DEFINE_SYSREG_RW_FUNCS(cntkctl_el12)
+DEFINE_SYSREG_RW_FUNCS(mdscr_el1)
+DEFINE_SYSREG_RW_FUNCS(mdccint_el1)
+DEFINE_SYSREG_RW_FUNCS(disr_el1)
+DEFINE_SYSREG_RW_FUNCS(cnrv_ctl_el02)
+DEFINE_SYSREG_RW_FUNCS(vtcr_el2)
+DEFINE_SYSREG_RW_FUNCS(vsesr_el2)
+DEFINE_SYSREG_RW_FUNCS(par_el1)
+DEFINE_SYSREG_READ_FUNC(id_pfr1_el1)
+DEFINE_RENAME_SYSREG_READ_FUNC(ID_AA64PFR0_EL1, id_aa64pfr0_el1)
+DEFINE_RENAME_SYSREG_READ_FUNC(ID_AA64PFR1_EL1, id_aa64pfr1_el1)
+DEFINE_RENAME_SYSREG_READ_FUNC(ID_AA64DFR0_EL1, id_aa64dfr0_el1)
+DEFINE_RENAME_SYSREG_READ_FUNC(ID_AA64DFR1_EL1, id_aa64dfr1_el1)
+DEFINE_RENAME_SYSREG_READ_FUNC(ID_AA64AFR0_EL1, id_aa64afr0_el1)
+DEFINE_RENAME_SYSREG_READ_FUNC(ID_AA64AFR1_EL1, id_aa64afr1_el1)
+DEFINE_RENAME_SYSREG_READ_FUNC(ID_AA64ISAR0_EL1, id_aa64isar0_el1)
+DEFINE_RENAME_SYSREG_READ_FUNC(ID_AA64ISAR1_EL1, id_aa64isar1_el1)
+DEFINE_RENAME_SYSREG_READ_FUNC(ID_AA64MMFR0_EL1, id_aa64mmfr0_el1)
+DEFINE_RENAME_SYSREG_READ_FUNC(ID_AA64MMFR1_EL1, id_aa64mmfr1_el1)
+DEFINE_RENAME_SYSREG_READ_FUNC(ID_AA64MMFR2_EL1, id_aa64mmfr1_el1)
+DEFINE_RENAME_SYSREG_RW_FUNCS(icc_hppir1_el1, ICC_HPPIR1_EL1)
+DEFINE_RENAME_SYSREG_RW_FUNCS(mpam0_el1, MPAM0_EL1)
+DEFINE_SYSREG_READ_FUNC(id_aa64pfr0_el1)
+DEFINE_SYSREG_READ_FUNC(id_afr0_el1)
+DEFINE_SYSREG_READ_FUNC(CurrentEl)
+DEFINE_SYSREG_READ_FUNC(ctr_el0)
+DEFINE_SYSREG_RW_FUNCS(daif)
+DEFINE_SYSREG_RW_FUNCS(spsr_el1)
+DEFINE_SYSREG_RW_FUNCS(spsr_el2)
+DEFINE_SYSREG_RW_FUNCS(elr_el1)
+DEFINE_SYSREG_RW_FUNCS(elr_el2)
+
+DEFINE_SYSREG_READ_FUNC(midr_el1)
+DEFINE_SYSREG_READ_FUNC(mpidr_el1)
+DEFINE_SYSREG_READ_FUNC(id_aa64mmfr0_el1)
+DEFINE_SYSREG_READ_FUNC(id_aa64mmfr1_el1)
+
+DEFINE_SYSREG_RW_FUNCS(hcr_el2)
+
+DEFINE_SYSREG_RW_FUNCS(vbar_el1)
+DEFINE_SYSREG_RW_FUNCS(vbar_el2)
+
+DEFINE_SYSREG_RW_FUNCS(sctlr_el1)
+DEFINE_SYSREG_RW_FUNCS(sctlr_el2)
+
+DEFINE_SYSREG_RW_FUNCS(actlr_el1)
+DEFINE_SYSREG_RW_FUNCS(actlr_el2)
+
+DEFINE_SYSREG_RW_FUNCS(esr_el1)
+DEFINE_SYSREG_RW_FUNCS(esr_el2)
+
+DEFINE_SYSREG_RW_FUNCS(afsr0_el1)
+DEFINE_SYSREG_RW_FUNCS(afsr0_el2)
+
+DEFINE_SYSREG_RW_FUNCS(afsr1_el1)
+DEFINE_SYSREG_RW_FUNCS(afsr1_el2)
+
+DEFINE_SYSREG_RW_FUNCS(far_el1)
+DEFINE_SYSREG_RW_FUNCS(far_el2)
+DEFINE_SYSREG_RW_FUNCS(hpfar_el2)
+
+DEFINE_SYSREG_RW_FUNCS(mair_el1)
+DEFINE_SYSREG_RW_FUNCS(mair_el2)
+
+DEFINE_SYSREG_RW_FUNCS(amair_el1)
+DEFINE_SYSREG_RW_FUNCS(amair_el2)
+
+DEFINE_SYSREG_READ_FUNC(rvbar_el1)
+DEFINE_SYSREG_READ_FUNC(rvbar_el2)
+
+DEFINE_SYSREG_RW_FUNCS(rmr_el1)
+DEFINE_SYSREG_RW_FUNCS(rmr_el2)
+
+DEFINE_SYSREG_RW_FUNCS(tcr_el1)
+DEFINE_SYSREG_RW_FUNCS(tcr_el2)
+
+DEFINE_SYSREG_RW_FUNCS(ttbr0_el1)
+DEFINE_SYSREG_RW_FUNCS(ttbr0_el2)
+DEFINE_SYSREG_RW_FUNCS(ttbr1_el2)
+
+DEFINE_SYSREG_RW_FUNCS(ttbr1_el1)
+
+DEFINE_SYSREG_RW_FUNCS(vttbr_el2)
+
+DEFINE_SYSREG_RW_FUNCS(cptr_el2)
+
+DEFINE_SYSREG_RW_FUNCS(cpacr_el1)
+
+DEFINE_SYSREG_RW_FUNCS(vpidr_el2)
+DEFINE_SYSREG_RW_FUNCS(vmpidr_el2)
+
+DEFINE_SYSREG_READ_FUNC(isr_el1)
+
+DEFINE_SYSREG_RW_FUNCS(mdcr_el2)
+DEFINE_SYSREG_RW_FUNCS(hstr_el2)
+DEFINE_SYSREG_RW_FUNCS(pmcr_el0)
+DEFINE_SYSREG_RW_FUNCS(mpam2_el2)
+DEFINE_SYSREG_RW_FUNCS(mpamhcr_el2)
+DEFINE_SYSREG_RW_FUNCS(pmscr_el2)
+
+/*******************************************************************************
+ * Timer register accessor prototypes
+ ******************************************************************************/
+DEFINE_SYSREG_RW_FUNCS(cntfrq_el0)
+DEFINE_SYSREG_RW_FUNCS(cnthp_ctl_el2)
+DEFINE_SYSREG_RW_FUNCS(cnthp_tval_el2)
+DEFINE_SYSREG_RW_FUNCS(cnthp_cval_el2)
+DEFINE_SYSREG_RW_FUNCS(cntps_ctl_el1)
+DEFINE_SYSREG_RW_FUNCS(cntps_tval_el1)
+DEFINE_SYSREG_RW_FUNCS(cntps_cval_el1)
+DEFINE_SYSREG_RW_FUNCS(cntp_ctl_el0)
+DEFINE_SYSREG_RW_FUNCS(cntp_tval_el0)
+DEFINE_SYSREG_RW_FUNCS(cntp_cval_el0)
+DEFINE_SYSREG_READ_FUNC(cntpct_el0)
+DEFINE_SYSREG_RW_FUNCS(cnthctl_el2)
+DEFINE_SYSREG_RW_FUNCS(cntp_ctl_el02)
+DEFINE_SYSREG_RW_FUNCS(cntp_cval_el02)
+DEFINE_SYSREG_RW_FUNCS(cntv_ctl_el02)
+DEFINE_SYSREG_RW_FUNCS(cntv_cval_el02)
+DEFINE_SYSREG_RW_FUNCS(cntvoff_el2)
+DEFINE_RENAME_SYSREG_RW_FUNCS(cntpoff_el2, CNTPOFF_EL2)
+
+/*******************************************************************************
+ * Interrupt Controller register accessor prototypes
+ ******************************************************************************/
+DEFINE_RENAME_SYSREG_RW_FUNCS(icc_sre_el2, ICC_SRE_EL2)
+DEFINE_RENAME_SYSREG_RW_FUNCS(icc_ctrl_el1, ICC_CTLR_EL1)
+DEFINE_RENAME_SYSREG_READ_FUNC(icc_hppir1_el2, ICC_HPPIR1_EL1)
+
+/*******************************************************************************
+ * Virtual GIC register accessor prototypes
+ ******************************************************************************/
+DEFINE_RENAME_SYSREG_RW_FUNCS(ich_ap0r0_el2, ICH_AP0R0_EL2)
+DEFINE_RENAME_SYSREG_RW_FUNCS(ich_ap0r1_el2, ICH_AP0R1_EL2)
+DEFINE_RENAME_SYSREG_RW_FUNCS(ich_ap0r2_el2, ICH_AP0R2_EL2)
+DEFINE_RENAME_SYSREG_RW_FUNCS(ich_ap0r3_el2, ICH_AP0R3_EL2)
+DEFINE_RENAME_SYSREG_RW_FUNCS(ich_ap1r0_el2, ICH_AP1R0_EL2)
+DEFINE_RENAME_SYSREG_RW_FUNCS(ich_ap1r1_el2, ICH_AP1R1_EL2)
+DEFINE_RENAME_SYSREG_RW_FUNCS(ich_ap1r2_el2, ICH_AP1R2_EL2)
+DEFINE_RENAME_SYSREG_RW_FUNCS(ich_ap1r3_el2, ICH_AP1R3_EL2)
+
+DEFINE_RENAME_SYSREG_RW_FUNCS(ich_lr0_el2, ICH_LR0_EL2)
+DEFINE_RENAME_SYSREG_RW_FUNCS(ich_lr1_el2, ICH_LR1_EL2)
+DEFINE_RENAME_SYSREG_RW_FUNCS(ich_lr2_el2, ICH_LR2_EL2)
+DEFINE_RENAME_SYSREG_RW_FUNCS(ich_lr3_el2, ICH_LR3_EL2)
+DEFINE_RENAME_SYSREG_RW_FUNCS(ich_lr4_el2, ICH_LR4_EL2)
+DEFINE_RENAME_SYSREG_RW_FUNCS(ich_lr5_el2, ICH_LR5_EL2)
+DEFINE_RENAME_SYSREG_RW_FUNCS(ich_lr6_el2, ICH_LR6_EL2)
+DEFINE_RENAME_SYSREG_RW_FUNCS(ich_lr7_el2, ICH_LR7_EL2)
+DEFINE_RENAME_SYSREG_RW_FUNCS(ich_lr8_el2, ICH_LR8_EL2)
+DEFINE_RENAME_SYSREG_RW_FUNCS(ich_lr9_el2, ICH_LR9_EL2)
+DEFINE_RENAME_SYSREG_RW_FUNCS(ich_lr10_el2, ICH_LR10_EL2)
+DEFINE_RENAME_SYSREG_RW_FUNCS(ich_lr11_el2, ICH_LR11_EL2)
+DEFINE_RENAME_SYSREG_RW_FUNCS(ich_lr12_el2, ICH_LR12_EL2)
+DEFINE_RENAME_SYSREG_RW_FUNCS(ich_lr13_el2, ICH_LR13_EL2)
+DEFINE_RENAME_SYSREG_RW_FUNCS(ich_lr14_el2, ICH_LR14_EL2)
+DEFINE_RENAME_SYSREG_RW_FUNCS(ich_lr15_el2, ICH_LR15_EL2)
+
+DEFINE_RENAME_SYSREG_RW_FUNCS(ich_hcr_el2, ICH_HCR_EL2)
+DEFINE_RENAME_SYSREG_RW_FUNCS(ich_vmcr_el2, ICH_VMCR_EL2)
+DEFINE_RENAME_SYSREG_READ_FUNC(ich_vtr_el2, ICH_VTR_EL2)
+DEFINE_RENAME_SYSREG_READ_FUNC(ich_misr_el2, ICH_MISR_EL2)
+
+/* Armv8.2 Registers */
+DEFINE_RENAME_SYSREG_READ_FUNC(id_aa64mmfr2_el1, ID_AA64MMFR2_EL1)
+
+/* Armv8.3 Pointer Authentication Registers */
+DEFINE_RENAME_SYSREG_RW_FUNCS(apiakeyhi_el1, APIAKeyHi_EL1)
+DEFINE_RENAME_SYSREG_RW_FUNCS(apiakeylo_el1, APIAKeyLo_EL1)
+
+/* Armv8.5 MTE Registers */
+DEFINE_RENAME_SYSREG_RW_FUNCS(tfsre0_el1, TFSRE0_EL1)
+DEFINE_RENAME_SYSREG_RW_FUNCS(tfsr_el1, TFSR_EL1)
+DEFINE_RENAME_SYSREG_RW_FUNCS(rgsr_el1, RGSR_EL1)
+DEFINE_RENAME_SYSREG_RW_FUNCS(gcr_el1, GCR_EL1)
+
+#endif /* ARCH_HELPERS_H */
diff --git a/lib/arch/include/esr.h b/lib/arch/include/esr.h
new file mode 100644
index 0000000..a88b71c
--- /dev/null
+++ b/lib/arch/include/esr.h
@@ -0,0 +1,78 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
+ */
+
+#ifndef ESR_H
+#define ESR_H
+
+#include <arch.h>
+#include <assert.h>
+#include <stdbool.h>
+#include <utils_def.h>
+
+static inline unsigned int esr_srt(unsigned long esr)
+{
+ return EXTRACT(ESR_EL2_ABORT_SRT, esr);
+}
+
+static inline bool esr_is_write(unsigned long esr)
+{
+ return ((esr & ESR_EL2_ABORT_WNR_BIT) != 0UL);
+}
+
+static inline bool esr_sign_extend(unsigned long esr)
+{
+ return ((esr & ESR_EL2_ABORT_SSE_BIT) != 0UL);
+}
+
+static inline bool esr_sixty_four(unsigned long esr)
+{
+ return ((esr & ESR_EL2_ABORT_SF_BIT) != 0UL);
+}
+
+static inline unsigned int esr_sas(unsigned long esr)
+{
+ return EXTRACT(ESR_EL2_ABORT_SAS, esr);
+}
+
+static inline unsigned int access_len(unsigned long esr)
+{
+ switch (esr_sas(esr)) {
+ case ESR_EL2_ABORT_SAS_BYTE_VAL:
+ return 1U;
+ case ESR_EL2_ABORT_SAS_HWORD_VAL:
+ return 2U;
+ case ESR_EL2_ABORT_SAS_WORD_VAL:
+ return 4U;
+ default:
+ assert(esr_sas(esr) == ESR_EL2_ABORT_SAS_DWORD_VAL);
+ return 8U;
+ }
+}
+
+static inline unsigned long access_mask(unsigned long esr)
+{
+ switch (esr_sas(esr)) {
+ case ESR_EL2_ABORT_SAS_BYTE_VAL:
+ return 0xffUL;
+ case ESR_EL2_ABORT_SAS_HWORD_VAL:
+ return 0xffffUL;
+ case ESR_EL2_ABORT_SAS_WORD_VAL:
+ return 0xffffffffUL;
+ default:
+ assert(esr_sas(esr) == ESR_EL2_ABORT_SAS_DWORD_VAL);
+ return ~(0UL);
+ }
+}
+
+/*
+ * For a trapped msr/mrs sysreg access, get the transfer register in the
+ * ESR_EL2.
+ */
+static inline unsigned int esr_sysreg_rt(unsigned long esr)
+{
+ return EXTRACT(ESR_EL2_SYSREG_TRAP_RT, esr);
+}
+
+#endif /* ESR_H */
diff --git a/lib/arch/include/fake_host/atomics.h b/lib/arch/include/fake_host/atomics.h
new file mode 100644
index 0000000..bda6784
--- /dev/null
+++ b/lib/arch/include/fake_host/atomics.h
@@ -0,0 +1,78 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
+ */
+
+#ifndef ATOMICS_H
+#define ATOMICS_H
+
+#include <stdbool.h>
+#include <stdint.h>
+
+/*
+ * Atomically adds @val to the 64-bit value stored at memory location @loc.
+ */
+static inline void atomic_add_64(uint64_t *loc, long val)
+{
+ *loc = *loc + val;
+}
+
+/*
+ * Atomically adds @val to the 64-bit value stored at memory location @loc.
+ * Stores to memory with release semantics.
+ * Returns the old value.
+ */
+static inline unsigned long atomic_load_add_release_64(uint64_t *loc, long val)
+{
+ unsigned long old_val = *loc;
+
+ *loc = *loc + val;
+ return old_val;
+}
+
+/*
+ * Atomically set bit @bit in value pointed to by @val with release semantics.
+ */
+static inline void atomic_bit_set_release_64(uint64_t *loc, int bit)
+{
+ uint64_t mask = (1UL << bit);
+
+ *loc = *loc | mask;
+}
+
+/*
+ * Atomically clear bit @bit in value pointed to by @loc with release semantics.
+ */
+static inline void atomic_bit_clear_release_64(uint64_t *loc, int bit)
+{
+ uint64_t mask = ~((uint64_t)(1UL << bit));
+
+ *loc = *loc & mask;
+}
+
+/*
+ * Test bit @bit in value pointed to by @loc with acquire semantics.
+ */
+static inline bool atomic_test_bit_acquire_64(uint64_t *loc, int bit)
+{
+ uint64_t val = *loc;
+ uint64_t mask = (1UL << bit);
+
+ return ((val & mask) != 0UL);
+}
+
+/*
+ * Atomically set bit @bit in value pointed to by @val
+ * with acquire and release semantics.
+ * Return True if the previous state of @bit was 1, False otherwise.
+ */
+static inline bool atomic_bit_set_acquire_release_64(uint64_t *loc, int bit)
+{
+ uint64_t mask = (1UL << bit);
+ unsigned long old_val = *loc & mask;
+
+ *loc |= mask;
+ return (old_val != 0UL);
+}
+
+#endif /* ATOMICS_H */
diff --git a/lib/arch/include/fake_host/cpuid.h b/lib/arch/include/fake_host/cpuid.h
new file mode 100644
index 0000000..c4c6a6e
--- /dev/null
+++ b/lib/arch/include/fake_host/cpuid.h
@@ -0,0 +1,14 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
+ */
+
+#ifndef CPUID_H
+#define CPUID_H
+
+static inline unsigned int my_cpuid(void)
+{
+ return 0UL;
+}
+
+#endif /* CPUID_H */
diff --git a/lib/arch/include/fake_host/entropy.h b/lib/arch/include/fake_host/entropy.h
new file mode 100644
index 0000000..ff70dcf
--- /dev/null
+++ b/lib/arch/include/fake_host/entropy.h
@@ -0,0 +1,20 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
+ */
+
+#ifndef ENTROPY_H
+#define ENTROPY_H
+
+#include <stdbool.h>
+#include <stdint.h>
+
+static inline bool arch_collect_entropy(uint64_t *random)
+{
+ static uint64_t val;
+
+ *random = val++;
+ return true;
+}
+
+#endif /* ENTROPY_H */
diff --git a/lib/arch/include/fake_host/instr_helpers.h b/lib/arch/include/fake_host/instr_helpers.h
new file mode 100644
index 0000000..5113cc6
--- /dev/null
+++ b/lib/arch/include/fake_host/instr_helpers.h
@@ -0,0 +1,65 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
+ */
+
+#ifndef INSTR_HELPERS_H
+#define INSTR_HELPERS_H
+
+#include <host_harness.h>
+#include <stddef.h>
+
+/**********************************************************************
+ * Macros which create inline functions to read or write CPU system
+ * registers
+ *********************************************************************/
+#define DEFINE_SYSREG_READ_FUNC_(_name, _reg_name) \
+static inline u_register_t read_ ## _name(void) \
+{ \
+ return host_read_sysreg(#_name); \
+}
+
+#define DEFINE_SYSREG_WRITE_FUNC_(_name, _reg_name) \
+static inline void write_ ## _name(u_register_t v) \
+{ \
+ host_write_sysreg(#_name, v); \
+}
+
+#define SYSREG_WRITE_CONST(reg_name, v) \
+ host_write_sysreg(#reg_name, v)
+
+/**********************************************************************
+ * Macros to create inline functions for system instructions
+ *********************************************************************/
+
+/* Define function for simple system instruction */
+#define DEFINE_SYSOP_FUNC(_op) \
+static inline void (_op)(void) \
+{ \
+ (void)_op; \
+}
+
+/* Define function for system instruction with register parameter */
+#define DEFINE_SYSOP_PARAM_FUNC(_op) \
+static inline void (_op)(uint64_t v) \
+{ \
+ (void)v; /* To avoid MISRA-C:2012-2.7 warnings */ \
+}
+
+/* Define function for system instruction with type specifier */
+#define DEFINE_SYSOP_TYPE_FUNC(_op, _type) \
+static inline void (_op ## _type)(void) \
+{ \
+}
+
+/* Define function for system instruction with register parameter */
+#define DEFINE_SYSOP_TYPE_PARAM_FUNC(_op, _type) \
+static inline void (_op ## _type)(uint64_t v) \
+{ \
+ (void)v; /* To avoid MISRA-C:2012-2.7 warnings */ \
+}
+
+#define dsb(scope)
+#define dmb(scope)
+
+#endif /* INSTR_HELPERS_H */
diff --git a/lib/arch/include/fake_host/memory.h b/lib/arch/include/fake_host/memory.h
new file mode 100644
index 0000000..3f10f44
--- /dev/null
+++ b/lib/arch/include/fake_host/memory.h
@@ -0,0 +1,40 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
+ */
+
+#ifndef MEMORY_H
+#define MEMORY_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+/* Single-Copy Atomic 64-bit write */
+static inline void __sca_write64(uint64_t *ptr, uint64_t val)
+{
+ *ptr = val;
+}
+#define SCA_WRITE64(_p, _v) __sca_write64((void *)(_p), ((uint64_t)(_v)))
+
+/* Single-Copy Atomic 64-bit write with RELEASE memory ordering semantics*/
+static inline void __sca_write64_release(uint64_t *ptr, uint64_t val)
+{
+ *ptr = val;
+}
+#define SCA_WRITE64_RELEASE(_p, _v) __sca_write64_release((void *)(_p), ((uint64_t)(_v)))
+
+/* Single-Copy Atomic 64-bit read */
+static inline uint64_t __sca_read64(uint64_t *ptr)
+{
+ return *ptr;
+}
+#define SCA_READ64(_p) ((typeof(*(_p)))__sca_read64((void *)(_p)))
+
+/* Single-Copy Atomic 64-bit read with ACQUIRE memory ordering semantics */
+static inline uint64_t __sca_read64_acquire(uint64_t *ptr)
+{
+ return *ptr;
+}
+#define SCA_READ64_ACQUIRE(_p) ((typeof(*(_p)))__sca_read64_acquire((void *)(_p)))
+
+#endif /* MEMORY_H */
diff --git a/lib/arch/include/fake_host/mmio.h b/lib/arch/include/fake_host/mmio.h
new file mode 100644
index 0000000..293f803
--- /dev/null
+++ b/lib/arch/include/fake_host/mmio.h
@@ -0,0 +1,51 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
+ */
+
+#ifndef MMIO_H
+#define MMIO_H
+
+#include <stdint.h>
+
+static inline uint8_t read8(volatile void *addr)
+{
+ return *(uint8_t *)addr;
+}
+
+static inline void write8(uint8_t val, volatile void *addr)
+{
+ *(uint8_t *)addr = val;
+}
+
+static inline uint16_t read16(volatile void *addr)
+{
+ return *(uint16_t *)addr;
+}
+
+static inline void write16(uint16_t val, volatile void *addr)
+{
+ *(uint16_t *)addr = val;
+}
+
+static inline uint32_t read32(volatile void *addr)
+{
+ return *(uint32_t *)addr;
+}
+
+static inline void write32(uint32_t val, volatile void *addr)
+{
+ *(uint32_t *)addr = val;
+}
+
+static inline uint64_t read64(volatile void *addr)
+{
+ return *(uint64_t *)addr;
+}
+
+static inline void write64(uint64_t val, volatile void *addr)
+{
+ *(uint64_t *)addr = val;
+}
+
+#endif /* MMIO_H */
diff --git a/lib/arch/include/fake_host/spinlock.h b/lib/arch/include/fake_host/spinlock.h
new file mode 100644
index 0000000..7aa4b29
--- /dev/null
+++ b/lib/arch/include/fake_host/spinlock.h
@@ -0,0 +1,25 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
+ */
+
+#ifndef SPINLOCK_H
+#define SPINLOCK_H
+
+#include <host_harness.h>
+
+typedef struct spinlock_s {
+ unsigned int val;
+} spinlock_t;
+
+static inline void spinlock_acquire(spinlock_t *l)
+{
+ host_spinlock_acquire(l);
+}
+
+static inline void spinlock_release(spinlock_t *l)
+{
+ host_spinlock_release(l);
+}
+
+#endif /* SPINLOCK_H */
diff --git a/lib/arch/include/fpu_helpers.h b/lib/arch/include/fpu_helpers.h
new file mode 100644
index 0000000..2581a3b
--- /dev/null
+++ b/lib/arch/include/fpu_helpers.h
@@ -0,0 +1,93 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
+ */
+
+#ifndef FPU_HELPERS_H
+#define FPU_HELPERS_H
+
+#include <arch_helpers.h>
+#include <assert.h>
+#include <cpuid.h>
+#include <stdbool.h>
+
+/* The FPU and SIMD register bank is 32 quadword (128 bits) Q registers. */
+#define FPU_Q_SIZE 16U
+#define FPU_Q_COUNT 32U
+
+/* These defines are needed by assembly code to access the context. */
+#define FPU_CTX_OFFSET_Q 0U
+#define FPU_CTX_OFFSET_FPSR 512U
+#define FPU_CTX_OFFSET_FPCR 520U
+
+#ifdef RMM_FPU_USE_AT_REL2
+#define FPU_ALLOW(expression) \
+ do { \
+ assert(fpu_is_my_state_saved(my_cpuid())); \
+ write_cptr_el2( \
+ (read_cptr_el2() & \
+ (~(CPTR_EL2_FPEN_MASK << CPTR_EL2_FPEN_SHIFT))) | \
+ (CPTR_EL2_FPEN_NO_TRAP_11 << CPTR_EL2_FPEN_SHIFT)); \
+ isb(); \
+ expression; \
+ write_cptr_el2( \
+ (read_cptr_el2() & \
+ (~(CPTR_EL2_FPEN_MASK << CPTR_EL2_FPEN_SHIFT))) | \
+ (CPTR_EL2_FPEN_TRAP_ALL_00 << CPTR_EL2_FPEN_SHIFT)); \
+ isb(); \
+ } while (0)
+
+#define IS_FPU_ALLOWED() \
+ (fpu_is_my_state_saved(my_cpuid()) && is_fpen_enabled())
+
+#else /* RMM_FPU_USE_AT_REL2 */
+#define FPU_ALLOW(expression) \
+ do { \
+ expression; \
+ } while (0)
+
+#define IS_FPU_ALLOWED() (true)
+
+#endif /* RMM_FPU_USE_AT_REL2 */
+
+struct fpu_state {
+ unsigned __int128 q[FPU_Q_COUNT];
+ unsigned long fpsr;
+ unsigned long fpcr;
+};
+
+/* Since we use these offsets in assembly code make sure they are correct. */
+COMPILER_ASSERT(__builtin_offsetof(struct fpu_state, q) ==
+ FPU_CTX_OFFSET_Q);
+COMPILER_ASSERT(__builtin_offsetof(struct fpu_state, fpsr) ==
+ FPU_CTX_OFFSET_FPSR);
+COMPILER_ASSERT(__builtin_offsetof(struct fpu_state, fpcr) ==
+ FPU_CTX_OFFSET_FPCR);
+
+/*
+ * Save/restore FPU context to/from the `fpu_state` passed as parameter. The FPU
+ * instruction trap needs to be disabled before calling these functions.
+ * Can be used for context switching.
+ */
+void fpu_save_state(struct fpu_state *fpu);
+void fpu_restore_state(struct fpu_state *fpu);
+
+/*
+ * Save/restore FPU state to/from a per-cpu buffer allocated within the
+ * library. The FPU instruction trap is disabled by this function during the
+ * access to the FPU registers.
+ * These functions are expected to be called before FPU is used by RMM to save
+ * the incoming FPU context.
+ */
+void fpu_save_my_state(void);
+void fpu_restore_my_state(void);
+
+/*
+ * Return true iff an fpu state is saved in the per-cpu buffer in this library.
+ *
+ * After calling 'fpu_save_my_state' this function returns true. After calling
+ * 'fpu_restore_my_state' this function returns false.
+ */
+bool fpu_is_my_state_saved(unsigned int cpu_id);
+
+#endif /* FPU_HELPERS_H */
diff --git a/lib/arch/src/aarch64/cache_helpers.S b/lib/arch/src/aarch64/cache_helpers.S
new file mode 100644
index 0000000..195cf61
--- /dev/null
+++ b/lib/arch/src/aarch64/cache_helpers.S
@@ -0,0 +1,57 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
+ */
+
+#include <arch.h>
+#include <asm_macros.S>
+
+ .globl flush_dcache_range
+ .globl clean_dcache_range
+ .globl inv_dcache_range
+
+/*
+ * This macro can be used for implementing various data cache operations `op`
+ */
+.macro do_dcache_maintenance_by_mva op
+ /* Exit early if size is zero */
+ cbz x1, exit_loop_\op
+ dcache_line_size x2, x3
+ add x1, x0, x1
+ sub x3, x2, #1
+ bic x0, x0, x3
+loop_\op:
+ dc \op, x0
+ add x0, x0, x2
+ cmp x0, x1
+ b.lo loop_\op
+ dsb sy
+exit_loop_\op:
+ ret
+.endm
+ /* ------------------------------------------
+ * Clean+Invalidate from base address till
+ * size. 'x0' = addr, 'x1' = size
+ * ------------------------------------------
+ */
+func flush_dcache_range
+ do_dcache_maintenance_by_mva civac
+endfunc flush_dcache_range
+
+ /* ------------------------------------------
+ * Clean from base address till size.
+ * 'x0' = addr, 'x1' = size
+ * ------------------------------------------
+ */
+func clean_dcache_range
+ do_dcache_maintenance_by_mva cvac
+endfunc clean_dcache_range
+
+ /* ------------------------------------------
+ * Invalidate from base address till
+ * size. 'x0' = addr, 'x1' = size
+ * ------------------------------------------
+ */
+func inv_dcache_range
+ do_dcache_maintenance_by_mva ivac
+endfunc inv_dcache_range
diff --git a/lib/arch/src/aarch64/fpu_helpers.S b/lib/arch/src/aarch64/fpu_helpers.S
new file mode 100644
index 0000000..6b7a1ef
--- /dev/null
+++ b/lib/arch/src/aarch64/fpu_helpers.S
@@ -0,0 +1,83 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
+ */
+
+#include <asm_macros.S>
+
+.globl fpu_save_state
+.globl fpu_restore_state
+
+/*
+ * void fpu_save_state(struct fpu_state *fpu);
+ *
+ * This function saves the FPU state in systems supporting FPU but NOT SVE.
+ * Systems with SVE support only use the SVE save/restore since these
+ * registers are mapped into SVE registers.
+ *
+ * Since we make assumptions about register fields in C structs here, there are
+ * compiler asserts in rec.h to help guarantee that these assumptions are true.
+ */
+func fpu_save_state
+ /* Save the SIMD/FPU register bank to memory. */
+ stp q0, q1, [x0], #32
+ stp q2, q3, [x0], #32
+ stp q4, q5, [x0], #32
+ stp q6, q7, [x0], #32
+ stp q8, q9, [x0], #32
+ stp q10, q11, [x0], #32
+ stp q12, q13, [x0], #32
+ stp q14, q15, [x0], #32
+ stp q16, q17, [x0], #32
+ stp q18, q19, [x0], #32
+ stp q20, q21, [x0], #32
+ stp q22, q23, [x0], #32
+ stp q24, q25, [x0], #32
+ stp q26, q27, [x0], #32
+ stp q28, q29, [x0], #32
+ stp q30, q31, [x0], #32
+
+ /* Save the FPSR/FPCR */
+ mrs x1, fpsr
+ mrs x2, fpcr
+ stp x1, x2, [x0]
+
+ ret
+endfunc fpu_save_state
+
+/*
+ * void fpu_restore_state(struct fpu_state *fpu);
+ *
+ * Function to restore a saved FPU register state on systems supporting FPU/SIMD
+ * but NOT SVE. Systems with SVE support enabled only use the SVE save/restore
+ * since these registers are mapped into SVE registers.
+ *
+ * Since we make assumptions about register fields in C structs here, there are
+ * compiler asserts in rec.h to help guarantee that these assumptions are true.
+ */
+func fpu_restore_state
+ /* Load the SIMD/FPU register bank from memory. */
+ ldp q0, q1, [x0], #32
+ ldp q2, q3, [x0], #32
+ ldp q4, q5, [x0], #32
+ ldp q6, q7, [x0], #32
+ ldp q8, q9, [x0], #32
+ ldp q10, q11, [x0], #32
+ ldp q12, q13, [x0], #32
+ ldp q14, q15, [x0], #32
+ ldp q16, q17, [x0], #32
+ ldp q18, q19, [x0], #32
+ ldp q20, q21, [x0], #32
+ ldp q22, q23, [x0], #32
+ ldp q24, q25, [x0], #32
+ ldp q26, q27, [x0], #32
+ ldp q28, q29, [x0], #32
+ ldp q30, q31, [x0], #32
+
+ /* Load the FPSR/FPCR */
+ ldp x1, x2, [x0]
+ msr fpsr, x1
+ msr fpcr, x2
+
+ ret
+endfunc fpu_restore_state
diff --git a/lib/arch/src/arch_features.c b/lib/arch/src/arch_features.c
new file mode 100644
index 0000000..67d70b6
--- /dev/null
+++ b/lib/arch/src/arch_features.c
@@ -0,0 +1,38 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
+ */
+
+#include <arch.h>
+#include <arch_helpers.h>
+#include <assert.h>
+#include <stdint.h>
+#include <utils_def.h>
+
+/*
+ * Return the PA width supported by the current system.
+ */
+unsigned int arch_feat_get_pa_width(void)
+{
+ /*
+ * Physical Address ranges supported in the AArch64 Memory Model.
+ * Value 0b110 is supported in ARMv8.2 onwards but not used in RMM.
+ */
+ static const unsigned int pa_range_bits_arr[] = {
+ PARANGE_0000_WIDTH, PARANGE_0001_WIDTH, PARANGE_0010_WIDTH,
+ PARANGE_0011_WIDTH, PARANGE_0100_WIDTH, PARANGE_0101_WIDTH,
+ /*
+ * FEAT_LPA/LPA2 is not supported yet in RMM,
+ * so max PA width is 48.
+ */
+ PARANGE_0101_WIDTH
+ };
+
+ register_t pa_range = (read_id_aa64mmfr0_el1() >>
+ ID_AA64MMFR0_EL1_PARANGE_SHIFT) &
+ ID_AA64MMFR0_EL1_PARANGE_MASK;
+
+ assert(pa_range < ARRAY_SIZE(pa_range_bits_arr));
+
+ return pa_range_bits_arr[pa_range];
+}
diff --git a/lib/arch/src/fake_host/cache_wrappers.c b/lib/arch/src/fake_host/cache_wrappers.c
new file mode 100644
index 0000000..6de91fa
--- /dev/null
+++ b/lib/arch/src/fake_host/cache_wrappers.c
@@ -0,0 +1,25 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
+ */
+
+#include <arch_helpers.h>
+
+/*******************************************************************************
+ * Cache management
+ ******************************************************************************/
+void flush_dcache_range(uintptr_t addr, size_t size)
+{
+ (void)addr;
+ (void)size;
+}
+void clean_dcache_range(uintptr_t addr, size_t size)
+{
+ (void)addr;
+ (void)size;
+}
+void inv_dcache_range(uintptr_t addr, size_t size)
+{
+ (void)addr;
+ (void)size;
+}
diff --git a/lib/arch/src/fake_host/fpu_helpers_host.c b/lib/arch/src/fake_host/fpu_helpers_host.c
new file mode 100644
index 0000000..4922251
--- /dev/null
+++ b/lib/arch/src/fake_host/fpu_helpers_host.c
@@ -0,0 +1,9 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
+ */
+
+#include <fpu_helpers.h>
+
+void fpu_save_state(struct fpu_state *fpu) {}
+void fpu_restore_state(struct fpu_state *fpu) {}
diff --git a/lib/arch/src/fpu_helpers.c b/lib/arch/src/fpu_helpers.c
new file mode 100644
index 0000000..70f79d9
--- /dev/null
+++ b/lib/arch/src/fpu_helpers.c
@@ -0,0 +1,51 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
+ */
+
+#include <cpuid.h>
+#include <fpu_helpers.h>
+#include <stdbool.h>
+
+struct rmm_fpu_state {
+ struct fpu_state state;
+ bool saved;
+};
+
+struct rmm_fpu_state rmm_fpu_state[MAX_CPUS];
+
+#ifdef RMM_FPU_USE_AT_REL2
+void fpu_save_my_state(void)
+{
+ struct rmm_fpu_state *rmm_state;
+ unsigned int cpu_id = my_cpuid();
+
+ rmm_state = rmm_fpu_state + cpu_id;
+ assert(!rmm_state->saved);
+ rmm_state->saved = true;
+ FPU_ALLOW(fpu_save_state(&(rmm_state->state)));
+}
+
+void fpu_restore_my_state(void)
+{
+ struct rmm_fpu_state *rmm_state;
+ unsigned int cpu_id = my_cpuid();
+
+ rmm_state = rmm_fpu_state + cpu_id;
+ assert(rmm_state->saved);
+ FPU_ALLOW(fpu_restore_state(&(rmm_state->state)));
+ rmm_state->saved = false;
+}
+
+bool fpu_is_my_state_saved(unsigned int cpu_id)
+{
+ assert(cpu_id < MAX_CPUS);
+ return rmm_fpu_state[cpu_id].saved;
+}
+
+#else /* RMM_FPU_USE_AT_REL2 */
+void fpu_save_my_state(void) {}
+
+void fpu_restore_my_state(void) {}
+
+#endif /* RMM_FPU_USE_AT_REL2 */