Enable MMU in EL2.
diff --git a/src/arch/aarch64/entry.S b/src/arch/aarch64/entry.S
index 3c8f860..7f8a4e1 100644
--- a/src/arch/aarch64/entry.S
+++ b/src/arch/aarch64/entry.S
@@ -1,14 +1,11 @@
#include "offsets.h"
-#define PECOFF_FILE_ALIGNMENT 0x200
-
.section .init.entry, "ax"
.global entry
entry:
-
- add x13, x18, #0x16
b 0f
+ .word 0
.quad 4096 /* text_offset */
.quad file_size /* image_size */
.quad 0 /* flags */
diff --git a/src/arch/aarch64/handler.c b/src/arch/aarch64/handler.c
index 48de85d..27ceacf 100644
--- a/src/arch/aarch64/handler.c
+++ b/src/arch/aarch64/handler.c
@@ -19,7 +19,21 @@
void sync_current_exception(uint64_t esr, uint64_t elr)
{
- dlog("Exception: esr=%#x, elr=%#x\n", esr, elr);
+ switch (esr >> 26) {
+ case 0x25: /* EC = 100101, Data abort. */
+ dlog("Data abort: pc=0x%x, esr=0x%x, ec=0x%x", elr, esr, esr >> 26);
+ if (!(esr & (1u << 10))) /* Check FnV bit. */
+ dlog(", far=0x%x, hpfar=0x%x", read_msr(far_el2), read_msr(hpfar_el2) << 8);
+ else
+ dlog(", far=invalid");
+
+ dlog("\n");
+ for (;;);
+
+ default:
+ dlog("Unknown sync exception pc=0x%x, esr=0x%x, ec=0x%x\n", elr, esr, esr >> 26);
+ for (;;);
+ }
for (;;);
}
diff --git a/src/arch/aarch64/inc/arch_mm.h b/src/arch/aarch64/inc/arch_mm.h
new file mode 100644
index 0000000..c65c488
--- /dev/null
+++ b/src/arch/aarch64/inc/arch_mm.h
@@ -0,0 +1,191 @@
+#ifndef _ARCH_MM_H
+#define _ARCH_MM_H
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+
+/* A phypiscal address. */
+typedef size_t paddr_t;
+
+/* A virtual address. */
+typedef size_t vaddr_t;
+
+/* A page table entry. */
+typedef size_t pte_t;
+
+#define PAGE_LEVEL_BITS 9
+#define PAGE_BITS 12
+
+struct arch_mm_ptable {
+ int max_level;
+};
+
+/**
+ * Initialises the architecture-dependents aspects of the page table.
+ */
+static inline void arch_mm_ptable_init(struct arch_mm_ptable *t)
+{
+ t->max_level = 2;
+}
+
+/**
+ * Determines the maximum level supported by the given page table.
+ */
+static inline int arch_mm_max_level(struct arch_mm_ptable *t)
+{
+ return t->max_level;
+}
+
+/**
+ * Converts a physical address to a table PTE.
+ *
+ * The spec says that 'Table descriptors for stage 2 translations do not
+ * include any attribute field', so we don't take any attributes as arguments.
+ */
+static inline pte_t arch_mm_pa_to_table_pte(paddr_t pa)
+{
+ return pa | 0x3;
+}
+
+/**
+ * Converts a physical address to a block PTE.
+ */
+static inline pte_t arch_mm_pa_to_block_pte(paddr_t pa, uint64_t attrs)
+{
+ return pa | attrs;
+}
+
+/**
+ * Converts a physical address to a page PTE.
+ */
+static inline pte_t arch_mm_pa_to_page_pte(paddr_t pa, uint64_t attrs)
+{
+ return pa | attrs | ((attrs & 1) << 1);
+}
+
+/**
+ * Converts a block PTE to a page PTE.
+ */
+static inline pte_t arch_mm_block_to_page_pte(pte_t pte)
+{
+ return pte | 2;
+}
+
+/**
+ * Specifies whether block mappings are acceptable at the given level.
+ */
+static inline bool arch_mm_is_block_allowed(int level)
+{
+ return level == 1 || level == 2;
+}
+
+/**
+ * Returns the encoding of a page table entry that isn't present.
+ */
+static inline pte_t arch_mm_absent_pte(void)
+{
+ return 0;
+}
+
+/**
+ * Determines if the given pte is present, i.e., if it points to another table,
+ * to a page, or a block of pages.
+ */
+static inline bool arch_mm_pte_is_present(pte_t pte)
+{
+ return (pte & 1) != 0;
+}
+
+/**
+ * Determines if the given pte references another table.
+ */
+static inline bool arch_mm_pte_is_table(pte_t pte)
+{
+ return (pte & 3) == 3;
+}
+
+/**
+ * Determines if the given pte references a block of pages.
+ */
+static inline bool arch_mm_pte_is_block(pte_t pte)
+{
+ return (pte & 3) == 1;
+}
+
+/**
+ * Clears the given virtual address, i.e., sets the ignored bits (from a page
+ * table perspective) to zero.
+ */
+static inline vaddr_t arch_mm_clear_va(vaddr_t addr)
+{
+ return addr & ~((1ull << PAGE_BITS) - 1) & ((1ull << 48) - 1);
+}
+
+/**
+ * Clears the given physical address, i.e., sets the ignored bits (from a page
+ * table perspective) to zero.
+ */
+static inline paddr_t arch_mm_clear_pa(paddr_t addr)
+{
+ return addr & ~((1ull << PAGE_BITS) - 1) & ((1ull << 48) - 1);
+}
+
+/**
+ * Extracts the physical address from a page table entry.
+ */
+static inline paddr_t arch_mm_pte_to_paddr(pte_t pte)
+{
+ return arch_mm_clear_pa(pte);
+}
+
+/**
+ * Extracts a page table pointer from the given page table entry.
+ */
+static inline pte_t *arch_mm_pte_to_table(pte_t pte)
+{
+ return (pte_t *)arch_mm_pte_to_paddr(pte);
+}
+
+/**
+ * Invalidates stage-1 TLB entries referring to the given virtual address range.
+ */
+static inline void arch_mm_invalidate_stage1_range(vaddr_t begin, vaddr_t end)
+{
+ vaddr_t it;
+
+ begin >>= 12;
+ end >>= 12;
+
+ __asm__ volatile("dsb ishst");
+
+ for (it = begin; it < end; it += (1ull << (PAGE_BITS - 12)))
+ __asm__("tlbi vae2is, %0" : : "r"(it));
+
+ __asm__ volatile("dsb ish");
+}
+
+/**
+ * Invalidates stage-2 TLB entries referring to the given virtual address range.
+ */
+static inline void arch_mm_invalidate_stage2_range(vaddr_t begin, vaddr_t end)
+{
+ vaddr_t it;
+
+ begin >>= 12;
+ end >>= 12;
+
+ __asm__ volatile("dsb ishst");
+
+ for (it = begin; it < end; it += (1ull << (PAGE_BITS - 12)))
+ __asm__("tlbi ipas2e1, %0" : : "r"(it));
+
+ __asm__ volatile("dsb ish\n"
+ "tlbi vmalle1is\n"
+ "dsb ish\n");
+}
+
+uint64_t arch_mm_mode_to_attrs(int mode);
+void arch_mm_init(paddr_t table);
+
+#endif /* _ARCH_MM_H */
diff --git a/src/arch/aarch64/mm.c b/src/arch/aarch64/mm.c
index ef446f3..225f1ae 100644
--- a/src/arch/aarch64/mm.c
+++ b/src/arch/aarch64/mm.c
@@ -1,10 +1,59 @@
#include "arch_cpu.h"
+#include "mm.h"
#include "msr.h"
+#define NON_SHAREABLE 0ull
+#define OUTER_SHAREABLE 2ull
+#define INNER_SHAREABLE 3ull
+
+#define STAGE1_XN (1ull << 54)
+#define STAGE1_CONTIGUOUS (1ull << 52)
+#define STAGE1_DBM (1ull << 51)
+#define STAGE1_NG (1ull << 11)
+#define STAGE1_AF (1ull << 10)
+#define STAGE1_SH(x) ((x) << 8)
+#define STAGE1_AP(x) ((x) << 6)
+#define STAGE1_NS (1ull << 5)
+#define STAGE1_ATTRINDX(x) ((x) << 2)
+
+#define STAGE1_READONLY 2ull
+#define STAGE1_READWRITE 0ull
+
+#define STAGE1_DEVICEINDX 0ull
+#define STAGE1_NORMALINDX 1ull
+
+#define STAGE2_XN(x) ((x) << 53)
+#define STAGE2_CONTIGUOUS (1ull << 52)
+#define STAGE2_DBM (1ull << 51)
+#define STAGE2_AF (1ull << 10)
+#define STAGE2_SH(x) ((x) << 8)
+#define STAGE2_S2AP(x) ((x) << 6)
+#define STAGE2_MEMATTR(x) ((x) << 2)
+
+#define STAGE2_EXECUTE_ALL 0ull
+#define STAGE2_EXECUTE_EL0 1ull
+#define STAGE2_EXECUTE_NONE 2ull
+#define STAGE2_EXECUTE_EL1 3ull
+
+/* The following are stage-2 memory attributes for normal memory. */
+#define STAGE2_NONCACHEABLE 1ull
+#define STAGE2_WRITETHROUGH 2ull
+#define STAGE2_WRITEBACK 3ull
+
+#define STAGE2_MEMATTR_NORMAL(outer, inner) ((outer << 2) | (inner))
+
+/* The following stage-2 memory attributes for device memory. */
+#define STAGE2_MEMATTR_DEVICE_nGnRnE 0ull
+#define STAGE2_MEMATTR_DEVICE_nGnRE 1ull
+#define STAGE2_MEMATTR_DEVICE_nGRE 2ull
+#define STAGE2_MEMATTR_DEVICE_GRE 3ull
+
+#define STAGE2_ACCESS_READ 1ull
+#define STAGE2_ACCESS_WRITE 2ull
+
void arch_vptable_init(struct arch_page_table *table)
{
uint64_t i;
- uint64_t v;
/* TODO: Check each bit. */
for (i = 0; i < 512; i++) {
@@ -25,9 +74,64 @@
table->first[0] = (uint64_t)&table->entry0[0] | 3;
table->first[1] = (uint64_t)&table->entry1[0] | 3;
+}
- /* TODO: Where should this go? */
- v =
+uint64_t arch_mm_mode_to_attrs(int mode)
+{
+ uint64_t attrs = 1; /* Present bit. */
+
+ if (mode & MM_MODE_STAGE1) {
+ attrs |= STAGE1_AF | STAGE1_SH(OUTER_SHAREABLE);
+
+ /* Define the execute bits. */
+ if (!(mode & MM_MODE_X))
+ attrs |= STAGE1_XN;
+
+ /* Define the read/write bits. */
+ if (mode & MM_MODE_W)
+ attrs |= STAGE1_AP(STAGE1_READWRITE);
+ else
+ attrs |= STAGE1_AP(STAGE1_READONLY);
+
+ /* Define the memory attribute bits. */
+ if (mode & MM_MODE_D)
+ attrs |= STAGE1_ATTRINDX(STAGE1_DEVICEINDX);
+ else
+ attrs |= STAGE1_ATTRINDX(STAGE1_NORMALINDX);
+ } else {
+ uint64_t access = 0;
+
+ attrs |= STAGE2_AF | STAGE2_SH(OUTER_SHAREABLE);
+
+ /* Define the read/write bits. */
+ if (mode & MM_MODE_R)
+ access |= STAGE2_ACCESS_READ;
+
+ if (mode & MM_MODE_W)
+ access |= STAGE2_ACCESS_WRITE;
+
+ attrs |= STAGE2_S2AP(access);
+
+ /* Define the execute bits. */
+ if (mode & MM_MODE_X)
+ attrs |= STAGE2_XN(STAGE2_EXECUTE_ALL);
+ else
+ attrs |= STAGE2_XN(STAGE2_EXECUTE_NONE);
+
+ /* Define the memory attribute bits. */
+ if (mode & MM_MODE_D)
+ attrs |= STAGE2_MEMATTR_DEVICE_nGnRnE;
+ else
+ attrs |= STAGE2_MEMATTR_NORMAL(STAGE2_WRITEBACK,
+ STAGE2_WRITEBACK);
+ }
+
+ return attrs;
+}
+
+void arch_mm_init(paddr_t table)
+{
+ uint64_t v =
(1u << 31) | /* RES1. */
(4 << 16) | /* PS: 44 bits. */
(0 << 14) | /* TG0: 4 KB granule. */
@@ -37,258 +141,16 @@
(2 << 6) | /* SL0: Start at level 0. */
(20 << 0); /* T0SZ: 44-bit input address size. */
write_msr(vtcr_el2, v);
-}
-#if 0
-#include "arch.h"
-
-#include <stdint.h>
-
-#include "alloc.h"
-#include "log.h"
-#include "msr.h"
-
-#define PAGE_BITS 12
-#define PAGE_SIZE (1 << PAGE_BITS)
-#define ENTRIES_PER_LEVEL (PAGE_SIZE / sizeof(uint64_t))
-#define INITIAL_LEVEL 1
-
-extern char text_begin[];
-extern char text_end[];
-extern char rodata_begin[];
-extern char rodata_end[];
-extern char data_begin[];
-extern char data_end[];
-extern char bin_end[];
-
-static uint64_t *ttbr;
-
-static inline size_t mm_entry_size(int level)
-{
- return 1ull << (PAGE_BITS + (3 - level) * (PAGE_BITS - 3));
-}
-
-static inline size_t mm_level_end(size_t va, int level)
-{
- size_t offset = (PAGE_BITS + (4 - level) * (PAGE_BITS - 3));
- return ((va >> offset) + 1) << offset;
-}
-
-static inline size_t mm_index(size_t va, int level)
-{
- size_t v = va >> (PAGE_BITS + (3 - level) * (PAGE_BITS - 3));
- return v & ((1 << (PAGE_BITS - 3)) - 1);
-}
-
-static inline uint64_t mm_clear_attrs(uint64_t v)
-{
- /* Clean bottom bits. */
- v &= ~((1 << PAGE_BITS) - 1);
-
- /* Clean top bits. */
- v &= ((1ull << 59) - 1);
-
- return v;
-}
-
-static inline uint64_t *mm_table_ptr(uint64_t pa)
-{
- return (uint64_t *)mm_clear_attrs(pa);
-}
-
-static inline uint64_t mm_mode_to_attrs(uint64_t mode)
-{
- uint64_t attrs =
- (1 << 10) | /* Access flag. */
- (2 << 8); /* sh -> outer shareable. */
-
- /* TODO: This is different in s2. */
- if (!(mode & MM_X)) {
- attrs |= (1ull << 54); /* XN or UXN, [user] execute never. */
-
- /* TODO: This is only ok in EL1, it is RES0 in EL2. */
- attrs |= (1ull << 53); /* PXN, privileged execute never. */
- }
-
- /* TODO: This is different in s2. */
- if (mode & MM_W)
- attrs |= (0 << 6); /* rw, no EL0 access. */
- else
- attrs |= (2 << 6); /* read-only, no EL0 access. */
-
- if (mode & MM_D)
- attrs |= (0 << 2); /* device memory in MAIR_ELx. */
- else
- attrs |= (1 << 2); /* normal memory in MAIR_ELx. */
-
- return attrs;
-}
-
-static uint64_t *mm_populate_table(uint64_t *table, uint64_t index)
-{
- uint64_t *ntable;
- uint64_t v = table[index];
- uint64_t i;
-
- /* Check if table entry already exists. */
- if (v & 1) {
- /* Fail if it's a block one. */
- if (!(v & 2))
- return NULL;
- return mm_table_ptr(v);
- }
-
- /* Allocate a new table entry and initialize it. */
- ntable = halloc_aligned(PAGE_SIZE, PAGE_SIZE);
- if (!ntable)
- return NULL;
-
- for (i = 0; i < ENTRIES_PER_LEVEL; i++)
- ntable[i] = 0;
-
- /* Fill in the new entry. */
- table[index] = (size_t)ntable | 0x3;
-
- return ntable;
-}
-
-static bool mm_map_level(size_t va, size_t va_end, size_t pa,
- uint64_t attrs, uint64_t *table, int level)
-{
- size_t i = mm_index(va, level);
- size_t va_level_end = mm_level_end(va, level);
- size_t entry_size = mm_entry_size(level);
-
- /* Cap va_end so that we don't go over of the current level max. */
- if (va_end > va_level_end)
- va_end = va_level_end;
-
- /* Fill each entry in the table. */
- while (va < va_end) {
- if (level == 3) {
- table[i] = pa | 0x3 | attrs;
- } else {
- uint64_t *nt = mm_populate_table(table, i);
- if (!nt) {
- /* TODO: Undo all the work so far? */
- return false;
- }
-
- if (!mm_map_level(va, va_end, pa, attrs, nt, level+1)) {
- /* TODO: Undo all the work so far? */
- return false;
- }
- }
-
- va += entry_size;
- pa += entry_size;
- i++;
- }
-
- return true;
-}
-
-bool mm_map_range(size_t va, size_t size, uint64_t pa, uint64_t mode)
-{
- uint64_t attrs = mm_mode_to_attrs(mode);
- uint64_t end = mm_clear_attrs(va + size + PAGE_SIZE - 1);
-
- va = mm_clear_attrs(va);
- pa = mm_clear_attrs(pa);
-
- return mm_map_level(va, end, pa, attrs, ttbr, INITIAL_LEVEL);
-}
-
-bool mm_map_page(size_t va, size_t pa, uint64_t mode)
-{
- size_t i;
- uint64_t attrs = mm_mode_to_attrs(mode);
- uint64_t *table = ttbr;
-
- va = mm_clear_attrs(va);
- pa = mm_clear_attrs(pa);
- for (i = INITIAL_LEVEL; i < 3; i++) {
- table = mm_populate_table(table, mm_index(va, i));
- if (!table)
- return false;
- }
-
- /* We reached level 3. */
- i = mm_index(va, 3);
- table[i] = pa | 0x3 | attrs;
- return true;
-}
-
-bool arch_init_mm(void)
-{
-#if 0
- size_t i;
-
- /* Allocate the first level, then zero it out. */
- ttbr = halloc_aligned(PAGE_SIZE, PAGE_SIZE);
- if (!ttbr)
- return false;
-
- for (i = 0; i < ENTRIES_PER_LEVEL; i++)
- ttbr[i] = 0;
-
- /* Map page for uart. */
- mm_map_page(PL011_BASE, PL011_BASE, MM_R | MM_W | MM_D);
-
- /* Map page for gic. */
- mm_map_page(GICD_BASE, GICD_BASE, MM_R | MM_W | MM_D);
- mm_map_page(GICC_BASE, GICC_BASE, MM_R | MM_W | MM_D);
-
- /* Map each section. */
- mm_map_range((size_t)text_begin, text_end - text_begin,
- (size_t)text_begin, MM_X);
-
- mm_map_range((size_t)rodata_begin, rodata_end - rodata_begin,
- (size_t)rodata_begin, MM_R);
-
- mm_map_range((size_t)data_begin, data_end - data_begin,
- (size_t)data_begin, MM_R | MM_W);
-
- mm_map_range((size_t)bin_end, 20 * 1024 * 1024, (size_t)bin_end,
- MM_R | MM_W);
-#endif
- log(INFO, "About to enable mmu.\n");
- enable_mmu(ttbr);
- log(INFO, "mmu is on.\n");
-
- return true;
-}
-
-static void arch_mm_dump_table(uint64_t *table, int level)
-{
- uint64_t i, j;
- for (i = 0; i < ENTRIES_PER_LEVEL; i++) {
- if ((table[i] & 1) == 0)
- continue;
-
- for (j = 1 * (level - INITIAL_LEVEL + 1); j; j--)
- log(INFO, "\t");
- log(INFO, "%x: %x\n", i, table[i]);
- if (level >= 3)
- continue;
-
- if ((table[i] & 3) == 3)
- arch_mm_dump_table(mm_table_ptr(table[i]), level + 1);
- }
-}
-
-void enable_mmu(uint64_t *table)
-{
- //uint32_t v;
-
- enable_s2();
-#if 0
/*
- * 0 -> Device-nGnRnE memory
- * 1 -> Normal memory, Inner/Outer Write-Back Non-transient,
- * Write-Alloc, Read-Alloc.
+ * 0 -> Device-nGnRnE memory
+ * 0xff -> Normal memory, Inner/Outer Write-Back Non-transient,
+ * Write-Alloc, Read-Alloc.
*/
- write_msr(mair_el2, 0xff00);
+ write_msr(mair_el2,
+ (0 << (8 * STAGE1_DEVICEINDX)) |
+ (0xff << (8 * STAGE1_NORMALINDX)));
+
write_msr(ttbr0_el2, table);
/*
@@ -308,8 +170,7 @@
v =
(1 << 0) | /* M, enable stage 1 EL2 MMU. */
(1 << 1) | /* A, enable alignment check faults. */
- // TODO: Enable this.
-// (1 << 2) | /* C, data cache enable. */
+ (1 << 2) | /* C, data cache enable. */
(1 << 3) | /* SA, enable stack alignment check. */
(3 << 4) | /* RES1 bits. */
(1 << 11) | /* RES1 bit. */
@@ -325,6 +186,4 @@
__asm volatile("isb");
write_msr(sctlr_el2, v);
__asm volatile("isb");
-#endif
}
-#endif