Andrew Scull | 1883487 | 2018-10-12 11:48:09 +0100 | [diff] [blame] | 1 | /* |
Andrew Walbran | 692b325 | 2019-03-07 15:51:31 +0000 | [diff] [blame] | 2 | * Copyright 2018 The Hafnium Authors. |
Andrew Scull | 1883487 | 2018-10-12 11:48:09 +0100 | [diff] [blame] | 3 | * |
Andrew Walbran | e959ec1 | 2020-06-17 15:01:09 +0100 | [diff] [blame] | 4 | * Use of this source code is governed by a BSD-style |
| 5 | * license that can be found in the LICENSE file or at |
| 6 | * https://opensource.org/licenses/BSD-3-Clause. |
Andrew Scull | 1883487 | 2018-10-12 11:48:09 +0100 | [diff] [blame] | 7 | */ |
| 8 | |
Andrew Scull | fbc938a | 2018-08-20 14:09:28 +0100 | [diff] [blame] | 9 | #pragma once |
Wedson Almeida Filho | fed6902 | 2018-07-11 15:39:12 +0100 | [diff] [blame] | 10 | |
Andrew Scull | 4e5f814 | 2018-10-12 14:37:19 +0100 | [diff] [blame] | 11 | #include <stdalign.h> |
Wedson Almeida Filho | fed6902 | 2018-07-11 15:39:12 +0100 | [diff] [blame] | 12 | #include <stdbool.h> |
J-Alves | 715d623 | 2023-02-16 16:33:28 +0000 | [diff] [blame] | 13 | #include <stddef.h> |
Wedson Almeida Filho | 84a30a0 | 2018-07-23 20:05:05 +0100 | [diff] [blame] | 14 | #include <stdint.h> |
Wedson Almeida Filho | fed6902 | 2018-07-11 15:39:12 +0100 | [diff] [blame] | 15 | |
Andrew Scull | 18c78fc | 2018-08-20 12:57:41 +0100 | [diff] [blame] | 16 | #include "hf/addr.h" |
Wedson Almeida Filho | 22d5eaa | 2018-12-16 00:38:49 +0000 | [diff] [blame] | 17 | #include "hf/mpool.h" |
Andrew Scull | 877ae4b | 2019-07-02 12:52:33 +0100 | [diff] [blame] | 18 | #include "hf/static_assert.h" |
Wedson Almeida Filho | fed6902 | 2018-07-11 15:39:12 +0100 | [diff] [blame] | 19 | |
Karl Meakin | 07a69ab | 2025-02-07 14:53:19 +0000 | [diff] [blame] | 20 | typedef uint32_t mm_mode_t; |
| 21 | typedef uint64_t mm_attr_t; |
Karl Meakin | a3a9f95 | 2025-02-08 00:11:16 +0000 | [diff] [blame] | 22 | |
| 23 | /** |
| 24 | * The level of a page table entry (i.e. how deep into the recursive tree |
| 25 | * structure it is). See also Arm ARM, table D8-14. |
| 26 | * |
| 27 | * - `level == 4`: table entries (root) |
| 28 | * - `level == 3`: table or block entries |
| 29 | * - `level == 2`: table or block entries |
| 30 | * - `level == 1`: table or block entries |
| 31 | * - `level == 0`: page entries |
| 32 | * |
| 33 | * NOTE: The Arm ARM uses levels in the opposite order to our code: in the Arm |
| 34 | * ARM, levels start at 0 (or -1 if 52 bits of PA are used, but that is not |
| 35 | * supported by Hafnium) and page entries are at level 3. We go in the opposite |
| 36 | * direction: levels start at 3 or 4 and page entries are at level 0. This is |
| 37 | * because it makes the arithmetic and bit manipulation easier. |
| 38 | */ |
Karl Meakin | 07a69ab | 2025-02-07 14:53:19 +0000 | [diff] [blame] | 39 | typedef uint8_t mm_level_t; |
| 40 | typedef uint16_t mm_asid_t; |
| 41 | |
Karl Meakin | 23122e1 | 2025-02-05 14:44:20 +0000 | [diff] [blame] | 42 | /* |
| 43 | * A page table entry (PTE) will take one of the following forms: |
| 44 | * |
| 45 | * 1. absent : There is no mapping. |
| 46 | * 2. invalid block : Represents a block that is not in the address space. |
| 47 | * 3. valid block : Represents a block that is in the address space. |
| 48 | * 4. table : Represents a reference to a table of PTEs. |
| 49 | * See Arm ARM, D8.3 (Translation table descriptor formats). |
| 50 | */ |
| 51 | enum mm_pte_type { |
| 52 | PTE_TYPE_ABSENT, |
| 53 | PTE_TYPE_INVALID_BLOCK, |
| 54 | PTE_TYPE_VALID_BLOCK, |
| 55 | PTE_TYPE_TABLE, |
| 56 | }; |
| 57 | |
Andrew Scull | c66a04d | 2018-12-07 13:41:56 +0000 | [diff] [blame] | 58 | /* Keep macro alignment */ |
| 59 | /* clang-format off */ |
| 60 | |
J-Alves | 715d623 | 2023-02-16 16:33:28 +0000 | [diff] [blame] | 61 | #define PAGE_SIZE ((size_t)(1 << PAGE_BITS)) |
Andrew Scull | 4e5f814 | 2018-10-12 14:37:19 +0100 | [diff] [blame] | 62 | #define MM_PTE_PER_PAGE (PAGE_SIZE / sizeof(pte_t)) |
| 63 | |
Andrew Scull | c66a04d | 2018-12-07 13:41:56 +0000 | [diff] [blame] | 64 | /* The following are arch-independent page mapping modes. */ |
Karl Meakin | 07a69ab | 2025-02-07 14:53:19 +0000 | [diff] [blame] | 65 | #define MM_MODE_R (1U << 0) /* read */ |
| 66 | #define MM_MODE_W (1U << 1) /* write */ |
| 67 | #define MM_MODE_X (1U << 2) /* execute */ |
| 68 | #define MM_MODE_D (1U << 3) /* device */ |
Andrew Scull | c66a04d | 2018-12-07 13:41:56 +0000 | [diff] [blame] | 69 | |
| 70 | /* |
| 71 | * Memory in stage-1 is either valid (present) or invalid (absent). |
| 72 | * |
| 73 | * Memory in stage-2 has more states to track sharing, borrowing and giving of |
| 74 | * memory. The states are made up of three parts: |
| 75 | * |
| 76 | * 1. V = valid/invalid : Whether the memory is part of the VM's address |
| 77 | * space. A fault will be generated if accessed when |
| 78 | * invalid. |
| 79 | * 2. O = owned/unowned : Whether the memory is owned by the VM. |
| 80 | * 3. X = exclusive/shared : Whether access is exclusive to the VM or shared |
| 81 | * with at most one other. |
| 82 | * |
| 83 | * These parts compose to form the following state: |
| 84 | * |
| 85 | * - V O X : Owner of memory with exclusive access. |
| 86 | * - V O !X : Owner of memory with access shared with at most one other VM. |
| 87 | * - V !O X : Borrower of memory with exclusive access. |
| 88 | * - V !O !X : Borrower of memory where access is shared with the owner. |
| 89 | * - !V O X : Owner of memory lent to a VM that has exclusive access. |
| 90 | * |
| 91 | * - !V O !X : Unused. Owner of shared memory always has access. |
Andrew Scull | 73b8954 | 2019-11-20 17:31:26 +0000 | [diff] [blame] | 92 | * - !V !O X : Unused. Next entry is used for invalid memory. |
Andrew Scull | c66a04d | 2018-12-07 13:41:56 +0000 | [diff] [blame] | 93 | * |
Andrew Scull | c66a04d | 2018-12-07 13:41:56 +0000 | [diff] [blame] | 94 | * - !V !O !X : Invalid memory. Memory is unrelated to the VM. |
| 95 | * |
| 96 | * Modes are selected so that owner of exclusive memory is the default. |
| 97 | */ |
Karl Meakin | 07a69ab | 2025-02-07 14:53:19 +0000 | [diff] [blame] | 98 | #define MM_MODE_INVALID (1U << 4) |
| 99 | #define MM_MODE_UNOWNED (1U << 5) |
| 100 | #define MM_MODE_SHARED (1U << 6) |
Raghu Krishnamurthy | 25daaea | 2021-02-10 13:19:16 -0800 | [diff] [blame] | 101 | |
| 102 | /* Map page as non-global. */ |
Karl Meakin | 07a69ab | 2025-02-07 14:53:19 +0000 | [diff] [blame] | 103 | #define MM_MODE_NG (1U << 8) |
| 104 | |
| 105 | /* Specifies if a mapping will be a user mapping(EL0). */ |
| 106 | #define MM_MODE_USER (1U << 9) |
Raghu Krishnamurthy | 25daaea | 2021-02-10 13:19:16 -0800 | [diff] [blame] | 107 | |
Andrew Scull | 73b8954 | 2019-11-20 17:31:26 +0000 | [diff] [blame] | 108 | /* The mask for a mode that is considered unmapped. */ |
| 109 | #define MM_MODE_UNMAPPED_MASK (MM_MODE_INVALID | MM_MODE_UNOWNED) |
| 110 | |
Andrew Scull | c66a04d | 2018-12-07 13:41:56 +0000 | [diff] [blame] | 111 | /* clang-format on */ |
| 112 | |
Karl Meakin | 1fd4b82 | 2025-02-01 17:13:47 +0000 | [diff] [blame] | 113 | struct mm_flags { |
| 114 | bool commit : 1; |
| 115 | bool unmap : 1; |
| 116 | bool stage1 : 1; |
| 117 | }; |
| 118 | |
Andrew Walbran | 5de9c3d | 2020-02-10 13:35:29 +0000 | [diff] [blame] | 119 | #define MM_PPOOL_ENTRY_SIZE sizeof(struct mm_page_table) |
| 120 | |
Andrew Scull | 4e5f814 | 2018-10-12 14:37:19 +0100 | [diff] [blame] | 121 | struct mm_page_table { |
| 122 | alignas(PAGE_SIZE) pte_t entries[MM_PTE_PER_PAGE]; |
| 123 | }; |
| 124 | static_assert(sizeof(struct mm_page_table) == PAGE_SIZE, |
| 125 | "A page table must take exactly one page."); |
| 126 | static_assert(alignof(struct mm_page_table) == PAGE_SIZE, |
| 127 | "A page table must be page aligned."); |
| 128 | |
Wedson Almeida Filho | fed6902 | 2018-07-11 15:39:12 +0100 | [diff] [blame] | 129 | struct mm_ptable { |
Raghu Krishnamurthy | 0132b51 | 2021-02-03 14:13:26 -0800 | [diff] [blame] | 130 | /** |
| 131 | * VMID/ASID associated with a page table. ASID 0 is reserved for use by |
| 132 | * the hypervisor. |
| 133 | */ |
Karl Meakin | 07a69ab | 2025-02-07 14:53:19 +0000 | [diff] [blame] | 134 | mm_asid_t id; |
Karl Meakin | e1aeb1d | 2025-02-08 00:35:14 +0000 | [diff] [blame] | 135 | /** |
| 136 | * Address of the root tables. |
| 137 | * At stage 1, concatenated tables are not used, so there is only one |
| 138 | * root table. |
| 139 | * At stage 2, concatenated tables are used, so there are multiple root |
| 140 | * tables (given by `arch_mm_root_table_count()`). The Arm ARM says |
| 141 | * there can be up to 16 root tables, but we only use 4. |
| 142 | */ |
| 143 | struct mm_page_table *root_tables; |
Wedson Almeida Filho | fed6902 | 2018-07-11 15:39:12 +0100 | [diff] [blame] | 144 | }; |
| 145 | |
David Brazdil | 711fbe9 | 2019-08-06 13:39:58 +0100 | [diff] [blame] | 146 | /** The type of addresses stored in the page table. */ |
| 147 | typedef uintvaddr_t ptable_addr_t; |
| 148 | |
Andrew Scull | ba79b0a | 2019-07-03 11:26:53 +0100 | [diff] [blame] | 149 | /** Represents the currently locked stage-1 page table of the hypervisor. */ |
Andrew Scull | 3c0a90a | 2019-07-01 11:55:53 +0100 | [diff] [blame] | 150 | struct mm_stage1_locked { |
| 151 | struct mm_ptable *ptable; |
| 152 | }; |
| 153 | |
Andrew Scull | da24197 | 2019-01-05 18:17:48 +0000 | [diff] [blame] | 154 | void mm_vm_enable_invalidation(void); |
| 155 | |
Karl Meakin | 07a69ab | 2025-02-07 14:53:19 +0000 | [diff] [blame] | 156 | bool mm_ptable_init(struct mm_ptable *ptable, mm_asid_t id, |
Karl Meakin | 1fd4b82 | 2025-02-01 17:13:47 +0000 | [diff] [blame] | 157 | struct mm_flags flags, struct mpool *ppool); |
| 158 | ptable_addr_t mm_ptable_addr_space_end(struct mm_flags flags); |
David Brazdil | 711fbe9 | 2019-08-06 13:39:58 +0100 | [diff] [blame] | 159 | |
Karl Meakin | 07a69ab | 2025-02-07 14:53:19 +0000 | [diff] [blame] | 160 | bool mm_vm_init(struct mm_ptable *ptable, mm_asid_t id, struct mpool *ppool); |
Karl Meakin | d64aaf8 | 2025-02-08 01:12:55 +0000 | [diff] [blame] | 161 | void mm_vm_fini(const struct mm_ptable *ptable, struct mpool *ppool); |
Raghu Krishnamurthy | 43fe93a | 2021-01-31 16:38:38 -0800 | [diff] [blame] | 162 | |
Karl Meakin | d64aaf8 | 2025-02-08 01:12:55 +0000 | [diff] [blame] | 163 | bool mm_identity_prepare(struct mm_ptable *ptable, paddr_t begin, paddr_t end, |
Karl Meakin | 07a69ab | 2025-02-07 14:53:19 +0000 | [diff] [blame] | 164 | mm_mode_t mode, struct mpool *ppool); |
Karl Meakin | d64aaf8 | 2025-02-08 01:12:55 +0000 | [diff] [blame] | 165 | void *mm_identity_commit(struct mm_ptable *ptable, paddr_t begin, paddr_t end, |
Karl Meakin | 07a69ab | 2025-02-07 14:53:19 +0000 | [diff] [blame] | 166 | mm_mode_t mode, struct mpool *ppool); |
Raghu Krishnamurthy | 43fe93a | 2021-01-31 16:38:38 -0800 | [diff] [blame] | 167 | |
Karl Meakin | d64aaf8 | 2025-02-08 01:12:55 +0000 | [diff] [blame] | 168 | bool mm_vm_identity_map(struct mm_ptable *ptable, paddr_t begin, paddr_t end, |
Karl Meakin | 07a69ab | 2025-02-07 14:53:19 +0000 | [diff] [blame] | 169 | mm_mode_t mode, struct mpool *ppool, ipaddr_t *ipa); |
Karl Meakin | d64aaf8 | 2025-02-08 01:12:55 +0000 | [diff] [blame] | 170 | bool mm_vm_identity_prepare(struct mm_ptable *ptable, paddr_t begin, |
Karl Meakin | 07a69ab | 2025-02-07 14:53:19 +0000 | [diff] [blame] | 171 | paddr_t end, mm_mode_t mode, struct mpool *ppool); |
Karl Meakin | d64aaf8 | 2025-02-08 01:12:55 +0000 | [diff] [blame] | 172 | void mm_vm_identity_commit(struct mm_ptable *ptable, paddr_t begin, paddr_t end, |
Karl Meakin | 07a69ab | 2025-02-07 14:53:19 +0000 | [diff] [blame] | 173 | mm_mode_t mode, struct mpool *ppool, ipaddr_t *ipa); |
Karl Meakin | d64aaf8 | 2025-02-08 01:12:55 +0000 | [diff] [blame] | 174 | bool mm_vm_unmap(struct mm_ptable *ptable, paddr_t begin, paddr_t end, |
Wedson Almeida Filho | 22d5eaa | 2018-12-16 00:38:49 +0000 | [diff] [blame] | 175 | struct mpool *ppool); |
Karl Meakin | d64aaf8 | 2025-02-08 01:12:55 +0000 | [diff] [blame] | 176 | void mm_stage1_defrag(struct mm_ptable *ptable, struct mpool *ppool); |
| 177 | void mm_vm_defrag(struct mm_ptable *ptable, struct mpool *ppool, |
| 178 | bool non_secure); |
| 179 | void mm_vm_dump(const struct mm_ptable *ptable); |
| 180 | bool mm_vm_get_mode(const struct mm_ptable *ptable, ipaddr_t begin, |
Karl Meakin | 07a69ab | 2025-02-07 14:53:19 +0000 | [diff] [blame] | 181 | ipaddr_t end, mm_mode_t *mode); |
Karl Meakin | d64aaf8 | 2025-02-08 01:12:55 +0000 | [diff] [blame] | 182 | bool mm_get_mode(const struct mm_ptable *ptable, vaddr_t begin, vaddr_t end, |
Karl Meakin | 07a69ab | 2025-02-07 14:53:19 +0000 | [diff] [blame] | 183 | mm_mode_t *mode); |
Andrew Scull | 8087132 | 2018-08-06 12:04:09 +0100 | [diff] [blame] | 184 | |
Raghu Krishnamurthy | d3ab8c3 | 2021-02-10 19:11:30 -0800 | [diff] [blame] | 185 | struct mm_stage1_locked mm_lock_ptable_unsafe(struct mm_ptable *ptable); |
Andrew Scull | 3c0a90a | 2019-07-01 11:55:53 +0100 | [diff] [blame] | 186 | struct mm_stage1_locked mm_lock_stage1(void); |
| 187 | void mm_unlock_stage1(struct mm_stage1_locked *lock); |
| 188 | void *mm_identity_map(struct mm_stage1_locked stage1_locked, paddr_t begin, |
Karl Meakin | 07a69ab | 2025-02-07 14:53:19 +0000 | [diff] [blame] | 189 | paddr_t end, mm_mode_t mode, struct mpool *ppool); |
Andrew Scull | 3c0a90a | 2019-07-01 11:55:53 +0100 | [diff] [blame] | 190 | bool mm_unmap(struct mm_stage1_locked stage1_locked, paddr_t begin, paddr_t end, |
| 191 | struct mpool *ppool); |
| 192 | void mm_defrag(struct mm_stage1_locked stage1_locked, struct mpool *ppool); |
| 193 | |
Wedson Almeida Filho | 22d5eaa | 2018-12-16 00:38:49 +0000 | [diff] [blame] | 194 | bool mm_init(struct mpool *ppool); |