Andrew Scull | 1883487 | 2018-10-12 11:48:09 +0100 | [diff] [blame] | 1 | /* |
Andrew Walbran | 692b325 | 2019-03-07 15:51:31 +0000 | [diff] [blame] | 2 | * Copyright 2018 The Hafnium Authors. |
Andrew Scull | 1883487 | 2018-10-12 11:48:09 +0100 | [diff] [blame] | 3 | * |
Andrew Walbran | e959ec1 | 2020-06-17 15:01:09 +0100 | [diff] [blame] | 4 | * Use of this source code is governed by a BSD-style |
| 5 | * license that can be found in the LICENSE file or at |
| 6 | * https://opensource.org/licenses/BSD-3-Clause. |
Andrew Scull | 1883487 | 2018-10-12 11:48:09 +0100 | [diff] [blame] | 7 | */ |
| 8 | |
Andrew Scull | fbc938a | 2018-08-20 14:09:28 +0100 | [diff] [blame] | 9 | #pragma once |
Wedson Almeida Filho | fed6902 | 2018-07-11 15:39:12 +0100 | [diff] [blame] | 10 | |
Andrew Scull | 4e5f814 | 2018-10-12 14:37:19 +0100 | [diff] [blame] | 11 | #include <stdalign.h> |
Wedson Almeida Filho | fed6902 | 2018-07-11 15:39:12 +0100 | [diff] [blame] | 12 | #include <stdbool.h> |
J-Alves | 715d623 | 2023-02-16 16:33:28 +0000 | [diff] [blame] | 13 | #include <stddef.h> |
Wedson Almeida Filho | 84a30a0 | 2018-07-23 20:05:05 +0100 | [diff] [blame] | 14 | #include <stdint.h> |
Wedson Almeida Filho | fed6902 | 2018-07-11 15:39:12 +0100 | [diff] [blame] | 15 | |
Andrew Scull | 18c78fc | 2018-08-20 12:57:41 +0100 | [diff] [blame] | 16 | #include "hf/addr.h" |
Wedson Almeida Filho | 22d5eaa | 2018-12-16 00:38:49 +0000 | [diff] [blame] | 17 | #include "hf/mpool.h" |
Andrew Scull | 877ae4b | 2019-07-02 12:52:33 +0100 | [diff] [blame] | 18 | #include "hf/static_assert.h" |
Wedson Almeida Filho | fed6902 | 2018-07-11 15:39:12 +0100 | [diff] [blame] | 19 | |
Karl Meakin | 07a69ab | 2025-02-07 14:53:19 +0000 | [diff] [blame] | 20 | typedef uint32_t mm_mode_t; |
| 21 | typedef uint64_t mm_attr_t; |
| 22 | typedef uint8_t mm_level_t; |
| 23 | typedef uint16_t mm_asid_t; |
| 24 | |
Karl Meakin | 23122e1 | 2025-02-05 14:44:20 +0000 | [diff] [blame] | 25 | /* |
| 26 | * A page table entry (PTE) will take one of the following forms: |
| 27 | * |
| 28 | * 1. absent : There is no mapping. |
| 29 | * 2. invalid block : Represents a block that is not in the address space. |
| 30 | * 3. valid block : Represents a block that is in the address space. |
| 31 | * 4. table : Represents a reference to a table of PTEs. |
| 32 | * See Arm ARM, D8.3 (Translation table descriptor formats). |
| 33 | */ |
| 34 | enum mm_pte_type { |
| 35 | PTE_TYPE_ABSENT, |
| 36 | PTE_TYPE_INVALID_BLOCK, |
| 37 | PTE_TYPE_VALID_BLOCK, |
| 38 | PTE_TYPE_TABLE, |
| 39 | }; |
| 40 | |
Andrew Scull | c66a04d | 2018-12-07 13:41:56 +0000 | [diff] [blame] | 41 | /* Keep macro alignment */ |
| 42 | /* clang-format off */ |
| 43 | |
J-Alves | 715d623 | 2023-02-16 16:33:28 +0000 | [diff] [blame] | 44 | #define PAGE_SIZE ((size_t)(1 << PAGE_BITS)) |
Andrew Scull | 4e5f814 | 2018-10-12 14:37:19 +0100 | [diff] [blame] | 45 | #define MM_PTE_PER_PAGE (PAGE_SIZE / sizeof(pte_t)) |
| 46 | |
Andrew Scull | c66a04d | 2018-12-07 13:41:56 +0000 | [diff] [blame] | 47 | /* The following are arch-independent page mapping modes. */ |
Karl Meakin | 07a69ab | 2025-02-07 14:53:19 +0000 | [diff] [blame] | 48 | #define MM_MODE_R (1U << 0) /* read */ |
| 49 | #define MM_MODE_W (1U << 1) /* write */ |
| 50 | #define MM_MODE_X (1U << 2) /* execute */ |
| 51 | #define MM_MODE_D (1U << 3) /* device */ |
Andrew Scull | c66a04d | 2018-12-07 13:41:56 +0000 | [diff] [blame] | 52 | |
| 53 | /* |
| 54 | * Memory in stage-1 is either valid (present) or invalid (absent). |
| 55 | * |
| 56 | * Memory in stage-2 has more states to track sharing, borrowing and giving of |
| 57 | * memory. The states are made up of three parts: |
| 58 | * |
| 59 | * 1. V = valid/invalid : Whether the memory is part of the VM's address |
| 60 | * space. A fault will be generated if accessed when |
| 61 | * invalid. |
| 62 | * 2. O = owned/unowned : Whether the memory is owned by the VM. |
| 63 | * 3. X = exclusive/shared : Whether access is exclusive to the VM or shared |
| 64 | * with at most one other. |
| 65 | * |
| 66 | * These parts compose to form the following state: |
| 67 | * |
| 68 | * - V O X : Owner of memory with exclusive access. |
| 69 | * - V O !X : Owner of memory with access shared with at most one other VM. |
| 70 | * - V !O X : Borrower of memory with exclusive access. |
| 71 | * - V !O !X : Borrower of memory where access is shared with the owner. |
| 72 | * - !V O X : Owner of memory lent to a VM that has exclusive access. |
| 73 | * |
| 74 | * - !V O !X : Unused. Owner of shared memory always has access. |
Andrew Scull | 73b8954 | 2019-11-20 17:31:26 +0000 | [diff] [blame] | 75 | * - !V !O X : Unused. Next entry is used for invalid memory. |
Andrew Scull | c66a04d | 2018-12-07 13:41:56 +0000 | [diff] [blame] | 76 | * |
Andrew Scull | c66a04d | 2018-12-07 13:41:56 +0000 | [diff] [blame] | 77 | * - !V !O !X : Invalid memory. Memory is unrelated to the VM. |
| 78 | * |
| 79 | * Modes are selected so that owner of exclusive memory is the default. |
| 80 | */ |
Karl Meakin | 07a69ab | 2025-02-07 14:53:19 +0000 | [diff] [blame] | 81 | #define MM_MODE_INVALID (1U << 4) |
| 82 | #define MM_MODE_UNOWNED (1U << 5) |
| 83 | #define MM_MODE_SHARED (1U << 6) |
Raghu Krishnamurthy | 25daaea | 2021-02-10 13:19:16 -0800 | [diff] [blame] | 84 | |
| 85 | /* Map page as non-global. */ |
Karl Meakin | 07a69ab | 2025-02-07 14:53:19 +0000 | [diff] [blame] | 86 | #define MM_MODE_NG (1U << 8) |
| 87 | |
| 88 | /* Specifies if a mapping will be a user mapping(EL0). */ |
| 89 | #define MM_MODE_USER (1U << 9) |
Raghu Krishnamurthy | 25daaea | 2021-02-10 13:19:16 -0800 | [diff] [blame] | 90 | |
Andrew Scull | 73b8954 | 2019-11-20 17:31:26 +0000 | [diff] [blame] | 91 | /* The mask for a mode that is considered unmapped. */ |
| 92 | #define MM_MODE_UNMAPPED_MASK (MM_MODE_INVALID | MM_MODE_UNOWNED) |
| 93 | |
Andrew Scull | c66a04d | 2018-12-07 13:41:56 +0000 | [diff] [blame] | 94 | /* clang-format on */ |
| 95 | |
Karl Meakin | 1fd4b82 | 2025-02-01 17:13:47 +0000 | [diff] [blame] | 96 | struct mm_flags { |
| 97 | bool commit : 1; |
| 98 | bool unmap : 1; |
| 99 | bool stage1 : 1; |
| 100 | }; |
| 101 | |
Andrew Walbran | 5de9c3d | 2020-02-10 13:35:29 +0000 | [diff] [blame] | 102 | #define MM_PPOOL_ENTRY_SIZE sizeof(struct mm_page_table) |
| 103 | |
Andrew Scull | 4e5f814 | 2018-10-12 14:37:19 +0100 | [diff] [blame] | 104 | struct mm_page_table { |
| 105 | alignas(PAGE_SIZE) pte_t entries[MM_PTE_PER_PAGE]; |
| 106 | }; |
| 107 | static_assert(sizeof(struct mm_page_table) == PAGE_SIZE, |
| 108 | "A page table must take exactly one page."); |
| 109 | static_assert(alignof(struct mm_page_table) == PAGE_SIZE, |
| 110 | "A page table must be page aligned."); |
| 111 | |
Wedson Almeida Filho | fed6902 | 2018-07-11 15:39:12 +0100 | [diff] [blame] | 112 | struct mm_ptable { |
Raghu Krishnamurthy | 0132b51 | 2021-02-03 14:13:26 -0800 | [diff] [blame] | 113 | /** |
| 114 | * VMID/ASID associated with a page table. ASID 0 is reserved for use by |
| 115 | * the hypervisor. |
| 116 | */ |
Karl Meakin | 07a69ab | 2025-02-07 14:53:19 +0000 | [diff] [blame] | 117 | mm_asid_t id; |
Andrew Scull | 1ba470e | 2018-10-31 15:14:31 +0000 | [diff] [blame] | 118 | /** Address of the root of the page table. */ |
| 119 | paddr_t root; |
Wedson Almeida Filho | fed6902 | 2018-07-11 15:39:12 +0100 | [diff] [blame] | 120 | }; |
| 121 | |
David Brazdil | 711fbe9 | 2019-08-06 13:39:58 +0100 | [diff] [blame] | 122 | /** The type of addresses stored in the page table. */ |
| 123 | typedef uintvaddr_t ptable_addr_t; |
| 124 | |
Andrew Scull | ba79b0a | 2019-07-03 11:26:53 +0100 | [diff] [blame] | 125 | /** Represents the currently locked stage-1 page table of the hypervisor. */ |
Andrew Scull | 3c0a90a | 2019-07-01 11:55:53 +0100 | [diff] [blame] | 126 | struct mm_stage1_locked { |
| 127 | struct mm_ptable *ptable; |
| 128 | }; |
| 129 | |
Andrew Scull | da24197 | 2019-01-05 18:17:48 +0000 | [diff] [blame] | 130 | void mm_vm_enable_invalidation(void); |
| 131 | |
Karl Meakin | 07a69ab | 2025-02-07 14:53:19 +0000 | [diff] [blame] | 132 | bool mm_ptable_init(struct mm_ptable *ptable, mm_asid_t id, |
Karl Meakin | 1fd4b82 | 2025-02-01 17:13:47 +0000 | [diff] [blame] | 133 | struct mm_flags flags, struct mpool *ppool); |
| 134 | ptable_addr_t mm_ptable_addr_space_end(struct mm_flags flags); |
David Brazdil | 711fbe9 | 2019-08-06 13:39:58 +0100 | [diff] [blame] | 135 | |
Karl Meakin | 07a69ab | 2025-02-07 14:53:19 +0000 | [diff] [blame] | 136 | bool mm_vm_init(struct mm_ptable *ptable, mm_asid_t id, struct mpool *ppool); |
Karl Meakin | d64aaf8 | 2025-02-08 01:12:55 +0000 | [diff] [blame] | 137 | void mm_vm_fini(const struct mm_ptable *ptable, struct mpool *ppool); |
Raghu Krishnamurthy | 43fe93a | 2021-01-31 16:38:38 -0800 | [diff] [blame] | 138 | |
Karl Meakin | d64aaf8 | 2025-02-08 01:12:55 +0000 | [diff] [blame] | 139 | bool mm_identity_prepare(struct mm_ptable *ptable, paddr_t begin, paddr_t end, |
Karl Meakin | 07a69ab | 2025-02-07 14:53:19 +0000 | [diff] [blame] | 140 | mm_mode_t mode, struct mpool *ppool); |
Karl Meakin | d64aaf8 | 2025-02-08 01:12:55 +0000 | [diff] [blame] | 141 | void *mm_identity_commit(struct mm_ptable *ptable, paddr_t begin, paddr_t end, |
Karl Meakin | 07a69ab | 2025-02-07 14:53:19 +0000 | [diff] [blame] | 142 | mm_mode_t mode, struct mpool *ppool); |
Raghu Krishnamurthy | 43fe93a | 2021-01-31 16:38:38 -0800 | [diff] [blame] | 143 | |
Karl Meakin | d64aaf8 | 2025-02-08 01:12:55 +0000 | [diff] [blame] | 144 | bool mm_vm_identity_map(struct mm_ptable *ptable, paddr_t begin, paddr_t end, |
Karl Meakin | 07a69ab | 2025-02-07 14:53:19 +0000 | [diff] [blame] | 145 | mm_mode_t mode, struct mpool *ppool, ipaddr_t *ipa); |
Karl Meakin | d64aaf8 | 2025-02-08 01:12:55 +0000 | [diff] [blame] | 146 | bool mm_vm_identity_prepare(struct mm_ptable *ptable, paddr_t begin, |
Karl Meakin | 07a69ab | 2025-02-07 14:53:19 +0000 | [diff] [blame] | 147 | paddr_t end, mm_mode_t mode, struct mpool *ppool); |
Karl Meakin | d64aaf8 | 2025-02-08 01:12:55 +0000 | [diff] [blame] | 148 | void mm_vm_identity_commit(struct mm_ptable *ptable, paddr_t begin, paddr_t end, |
Karl Meakin | 07a69ab | 2025-02-07 14:53:19 +0000 | [diff] [blame] | 149 | mm_mode_t mode, struct mpool *ppool, ipaddr_t *ipa); |
Karl Meakin | d64aaf8 | 2025-02-08 01:12:55 +0000 | [diff] [blame] | 150 | bool mm_vm_unmap(struct mm_ptable *ptable, paddr_t begin, paddr_t end, |
Wedson Almeida Filho | 22d5eaa | 2018-12-16 00:38:49 +0000 | [diff] [blame] | 151 | struct mpool *ppool); |
Karl Meakin | d64aaf8 | 2025-02-08 01:12:55 +0000 | [diff] [blame] | 152 | void mm_stage1_defrag(struct mm_ptable *ptable, struct mpool *ppool); |
| 153 | void mm_vm_defrag(struct mm_ptable *ptable, struct mpool *ppool, |
| 154 | bool non_secure); |
| 155 | void mm_vm_dump(const struct mm_ptable *ptable); |
| 156 | bool mm_vm_get_mode(const struct mm_ptable *ptable, ipaddr_t begin, |
Karl Meakin | 07a69ab | 2025-02-07 14:53:19 +0000 | [diff] [blame] | 157 | ipaddr_t end, mm_mode_t *mode); |
Karl Meakin | d64aaf8 | 2025-02-08 01:12:55 +0000 | [diff] [blame] | 158 | bool mm_get_mode(const struct mm_ptable *ptable, vaddr_t begin, vaddr_t end, |
Karl Meakin | 07a69ab | 2025-02-07 14:53:19 +0000 | [diff] [blame] | 159 | mm_mode_t *mode); |
Andrew Scull | 8087132 | 2018-08-06 12:04:09 +0100 | [diff] [blame] | 160 | |
Raghu Krishnamurthy | d3ab8c3 | 2021-02-10 19:11:30 -0800 | [diff] [blame] | 161 | struct mm_stage1_locked mm_lock_ptable_unsafe(struct mm_ptable *ptable); |
Andrew Scull | 3c0a90a | 2019-07-01 11:55:53 +0100 | [diff] [blame] | 162 | struct mm_stage1_locked mm_lock_stage1(void); |
| 163 | void mm_unlock_stage1(struct mm_stage1_locked *lock); |
| 164 | void *mm_identity_map(struct mm_stage1_locked stage1_locked, paddr_t begin, |
Karl Meakin | 07a69ab | 2025-02-07 14:53:19 +0000 | [diff] [blame] | 165 | paddr_t end, mm_mode_t mode, struct mpool *ppool); |
Andrew Scull | 3c0a90a | 2019-07-01 11:55:53 +0100 | [diff] [blame] | 166 | bool mm_unmap(struct mm_stage1_locked stage1_locked, paddr_t begin, paddr_t end, |
| 167 | struct mpool *ppool); |
| 168 | void mm_defrag(struct mm_stage1_locked stage1_locked, struct mpool *ppool); |
| 169 | |
Wedson Almeida Filho | 22d5eaa | 2018-12-16 00:38:49 +0000 | [diff] [blame] | 170 | bool mm_init(struct mpool *ppool); |