Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 1 | /* |
| 2 | * SPDX-License-Identifier: BSD-3-Clause |
| 3 | * SPDX-FileCopyrightText: Copyright TF-RMM Contributors. |
| 4 | */ |
| 5 | |
| 6 | #include <arch_helpers.h> |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 7 | #include <bitmap.h> |
| 8 | #include <buffer.h> |
| 9 | #include <gic.h> |
| 10 | #include <granule.h> |
| 11 | #include <memory_alloc.h> |
| 12 | #include <realm.h> |
| 13 | #include <ripas.h> |
| 14 | #include <smc.h> |
| 15 | #include <status.h> |
| 16 | #include <stddef.h> |
| 17 | #include <string.h> |
| 18 | #include <table.h> |
| 19 | |
| 20 | /* |
| 21 | * For prototyping we assume 4K pages |
| 22 | */ |
| 23 | #define BLOCK_L2_SIZE (GRANULE_SIZE * S2TTES_PER_S2TT) |
| 24 | |
| 25 | /* |
| 26 | * The maximum number of bits supported by the RMM for a stage 2 translation |
| 27 | * output address (including stage 2 table entries). |
| 28 | */ |
AlexeiFedorov | 93f5ec5 | 2023-08-31 14:26:53 +0100 | [diff] [blame] | 29 | #define S2TTE_OA_BITS 48U |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 30 | |
AlexeiFedorov | c53b1f7 | 2023-07-04 15:37:03 +0100 | [diff] [blame] | 31 | #define DESC_TYPE_MASK 3UL |
| 32 | #define S2TTE_Lx_INVALID 0UL |
| 33 | #define S2TTE_L012_BLOCK 1UL |
| 34 | #define S2TTE_L012_TABLE 3UL |
| 35 | #define S2TTE_L3_PAGE 3UL |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 36 | |
| 37 | /* |
| 38 | * The following constants for the mapping attributes (S2_TTE_MEMATTR_*) |
| 39 | * assume that HCR_EL2.FWB is set. |
| 40 | */ |
| 41 | #define S2TTE_MEMATTR_SHIFT 2 |
| 42 | #define S2TTE_MEMATTR_MASK (0x7UL << S2TTE_MEMATTR_SHIFT) |
| 43 | #define S2TTE_MEMATTR_FWB_NORMAL_WB ((1UL << 4) | (2UL << 2)) |
| 44 | #define S2TTE_MEMATTR_FWB_RESERVED ((1UL << 4) | (0UL << 2)) |
| 45 | |
| 46 | #define S2TTE_AP_SHIFT 6 |
| 47 | #define S2TTE_AP_MASK (3UL << S2TTE_AP_SHIFT) |
| 48 | #define S2TTE_AP_RW (3UL << S2TTE_AP_SHIFT) |
| 49 | |
| 50 | #define S2TTE_SH_SHIFT 8 |
| 51 | #define S2TTE_SH_MASK (3UL << S2TTE_SH_SHIFT) |
| 52 | #define S2TTE_SH_NS (0UL << S2TTE_SH_SHIFT) |
| 53 | #define S2TTE_SH_RESERVED (1UL << S2TTE_SH_SHIFT) |
| 54 | #define S2TTE_SH_OS (2UL << S2TTE_SH_SHIFT) |
| 55 | #define S2TTE_SH_IS (3UL << S2TTE_SH_SHIFT) /* Inner Shareable */ |
| 56 | |
| 57 | /* |
| 58 | * We set HCR_EL2.FWB So we set bit[4] to 1 and bits[3:2] to 2 and force |
| 59 | * everyting to be Normal Write-Back |
| 60 | */ |
| 61 | #define S2TTE_MEMATTR_FWB_NORMAL_WB ((1UL << 4) | (2UL << 2)) |
| 62 | #define S2TTE_AF (1UL << 10) |
| 63 | #define S2TTE_XN (2UL << 53) |
| 64 | #define S2TTE_NS (1UL << 55) |
| 65 | |
| 66 | #define S2TTE_ATTRS (S2TTE_MEMATTR_FWB_NORMAL_WB | S2TTE_AP_RW | \ |
| 67 | S2TTE_SH_IS | S2TTE_AF) |
Javier Almansa Sobrino | 15fc44e | 2023-09-29 13:52:04 +0100 | [diff] [blame] | 68 | #define S2TTE_NS_ATTR_MASK (S2TTE_MEMATTR_MASK | S2TTE_AP_MASK | \ |
| 69 | S2TTE_SH_MASK) |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 70 | |
| 71 | #define S2TTE_TABLE S2TTE_L012_TABLE |
| 72 | #define S2TTE_BLOCK (S2TTE_ATTRS | S2TTE_L012_BLOCK) |
| 73 | #define S2TTE_PAGE (S2TTE_ATTRS | S2TTE_L3_PAGE) |
| 74 | #define S2TTE_BLOCK_NS (S2TTE_NS | S2TTE_XN | S2TTE_AF | S2TTE_L012_BLOCK) |
| 75 | #define S2TTE_PAGE_NS (S2TTE_NS | S2TTE_XN | S2TTE_AF | S2TTE_L3_PAGE) |
AlexeiFedorov | c53b1f7 | 2023-07-04 15:37:03 +0100 | [diff] [blame] | 76 | #define S2TTE_INVALID S2TTE_Lx_INVALID |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 77 | |
| 78 | /* |
AlexeiFedorov | 5ceff35 | 2023-04-12 16:17:00 +0100 | [diff] [blame] | 79 | * The type of stage 2 translation table entry (s2tte) is defined by: |
| 80 | * 1. Table level where it resides |
| 81 | * 2. DESC_TYPE field[1:0] |
AlexeiFedorov | c53b1f7 | 2023-07-04 15:37:03 +0100 | [diff] [blame] | 82 | * 4. HIPAS field [4:2] |
| 83 | * 4. RIPAS field [6:5] |
AlexeiFedorov | 5ceff35 | 2023-04-12 16:17:00 +0100 | [diff] [blame] | 84 | * 5. NS field [55] |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 85 | * |
AlexeiFedorov | 63614ea | 2023-07-14 17:07:20 +0100 | [diff] [blame] | 86 | * ====================================================================================== |
| 87 | * s2tte type level DESC_TYPE[1:0] HIPAS[4:2] RIPAS[6:5] NS OA alignment |
| 88 | * ====================================================================================== |
| 89 | * unassigned_empty any invalid[0] unassigned[0] empty[0] 0 n/a |
| 90 | * -------------------------------------------------------------------------------------- |
| 91 | * unassigned_ram any invalid[0] unassigned[0] ram[1] 0 n/a |
| 92 | * -------------------------------------------------------------------------------------- |
| 93 | * unassigned_destroyed any invalid[0] unassigned[0] destroyed[2] 0 n/a |
| 94 | * -------------------------------------------------------------------------------------- |
| 95 | * assigned_empty 2,3 invalid[0] assigned[1] empty[0] 0 to level |
| 96 | * -------------------------------------------------------------------------------------- |
| 97 | * assigned_ram 3 page[3] n/a n/a 0 to level |
| 98 | * 2 block[1] n/a n/a 0 to level |
| 99 | * -------------------------------------------------------------------------------------- |
| 100 | * assigned_destroyed any invalid[0] assigned[1] destroyed[2] 0 n/a |
| 101 | * ====================================================================================== |
| 102 | * unassigned_ns any invalid[0] unassigned[0] n/a 1 n/a |
| 103 | * -------------------------------------------------------------------------------------- |
| 104 | * assigned_ns 3 page[3] n/a n/a 1 to level |
| 105 | * 2 block[1] n/a n/a 1 to level |
| 106 | * ====================================================================================== |
| 107 | * table <=2 table[3] n/a n/a n/a to 4K |
| 108 | * ====================================================================================== |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 109 | */ |
| 110 | |
| 111 | #define S2TTE_INVALID_HIPAS_SHIFT 2 |
AlexeiFedorov | 93f5ec5 | 2023-08-31 14:26:53 +0100 | [diff] [blame] | 112 | #define S2TTE_INVALID_HIPAS_WIDTH 3U |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 113 | #define S2TTE_INVALID_HIPAS_MASK MASK(S2TTE_INVALID_HIPAS) |
| 114 | |
AlexeiFedorov | 93f5ec5 | 2023-08-31 14:26:53 +0100 | [diff] [blame] | 115 | #define S2TTE_INVALID_HIPAS_UNASSIGNED (INPLACE(S2TTE_INVALID_HIPAS, 0UL)) |
| 116 | #define S2TTE_INVALID_HIPAS_ASSIGNED (INPLACE(S2TTE_INVALID_HIPAS, 1UL)) |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 117 | |
AlexeiFedorov | c53b1f7 | 2023-07-04 15:37:03 +0100 | [diff] [blame] | 118 | #define S2TTE_INVALID_RIPAS_SHIFT 5 |
AlexeiFedorov | 93f5ec5 | 2023-08-31 14:26:53 +0100 | [diff] [blame] | 119 | #define S2TTE_INVALID_RIPAS_WIDTH 2U |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 120 | #define S2TTE_INVALID_RIPAS_MASK MASK(S2TTE_INVALID_RIPAS) |
| 121 | |
AlexeiFedorov | 93f5ec5 | 2023-08-31 14:26:53 +0100 | [diff] [blame] | 122 | #define S2TTE_INVALID_RIPAS_EMPTY (INPLACE(S2TTE_INVALID_RIPAS, 0UL)) |
| 123 | #define S2TTE_INVALID_RIPAS_RAM (INPLACE(S2TTE_INVALID_RIPAS, 1UL)) |
| 124 | #define S2TTE_INVALID_RIPAS_DESTROYED (INPLACE(S2TTE_INVALID_RIPAS, 2UL)) |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 125 | |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 126 | #define S2TTE_INVALID_UNPROTECTED 0x0UL |
| 127 | |
| 128 | #define NR_RTT_LEVELS 4 |
| 129 | |
| 130 | /* |
| 131 | * Invalidates S2 TLB entries from [ipa, ipa + size] region tagged with `vmid`. |
| 132 | */ |
| 133 | static void stage2_tlbi_ipa(const struct realm_s2_context *s2_ctx, |
| 134 | unsigned long ipa, |
| 135 | unsigned long size) |
| 136 | { |
| 137 | /* |
| 138 | * Notes: |
| 139 | * |
| 140 | * - This follows the description provided in the Arm ARM on |
| 141 | * "Invalidation of TLB entries from stage 2 translations". |
| 142 | * |
| 143 | * - @TODO: Provide additional information to this primitive so that |
| 144 | * we can utilize: |
| 145 | * - The TTL level hint, see FEAT_TTL, |
| 146 | * - Final level lookup only invalidation, |
| 147 | * - Address range invalidation. |
| 148 | */ |
| 149 | |
| 150 | /* |
| 151 | * Save the current content of vttb_el2. |
| 152 | */ |
| 153 | unsigned long old_vttbr_el2 = read_vttbr_el2(); |
| 154 | |
| 155 | /* |
| 156 | * Make 'vmid' the `current vmid`. Note that the tlbi instructions |
| 157 | * bellow target the TLB entries that match the `current vmid`. |
| 158 | */ |
| 159 | write_vttbr_el2(INPLACE(VTTBR_EL2_VMID, s2_ctx->vmid)); |
| 160 | isb(); |
| 161 | |
| 162 | /* |
| 163 | * Invalidate entries in S2 TLB caches that |
| 164 | * match both `ipa` & the `current vmid`. |
| 165 | */ |
| 166 | while (size != 0UL) { |
| 167 | tlbiipas2e1is(ipa >> 12); |
| 168 | size -= GRANULE_SIZE; |
| 169 | ipa += GRANULE_SIZE; |
| 170 | } |
| 171 | dsb(ish); |
| 172 | |
| 173 | /* |
| 174 | * The architecture does not require TLB invalidation by IPA to affect |
| 175 | * combined Stage-1 + Stage-2 TLBs. Therefore we must invalidate all of |
| 176 | * Stage-1 (tagged with the `current vmid`) after invalidating Stage-2. |
| 177 | */ |
| 178 | tlbivmalle1is(); |
| 179 | dsb(ish); |
| 180 | isb(); |
| 181 | |
| 182 | /* |
| 183 | * Restore the old content of vttb_el2. |
| 184 | */ |
| 185 | write_vttbr_el2(old_vttbr_el2); |
| 186 | isb(); |
| 187 | } |
| 188 | |
| 189 | /* |
| 190 | * Invalidate S2 TLB entries with "addr" IPA. |
| 191 | * Call this function after: |
| 192 | * 1. A L3 page desc has been removed. |
| 193 | */ |
| 194 | void invalidate_page(const struct realm_s2_context *s2_ctx, unsigned long addr) |
| 195 | { |
| 196 | stage2_tlbi_ipa(s2_ctx, addr, GRANULE_SIZE); |
| 197 | } |
| 198 | |
| 199 | /* |
| 200 | * Invalidate S2 TLB entries with "addr" IPA. |
| 201 | * Call this function after: |
| 202 | * 1. A L2 block desc has been removed, or |
| 203 | * 2a. A L2 table desc has been removed, where |
| 204 | * 2b. All S2TTEs in L3 table that the L2 table desc was pointed to were invalid. |
| 205 | */ |
| 206 | void invalidate_block(const struct realm_s2_context *s2_ctx, unsigned long addr) |
| 207 | { |
| 208 | stage2_tlbi_ipa(s2_ctx, addr, GRANULE_SIZE); |
| 209 | } |
| 210 | |
| 211 | /* |
| 212 | * Invalidate S2 TLB entries with "addr" IPA. |
| 213 | * Call this function after: |
| 214 | * 1a. A L2 table desc has been removed, where |
| 215 | * 1b. Some S2TTEs in the table that the L2 table desc was pointed to were valid. |
| 216 | */ |
| 217 | void invalidate_pages_in_block(const struct realm_s2_context *s2_ctx, unsigned long addr) |
| 218 | { |
| 219 | stage2_tlbi_ipa(s2_ctx, addr, BLOCK_L2_SIZE); |
| 220 | } |
| 221 | |
| 222 | /* |
| 223 | * Return the index of the entry describing @addr in the translation table at |
| 224 | * level @level. This only works for non-concatenated page tables, so should |
| 225 | * not be called to get the index for the starting level. |
| 226 | * |
| 227 | * See the library pseudocode |
| 228 | * aarch64/translation/vmsa_addrcalc/AArch64.TTEntryAddress on which this is |
| 229 | * modeled. |
| 230 | */ |
| 231 | static unsigned long s2_addr_to_idx(unsigned long addr, long level) |
| 232 | { |
AlexeiFedorov | bb1f1c8 | 2023-08-25 10:44:49 +0100 | [diff] [blame] | 233 | unsigned int levels, lsb; |
| 234 | |
| 235 | assert(level <= RTT_PAGE_LEVEL); |
| 236 | |
AlexeiFedorov | 4faab85 | 2023-08-30 15:06:49 +0100 | [diff] [blame] | 237 | levels = (unsigned int)(RTT_PAGE_LEVEL - level); |
AlexeiFedorov | 56e1a8e | 2023-09-01 17:06:13 +0100 | [diff] [blame] | 238 | lsb = (levels * S2TTE_STRIDE) + GRANULE_SHIFT; |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 239 | |
| 240 | addr >>= lsb; |
AlexeiFedorov | bb1f1c8 | 2023-08-25 10:44:49 +0100 | [diff] [blame] | 241 | addr &= (1UL << S2TTE_STRIDE) - 1UL; |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 242 | return addr; |
| 243 | } |
| 244 | |
| 245 | /* |
| 246 | * Return the index of the entry describing @addr in the translation table |
| 247 | * starting level. This may return an index >= S2TTES_PER_S2TT when the |
| 248 | * combination of @start_level and @ipa_bits implies concatenated |
| 249 | * stage 2 tables. |
| 250 | * |
| 251 | * See the library pseudocode |
| 252 | * aarch64/translation/vmsa_addrcalc/AArch64.S2SLTTEntryAddress on which |
| 253 | * this is modeled. |
| 254 | */ |
| 255 | static unsigned long s2_sl_addr_to_idx(unsigned long addr, int start_level, |
| 256 | unsigned long ipa_bits) |
| 257 | { |
AlexeiFedorov | bb1f1c8 | 2023-08-25 10:44:49 +0100 | [diff] [blame] | 258 | unsigned int levels, lsb; |
| 259 | |
| 260 | assert(start_level <= RTT_PAGE_LEVEL); |
| 261 | |
AlexeiFedorov | 4faab85 | 2023-08-30 15:06:49 +0100 | [diff] [blame] | 262 | levels = (unsigned int)(RTT_PAGE_LEVEL - start_level); |
AlexeiFedorov | 56e1a8e | 2023-09-01 17:06:13 +0100 | [diff] [blame] | 263 | lsb = (levels * S2TTE_STRIDE) + GRANULE_SHIFT; |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 264 | |
| 265 | addr &= (1UL << ipa_bits) - 1UL; |
| 266 | addr >>= lsb; |
| 267 | return addr; |
| 268 | } |
| 269 | |
AlexeiFedorov | 7641a81 | 2023-08-22 14:31:27 +0100 | [diff] [blame] | 270 | static unsigned long addr_level_mask(unsigned long addr, long level) |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 271 | { |
AlexeiFedorov | bb1f1c8 | 2023-08-25 10:44:49 +0100 | [diff] [blame] | 272 | unsigned int levels, lsb, msb; |
| 273 | |
| 274 | assert(level <= RTT_PAGE_LEVEL); |
| 275 | |
AlexeiFedorov | 4faab85 | 2023-08-30 15:06:49 +0100 | [diff] [blame] | 276 | levels = (unsigned int)(RTT_PAGE_LEVEL - level); |
AlexeiFedorov | 56e1a8e | 2023-09-01 17:06:13 +0100 | [diff] [blame] | 277 | lsb = (levels * S2TTE_STRIDE) + GRANULE_SHIFT; |
AlexeiFedorov | bb1f1c8 | 2023-08-25 10:44:49 +0100 | [diff] [blame] | 278 | msb = S2TTE_OA_BITS - 1U; |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 279 | |
AlexeiFedorov | c53b1f7 | 2023-07-04 15:37:03 +0100 | [diff] [blame] | 280 | return (addr & BIT_MASK_ULL(msb, lsb)); |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 281 | } |
| 282 | |
| 283 | static inline unsigned long table_entry_to_phys(unsigned long entry) |
| 284 | { |
| 285 | return addr_level_mask(entry, RTT_PAGE_LEVEL); |
| 286 | } |
| 287 | |
| 288 | static inline bool entry_is_table(unsigned long entry) |
| 289 | { |
AlexeiFedorov | c53b1f7 | 2023-07-04 15:37:03 +0100 | [diff] [blame] | 290 | return ((entry & DESC_TYPE_MASK) == S2TTE_L012_TABLE); |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 291 | } |
| 292 | |
| 293 | static unsigned long __table_get_entry(struct granule *g_tbl, |
| 294 | unsigned long idx) |
| 295 | { |
| 296 | unsigned long *table, entry; |
| 297 | |
| 298 | table = granule_map(g_tbl, SLOT_RTT); |
AlexeiFedorov | 9a9062c | 2023-08-21 15:41:48 +0100 | [diff] [blame] | 299 | assert(table != NULL); |
| 300 | |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 301 | entry = s2tte_read(&table[idx]); |
| 302 | buffer_unmap(table); |
| 303 | |
| 304 | return entry; |
| 305 | } |
| 306 | |
| 307 | static struct granule *__find_next_level_idx(struct granule *g_tbl, |
| 308 | unsigned long idx) |
| 309 | { |
| 310 | const unsigned long entry = __table_get_entry(g_tbl, idx); |
| 311 | |
| 312 | if (!entry_is_table(entry)) { |
| 313 | return NULL; |
| 314 | } |
| 315 | |
| 316 | return addr_to_granule(table_entry_to_phys(entry)); |
| 317 | } |
| 318 | |
| 319 | static struct granule *__find_lock_next_level(struct granule *g_tbl, |
| 320 | unsigned long map_addr, |
| 321 | long level) |
| 322 | { |
| 323 | const unsigned long idx = s2_addr_to_idx(map_addr, level); |
| 324 | struct granule *g = __find_next_level_idx(g_tbl, idx); |
| 325 | |
| 326 | if (g != NULL) { |
| 327 | granule_lock(g, GRANULE_STATE_RTT); |
| 328 | } |
| 329 | |
| 330 | return g; |
| 331 | } |
| 332 | |
| 333 | /* |
| 334 | * Walk an RTT until level @level using @map_addr. |
| 335 | * @g_root is the root (level 0) table and must be locked before the call. |
| 336 | * @start_level is the initial lookup level used for the stage 2 translation |
| 337 | * tables which may depend on the configuration of the realm, factoring in the |
| 338 | * IPA size of the realm and the desired starting level (within the limits |
| 339 | * defined by the Armv8 VMSA including options for stage 2 table concatenation). |
| 340 | * The function uses hand-over-hand locking to avoid race conditions and allow |
| 341 | * concurrent access to RTT tree which is not part of the current walk; when a |
| 342 | * next level table is reached it is locked before releasing previously locked |
| 343 | * table. |
| 344 | * The walk stops when either: |
| 345 | * - The entry found is a leaf entry (not an RTT Table entry), or |
| 346 | * - Level @level is reached. |
| 347 | * |
| 348 | * On return: |
| 349 | * - rtt_walk::last_level is the last level that has been reached by the walk. |
| 350 | * - rtt_walk.g_llt points to the TABLE granule at level @rtt_walk::level. |
| 351 | * The granule is locked. |
| 352 | * - rtt_walk::index is the entry index at rtt_walk.g_llt for @map_addr. |
| 353 | */ |
| 354 | void rtt_walk_lock_unlock(struct granule *g_root, |
| 355 | int start_level, |
| 356 | unsigned long ipa_bits, |
| 357 | unsigned long map_addr, |
| 358 | long level, |
| 359 | struct rtt_walk *wi) |
| 360 | { |
| 361 | struct granule *g_tbls[NR_RTT_LEVELS] = { NULL }; |
| 362 | unsigned long sl_idx; |
| 363 | int i, last_level; |
| 364 | |
| 365 | assert(start_level >= MIN_STARTING_LEVEL); |
| 366 | assert(level >= start_level); |
| 367 | assert(map_addr < (1UL << ipa_bits)); |
| 368 | assert(wi != NULL); |
| 369 | |
| 370 | /* Handle concatenated starting level (SL) tables */ |
| 371 | sl_idx = s2_sl_addr_to_idx(map_addr, start_level, ipa_bits); |
| 372 | if (sl_idx >= S2TTES_PER_S2TT) { |
AlexeiFedorov | 4faab85 | 2023-08-30 15:06:49 +0100 | [diff] [blame] | 373 | unsigned int tt_num = (unsigned int)(sl_idx >> S2TTE_STRIDE); |
AlexeiFedorov | 3f5d627 | 2023-10-23 16:27:37 +0100 | [diff] [blame] | 374 | struct granule *g_concat_root = (struct granule *)((uintptr_t)g_root + |
| 375 | (tt_num * sizeof(struct granule))); |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 376 | |
| 377 | granule_lock(g_concat_root, GRANULE_STATE_RTT); |
| 378 | granule_unlock(g_root); |
| 379 | g_root = g_concat_root; |
| 380 | } |
| 381 | |
| 382 | g_tbls[start_level] = g_root; |
| 383 | for (i = start_level; i < level; i++) { |
| 384 | /* |
| 385 | * Lock next RTT level. Correct locking order is guaranteed |
| 386 | * because reference is obtained from a locked granule |
| 387 | * (previous level). Also, hand-over-hand locking/unlocking is |
| 388 | * used to avoid race conditions. |
| 389 | */ |
| 390 | g_tbls[i + 1] = __find_lock_next_level(g_tbls[i], map_addr, i); |
| 391 | if (g_tbls[i + 1] == NULL) { |
| 392 | last_level = i; |
| 393 | goto out; |
| 394 | } |
| 395 | granule_unlock(g_tbls[i]); |
| 396 | } |
| 397 | |
AlexeiFedorov | 4faab85 | 2023-08-30 15:06:49 +0100 | [diff] [blame] | 398 | last_level = (int)level; |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 399 | out: |
| 400 | wi->last_level = last_level; |
| 401 | wi->g_llt = g_tbls[last_level]; |
| 402 | wi->index = s2_addr_to_idx(map_addr, last_level); |
| 403 | } |
| 404 | |
| 405 | /* |
AlexeiFedorov | c07a638 | 2023-04-14 11:59:18 +0100 | [diff] [blame] | 406 | * Creates an unassigned_empty s2tte. |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 407 | */ |
AlexeiFedorov | c07a638 | 2023-04-14 11:59:18 +0100 | [diff] [blame] | 408 | unsigned long s2tte_create_unassigned_empty(void) |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 409 | { |
AlexeiFedorov | c53b1f7 | 2023-07-04 15:37:03 +0100 | [diff] [blame] | 410 | return (S2TTE_INVALID_HIPAS_UNASSIGNED | S2TTE_INVALID_RIPAS_EMPTY); |
AlexeiFedorov | c07a638 | 2023-04-14 11:59:18 +0100 | [diff] [blame] | 411 | } |
| 412 | |
| 413 | /* |
| 414 | * Creates an unassigned_ram s2tte. |
| 415 | */ |
| 416 | unsigned long s2tte_create_unassigned_ram(void) |
| 417 | { |
AlexeiFedorov | c53b1f7 | 2023-07-04 15:37:03 +0100 | [diff] [blame] | 418 | return (S2TTE_INVALID_HIPAS_UNASSIGNED | S2TTE_INVALID_RIPAS_RAM); |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 419 | } |
| 420 | |
| 421 | /* |
AlexeiFedorov | c53b1f7 | 2023-07-04 15:37:03 +0100 | [diff] [blame] | 422 | * Creates an unassigned_destroyed s2tte. |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 423 | */ |
AlexeiFedorov | c53b1f7 | 2023-07-04 15:37:03 +0100 | [diff] [blame] | 424 | unsigned long s2tte_create_unassigned_destroyed(void) |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 425 | { |
AlexeiFedorov | c53b1f7 | 2023-07-04 15:37:03 +0100 | [diff] [blame] | 426 | return (S2TTE_INVALID_HIPAS_UNASSIGNED | S2TTE_INVALID_RIPAS_DESTROYED); |
| 427 | } |
| 428 | |
| 429 | /* |
| 430 | * Creates an invalid s2tte with output address @pa, HIPAS=ASSIGNED and |
| 431 | * RIPAS=DESTROYED, at level @level. |
| 432 | */ |
| 433 | unsigned long s2tte_create_assigned_destroyed(unsigned long pa, long level) |
| 434 | { |
Chuyue Luo | 5ef71d9 | 2023-10-24 14:29:20 +0100 | [diff] [blame^] | 435 | (void)level; |
| 436 | |
AlexeiFedorov | c53b1f7 | 2023-07-04 15:37:03 +0100 | [diff] [blame] | 437 | assert(level >= RTT_MIN_BLOCK_LEVEL); |
| 438 | assert(addr_is_level_aligned(pa, level)); |
| 439 | return (pa | S2TTE_INVALID_HIPAS_ASSIGNED | S2TTE_INVALID_RIPAS_DESTROYED); |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 440 | } |
| 441 | |
| 442 | /* |
| 443 | * Creates an invalid s2tte with output address @pa, HIPAS=ASSIGNED and |
| 444 | * RIPAS=EMPTY, at level @level. |
| 445 | */ |
| 446 | unsigned long s2tte_create_assigned_empty(unsigned long pa, long level) |
| 447 | { |
Chuyue Luo | 5ef71d9 | 2023-10-24 14:29:20 +0100 | [diff] [blame^] | 448 | (void)level; |
| 449 | |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 450 | assert(level >= RTT_MIN_BLOCK_LEVEL); |
| 451 | assert(addr_is_level_aligned(pa, level)); |
| 452 | return (pa | S2TTE_INVALID_HIPAS_ASSIGNED | S2TTE_INVALID_RIPAS_EMPTY); |
| 453 | } |
| 454 | |
| 455 | /* |
AlexeiFedorov | 3a73933 | 2023-04-13 13:54:04 +0100 | [diff] [blame] | 456 | * Creates an assigned_ram s2tte with output address @pa. |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 457 | */ |
AlexeiFedorov | 3a73933 | 2023-04-13 13:54:04 +0100 | [diff] [blame] | 458 | unsigned long s2tte_create_assigned_ram(unsigned long pa, long level) |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 459 | { |
| 460 | assert(level >= RTT_MIN_BLOCK_LEVEL); |
| 461 | assert(addr_is_level_aligned(pa, level)); |
| 462 | if (level == RTT_PAGE_LEVEL) { |
| 463 | return (pa | S2TTE_PAGE); |
| 464 | } |
| 465 | return (pa | S2TTE_BLOCK); |
| 466 | } |
| 467 | |
| 468 | /* |
Javier Almansa Sobrino | 84a1b16 | 2023-09-26 17:32:44 +0100 | [diff] [blame] | 469 | * Creates an assigned s2tte with output address @pa and the same |
| 470 | * RIPAS as passed on @s2tte. |
| 471 | */ |
| 472 | unsigned long s2tte_create_assigned_unchanged(unsigned long s2tte, |
| 473 | unsigned long pa, |
| 474 | long level) |
| 475 | { |
| 476 | unsigned long current_ripas = s2tte & S2TTE_INVALID_RIPAS_MASK; |
| 477 | |
| 478 | assert(level >= RTT_MIN_BLOCK_LEVEL); |
| 479 | assert(addr_is_level_aligned(pa, level)); |
| 480 | assert(EXTRACT(S2TTE_INVALID_RIPAS, current_ripas) <= |
| 481 | EXTRACT(S2TTE_INVALID_RIPAS, S2TTE_INVALID_RIPAS_DESTROYED)); |
| 482 | |
| 483 | if (current_ripas != S2TTE_INVALID_RIPAS_RAM) { |
| 484 | return (pa | S2TTE_INVALID_HIPAS_ASSIGNED | current_ripas); |
| 485 | } |
| 486 | |
| 487 | return s2tte_create_assigned_ram(pa, level); |
| 488 | } |
| 489 | |
| 490 | /* |
AlexeiFedorov | 5ceff35 | 2023-04-12 16:17:00 +0100 | [diff] [blame] | 491 | * Creates an unassigned_ns s2tte. |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 492 | */ |
AlexeiFedorov | 5ceff35 | 2023-04-12 16:17:00 +0100 | [diff] [blame] | 493 | unsigned long s2tte_create_unassigned_ns(void) |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 494 | { |
AlexeiFedorov | c53b1f7 | 2023-07-04 15:37:03 +0100 | [diff] [blame] | 495 | return (S2TTE_NS | S2TTE_INVALID_HIPAS_UNASSIGNED | |
| 496 | S2TTE_INVALID_UNPROTECTED); |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 497 | } |
| 498 | |
| 499 | /* |
AlexeiFedorov | 3a73933 | 2023-04-13 13:54:04 +0100 | [diff] [blame] | 500 | * Creates an assigned_ns s2tte at level @level. |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 501 | * |
| 502 | * The following S2 TTE fields are provided through @s2tte argument: |
| 503 | * - The physical address |
| 504 | * - MemAttr |
| 505 | * - S2AP |
| 506 | * - Shareability |
Javier Almansa Sobrino | 15fc44e | 2023-09-29 13:52:04 +0100 | [diff] [blame] | 507 | * Any other field on @s2tte is masked out. |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 508 | */ |
Javier Almansa Sobrino | 15fc44e | 2023-09-29 13:52:04 +0100 | [diff] [blame] | 509 | unsigned long s2tte_create_assigned_ns(unsigned long s2tte, long level) |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 510 | { |
Javier Almansa Sobrino | 15fc44e | 2023-09-29 13:52:04 +0100 | [diff] [blame] | 511 | unsigned long new_s2tte = s2tte & ~DESC_TYPE_MASK; |
| 512 | |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 513 | assert(level >= RTT_MIN_BLOCK_LEVEL); |
| 514 | if (level == RTT_PAGE_LEVEL) { |
Javier Almansa Sobrino | 15fc44e | 2023-09-29 13:52:04 +0100 | [diff] [blame] | 515 | return (new_s2tte | S2TTE_PAGE_NS); |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 516 | } |
Javier Almansa Sobrino | 15fc44e | 2023-09-29 13:52:04 +0100 | [diff] [blame] | 517 | return (new_s2tte | S2TTE_BLOCK_NS); |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 518 | } |
| 519 | |
| 520 | /* |
| 521 | * Validate the portion of NS S2TTE that is provided by the host. |
| 522 | */ |
| 523 | bool host_ns_s2tte_is_valid(unsigned long s2tte, long level) |
| 524 | { |
| 525 | unsigned long mask = addr_level_mask(~0UL, level) | |
Javier Almansa Sobrino | 15fc44e | 2023-09-29 13:52:04 +0100 | [diff] [blame] | 526 | S2TTE_NS_ATTR_MASK; |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 527 | |
| 528 | /* |
| 529 | * Test that all fields that are not controlled by the host are zero |
| 530 | * and that the output address is correctly aligned. Note that |
| 531 | * the host is permitted to map any physical address outside PAR. |
| 532 | */ |
| 533 | if ((s2tte & ~mask) != 0UL) { |
| 534 | return false; |
| 535 | } |
| 536 | |
| 537 | /* |
| 538 | * Only one value masked by S2TTE_MEMATTR_MASK is invalid/reserved. |
| 539 | */ |
| 540 | if ((s2tte & S2TTE_MEMATTR_MASK) == S2TTE_MEMATTR_FWB_RESERVED) { |
| 541 | return false; |
| 542 | } |
| 543 | |
| 544 | /* |
| 545 | * Only one value masked by S2TTE_SH_MASK is invalid/reserved. |
| 546 | */ |
| 547 | if ((s2tte & S2TTE_SH_MASK) == S2TTE_SH_RESERVED) { |
| 548 | return false; |
| 549 | } |
| 550 | |
| 551 | /* |
| 552 | * Note that all the values that are masked by S2TTE_AP_MASK are valid. |
| 553 | */ |
| 554 | return true; |
| 555 | } |
| 556 | |
| 557 | /* |
| 558 | * Returns the portion of NS S2TTE that is set by the host. |
| 559 | */ |
| 560 | unsigned long host_ns_s2tte(unsigned long s2tte, long level) |
| 561 | { |
| 562 | unsigned long mask = addr_level_mask(~0UL, level) | |
Javier Almansa Sobrino | 15fc44e | 2023-09-29 13:52:04 +0100 | [diff] [blame] | 563 | S2TTE_NS_ATTR_MASK; |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 564 | return (s2tte & mask); |
| 565 | } |
| 566 | |
| 567 | /* |
| 568 | * Creates a table s2tte at level @level with output address @pa. |
| 569 | */ |
| 570 | unsigned long s2tte_create_table(unsigned long pa, long level) |
| 571 | { |
Chuyue Luo | 5ef71d9 | 2023-10-24 14:29:20 +0100 | [diff] [blame^] | 572 | (void)level; |
| 573 | |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 574 | assert(level < RTT_PAGE_LEVEL); |
| 575 | assert(GRANULE_ALIGNED(pa)); |
| 576 | |
| 577 | return (pa | S2TTE_TABLE); |
| 578 | } |
| 579 | |
| 580 | /* |
AlexeiFedorov | 0fb4455 | 2023-04-14 15:37:58 +0100 | [diff] [blame] | 581 | * Returns true if s2tte has defined ripas value, namely if it is one of: |
| 582 | * - unassigned_empty |
| 583 | * - unassigned_ram |
AlexeiFedorov | c53b1f7 | 2023-07-04 15:37:03 +0100 | [diff] [blame] | 584 | * - unassigned_destroyed |
AlexeiFedorov | 0fb4455 | 2023-04-14 15:37:58 +0100 | [diff] [blame] | 585 | * - assigned_empty |
| 586 | * - assigned_ram |
AlexeiFedorov | c53b1f7 | 2023-07-04 15:37:03 +0100 | [diff] [blame] | 587 | * - assigned_destroyed |
AlexeiFedorov | 0fb4455 | 2023-04-14 15:37:58 +0100 | [diff] [blame] | 588 | */ |
| 589 | bool s2tte_has_ripas(unsigned long s2tte, long level) |
| 590 | { |
AlexeiFedorov | c53b1f7 | 2023-07-04 15:37:03 +0100 | [diff] [blame] | 591 | return (((s2tte & S2TTE_NS) == 0UL) && !s2tte_is_table(s2tte, level)); |
AlexeiFedorov | 0fb4455 | 2023-04-14 15:37:58 +0100 | [diff] [blame] | 592 | } |
| 593 | |
| 594 | /* |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 595 | * Returns true if @s2tte has HIPAS=@hipas. |
| 596 | */ |
| 597 | static bool s2tte_has_hipas(unsigned long s2tte, unsigned long hipas) |
| 598 | { |
| 599 | unsigned long desc_type = s2tte & DESC_TYPE_MASK; |
| 600 | unsigned long invalid_desc_hipas = s2tte & S2TTE_INVALID_HIPAS_MASK; |
| 601 | |
AlexeiFedorov | 63614ea | 2023-07-14 17:07:20 +0100 | [diff] [blame] | 602 | return ((desc_type == S2TTE_INVALID) && (invalid_desc_hipas == hipas)); |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 603 | } |
| 604 | |
| 605 | /* |
AlexeiFedorov | c07a638 | 2023-04-14 11:59:18 +0100 | [diff] [blame] | 606 | * Returns true if @s2tte has HIPAS=UNASSIGNED and RIPAS=@ripas. |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 607 | */ |
AlexeiFedorov | c07a638 | 2023-04-14 11:59:18 +0100 | [diff] [blame] | 608 | static bool s2tte_has_unassigned_ripas(unsigned long s2tte, unsigned long ripas) |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 609 | { |
AlexeiFedorov | c53b1f7 | 2023-07-04 15:37:03 +0100 | [diff] [blame] | 610 | return (((s2tte & S2TTE_INVALID_RIPAS_MASK) == ripas) && |
| 611 | s2tte_has_hipas(s2tte, S2TTE_INVALID_HIPAS_UNASSIGNED)); |
| 612 | } |
AlexeiFedorov | c07a638 | 2023-04-14 11:59:18 +0100 | [diff] [blame] | 613 | |
AlexeiFedorov | c53b1f7 | 2023-07-04 15:37:03 +0100 | [diff] [blame] | 614 | /* |
| 615 | * Returns true if @s2tte has HIPAS=ASSIGNED and RIPAS=@ripas. |
| 616 | */ |
| 617 | static bool s2tte_has_assigned_ripas(unsigned long s2tte, unsigned long ripas) |
| 618 | { |
| 619 | return (((s2tte & S2TTE_INVALID_RIPAS_MASK) == ripas) && |
| 620 | s2tte_has_hipas(s2tte, S2TTE_INVALID_HIPAS_ASSIGNED)); |
AlexeiFedorov | c07a638 | 2023-04-14 11:59:18 +0100 | [diff] [blame] | 621 | } |
| 622 | |
| 623 | /* |
AlexeiFedorov | 0f9cd1f | 2023-07-10 17:04:58 +0100 | [diff] [blame] | 624 | * Returns true if @s2tte has HIPAS=UNASSIGNED. |
| 625 | */ |
| 626 | bool s2tte_is_unassigned(unsigned long s2tte) |
| 627 | { |
| 628 | return s2tte_has_hipas(s2tte, S2TTE_INVALID_HIPAS_UNASSIGNED); |
| 629 | } |
| 630 | |
| 631 | /* |
AlexeiFedorov | c07a638 | 2023-04-14 11:59:18 +0100 | [diff] [blame] | 632 | * Returns true if @s2tte is an unassigned_empty. |
| 633 | */ |
| 634 | bool s2tte_is_unassigned_empty(unsigned long s2tte) |
| 635 | { |
AlexeiFedorov | c53b1f7 | 2023-07-04 15:37:03 +0100 | [diff] [blame] | 636 | return (((s2tte & S2TTE_NS) == 0UL) && |
| 637 | s2tte_has_unassigned_ripas(s2tte, S2TTE_INVALID_RIPAS_EMPTY)); |
AlexeiFedorov | c07a638 | 2023-04-14 11:59:18 +0100 | [diff] [blame] | 638 | } |
| 639 | |
| 640 | /* |
| 641 | * Returns true if @s2tte is an unassigned_ram. |
| 642 | */ |
| 643 | bool s2tte_is_unassigned_ram(unsigned long s2tte) |
| 644 | { |
| 645 | return s2tte_has_unassigned_ripas(s2tte, S2TTE_INVALID_RIPAS_RAM); |
| 646 | } |
| 647 | |
| 648 | /* |
AlexeiFedorov | 5ceff35 | 2023-04-12 16:17:00 +0100 | [diff] [blame] | 649 | * Returns true if @s2tte is unassigned_ns. |
| 650 | */ |
| 651 | bool s2tte_is_unassigned_ns(unsigned long s2tte) |
| 652 | { |
AlexeiFedorov | c53b1f7 | 2023-07-04 15:37:03 +0100 | [diff] [blame] | 653 | return (((s2tte & S2TTE_NS) != 0UL) && |
| 654 | s2tte_has_hipas(s2tte, S2TTE_INVALID_HIPAS_UNASSIGNED)); |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 655 | } |
| 656 | |
| 657 | /* |
AlexeiFedorov | c53b1f7 | 2023-07-04 15:37:03 +0100 | [diff] [blame] | 658 | * Returns true if @s2tte has RIPAS=DESTROYED. |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 659 | */ |
AlexeiFedorov | c53b1f7 | 2023-07-04 15:37:03 +0100 | [diff] [blame] | 660 | bool s2tte_is_unassigned_destroyed(unsigned long s2tte) |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 661 | { |
AlexeiFedorov | c53b1f7 | 2023-07-04 15:37:03 +0100 | [diff] [blame] | 662 | return s2tte_has_unassigned_ripas(s2tte, S2TTE_INVALID_RIPAS_DESTROYED); |
| 663 | } |
| 664 | |
| 665 | /* |
| 666 | * Returns true if @s2tte is an assigned_destroyed. |
| 667 | */ |
| 668 | bool s2tte_is_assigned_destroyed(unsigned long s2tte, long level) |
| 669 | { |
| 670 | (void)level; |
| 671 | |
| 672 | return s2tte_has_assigned_ripas(s2tte, S2TTE_INVALID_RIPAS_DESTROYED); |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 673 | } |
| 674 | |
| 675 | /* |
AlexeiFedorov | 3a73933 | 2023-04-13 13:54:04 +0100 | [diff] [blame] | 676 | * Returns true if @s2tte is an assigned_empty. |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 677 | */ |
AlexeiFedorov | 3a73933 | 2023-04-13 13:54:04 +0100 | [diff] [blame] | 678 | bool s2tte_is_assigned_empty(unsigned long s2tte, long level) |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 679 | { |
| 680 | (void)level; |
| 681 | |
AlexeiFedorov | c53b1f7 | 2023-07-04 15:37:03 +0100 | [diff] [blame] | 682 | return s2tte_has_assigned_ripas(s2tte, S2TTE_INVALID_RIPAS_EMPTY); |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 683 | } |
| 684 | |
| 685 | static bool s2tte_check(unsigned long s2tte, long level, unsigned long ns) |
| 686 | { |
| 687 | unsigned long desc_type; |
| 688 | |
| 689 | if ((s2tte & S2TTE_NS) != ns) { |
| 690 | return false; |
| 691 | } |
| 692 | |
| 693 | desc_type = s2tte & DESC_TYPE_MASK; |
| 694 | |
| 695 | /* Only pages at L3 and valid blocks at L2 allowed */ |
| 696 | if (((level == RTT_PAGE_LEVEL) && (desc_type == S2TTE_L3_PAGE)) || |
AlexeiFedorov | 80c2f04 | 2022-11-25 14:54:46 +0000 | [diff] [blame] | 697 | ((level == RTT_MIN_BLOCK_LEVEL) && (desc_type == S2TTE_L012_BLOCK))) { |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 698 | return true; |
| 699 | } |
| 700 | |
| 701 | return false; |
| 702 | } |
| 703 | |
| 704 | /* |
AlexeiFedorov | 3a73933 | 2023-04-13 13:54:04 +0100 | [diff] [blame] | 705 | * Returns true if @s2tte is an assigned_ram. |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 706 | */ |
AlexeiFedorov | 3a73933 | 2023-04-13 13:54:04 +0100 | [diff] [blame] | 707 | bool s2tte_is_assigned_ram(unsigned long s2tte, long level) |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 708 | { |
| 709 | return s2tte_check(s2tte, level, 0UL); |
| 710 | } |
| 711 | |
| 712 | /* |
AlexeiFedorov | 3a73933 | 2023-04-13 13:54:04 +0100 | [diff] [blame] | 713 | * Returns true if @s2tte is an assigned_ns s2tte. |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 714 | */ |
AlexeiFedorov | 3a73933 | 2023-04-13 13:54:04 +0100 | [diff] [blame] | 715 | bool s2tte_is_assigned_ns(unsigned long s2tte, long level) |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 716 | { |
| 717 | return s2tte_check(s2tte, level, S2TTE_NS); |
| 718 | } |
| 719 | |
| 720 | /* |
| 721 | * Returns true if @s2tte is a table at level @level. |
| 722 | */ |
| 723 | bool s2tte_is_table(unsigned long s2tte, long level) |
| 724 | { |
AlexeiFedorov | c53b1f7 | 2023-07-04 15:37:03 +0100 | [diff] [blame] | 725 | return ((level < RTT_PAGE_LEVEL) && |
| 726 | ((s2tte & DESC_TYPE_MASK) == S2TTE_L012_TABLE)); |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 727 | } |
| 728 | |
| 729 | /* |
| 730 | * Returns RIPAS of @s2tte. |
| 731 | * |
| 732 | * Caller should ensure that HIPAS=UNASSIGNED or HIPAS=ASSIGNED. |
| 733 | * The s2tte must be not valid/invalid descriptor. |
| 734 | */ |
| 735 | enum ripas s2tte_get_ripas(unsigned long s2tte) |
| 736 | { |
| 737 | unsigned long desc_ripas = s2tte & S2TTE_INVALID_RIPAS_MASK; |
| 738 | |
| 739 | /* |
| 740 | * If valid s2tte descriptor is passed, then ensure S2AP[0] |
| 741 | * bit is 1 (S2AP is set to RW for lower EL), which corresponds |
AlexeiFedorov | c53b1f7 | 2023-07-04 15:37:03 +0100 | [diff] [blame] | 742 | * to RIPAS_RAM (bits[6:5] = b01) on a valid descriptor. |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 743 | */ |
AlexeiFedorov | 63614ea | 2023-07-14 17:07:20 +0100 | [diff] [blame] | 744 | if (((s2tte & DESC_TYPE_MASK) != S2TTE_INVALID) && |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 745 | (desc_ripas != S2TTE_INVALID_RIPAS_RAM)) { |
| 746 | assert(false); |
| 747 | } |
| 748 | |
AlexeiFedorov | c53b1f7 | 2023-07-04 15:37:03 +0100 | [diff] [blame] | 749 | switch (desc_ripas) { |
| 750 | case S2TTE_INVALID_RIPAS_EMPTY: |
Yousuf A | 6280815 | 2022-10-31 10:35:42 +0000 | [diff] [blame] | 751 | return RIPAS_EMPTY; |
AlexeiFedorov | c53b1f7 | 2023-07-04 15:37:03 +0100 | [diff] [blame] | 752 | case S2TTE_INVALID_RIPAS_RAM: |
| 753 | return RIPAS_RAM; |
| 754 | default: |
| 755 | assert(desc_ripas == S2TTE_INVALID_RIPAS_DESTROYED); |
| 756 | return RIPAS_DESTROYED; |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 757 | } |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 758 | } |
| 759 | |
| 760 | /* |
AlexeiFedorov | c07a638 | 2023-04-14 11:59:18 +0100 | [diff] [blame] | 761 | * Populates @s2tt with unassigned_empty s2ttes. |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 762 | * |
| 763 | * The granule is populated before it is made a table, |
| 764 | * hence, don't use s2tte_write for access. |
| 765 | */ |
AlexeiFedorov | c07a638 | 2023-04-14 11:59:18 +0100 | [diff] [blame] | 766 | void s2tt_init_unassigned_empty(unsigned long *s2tt) |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 767 | { |
| 768 | for (unsigned int i = 0U; i < S2TTES_PER_S2TT; i++) { |
AlexeiFedorov | c07a638 | 2023-04-14 11:59:18 +0100 | [diff] [blame] | 769 | s2tt[i] = s2tte_create_unassigned_empty(); |
| 770 | } |
| 771 | |
| 772 | dsb(ish); |
| 773 | } |
| 774 | |
| 775 | /* |
| 776 | * Populates @s2tt with unassigned_ram s2ttes. |
| 777 | * |
| 778 | * The granule is populated before it is made a table, |
| 779 | * hence, don't use s2tte_write for access. |
| 780 | */ |
| 781 | void s2tt_init_unassigned_ram(unsigned long *s2tt) |
| 782 | { |
| 783 | for (unsigned int i = 0U; i < S2TTES_PER_S2TT; i++) { |
| 784 | s2tt[i] = s2tte_create_unassigned_ram(); |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 785 | } |
| 786 | |
| 787 | dsb(ish); |
| 788 | } |
| 789 | |
| 790 | /* |
AlexeiFedorov | 5ceff35 | 2023-04-12 16:17:00 +0100 | [diff] [blame] | 791 | * Populates @s2tt with unassigned_ns s2ttes. |
| 792 | * |
| 793 | * The granule is populated before it is made a table, |
| 794 | * hence, don't use s2tte_write for access. |
| 795 | */ |
| 796 | void s2tt_init_unassigned_ns(unsigned long *s2tt) |
| 797 | { |
| 798 | for (unsigned int i = 0U; i < S2TTES_PER_S2TT; i++) { |
| 799 | s2tt[i] = s2tte_create_unassigned_ns(); |
| 800 | } |
| 801 | |
| 802 | dsb(ish); |
| 803 | } |
| 804 | |
| 805 | /* |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 806 | * Populates @s2tt with s2ttes which have HIPAS=DESTROYED. |
| 807 | * |
| 808 | * The granule is populated before it is made a table, |
| 809 | * hence, don't use s2tte_write for access. |
| 810 | */ |
AlexeiFedorov | c53b1f7 | 2023-07-04 15:37:03 +0100 | [diff] [blame] | 811 | void s2tt_init_unassigned_destroyed(unsigned long *s2tt) |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 812 | { |
| 813 | for (unsigned int i = 0U; i < S2TTES_PER_S2TT; i++) { |
AlexeiFedorov | c53b1f7 | 2023-07-04 15:37:03 +0100 | [diff] [blame] | 814 | s2tt[i] = s2tte_create_unassigned_destroyed(); |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 815 | } |
AlexeiFedorov | c53b1f7 | 2023-07-04 15:37:03 +0100 | [diff] [blame] | 816 | dsb(ish); |
| 817 | } |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 818 | |
AlexeiFedorov | c53b1f7 | 2023-07-04 15:37:03 +0100 | [diff] [blame] | 819 | /* |
| 820 | * Populates @s2tt with HIPAS=ASSIGNED, RIPAS=DESTROYED s2ttes that refer to a |
| 821 | * contiguous memory block starting at @pa, and mapped at level @level. |
| 822 | * |
| 823 | * The granule is populated before it is made a table, |
| 824 | * hence, don't use s2tte_write for access. |
| 825 | */ |
| 826 | void s2tt_init_assigned_destroyed(unsigned long *s2tt, unsigned long pa, long level) |
| 827 | { |
| 828 | const unsigned long map_size = s2tte_map_size(level); |
| 829 | |
| 830 | for (unsigned int i = 0U; i < S2TTES_PER_S2TT; i++) { |
| 831 | s2tt[i] = s2tte_create_assigned_destroyed(pa, level); |
| 832 | pa += map_size; |
| 833 | } |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 834 | dsb(ish); |
| 835 | } |
| 836 | |
AlexeiFedorov | 4faab85 | 2023-08-30 15:06:49 +0100 | [diff] [blame] | 837 | unsigned long s2tte_map_size(long level) |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 838 | { |
AlexeiFedorov | bb1f1c8 | 2023-08-25 10:44:49 +0100 | [diff] [blame] | 839 | unsigned int levels, lsb; |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 840 | |
| 841 | assert(level <= RTT_PAGE_LEVEL); |
| 842 | |
AlexeiFedorov | 4faab85 | 2023-08-30 15:06:49 +0100 | [diff] [blame] | 843 | levels = (unsigned int)(RTT_PAGE_LEVEL - level); |
AlexeiFedorov | 56e1a8e | 2023-09-01 17:06:13 +0100 | [diff] [blame] | 844 | lsb = (levels * S2TTE_STRIDE) + GRANULE_SHIFT; |
AlexeiFedorov | c53b1f7 | 2023-07-04 15:37:03 +0100 | [diff] [blame] | 845 | return (1UL << lsb); |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 846 | } |
| 847 | |
| 848 | /* |
| 849 | * Populates @s2tt with HIPAS=ASSIGNED, RIPAS=EMPTY s2ttes that refer to a |
| 850 | * contiguous memory block starting at @pa, and mapped at level @level. |
| 851 | * |
| 852 | * The granule is populated before it is made a table, |
| 853 | * hence, don't use s2tte_write for access. |
| 854 | */ |
| 855 | void s2tt_init_assigned_empty(unsigned long *s2tt, unsigned long pa, long level) |
| 856 | { |
| 857 | const unsigned long map_size = s2tte_map_size(level); |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 858 | |
AlexeiFedorov | 3a73933 | 2023-04-13 13:54:04 +0100 | [diff] [blame] | 859 | for (unsigned int i = 0U; i < S2TTES_PER_S2TT; i++) { |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 860 | s2tt[i] = s2tte_create_assigned_empty(pa, level); |
| 861 | pa += map_size; |
| 862 | } |
| 863 | dsb(ish); |
| 864 | } |
| 865 | |
| 866 | /* |
AlexeiFedorov | 3a73933 | 2023-04-13 13:54:04 +0100 | [diff] [blame] | 867 | * Populates @s2tt with assigned_ram s2ttes that refer to a |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 868 | * contiguous memory block starting at @pa, and mapped at level @level. |
| 869 | * |
| 870 | * The granule is populated before it is made a table, |
| 871 | * hence, don't use s2tte_write for access. |
| 872 | */ |
AlexeiFedorov | 3a73933 | 2023-04-13 13:54:04 +0100 | [diff] [blame] | 873 | void s2tt_init_assigned_ram(unsigned long *s2tt, unsigned long pa, long level) |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 874 | { |
| 875 | const unsigned long map_size = s2tte_map_size(level); |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 876 | |
AlexeiFedorov | 3a73933 | 2023-04-13 13:54:04 +0100 | [diff] [blame] | 877 | for (unsigned int i = 0U; i < S2TTES_PER_S2TT; i++) { |
| 878 | s2tt[i] = s2tte_create_assigned_ram(pa, level); |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 879 | pa += map_size; |
| 880 | } |
| 881 | dsb(ish); |
| 882 | } |
| 883 | |
| 884 | /* |
Javier Almansa Sobrino | 15fc44e | 2023-09-29 13:52:04 +0100 | [diff] [blame] | 885 | * Populates @s2tt with NS attributes @attrs that refer to a |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 886 | * contiguous memory block starting at @pa, and mapped at level @level. |
| 887 | * |
| 888 | * The granule is populated before it is made a table, |
| 889 | * hence, don't use s2tte_write for access. |
| 890 | */ |
Javier Almansa Sobrino | 15fc44e | 2023-09-29 13:52:04 +0100 | [diff] [blame] | 891 | void s2tt_init_assigned_ns(unsigned long *s2tt, unsigned long attrs, |
| 892 | unsigned long pa, long level) |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 893 | { |
| 894 | const unsigned long map_size = s2tte_map_size(level); |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 895 | |
Javier Almansa Sobrino | 15fc44e | 2023-09-29 13:52:04 +0100 | [diff] [blame] | 896 | assert(addr_is_level_aligned(pa, level)); |
| 897 | |
AlexeiFedorov | 3a73933 | 2023-04-13 13:54:04 +0100 | [diff] [blame] | 898 | for (unsigned int i = 0U; i < S2TTES_PER_S2TT; i++) { |
Javier Almansa Sobrino | 15fc44e | 2023-09-29 13:52:04 +0100 | [diff] [blame] | 899 | unsigned long s2tte = attrs & S2TTE_NS_ATTR_MASK; |
| 900 | |
| 901 | s2tt[i] = s2tte_create_assigned_ns(s2tte | pa, level); |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 902 | pa += map_size; |
| 903 | } |
Javier Almansa Sobrino | 15fc44e | 2023-09-29 13:52:04 +0100 | [diff] [blame] | 904 | |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 905 | dsb(ish); |
| 906 | } |
| 907 | |
AlexeiFedorov | c07a638 | 2023-04-14 11:59:18 +0100 | [diff] [blame] | 908 | /* |
| 909 | * Returns true if s2tte has 'output address' field, namely, if it is one of: |
| 910 | * - assigned_empty |
| 911 | * - assigned_ram |
| 912 | * - assigned_ns |
AlexeiFedorov | 3ebd462 | 2023-07-18 16:27:39 +0100 | [diff] [blame] | 913 | * - assigned_destroyed |
AlexeiFedorov | c07a638 | 2023-04-14 11:59:18 +0100 | [diff] [blame] | 914 | * - table |
| 915 | */ |
AlexeiFedorov | 7641a81 | 2023-08-22 14:31:27 +0100 | [diff] [blame] | 916 | static bool s2tte_has_pa(unsigned long s2tte, long level) |
AlexeiFedorov | c07a638 | 2023-04-14 11:59:18 +0100 | [diff] [blame] | 917 | { |
| 918 | unsigned long desc_type = s2tte & DESC_TYPE_MASK; |
| 919 | |
AlexeiFedorov | 3ebd462 | 2023-07-18 16:27:39 +0100 | [diff] [blame] | 920 | return ((desc_type != S2TTE_INVALID) || /* block, page or table */ |
| 921 | s2tte_is_assigned_empty(s2tte, level) || |
| 922 | s2tte_is_assigned_destroyed(s2tte, level)); |
AlexeiFedorov | c07a638 | 2023-04-14 11:59:18 +0100 | [diff] [blame] | 923 | } |
| 924 | |
AlexeiFedorov | 917eabf | 2023-04-24 12:20:41 +0100 | [diff] [blame] | 925 | /* |
| 926 | * Returns true if s2tte is a live RTTE entry. i.e., |
AlexeiFedorov | 3ebd462 | 2023-07-18 16:27:39 +0100 | [diff] [blame] | 927 | * HIPAS is ASSIGNED. |
AlexeiFedorov | 917eabf | 2023-04-24 12:20:41 +0100 | [diff] [blame] | 928 | * |
| 929 | * NOTE: For now, only the RTTE with PA are live. |
| 930 | * This could change with EXPORT/IMPORT support. |
| 931 | */ |
AlexeiFedorov | 7641a81 | 2023-08-22 14:31:27 +0100 | [diff] [blame] | 932 | static bool s2tte_is_live(unsigned long s2tte, long level) |
AlexeiFedorov | 917eabf | 2023-04-24 12:20:41 +0100 | [diff] [blame] | 933 | { |
| 934 | return s2tte_has_pa(s2tte, level); |
| 935 | } |
| 936 | |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 937 | /* Returns physical address of a page entry or block */ |
| 938 | unsigned long s2tte_pa(unsigned long s2tte, long level) |
| 939 | { |
AlexeiFedorov | c07a638 | 2023-04-14 11:59:18 +0100 | [diff] [blame] | 940 | if (!s2tte_has_pa(s2tte, level)) { |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 941 | assert(false); |
| 942 | } |
AlexeiFedorov | c07a638 | 2023-04-14 11:59:18 +0100 | [diff] [blame] | 943 | |
| 944 | if (s2tte_is_table(s2tte, level)) { |
| 945 | return addr_level_mask(s2tte, RTT_PAGE_LEVEL); |
| 946 | } |
| 947 | |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 948 | return addr_level_mask(s2tte, level); |
| 949 | } |
| 950 | |
| 951 | /* Returns physical address of a table entry */ |
| 952 | unsigned long s2tte_pa_table(unsigned long s2tte, long level) |
| 953 | { |
Chuyue Luo | 5ef71d9 | 2023-10-24 14:29:20 +0100 | [diff] [blame^] | 954 | (void)level; |
| 955 | |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 956 | assert(s2tte_is_table(s2tte, level)); |
| 957 | return addr_level_mask(s2tte, RTT_PAGE_LEVEL); |
| 958 | } |
| 959 | |
| 960 | bool addr_is_level_aligned(unsigned long addr, long level) |
| 961 | { |
| 962 | return (addr == addr_level_mask(addr, level)); |
| 963 | } |
| 964 | |
| 965 | typedef bool (*s2tte_type_checker)(unsigned long s2tte); |
| 966 | |
| 967 | static bool __table_is_uniform_block(unsigned long *table, |
AlexeiFedorov | 377d81f | 2023-04-14 16:29:12 +0100 | [diff] [blame] | 968 | s2tte_type_checker s2tte_is_x) |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 969 | { |
AlexeiFedorov | 377d81f | 2023-04-14 16:29:12 +0100 | [diff] [blame] | 970 | for (unsigned int i = 0U; i < S2TTES_PER_S2TT; i++) { |
| 971 | unsigned long s2tte = s2tte_read(&table[i]); |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 972 | |
| 973 | if (!s2tte_is_x(s2tte)) { |
| 974 | return false; |
| 975 | } |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 976 | } |
| 977 | |
| 978 | return true; |
| 979 | } |
| 980 | |
| 981 | /* |
AlexeiFedorov | c07a638 | 2023-04-14 11:59:18 +0100 | [diff] [blame] | 982 | * Returns true if all s2ttes in @table are unassigned_empty. |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 983 | */ |
AlexeiFedorov | c07a638 | 2023-04-14 11:59:18 +0100 | [diff] [blame] | 984 | bool table_is_unassigned_empty_block(unsigned long *table) |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 985 | { |
AlexeiFedorov | 377d81f | 2023-04-14 16:29:12 +0100 | [diff] [blame] | 986 | return __table_is_uniform_block(table, s2tte_is_unassigned_empty); |
AlexeiFedorov | c07a638 | 2023-04-14 11:59:18 +0100 | [diff] [blame] | 987 | } |
| 988 | |
| 989 | /* |
| 990 | * Returns true if all s2ttes in @table are unassigned_ram. |
| 991 | */ |
| 992 | bool table_is_unassigned_ram_block(unsigned long *table) |
| 993 | { |
AlexeiFedorov | 377d81f | 2023-04-14 16:29:12 +0100 | [diff] [blame] | 994 | return __table_is_uniform_block(table, s2tte_is_unassigned_ram); |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 995 | } |
| 996 | |
| 997 | /* |
AlexeiFedorov | 5ceff35 | 2023-04-12 16:17:00 +0100 | [diff] [blame] | 998 | * Returns true if all s2ttes in @table are unassigned_ns |
| 999 | */ |
| 1000 | bool table_is_unassigned_ns_block(unsigned long *table) |
| 1001 | { |
AlexeiFedorov | 377d81f | 2023-04-14 16:29:12 +0100 | [diff] [blame] | 1002 | return __table_is_uniform_block(table, s2tte_is_unassigned_ns); |
AlexeiFedorov | 5ceff35 | 2023-04-12 16:17:00 +0100 | [diff] [blame] | 1003 | } |
| 1004 | |
| 1005 | /* |
AlexeiFedorov | c53b1f7 | 2023-07-04 15:37:03 +0100 | [diff] [blame] | 1006 | * Returns true if all s2ttes in @table are unassigned_destroyed |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 1007 | */ |
AlexeiFedorov | c53b1f7 | 2023-07-04 15:37:03 +0100 | [diff] [blame] | 1008 | bool table_is_unassigned_destroyed_block(unsigned long *table) |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 1009 | { |
AlexeiFedorov | c53b1f7 | 2023-07-04 15:37:03 +0100 | [diff] [blame] | 1010 | return __table_is_uniform_block(table, s2tte_is_unassigned_destroyed); |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 1011 | } |
| 1012 | |
| 1013 | typedef bool (*s2tte_type_level_checker)(unsigned long s2tte, long level); |
| 1014 | |
| 1015 | static bool __table_maps_block(unsigned long *table, |
| 1016 | long level, |
Javier Almansa Sobrino | 15fc44e | 2023-09-29 13:52:04 +0100 | [diff] [blame] | 1017 | s2tte_type_level_checker s2tte_is_x, |
| 1018 | bool check_ns_attrs) |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 1019 | { |
| 1020 | unsigned long base_pa; |
| 1021 | unsigned long map_size = s2tte_map_size(level); |
| 1022 | unsigned long s2tte = s2tte_read(&table[0]); |
| 1023 | unsigned int i; |
| 1024 | |
| 1025 | if (!s2tte_is_x(s2tte, level)) { |
| 1026 | return false; |
| 1027 | } |
| 1028 | |
| 1029 | base_pa = s2tte_pa(s2tte, level); |
| 1030 | if (!addr_is_level_aligned(base_pa, level - 1L)) { |
| 1031 | return false; |
| 1032 | } |
| 1033 | |
| 1034 | for (i = 1U; i < S2TTES_PER_S2TT; i++) { |
| 1035 | unsigned long expected_pa = base_pa + (i * map_size); |
| 1036 | |
| 1037 | s2tte = s2tte_read(&table[i]); |
| 1038 | |
| 1039 | if (!s2tte_is_x(s2tte, level)) { |
| 1040 | return false; |
| 1041 | } |
| 1042 | |
| 1043 | if (s2tte_pa(s2tte, level) != expected_pa) { |
| 1044 | return false; |
| 1045 | } |
Javier Almansa Sobrino | 15fc44e | 2023-09-29 13:52:04 +0100 | [diff] [blame] | 1046 | |
| 1047 | if (check_ns_attrs) { |
| 1048 | unsigned long ns_attrs = s2tte & S2TTE_NS_ATTR_MASK; |
| 1049 | |
| 1050 | /* |
| 1051 | * We match all the attributes in the S2TTE |
| 1052 | * except for the AF bit. |
| 1053 | */ |
| 1054 | if ((s2tte & S2TTE_NS_ATTR_MASK) != ns_attrs) { |
| 1055 | return false; |
| 1056 | } |
| 1057 | } |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 1058 | } |
| 1059 | |
| 1060 | return true; |
| 1061 | } |
| 1062 | |
| 1063 | /* |
AlexeiFedorov | 3a73933 | 2023-04-13 13:54:04 +0100 | [diff] [blame] | 1064 | * Returns true if all s2ttes are assigned_empty |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 1065 | * and refer to a contiguous block of granules aligned to @level - 1. |
| 1066 | */ |
AlexeiFedorov | 3a73933 | 2023-04-13 13:54:04 +0100 | [diff] [blame] | 1067 | bool table_maps_assigned_empty_block(unsigned long *table, long level) |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 1068 | { |
Javier Almansa Sobrino | 15fc44e | 2023-09-29 13:52:04 +0100 | [diff] [blame] | 1069 | return __table_maps_block(table, level, s2tte_is_assigned_empty, |
| 1070 | false); |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 1071 | } |
| 1072 | |
| 1073 | /* |
AlexeiFedorov | 3a73933 | 2023-04-13 13:54:04 +0100 | [diff] [blame] | 1074 | * Returns true if all s2ttes are assigned_ram and |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 1075 | * refer to a contiguous block of granules aligned to @level - 1. |
| 1076 | */ |
AlexeiFedorov | 3a73933 | 2023-04-13 13:54:04 +0100 | [diff] [blame] | 1077 | bool table_maps_assigned_ram_block(unsigned long *table, long level) |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 1078 | { |
Javier Almansa Sobrino | 15fc44e | 2023-09-29 13:52:04 +0100 | [diff] [blame] | 1079 | return __table_maps_block(table, level, s2tte_is_assigned_ram, false); |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 1080 | } |
| 1081 | |
| 1082 | /* |
Javier Almansa Sobrino | 15fc44e | 2023-09-29 13:52:04 +0100 | [diff] [blame] | 1083 | * Returns true if |
| 1084 | * - all s2ttes in @table are assigned_ns s2ttes and |
| 1085 | * - they refer to a contiguous block of granules aligned to @level - 1 and |
| 1086 | * - all the s2tte attributes in @table controlled by the host are identical |
AlexeiFedorov | 3a73933 | 2023-04-13 13:54:04 +0100 | [diff] [blame] | 1087 | * |
| 1088 | * @pre: @table maps IPA outside PAR. |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 1089 | */ |
AlexeiFedorov | 3a73933 | 2023-04-13 13:54:04 +0100 | [diff] [blame] | 1090 | bool table_maps_assigned_ns_block(unsigned long *table, long level) |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 1091 | { |
Javier Almansa Sobrino | 15fc44e | 2023-09-29 13:52:04 +0100 | [diff] [blame] | 1092 | return __table_maps_block(table, level, s2tte_is_assigned_ns, true); |
Soby Mathew | b4c6df4 | 2022-11-09 11:13:29 +0000 | [diff] [blame] | 1093 | } |
AlexeiFedorov | 917eabf | 2023-04-24 12:20:41 +0100 | [diff] [blame] | 1094 | |
| 1095 | /* |
AlexeiFedorov | 3f840a0 | 2023-07-19 10:55:05 +0100 | [diff] [blame] | 1096 | * Returns true if all s2ttes are assigned_destroyed and |
| 1097 | * refer to a contiguous block of granules aligned to @level - 1. |
| 1098 | */ |
| 1099 | bool table_maps_assigned_destroyed_block(unsigned long *table, long level) |
| 1100 | { |
Javier Almansa Sobrino | 15fc44e | 2023-09-29 13:52:04 +0100 | [diff] [blame] | 1101 | return __table_maps_block(table, level, s2tte_is_assigned_destroyed, |
| 1102 | false); |
AlexeiFedorov | 3f840a0 | 2023-07-19 10:55:05 +0100 | [diff] [blame] | 1103 | } |
| 1104 | |
| 1105 | /* |
AlexeiFedorov | 917eabf | 2023-04-24 12:20:41 +0100 | [diff] [blame] | 1106 | * Scan the RTT @s2tt (which is @wi.level), from the entry (@wi.index) and |
AlexeiFedorov | 3ebd462 | 2023-07-18 16:27:39 +0100 | [diff] [blame] | 1107 | * skip the non-live entries (i.e., HIPAS=UNASSIGNED). |
AlexeiFedorov | 917eabf | 2023-04-24 12:20:41 +0100 | [diff] [blame] | 1108 | * In other words, the scanning stops when a live RTTE is encountered or we |
| 1109 | * reach the end of this RTT. |
| 1110 | * |
| 1111 | * For now an RTTE can be considered non-live if it doesn't have a PA. |
| 1112 | * NOTE: This would change with EXPORT/IMPORT where we may have metadata stored |
| 1113 | * in the RTTE. |
| 1114 | * |
| 1115 | * @addr is not necessarily aligned to the wi.last_level (e.g., if we were called |
| 1116 | * with RMI_ERROR_RTT). |
| 1117 | * |
| 1118 | * Returns: |
| 1119 | * - If the entry @wi.index is live, returns @addr. |
| 1120 | * - If none of the entries in the @s2tt are "live", returns the address of the |
| 1121 | * first entry in the next table. |
| 1122 | * - Otherwise, the address of the first live entry in @s2tt |
| 1123 | */ |
| 1124 | unsigned long skip_non_live_entries(unsigned long addr, |
| 1125 | unsigned long *s2tt, |
| 1126 | const struct rtt_walk *wi) |
| 1127 | { |
AlexeiFedorov | 56e1a8e | 2023-09-01 17:06:13 +0100 | [diff] [blame] | 1128 | unsigned long i, index = wi->index; |
AlexeiFedorov | 917eabf | 2023-04-24 12:20:41 +0100 | [diff] [blame] | 1129 | long level = wi->last_level; |
| 1130 | unsigned long map_size; |
| 1131 | |
| 1132 | /* |
| 1133 | * If the entry for the map_addr is live, |
| 1134 | * return @addr. |
| 1135 | */ |
| 1136 | if (s2tte_is_live(s2tte_read(&s2tt[index]), level)) { |
| 1137 | return addr; |
| 1138 | } |
| 1139 | |
| 1140 | /* |
| 1141 | * Align the address DOWN to the map_size, as expected for the @level, |
| 1142 | * so that we can compute the correct address by using the index. |
| 1143 | */ |
| 1144 | map_size = s2tte_map_size(level); |
| 1145 | addr &= ~(map_size - 1UL); |
| 1146 | |
| 1147 | /* Skip the "index" */ |
AlexeiFedorov | 56e1a8e | 2023-09-01 17:06:13 +0100 | [diff] [blame] | 1148 | for (i = index + 1UL; i < S2TTES_PER_S2TT; i++) { |
AlexeiFedorov | 917eabf | 2023-04-24 12:20:41 +0100 | [diff] [blame] | 1149 | unsigned long s2tte = s2tte_read(&s2tt[i]); |
| 1150 | |
| 1151 | if (s2tte_is_live(s2tte, level)) { |
| 1152 | break; |
| 1153 | } |
| 1154 | } |
| 1155 | |
AlexeiFedorov | 56e1a8e | 2023-09-01 17:06:13 +0100 | [diff] [blame] | 1156 | return (addr + ((i - index) * map_size)); |
AlexeiFedorov | 917eabf | 2023-04-24 12:20:41 +0100 | [diff] [blame] | 1157 | } |