blob: 602381717a5d15ec1040db3c36f8d35db7dec04d [file] [log] [blame]
Andrew Scull18834872018-10-12 11:48:09 +01001/*
Andrew Walbran692b3252019-03-07 15:51:31 +00002 * Copyright 2018 The Hafnium Authors.
Andrew Scull18834872018-10-12 11:48:09 +01003 *
Andrew Walbrane959ec12020-06-17 15:01:09 +01004 * Use of this source code is governed by a BSD-style
5 * license that can be found in the LICENSE file or at
6 * https://opensource.org/licenses/BSD-3-Clause.
Andrew Scull18834872018-10-12 11:48:09 +01007 */
8
Andrew Scullfbc938a2018-08-20 14:09:28 +01009#pragma once
Wedson Almeida Filhofed69022018-07-11 15:39:12 +010010
Andrew Scull4e5f8142018-10-12 14:37:19 +010011#include <stdalign.h>
Wedson Almeida Filhofed69022018-07-11 15:39:12 +010012#include <stdbool.h>
J-Alves715d6232023-02-16 16:33:28 +000013#include <stddef.h>
Wedson Almeida Filho84a30a02018-07-23 20:05:05 +010014#include <stdint.h>
Wedson Almeida Filhofed69022018-07-11 15:39:12 +010015
Andrew Scull18c78fc2018-08-20 12:57:41 +010016#include "hf/addr.h"
Wedson Almeida Filho22d5eaa2018-12-16 00:38:49 +000017#include "hf/mpool.h"
Andrew Scull877ae4b2019-07-02 12:52:33 +010018#include "hf/static_assert.h"
Wedson Almeida Filhofed69022018-07-11 15:39:12 +010019
Karl Meakin07a69ab2025-02-07 14:53:19 +000020typedef uint32_t mm_mode_t;
21typedef uint64_t mm_attr_t;
22typedef uint8_t mm_level_t;
23typedef uint16_t mm_asid_t;
24
Andrew Scullc66a04d2018-12-07 13:41:56 +000025/* Keep macro alignment */
26/* clang-format off */
27
J-Alves715d6232023-02-16 16:33:28 +000028#define PAGE_SIZE ((size_t)(1 << PAGE_BITS))
Andrew Scull4e5f8142018-10-12 14:37:19 +010029#define MM_PTE_PER_PAGE (PAGE_SIZE / sizeof(pte_t))
30
Andrew Scullc66a04d2018-12-07 13:41:56 +000031/* The following are arch-independent page mapping modes. */
Karl Meakin07a69ab2025-02-07 14:53:19 +000032#define MM_MODE_R (1U << 0) /* read */
33#define MM_MODE_W (1U << 1) /* write */
34#define MM_MODE_X (1U << 2) /* execute */
35#define MM_MODE_D (1U << 3) /* device */
Andrew Scullc66a04d2018-12-07 13:41:56 +000036
37/*
38 * Memory in stage-1 is either valid (present) or invalid (absent).
39 *
40 * Memory in stage-2 has more states to track sharing, borrowing and giving of
41 * memory. The states are made up of three parts:
42 *
43 * 1. V = valid/invalid : Whether the memory is part of the VM's address
44 * space. A fault will be generated if accessed when
45 * invalid.
46 * 2. O = owned/unowned : Whether the memory is owned by the VM.
47 * 3. X = exclusive/shared : Whether access is exclusive to the VM or shared
48 * with at most one other.
49 *
50 * These parts compose to form the following state:
51 *
52 * - V O X : Owner of memory with exclusive access.
53 * - V O !X : Owner of memory with access shared with at most one other VM.
54 * - V !O X : Borrower of memory with exclusive access.
55 * - V !O !X : Borrower of memory where access is shared with the owner.
56 * - !V O X : Owner of memory lent to a VM that has exclusive access.
57 *
58 * - !V O !X : Unused. Owner of shared memory always has access.
Andrew Scull73b89542019-11-20 17:31:26 +000059 * - !V !O X : Unused. Next entry is used for invalid memory.
Andrew Scullc66a04d2018-12-07 13:41:56 +000060 *
Andrew Scullc66a04d2018-12-07 13:41:56 +000061 * - !V !O !X : Invalid memory. Memory is unrelated to the VM.
62 *
63 * Modes are selected so that owner of exclusive memory is the default.
64 */
Karl Meakin07a69ab2025-02-07 14:53:19 +000065#define MM_MODE_INVALID (1U << 4)
66#define MM_MODE_UNOWNED (1U << 5)
67#define MM_MODE_SHARED (1U << 6)
Raghu Krishnamurthy25daaea2021-02-10 13:19:16 -080068
69/* Map page as non-global. */
Karl Meakin07a69ab2025-02-07 14:53:19 +000070#define MM_MODE_NG (1U << 8)
71
72/* Specifies if a mapping will be a user mapping(EL0). */
73#define MM_MODE_USER (1U << 9)
Raghu Krishnamurthy25daaea2021-02-10 13:19:16 -080074
Andrew Scull73b89542019-11-20 17:31:26 +000075/* The mask for a mode that is considered unmapped. */
76#define MM_MODE_UNMAPPED_MASK (MM_MODE_INVALID | MM_MODE_UNOWNED)
77
Andrew Scullc66a04d2018-12-07 13:41:56 +000078/* clang-format on */
79
Karl Meakin1fd4b822025-02-01 17:13:47 +000080struct mm_flags {
81 bool commit : 1;
82 bool unmap : 1;
83 bool stage1 : 1;
84};
85
Andrew Walbran5de9c3d2020-02-10 13:35:29 +000086#define MM_PPOOL_ENTRY_SIZE sizeof(struct mm_page_table)
87
Andrew Scull4e5f8142018-10-12 14:37:19 +010088struct mm_page_table {
89 alignas(PAGE_SIZE) pte_t entries[MM_PTE_PER_PAGE];
90};
91static_assert(sizeof(struct mm_page_table) == PAGE_SIZE,
92 "A page table must take exactly one page.");
93static_assert(alignof(struct mm_page_table) == PAGE_SIZE,
94 "A page table must be page aligned.");
95
Wedson Almeida Filhofed69022018-07-11 15:39:12 +010096struct mm_ptable {
Raghu Krishnamurthy0132b512021-02-03 14:13:26 -080097 /**
98 * VMID/ASID associated with a page table. ASID 0 is reserved for use by
99 * the hypervisor.
100 */
Karl Meakin07a69ab2025-02-07 14:53:19 +0000101 mm_asid_t id;
Andrew Scull1ba470e2018-10-31 15:14:31 +0000102 /** Address of the root of the page table. */
103 paddr_t root;
Wedson Almeida Filhofed69022018-07-11 15:39:12 +0100104};
105
David Brazdil711fbe92019-08-06 13:39:58 +0100106/** The type of addresses stored in the page table. */
107typedef uintvaddr_t ptable_addr_t;
108
Andrew Scullba79b0a2019-07-03 11:26:53 +0100109/** Represents the currently locked stage-1 page table of the hypervisor. */
Andrew Scull3c0a90a2019-07-01 11:55:53 +0100110struct mm_stage1_locked {
111 struct mm_ptable *ptable;
112};
113
Andrew Scullda241972019-01-05 18:17:48 +0000114void mm_vm_enable_invalidation(void);
115
Karl Meakin07a69ab2025-02-07 14:53:19 +0000116bool mm_ptable_init(struct mm_ptable *ptable, mm_asid_t id,
Karl Meakin1fd4b822025-02-01 17:13:47 +0000117 struct mm_flags flags, struct mpool *ppool);
118ptable_addr_t mm_ptable_addr_space_end(struct mm_flags flags);
David Brazdil711fbe92019-08-06 13:39:58 +0100119
Karl Meakin07a69ab2025-02-07 14:53:19 +0000120bool mm_vm_init(struct mm_ptable *ptable, mm_asid_t id, struct mpool *ppool);
Karl Meakind64aaf82025-02-08 01:12:55 +0000121void mm_vm_fini(const struct mm_ptable *ptable, struct mpool *ppool);
Raghu Krishnamurthy43fe93a2021-01-31 16:38:38 -0800122
Karl Meakind64aaf82025-02-08 01:12:55 +0000123bool mm_identity_prepare(struct mm_ptable *ptable, paddr_t begin, paddr_t end,
Karl Meakin07a69ab2025-02-07 14:53:19 +0000124 mm_mode_t mode, struct mpool *ppool);
Karl Meakind64aaf82025-02-08 01:12:55 +0000125void *mm_identity_commit(struct mm_ptable *ptable, paddr_t begin, paddr_t end,
Karl Meakin07a69ab2025-02-07 14:53:19 +0000126 mm_mode_t mode, struct mpool *ppool);
Raghu Krishnamurthy43fe93a2021-01-31 16:38:38 -0800127
Karl Meakind64aaf82025-02-08 01:12:55 +0000128bool mm_vm_identity_map(struct mm_ptable *ptable, paddr_t begin, paddr_t end,
Karl Meakin07a69ab2025-02-07 14:53:19 +0000129 mm_mode_t mode, struct mpool *ppool, ipaddr_t *ipa);
Karl Meakind64aaf82025-02-08 01:12:55 +0000130bool mm_vm_identity_prepare(struct mm_ptable *ptable, paddr_t begin,
Karl Meakin07a69ab2025-02-07 14:53:19 +0000131 paddr_t end, mm_mode_t mode, struct mpool *ppool);
Karl Meakind64aaf82025-02-08 01:12:55 +0000132void mm_vm_identity_commit(struct mm_ptable *ptable, paddr_t begin, paddr_t end,
Karl Meakin07a69ab2025-02-07 14:53:19 +0000133 mm_mode_t mode, struct mpool *ppool, ipaddr_t *ipa);
Karl Meakind64aaf82025-02-08 01:12:55 +0000134bool mm_vm_unmap(struct mm_ptable *ptable, paddr_t begin, paddr_t end,
Wedson Almeida Filho22d5eaa2018-12-16 00:38:49 +0000135 struct mpool *ppool);
Karl Meakind64aaf82025-02-08 01:12:55 +0000136void mm_stage1_defrag(struct mm_ptable *ptable, struct mpool *ppool);
137void mm_vm_defrag(struct mm_ptable *ptable, struct mpool *ppool,
138 bool non_secure);
139void mm_vm_dump(const struct mm_ptable *ptable);
140bool mm_vm_get_mode(const struct mm_ptable *ptable, ipaddr_t begin,
Karl Meakin07a69ab2025-02-07 14:53:19 +0000141 ipaddr_t end, mm_mode_t *mode);
Karl Meakind64aaf82025-02-08 01:12:55 +0000142bool mm_get_mode(const struct mm_ptable *ptable, vaddr_t begin, vaddr_t end,
Karl Meakin07a69ab2025-02-07 14:53:19 +0000143 mm_mode_t *mode);
Andrew Scull80871322018-08-06 12:04:09 +0100144
Raghu Krishnamurthyd3ab8c32021-02-10 19:11:30 -0800145struct mm_stage1_locked mm_lock_ptable_unsafe(struct mm_ptable *ptable);
Andrew Scull3c0a90a2019-07-01 11:55:53 +0100146struct mm_stage1_locked mm_lock_stage1(void);
147void mm_unlock_stage1(struct mm_stage1_locked *lock);
148void *mm_identity_map(struct mm_stage1_locked stage1_locked, paddr_t begin,
Karl Meakin07a69ab2025-02-07 14:53:19 +0000149 paddr_t end, mm_mode_t mode, struct mpool *ppool);
Andrew Scull3c0a90a2019-07-01 11:55:53 +0100150bool mm_unmap(struct mm_stage1_locked stage1_locked, paddr_t begin, paddr_t end,
151 struct mpool *ppool);
152void mm_defrag(struct mm_stage1_locked stage1_locked, struct mpool *ppool);
153
Wedson Almeida Filho22d5eaa2018-12-16 00:38:49 +0000154bool mm_init(struct mpool *ppool);