Implement SPCI pull model for memory sharing.
Bug: 132420445
Change-Id: Iac21d7949bc54da13a4f25a317f5e83b2727638c
diff --git a/inc/hf/api.h b/inc/hf/api.h
index c5ebe2a..d6de2cb 100644
--- a/inc/hf/api.h
+++ b/inc/hf/api.h
@@ -65,3 +65,11 @@
uint32_t fragment_length, uint32_t length,
uint32_t cookie, struct vcpu *current,
struct vcpu **next);
+struct spci_value api_spci_mem_retrieve_req(ipaddr_t address,
+ uint32_t page_count,
+ uint32_t fragment_length,
+ uint32_t length, uint32_t cookie,
+ struct vcpu *current);
+struct spci_value api_spci_mem_relinquish(struct vcpu *current);
+struct spci_value api_spci_mem_reclaim(uint32_t handle, uint32_t flags,
+ struct vcpu *current);
diff --git a/inc/hf/mm.h b/inc/hf/mm.h
index 43d4c79..6fb9296 100644
--- a/inc/hf/mm.h
+++ b/inc/hf/mm.h
@@ -79,6 +79,8 @@
/* clang-format on */
+#define MM_PPOOL_ENTRY_SIZE sizeof(struct mm_page_table)
+
struct mm_page_table {
alignas(PAGE_SIZE) pte_t entries[MM_PTE_PER_PAGE];
};
diff --git a/inc/hf/spci_memory.h b/inc/hf/spci_memory.h
index fc21e09..023a8ce 100644
--- a/inc/hf/spci_memory.h
+++ b/inc/hf/spci_memory.h
@@ -21,9 +21,19 @@
#include "vmapi/hf/spci.h"
-struct spci_value spci_memory_send(struct vm_locked to_locked,
- struct vm_locked from_locked,
+struct spci_value spci_memory_send(struct vm *to, struct vm_locked from_locked,
struct spci_memory_region *memory_region,
uint32_t memory_share_size,
uint32_t share_func,
struct mpool *page_pool);
+struct spci_value spci_memory_retrieve(
+ struct vm_locked to_locked,
+ struct spci_memory_retrieve_request *retrieve_request,
+ uint32_t retrieve_request_size, struct mpool *page_pool);
+struct spci_value spci_memory_relinquish(
+ struct vm_locked from_locked,
+ struct spci_mem_relinquish *relinquish_request,
+ struct mpool *page_pool);
+struct spci_value spci_memory_reclaim(struct vm_locked to_locked,
+ spci_memory_handle_t handle, bool clear,
+ struct mpool *page_pool);
diff --git a/inc/vmapi/hf/abi.h b/inc/vmapi/hf/abi.h
index 2802479..ed004b7 100644
--- a/inc/vmapi/hf/abi.h
+++ b/inc/vmapi/hf/abi.h
@@ -35,9 +35,6 @@
#define HF_SPCI_RUN_WAIT_FOR_INTERRUPT 0xff09
#define HF_SPCI_RUN_WAKE_UP 0xff0a
-/* Custom SPCI-like call for relinquishing memory in the push model. */
-#define HF_SPCI_MEM_RELINQUISH 0xffab
-
/* This matches what Trusty and its ATF module currently use. */
#define HF_DEBUG_LOG 0xbd000000
diff --git a/inc/vmapi/hf/call.h b/inc/vmapi/hf/call.h
index 5a1e097..f3557b6 100644
--- a/inc/vmapi/hf/call.h
+++ b/inc/vmapi/hf/call.h
@@ -159,16 +159,28 @@
.arg5 = cookie});
}
-static inline struct spci_value hf_spci_mem_relinquish(uint32_t fragment_length,
- uint32_t length,
- uint32_t cookie)
+static inline struct spci_value spci_mem_retrieve_req(uint32_t fragment_length,
+ uint32_t length,
+ uint32_t cookie)
{
- return spci_call((struct spci_value){.func = HF_SPCI_MEM_RELINQUISH,
+ return spci_call((struct spci_value){.func = SPCI_MEM_RETRIEVE_REQ_32,
.arg3 = fragment_length,
.arg4 = length,
.arg5 = cookie});
}
+static inline struct spci_value spci_mem_relinquish(void)
+{
+ return spci_call((struct spci_value){.func = SPCI_MEM_RELINQUISH_32});
+}
+
+static inline struct spci_value spci_mem_reclaim(uint32_t handle,
+ uint32_t flags)
+{
+ return spci_call((struct spci_value){
+ .func = SPCI_MEM_RECLAIM_32, .arg1 = handle, .arg2 = flags});
+}
+
/**
* Called by secondary VMs to receive a message. This will block until a message
* is received.
diff --git a/inc/vmapi/hf/spci.h b/inc/vmapi/hf/spci.h
index 98fad84..0422415 100644
--- a/inc/vmapi/hf/spci.h
+++ b/inc/vmapi/hf/spci.h
@@ -47,6 +47,10 @@
#define SPCI_MEM_DONATE_32 0x84000071
#define SPCI_MEM_LEND_32 0x84000072
#define SPCI_MEM_SHARE_32 0x84000073
+#define SPCI_MEM_RETRIEVE_REQ_32 0x84000074
+#define SPCI_MEM_RETRIEVE_RESP_32 0x84000075
+#define SPCI_MEM_RELINQUISH_32 0x84000076
+#define SPCI_MEM_RECLAIM_32 0x84000077
/* SPCI error codes. */
#define SPCI_NOT_SUPPORTED INT32_C(-1)
@@ -65,6 +69,8 @@
#define SPCI_MSG_SEND_NOTIFY 0x1
#define SPCI_MSG_SEND_NOTIFY_MASK 0x1
+#define SPCI_MEM_RECLAIM_CLEAR 0x1
+
#define SPCI_SLEEP_INDEFINITE 0
/**
@@ -157,6 +163,10 @@
/* clang-format on */
+#define SPCI_MEMORY_HANDLE_ALLOCATOR_MASK ((spci_memory_handle_t)(1U << 31))
+#define SPCI_MEMORY_HANDLE_ALLOCATOR_HYPERVISOR \
+ ((spci_memory_handle_t)(1U << 31))
+
/** The ID of a VM. These are assigned sequentially starting with an offset. */
typedef uint16_t spci_vm_id_t;
typedef uint32_t spci_memory_handle_t;
@@ -208,6 +218,12 @@
return args.arg4;
}
+static inline spci_memory_handle_t spci_mem_success_handle(
+ struct spci_value args)
+{
+ return args.arg2;
+}
+
static inline spci_vm_id_t spci_vm_id(struct spci_value args)
{
return (args.arg1 >> 16) & 0xffff;
@@ -304,6 +320,132 @@
struct spci_memory_region_attributes attributes[];
};
+/**
+ * Retrieval attributes for a single receiver. This corresponds to table 138 of
+ * the SPCI beta specification, "Descriptor with properties to retrieve a memory
+ * region".
+ */
+struct spci_memory_retrieve_properties {
+ struct spci_memory_region_attributes attributes;
+ uint32_t page_count;
+ uint32_t constituent_count;
+ /** Reserved field, must be 0. */
+ uint32_t reserved;
+ struct spci_memory_region_constituent constituents[];
+};
+
+/**
+ * Descriptor used for SPCI_MEM_RETRIEVE_REQ. This corresponds to table 137 of
+ * the SPCI beta specification, "Descriptor to retrieve a donated, lent or
+ * shared memory region".
+ */
+struct spci_memory_retrieve_request {
+ /** Globally unique handle to identify the memory transaction. */
+ spci_memory_handle_t handle;
+ /** ID of the VM which sent the memory. */
+ spci_vm_id_t sender;
+ /** Reserved field, must be 0. */
+ uint16_t reserved_0;
+ /** Function ID of the memory sharing operation. */
+ uint32_t share_func;
+ /**
+ * An implementation defined value associated with the receiver and the
+ * memory region.
+ */
+ uint32_t tag;
+ /**
+ * The number of descriptors to specify the attributes with which the
+ * memory region will be mapped in the other recipients. Hafnium doesn't
+ * support multi-way memory sharing so this should always be 0.
+ */
+ uint32_t attribute_count;
+ /**
+ * The offset in bytes from the base address of this
+ * `spci_memory_retrieve_request` to the start of the first attribute
+ * descriptor.
+ */
+ uint32_t attribute_offset;
+ /**
+ * The number of `spci_memory_retrieve_properties` entries included
+ * in this retrieve request, i.e. the number of receivers including the
+ * caller and any stream endpoints for which the caller is a proxy.
+ * Hafnium doesn't support stream endpoints, so this should always be 1.
+ */
+ uint32_t retrieve_properties_count;
+ /** Reserved field, must be 0. */
+ uint32_t reserved_1;
+ /*
+ * 'Array' of `struct spci_memory_retrieve_properties` goes here, but it
+ * is not included in the struct as the entries are variable length. Use
+ * `spci_memory_retrieve_request_first_retrieve_properties` to get the
+ * first one.
+ */
+};
+
+/**
+ * Receiver address range descriptor. This corresponds to table 144 of the SPCI
+ * beta specification, "Descriptor with address ranges of retrieved memory
+ * region", and is included as part of the `spci_retrieved_memory_region`.
+ */
+struct spci_receiver_address_range {
+ /** Receiver VM ID. */
+ spci_vm_id_t receiver;
+ /** Reserved field, must be 0. */
+ uint16_t reserved_0;
+ /**
+ * The total number of 4 kiB pages included in this memory region. This
+ * must be equal to the sum of page counts specified in each
+ * `spci_memory_region_constituent`.
+ */
+ uint32_t page_count;
+ /**
+ * The number of constituents (`spci_memory_region_constituent`)
+ * included in this memory region range.
+ */
+ uint32_t constituent_count;
+ /** Reserved field, must be 0. */
+ uint32_t reserved_1;
+ /** An array of `constituent_count` memory region constituents. */
+ struct spci_memory_region_constituent constituents[];
+};
+
+/**
+ * Descriptor used for SPCI_MEM_RETRIEVE_RESP. This corresponds to table 143 of
+ * the SPCI beta specification, "Encoding of mapped address ranges of retrieved
+ * memory region".
+ */
+struct spci_retrieved_memory_region {
+ /**
+ * The number of `spci_receiver_address_range` entries included
+ * in this memory region.
+ */
+ uint32_t receiver_count;
+ /** Reserved field, must be 0. */
+ uint32_t reserved_0;
+ /** Reserved field, must be 0. */
+ uint64_t reserved_1;
+ /*
+ * 'Array' of `struct spci_receiver_address_range` goes here, but it is
+ * not included in the struct as the entries are variable length. Use
+ * `spci_retrieved_memory_region_first_receiver_range` to get the first
+ * one.
+ */
+};
+
+/**
+ * Descriptor used for SPCI_MEM_RELINQUISH requests. This corresponds to table
+ * 146 of the SPCI beta specification, "Descriptor to relinquish a memory
+ * region".
+ */
+struct spci_mem_relinquish {
+ spci_memory_handle_t handle;
+ uint32_t flags;
+ spci_vm_id_t sender;
+ uint16_t reserved;
+ uint32_t endpoint_count;
+ spci_vm_id_t endpoints[];
+};
+
static inline struct spci_memory_region_constituent
spci_memory_region_constituent_init(uint64_t address, uint32_t page_count)
{
@@ -332,6 +474,33 @@
memory_region->constituent_offset);
}
+/**
+ * Gets the first descriptor with address ranges of a retrieved memory region.
+ *
+ * Note that getting the second requires parsing the first, as they are variable
+ * length due to the variable number of constituents.
+ */
+static inline struct spci_receiver_address_range *
+spci_retrieved_memory_region_first_receiver_range(
+ struct spci_retrieved_memory_region *memory_region)
+{
+ return (struct spci_receiver_address_range *)(memory_region + 1);
+}
+
+/**
+ * Gets the first retrieval attributes descriptor of a memory region retrieval
+ * request.
+ *
+ * Note that getting the second requires parsing the first, as they are variable
+ * length due to the variable number of constituents.
+ */
+static inline struct spci_memory_retrieve_properties *
+spci_memory_retrieve_request_first_retrieve_properties(
+ struct spci_memory_retrieve_request *retrieve_request)
+{
+ return (struct spci_memory_retrieve_properties *)(retrieve_request + 1);
+}
+
uint32_t spci_memory_region_init(
struct spci_memory_region *memory_region, spci_vm_id_t sender,
spci_vm_id_t receiver,
@@ -340,3 +509,15 @@
spci_memory_region_flags_t flags, enum spci_memory_access access,
enum spci_memory_type type, enum spci_memory_cacheability cacheability,
enum spci_memory_shareability shareability);
+uint32_t spci_memory_retrieve_request_init(
+ struct spci_memory_retrieve_request *request,
+ spci_memory_handle_t handle, spci_vm_id_t sender, spci_vm_id_t receiver,
+ uint32_t share_func, uint32_t tag, uint32_t page_count,
+ enum spci_memory_access access, enum spci_memory_type type,
+ enum spci_memory_cacheability cacheability,
+ enum spci_memory_shareability shareability);
+uint32_t spci_retrieved_memory_region_init(
+ struct spci_retrieved_memory_region *response, size_t response_max_size,
+ spci_vm_id_t receiver,
+ const struct spci_memory_region_constituent constituents[],
+ uint32_t constituent_count, uint32_t page_count);
diff --git a/inc/vmapi/hf/types.h b/inc/vmapi/hf/types.h
index dc9f7a1..d0989c7 100644
--- a/inc/vmapi/hf/types.h
+++ b/inc/vmapi/hf/types.h
@@ -35,6 +35,8 @@
#endif
+#define HF_HYPERVISOR_VM_ID 0
+
/**
* An offset to use when assigning VM IDs.
* The offset is needed because VM ID 0 is reserved.