Add support for notifying VMs when mailboxes become writable.
This allows VMs to know when they can retry sending messages to another
VM after an attempt fails because the recipient's mailbox is not
available.
Change-Id: Ib20159f7ecf81544e40149edcd663876c58851c7
diff --git a/inc/hf/api.h b/inc/hf/api.h
index f3e189a..8357a89 100644
--- a/inc/hf/api.h
+++ b/inc/hf/api.h
@@ -29,14 +29,16 @@
struct hf_vcpu_run_return api_vcpu_run(uint32_t vm_id, uint32_t vcpu_idx,
const struct vcpu *current,
struct vcpu **next);
-int64_t api_vm_configure(ipaddr_t send, ipaddr_t recv,
- const struct vcpu *current);
+int64_t api_vm_configure(ipaddr_t send, ipaddr_t recv, struct vcpu *current,
+ struct vcpu **next);
int64_t api_mailbox_send(uint32_t vm_id, size_t size, bool notify,
struct vcpu *current, struct vcpu **next);
struct hf_mailbox_receive_return api_mailbox_receive(bool block,
struct vcpu *current,
struct vcpu **next);
-int64_t api_mailbox_clear(const struct vcpu *current);
+int64_t api_mailbox_clear(struct vcpu *current, struct vcpu **next);
+int64_t api_mailbox_writable_get(const struct vcpu *current);
+int64_t api_mailbox_waiter_get(uint32_t vm_id, const struct vcpu *current);
struct vcpu *api_preempt(struct vcpu *current);
struct vcpu *api_yield(struct vcpu *current);
diff --git a/inc/hf/vm.h b/inc/hf/vm.h
index 6a140b8..75096a5 100644
--- a/inc/hf/vm.h
+++ b/inc/hf/vm.h
@@ -17,6 +17,7 @@
#pragma once
#include "hf/cpu.h"
+#include "hf/list.h"
#include "hf/mm.h"
#include "hf/mpool.h"
@@ -31,6 +32,23 @@
mailbox_state_read,
};
+struct wait_entry {
+ /** The VM that is waiting for a mailbox to become writable. */
+ struct vm *waiting_vm;
+
+ /**
+ * Links used to add entry to a VM's waiter_list. This is protected by
+ * the notifying VM's lock.
+ */
+ struct list_entry wait_links;
+
+ /**
+ * Links used to add entry to a VM's ready_list. This is protected by
+ * the waiting VM's lock.
+ */
+ struct list_entry ready_links;
+};
+
struct mailbox {
enum mailbox_state state;
uint32_t recv_from_id;
@@ -38,6 +56,20 @@
void *recv;
const void *send;
struct vcpu *recv_waiter;
+
+ /**
+ * List of wait_entry structs representing VMs that want to be notified
+ * when the mailbox becomes writable. Once the mailbox does become
+ * writable, the entry is removed from this list and added to the
+ * waiting VM's ready_list.
+ */
+ struct list_entry waiter_list;
+
+ /**
+ * List of wait_entry structs representing VMs whose mailboxes became
+ * writable since the owner of the mailbox registers for notification.
+ */
+ struct list_entry ready_list;
};
struct vm {
@@ -48,9 +80,19 @@
struct vcpu vcpus[MAX_CPUS];
struct mm_ptable ptable;
struct mailbox mailbox;
+
+ /** Wait entries to be used when waiting on other VM mailboxes. */
+ struct wait_entry wentry[MAX_VMS];
+};
+
+/** Encapsulates a VM whose lock is held. */
+struct vm_locked {
+ struct vm *vm;
};
bool vm_init(uint32_t vcpu_count, struct mpool *ppool, struct vm **new_vm);
uint32_t vm_get_count(void);
struct vm *vm_get(uint32_t id);
void vm_start_vcpu(struct vm *vm, size_t index, ipaddr_t entry, uintreg_t arg);
+void vm_lock(struct vm *vm, struct vm_locked *locked);
+void vm_unlock(struct vm_locked *locked);
diff --git a/inc/vmapi/hf/abi.h b/inc/vmapi/hf/abi.h
index 6f43360..451de41 100644
--- a/inc/vmapi/hf/abi.h
+++ b/inc/vmapi/hf/abi.h
@@ -64,6 +64,13 @@
* `hf_vcpu_run` on it again.
*/
HF_VCPU_RUN_SLEEP,
+
+ /**
+ * The vCPU has made the mailbox writable and there are pending waiters.
+ * The scheduler MUST call hf_mailbox_waiter_get() repeatedly and notify
+ * all waiters by injecting an HF_MAILBOX_WRITABLE_INTID interrupt.
+ */
+ HF_VCPU_RUN_NOTIFY_WAITERS,
};
struct hf_vcpu_run_return {
diff --git a/inc/vmapi/hf/call.h b/inc/vmapi/hf/call.h
index 3922f7a..8014f8c 100644
--- a/inc/vmapi/hf/call.h
+++ b/inc/vmapi/hf/call.h
@@ -23,18 +23,20 @@
/* clang-format off */
/* TODO: Define constants below according to spec. */
-#define HF_VM_GET_ID 0xff00
-#define HF_VM_GET_COUNT 0xff01
-#define HF_VCPU_GET_COUNT 0xff02
-#define HF_VCPU_RUN 0xff03
-#define HF_VCPU_YIELD 0xff04
-#define HF_VM_CONFIGURE 0xff05
-#define HF_MAILBOX_SEND 0xff06
-#define HF_MAILBOX_RECEIVE 0xff07
-#define HF_MAILBOX_CLEAR 0xff08
-#define HF_INTERRUPT_ENABLE 0xff09
-#define HF_INTERRUPT_GET 0xff0a
-#define HF_INTERRUPT_INJECT 0xff0b
+#define HF_VM_GET_ID 0xff00
+#define HF_VM_GET_COUNT 0xff01
+#define HF_VCPU_GET_COUNT 0xff02
+#define HF_VCPU_RUN 0xff03
+#define HF_VCPU_YIELD 0xff04
+#define HF_VM_CONFIGURE 0xff05
+#define HF_MAILBOX_SEND 0xff06
+#define HF_MAILBOX_RECEIVE 0xff07
+#define HF_MAILBOX_CLEAR 0xff08
+#define HF_MAILBOX_WRITABLE_GET 0xff09
+#define HF_MAILBOX_WAITER_GET 0xff0a
+#define HF_INTERRUPT_ENABLE 0xff0b
+#define HF_INTERRUPT_GET 0xff0c
+#define HF_INTERRUPT_INJECT 0xff0d
/** The amount of data that can be sent to a mailbox. */
#define HF_MAILBOX_SIZE 4096
@@ -95,7 +97,11 @@
* Configures the pages to send/receive data through. The pages must not be
* shared.
*
- * Returns 0 on success or -1 or failure.
+ * Returns:
+ * - -1 on failure.
+ * - 0 on success if no further action is needed.
+ * - 1 if it was called by the primary VM and the primary VM now needs to wake
+ * up or kick waiters.
*/
static inline int64_t hf_vm_configure(hf_ipaddr_t send, hf_ipaddr_t recv)
{
@@ -137,8 +143,12 @@
/**
* Clears the caller's mailbox so a new message can be received.
*
- * Returns 0 on success, or -1 if the mailbox hasn't been read or is already
- * empty.
+ * Returns:
+ * - -1 on failure, if the mailbox hasn't been read or is already empty.
+ * - 0 on success if no further action is needed.
+ * - 1 if it was called by the primary VM and the primary VM now needs to wake
+ * up or kick waiters. Waiters should be retrieved by calling
+ * hf_mailbox_waiter_get.
*/
static inline int64_t hf_mailbox_clear(void)
{
@@ -146,6 +156,34 @@
}
/**
+ * Retrieves the next VM whose mailbox became writable. For a VM to be notified
+ * by this function, the caller must have called api_mailbox_send before with
+ * the notify argument set to true, and this call must have failed because the
+ * mailbox was not available.
+ *
+ * It should be called repeatedly to retreive a list of VMs.
+ *
+ * Returns -1 if no VM became writable, or the id of the VM whose mailbox
+ * became writable.
+ */
+static inline int64_t hf_mailbox_writable_get(void)
+{
+ return hf_call(HF_MAILBOX_WRITABLE_GET, 0, 0, 0);
+}
+
+/**
+ * Retrieves the next VM waiting to be notified that the mailbox of the
+ * specified VM became writable. Only primary VMs are allowed to call this.
+ *
+ * Returns -1 if there are no waiters, or the VM id of the next waiter
+ * otherwise.
+ */
+static inline int64_t hf_mailbox_waiter_get(uint32_t vm_id)
+{
+ return hf_call(HF_MAILBOX_WAITER_GET, vm_id, 0, 0);
+}
+
+/**
* Enables or disables a given interrupt ID.
*
* Returns 0 on success, or -1 if the intid is invalid.
diff --git a/inc/vmapi/hf/types.h b/inc/vmapi/hf/types.h
index 697a990..f0b39d9 100644
--- a/inc/vmapi/hf/types.h
+++ b/inc/vmapi/hf/types.h
@@ -45,3 +45,9 @@
/** Interrupt ID returned when there is no interrupt pending. */
#define HF_INVALID_INTID 0xffffffff
+
+/** Interrupt ID indicating the mailbox is readable. */
+#define HF_MAILBOX_READBLE_INTID 1
+
+/** Interrupt ID indicating a mailbox is writable. */
+#define HF_MAILBOX_WRITABLE_INTID 2