blob: dae8683c7c84ab30a5c9718d1d76a21a8f529863 [file] [log] [blame]
Andrew Scull18834872018-10-12 11:48:09 +01001/*
Andrew Walbran692b3252019-03-07 15:51:31 +00002 * Copyright 2018 The Hafnium Authors.
Andrew Scull18834872018-10-12 11:48:09 +01003 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * https://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
Andrew Scull18c78fc2018-08-20 12:57:41 +010017#include "hf/api.h"
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +010018
Andrew Walbran318f5732018-11-20 16:23:42 +000019#include "hf/arch/cpu.h"
Andrew Walbran508e63c2018-12-20 17:02:37 +000020#include "hf/arch/timer.h"
Andrew Walbran318f5732018-11-20 16:23:42 +000021
Andrew Scull5c496a32019-04-04 11:57:33 +010022#include "hf/assert.h"
Andrew Walbran318f5732018-11-20 16:23:42 +000023#include "hf/dlog.h"
Andrew Scull6386f252018-12-06 13:29:10 +000024#include "hf/mm.h"
Andrew Walbranb037d5b2019-06-25 17:19:41 +010025#include "hf/spci.h"
Andrew Scull6386f252018-12-06 13:29:10 +000026#include "hf/spinlock.h"
Andrew Scull8d9e1212019-04-05 13:52:55 +010027#include "hf/std.h"
Andrew Scull18c78fc2018-08-20 12:57:41 +010028#include "hf/vm.h"
29
Andrew Scullf35a5c92018-08-07 18:09:46 +010030#include "vmapi/hf/call.h"
Jose Marinhoa1dfeda2019-02-27 16:46:03 +000031#include "vmapi/hf/spci.h"
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +010032
Wedson Almeida Filhoba641ef2018-12-03 04:19:44 +000033/*
34 * To eliminate the risk of deadlocks, we define a partial order for the
35 * acquisition of locks held concurrently by the same physical CPU. Our current
36 * ordering requirements are as follows:
37 *
Andrew Scull3c0a90a2019-07-01 11:55:53 +010038 * vm::stage1_locked -> vcpu::stage1_locked -> mm_stage1_lock
Andrew Scull6386f252018-12-06 13:29:10 +000039 *
Andrew Scull3c0a90a2019-07-01 11:55:53 +010040 * Locks of the same kind require the stage1_locked of lowest address to be
41 * locked first, see `sl_lock_both()`.
Wedson Almeida Filhoba641ef2018-12-03 04:19:44 +000042 */
43
Andrew Scullaa039b32018-10-04 15:02:26 +010044static_assert(HF_MAILBOX_SIZE == PAGE_SIZE,
Andrew Scull13652af2018-09-17 14:49:08 +010045 "Currently, a page is mapped for the send and receive buffers so "
46 "the maximum request is the size of a page.");
47
Wedson Almeida Filho9ed8da52018-12-17 16:09:11 +000048static struct mpool api_page_pool;
Wedson Almeida Filho22d5eaa2018-12-16 00:38:49 +000049
50/**
Wedson Almeida Filho81568c42019-01-04 13:33:02 +000051 * Initialises the API page pool by taking ownership of the contents of the
52 * given page pool.
Wedson Almeida Filho22d5eaa2018-12-16 00:38:49 +000053 */
54void api_init(struct mpool *ppool)
55{
Wedson Almeida Filho9ed8da52018-12-17 16:09:11 +000056 mpool_init_from(&api_page_pool, ppool);
Wedson Almeida Filho22d5eaa2018-12-16 00:38:49 +000057}
58
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +010059/**
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +010060 * Switches the physical CPU back to the corresponding vcpu of the primary VM.
Andrew Scullaa039b32018-10-04 15:02:26 +010061 *
62 * This triggers the scheduling logic to run. Run in the context of secondary VM
63 * to cause HF_VCPU_RUN to return and the primary VM to regain control of the
64 * cpu.
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +010065 */
Wedson Almeida Filho00df6c72018-10-18 11:19:24 +010066static struct vcpu *api_switch_to_primary(struct vcpu *current,
Wedson Almeida Filhoba641ef2018-12-03 04:19:44 +000067 struct hf_vcpu_run_return primary_ret,
68 enum vcpu_state secondary_state)
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +010069{
Andrew Walbran42347a92019-05-09 13:59:03 +010070 struct vm *primary = vm_find(HF_PRIMARY_VM_ID);
Andrew Walbrane1310df2019-04-29 17:28:28 +010071 struct vcpu *next = vm_get_vcpu(primary, cpu_index(current->cpu));
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +010072
Andrew Walbran508e63c2018-12-20 17:02:37 +000073 /*
74 * If the secondary is blocked but has a timer running, sleep until the
75 * timer fires rather than indefinitely.
76 */
Andrew Scullb06d1752019-02-04 10:15:48 +000077 switch (primary_ret.code) {
78 case HF_VCPU_RUN_WAIT_FOR_INTERRUPT:
79 case HF_VCPU_RUN_WAIT_FOR_MESSAGE:
80 primary_ret.sleep.ns =
81 arch_timer_enabled_current()
82 ? arch_timer_remaining_ns_current()
83 : HF_SLEEP_INDEFINITE;
84 break;
85
86 default:
87 /* Do nothing. */
88 break;
Andrew Walbran508e63c2018-12-20 17:02:37 +000089 }
90
Wedson Almeida Filho00df6c72018-10-18 11:19:24 +010091 /* Set the return value for the primary VM's call to HF_VCPU_RUN. */
Andrew Scull6d2db332018-10-10 15:28:17 +010092 arch_regs_set_retval(&next->regs,
93 hf_vcpu_run_return_encode(primary_ret));
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +010094
Wedson Almeida Filhoba641ef2018-12-03 04:19:44 +000095 /* Mark the current vcpu as waiting. */
96 sl_lock(&current->lock);
97 current->state = secondary_state;
98 sl_unlock(&current->lock);
99
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100100 return next;
101}
102
103/**
Andrew Scull33fecd32019-01-08 14:48:27 +0000104 * Returns to the primary vm and signals that the vcpu still has work to do so.
105 */
106struct vcpu *api_preempt(struct vcpu *current)
107{
108 struct hf_vcpu_run_return ret = {
109 .code = HF_VCPU_RUN_PREEMPTED,
110 };
111
Andrew Sculld6ee1102019-04-05 22:12:42 +0100112 return api_switch_to_primary(current, ret, VCPU_STATE_READY);
Andrew Scull33fecd32019-01-08 14:48:27 +0000113}
114
115/**
Andrew Scullaa039b32018-10-04 15:02:26 +0100116 * Puts the current vcpu in wait for interrupt mode, and returns to the primary
117 * vm.
118 */
Wedson Almeida Filho00df6c72018-10-18 11:19:24 +0100119struct vcpu *api_wait_for_interrupt(struct vcpu *current)
Andrew Scullaa039b32018-10-04 15:02:26 +0100120{
Andrew Scull6d2db332018-10-10 15:28:17 +0100121 struct hf_vcpu_run_return ret = {
122 .code = HF_VCPU_RUN_WAIT_FOR_INTERRUPT,
123 };
Wedson Almeida Filho81568c42019-01-04 13:33:02 +0000124
Wedson Almeida Filhoba641ef2018-12-03 04:19:44 +0000125 return api_switch_to_primary(current, ret,
Andrew Sculld6ee1102019-04-05 22:12:42 +0100126 VCPU_STATE_BLOCKED_INTERRUPT);
Andrew Scullaa039b32018-10-04 15:02:26 +0100127}
128
129/**
Andrew Walbran33645652019-04-15 12:29:31 +0100130 * Puts the current vCPU in off mode, and returns to the primary VM.
131 */
132struct vcpu *api_vcpu_off(struct vcpu *current)
133{
134 struct hf_vcpu_run_return ret = {
135 .code = HF_VCPU_RUN_WAIT_FOR_INTERRUPT,
136 };
137
138 /*
139 * Disable the timer, so the scheduler doesn't get told to call back
140 * based on it.
141 */
142 arch_timer_disable_current();
143
144 return api_switch_to_primary(current, ret, VCPU_STATE_OFF);
145}
146
147/**
Andrew Scull66d62bf2019-02-01 13:54:10 +0000148 * Returns to the primary vm to allow this cpu to be used for other tasks as the
149 * vcpu does not have work to do at this moment. The current vcpu is marked as
Jose Marinho135dff32019-02-28 10:25:57 +0000150 * ready to be scheduled again. This SPCI function always returns SPCI_SUCCESS.
Andrew Scull66d62bf2019-02-01 13:54:10 +0000151 */
Jose Marinho135dff32019-02-28 10:25:57 +0000152int32_t api_spci_yield(struct vcpu *current, struct vcpu **next)
Andrew Scull66d62bf2019-02-01 13:54:10 +0000153{
154 struct hf_vcpu_run_return ret = {
155 .code = HF_VCPU_RUN_YIELD,
156 };
157
158 if (current->vm->id == HF_PRIMARY_VM_ID) {
Andrew Scullb06d1752019-02-04 10:15:48 +0000159 /* Noop on the primary as it makes the scheduling decisions. */
Jose Marinho135dff32019-02-28 10:25:57 +0000160 return SPCI_SUCCESS;
Andrew Scull66d62bf2019-02-01 13:54:10 +0000161 }
162
Jose Marinho135dff32019-02-28 10:25:57 +0000163 *next = api_switch_to_primary(current, ret, VCPU_STATE_READY);
164
165 /* SPCI_YIELD always returns SPCI_SUCCESS. */
166 return SPCI_SUCCESS;
Andrew Scull66d62bf2019-02-01 13:54:10 +0000167}
168
169/**
Andrew Walbran33645652019-04-15 12:29:31 +0100170 * Switches to the primary so that it can switch to the target, or kick it if it
171 * is already running on a different physical CPU.
172 */
173struct vcpu *api_wake_up(struct vcpu *current, struct vcpu *target_vcpu)
174{
175 struct hf_vcpu_run_return ret = {
176 .code = HF_VCPU_RUN_WAKE_UP,
177 .wake_up.vm_id = target_vcpu->vm->id,
178 .wake_up.vcpu = vcpu_index(target_vcpu),
179 };
180 return api_switch_to_primary(current, ret, VCPU_STATE_READY);
181}
182
183/**
Andrew Scull38772ab2019-01-24 15:16:50 +0000184 * Aborts the vCPU and triggers its VM to abort fully.
Andrew Scull9726c252019-01-23 13:44:19 +0000185 */
186struct vcpu *api_abort(struct vcpu *current)
187{
188 struct hf_vcpu_run_return ret = {
189 .code = HF_VCPU_RUN_ABORTED,
190 };
191
192 dlog("Aborting VM %u vCPU %u\n", current->vm->id, vcpu_index(current));
193
194 if (current->vm->id == HF_PRIMARY_VM_ID) {
195 /* TODO: what to do when the primary aborts? */
196 for (;;) {
197 /* Do nothing. */
198 }
199 }
200
201 atomic_store_explicit(&current->vm->aborting, true,
202 memory_order_relaxed);
203
204 /* TODO: free resources once all vCPUs abort. */
205
Andrew Sculld6ee1102019-04-05 22:12:42 +0100206 return api_switch_to_primary(current, ret, VCPU_STATE_ABORTED);
Andrew Scull9726c252019-01-23 13:44:19 +0000207}
208
209/**
Andrew Scull55c4d8b2018-12-18 18:50:18 +0000210 * Returns the ID of the VM.
211 */
Andrew Walbranfc6cd9d2019-06-25 15:23:27 +0100212spci_vm_id_t api_vm_get_id(const struct vcpu *current)
Andrew Scull55c4d8b2018-12-18 18:50:18 +0000213{
214 return current->vm->id;
215}
216
217/**
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +0100218 * Returns the number of VMs configured to run.
219 */
Andrew Walbran52d99672019-06-25 15:51:11 +0100220spci_vm_count_t api_vm_get_count(void)
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +0100221{
Andrew Scull19503262018-09-20 14:48:39 +0100222 return vm_get_count();
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +0100223}
224
225/**
226 * Returns the number of vcpus configured in the given VM.
227 */
Andrew Walbran42347a92019-05-09 13:59:03 +0100228int64_t api_vcpu_get_count(spci_vm_id_t vm_id, const struct vcpu *current)
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +0100229{
Andrew Scull19503262018-09-20 14:48:39 +0100230 struct vm *vm;
231
232 /* Only the primary VM needs to know about vcpus for scheduling. */
Wedson Almeida Filho00df6c72018-10-18 11:19:24 +0100233 if (current->vm->id != HF_PRIMARY_VM_ID) {
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +0100234 return -1;
Andrew Scull7364a8e2018-07-19 15:39:29 +0100235 }
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +0100236
Andrew Walbran42347a92019-05-09 13:59:03 +0100237 vm = vm_find(vm_id);
Andrew Scull19503262018-09-20 14:48:39 +0100238 if (vm == NULL) {
239 return -1;
240 }
241
242 return vm->vcpu_count;
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +0100243}
244
245/**
Wedson Almeida Filho03306112018-11-26 00:08:03 +0000246 * This function is called by the architecture-specific context switching
247 * function to indicate that register state for the given vcpu has been saved
248 * and can therefore be used by other pcpus.
249 */
250void api_regs_state_saved(struct vcpu *vcpu)
251{
252 sl_lock(&vcpu->lock);
253 vcpu->regs_available = true;
254 sl_unlock(&vcpu->lock);
255}
256
257/**
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +0000258 * Retrieves the next waiter and removes it from the wait list if the VM's
259 * mailbox is in a writable state.
260 */
261static struct wait_entry *api_fetch_waiter(struct vm_locked locked_vm)
262{
263 struct wait_entry *entry;
264 struct vm *vm = locked_vm.vm;
265
Andrew Sculld6ee1102019-04-05 22:12:42 +0100266 if (vm->mailbox.state != MAILBOX_STATE_EMPTY ||
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +0000267 vm->mailbox.recv == NULL || list_empty(&vm->mailbox.waiter_list)) {
268 /* The mailbox is not writable or there are no waiters. */
269 return NULL;
270 }
271
272 /* Remove waiter from the wait list. */
273 entry = CONTAINER_OF(vm->mailbox.waiter_list.next, struct wait_entry,
274 wait_links);
275 list_remove(&entry->wait_links);
276 return entry;
277}
278
279/**
Andrew Walbran508e63c2018-12-20 17:02:37 +0000280 * Assuming that the arguments have already been checked by the caller, injects
281 * a virtual interrupt of the given ID into the given target vCPU. This doesn't
282 * cause the vCPU to actually be run immediately; it will be taken when the vCPU
283 * is next run, which is up to the scheduler.
284 *
285 * Returns:
286 * - 0 on success if no further action is needed.
287 * - 1 if it was called by the primary VM and the primary VM now needs to wake
288 * up or kick the target vCPU.
289 */
Andrew Walbranfc9d4382019-05-10 18:07:21 +0100290static int64_t internal_interrupt_inject(struct vcpu *target_vcpu,
Andrew Walbran508e63c2018-12-20 17:02:37 +0000291 uint32_t intid, struct vcpu *current,
292 struct vcpu **next)
293{
294 uint32_t intid_index = intid / INTERRUPT_REGISTER_BITS;
295 uint32_t intid_mask = 1u << (intid % INTERRUPT_REGISTER_BITS);
Andrew Walbran508e63c2018-12-20 17:02:37 +0000296 int64_t ret = 0;
297
298 sl_lock(&target_vcpu->lock);
Andrew Walbran508e63c2018-12-20 17:02:37 +0000299
300 /*
301 * We only need to change state and (maybe) trigger a virtual IRQ if it
302 * is enabled and was not previously pending. Otherwise we can skip
303 * everything except setting the pending bit.
304 *
305 * If you change this logic make sure to update the need_vm_lock logic
306 * above to match.
307 */
308 if (!(target_vcpu->interrupts.interrupt_enabled[intid_index] &
309 ~target_vcpu->interrupts.interrupt_pending[intid_index] &
310 intid_mask)) {
311 goto out;
312 }
313
314 /* Increment the count. */
315 target_vcpu->interrupts.enabled_and_pending_count++;
316
317 /*
318 * Only need to update state if there was not already an
319 * interrupt enabled and pending.
320 */
321 if (target_vcpu->interrupts.enabled_and_pending_count != 1) {
322 goto out;
323 }
324
Andrew Walbran508e63c2018-12-20 17:02:37 +0000325 if (current->vm->id == HF_PRIMARY_VM_ID) {
326 /*
327 * If the call came from the primary VM, let it know that it
328 * should run or kick the target vCPU.
329 */
330 ret = 1;
331 } else if (current != target_vcpu && next != NULL) {
Andrew Walbran33645652019-04-15 12:29:31 +0100332 *next = api_wake_up(current, target_vcpu);
Andrew Walbran508e63c2018-12-20 17:02:37 +0000333 }
334
335out:
336 /* Either way, make it pending. */
337 target_vcpu->interrupts.interrupt_pending[intid_index] |= intid_mask;
338
339 sl_unlock(&target_vcpu->lock);
Andrew Walbran508e63c2018-12-20 17:02:37 +0000340
341 return ret;
342}
343
344/**
Wedson Almeida Filho03306112018-11-26 00:08:03 +0000345 * Prepares the vcpu to run by updating its state and fetching whether a return
346 * value needs to be forced onto the vCPU.
347 */
Andrew Scull38772ab2019-01-24 15:16:50 +0000348static bool api_vcpu_prepare_run(const struct vcpu *current, struct vcpu *vcpu,
Andrew Walbran508e63c2018-12-20 17:02:37 +0000349 struct hf_vcpu_run_return *run_ret)
Wedson Almeida Filho03306112018-11-26 00:08:03 +0000350{
Andrew Scullb06d1752019-02-04 10:15:48 +0000351 bool need_vm_lock;
Wedson Almeida Filho03306112018-11-26 00:08:03 +0000352 bool ret;
353
Andrew Scullb06d1752019-02-04 10:15:48 +0000354 /*
355 * Wait until the registers become available. All locks must be
356 * released between iterations of this loop to avoid potential deadlocks
Andrew Scull3c0a90a2019-07-01 11:55:53 +0100357 * if, on any path, a stage1_locked needs to be taken after taking the
358 * decision to switch context but before the registers have been saved.
Andrew Scullb06d1752019-02-04 10:15:48 +0000359 *
Andrew Scull3c0a90a2019-07-01 11:55:53 +0100360 * The VM stage1_locked is not needed in the common case so it must only
361 * be taken when it is going to be needed. This ensures there are no
362 * inter-vCPU dependencies in the common run case meaning the sensitive
363 * context switch performance is consistent.
Andrew Scullb06d1752019-02-04 10:15:48 +0000364 */
365 for (;;) {
366 sl_lock(&vcpu->lock);
367
368 /* The VM needs to be locked to deliver mailbox messages. */
Andrew Sculld6ee1102019-04-05 22:12:42 +0100369 need_vm_lock = vcpu->state == VCPU_STATE_BLOCKED_MAILBOX;
Andrew Scullb06d1752019-02-04 10:15:48 +0000370 if (need_vm_lock) {
371 sl_unlock(&vcpu->lock);
372 sl_lock(&vcpu->vm->lock);
373 sl_lock(&vcpu->lock);
374 }
375
376 if (vcpu->regs_available) {
377 break;
378 }
379
Andrew Sculld6ee1102019-04-05 22:12:42 +0100380 if (vcpu->state == VCPU_STATE_RUNNING) {
Andrew Scullb06d1752019-02-04 10:15:48 +0000381 /*
382 * vCPU is running on another pCPU.
383 *
Andrew Walbranabf88fb2019-06-21 12:17:47 +0100384 * It's ok not to return the sleep duration here because
Andrew Scullb06d1752019-02-04 10:15:48 +0000385 * the other physical CPU that is currently running this
Andrew Walbranabf88fb2019-06-21 12:17:47 +0100386 * vCPU will return the sleep duration if needed. The
387 * default return value is
388 * HF_VCPU_RUN_WAIT_FOR_INTERRUPT, so no need to set it
389 * explicitly.
Andrew Scullb06d1752019-02-04 10:15:48 +0000390 */
391 ret = false;
392 goto out;
393 }
394
395 sl_unlock(&vcpu->lock);
396 if (need_vm_lock) {
397 sl_unlock(&vcpu->vm->lock);
398 }
399 }
Andrew Scull9726c252019-01-23 13:44:19 +0000400
401 if (atomic_load_explicit(&vcpu->vm->aborting, memory_order_relaxed)) {
Andrew Sculld6ee1102019-04-05 22:12:42 +0100402 if (vcpu->state != VCPU_STATE_ABORTED) {
Andrew Scull82331282019-01-25 10:29:34 +0000403 dlog("Aborting VM %u vCPU %u\n", vcpu->vm->id,
404 vcpu_index(vcpu));
Andrew Sculld6ee1102019-04-05 22:12:42 +0100405 vcpu->state = VCPU_STATE_ABORTED;
Andrew Scull9726c252019-01-23 13:44:19 +0000406 }
407 ret = false;
408 goto out;
409 }
410
Andrew Walbran508e63c2018-12-20 17:02:37 +0000411 switch (vcpu->state) {
Andrew Sculld6ee1102019-04-05 22:12:42 +0100412 case VCPU_STATE_RUNNING:
413 case VCPU_STATE_OFF:
414 case VCPU_STATE_ABORTED:
Wedson Almeida Filho03306112018-11-26 00:08:03 +0000415 ret = false;
416 goto out;
Andrew Scullb06d1752019-02-04 10:15:48 +0000417
Andrew Sculld6ee1102019-04-05 22:12:42 +0100418 case VCPU_STATE_BLOCKED_MAILBOX:
Andrew Scullb06d1752019-02-04 10:15:48 +0000419 /*
420 * A pending message allows the vCPU to run so the message can
421 * be delivered directly.
422 */
Andrew Sculld6ee1102019-04-05 22:12:42 +0100423 if (vcpu->vm->mailbox.state == MAILBOX_STATE_RECEIVED) {
Jose Marinho3e2442f2019-03-12 13:30:37 +0000424 arch_regs_set_retval(&vcpu->regs, SPCI_SUCCESS);
Andrew Sculld6ee1102019-04-05 22:12:42 +0100425 vcpu->vm->mailbox.state = MAILBOX_STATE_READ;
Andrew Scullb06d1752019-02-04 10:15:48 +0000426 break;
427 }
428 /* Fall through. */
Andrew Sculld6ee1102019-04-05 22:12:42 +0100429 case VCPU_STATE_BLOCKED_INTERRUPT:
Andrew Scullb06d1752019-02-04 10:15:48 +0000430 /* Allow virtual interrupts to be delivered. */
431 if (vcpu->interrupts.enabled_and_pending_count > 0) {
432 break;
433 }
434
435 /* The timer expired so allow the interrupt to be delivered. */
Andrew Walbran508e63c2018-12-20 17:02:37 +0000436 if (arch_timer_pending(&vcpu->regs)) {
437 break;
438 }
439
440 /*
441 * The vCPU is not ready to run, return the appropriate code to
442 * the primary which called vcpu_run.
443 */
444 if (arch_timer_enabled(&vcpu->regs)) {
Andrew Scullb06d1752019-02-04 10:15:48 +0000445 run_ret->code =
Andrew Sculld6ee1102019-04-05 22:12:42 +0100446 vcpu->state == VCPU_STATE_BLOCKED_MAILBOX
Andrew Scullb06d1752019-02-04 10:15:48 +0000447 ? HF_VCPU_RUN_WAIT_FOR_MESSAGE
448 : HF_VCPU_RUN_WAIT_FOR_INTERRUPT;
Andrew Walbran508e63c2018-12-20 17:02:37 +0000449 run_ret->sleep.ns =
450 arch_timer_remaining_ns(&vcpu->regs);
451 }
452
453 ret = false;
454 goto out;
Andrew Scullb06d1752019-02-04 10:15:48 +0000455
Andrew Sculld6ee1102019-04-05 22:12:42 +0100456 case VCPU_STATE_READY:
Andrew Walbran508e63c2018-12-20 17:02:37 +0000457 break;
Wedson Almeida Filho03306112018-11-26 00:08:03 +0000458 }
459
Andrew Scullb06d1752019-02-04 10:15:48 +0000460 /* It has been decided that the vCPU should be run. */
Wedson Almeida Filho03306112018-11-26 00:08:03 +0000461 vcpu->cpu = current->cpu;
Andrew Sculld6ee1102019-04-05 22:12:42 +0100462 vcpu->state = VCPU_STATE_RUNNING;
Wedson Almeida Filho03306112018-11-26 00:08:03 +0000463
Wedson Almeida Filho03306112018-11-26 00:08:03 +0000464 /*
Wedson Almeida Filho03306112018-11-26 00:08:03 +0000465 * Mark the registers as unavailable now that we're about to reflect
466 * them onto the real registers. This will also prevent another physical
467 * CPU from trying to read these registers.
468 */
469 vcpu->regs_available = false;
470
471 ret = true;
472
473out:
474 sl_unlock(&vcpu->lock);
Andrew Scullb06d1752019-02-04 10:15:48 +0000475 if (need_vm_lock) {
476 sl_unlock(&vcpu->vm->lock);
477 }
478
Wedson Almeida Filho03306112018-11-26 00:08:03 +0000479 return ret;
480}
481
482/**
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +0100483 * Runs the given vcpu of the given vm.
484 */
Andrew Walbranb037d5b2019-06-25 17:19:41 +0100485struct hf_vcpu_run_return api_vcpu_run(spci_vm_id_t vm_id,
486 spci_vcpu_index_t vcpu_idx,
Andrew Scull38772ab2019-01-24 15:16:50 +0000487 const struct vcpu *current,
488 struct vcpu **next)
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +0100489{
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100490 struct vm *vm;
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +0100491 struct vcpu *vcpu;
Andrew Scull6d2db332018-10-10 15:28:17 +0100492 struct hf_vcpu_run_return ret = {
493 .code = HF_VCPU_RUN_WAIT_FOR_INTERRUPT,
Andrew Scullb06d1752019-02-04 10:15:48 +0000494 .sleep.ns = HF_SLEEP_INDEFINITE,
Andrew Scull6d2db332018-10-10 15:28:17 +0100495 };
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +0100496
497 /* Only the primary VM can switch vcpus. */
Wedson Almeida Filho00df6c72018-10-18 11:19:24 +0100498 if (current->vm->id != HF_PRIMARY_VM_ID) {
Andrew Scull6d2db332018-10-10 15:28:17 +0100499 goto out;
Andrew Scull7364a8e2018-07-19 15:39:29 +0100500 }
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +0100501
Andrew Scull19503262018-09-20 14:48:39 +0100502 /* Only secondary VM vcpus can be run. */
503 if (vm_id == HF_PRIMARY_VM_ID) {
Andrew Scull6d2db332018-10-10 15:28:17 +0100504 goto out;
Andrew Scull7364a8e2018-07-19 15:39:29 +0100505 }
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +0100506
Andrew Scull19503262018-09-20 14:48:39 +0100507 /* The requested VM must exist. */
Andrew Walbran42347a92019-05-09 13:59:03 +0100508 vm = vm_find(vm_id);
Andrew Scull19503262018-09-20 14:48:39 +0100509 if (vm == NULL) {
Andrew Scull6d2db332018-10-10 15:28:17 +0100510 goto out;
Andrew Scull19503262018-09-20 14:48:39 +0100511 }
512
513 /* The requested vcpu must exist. */
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100514 if (vcpu_idx >= vm->vcpu_count) {
Andrew Scull6d2db332018-10-10 15:28:17 +0100515 goto out;
Andrew Scull7364a8e2018-07-19 15:39:29 +0100516 }
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +0100517
Wedson Almeida Filho03306112018-11-26 00:08:03 +0000518 /* Update state if allowed. */
Andrew Walbrane1310df2019-04-29 17:28:28 +0100519 vcpu = vm_get_vcpu(vm, vcpu_idx);
Andrew Scullb06d1752019-02-04 10:15:48 +0000520 if (!api_vcpu_prepare_run(current, vcpu, &ret)) {
Wedson Almeida Filho03306112018-11-26 00:08:03 +0000521 goto out;
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100522 }
Wedson Almeida Filho03306112018-11-26 00:08:03 +0000523
Andrew Walbran508e63c2018-12-20 17:02:37 +0000524 /*
525 * Inject timer interrupt if timer has expired. It's safe to access
526 * vcpu->regs here because api_vcpu_prepare_run already made sure that
527 * regs_available was true (and then set it to false) before returning
528 * true.
529 */
530 if (arch_timer_pending(&vcpu->regs)) {
531 /* Make virtual timer interrupt pending. */
Andrew Walbranfc9d4382019-05-10 18:07:21 +0100532 internal_interrupt_inject(vcpu, HF_VIRTUAL_TIMER_INTID, vcpu,
533 NULL);
Andrew Walbran508e63c2018-12-20 17:02:37 +0000534
535 /*
536 * Set the mask bit so the hardware interrupt doesn't fire
537 * again. Ideally we wouldn't do this because it affects what
538 * the secondary vCPU sees, but if we don't then we end up with
539 * a loop of the interrupt firing each time we try to return to
540 * the secondary vCPU.
541 */
542 arch_timer_mask(&vcpu->regs);
543 }
544
Andrew Scull33fecd32019-01-08 14:48:27 +0000545 /* Switch to the vcpu. */
Wedson Almeida Filho03306112018-11-26 00:08:03 +0000546 *next = vcpu;
Wedson Almeida Filho03306112018-11-26 00:08:03 +0000547
Andrew Scull33fecd32019-01-08 14:48:27 +0000548 /*
549 * Set a placeholder return code to the scheduler. This will be
550 * overwritten when the switch back to the primary occurs.
551 */
552 ret.code = HF_VCPU_RUN_PREEMPTED;
553
Andrew Scull6d2db332018-10-10 15:28:17 +0100554out:
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100555 return ret;
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +0100556}
557
558/**
Andrew Scull81e85092018-12-12 12:56:20 +0000559 * Check that the mode indicates memory that is valid, owned and exclusive.
560 */
Andrew Scullcbefbdb2019-01-11 16:36:26 +0000561static bool api_mode_valid_owned_and_exclusive(int mode)
Andrew Scull81e85092018-12-12 12:56:20 +0000562{
563 return (mode & (MM_MODE_INVALID | MM_MODE_UNOWNED | MM_MODE_SHARED)) ==
564 0;
565}
566
567/**
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +0000568 * Determines the value to be returned by api_vm_configure and api_mailbox_clear
569 * after they've succeeded. If a secondary VM is running and there are waiters,
570 * it also switches back to the primary VM for it to wake waiters up.
571 */
572static int64_t api_waiter_result(struct vm_locked locked_vm,
573 struct vcpu *current, struct vcpu **next)
574{
575 struct vm *vm = locked_vm.vm;
576 struct hf_vcpu_run_return ret = {
577 .code = HF_VCPU_RUN_NOTIFY_WAITERS,
578 };
579
580 if (list_empty(&vm->mailbox.waiter_list)) {
581 /* No waiters, nothing else to do. */
582 return 0;
583 }
584
585 if (vm->id == HF_PRIMARY_VM_ID) {
586 /* The caller is the primary VM. Tell it to wake up waiters. */
587 return 1;
588 }
589
590 /*
591 * Switch back to the primary VM, informing it that there are waiters
592 * that need to be notified.
593 */
Andrew Sculld6ee1102019-04-05 22:12:42 +0100594 *next = api_switch_to_primary(current, ret, VCPU_STATE_READY);
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +0000595
596 return 0;
597}
598
599/**
Andrew Sculle1322792019-07-01 17:46:10 +0100600 * Configures the hypervisor's stage-1 view of the send and receive pages. The
601 * stage-1 page tables must be locked so memory cannot be taken by another core
602 * which could result in this transaction being unable to roll back in the case
603 * of an error.
604 */
605static bool api_vm_configure_stage1(struct vm_locked vm_locked,
606 paddr_t pa_send_begin, paddr_t pa_send_end,
607 paddr_t pa_recv_begin, paddr_t pa_recv_end,
608 struct mpool *local_page_pool)
609{
610 bool ret;
611 struct mm_stage1_locked mm_stage1_locked = mm_lock_stage1();
612
613 /* Map the send page as read-only in the hypervisor address space. */
614 vm_locked.vm->mailbox.send =
615 mm_identity_map(mm_stage1_locked, pa_send_begin, pa_send_end,
616 MM_MODE_R, local_page_pool);
617 if (!vm_locked.vm->mailbox.send) {
618 /* TODO: partial defrag of failed range. */
619 /* Recover any memory consumed in failed mapping. */
620 mm_defrag(mm_stage1_locked, local_page_pool);
621 goto fail;
622 }
623
624 /*
625 * Map the receive page as writable in the hypervisor address space. On
626 * failure, unmap the send page before returning.
627 */
628 vm_locked.vm->mailbox.recv =
629 mm_identity_map(mm_stage1_locked, pa_recv_begin, pa_recv_end,
630 MM_MODE_W, local_page_pool);
631 if (!vm_locked.vm->mailbox.recv) {
632 /* TODO: partial defrag of failed range. */
633 /* Recover any memory consumed in failed mapping. */
634 mm_defrag(mm_stage1_locked, local_page_pool);
635 goto fail_undo_send;
636 }
637
638 ret = true;
639 goto out;
640
641 /*
642 * The following mappings will not require more memory than is available
643 * in the local pool.
644 */
645fail_undo_send:
646 vm_locked.vm->mailbox.send = NULL;
647 mm_unmap(mm_stage1_locked, pa_send_begin, pa_send_end, local_page_pool);
648
649fail:
650 ret = false;
651
652out:
653 mm_unlock_stage1(&mm_stage1_locked);
654
655 return ret;
656}
657
658/**
659 * Configures the send and receive pages in the VM stage-2 and hypervisor
660 * stage-1 page tables. Locking of the page tables combined with a local memory
661 * pool ensures there will always be enough memory to recover from any errors
662 * that arise.
663 */
664static bool api_vm_configure_pages(struct vm_locked vm_locked,
665 paddr_t pa_send_begin, paddr_t pa_send_end,
666 int orig_send_mode, paddr_t pa_recv_begin,
667 paddr_t pa_recv_end, int orig_recv_mode)
668{
669 bool ret;
670 struct mpool local_page_pool;
671
672 /*
673 * Create a local pool so any freed memory can't be used by another
674 * thread. This is to ensure the original mapping can be restored if any
675 * stage of the process fails.
676 */
677 mpool_init_with_fallback(&local_page_pool, &api_page_pool);
678
679 /* Take memory ownership away from the VM and mark as shared. */
680 if (!mm_vm_identity_map(
681 &vm_locked.vm->ptable, pa_send_begin, pa_send_end,
682 MM_MODE_UNOWNED | MM_MODE_SHARED | MM_MODE_R | MM_MODE_W,
683 NULL, &local_page_pool)) {
684 goto fail;
685 }
686
687 if (!mm_vm_identity_map(&vm_locked.vm->ptable, pa_recv_begin,
688 pa_recv_end,
689 MM_MODE_UNOWNED | MM_MODE_SHARED | MM_MODE_R,
690 NULL, &local_page_pool)) {
691 /* TODO: partial defrag of failed range. */
692 /* Recover any memory consumed in failed mapping. */
693 mm_vm_defrag(&vm_locked.vm->ptable, &local_page_pool);
694 goto fail_undo_send;
695 }
696
697 if (!api_vm_configure_stage1(vm_locked, pa_send_begin, pa_send_end,
698 pa_recv_begin, pa_recv_end,
699 &local_page_pool)) {
700 goto fail_undo_send_and_recv;
701 }
702
703 ret = true;
704 goto out;
705
706 /*
707 * The following mappings will not require more memory than is available
708 * in the local pool.
709 */
710fail_undo_send_and_recv:
711 mm_vm_identity_map(&vm_locked.vm->ptable, pa_recv_begin, pa_recv_end,
712 orig_recv_mode, NULL, &local_page_pool);
713
714fail_undo_send:
715 mm_vm_identity_map(&vm_locked.vm->ptable, pa_send_begin, pa_send_end,
716 orig_send_mode, NULL, &local_page_pool);
717
718fail:
719 ret = false;
720
721out:
722 mpool_fini(&local_page_pool);
723
724 return ret;
725}
726
727/**
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100728 * Configures the VM to send/receive data through the specified pages. The pages
729 * must not be shared.
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +0000730 *
731 * Returns:
732 * - -1 on failure.
733 * - 0 on success if no further action is needed.
734 * - 1 if it was called by the primary VM and the primary VM now needs to wake
735 * up or kick waiters. Waiters should be retrieved by calling
736 * hf_mailbox_waiter_get.
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100737 */
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +0000738int64_t api_vm_configure(ipaddr_t send, ipaddr_t recv, struct vcpu *current,
739 struct vcpu **next)
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100740{
Wedson Almeida Filho00df6c72018-10-18 11:19:24 +0100741 struct vm *vm = current->vm;
Andrew Sculle1322792019-07-01 17:46:10 +0100742 struct vm_locked vm_locked;
Andrew Scull80871322018-08-06 12:04:09 +0100743 paddr_t pa_send_begin;
744 paddr_t pa_send_end;
745 paddr_t pa_recv_begin;
746 paddr_t pa_recv_end;
Andrew Scull220e6212018-12-21 18:09:00 +0000747 int orig_send_mode;
748 int orig_recv_mode;
Andrew Scullc0e569a2018-10-02 18:05:21 +0100749 int64_t ret;
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100750
751 /* Fail if addresses are not page-aligned. */
Alfredo Mazzinghieb1997c2019-02-07 18:00:01 +0000752 if (!is_aligned(ipa_addr(send), PAGE_SIZE) ||
753 !is_aligned(ipa_addr(recv), PAGE_SIZE)) {
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100754 return -1;
755 }
756
Andrew Scullc2eb6a32018-12-13 16:54:24 +0000757 /* Convert to physical addresses. */
758 pa_send_begin = pa_from_ipa(send);
759 pa_send_end = pa_add(pa_send_begin, PAGE_SIZE);
760
761 pa_recv_begin = pa_from_ipa(recv);
762 pa_recv_end = pa_add(pa_recv_begin, PAGE_SIZE);
763
Andrew Scullc9ccb3f2018-08-13 15:27:12 +0100764 /* Fail if the same page is used for the send and receive pages. */
765 if (pa_addr(pa_send_begin) == pa_addr(pa_recv_begin)) {
Andrew Scull220e6212018-12-21 18:09:00 +0000766 return -1;
767 }
768
Andrew Scull3c0a90a2019-07-01 11:55:53 +0100769 /*
770 * The hypervisor's memory map must be locked for the duration of this
771 * operation to ensure there will be sufficient memory to recover from
772 * any failures.
773 *
774 * TODO: the scope of the can be reduced but will require restructuring
775 * to keep a single unlock point.
776 */
Andrew Sculle1322792019-07-01 17:46:10 +0100777 vm_locked = vm_lock(vm);
Andrew Scull220e6212018-12-21 18:09:00 +0000778
779 /* We only allow these to be setup once. */
780 if (vm->mailbox.send || vm->mailbox.recv) {
781 goto fail;
782 }
783
784 /*
785 * Ensure the pages are valid, owned and exclusive to the VM and that
786 * the VM has the required access to the memory.
787 */
788 if (!mm_vm_get_mode(&vm->ptable, send, ipa_add(send, PAGE_SIZE),
789 &orig_send_mode) ||
790 !api_mode_valid_owned_and_exclusive(orig_send_mode) ||
791 (orig_send_mode & MM_MODE_R) == 0 ||
792 (orig_send_mode & MM_MODE_W) == 0) {
793 goto fail;
794 }
795
796 if (!mm_vm_get_mode(&vm->ptable, recv, ipa_add(recv, PAGE_SIZE),
797 &orig_recv_mode) ||
798 !api_mode_valid_owned_and_exclusive(orig_recv_mode) ||
799 (orig_recv_mode & MM_MODE_R) == 0) {
800 goto fail;
801 }
802
Andrew Sculle1322792019-07-01 17:46:10 +0100803 if (!api_vm_configure_pages(vm_locked, pa_send_begin, pa_send_end,
804 orig_send_mode, pa_recv_begin, pa_recv_end,
805 orig_recv_mode)) {
806 goto fail;
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100807 }
808
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +0000809 /* Tell caller about waiters, if any. */
Andrew Sculle1322792019-07-01 17:46:10 +0100810 ret = api_waiter_result(vm_locked, current, next);
Andrew Scull220e6212018-12-21 18:09:00 +0000811 goto exit;
812
Andrew Scull220e6212018-12-21 18:09:00 +0000813fail:
814 ret = -1;
815
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100816exit:
Andrew Sculle1322792019-07-01 17:46:10 +0100817 vm_unlock(&vm_locked);
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100818
819 return ret;
820}
821
822/**
Andrew Scullaa039b32018-10-04 15:02:26 +0100823 * Copies data from the sender's send buffer to the recipient's receive buffer
824 * and notifies the recipient.
Wedson Almeida Filho17c997f2019-01-09 18:50:09 +0000825 *
826 * If the recipient's receive buffer is busy, it can optionally register the
827 * caller to be notified when the recipient's receive buffer becomes available.
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100828 */
Jose Marinhoa1dfeda2019-02-27 16:46:03 +0000829int32_t api_spci_msg_send(uint32_t attributes, struct vcpu *current,
830 struct vcpu **next)
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100831{
Wedson Almeida Filho00df6c72018-10-18 11:19:24 +0100832 struct vm *from = current->vm;
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100833 struct vm *to;
Andrew Scullb06d1752019-02-04 10:15:48 +0000834 struct hf_vcpu_run_return primary_ret = {
835 .code = HF_VCPU_RUN_MESSAGE,
836 };
Jose Marinhoa1dfeda2019-02-27 16:46:03 +0000837 struct spci_message from_msg_replica;
838 struct spci_message *to_msg;
839 const struct spci_message *from_msg;
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100840
Jose Marinhoa1dfeda2019-02-27 16:46:03 +0000841 uint32_t size;
Andrew Scull19503262018-09-20 14:48:39 +0100842
Jose Marinhoa1dfeda2019-02-27 16:46:03 +0000843 int64_t ret;
844 bool notify = (attributes & SPCI_MSG_SEND_NOTIFY_MASK) ==
845 SPCI_MSG_SEND_NOTIFY;
Andrew Scull19503262018-09-20 14:48:39 +0100846
Jose Marinhoa1dfeda2019-02-27 16:46:03 +0000847 /*
848 * Check that the sender has configured its send buffer. Copy the
849 * message header. If the tx mailbox at from_msg is configured (i.e.
850 * from_msg != NULL) then it can be safely accessed after releasing the
851 * lock since the tx mailbox address can only be configured once.
852 */
853 sl_lock(&from->lock);
854 from_msg = from->mailbox.send;
855 sl_unlock(&from->lock);
856
857 if (from_msg == NULL) {
858 return SPCI_INVALID_PARAMETERS;
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100859 }
860
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100861 /*
Jose Marinhoa1dfeda2019-02-27 16:46:03 +0000862 * Note that the payload is not copied when the message header is.
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100863 */
Jose Marinhoa1dfeda2019-02-27 16:46:03 +0000864 from_msg_replica = *from_msg;
865
866 /* Ensure source VM id corresponds to the current VM. */
867 if (from_msg_replica.source_vm_id != from->id) {
868 return SPCI_INVALID_PARAMETERS;
869 }
870
871 size = from_msg_replica.length;
872 /* Limit the size of transfer. */
Andrew Scull1262ac22019-04-05 12:44:26 +0100873 if (size > SPCI_MSG_PAYLOAD_MAX) {
Jose Marinhoa1dfeda2019-02-27 16:46:03 +0000874 return SPCI_INVALID_PARAMETERS;
875 }
876
877 /* Disallow reflexive requests as this suggests an error in the VM. */
878 if (from_msg_replica.target_vm_id == from->id) {
879 return SPCI_INVALID_PARAMETERS;
880 }
881
882 /* Ensure the target VM exists. */
Andrew Walbran42347a92019-05-09 13:59:03 +0100883 to = vm_find(from_msg_replica.target_vm_id);
Jose Marinhoa1dfeda2019-02-27 16:46:03 +0000884 if (to == NULL) {
885 return SPCI_INVALID_PARAMETERS;
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100886 }
887
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100888 sl_lock(&to->lock);
889
Andrew Sculld6ee1102019-04-05 22:12:42 +0100890 if (to->mailbox.state != MAILBOX_STATE_EMPTY ||
Andrew Scullaa039b32018-10-04 15:02:26 +0100891 to->mailbox.recv == NULL) {
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +0000892 /*
893 * Fail if the target isn't currently ready to receive data,
894 * setting up for notification if requested.
895 */
896 if (notify) {
Wedson Almeida Filhob790f652019-01-22 23:41:56 +0000897 struct wait_entry *entry =
Jose Marinhoa1dfeda2019-02-27 16:46:03 +0000898 &current->vm->wait_entries
899 [from_msg_replica.target_vm_id];
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +0000900
901 /* Append waiter only if it's not there yet. */
902 if (list_empty(&entry->wait_links)) {
903 list_append(&to->mailbox.waiter_list,
904 &entry->wait_links);
905 }
906 }
907
Jose Marinhoa1dfeda2019-02-27 16:46:03 +0000908 ret = SPCI_BUSY;
Andrew Scullaa039b32018-10-04 15:02:26 +0100909 goto out;
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100910 }
911
Andrew Scullaa039b32018-10-04 15:02:26 +0100912 /* Copy data. */
Jose Marinhoa1dfeda2019-02-27 16:46:03 +0000913 to_msg = to->mailbox.recv;
914 *to_msg = from_msg_replica;
Andrew Sculla1aa2ba2019-04-05 11:49:02 +0100915 memcpy_s(to_msg->payload, SPCI_MSG_PAYLOAD_MAX,
916 from->mailbox.send->payload, size);
Andrew Scullb06d1752019-02-04 10:15:48 +0000917 primary_ret.message.vm_id = to->id;
Jose Marinhoa1dfeda2019-02-27 16:46:03 +0000918 ret = SPCI_SUCCESS;
Andrew Scullaa039b32018-10-04 15:02:26 +0100919
920 /* Messages for the primary VM are delivered directly. */
921 if (to->id == HF_PRIMARY_VM_ID) {
Andrew Sculld6ee1102019-04-05 22:12:42 +0100922 to->mailbox.state = MAILBOX_STATE_READ;
Wedson Almeida Filhoba641ef2018-12-03 04:19:44 +0000923 *next = api_switch_to_primary(current, primary_ret,
Andrew Sculld6ee1102019-04-05 22:12:42 +0100924 VCPU_STATE_READY);
Andrew Scullaa039b32018-10-04 15:02:26 +0100925 goto out;
926 }
927
Andrew Sculld6ee1102019-04-05 22:12:42 +0100928 to->mailbox.state = MAILBOX_STATE_RECEIVED;
Andrew Scullaa039b32018-10-04 15:02:26 +0100929
930 /* Return to the primary VM directly or with a switch. */
Andrew Scullb06d1752019-02-04 10:15:48 +0000931 if (from->id != HF_PRIMARY_VM_ID) {
Wedson Almeida Filhoba641ef2018-12-03 04:19:44 +0000932 *next = api_switch_to_primary(current, primary_ret,
Andrew Sculld6ee1102019-04-05 22:12:42 +0100933 VCPU_STATE_READY);
Wedson Almeida Filho80eb4a32018-11-30 17:11:15 +0000934 }
Andrew Scullaa039b32018-10-04 15:02:26 +0100935
936out:
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100937 sl_unlock(&to->lock);
938
Wedson Almeida Filho80eb4a32018-11-30 17:11:15 +0000939 return ret;
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100940}
941
942/**
Andrew Scullaa039b32018-10-04 15:02:26 +0100943 * Receives a message from the mailbox. If one isn't available, this function
944 * can optionally block the caller until one becomes available.
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100945 *
Andrew Scullaa039b32018-10-04 15:02:26 +0100946 * No new messages can be received until the mailbox has been cleared.
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100947 */
Jose Marinho3e2442f2019-03-12 13:30:37 +0000948int32_t api_spci_msg_recv(uint32_t attributes, struct vcpu *current,
949 struct vcpu **next)
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100950{
Wedson Almeida Filho00df6c72018-10-18 11:19:24 +0100951 struct vm *vm = current->vm;
Jose Marinho3e2442f2019-03-12 13:30:37 +0000952 int32_t return_code;
953 bool block =
954 (attributes & SPCI_MSG_RECV_BLOCK_MASK) == SPCI_MSG_RECV_BLOCK;
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100955
Andrew Scullaa039b32018-10-04 15:02:26 +0100956 /*
957 * The primary VM will receive messages as a status code from running
958 * vcpus and must not call this function.
959 */
Andrew Scull19503262018-09-20 14:48:39 +0100960 if (vm->id == HF_PRIMARY_VM_ID) {
Jose Marinho3e2442f2019-03-12 13:30:37 +0000961 return SPCI_INTERRUPTED;
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100962 }
963
964 sl_lock(&vm->lock);
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100965
Andrew Scullaa039b32018-10-04 15:02:26 +0100966 /* Return pending messages without blocking. */
Andrew Sculld6ee1102019-04-05 22:12:42 +0100967 if (vm->mailbox.state == MAILBOX_STATE_RECEIVED) {
968 vm->mailbox.state = MAILBOX_STATE_READ;
Jose Marinho3e2442f2019-03-12 13:30:37 +0000969 return_code = SPCI_SUCCESS;
970 goto out;
971 }
972
973 /* No pending message so fail if not allowed to block. */
974 if (!block) {
975 return_code = SPCI_RETRY;
Andrew Scullaa039b32018-10-04 15:02:26 +0100976 goto out;
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100977 }
Andrew Scullaa039b32018-10-04 15:02:26 +0100978
Andrew Walbran9311c9a2019-03-12 16:59:04 +0000979 /*
Jose Marinho3e2442f2019-03-12 13:30:37 +0000980 * From this point onward this call can only be interrupted or a message
981 * received. If a message is received the return value will be set at
982 * that time to SPCI_SUCCESS.
Andrew Walbran9311c9a2019-03-12 16:59:04 +0000983 */
Jose Marinho3e2442f2019-03-12 13:30:37 +0000984 return_code = SPCI_INTERRUPTED;
985
986 /*
987 * Don't block if there are enabled and pending interrupts, to match
988 * behaviour of wait_for_interrupt.
989 */
990 if (current->interrupts.enabled_and_pending_count > 0) {
Andrew Scullaa039b32018-10-04 15:02:26 +0100991 goto out;
992 }
993
Andrew Scullaa039b32018-10-04 15:02:26 +0100994 /* Switch back to primary vm to block. */
Andrew Walbranb4816552018-12-05 17:35:42 +0000995 {
996 struct hf_vcpu_run_return run_return = {
Andrew Scullb06d1752019-02-04 10:15:48 +0000997 .code = HF_VCPU_RUN_WAIT_FOR_MESSAGE,
Andrew Walbranb4816552018-12-05 17:35:42 +0000998 };
Wedson Almeida Filho81568c42019-01-04 13:33:02 +0000999
Andrew Walbranb4816552018-12-05 17:35:42 +00001000 *next = api_switch_to_primary(current, run_return,
Andrew Sculld6ee1102019-04-05 22:12:42 +01001001 VCPU_STATE_BLOCKED_MAILBOX);
Andrew Walbranb4816552018-12-05 17:35:42 +00001002 }
Andrew Scullaa039b32018-10-04 15:02:26 +01001003out:
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +01001004 sl_unlock(&vm->lock);
1005
Jose Marinho3e2442f2019-03-12 13:30:37 +00001006 return return_code;
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +01001007}
1008
1009/**
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +00001010 * Retrieves the next VM whose mailbox became writable. For a VM to be notified
1011 * by this function, the caller must have called api_mailbox_send before with
1012 * the notify argument set to true, and this call must have failed because the
1013 * mailbox was not available.
1014 *
1015 * It should be called repeatedly to retrieve a list of VMs.
1016 *
1017 * Returns -1 if no VM became writable, or the id of the VM whose mailbox
1018 * became writable.
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +01001019 */
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +00001020int64_t api_mailbox_writable_get(const struct vcpu *current)
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +01001021{
Wedson Almeida Filho00df6c72018-10-18 11:19:24 +01001022 struct vm *vm = current->vm;
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +00001023 struct wait_entry *entry;
Andrew Scullc0e569a2018-10-02 18:05:21 +01001024 int64_t ret;
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +01001025
1026 sl_lock(&vm->lock);
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +00001027 if (list_empty(&vm->mailbox.ready_list)) {
1028 ret = -1;
1029 goto exit;
1030 }
1031
1032 entry = CONTAINER_OF(vm->mailbox.ready_list.next, struct wait_entry,
1033 ready_links);
1034 list_remove(&entry->ready_links);
Wedson Almeida Filhob790f652019-01-22 23:41:56 +00001035 ret = entry - vm->wait_entries;
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +00001036
1037exit:
1038 sl_unlock(&vm->lock);
1039 return ret;
1040}
1041
1042/**
1043 * Retrieves the next VM waiting to be notified that the mailbox of the
1044 * specified VM became writable. Only primary VMs are allowed to call this.
1045 *
Wedson Almeida Filhob790f652019-01-22 23:41:56 +00001046 * Returns -1 on failure or if there are no waiters; the VM id of the next
1047 * waiter otherwise.
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +00001048 */
Andrew Walbran42347a92019-05-09 13:59:03 +01001049int64_t api_mailbox_waiter_get(spci_vm_id_t vm_id, const struct vcpu *current)
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +00001050{
1051 struct vm *vm;
1052 struct vm_locked locked;
1053 struct wait_entry *entry;
1054 struct vm *waiting_vm;
1055
1056 /* Only primary VMs are allowed to call this function. */
1057 if (current->vm->id != HF_PRIMARY_VM_ID) {
1058 return -1;
1059 }
1060
Andrew Walbran42347a92019-05-09 13:59:03 +01001061 vm = vm_find(vm_id);
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +00001062 if (vm == NULL) {
1063 return -1;
1064 }
1065
1066 /* Check if there are outstanding notifications from given vm. */
Andrew Walbran7e932bd2019-04-29 16:47:06 +01001067 locked = vm_lock(vm);
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +00001068 entry = api_fetch_waiter(locked);
1069 vm_unlock(&locked);
1070
1071 if (entry == NULL) {
1072 return -1;
1073 }
1074
1075 /* Enqueue notification to waiting VM. */
1076 waiting_vm = entry->waiting_vm;
1077
1078 sl_lock(&waiting_vm->lock);
1079 if (list_empty(&entry->ready_links)) {
1080 list_append(&waiting_vm->mailbox.ready_list,
1081 &entry->ready_links);
1082 }
1083 sl_unlock(&waiting_vm->lock);
1084
1085 return waiting_vm->id;
1086}
1087
1088/**
1089 * Clears the caller's mailbox so that a new message can be received. The caller
1090 * must have copied out all data they wish to preserve as new messages will
1091 * overwrite the old and will arrive asynchronously.
1092 *
1093 * Returns:
Andrew Scullaa7db8e2019-02-01 14:12:19 +00001094 * - -1 on failure, if the mailbox hasn't been read.
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +00001095 * - 0 on success if no further action is needed.
1096 * - 1 if it was called by the primary VM and the primary VM now needs to wake
1097 * up or kick waiters. Waiters should be retrieved by calling
1098 * hf_mailbox_waiter_get.
1099 */
1100int64_t api_mailbox_clear(struct vcpu *current, struct vcpu **next)
1101{
1102 struct vm *vm = current->vm;
1103 struct vm_locked locked;
1104 int64_t ret;
1105
Andrew Walbran7e932bd2019-04-29 16:47:06 +01001106 locked = vm_lock(vm);
Andrew Scullaa7db8e2019-02-01 14:12:19 +00001107 switch (vm->mailbox.state) {
Andrew Sculld6ee1102019-04-05 22:12:42 +01001108 case MAILBOX_STATE_EMPTY:
Andrew Scullaa7db8e2019-02-01 14:12:19 +00001109 ret = 0;
1110 break;
1111
Andrew Sculld6ee1102019-04-05 22:12:42 +01001112 case MAILBOX_STATE_RECEIVED:
Andrew Scullaa7db8e2019-02-01 14:12:19 +00001113 ret = -1;
1114 break;
1115
Andrew Sculld6ee1102019-04-05 22:12:42 +01001116 case MAILBOX_STATE_READ:
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +00001117 ret = api_waiter_result(locked, current, next);
Andrew Sculld6ee1102019-04-05 22:12:42 +01001118 vm->mailbox.state = MAILBOX_STATE_EMPTY;
Andrew Scullaa7db8e2019-02-01 14:12:19 +00001119 break;
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +01001120 }
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +00001121 vm_unlock(&locked);
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +01001122
1123 return ret;
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +01001124}
Andrew Walbran318f5732018-11-20 16:23:42 +00001125
1126/**
1127 * Enables or disables a given interrupt ID for the calling vCPU.
1128 *
1129 * Returns 0 on success, or -1 if the intid is invalid.
1130 */
Wedson Almeida Filhoc559d132019-01-09 19:33:40 +00001131int64_t api_interrupt_enable(uint32_t intid, bool enable, struct vcpu *current)
Andrew Walbran318f5732018-11-20 16:23:42 +00001132{
1133 uint32_t intid_index = intid / INTERRUPT_REGISTER_BITS;
1134 uint32_t intid_mask = 1u << (intid % INTERRUPT_REGISTER_BITS);
Wedson Almeida Filho81568c42019-01-04 13:33:02 +00001135
Andrew Walbran318f5732018-11-20 16:23:42 +00001136 if (intid >= HF_NUM_INTIDS) {
1137 return -1;
1138 }
1139
1140 sl_lock(&current->lock);
1141 if (enable) {
Andrew Walbran3d84a262018-12-13 14:41:19 +00001142 /*
1143 * If it is pending and was not enabled before, increment the
1144 * count.
1145 */
1146 if (current->interrupts.interrupt_pending[intid_index] &
1147 ~current->interrupts.interrupt_enabled[intid_index] &
1148 intid_mask) {
1149 current->interrupts.enabled_and_pending_count++;
1150 }
Andrew Walbran318f5732018-11-20 16:23:42 +00001151 current->interrupts.interrupt_enabled[intid_index] |=
1152 intid_mask;
Andrew Walbran318f5732018-11-20 16:23:42 +00001153 } else {
Andrew Walbran3d84a262018-12-13 14:41:19 +00001154 /*
1155 * If it is pending and was enabled before, decrement the count.
1156 */
1157 if (current->interrupts.interrupt_pending[intid_index] &
1158 current->interrupts.interrupt_enabled[intid_index] &
1159 intid_mask) {
1160 current->interrupts.enabled_and_pending_count--;
1161 }
Andrew Walbran318f5732018-11-20 16:23:42 +00001162 current->interrupts.interrupt_enabled[intid_index] &=
1163 ~intid_mask;
1164 }
1165
1166 sl_unlock(&current->lock);
1167 return 0;
1168}
1169
1170/**
1171 * Returns the ID of the next pending interrupt for the calling vCPU, and
1172 * acknowledges it (i.e. marks it as no longer pending). Returns
1173 * HF_INVALID_INTID if there are no pending interrupts.
1174 */
Wedson Almeida Filhoc559d132019-01-09 19:33:40 +00001175uint32_t api_interrupt_get(struct vcpu *current)
Andrew Walbran318f5732018-11-20 16:23:42 +00001176{
1177 uint8_t i;
1178 uint32_t first_interrupt = HF_INVALID_INTID;
Andrew Walbran318f5732018-11-20 16:23:42 +00001179
1180 /*
1181 * Find the first enabled and pending interrupt ID, return it, and
1182 * deactivate it.
1183 */
1184 sl_lock(&current->lock);
1185 for (i = 0; i < HF_NUM_INTIDS / INTERRUPT_REGISTER_BITS; ++i) {
1186 uint32_t enabled_and_pending =
1187 current->interrupts.interrupt_enabled[i] &
1188 current->interrupts.interrupt_pending[i];
Wedson Almeida Filho81568c42019-01-04 13:33:02 +00001189
Andrew Walbran318f5732018-11-20 16:23:42 +00001190 if (enabled_and_pending != 0) {
Andrew Walbran3d84a262018-12-13 14:41:19 +00001191 uint8_t bit_index = ctz(enabled_and_pending);
1192 /*
1193 * Mark it as no longer pending and decrement the count.
1194 */
1195 current->interrupts.interrupt_pending[i] &=
1196 ~(1u << bit_index);
1197 current->interrupts.enabled_and_pending_count--;
1198 first_interrupt =
1199 i * INTERRUPT_REGISTER_BITS + bit_index;
Andrew Walbran318f5732018-11-20 16:23:42 +00001200 break;
1201 }
1202 }
Andrew Walbran318f5732018-11-20 16:23:42 +00001203
1204 sl_unlock(&current->lock);
1205 return first_interrupt;
1206}
1207
1208/**
Andrew Walbran4cf217a2018-12-14 15:24:50 +00001209 * Returns whether the current vCPU is allowed to inject an interrupt into the
Andrew Walbran318f5732018-11-20 16:23:42 +00001210 * given VM and vCPU.
1211 */
1212static inline bool is_injection_allowed(uint32_t target_vm_id,
1213 struct vcpu *current)
1214{
1215 uint32_t current_vm_id = current->vm->id;
Wedson Almeida Filho81568c42019-01-04 13:33:02 +00001216
Andrew Walbran318f5732018-11-20 16:23:42 +00001217 /*
1218 * The primary VM is allowed to inject interrupts into any VM. Secondary
1219 * VMs are only allowed to inject interrupts into their own vCPUs.
1220 */
1221 return current_vm_id == HF_PRIMARY_VM_ID ||
1222 current_vm_id == target_vm_id;
1223}
1224
1225/**
1226 * Injects a virtual interrupt of the given ID into the given target vCPU.
1227 * This doesn't cause the vCPU to actually be run immediately; it will be taken
1228 * when the vCPU is next run, which is up to the scheduler.
1229 *
Andrew Walbran3d84a262018-12-13 14:41:19 +00001230 * Returns:
1231 * - -1 on failure because the target VM or vCPU doesn't exist, the interrupt
1232 * ID is invalid, or the current VM is not allowed to inject interrupts to
1233 * the target VM.
1234 * - 0 on success if no further action is needed.
1235 * - 1 if it was called by the primary VM and the primary VM now needs to wake
1236 * up or kick the target vCPU.
Andrew Walbran318f5732018-11-20 16:23:42 +00001237 */
Andrew Walbran42347a92019-05-09 13:59:03 +01001238int64_t api_interrupt_inject(spci_vm_id_t target_vm_id,
Andrew Walbranb037d5b2019-06-25 17:19:41 +01001239 spci_vcpu_index_t target_vcpu_idx, uint32_t intid,
Andrew Walbran42347a92019-05-09 13:59:03 +01001240 struct vcpu *current, struct vcpu **next)
Andrew Walbran318f5732018-11-20 16:23:42 +00001241{
Andrew Walbran318f5732018-11-20 16:23:42 +00001242 struct vcpu *target_vcpu;
Andrew Walbran42347a92019-05-09 13:59:03 +01001243 struct vm *target_vm = vm_find(target_vm_id);
Andrew Walbran318f5732018-11-20 16:23:42 +00001244
1245 if (intid >= HF_NUM_INTIDS) {
1246 return -1;
1247 }
Wedson Almeida Filho81568c42019-01-04 13:33:02 +00001248
Andrew Walbran318f5732018-11-20 16:23:42 +00001249 if (target_vm == NULL) {
1250 return -1;
1251 }
Wedson Almeida Filho81568c42019-01-04 13:33:02 +00001252
Andrew Walbran318f5732018-11-20 16:23:42 +00001253 if (target_vcpu_idx >= target_vm->vcpu_count) {
1254 /* The requested vcpu must exist. */
1255 return -1;
1256 }
Wedson Almeida Filho81568c42019-01-04 13:33:02 +00001257
Andrew Walbran318f5732018-11-20 16:23:42 +00001258 if (!is_injection_allowed(target_vm_id, current)) {
1259 return -1;
1260 }
Wedson Almeida Filho81568c42019-01-04 13:33:02 +00001261
Andrew Walbrane1310df2019-04-29 17:28:28 +01001262 target_vcpu = vm_get_vcpu(target_vm, target_vcpu_idx);
Andrew Walbran318f5732018-11-20 16:23:42 +00001263
1264 dlog("Injecting IRQ %d for VM %d VCPU %d from VM %d VCPU %d\n", intid,
1265 target_vm_id, target_vcpu_idx, current->vm->id, current->cpu->id);
Andrew Walbranfc9d4382019-05-10 18:07:21 +01001266 return internal_interrupt_inject(target_vcpu, intid, current, next);
Andrew Walbran318f5732018-11-20 16:23:42 +00001267}
Andrew Scull6386f252018-12-06 13:29:10 +00001268
1269/**
1270 * Clears a region of physical memory by overwriting it with zeros. The data is
1271 * flushed from the cache so the memory has been cleared across the system.
1272 */
1273static bool api_clear_memory(paddr_t begin, paddr_t end, struct mpool *ppool)
1274{
1275 /*
1276 * TODO: change this to a cpu local single page window rather than a
1277 * global mapping of the whole range. Such an approach will limit
1278 * the changes to stage-1 tables and will allow only local
1279 * invalidation.
1280 */
Andrew Scull3c0a90a2019-07-01 11:55:53 +01001281 bool ret;
1282 struct mm_stage1_locked stage1_locked = mm_lock_stage1();
1283 void *ptr =
1284 mm_identity_map(stage1_locked, begin, end, MM_MODE_W, ppool);
Andrew Walbran2cb43392019-04-17 12:52:45 +01001285 size_t size = pa_difference(begin, end);
Andrew Scull6386f252018-12-06 13:29:10 +00001286
1287 if (!ptr) {
1288 /* TODO: partial defrag of failed range. */
1289 /* Recover any memory consumed in failed mapping. */
Andrew Scull3c0a90a2019-07-01 11:55:53 +01001290 mm_defrag(stage1_locked, ppool);
1291 goto fail;
Andrew Scull6386f252018-12-06 13:29:10 +00001292 }
1293
Andrew Scull2b5fbad2019-04-05 13:55:56 +01001294 memset_s(ptr, size, 0, size);
Andrew Scull6386f252018-12-06 13:29:10 +00001295 arch_mm_write_back_dcache(ptr, size);
Andrew Scull3c0a90a2019-07-01 11:55:53 +01001296 mm_unmap(stage1_locked, begin, end, ppool);
Andrew Scull6386f252018-12-06 13:29:10 +00001297
Andrew Scull3c0a90a2019-07-01 11:55:53 +01001298 ret = true;
1299 goto out;
1300
1301fail:
1302 ret = false;
1303
1304out:
1305 mm_unlock_stage1(&stage1_locked);
1306
1307 return ret;
Andrew Scull6386f252018-12-06 13:29:10 +00001308}
1309
1310/**
1311 * Shares memory from the calling VM with another. The memory can be shared in
1312 * different modes.
1313 *
1314 * TODO: the interface for sharing memory will need to be enhanced to allow
1315 * sharing with different modes e.g. read-only, informing the recipient
1316 * of the memory they have been given, opting to not wipe the memory and
1317 * possibly allowing multiple blocks to be transferred. What this will
1318 * look like is TBD.
1319 */
Andrew Walbran42347a92019-05-09 13:59:03 +01001320int64_t api_share_memory(spci_vm_id_t vm_id, ipaddr_t addr, size_t size,
Andrew Scull6386f252018-12-06 13:29:10 +00001321 enum hf_share share, struct vcpu *current)
1322{
1323 struct vm *from = current->vm;
1324 struct vm *to;
1325 int orig_from_mode;
1326 int from_mode;
1327 int to_mode;
1328 ipaddr_t begin;
1329 ipaddr_t end;
1330 paddr_t pa_begin;
1331 paddr_t pa_end;
1332 struct mpool local_page_pool;
1333 int64_t ret;
1334
1335 /* Disallow reflexive shares as this suggests an error in the VM. */
1336 if (vm_id == from->id) {
1337 return -1;
1338 }
1339
1340 /* Ensure the target VM exists. */
Andrew Walbran42347a92019-05-09 13:59:03 +01001341 to = vm_find(vm_id);
Andrew Scull6386f252018-12-06 13:29:10 +00001342 if (to == NULL) {
1343 return -1;
1344 }
1345
1346 begin = addr;
1347 end = ipa_add(addr, size);
1348
1349 /* Fail if addresses are not page-aligned. */
Alfredo Mazzinghieb1997c2019-02-07 18:00:01 +00001350 if (!is_aligned(ipa_addr(begin), PAGE_SIZE) ||
1351 !is_aligned(ipa_addr(end), PAGE_SIZE)) {
Andrew Scull6386f252018-12-06 13:29:10 +00001352 return -1;
1353 }
1354
1355 /* Convert the sharing request to memory management modes. */
1356 switch (share) {
1357 case HF_MEMORY_GIVE:
1358 from_mode = MM_MODE_INVALID | MM_MODE_UNOWNED;
1359 to_mode = MM_MODE_R | MM_MODE_W | MM_MODE_X;
1360 break;
1361
1362 case HF_MEMORY_LEND:
1363 from_mode = MM_MODE_INVALID;
1364 to_mode = MM_MODE_R | MM_MODE_W | MM_MODE_X | MM_MODE_UNOWNED;
1365 break;
1366
1367 case HF_MEMORY_SHARE:
1368 from_mode = MM_MODE_R | MM_MODE_W | MM_MODE_X | MM_MODE_SHARED;
1369 to_mode = MM_MODE_R | MM_MODE_W | MM_MODE_X | MM_MODE_UNOWNED |
1370 MM_MODE_SHARED;
1371 break;
1372
1373 default:
1374 /* The input is untrusted so might not be a valid value. */
1375 return -1;
1376 }
1377
1378 /*
1379 * Create a local pool so any freed memory can't be used by another
1380 * thread. This is to ensure the original mapping can be restored if any
1381 * stage of the process fails.
1382 */
1383 mpool_init_with_fallback(&local_page_pool, &api_page_pool);
1384
1385 sl_lock_both(&from->lock, &to->lock);
1386
1387 /*
1388 * Ensure that the memory range is mapped with the same mode so that
1389 * changes can be reverted if the process fails.
1390 */
1391 if (!mm_vm_get_mode(&from->ptable, begin, end, &orig_from_mode)) {
1392 goto fail;
1393 }
1394
1395 /*
1396 * Ensure the memory range is valid for the sender. If it isn't, the
1397 * sender has either shared it with another VM already or has no claim
1398 * to the memory.
1399 */
1400 if (orig_from_mode & MM_MODE_INVALID) {
1401 goto fail;
1402 }
1403
1404 /*
1405 * The sender must own the memory and have exclusive access to it in
1406 * order to share it. Alternatively, it is giving memory back to the
1407 * owning VM.
1408 */
1409 if (orig_from_mode & MM_MODE_UNOWNED) {
1410 int orig_to_mode;
1411
1412 if (share != HF_MEMORY_GIVE ||
1413 !mm_vm_get_mode(&to->ptable, begin, end, &orig_to_mode) ||
1414 orig_to_mode & MM_MODE_UNOWNED) {
1415 goto fail;
1416 }
1417 } else if (orig_from_mode & MM_MODE_SHARED) {
1418 goto fail;
1419 }
1420
1421 pa_begin = pa_from_ipa(begin);
1422 pa_end = pa_from_ipa(end);
1423
1424 /*
1425 * First update the mapping for the sender so there is not overlap with
1426 * the recipient.
1427 */
1428 if (!mm_vm_identity_map(&from->ptable, pa_begin, pa_end, from_mode,
1429 NULL, &local_page_pool)) {
1430 goto fail;
1431 }
1432
1433 /* Clear the memory so no VM or device can see the previous contents. */
1434 if (!api_clear_memory(pa_begin, pa_end, &local_page_pool)) {
1435 goto fail_return_to_sender;
1436 }
1437
1438 /* Complete the transfer by mapping the memory into the recipient. */
1439 if (!mm_vm_identity_map(&to->ptable, pa_begin, pa_end, to_mode, NULL,
1440 &local_page_pool)) {
1441 /* TODO: partial defrag of failed range. */
1442 /* Recover any memory consumed in failed mapping. */
1443 mm_vm_defrag(&from->ptable, &local_page_pool);
1444 goto fail_return_to_sender;
1445 }
1446
1447 ret = 0;
1448 goto out;
1449
1450fail_return_to_sender:
1451 mm_vm_identity_map(&from->ptable, pa_begin, pa_end, orig_from_mode,
1452 NULL, &local_page_pool);
1453
1454fail:
1455 ret = -1;
1456
1457out:
1458 sl_unlock(&from->lock);
1459 sl_unlock(&to->lock);
1460
1461 mpool_fini(&local_page_pool);
1462
1463 return ret;
1464}
Jose Marinhofc0b2b62019-06-06 11:18:45 +01001465
1466/** Returns the version of the implemented SPCI specification. */
1467int32_t api_spci_version(void)
1468{
1469 /*
1470 * Ensure that both major and minor revision representation occupies at
1471 * most 15 bits.
1472 */
1473 static_assert(0x8000 > SPCI_VERSION_MAJOR,
1474 "Major revision representation take more than 15 bits.");
1475 static_assert(0x10000 > SPCI_VERSION_MINOR,
1476 "Minor revision representation take more than 16 bits.");
1477
1478 return (SPCI_VERSION_MAJOR << SPCI_VERSION_MAJOR_OFFSET) |
1479 SPCI_VERSION_MINOR;
1480}