blob: 1fdb54add935bf27150b385c1ddbde3abb25b699 [file] [log] [blame]
Andrew Scull18834872018-10-12 11:48:09 +01001/*
Andrew Walbran692b3252019-03-07 15:51:31 +00002 * Copyright 2018 The Hafnium Authors.
Andrew Scull18834872018-10-12 11:48:09 +01003 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * https://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
Andrew Scull18c78fc2018-08-20 12:57:41 +010017#include "hf/api.h"
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +010018
Andrew Scull13652af2018-09-17 14:49:08 +010019#include <assert.h>
20
Andrew Walbran318f5732018-11-20 16:23:42 +000021#include "hf/arch/cpu.h"
Andrew Walbran4a53ba62019-03-05 17:26:12 +000022#include "hf/arch/std.h"
Andrew Walbran508e63c2018-12-20 17:02:37 +000023#include "hf/arch/timer.h"
Andrew Walbran318f5732018-11-20 16:23:42 +000024
25#include "hf/dlog.h"
Andrew Scull6386f252018-12-06 13:29:10 +000026#include "hf/mm.h"
27#include "hf/spinlock.h"
Andrew Scull18c78fc2018-08-20 12:57:41 +010028#include "hf/vm.h"
29
Andrew Scullf35a5c92018-08-07 18:09:46 +010030#include "vmapi/hf/call.h"
Jose Marinhoa1dfeda2019-02-27 16:46:03 +000031#include "vmapi/hf/spci.h"
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +010032
Wedson Almeida Filhoba641ef2018-12-03 04:19:44 +000033/*
34 * To eliminate the risk of deadlocks, we define a partial order for the
35 * acquisition of locks held concurrently by the same physical CPU. Our current
36 * ordering requirements are as follows:
37 *
38 * vm::lock -> vcpu::lock
Andrew Scull6386f252018-12-06 13:29:10 +000039 *
40 * Locks of the same kind require the lock of lowest address to be locked first,
41 * see `sl_lock_both()`.
Wedson Almeida Filhoba641ef2018-12-03 04:19:44 +000042 */
43
Andrew Scullaa039b32018-10-04 15:02:26 +010044static_assert(HF_MAILBOX_SIZE == PAGE_SIZE,
Andrew Scull13652af2018-09-17 14:49:08 +010045 "Currently, a page is mapped for the send and receive buffers so "
46 "the maximum request is the size of a page.");
47
Wedson Almeida Filho9ed8da52018-12-17 16:09:11 +000048static struct mpool api_page_pool;
Wedson Almeida Filho22d5eaa2018-12-16 00:38:49 +000049
50/**
Wedson Almeida Filho81568c42019-01-04 13:33:02 +000051 * Initialises the API page pool by taking ownership of the contents of the
52 * given page pool.
Wedson Almeida Filho22d5eaa2018-12-16 00:38:49 +000053 */
54void api_init(struct mpool *ppool)
55{
Wedson Almeida Filho9ed8da52018-12-17 16:09:11 +000056 mpool_init_from(&api_page_pool, ppool);
Wedson Almeida Filho22d5eaa2018-12-16 00:38:49 +000057}
58
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +010059/**
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +010060 * Switches the physical CPU back to the corresponding vcpu of the primary VM.
Andrew Scullaa039b32018-10-04 15:02:26 +010061 *
62 * This triggers the scheduling logic to run. Run in the context of secondary VM
63 * to cause HF_VCPU_RUN to return and the primary VM to regain control of the
64 * cpu.
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +010065 */
Wedson Almeida Filho00df6c72018-10-18 11:19:24 +010066static struct vcpu *api_switch_to_primary(struct vcpu *current,
Wedson Almeida Filhoba641ef2018-12-03 04:19:44 +000067 struct hf_vcpu_run_return primary_ret,
68 enum vcpu_state secondary_state)
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +010069{
Andrew Scull19503262018-09-20 14:48:39 +010070 struct vm *primary = vm_get(HF_PRIMARY_VM_ID);
Wedson Almeida Filho00df6c72018-10-18 11:19:24 +010071 struct vcpu *next = &primary->vcpus[cpu_index(current->cpu)];
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +010072
Andrew Walbran508e63c2018-12-20 17:02:37 +000073 /*
74 * If the secondary is blocked but has a timer running, sleep until the
75 * timer fires rather than indefinitely.
76 */
Andrew Scullb06d1752019-02-04 10:15:48 +000077 switch (primary_ret.code) {
78 case HF_VCPU_RUN_WAIT_FOR_INTERRUPT:
79 case HF_VCPU_RUN_WAIT_FOR_MESSAGE:
80 primary_ret.sleep.ns =
81 arch_timer_enabled_current()
82 ? arch_timer_remaining_ns_current()
83 : HF_SLEEP_INDEFINITE;
84 break;
85
86 default:
87 /* Do nothing. */
88 break;
Andrew Walbran508e63c2018-12-20 17:02:37 +000089 }
90
Wedson Almeida Filho00df6c72018-10-18 11:19:24 +010091 /* Set the return value for the primary VM's call to HF_VCPU_RUN. */
Andrew Scull6d2db332018-10-10 15:28:17 +010092 arch_regs_set_retval(&next->regs,
93 hf_vcpu_run_return_encode(primary_ret));
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +010094
Wedson Almeida Filhoba641ef2018-12-03 04:19:44 +000095 /* Mark the current vcpu as waiting. */
96 sl_lock(&current->lock);
97 current->state = secondary_state;
98 sl_unlock(&current->lock);
99
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100100 return next;
101}
102
103/**
Andrew Scull33fecd32019-01-08 14:48:27 +0000104 * Returns to the primary vm and signals that the vcpu still has work to do so.
105 */
106struct vcpu *api_preempt(struct vcpu *current)
107{
108 struct hf_vcpu_run_return ret = {
109 .code = HF_VCPU_RUN_PREEMPTED,
110 };
111
112 return api_switch_to_primary(current, ret, vcpu_state_ready);
113}
114
115/**
Andrew Scullaa039b32018-10-04 15:02:26 +0100116 * Puts the current vcpu in wait for interrupt mode, and returns to the primary
117 * vm.
118 */
Wedson Almeida Filho00df6c72018-10-18 11:19:24 +0100119struct vcpu *api_wait_for_interrupt(struct vcpu *current)
Andrew Scullaa039b32018-10-04 15:02:26 +0100120{
Andrew Scull6d2db332018-10-10 15:28:17 +0100121 struct hf_vcpu_run_return ret = {
122 .code = HF_VCPU_RUN_WAIT_FOR_INTERRUPT,
123 };
Wedson Almeida Filho81568c42019-01-04 13:33:02 +0000124
Wedson Almeida Filhoba641ef2018-12-03 04:19:44 +0000125 return api_switch_to_primary(current, ret,
126 vcpu_state_blocked_interrupt);
Andrew Scullaa039b32018-10-04 15:02:26 +0100127}
128
129/**
Andrew Scull66d62bf2019-02-01 13:54:10 +0000130 * Returns to the primary vm to allow this cpu to be used for other tasks as the
131 * vcpu does not have work to do at this moment. The current vcpu is marked as
132 * ready to be scheduled again.
133 */
134struct vcpu *api_yield(struct vcpu *current)
135{
136 struct hf_vcpu_run_return ret = {
137 .code = HF_VCPU_RUN_YIELD,
138 };
139
140 if (current->vm->id == HF_PRIMARY_VM_ID) {
Andrew Scullb06d1752019-02-04 10:15:48 +0000141 /* Noop on the primary as it makes the scheduling decisions. */
Andrew Scull66d62bf2019-02-01 13:54:10 +0000142 return NULL;
143 }
144
145 return api_switch_to_primary(current, ret, vcpu_state_ready);
146}
147
148/**
Andrew Scull38772ab2019-01-24 15:16:50 +0000149 * Aborts the vCPU and triggers its VM to abort fully.
Andrew Scull9726c252019-01-23 13:44:19 +0000150 */
151struct vcpu *api_abort(struct vcpu *current)
152{
153 struct hf_vcpu_run_return ret = {
154 .code = HF_VCPU_RUN_ABORTED,
155 };
156
157 dlog("Aborting VM %u vCPU %u\n", current->vm->id, vcpu_index(current));
158
159 if (current->vm->id == HF_PRIMARY_VM_ID) {
160 /* TODO: what to do when the primary aborts? */
161 for (;;) {
162 /* Do nothing. */
163 }
164 }
165
166 atomic_store_explicit(&current->vm->aborting, true,
167 memory_order_relaxed);
168
169 /* TODO: free resources once all vCPUs abort. */
170
171 return api_switch_to_primary(current, ret, vcpu_state_aborted);
172}
173
174/**
Andrew Scull55c4d8b2018-12-18 18:50:18 +0000175 * Returns the ID of the VM.
176 */
177int64_t api_vm_get_id(const struct vcpu *current)
178{
179 return current->vm->id;
180}
181
182/**
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +0100183 * Returns the number of VMs configured to run.
184 */
Andrew Scullc0e569a2018-10-02 18:05:21 +0100185int64_t api_vm_get_count(void)
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +0100186{
Andrew Scull19503262018-09-20 14:48:39 +0100187 return vm_get_count();
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +0100188}
189
190/**
191 * Returns the number of vcpus configured in the given VM.
192 */
Wedson Almeida Filho00df6c72018-10-18 11:19:24 +0100193int64_t api_vcpu_get_count(uint32_t vm_id, const struct vcpu *current)
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +0100194{
Andrew Scull19503262018-09-20 14:48:39 +0100195 struct vm *vm;
196
197 /* Only the primary VM needs to know about vcpus for scheduling. */
Wedson Almeida Filho00df6c72018-10-18 11:19:24 +0100198 if (current->vm->id != HF_PRIMARY_VM_ID) {
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +0100199 return -1;
Andrew Scull7364a8e2018-07-19 15:39:29 +0100200 }
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +0100201
Andrew Scull19503262018-09-20 14:48:39 +0100202 vm = vm_get(vm_id);
203 if (vm == NULL) {
204 return -1;
205 }
206
207 return vm->vcpu_count;
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +0100208}
209
210/**
Wedson Almeida Filho03306112018-11-26 00:08:03 +0000211 * This function is called by the architecture-specific context switching
212 * function to indicate that register state for the given vcpu has been saved
213 * and can therefore be used by other pcpus.
214 */
215void api_regs_state_saved(struct vcpu *vcpu)
216{
217 sl_lock(&vcpu->lock);
218 vcpu->regs_available = true;
219 sl_unlock(&vcpu->lock);
220}
221
222/**
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +0000223 * Retrieves the next waiter and removes it from the wait list if the VM's
224 * mailbox is in a writable state.
225 */
226static struct wait_entry *api_fetch_waiter(struct vm_locked locked_vm)
227{
228 struct wait_entry *entry;
229 struct vm *vm = locked_vm.vm;
230
231 if (vm->mailbox.state != mailbox_state_empty ||
232 vm->mailbox.recv == NULL || list_empty(&vm->mailbox.waiter_list)) {
233 /* The mailbox is not writable or there are no waiters. */
234 return NULL;
235 }
236
237 /* Remove waiter from the wait list. */
238 entry = CONTAINER_OF(vm->mailbox.waiter_list.next, struct wait_entry,
239 wait_links);
240 list_remove(&entry->wait_links);
241 return entry;
242}
243
244/**
Andrew Walbran508e63c2018-12-20 17:02:37 +0000245 * Assuming that the arguments have already been checked by the caller, injects
246 * a virtual interrupt of the given ID into the given target vCPU. This doesn't
247 * cause the vCPU to actually be run immediately; it will be taken when the vCPU
248 * is next run, which is up to the scheduler.
249 *
250 * Returns:
251 * - 0 on success if no further action is needed.
252 * - 1 if it was called by the primary VM and the primary VM now needs to wake
253 * up or kick the target vCPU.
254 */
255static int64_t internal_interrupt_inject(struct vm *target_vm,
256 struct vcpu *target_vcpu,
257 uint32_t intid, struct vcpu *current,
258 struct vcpu **next)
259{
260 uint32_t intid_index = intid / INTERRUPT_REGISTER_BITS;
261 uint32_t intid_mask = 1u << (intid % INTERRUPT_REGISTER_BITS);
Andrew Walbran508e63c2018-12-20 17:02:37 +0000262 int64_t ret = 0;
263
264 sl_lock(&target_vcpu->lock);
Andrew Walbran508e63c2018-12-20 17:02:37 +0000265
266 /*
267 * We only need to change state and (maybe) trigger a virtual IRQ if it
268 * is enabled and was not previously pending. Otherwise we can skip
269 * everything except setting the pending bit.
270 *
271 * If you change this logic make sure to update the need_vm_lock logic
272 * above to match.
273 */
274 if (!(target_vcpu->interrupts.interrupt_enabled[intid_index] &
275 ~target_vcpu->interrupts.interrupt_pending[intid_index] &
276 intid_mask)) {
277 goto out;
278 }
279
280 /* Increment the count. */
281 target_vcpu->interrupts.enabled_and_pending_count++;
282
283 /*
284 * Only need to update state if there was not already an
285 * interrupt enabled and pending.
286 */
287 if (target_vcpu->interrupts.enabled_and_pending_count != 1) {
288 goto out;
289 }
290
Andrew Walbran508e63c2018-12-20 17:02:37 +0000291 if (current->vm->id == HF_PRIMARY_VM_ID) {
292 /*
293 * If the call came from the primary VM, let it know that it
294 * should run or kick the target vCPU.
295 */
296 ret = 1;
297 } else if (current != target_vcpu && next != NULL) {
298 /*
299 * Switch to the primary so that it can switch to the target, or
300 * kick it if it is already running on a different physical CPU.
301 */
302 struct hf_vcpu_run_return ret = {
303 .code = HF_VCPU_RUN_WAKE_UP,
304 .wake_up.vm_id = target_vm->id,
305 .wake_up.vcpu = target_vcpu - target_vm->vcpus,
306 };
307 *next = api_switch_to_primary(current, ret, vcpu_state_ready);
308 }
309
310out:
311 /* Either way, make it pending. */
312 target_vcpu->interrupts.interrupt_pending[intid_index] |= intid_mask;
313
314 sl_unlock(&target_vcpu->lock);
Andrew Walbran508e63c2018-12-20 17:02:37 +0000315
316 return ret;
317}
318
319/**
Wedson Almeida Filho03306112018-11-26 00:08:03 +0000320 * Prepares the vcpu to run by updating its state and fetching whether a return
321 * value needs to be forced onto the vCPU.
322 */
Andrew Scull38772ab2019-01-24 15:16:50 +0000323static bool api_vcpu_prepare_run(const struct vcpu *current, struct vcpu *vcpu,
Andrew Walbran508e63c2018-12-20 17:02:37 +0000324 struct hf_vcpu_run_return *run_ret)
Wedson Almeida Filho03306112018-11-26 00:08:03 +0000325{
Andrew Scullb06d1752019-02-04 10:15:48 +0000326 bool need_vm_lock;
Wedson Almeida Filho03306112018-11-26 00:08:03 +0000327 bool ret;
328
Andrew Scullb06d1752019-02-04 10:15:48 +0000329 /*
330 * Wait until the registers become available. All locks must be
331 * released between iterations of this loop to avoid potential deadlocks
332 * if, on any path, a lock needs to be taken after taking the decision
333 * to switch context but before the registers have been saved.
334 *
335 * The VM lock is not needed in the common case so it must only be taken
336 * when it is going to be needed. This ensures there are no inter-vCPU
337 * dependencies in the common run case meaning the sensitive context
338 * switch performance is consistent.
339 */
340 for (;;) {
341 sl_lock(&vcpu->lock);
342
343 /* The VM needs to be locked to deliver mailbox messages. */
344 need_vm_lock = vcpu->state == vcpu_state_blocked_mailbox;
345 if (need_vm_lock) {
346 sl_unlock(&vcpu->lock);
347 sl_lock(&vcpu->vm->lock);
348 sl_lock(&vcpu->lock);
349 }
350
351 if (vcpu->regs_available) {
352 break;
353 }
354
355 if (vcpu->state == vcpu_state_running) {
356 /*
357 * vCPU is running on another pCPU.
358 *
359 * It's ok to not return the sleep duration here because
360 * the other physical CPU that is currently running this
361 * vCPU will return sleep duration if neeed. The default
362 * return value is HF_VCPU_RUN_WAIT_FOR_INTERRUPT, so no
363 * need to set it explicitly.
364 */
365 ret = false;
366 goto out;
367 }
368
369 sl_unlock(&vcpu->lock);
370 if (need_vm_lock) {
371 sl_unlock(&vcpu->vm->lock);
372 }
373 }
Andrew Scull9726c252019-01-23 13:44:19 +0000374
375 if (atomic_load_explicit(&vcpu->vm->aborting, memory_order_relaxed)) {
376 if (vcpu->state != vcpu_state_aborted) {
Andrew Scull82331282019-01-25 10:29:34 +0000377 dlog("Aborting VM %u vCPU %u\n", vcpu->vm->id,
378 vcpu_index(vcpu));
Andrew Scull9726c252019-01-23 13:44:19 +0000379 vcpu->state = vcpu_state_aborted;
380 }
381 ret = false;
382 goto out;
383 }
384
Andrew Walbran508e63c2018-12-20 17:02:37 +0000385 switch (vcpu->state) {
386 case vcpu_state_running:
387 case vcpu_state_off:
388 case vcpu_state_aborted:
Wedson Almeida Filho03306112018-11-26 00:08:03 +0000389 ret = false;
390 goto out;
Andrew Scullb06d1752019-02-04 10:15:48 +0000391
Andrew Walbran508e63c2018-12-20 17:02:37 +0000392 case vcpu_state_blocked_mailbox:
Andrew Scullb06d1752019-02-04 10:15:48 +0000393 /*
394 * A pending message allows the vCPU to run so the message can
395 * be delivered directly.
396 */
397 if (vcpu->vm->mailbox.state == mailbox_state_received) {
398 arch_regs_set_retval(
399 &vcpu->regs,
400 hf_mailbox_receive_return_encode((
401 struct hf_mailbox_receive_return){
402 .vm_id = vcpu->vm->mailbox.recv_from_id,
403 .size = vcpu->vm->mailbox.recv_bytes,
404 }));
405 vcpu->vm->mailbox.state = mailbox_state_read;
406 break;
407 }
408 /* Fall through. */
409 case vcpu_state_blocked_interrupt:
410 /* Allow virtual interrupts to be delivered. */
411 if (vcpu->interrupts.enabled_and_pending_count > 0) {
412 break;
413 }
414
415 /* The timer expired so allow the interrupt to be delivered. */
Andrew Walbran508e63c2018-12-20 17:02:37 +0000416 if (arch_timer_pending(&vcpu->regs)) {
417 break;
418 }
419
420 /*
421 * The vCPU is not ready to run, return the appropriate code to
422 * the primary which called vcpu_run.
423 */
424 if (arch_timer_enabled(&vcpu->regs)) {
Andrew Scullb06d1752019-02-04 10:15:48 +0000425 run_ret->code =
426 vcpu->state == vcpu_state_blocked_mailbox
427 ? HF_VCPU_RUN_WAIT_FOR_MESSAGE
428 : HF_VCPU_RUN_WAIT_FOR_INTERRUPT;
Andrew Walbran508e63c2018-12-20 17:02:37 +0000429 run_ret->sleep.ns =
430 arch_timer_remaining_ns(&vcpu->regs);
431 }
432
433 ret = false;
434 goto out;
Andrew Scullb06d1752019-02-04 10:15:48 +0000435
Andrew Walbran508e63c2018-12-20 17:02:37 +0000436 case vcpu_state_ready:
437 break;
Wedson Almeida Filho03306112018-11-26 00:08:03 +0000438 }
439
Andrew Scullb06d1752019-02-04 10:15:48 +0000440 /* It has been decided that the vCPU should be run. */
Wedson Almeida Filho03306112018-11-26 00:08:03 +0000441 vcpu->cpu = current->cpu;
442 vcpu->state = vcpu_state_running;
443
Wedson Almeida Filho03306112018-11-26 00:08:03 +0000444 /*
Wedson Almeida Filho03306112018-11-26 00:08:03 +0000445 * Mark the registers as unavailable now that we're about to reflect
446 * them onto the real registers. This will also prevent another physical
447 * CPU from trying to read these registers.
448 */
449 vcpu->regs_available = false;
450
451 ret = true;
452
453out:
454 sl_unlock(&vcpu->lock);
Andrew Scullb06d1752019-02-04 10:15:48 +0000455 if (need_vm_lock) {
456 sl_unlock(&vcpu->vm->lock);
457 }
458
Wedson Almeida Filho03306112018-11-26 00:08:03 +0000459 return ret;
460}
461
462/**
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +0100463 * Runs the given vcpu of the given vm.
464 */
Andrew Scull6d2db332018-10-10 15:28:17 +0100465struct hf_vcpu_run_return api_vcpu_run(uint32_t vm_id, uint32_t vcpu_idx,
Andrew Scull38772ab2019-01-24 15:16:50 +0000466 const struct vcpu *current,
467 struct vcpu **next)
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +0100468{
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100469 struct vm *vm;
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +0100470 struct vcpu *vcpu;
Andrew Scull6d2db332018-10-10 15:28:17 +0100471 struct hf_vcpu_run_return ret = {
472 .code = HF_VCPU_RUN_WAIT_FOR_INTERRUPT,
Andrew Scullb06d1752019-02-04 10:15:48 +0000473 .sleep.ns = HF_SLEEP_INDEFINITE,
Andrew Scull6d2db332018-10-10 15:28:17 +0100474 };
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +0100475
476 /* Only the primary VM can switch vcpus. */
Wedson Almeida Filho00df6c72018-10-18 11:19:24 +0100477 if (current->vm->id != HF_PRIMARY_VM_ID) {
Andrew Scull6d2db332018-10-10 15:28:17 +0100478 goto out;
Andrew Scull7364a8e2018-07-19 15:39:29 +0100479 }
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +0100480
Andrew Scull19503262018-09-20 14:48:39 +0100481 /* Only secondary VM vcpus can be run. */
482 if (vm_id == HF_PRIMARY_VM_ID) {
Andrew Scull6d2db332018-10-10 15:28:17 +0100483 goto out;
Andrew Scull7364a8e2018-07-19 15:39:29 +0100484 }
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +0100485
Andrew Scull19503262018-09-20 14:48:39 +0100486 /* The requested VM must exist. */
487 vm = vm_get(vm_id);
488 if (vm == NULL) {
Andrew Scull6d2db332018-10-10 15:28:17 +0100489 goto out;
Andrew Scull19503262018-09-20 14:48:39 +0100490 }
491
492 /* The requested vcpu must exist. */
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100493 if (vcpu_idx >= vm->vcpu_count) {
Andrew Scull6d2db332018-10-10 15:28:17 +0100494 goto out;
Andrew Scull7364a8e2018-07-19 15:39:29 +0100495 }
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +0100496
Wedson Almeida Filho03306112018-11-26 00:08:03 +0000497 /* Update state if allowed. */
Andrew Scullf3d45592018-09-20 14:30:22 +0100498 vcpu = &vm->vcpus[vcpu_idx];
Andrew Scullb06d1752019-02-04 10:15:48 +0000499 if (!api_vcpu_prepare_run(current, vcpu, &ret)) {
Wedson Almeida Filho03306112018-11-26 00:08:03 +0000500 goto out;
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100501 }
Wedson Almeida Filho03306112018-11-26 00:08:03 +0000502
Andrew Walbran508e63c2018-12-20 17:02:37 +0000503 /*
504 * Inject timer interrupt if timer has expired. It's safe to access
505 * vcpu->regs here because api_vcpu_prepare_run already made sure that
506 * regs_available was true (and then set it to false) before returning
507 * true.
508 */
509 if (arch_timer_pending(&vcpu->regs)) {
510 /* Make virtual timer interrupt pending. */
511 internal_interrupt_inject(vm, vcpu, HF_VIRTUAL_TIMER_INTID,
512 vcpu, NULL);
513
514 /*
515 * Set the mask bit so the hardware interrupt doesn't fire
516 * again. Ideally we wouldn't do this because it affects what
517 * the secondary vCPU sees, but if we don't then we end up with
518 * a loop of the interrupt firing each time we try to return to
519 * the secondary vCPU.
520 */
521 arch_timer_mask(&vcpu->regs);
522 }
523
Andrew Scull33fecd32019-01-08 14:48:27 +0000524 /* Switch to the vcpu. */
Wedson Almeida Filho03306112018-11-26 00:08:03 +0000525 *next = vcpu;
Wedson Almeida Filho03306112018-11-26 00:08:03 +0000526
Andrew Scull33fecd32019-01-08 14:48:27 +0000527 /*
528 * Set a placeholder return code to the scheduler. This will be
529 * overwritten when the switch back to the primary occurs.
530 */
531 ret.code = HF_VCPU_RUN_PREEMPTED;
532
Andrew Scull6d2db332018-10-10 15:28:17 +0100533out:
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100534 return ret;
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +0100535}
536
537/**
Andrew Scull81e85092018-12-12 12:56:20 +0000538 * Check that the mode indicates memory that is valid, owned and exclusive.
539 */
Andrew Scullcbefbdb2019-01-11 16:36:26 +0000540static bool api_mode_valid_owned_and_exclusive(int mode)
Andrew Scull81e85092018-12-12 12:56:20 +0000541{
542 return (mode & (MM_MODE_INVALID | MM_MODE_UNOWNED | MM_MODE_SHARED)) ==
543 0;
544}
545
546/**
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +0000547 * Determines the value to be returned by api_vm_configure and api_mailbox_clear
548 * after they've succeeded. If a secondary VM is running and there are waiters,
549 * it also switches back to the primary VM for it to wake waiters up.
550 */
551static int64_t api_waiter_result(struct vm_locked locked_vm,
552 struct vcpu *current, struct vcpu **next)
553{
554 struct vm *vm = locked_vm.vm;
555 struct hf_vcpu_run_return ret = {
556 .code = HF_VCPU_RUN_NOTIFY_WAITERS,
557 };
558
559 if (list_empty(&vm->mailbox.waiter_list)) {
560 /* No waiters, nothing else to do. */
561 return 0;
562 }
563
564 if (vm->id == HF_PRIMARY_VM_ID) {
565 /* The caller is the primary VM. Tell it to wake up waiters. */
566 return 1;
567 }
568
569 /*
570 * Switch back to the primary VM, informing it that there are waiters
571 * that need to be notified.
572 */
573 *next = api_switch_to_primary(current, ret, vcpu_state_ready);
574
575 return 0;
576}
577
578/**
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100579 * Configures the VM to send/receive data through the specified pages. The pages
580 * must not be shared.
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +0000581 *
582 * Returns:
583 * - -1 on failure.
584 * - 0 on success if no further action is needed.
585 * - 1 if it was called by the primary VM and the primary VM now needs to wake
586 * up or kick waiters. Waiters should be retrieved by calling
587 * hf_mailbox_waiter_get.
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100588 */
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +0000589int64_t api_vm_configure(ipaddr_t send, ipaddr_t recv, struct vcpu *current,
590 struct vcpu **next)
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100591{
Wedson Almeida Filho00df6c72018-10-18 11:19:24 +0100592 struct vm *vm = current->vm;
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +0000593 struct vm_locked locked;
Andrew Scull80871322018-08-06 12:04:09 +0100594 paddr_t pa_send_begin;
595 paddr_t pa_send_end;
596 paddr_t pa_recv_begin;
597 paddr_t pa_recv_end;
Andrew Scull220e6212018-12-21 18:09:00 +0000598 int orig_send_mode;
599 int orig_recv_mode;
600 struct mpool local_page_pool;
Andrew Scullc0e569a2018-10-02 18:05:21 +0100601 int64_t ret;
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100602
603 /* Fail if addresses are not page-aligned. */
Alfredo Mazzinghieb1997c2019-02-07 18:00:01 +0000604 if (!is_aligned(ipa_addr(send), PAGE_SIZE) ||
605 !is_aligned(ipa_addr(recv), PAGE_SIZE)) {
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100606 return -1;
607 }
608
Andrew Scullc2eb6a32018-12-13 16:54:24 +0000609 /* Convert to physical addresses. */
610 pa_send_begin = pa_from_ipa(send);
611 pa_send_end = pa_add(pa_send_begin, PAGE_SIZE);
612
613 pa_recv_begin = pa_from_ipa(recv);
614 pa_recv_end = pa_add(pa_recv_begin, PAGE_SIZE);
615
Andrew Scullc9ccb3f2018-08-13 15:27:12 +0100616 /* Fail if the same page is used for the send and receive pages. */
617 if (pa_addr(pa_send_begin) == pa_addr(pa_recv_begin)) {
Andrew Scull220e6212018-12-21 18:09:00 +0000618 return -1;
619 }
620
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +0000621 vm_lock(vm, &locked);
Andrew Scull220e6212018-12-21 18:09:00 +0000622
623 /* We only allow these to be setup once. */
624 if (vm->mailbox.send || vm->mailbox.recv) {
625 goto fail;
626 }
627
628 /*
629 * Ensure the pages are valid, owned and exclusive to the VM and that
630 * the VM has the required access to the memory.
631 */
632 if (!mm_vm_get_mode(&vm->ptable, send, ipa_add(send, PAGE_SIZE),
633 &orig_send_mode) ||
634 !api_mode_valid_owned_and_exclusive(orig_send_mode) ||
635 (orig_send_mode & MM_MODE_R) == 0 ||
636 (orig_send_mode & MM_MODE_W) == 0) {
637 goto fail;
638 }
639
640 if (!mm_vm_get_mode(&vm->ptable, recv, ipa_add(recv, PAGE_SIZE),
641 &orig_recv_mode) ||
642 !api_mode_valid_owned_and_exclusive(orig_recv_mode) ||
643 (orig_recv_mode & MM_MODE_R) == 0) {
644 goto fail;
645 }
646
647 /*
648 * Create a local pool so any freed memory can't be used by another
649 * thread. This is to ensure the original mapping can be restored if any
650 * stage of the process fails.
651 */
652 mpool_init_with_fallback(&local_page_pool, &api_page_pool);
653
654 /* Take memory ownership away from the VM and mark as shared. */
655 if (!mm_vm_identity_map(
656 &vm->ptable, pa_send_begin, pa_send_end,
657 MM_MODE_UNOWNED | MM_MODE_SHARED | MM_MODE_R | MM_MODE_W,
658 NULL, &local_page_pool)) {
659 goto fail_free_pool;
660 }
661
662 if (!mm_vm_identity_map(&vm->ptable, pa_recv_begin, pa_recv_end,
663 MM_MODE_UNOWNED | MM_MODE_SHARED | MM_MODE_R,
664 NULL, &local_page_pool)) {
665 /* TODO: partial defrag of failed range. */
666 /* Recover any memory consumed in failed mapping. */
Andrew Scullda3df7f2019-01-05 17:49:27 +0000667 mm_vm_defrag(&vm->ptable, &local_page_pool);
Andrew Scull220e6212018-12-21 18:09:00 +0000668 goto fail_undo_send;
Andrew Scullc9ccb3f2018-08-13 15:27:12 +0100669 }
670
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100671 /* Map the send page as read-only in the hypervisor address space. */
Wedson Almeida Filho22d5eaa2018-12-16 00:38:49 +0000672 vm->mailbox.send = mm_identity_map(pa_send_begin, pa_send_end,
Andrew Scull220e6212018-12-21 18:09:00 +0000673 MM_MODE_R, &local_page_pool);
Andrew Scullaa039b32018-10-04 15:02:26 +0100674 if (!vm->mailbox.send) {
Andrew Scull220e6212018-12-21 18:09:00 +0000675 /* TODO: partial defrag of failed range. */
676 /* Recover any memory consumed in failed mapping. */
677 mm_defrag(&local_page_pool);
678 goto fail_undo_send_and_recv;
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100679 }
680
681 /*
682 * Map the receive page as writable in the hypervisor address space. On
683 * failure, unmap the send page before returning.
684 */
Wedson Almeida Filho22d5eaa2018-12-16 00:38:49 +0000685 vm->mailbox.recv = mm_identity_map(pa_recv_begin, pa_recv_end,
Andrew Scull220e6212018-12-21 18:09:00 +0000686 MM_MODE_W, &local_page_pool);
Andrew Scullaa039b32018-10-04 15:02:26 +0100687 if (!vm->mailbox.recv) {
Andrew Scull220e6212018-12-21 18:09:00 +0000688 /* TODO: partial defrag of failed range. */
689 /* Recover any memory consumed in failed mapping. */
690 mm_defrag(&local_page_pool);
691 goto fail_undo_all;
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100692 }
693
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +0000694 /* Tell caller about waiters, if any. */
695 ret = api_waiter_result(locked, current, next);
Andrew Scull220e6212018-12-21 18:09:00 +0000696 goto exit;
697
698 /*
699 * The following mappings will not require more memory than is available
700 * in the local pool.
701 */
702fail_undo_all:
703 vm->mailbox.send = NULL;
Andrew Scullda241972019-01-05 18:17:48 +0000704 mm_unmap(pa_send_begin, pa_send_end, &local_page_pool);
Andrew Scull220e6212018-12-21 18:09:00 +0000705
706fail_undo_send_and_recv:
707 mm_vm_identity_map(&vm->ptable, pa_recv_begin, pa_recv_end,
708 orig_recv_mode, NULL, &local_page_pool);
709
710fail_undo_send:
711 mm_vm_identity_map(&vm->ptable, pa_send_begin, pa_send_end,
712 orig_send_mode, NULL, &local_page_pool);
713
714fail_free_pool:
715 mpool_fini(&local_page_pool);
716
717fail:
718 ret = -1;
719
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100720exit:
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +0000721 vm_unlock(&locked);
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100722
723 return ret;
724}
725
726/**
Andrew Scullaa039b32018-10-04 15:02:26 +0100727 * Copies data from the sender's send buffer to the recipient's receive buffer
728 * and notifies the recipient.
Wedson Almeida Filho17c997f2019-01-09 18:50:09 +0000729 *
730 * If the recipient's receive buffer is busy, it can optionally register the
731 * caller to be notified when the recipient's receive buffer becomes available.
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100732 */
Jose Marinhoa1dfeda2019-02-27 16:46:03 +0000733int32_t api_spci_msg_send(uint32_t attributes, struct vcpu *current,
734 struct vcpu **next)
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100735{
Wedson Almeida Filho00df6c72018-10-18 11:19:24 +0100736 struct vm *from = current->vm;
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100737 struct vm *to;
Andrew Scullb06d1752019-02-04 10:15:48 +0000738 struct hf_vcpu_run_return primary_ret = {
739 .code = HF_VCPU_RUN_MESSAGE,
740 };
Jose Marinhoa1dfeda2019-02-27 16:46:03 +0000741 struct spci_message from_msg_replica;
742 struct spci_message *to_msg;
743 const struct spci_message *from_msg;
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100744
Jose Marinhoa1dfeda2019-02-27 16:46:03 +0000745 uint32_t size;
Andrew Scull19503262018-09-20 14:48:39 +0100746
Jose Marinhoa1dfeda2019-02-27 16:46:03 +0000747 int64_t ret;
748 bool notify = (attributes & SPCI_MSG_SEND_NOTIFY_MASK) ==
749 SPCI_MSG_SEND_NOTIFY;
Andrew Scull19503262018-09-20 14:48:39 +0100750
Jose Marinhoa1dfeda2019-02-27 16:46:03 +0000751 /*
752 * Check that the sender has configured its send buffer. Copy the
753 * message header. If the tx mailbox at from_msg is configured (i.e.
754 * from_msg != NULL) then it can be safely accessed after releasing the
755 * lock since the tx mailbox address can only be configured once.
756 */
757 sl_lock(&from->lock);
758 from_msg = from->mailbox.send;
759 sl_unlock(&from->lock);
760
761 if (from_msg == NULL) {
762 return SPCI_INVALID_PARAMETERS;
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100763 }
764
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100765 /*
Jose Marinhoa1dfeda2019-02-27 16:46:03 +0000766 * Note that the payload is not copied when the message header is.
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100767 */
Jose Marinhoa1dfeda2019-02-27 16:46:03 +0000768 from_msg_replica = *from_msg;
769
770 /* Ensure source VM id corresponds to the current VM. */
771 if (from_msg_replica.source_vm_id != from->id) {
772 return SPCI_INVALID_PARAMETERS;
773 }
774
775 size = from_msg_replica.length;
776 /* Limit the size of transfer. */
777 if (size > HF_MAILBOX_SIZE - sizeof(struct spci_message)) {
778 return SPCI_INVALID_PARAMETERS;
779 }
780
781 /* Disallow reflexive requests as this suggests an error in the VM. */
782 if (from_msg_replica.target_vm_id == from->id) {
783 return SPCI_INVALID_PARAMETERS;
784 }
785
786 /* Ensure the target VM exists. */
787 to = vm_get(from_msg_replica.target_vm_id);
788 if (to == NULL) {
789 return SPCI_INVALID_PARAMETERS;
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100790 }
791
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100792 sl_lock(&to->lock);
793
Andrew Scullaa039b32018-10-04 15:02:26 +0100794 if (to->mailbox.state != mailbox_state_empty ||
795 to->mailbox.recv == NULL) {
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +0000796 /*
797 * Fail if the target isn't currently ready to receive data,
798 * setting up for notification if requested.
799 */
800 if (notify) {
Wedson Almeida Filhob790f652019-01-22 23:41:56 +0000801 struct wait_entry *entry =
Jose Marinhoa1dfeda2019-02-27 16:46:03 +0000802 &current->vm->wait_entries
803 [from_msg_replica.target_vm_id];
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +0000804
805 /* Append waiter only if it's not there yet. */
806 if (list_empty(&entry->wait_links)) {
807 list_append(&to->mailbox.waiter_list,
808 &entry->wait_links);
809 }
810 }
811
Jose Marinhoa1dfeda2019-02-27 16:46:03 +0000812 ret = SPCI_BUSY;
Andrew Scullaa039b32018-10-04 15:02:26 +0100813 goto out;
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100814 }
815
Andrew Scullaa039b32018-10-04 15:02:26 +0100816 /* Copy data. */
Jose Marinhoa1dfeda2019-02-27 16:46:03 +0000817 to_msg = to->mailbox.recv;
818 *to_msg = from_msg_replica;
819 memcpy(to_msg->payload, from->mailbox.send->payload, size);
Andrew Scullaa039b32018-10-04 15:02:26 +0100820 to->mailbox.recv_bytes = size;
821 to->mailbox.recv_from_id = from->id;
Andrew Scullb06d1752019-02-04 10:15:48 +0000822 primary_ret.message.vm_id = to->id;
Jose Marinhoa1dfeda2019-02-27 16:46:03 +0000823 ret = SPCI_SUCCESS;
Andrew Scullaa039b32018-10-04 15:02:26 +0100824
825 /* Messages for the primary VM are delivered directly. */
826 if (to->id == HF_PRIMARY_VM_ID) {
Andrew Scullb06d1752019-02-04 10:15:48 +0000827 primary_ret.message.size = size,
828 to->mailbox.state = mailbox_state_read;
Wedson Almeida Filhoba641ef2018-12-03 04:19:44 +0000829 *next = api_switch_to_primary(current, primary_ret,
830 vcpu_state_ready);
Andrew Scullaa039b32018-10-04 15:02:26 +0100831 goto out;
832 }
833
Andrew Scullb06d1752019-02-04 10:15:48 +0000834 to->mailbox.state = mailbox_state_received;
Andrew Scullaa039b32018-10-04 15:02:26 +0100835
836 /* Return to the primary VM directly or with a switch. */
Andrew Scullb06d1752019-02-04 10:15:48 +0000837 if (from->id != HF_PRIMARY_VM_ID) {
Wedson Almeida Filhoba641ef2018-12-03 04:19:44 +0000838 *next = api_switch_to_primary(current, primary_ret,
839 vcpu_state_ready);
Wedson Almeida Filho80eb4a32018-11-30 17:11:15 +0000840 }
Andrew Scullaa039b32018-10-04 15:02:26 +0100841
842out:
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100843 sl_unlock(&to->lock);
844
Wedson Almeida Filho80eb4a32018-11-30 17:11:15 +0000845 return ret;
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100846}
847
848/**
Andrew Scullaa039b32018-10-04 15:02:26 +0100849 * Receives a message from the mailbox. If one isn't available, this function
850 * can optionally block the caller until one becomes available.
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100851 *
Andrew Scullaa039b32018-10-04 15:02:26 +0100852 * No new messages can be received until the mailbox has been cleared.
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100853 */
Andrew Scull6d2db332018-10-10 15:28:17 +0100854struct hf_mailbox_receive_return api_mailbox_receive(bool block,
Wedson Almeida Filho00df6c72018-10-18 11:19:24 +0100855 struct vcpu *current,
Andrew Scull6d2db332018-10-10 15:28:17 +0100856 struct vcpu **next)
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100857{
Wedson Almeida Filho00df6c72018-10-18 11:19:24 +0100858 struct vm *vm = current->vm;
Andrew Scull6d2db332018-10-10 15:28:17 +0100859 struct hf_mailbox_receive_return ret = {
860 .vm_id = HF_INVALID_VM_ID,
861 };
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100862
Andrew Scullaa039b32018-10-04 15:02:26 +0100863 /*
864 * The primary VM will receive messages as a status code from running
865 * vcpus and must not call this function.
866 */
Andrew Scull19503262018-09-20 14:48:39 +0100867 if (vm->id == HF_PRIMARY_VM_ID) {
Andrew Scull6d2db332018-10-10 15:28:17 +0100868 return ret;
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100869 }
870
871 sl_lock(&vm->lock);
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100872
Andrew Scullaa039b32018-10-04 15:02:26 +0100873 /* Return pending messages without blocking. */
874 if (vm->mailbox.state == mailbox_state_received) {
875 vm->mailbox.state = mailbox_state_read;
Andrew Scull6d2db332018-10-10 15:28:17 +0100876 ret.vm_id = vm->mailbox.recv_from_id;
877 ret.size = vm->mailbox.recv_bytes;
Andrew Scullaa039b32018-10-04 15:02:26 +0100878 goto out;
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100879 }
Andrew Scullaa039b32018-10-04 15:02:26 +0100880
Andrew Walbran9311c9a2019-03-12 16:59:04 +0000881 /*
882 * No pending message so fail if not allowed to block. Don't block if
883 * there are enabled and pending interrupts, to match behaviour of
884 * wait_for_interrupt.
885 */
886 if (!block || current->interrupts.enabled_and_pending_count > 0) {
Andrew Scullaa039b32018-10-04 15:02:26 +0100887 goto out;
888 }
889
Andrew Scullaa039b32018-10-04 15:02:26 +0100890 /* Switch back to primary vm to block. */
Andrew Walbranb4816552018-12-05 17:35:42 +0000891 {
892 struct hf_vcpu_run_return run_return = {
Andrew Scullb06d1752019-02-04 10:15:48 +0000893 .code = HF_VCPU_RUN_WAIT_FOR_MESSAGE,
Andrew Walbranb4816552018-12-05 17:35:42 +0000894 };
Wedson Almeida Filho81568c42019-01-04 13:33:02 +0000895
Andrew Walbranb4816552018-12-05 17:35:42 +0000896 *next = api_switch_to_primary(current, run_return,
897 vcpu_state_blocked_mailbox);
898 }
Andrew Scullaa039b32018-10-04 15:02:26 +0100899out:
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100900 sl_unlock(&vm->lock);
901
902 return ret;
903}
904
905/**
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +0000906 * Retrieves the next VM whose mailbox became writable. For a VM to be notified
907 * by this function, the caller must have called api_mailbox_send before with
908 * the notify argument set to true, and this call must have failed because the
909 * mailbox was not available.
910 *
911 * It should be called repeatedly to retrieve a list of VMs.
912 *
913 * Returns -1 if no VM became writable, or the id of the VM whose mailbox
914 * became writable.
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100915 */
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +0000916int64_t api_mailbox_writable_get(const struct vcpu *current)
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100917{
Wedson Almeida Filho00df6c72018-10-18 11:19:24 +0100918 struct vm *vm = current->vm;
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +0000919 struct wait_entry *entry;
Andrew Scullc0e569a2018-10-02 18:05:21 +0100920 int64_t ret;
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +0100921
922 sl_lock(&vm->lock);
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +0000923 if (list_empty(&vm->mailbox.ready_list)) {
924 ret = -1;
925 goto exit;
926 }
927
928 entry = CONTAINER_OF(vm->mailbox.ready_list.next, struct wait_entry,
929 ready_links);
930 list_remove(&entry->ready_links);
Wedson Almeida Filhob790f652019-01-22 23:41:56 +0000931 ret = entry - vm->wait_entries;
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +0000932
933exit:
934 sl_unlock(&vm->lock);
935 return ret;
936}
937
938/**
939 * Retrieves the next VM waiting to be notified that the mailbox of the
940 * specified VM became writable. Only primary VMs are allowed to call this.
941 *
Wedson Almeida Filhob790f652019-01-22 23:41:56 +0000942 * Returns -1 on failure or if there are no waiters; the VM id of the next
943 * waiter otherwise.
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +0000944 */
945int64_t api_mailbox_waiter_get(uint32_t vm_id, const struct vcpu *current)
946{
947 struct vm *vm;
948 struct vm_locked locked;
949 struct wait_entry *entry;
950 struct vm *waiting_vm;
951
952 /* Only primary VMs are allowed to call this function. */
953 if (current->vm->id != HF_PRIMARY_VM_ID) {
954 return -1;
955 }
956
957 vm = vm_get(vm_id);
958 if (vm == NULL) {
959 return -1;
960 }
961
962 /* Check if there are outstanding notifications from given vm. */
963 vm_lock(vm, &locked);
964 entry = api_fetch_waiter(locked);
965 vm_unlock(&locked);
966
967 if (entry == NULL) {
968 return -1;
969 }
970
971 /* Enqueue notification to waiting VM. */
972 waiting_vm = entry->waiting_vm;
973
974 sl_lock(&waiting_vm->lock);
975 if (list_empty(&entry->ready_links)) {
976 list_append(&waiting_vm->mailbox.ready_list,
977 &entry->ready_links);
978 }
979 sl_unlock(&waiting_vm->lock);
980
981 return waiting_vm->id;
982}
983
984/**
985 * Clears the caller's mailbox so that a new message can be received. The caller
986 * must have copied out all data they wish to preserve as new messages will
987 * overwrite the old and will arrive asynchronously.
988 *
989 * Returns:
Andrew Scullaa7db8e2019-02-01 14:12:19 +0000990 * - -1 on failure, if the mailbox hasn't been read.
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +0000991 * - 0 on success if no further action is needed.
992 * - 1 if it was called by the primary VM and the primary VM now needs to wake
993 * up or kick waiters. Waiters should be retrieved by calling
994 * hf_mailbox_waiter_get.
995 */
996int64_t api_mailbox_clear(struct vcpu *current, struct vcpu **next)
997{
998 struct vm *vm = current->vm;
999 struct vm_locked locked;
1000 int64_t ret;
1001
1002 vm_lock(vm, &locked);
Andrew Scullaa7db8e2019-02-01 14:12:19 +00001003 switch (vm->mailbox.state) {
1004 case mailbox_state_empty:
1005 ret = 0;
1006 break;
1007
1008 case mailbox_state_received:
1009 ret = -1;
1010 break;
1011
1012 case mailbox_state_read:
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +00001013 ret = api_waiter_result(locked, current, next);
Andrew Scullaa039b32018-10-04 15:02:26 +01001014 vm->mailbox.state = mailbox_state_empty;
Andrew Scullaa7db8e2019-02-01 14:12:19 +00001015 break;
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +01001016 }
Wedson Almeida Filhoea62e2e2019-01-09 19:14:59 +00001017 vm_unlock(&locked);
Wedson Almeida Filho2f94ec12018-07-26 16:00:48 +01001018
1019 return ret;
Wedson Almeida Filho3fcbcff2018-07-10 23:53:39 +01001020}
Andrew Walbran318f5732018-11-20 16:23:42 +00001021
1022/**
1023 * Enables or disables a given interrupt ID for the calling vCPU.
1024 *
1025 * Returns 0 on success, or -1 if the intid is invalid.
1026 */
Wedson Almeida Filhoc559d132019-01-09 19:33:40 +00001027int64_t api_interrupt_enable(uint32_t intid, bool enable, struct vcpu *current)
Andrew Walbran318f5732018-11-20 16:23:42 +00001028{
1029 uint32_t intid_index = intid / INTERRUPT_REGISTER_BITS;
1030 uint32_t intid_mask = 1u << (intid % INTERRUPT_REGISTER_BITS);
Wedson Almeida Filho81568c42019-01-04 13:33:02 +00001031
Andrew Walbran318f5732018-11-20 16:23:42 +00001032 if (intid >= HF_NUM_INTIDS) {
1033 return -1;
1034 }
1035
1036 sl_lock(&current->lock);
1037 if (enable) {
Andrew Walbran3d84a262018-12-13 14:41:19 +00001038 /*
1039 * If it is pending and was not enabled before, increment the
1040 * count.
1041 */
1042 if (current->interrupts.interrupt_pending[intid_index] &
1043 ~current->interrupts.interrupt_enabled[intid_index] &
1044 intid_mask) {
1045 current->interrupts.enabled_and_pending_count++;
1046 }
Andrew Walbran318f5732018-11-20 16:23:42 +00001047 current->interrupts.interrupt_enabled[intid_index] |=
1048 intid_mask;
Andrew Walbran318f5732018-11-20 16:23:42 +00001049 } else {
Andrew Walbran3d84a262018-12-13 14:41:19 +00001050 /*
1051 * If it is pending and was enabled before, decrement the count.
1052 */
1053 if (current->interrupts.interrupt_pending[intid_index] &
1054 current->interrupts.interrupt_enabled[intid_index] &
1055 intid_mask) {
1056 current->interrupts.enabled_and_pending_count--;
1057 }
Andrew Walbran318f5732018-11-20 16:23:42 +00001058 current->interrupts.interrupt_enabled[intid_index] &=
1059 ~intid_mask;
1060 }
1061
1062 sl_unlock(&current->lock);
1063 return 0;
1064}
1065
1066/**
1067 * Returns the ID of the next pending interrupt for the calling vCPU, and
1068 * acknowledges it (i.e. marks it as no longer pending). Returns
1069 * HF_INVALID_INTID if there are no pending interrupts.
1070 */
Wedson Almeida Filhoc559d132019-01-09 19:33:40 +00001071uint32_t api_interrupt_get(struct vcpu *current)
Andrew Walbran318f5732018-11-20 16:23:42 +00001072{
1073 uint8_t i;
1074 uint32_t first_interrupt = HF_INVALID_INTID;
Andrew Walbran318f5732018-11-20 16:23:42 +00001075
1076 /*
1077 * Find the first enabled and pending interrupt ID, return it, and
1078 * deactivate it.
1079 */
1080 sl_lock(&current->lock);
1081 for (i = 0; i < HF_NUM_INTIDS / INTERRUPT_REGISTER_BITS; ++i) {
1082 uint32_t enabled_and_pending =
1083 current->interrupts.interrupt_enabled[i] &
1084 current->interrupts.interrupt_pending[i];
Wedson Almeida Filho81568c42019-01-04 13:33:02 +00001085
Andrew Walbran318f5732018-11-20 16:23:42 +00001086 if (enabled_and_pending != 0) {
Andrew Walbran3d84a262018-12-13 14:41:19 +00001087 uint8_t bit_index = ctz(enabled_and_pending);
1088 /*
1089 * Mark it as no longer pending and decrement the count.
1090 */
1091 current->interrupts.interrupt_pending[i] &=
1092 ~(1u << bit_index);
1093 current->interrupts.enabled_and_pending_count--;
1094 first_interrupt =
1095 i * INTERRUPT_REGISTER_BITS + bit_index;
Andrew Walbran318f5732018-11-20 16:23:42 +00001096 break;
1097 }
1098 }
Andrew Walbran318f5732018-11-20 16:23:42 +00001099
1100 sl_unlock(&current->lock);
1101 return first_interrupt;
1102}
1103
1104/**
Andrew Walbran4cf217a2018-12-14 15:24:50 +00001105 * Returns whether the current vCPU is allowed to inject an interrupt into the
Andrew Walbran318f5732018-11-20 16:23:42 +00001106 * given VM and vCPU.
1107 */
1108static inline bool is_injection_allowed(uint32_t target_vm_id,
1109 struct vcpu *current)
1110{
1111 uint32_t current_vm_id = current->vm->id;
Wedson Almeida Filho81568c42019-01-04 13:33:02 +00001112
Andrew Walbran318f5732018-11-20 16:23:42 +00001113 /*
1114 * The primary VM is allowed to inject interrupts into any VM. Secondary
1115 * VMs are only allowed to inject interrupts into their own vCPUs.
1116 */
1117 return current_vm_id == HF_PRIMARY_VM_ID ||
1118 current_vm_id == target_vm_id;
1119}
1120
1121/**
1122 * Injects a virtual interrupt of the given ID into the given target vCPU.
1123 * This doesn't cause the vCPU to actually be run immediately; it will be taken
1124 * when the vCPU is next run, which is up to the scheduler.
1125 *
Andrew Walbran3d84a262018-12-13 14:41:19 +00001126 * Returns:
1127 * - -1 on failure because the target VM or vCPU doesn't exist, the interrupt
1128 * ID is invalid, or the current VM is not allowed to inject interrupts to
1129 * the target VM.
1130 * - 0 on success if no further action is needed.
1131 * - 1 if it was called by the primary VM and the primary VM now needs to wake
1132 * up or kick the target vCPU.
Andrew Walbran318f5732018-11-20 16:23:42 +00001133 */
Wedson Almeida Filhoc559d132019-01-09 19:33:40 +00001134int64_t api_interrupt_inject(uint32_t target_vm_id, uint32_t target_vcpu_idx,
Andrew Walbran318f5732018-11-20 16:23:42 +00001135 uint32_t intid, struct vcpu *current,
1136 struct vcpu **next)
1137{
Andrew Walbran318f5732018-11-20 16:23:42 +00001138 struct vcpu *target_vcpu;
1139 struct vm *target_vm = vm_get(target_vm_id);
1140
1141 if (intid >= HF_NUM_INTIDS) {
1142 return -1;
1143 }
Wedson Almeida Filho81568c42019-01-04 13:33:02 +00001144
Andrew Walbran318f5732018-11-20 16:23:42 +00001145 if (target_vm == NULL) {
1146 return -1;
1147 }
Wedson Almeida Filho81568c42019-01-04 13:33:02 +00001148
Andrew Walbran318f5732018-11-20 16:23:42 +00001149 if (target_vcpu_idx >= target_vm->vcpu_count) {
1150 /* The requested vcpu must exist. */
1151 return -1;
1152 }
Wedson Almeida Filho81568c42019-01-04 13:33:02 +00001153
Andrew Walbran318f5732018-11-20 16:23:42 +00001154 if (!is_injection_allowed(target_vm_id, current)) {
1155 return -1;
1156 }
Wedson Almeida Filho81568c42019-01-04 13:33:02 +00001157
Andrew Walbran318f5732018-11-20 16:23:42 +00001158 target_vcpu = &target_vm->vcpus[target_vcpu_idx];
1159
1160 dlog("Injecting IRQ %d for VM %d VCPU %d from VM %d VCPU %d\n", intid,
1161 target_vm_id, target_vcpu_idx, current->vm->id, current->cpu->id);
Andrew Walbran508e63c2018-12-20 17:02:37 +00001162 return internal_interrupt_inject(target_vm, target_vcpu, intid, current,
1163 next);
Andrew Walbran318f5732018-11-20 16:23:42 +00001164}
Andrew Scull6386f252018-12-06 13:29:10 +00001165
1166/**
1167 * Clears a region of physical memory by overwriting it with zeros. The data is
1168 * flushed from the cache so the memory has been cleared across the system.
1169 */
1170static bool api_clear_memory(paddr_t begin, paddr_t end, struct mpool *ppool)
1171{
1172 /*
1173 * TODO: change this to a cpu local single page window rather than a
1174 * global mapping of the whole range. Such an approach will limit
1175 * the changes to stage-1 tables and will allow only local
1176 * invalidation.
1177 */
1178 void *ptr = mm_identity_map(begin, end, MM_MODE_W, ppool);
1179 size_t size = pa_addr(end) - pa_addr(begin);
1180
1181 if (!ptr) {
1182 /* TODO: partial defrag of failed range. */
1183 /* Recover any memory consumed in failed mapping. */
1184 mm_defrag(ppool);
1185 return false;
1186 }
1187
1188 memset(ptr, 0, size);
1189 arch_mm_write_back_dcache(ptr, size);
1190 mm_unmap(begin, end, ppool);
1191
1192 return true;
1193}
1194
1195/**
1196 * Shares memory from the calling VM with another. The memory can be shared in
1197 * different modes.
1198 *
1199 * TODO: the interface for sharing memory will need to be enhanced to allow
1200 * sharing with different modes e.g. read-only, informing the recipient
1201 * of the memory they have been given, opting to not wipe the memory and
1202 * possibly allowing multiple blocks to be transferred. What this will
1203 * look like is TBD.
1204 */
1205int64_t api_share_memory(uint32_t vm_id, ipaddr_t addr, size_t size,
1206 enum hf_share share, struct vcpu *current)
1207{
1208 struct vm *from = current->vm;
1209 struct vm *to;
1210 int orig_from_mode;
1211 int from_mode;
1212 int to_mode;
1213 ipaddr_t begin;
1214 ipaddr_t end;
1215 paddr_t pa_begin;
1216 paddr_t pa_end;
1217 struct mpool local_page_pool;
1218 int64_t ret;
1219
1220 /* Disallow reflexive shares as this suggests an error in the VM. */
1221 if (vm_id == from->id) {
1222 return -1;
1223 }
1224
1225 /* Ensure the target VM exists. */
1226 to = vm_get(vm_id);
1227 if (to == NULL) {
1228 return -1;
1229 }
1230
1231 begin = addr;
1232 end = ipa_add(addr, size);
1233
1234 /* Fail if addresses are not page-aligned. */
Alfredo Mazzinghieb1997c2019-02-07 18:00:01 +00001235 if (!is_aligned(ipa_addr(begin), PAGE_SIZE) ||
1236 !is_aligned(ipa_addr(end), PAGE_SIZE)) {
Andrew Scull6386f252018-12-06 13:29:10 +00001237 return -1;
1238 }
1239
1240 /* Convert the sharing request to memory management modes. */
1241 switch (share) {
1242 case HF_MEMORY_GIVE:
1243 from_mode = MM_MODE_INVALID | MM_MODE_UNOWNED;
1244 to_mode = MM_MODE_R | MM_MODE_W | MM_MODE_X;
1245 break;
1246
1247 case HF_MEMORY_LEND:
1248 from_mode = MM_MODE_INVALID;
1249 to_mode = MM_MODE_R | MM_MODE_W | MM_MODE_X | MM_MODE_UNOWNED;
1250 break;
1251
1252 case HF_MEMORY_SHARE:
1253 from_mode = MM_MODE_R | MM_MODE_W | MM_MODE_X | MM_MODE_SHARED;
1254 to_mode = MM_MODE_R | MM_MODE_W | MM_MODE_X | MM_MODE_UNOWNED |
1255 MM_MODE_SHARED;
1256 break;
1257
1258 default:
1259 /* The input is untrusted so might not be a valid value. */
1260 return -1;
1261 }
1262
1263 /*
1264 * Create a local pool so any freed memory can't be used by another
1265 * thread. This is to ensure the original mapping can be restored if any
1266 * stage of the process fails.
1267 */
1268 mpool_init_with_fallback(&local_page_pool, &api_page_pool);
1269
1270 sl_lock_both(&from->lock, &to->lock);
1271
1272 /*
1273 * Ensure that the memory range is mapped with the same mode so that
1274 * changes can be reverted if the process fails.
1275 */
1276 if (!mm_vm_get_mode(&from->ptable, begin, end, &orig_from_mode)) {
1277 goto fail;
1278 }
1279
1280 /*
1281 * Ensure the memory range is valid for the sender. If it isn't, the
1282 * sender has either shared it with another VM already or has no claim
1283 * to the memory.
1284 */
1285 if (orig_from_mode & MM_MODE_INVALID) {
1286 goto fail;
1287 }
1288
1289 /*
1290 * The sender must own the memory and have exclusive access to it in
1291 * order to share it. Alternatively, it is giving memory back to the
1292 * owning VM.
1293 */
1294 if (orig_from_mode & MM_MODE_UNOWNED) {
1295 int orig_to_mode;
1296
1297 if (share != HF_MEMORY_GIVE ||
1298 !mm_vm_get_mode(&to->ptable, begin, end, &orig_to_mode) ||
1299 orig_to_mode & MM_MODE_UNOWNED) {
1300 goto fail;
1301 }
1302 } else if (orig_from_mode & MM_MODE_SHARED) {
1303 goto fail;
1304 }
1305
1306 pa_begin = pa_from_ipa(begin);
1307 pa_end = pa_from_ipa(end);
1308
1309 /*
1310 * First update the mapping for the sender so there is not overlap with
1311 * the recipient.
1312 */
1313 if (!mm_vm_identity_map(&from->ptable, pa_begin, pa_end, from_mode,
1314 NULL, &local_page_pool)) {
1315 goto fail;
1316 }
1317
1318 /* Clear the memory so no VM or device can see the previous contents. */
1319 if (!api_clear_memory(pa_begin, pa_end, &local_page_pool)) {
1320 goto fail_return_to_sender;
1321 }
1322
1323 /* Complete the transfer by mapping the memory into the recipient. */
1324 if (!mm_vm_identity_map(&to->ptable, pa_begin, pa_end, to_mode, NULL,
1325 &local_page_pool)) {
1326 /* TODO: partial defrag of failed range. */
1327 /* Recover any memory consumed in failed mapping. */
1328 mm_vm_defrag(&from->ptable, &local_page_pool);
1329 goto fail_return_to_sender;
1330 }
1331
1332 ret = 0;
1333 goto out;
1334
1335fail_return_to_sender:
1336 mm_vm_identity_map(&from->ptable, pa_begin, pa_end, orig_from_mode,
1337 NULL, &local_page_pool);
1338
1339fail:
1340 ret = -1;
1341
1342out:
1343 sl_unlock(&from->lock);
1344 sl_unlock(&to->lock);
1345
1346 mpool_fini(&local_page_pool);
1347
1348 return ret;
1349}