blob: 4c08fd009768c07ec8ae012e254fdfab6cc51753 [file] [log] [blame]
David Brazdil0f672f62019-12-10 10:32:29 +00001// SPDX-License-Identifier: GPL-2.0-only
Andrew Scullb4b6d4a2019-01-02 15:54:55 +00002/*
3 * Copyright (C) 2015 Linaro Ltd.
4 * Author: Shannon Zhao <shannon.zhao@linaro.org>
Andrew Scullb4b6d4a2019-01-02 15:54:55 +00005 */
6
7#include <linux/cpu.h>
8#include <linux/kvm.h>
9#include <linux/kvm_host.h>
10#include <linux/perf_event.h>
David Brazdil0f672f62019-12-10 10:32:29 +000011#include <linux/perf/arm_pmu.h>
Andrew Scullb4b6d4a2019-01-02 15:54:55 +000012#include <linux/uaccess.h>
13#include <asm/kvm_emulate.h>
14#include <kvm/arm_pmu.h>
15#include <kvm/arm_vgic.h>
16
David Brazdil0f672f62019-12-10 10:32:29 +000017static void kvm_pmu_create_perf_event(struct kvm_vcpu *vcpu, u64 select_idx);
18
19#define PERF_ATTR_CFG1_KVM_PMU_CHAINED 0x1
20
21/**
22 * kvm_pmu_idx_is_64bit - determine if select_idx is a 64bit counter
23 * @vcpu: The vcpu pointer
24 * @select_idx: The counter index
25 */
26static bool kvm_pmu_idx_is_64bit(struct kvm_vcpu *vcpu, u64 select_idx)
27{
28 return (select_idx == ARMV8_PMU_CYCLE_IDX &&
29 __vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_LC);
30}
31
32static struct kvm_vcpu *kvm_pmc_to_vcpu(struct kvm_pmc *pmc)
33{
34 struct kvm_pmu *pmu;
35 struct kvm_vcpu_arch *vcpu_arch;
36
37 pmc -= pmc->idx;
38 pmu = container_of(pmc, struct kvm_pmu, pmc[0]);
39 vcpu_arch = container_of(pmu, struct kvm_vcpu_arch, pmu);
40 return container_of(vcpu_arch, struct kvm_vcpu, arch);
41}
42
43/**
44 * kvm_pmu_pmc_is_chained - determine if the pmc is chained
45 * @pmc: The PMU counter pointer
46 */
47static bool kvm_pmu_pmc_is_chained(struct kvm_pmc *pmc)
48{
49 struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc);
50
51 return test_bit(pmc->idx >> 1, vcpu->arch.pmu.chained);
52}
53
54/**
55 * kvm_pmu_idx_is_high_counter - determine if select_idx is a high/low counter
56 * @select_idx: The counter index
57 */
58static bool kvm_pmu_idx_is_high_counter(u64 select_idx)
59{
60 return select_idx & 0x1;
61}
62
63/**
64 * kvm_pmu_get_canonical_pmc - obtain the canonical pmc
65 * @pmc: The PMU counter pointer
66 *
67 * When a pair of PMCs are chained together we use the low counter (canonical)
68 * to hold the underlying perf event.
69 */
70static struct kvm_pmc *kvm_pmu_get_canonical_pmc(struct kvm_pmc *pmc)
71{
72 if (kvm_pmu_pmc_is_chained(pmc) &&
73 kvm_pmu_idx_is_high_counter(pmc->idx))
74 return pmc - 1;
75
76 return pmc;
77}
78
79/**
80 * kvm_pmu_idx_has_chain_evtype - determine if the event type is chain
81 * @vcpu: The vcpu pointer
82 * @select_idx: The counter index
83 */
84static bool kvm_pmu_idx_has_chain_evtype(struct kvm_vcpu *vcpu, u64 select_idx)
85{
86 u64 eventsel, reg;
87
88 select_idx |= 0x1;
89
90 if (select_idx == ARMV8_PMU_CYCLE_IDX)
91 return false;
92
93 reg = PMEVTYPER0_EL0 + select_idx;
94 eventsel = __vcpu_sys_reg(vcpu, reg) & ARMV8_PMU_EVTYPE_EVENT;
95
96 return eventsel == ARMV8_PMUV3_PERFCTR_CHAIN;
97}
98
99/**
100 * kvm_pmu_get_pair_counter_value - get PMU counter value
101 * @vcpu: The vcpu pointer
102 * @pmc: The PMU counter pointer
103 */
104static u64 kvm_pmu_get_pair_counter_value(struct kvm_vcpu *vcpu,
105 struct kvm_pmc *pmc)
106{
107 u64 counter, counter_high, reg, enabled, running;
108
109 if (kvm_pmu_pmc_is_chained(pmc)) {
110 pmc = kvm_pmu_get_canonical_pmc(pmc);
111 reg = PMEVCNTR0_EL0 + pmc->idx;
112
113 counter = __vcpu_sys_reg(vcpu, reg);
114 counter_high = __vcpu_sys_reg(vcpu, reg + 1);
115
116 counter = lower_32_bits(counter) | (counter_high << 32);
117 } else {
118 reg = (pmc->idx == ARMV8_PMU_CYCLE_IDX)
119 ? PMCCNTR_EL0 : PMEVCNTR0_EL0 + pmc->idx;
120 counter = __vcpu_sys_reg(vcpu, reg);
121 }
122
123 /*
124 * The real counter value is equal to the value of counter register plus
125 * the value perf event counts.
126 */
127 if (pmc->perf_event)
128 counter += perf_event_read_value(pmc->perf_event, &enabled,
129 &running);
130
131 return counter;
132}
133
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000134/**
135 * kvm_pmu_get_counter_value - get PMU counter value
136 * @vcpu: The vcpu pointer
137 * @select_idx: The counter index
138 */
139u64 kvm_pmu_get_counter_value(struct kvm_vcpu *vcpu, u64 select_idx)
140{
David Brazdil0f672f62019-12-10 10:32:29 +0000141 u64 counter;
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000142 struct kvm_pmu *pmu = &vcpu->arch.pmu;
143 struct kvm_pmc *pmc = &pmu->pmc[select_idx];
144
David Brazdil0f672f62019-12-10 10:32:29 +0000145 counter = kvm_pmu_get_pair_counter_value(vcpu, pmc);
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000146
David Brazdil0f672f62019-12-10 10:32:29 +0000147 if (kvm_pmu_pmc_is_chained(pmc) &&
148 kvm_pmu_idx_is_high_counter(select_idx))
149 counter = upper_32_bits(counter);
150 else if (select_idx != ARMV8_PMU_CYCLE_IDX)
151 counter = lower_32_bits(counter);
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000152
David Brazdil0f672f62019-12-10 10:32:29 +0000153 return counter;
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000154}
155
156/**
157 * kvm_pmu_set_counter_value - set PMU counter value
158 * @vcpu: The vcpu pointer
159 * @select_idx: The counter index
160 * @val: The counter value
161 */
162void kvm_pmu_set_counter_value(struct kvm_vcpu *vcpu, u64 select_idx, u64 val)
163{
164 u64 reg;
165
166 reg = (select_idx == ARMV8_PMU_CYCLE_IDX)
167 ? PMCCNTR_EL0 : PMEVCNTR0_EL0 + select_idx;
168 __vcpu_sys_reg(vcpu, reg) += (s64)val - kvm_pmu_get_counter_value(vcpu, select_idx);
David Brazdil0f672f62019-12-10 10:32:29 +0000169
170 /* Recreate the perf event to reflect the updated sample_period */
171 kvm_pmu_create_perf_event(vcpu, select_idx);
172}
173
174/**
175 * kvm_pmu_release_perf_event - remove the perf event
176 * @pmc: The PMU counter pointer
177 */
178static void kvm_pmu_release_perf_event(struct kvm_pmc *pmc)
179{
180 pmc = kvm_pmu_get_canonical_pmc(pmc);
181 if (pmc->perf_event) {
182 perf_event_disable(pmc->perf_event);
183 perf_event_release_kernel(pmc->perf_event);
184 pmc->perf_event = NULL;
185 }
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000186}
187
188/**
189 * kvm_pmu_stop_counter - stop PMU counter
190 * @pmc: The PMU counter pointer
191 *
192 * If this counter has been configured to monitor some event, release it here.
193 */
194static void kvm_pmu_stop_counter(struct kvm_vcpu *vcpu, struct kvm_pmc *pmc)
195{
David Brazdil0f672f62019-12-10 10:32:29 +0000196 u64 counter, reg, val;
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000197
David Brazdil0f672f62019-12-10 10:32:29 +0000198 pmc = kvm_pmu_get_canonical_pmc(pmc);
199 if (!pmc->perf_event)
200 return;
201
202 counter = kvm_pmu_get_pair_counter_value(vcpu, pmc);
203
204 if (pmc->idx == ARMV8_PMU_CYCLE_IDX) {
205 reg = PMCCNTR_EL0;
206 val = counter;
207 } else {
208 reg = PMEVCNTR0_EL0 + pmc->idx;
209 val = lower_32_bits(counter);
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000210 }
David Brazdil0f672f62019-12-10 10:32:29 +0000211
212 __vcpu_sys_reg(vcpu, reg) = val;
213
214 if (kvm_pmu_pmc_is_chained(pmc))
215 __vcpu_sys_reg(vcpu, reg + 1) = upper_32_bits(counter);
216
217 kvm_pmu_release_perf_event(pmc);
218}
219
220/**
221 * kvm_pmu_vcpu_init - assign pmu counter idx for cpu
222 * @vcpu: The vcpu pointer
223 *
224 */
225void kvm_pmu_vcpu_init(struct kvm_vcpu *vcpu)
226{
227 int i;
228 struct kvm_pmu *pmu = &vcpu->arch.pmu;
229
230 for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++)
231 pmu->pmc[i].idx = i;
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000232}
233
234/**
235 * kvm_pmu_vcpu_reset - reset pmu state for cpu
236 * @vcpu: The vcpu pointer
237 *
238 */
239void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu)
240{
241 int i;
242 struct kvm_pmu *pmu = &vcpu->arch.pmu;
243
David Brazdil0f672f62019-12-10 10:32:29 +0000244 for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++)
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000245 kvm_pmu_stop_counter(vcpu, &pmu->pmc[i]);
David Brazdil0f672f62019-12-10 10:32:29 +0000246
247 bitmap_zero(vcpu->arch.pmu.chained, ARMV8_PMU_MAX_COUNTER_PAIRS);
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000248}
249
250/**
251 * kvm_pmu_vcpu_destroy - free perf event of PMU for cpu
252 * @vcpu: The vcpu pointer
253 *
254 */
255void kvm_pmu_vcpu_destroy(struct kvm_vcpu *vcpu)
256{
257 int i;
258 struct kvm_pmu *pmu = &vcpu->arch.pmu;
259
David Brazdil0f672f62019-12-10 10:32:29 +0000260 for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++)
261 kvm_pmu_release_perf_event(&pmu->pmc[i]);
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000262}
263
264u64 kvm_pmu_valid_counter_mask(struct kvm_vcpu *vcpu)
265{
266 u64 val = __vcpu_sys_reg(vcpu, PMCR_EL0) >> ARMV8_PMU_PMCR_N_SHIFT;
267
268 val &= ARMV8_PMU_PMCR_N_MASK;
269 if (val == 0)
270 return BIT(ARMV8_PMU_CYCLE_IDX);
271 else
272 return GENMASK(val - 1, 0) | BIT(ARMV8_PMU_CYCLE_IDX);
273}
274
275/**
David Brazdil0f672f62019-12-10 10:32:29 +0000276 * kvm_pmu_enable_counter_mask - enable selected PMU counters
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000277 * @vcpu: The vcpu pointer
278 * @val: the value guest writes to PMCNTENSET register
279 *
280 * Call perf_event_enable to start counting the perf event
281 */
David Brazdil0f672f62019-12-10 10:32:29 +0000282void kvm_pmu_enable_counter_mask(struct kvm_vcpu *vcpu, u64 val)
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000283{
284 int i;
285 struct kvm_pmu *pmu = &vcpu->arch.pmu;
286 struct kvm_pmc *pmc;
287
288 if (!(__vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E) || !val)
289 return;
290
291 for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++) {
292 if (!(val & BIT(i)))
293 continue;
294
295 pmc = &pmu->pmc[i];
David Brazdil0f672f62019-12-10 10:32:29 +0000296
297 /*
298 * For high counters of chained events we must recreate the
299 * perf event with the long (64bit) attribute set.
300 */
301 if (kvm_pmu_pmc_is_chained(pmc) &&
302 kvm_pmu_idx_is_high_counter(i)) {
303 kvm_pmu_create_perf_event(vcpu, i);
304 continue;
305 }
306
307 /* At this point, pmc must be the canonical */
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000308 if (pmc->perf_event) {
309 perf_event_enable(pmc->perf_event);
310 if (pmc->perf_event->state != PERF_EVENT_STATE_ACTIVE)
311 kvm_debug("fail to enable perf event\n");
312 }
313 }
314}
315
316/**
David Brazdil0f672f62019-12-10 10:32:29 +0000317 * kvm_pmu_disable_counter_mask - disable selected PMU counters
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000318 * @vcpu: The vcpu pointer
319 * @val: the value guest writes to PMCNTENCLR register
320 *
321 * Call perf_event_disable to stop counting the perf event
322 */
David Brazdil0f672f62019-12-10 10:32:29 +0000323void kvm_pmu_disable_counter_mask(struct kvm_vcpu *vcpu, u64 val)
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000324{
325 int i;
326 struct kvm_pmu *pmu = &vcpu->arch.pmu;
327 struct kvm_pmc *pmc;
328
329 if (!val)
330 return;
331
332 for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++) {
333 if (!(val & BIT(i)))
334 continue;
335
336 pmc = &pmu->pmc[i];
David Brazdil0f672f62019-12-10 10:32:29 +0000337
338 /*
339 * For high counters of chained events we must recreate the
340 * perf event with the long (64bit) attribute unset.
341 */
342 if (kvm_pmu_pmc_is_chained(pmc) &&
343 kvm_pmu_idx_is_high_counter(i)) {
344 kvm_pmu_create_perf_event(vcpu, i);
345 continue;
346 }
347
348 /* At this point, pmc must be the canonical */
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000349 if (pmc->perf_event)
350 perf_event_disable(pmc->perf_event);
351 }
352}
353
354static u64 kvm_pmu_overflow_status(struct kvm_vcpu *vcpu)
355{
356 u64 reg = 0;
357
358 if ((__vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E)) {
359 reg = __vcpu_sys_reg(vcpu, PMOVSSET_EL0);
360 reg &= __vcpu_sys_reg(vcpu, PMCNTENSET_EL0);
361 reg &= __vcpu_sys_reg(vcpu, PMINTENSET_EL1);
362 reg &= kvm_pmu_valid_counter_mask(vcpu);
363 }
364
365 return reg;
366}
367
368static void kvm_pmu_update_state(struct kvm_vcpu *vcpu)
369{
370 struct kvm_pmu *pmu = &vcpu->arch.pmu;
371 bool overflow;
372
373 if (!kvm_arm_pmu_v3_ready(vcpu))
374 return;
375
376 overflow = !!kvm_pmu_overflow_status(vcpu);
377 if (pmu->irq_level == overflow)
378 return;
379
380 pmu->irq_level = overflow;
381
382 if (likely(irqchip_in_kernel(vcpu->kvm))) {
383 int ret = kvm_vgic_inject_irq(vcpu->kvm, vcpu->vcpu_id,
384 pmu->irq_num, overflow, pmu);
385 WARN_ON(ret);
386 }
387}
388
389bool kvm_pmu_should_notify_user(struct kvm_vcpu *vcpu)
390{
391 struct kvm_pmu *pmu = &vcpu->arch.pmu;
392 struct kvm_sync_regs *sregs = &vcpu->run->s.regs;
393 bool run_level = sregs->device_irq_level & KVM_ARM_DEV_PMU;
394
395 if (likely(irqchip_in_kernel(vcpu->kvm)))
396 return false;
397
398 return pmu->irq_level != run_level;
399}
400
401/*
402 * Reflect the PMU overflow interrupt output level into the kvm_run structure
403 */
404void kvm_pmu_update_run(struct kvm_vcpu *vcpu)
405{
406 struct kvm_sync_regs *regs = &vcpu->run->s.regs;
407
408 /* Populate the timer bitmap for user space */
409 regs->device_irq_level &= ~KVM_ARM_DEV_PMU;
410 if (vcpu->arch.pmu.irq_level)
411 regs->device_irq_level |= KVM_ARM_DEV_PMU;
412}
413
414/**
415 * kvm_pmu_flush_hwstate - flush pmu state to cpu
416 * @vcpu: The vcpu pointer
417 *
418 * Check if the PMU has overflowed while we were running in the host, and inject
419 * an interrupt if that was the case.
420 */
421void kvm_pmu_flush_hwstate(struct kvm_vcpu *vcpu)
422{
423 kvm_pmu_update_state(vcpu);
424}
425
426/**
427 * kvm_pmu_sync_hwstate - sync pmu state from cpu
428 * @vcpu: The vcpu pointer
429 *
430 * Check if the PMU has overflowed while we were running in the guest, and
431 * inject an interrupt if that was the case.
432 */
433void kvm_pmu_sync_hwstate(struct kvm_vcpu *vcpu)
434{
435 kvm_pmu_update_state(vcpu);
436}
437
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000438/**
439 * When the perf event overflows, set the overflow status and inform the vcpu.
440 */
441static void kvm_pmu_perf_overflow(struct perf_event *perf_event,
442 struct perf_sample_data *data,
443 struct pt_regs *regs)
444{
445 struct kvm_pmc *pmc = perf_event->overflow_handler_context;
David Brazdil0f672f62019-12-10 10:32:29 +0000446 struct arm_pmu *cpu_pmu = to_arm_pmu(perf_event->pmu);
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000447 struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc);
448 int idx = pmc->idx;
David Brazdil0f672f62019-12-10 10:32:29 +0000449 u64 period;
450
451 cpu_pmu->pmu.stop(perf_event, PERF_EF_UPDATE);
452
453 /*
454 * Reset the sample period to the architectural limit,
455 * i.e. the point where the counter overflows.
456 */
457 period = -(local64_read(&perf_event->count));
458
459 if (!kvm_pmu_idx_is_64bit(vcpu, pmc->idx))
460 period &= GENMASK(31, 0);
461
462 local64_set(&perf_event->hw.period_left, 0);
463 perf_event->attr.sample_period = period;
464 perf_event->hw.sample_period = period;
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000465
466 __vcpu_sys_reg(vcpu, PMOVSSET_EL0) |= BIT(idx);
467
468 if (kvm_pmu_overflow_status(vcpu)) {
469 kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu);
470 kvm_vcpu_kick(vcpu);
471 }
David Brazdil0f672f62019-12-10 10:32:29 +0000472
473 cpu_pmu->pmu.start(perf_event, PERF_EF_RELOAD);
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000474}
475
476/**
477 * kvm_pmu_software_increment - do software increment
478 * @vcpu: The vcpu pointer
479 * @val: the value guest writes to PMSWINC register
480 */
481void kvm_pmu_software_increment(struct kvm_vcpu *vcpu, u64 val)
482{
Olivier Deprez0e641232021-09-23 10:07:05 +0200483 struct kvm_pmu *pmu = &vcpu->arch.pmu;
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000484 int i;
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000485
Olivier Deprez0e641232021-09-23 10:07:05 +0200486 if (!(__vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E))
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000487 return;
488
Olivier Deprez0e641232021-09-23 10:07:05 +0200489 /* Weed out disabled counters */
490 val &= __vcpu_sys_reg(vcpu, PMCNTENSET_EL0);
491
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000492 for (i = 0; i < ARMV8_PMU_CYCLE_IDX; i++) {
Olivier Deprez0e641232021-09-23 10:07:05 +0200493 u64 type, reg;
494
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000495 if (!(val & BIT(i)))
496 continue;
Olivier Deprez0e641232021-09-23 10:07:05 +0200497
498 /* PMSWINC only applies to ... SW_INC! */
499 type = __vcpu_sys_reg(vcpu, PMEVTYPER0_EL0 + i);
500 type &= ARMV8_PMU_EVTYPE_EVENT;
501 if (type != ARMV8_PMUV3_PERFCTR_SW_INCR)
502 continue;
503
504 /* increment this even SW_INC counter */
505 reg = __vcpu_sys_reg(vcpu, PMEVCNTR0_EL0 + i) + 1;
506 reg = lower_32_bits(reg);
507 __vcpu_sys_reg(vcpu, PMEVCNTR0_EL0 + i) = reg;
508
509 if (reg) /* no overflow on the low part */
510 continue;
511
512 if (kvm_pmu_pmc_is_chained(&pmu->pmc[i])) {
513 /* increment the high counter */
514 reg = __vcpu_sys_reg(vcpu, PMEVCNTR0_EL0 + i + 1) + 1;
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000515 reg = lower_32_bits(reg);
Olivier Deprez0e641232021-09-23 10:07:05 +0200516 __vcpu_sys_reg(vcpu, PMEVCNTR0_EL0 + i + 1) = reg;
517 if (!reg) /* mark overflow on the high counter */
518 __vcpu_sys_reg(vcpu, PMOVSSET_EL0) |= BIT(i + 1);
519 } else {
520 /* mark overflow on low counter */
521 __vcpu_sys_reg(vcpu, PMOVSSET_EL0) |= BIT(i);
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000522 }
523 }
524}
525
526/**
527 * kvm_pmu_handle_pmcr - handle PMCR register
528 * @vcpu: The vcpu pointer
529 * @val: the value guest writes to PMCR register
530 */
531void kvm_pmu_handle_pmcr(struct kvm_vcpu *vcpu, u64 val)
532{
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000533 u64 mask;
534 int i;
535
536 mask = kvm_pmu_valid_counter_mask(vcpu);
537 if (val & ARMV8_PMU_PMCR_E) {
David Brazdil0f672f62019-12-10 10:32:29 +0000538 kvm_pmu_enable_counter_mask(vcpu,
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000539 __vcpu_sys_reg(vcpu, PMCNTENSET_EL0) & mask);
540 } else {
David Brazdil0f672f62019-12-10 10:32:29 +0000541 kvm_pmu_disable_counter_mask(vcpu, mask);
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000542 }
543
544 if (val & ARMV8_PMU_PMCR_C)
545 kvm_pmu_set_counter_value(vcpu, ARMV8_PMU_CYCLE_IDX, 0);
546
547 if (val & ARMV8_PMU_PMCR_P) {
548 for (i = 0; i < ARMV8_PMU_CYCLE_IDX; i++)
549 kvm_pmu_set_counter_value(vcpu, i, 0);
550 }
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000551}
552
553static bool kvm_pmu_counter_is_enabled(struct kvm_vcpu *vcpu, u64 select_idx)
554{
555 return (__vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E) &&
556 (__vcpu_sys_reg(vcpu, PMCNTENSET_EL0) & BIT(select_idx));
557}
558
559/**
David Brazdil0f672f62019-12-10 10:32:29 +0000560 * kvm_pmu_create_perf_event - create a perf event for a counter
561 * @vcpu: The vcpu pointer
562 * @select_idx: The number of selected counter
563 */
564static void kvm_pmu_create_perf_event(struct kvm_vcpu *vcpu, u64 select_idx)
565{
566 struct kvm_pmu *pmu = &vcpu->arch.pmu;
567 struct kvm_pmc *pmc;
568 struct perf_event *event;
569 struct perf_event_attr attr;
570 u64 eventsel, counter, reg, data;
571
572 /*
573 * For chained counters the event type and filtering attributes are
574 * obtained from the low/even counter. We also use this counter to
575 * determine if the event is enabled/disabled.
576 */
577 pmc = kvm_pmu_get_canonical_pmc(&pmu->pmc[select_idx]);
578
579 reg = (pmc->idx == ARMV8_PMU_CYCLE_IDX)
580 ? PMCCFILTR_EL0 : PMEVTYPER0_EL0 + pmc->idx;
581 data = __vcpu_sys_reg(vcpu, reg);
582
583 kvm_pmu_stop_counter(vcpu, pmc);
584 eventsel = data & ARMV8_PMU_EVTYPE_EVENT;
585
586 /* Software increment event does't need to be backed by a perf event */
587 if (eventsel == ARMV8_PMUV3_PERFCTR_SW_INCR &&
588 pmc->idx != ARMV8_PMU_CYCLE_IDX)
589 return;
590
591 memset(&attr, 0, sizeof(struct perf_event_attr));
592 attr.type = PERF_TYPE_RAW;
593 attr.size = sizeof(attr);
594 attr.pinned = 1;
595 attr.disabled = !kvm_pmu_counter_is_enabled(vcpu, pmc->idx);
596 attr.exclude_user = data & ARMV8_PMU_EXCLUDE_EL0 ? 1 : 0;
597 attr.exclude_kernel = data & ARMV8_PMU_EXCLUDE_EL1 ? 1 : 0;
598 attr.exclude_hv = 1; /* Don't count EL2 events */
599 attr.exclude_host = 1; /* Don't count host events */
600 attr.config = (pmc->idx == ARMV8_PMU_CYCLE_IDX) ?
601 ARMV8_PMUV3_PERFCTR_CPU_CYCLES : eventsel;
602
603 counter = kvm_pmu_get_pair_counter_value(vcpu, pmc);
604
605 if (kvm_pmu_idx_has_chain_evtype(vcpu, pmc->idx)) {
606 /**
607 * The initial sample period (overflow count) of an event. For
608 * chained counters we only support overflow interrupts on the
609 * high counter.
610 */
611 attr.sample_period = (-counter) & GENMASK(63, 0);
612 if (kvm_pmu_counter_is_enabled(vcpu, pmc->idx + 1))
613 attr.config1 |= PERF_ATTR_CFG1_KVM_PMU_CHAINED;
614
615 event = perf_event_create_kernel_counter(&attr, -1, current,
616 kvm_pmu_perf_overflow,
617 pmc + 1);
618 } else {
619 /* The initial sample period (overflow count) of an event. */
620 if (kvm_pmu_idx_is_64bit(vcpu, pmc->idx))
621 attr.sample_period = (-counter) & GENMASK(63, 0);
622 else
623 attr.sample_period = (-counter) & GENMASK(31, 0);
624
625 event = perf_event_create_kernel_counter(&attr, -1, current,
626 kvm_pmu_perf_overflow, pmc);
627 }
628
629 if (IS_ERR(event)) {
630 pr_err_once("kvm: pmu event creation failed %ld\n",
631 PTR_ERR(event));
632 return;
633 }
634
635 pmc->perf_event = event;
636}
637
638/**
639 * kvm_pmu_update_pmc_chained - update chained bitmap
640 * @vcpu: The vcpu pointer
641 * @select_idx: The number of selected counter
642 *
643 * Update the chained bitmap based on the event type written in the
644 * typer register.
645 */
646static void kvm_pmu_update_pmc_chained(struct kvm_vcpu *vcpu, u64 select_idx)
647{
648 struct kvm_pmu *pmu = &vcpu->arch.pmu;
649 struct kvm_pmc *pmc = &pmu->pmc[select_idx];
650
651 if (kvm_pmu_idx_has_chain_evtype(vcpu, pmc->idx)) {
652 /*
653 * During promotion from !chained to chained we must ensure
654 * the adjacent counter is stopped and its event destroyed
655 */
656 if (!kvm_pmu_pmc_is_chained(pmc))
657 kvm_pmu_stop_counter(vcpu, pmc);
658
659 set_bit(pmc->idx >> 1, vcpu->arch.pmu.chained);
660 } else {
661 clear_bit(pmc->idx >> 1, vcpu->arch.pmu.chained);
662 }
663}
664
665/**
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000666 * kvm_pmu_set_counter_event_type - set selected counter to monitor some event
667 * @vcpu: The vcpu pointer
668 * @data: The data guest writes to PMXEVTYPER_EL0
669 * @select_idx: The number of selected counter
670 *
671 * When OS accesses PMXEVTYPER_EL0, that means it wants to set a PMC to count an
672 * event with given hardware event number. Here we call perf_event API to
673 * emulate this action and create a kernel perf event for it.
674 */
675void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, u64 data,
676 u64 select_idx)
677{
David Brazdil0f672f62019-12-10 10:32:29 +0000678 u64 reg, event_type = data & ARMV8_PMU_EVTYPE_MASK;
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000679
David Brazdil0f672f62019-12-10 10:32:29 +0000680 reg = (select_idx == ARMV8_PMU_CYCLE_IDX)
681 ? PMCCFILTR_EL0 : PMEVTYPER0_EL0 + select_idx;
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000682
David Brazdil0f672f62019-12-10 10:32:29 +0000683 __vcpu_sys_reg(vcpu, reg) = event_type;
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000684
David Brazdil0f672f62019-12-10 10:32:29 +0000685 kvm_pmu_update_pmc_chained(vcpu, select_idx);
686 kvm_pmu_create_perf_event(vcpu, select_idx);
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000687}
688
689bool kvm_arm_support_pmu_v3(void)
690{
691 /*
692 * Check if HW_PERF_EVENTS are supported by checking the number of
693 * hardware performance counters. This could ensure the presence of
694 * a physical PMU and CONFIG_PERF_EVENT is selected.
695 */
696 return (perf_num_counters() > 0);
697}
698
699int kvm_arm_pmu_v3_enable(struct kvm_vcpu *vcpu)
700{
701 if (!vcpu->arch.pmu.created)
702 return 0;
703
704 /*
705 * A valid interrupt configuration for the PMU is either to have a
706 * properly configured interrupt number and using an in-kernel
707 * irqchip, or to not have an in-kernel GIC and not set an IRQ.
708 */
709 if (irqchip_in_kernel(vcpu->kvm)) {
710 int irq = vcpu->arch.pmu.irq_num;
711 if (!kvm_arm_pmu_irq_initialized(vcpu))
712 return -EINVAL;
713
714 /*
715 * If we are using an in-kernel vgic, at this point we know
716 * the vgic will be initialized, so we can check the PMU irq
717 * number against the dimensions of the vgic and make sure
718 * it's valid.
719 */
720 if (!irq_is_ppi(irq) && !vgic_valid_spi(vcpu->kvm, irq))
721 return -EINVAL;
722 } else if (kvm_arm_pmu_irq_initialized(vcpu)) {
723 return -EINVAL;
724 }
725
726 kvm_pmu_vcpu_reset(vcpu);
727 vcpu->arch.pmu.ready = true;
728
729 return 0;
730}
731
732static int kvm_arm_pmu_v3_init(struct kvm_vcpu *vcpu)
733{
734 if (!kvm_arm_support_pmu_v3())
735 return -ENODEV;
736
737 if (!test_bit(KVM_ARM_VCPU_PMU_V3, vcpu->arch.features))
738 return -ENXIO;
739
740 if (vcpu->arch.pmu.created)
741 return -EBUSY;
742
743 if (irqchip_in_kernel(vcpu->kvm)) {
744 int ret;
745
746 /*
747 * If using the PMU with an in-kernel virtual GIC
748 * implementation, we require the GIC to be already
749 * initialized when initializing the PMU.
750 */
751 if (!vgic_initialized(vcpu->kvm))
752 return -ENODEV;
753
754 if (!kvm_arm_pmu_irq_initialized(vcpu))
755 return -ENXIO;
756
757 ret = kvm_vgic_set_owner(vcpu, vcpu->arch.pmu.irq_num,
758 &vcpu->arch.pmu);
759 if (ret)
760 return ret;
761 }
762
763 vcpu->arch.pmu.created = true;
764 return 0;
765}
766
767/*
768 * For one VM the interrupt type must be same for each vcpu.
769 * As a PPI, the interrupt number is the same for all vcpus,
770 * while as an SPI it must be a separate number per vcpu.
771 */
772static bool pmu_irq_is_valid(struct kvm *kvm, int irq)
773{
774 int i;
775 struct kvm_vcpu *vcpu;
776
777 kvm_for_each_vcpu(i, vcpu, kvm) {
778 if (!kvm_arm_pmu_irq_initialized(vcpu))
779 continue;
780
781 if (irq_is_ppi(irq)) {
782 if (vcpu->arch.pmu.irq_num != irq)
783 return false;
784 } else {
785 if (vcpu->arch.pmu.irq_num == irq)
786 return false;
787 }
788 }
789
790 return true;
791}
792
793int kvm_arm_pmu_v3_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
794{
795 switch (attr->attr) {
796 case KVM_ARM_VCPU_PMU_V3_IRQ: {
797 int __user *uaddr = (int __user *)(long)attr->addr;
798 int irq;
799
800 if (!irqchip_in_kernel(vcpu->kvm))
801 return -EINVAL;
802
803 if (!test_bit(KVM_ARM_VCPU_PMU_V3, vcpu->arch.features))
804 return -ENODEV;
805
806 if (get_user(irq, uaddr))
807 return -EFAULT;
808
809 /* The PMU overflow interrupt can be a PPI or a valid SPI. */
810 if (!(irq_is_ppi(irq) || irq_is_spi(irq)))
811 return -EINVAL;
812
813 if (!pmu_irq_is_valid(vcpu->kvm, irq))
814 return -EINVAL;
815
816 if (kvm_arm_pmu_irq_initialized(vcpu))
817 return -EBUSY;
818
819 kvm_debug("Set kvm ARM PMU irq: %d\n", irq);
820 vcpu->arch.pmu.irq_num = irq;
821 return 0;
822 }
823 case KVM_ARM_VCPU_PMU_V3_INIT:
824 return kvm_arm_pmu_v3_init(vcpu);
825 }
826
827 return -ENXIO;
828}
829
830int kvm_arm_pmu_v3_get_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
831{
832 switch (attr->attr) {
833 case KVM_ARM_VCPU_PMU_V3_IRQ: {
834 int __user *uaddr = (int __user *)(long)attr->addr;
835 int irq;
836
837 if (!irqchip_in_kernel(vcpu->kvm))
838 return -EINVAL;
839
840 if (!test_bit(KVM_ARM_VCPU_PMU_V3, vcpu->arch.features))
841 return -ENODEV;
842
843 if (!kvm_arm_pmu_irq_initialized(vcpu))
844 return -ENXIO;
845
846 irq = vcpu->arch.pmu.irq_num;
847 return put_user(irq, uaddr);
848 }
849 }
850
851 return -ENXIO;
852}
853
854int kvm_arm_pmu_v3_has_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
855{
856 switch (attr->attr) {
857 case KVM_ARM_VCPU_PMU_V3_IRQ:
858 case KVM_ARM_VCPU_PMU_V3_INIT:
859 if (kvm_arm_support_pmu_v3() &&
860 test_bit(KVM_ARM_VCPU_PMU_V3, vcpu->arch.features))
861 return 0;
862 }
863
864 return -ENXIO;
865}