blob: 011e681a23366bd3519fd9324f5047547be97b3b [file] [log] [blame]
David Brazdil0f672f62019-12-10 10:32:29 +00001/* SPDX-License-Identifier: GPL-2.0-only */
Andrew Scullb4b6d4a2019-01-02 15:54:55 +00002/*
3 * Based on arch/arm/include/asm/assembler.h, arch/arm/mm/proc-macros.S
4 *
5 * Copyright (C) 1996-2000 Russell King
6 * Copyright (C) 2012 ARM Ltd.
Andrew Scullb4b6d4a2019-01-02 15:54:55 +00007 */
8#ifndef __ASSEMBLY__
9#error "Only include this from assembly code"
10#endif
11
12#ifndef __ASM_ASSEMBLER_H
13#define __ASM_ASSEMBLER_H
14
David Brazdil0f672f62019-12-10 10:32:29 +000015#include <asm-generic/export.h>
16
Andrew Scullb4b6d4a2019-01-02 15:54:55 +000017#include <asm/asm-offsets.h>
18#include <asm/cpufeature.h>
David Brazdil0f672f62019-12-10 10:32:29 +000019#include <asm/cputype.h>
Andrew Scullb4b6d4a2019-01-02 15:54:55 +000020#include <asm/debug-monitors.h>
21#include <asm/page.h>
22#include <asm/pgtable-hwdef.h>
23#include <asm/ptrace.h>
24#include <asm/thread_info.h>
25
26 .macro save_and_disable_daif, flags
27 mrs \flags, daif
28 msr daifset, #0xf
29 .endm
30
31 .macro disable_daif
32 msr daifset, #0xf
33 .endm
34
35 .macro enable_daif
36 msr daifclr, #0xf
37 .endm
38
39 .macro restore_daif, flags:req
40 msr daif, \flags
41 .endm
42
Andrew Scullb4b6d4a2019-01-02 15:54:55 +000043 /* IRQ is the lowest priority flag, unconditionally unmask the rest. */
44 .macro enable_da_f
45 msr daifclr, #(8 | 4 | 1)
46 .endm
47
48/*
David Brazdil0f672f62019-12-10 10:32:29 +000049 * Save/restore interrupts.
Andrew Scullb4b6d4a2019-01-02 15:54:55 +000050 */
Andrew Scullb4b6d4a2019-01-02 15:54:55 +000051 .macro save_and_disable_irq, flags
52 mrs \flags, daif
53 msr daifset, #2
54 .endm
55
56 .macro restore_irq, flags
57 msr daif, \flags
58 .endm
59
60 .macro enable_dbg
61 msr daifclr, #8
62 .endm
63
64 .macro disable_step_tsk, flgs, tmp
65 tbz \flgs, #TIF_SINGLESTEP, 9990f
66 mrs \tmp, mdscr_el1
67 bic \tmp, \tmp, #DBG_MDSCR_SS
68 msr mdscr_el1, \tmp
69 isb // Synchronise with enable_dbg
709990:
71 .endm
72
73 /* call with daif masked */
74 .macro enable_step_tsk, flgs, tmp
75 tbz \flgs, #TIF_SINGLESTEP, 9990f
76 mrs \tmp, mdscr_el1
77 orr \tmp, \tmp, #DBG_MDSCR_SS
78 msr mdscr_el1, \tmp
799990:
80 .endm
81
82/*
Andrew Scullb4b6d4a2019-01-02 15:54:55 +000083 * RAS Error Synchronization barrier
84 */
85 .macro esb
David Brazdil0f672f62019-12-10 10:32:29 +000086#ifdef CONFIG_ARM64_RAS_EXTN
Andrew Scullb4b6d4a2019-01-02 15:54:55 +000087 hint #16
David Brazdil0f672f62019-12-10 10:32:29 +000088#else
89 nop
90#endif
Andrew Scullb4b6d4a2019-01-02 15:54:55 +000091 .endm
92
93/*
94 * Value prediction barrier
95 */
96 .macro csdb
97 hint #20
98 .endm
99
100/*
Olivier Deprez157378f2022-04-04 15:47:50 +0200101 * Clear Branch History instruction
102 */
103 .macro clearbhb
104 hint #22
105 .endm
106
107/*
David Brazdil0f672f62019-12-10 10:32:29 +0000108 * Speculation barrier
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000109 */
David Brazdil0f672f62019-12-10 10:32:29 +0000110 .macro sb
111alternative_if_not ARM64_HAS_SB
112 dsb nsh
113 isb
114alternative_else
115 SB_BARRIER_INSN
116 nop
117alternative_endif
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000118 .endm
119
120/*
121 * NOP sequence
122 */
123 .macro nops, num
124 .rept \num
125 nop
126 .endr
127 .endm
128
129/*
130 * Emit an entry into the exception table
131 */
132 .macro _asm_extable, from, to
133 .pushsection __ex_table, "a"
134 .align 3
135 .long (\from - .), (\to - .)
136 .popsection
137 .endm
138
139#define USER(l, x...) \
1409999: x; \
141 _asm_extable 9999b, l
142
143/*
144 * Register aliases.
145 */
146lr .req x30 // link register
147
148/*
149 * Vector entry
150 */
151 .macro ventry label
152 .align 7
153 b \label
154 .endm
155
156/*
157 * Select code when configured for BE.
158 */
159#ifdef CONFIG_CPU_BIG_ENDIAN
160#define CPU_BE(code...) code
161#else
162#define CPU_BE(code...)
163#endif
164
165/*
166 * Select code when configured for LE.
167 */
168#ifdef CONFIG_CPU_BIG_ENDIAN
169#define CPU_LE(code...)
170#else
171#define CPU_LE(code...) code
172#endif
173
174/*
175 * Define a macro that constructs a 64-bit value by concatenating two
176 * 32-bit registers. Note that on big endian systems the order of the
177 * registers is swapped.
178 */
179#ifndef CONFIG_CPU_BIG_ENDIAN
180 .macro regs_to_64, rd, lbits, hbits
181#else
182 .macro regs_to_64, rd, hbits, lbits
183#endif
184 orr \rd, \lbits, \hbits, lsl #32
185 .endm
186
187/*
188 * Pseudo-ops for PC-relative adr/ldr/str <reg>, <symbol> where
189 * <symbol> is within the range +/- 4 GB of the PC.
190 */
191 /*
192 * @dst: destination register (64 bit wide)
193 * @sym: name of the symbol
194 */
195 .macro adr_l, dst, sym
196 adrp \dst, \sym
197 add \dst, \dst, :lo12:\sym
198 .endm
199
200 /*
201 * @dst: destination register (32 or 64 bit wide)
202 * @sym: name of the symbol
203 * @tmp: optional 64-bit scratch register to be used if <dst> is a
204 * 32-bit wide register, in which case it cannot be used to hold
205 * the address
206 */
207 .macro ldr_l, dst, sym, tmp=
208 .ifb \tmp
209 adrp \dst, \sym
210 ldr \dst, [\dst, :lo12:\sym]
211 .else
212 adrp \tmp, \sym
213 ldr \dst, [\tmp, :lo12:\sym]
214 .endif
215 .endm
216
217 /*
218 * @src: source register (32 or 64 bit wide)
219 * @sym: name of the symbol
220 * @tmp: mandatory 64-bit scratch register to calculate the address
221 * while <src> needs to be preserved.
222 */
223 .macro str_l, src, sym, tmp
224 adrp \tmp, \sym
225 str \src, [\tmp, :lo12:\sym]
226 .endm
227
228 /*
Olivier Deprez157378f2022-04-04 15:47:50 +0200229 * @dst: destination register
230 */
231#if defined(__KVM_NVHE_HYPERVISOR__) || defined(__KVM_VHE_HYPERVISOR__)
232 .macro this_cpu_offset, dst
233 mrs \dst, tpidr_el2
234 .endm
235#else
236 .macro this_cpu_offset, dst
237alternative_if_not ARM64_HAS_VIRT_HOST_EXTN
238 mrs \dst, tpidr_el1
239alternative_else
240 mrs \dst, tpidr_el2
241alternative_endif
242 .endm
243#endif
244
245 /*
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000246 * @dst: Result of per_cpu(sym, smp_processor_id()) (can be SP)
247 * @sym: The name of the per-cpu variable
248 * @tmp: scratch register
249 */
250 .macro adr_this_cpu, dst, sym, tmp
251 adrp \tmp, \sym
252 add \dst, \tmp, #:lo12:\sym
Olivier Deprez157378f2022-04-04 15:47:50 +0200253 this_cpu_offset \tmp
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000254 add \dst, \dst, \tmp
255 .endm
256
257 /*
258 * @dst: Result of READ_ONCE(per_cpu(sym, smp_processor_id()))
259 * @sym: The name of the per-cpu variable
260 * @tmp: scratch register
261 */
262 .macro ldr_this_cpu dst, sym, tmp
263 adr_l \dst, \sym
Olivier Deprez157378f2022-04-04 15:47:50 +0200264 this_cpu_offset \tmp
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000265 ldr \dst, [\dst, \tmp]
266 .endm
267
268/*
269 * vma_vm_mm - get mm pointer from vma pointer (vma->vm_mm)
270 */
271 .macro vma_vm_mm, rd, rn
272 ldr \rd, [\rn, #VMA_VM_MM]
273 .endm
274
275/*
David Brazdil0f672f62019-12-10 10:32:29 +0000276 * read_ctr - read CTR_EL0. If the system has mismatched register fields,
277 * provide the system wide safe value from arm64_ftr_reg_ctrel0.sys_val
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000278 */
279 .macro read_ctr, reg
David Brazdil0f672f62019-12-10 10:32:29 +0000280alternative_if_not ARM64_MISMATCHED_CACHE_TYPE
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000281 mrs \reg, ctr_el0 // read CTR
282 nop
283alternative_else
284 ldr_l \reg, arm64_ftr_reg_ctrel0 + ARM64_FTR_SYSVAL
285alternative_endif
286 .endm
287
288
289/*
290 * raw_dcache_line_size - get the minimum D-cache line size on this CPU
291 * from the CTR register.
292 */
293 .macro raw_dcache_line_size, reg, tmp
294 mrs \tmp, ctr_el0 // read CTR
295 ubfm \tmp, \tmp, #16, #19 // cache line size encoding
296 mov \reg, #4 // bytes per word
297 lsl \reg, \reg, \tmp // actual cache line size
298 .endm
299
300/*
301 * dcache_line_size - get the safe D-cache line size across all CPUs
302 */
303 .macro dcache_line_size, reg, tmp
304 read_ctr \tmp
305 ubfm \tmp, \tmp, #16, #19 // cache line size encoding
306 mov \reg, #4 // bytes per word
307 lsl \reg, \reg, \tmp // actual cache line size
308 .endm
309
310/*
311 * raw_icache_line_size - get the minimum I-cache line size on this CPU
312 * from the CTR register.
313 */
314 .macro raw_icache_line_size, reg, tmp
315 mrs \tmp, ctr_el0 // read CTR
316 and \tmp, \tmp, #0xf // cache line size encoding
317 mov \reg, #4 // bytes per word
318 lsl \reg, \reg, \tmp // actual cache line size
319 .endm
320
321/*
322 * icache_line_size - get the safe I-cache line size across all CPUs
323 */
324 .macro icache_line_size, reg, tmp
325 read_ctr \tmp
326 and \tmp, \tmp, #0xf // cache line size encoding
327 mov \reg, #4 // bytes per word
328 lsl \reg, \reg, \tmp // actual cache line size
329 .endm
330
331/*
David Brazdil0f672f62019-12-10 10:32:29 +0000332 * tcr_set_t0sz - update TCR.T0SZ so that we can load the ID map
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000333 */
David Brazdil0f672f62019-12-10 10:32:29 +0000334 .macro tcr_set_t0sz, valreg, t0sz
335 bfi \valreg, \t0sz, #TCR_T0SZ_OFFSET, #TCR_TxSZ_WIDTH
336 .endm
337
338/*
339 * tcr_set_t1sz - update TCR.T1SZ
340 */
341 .macro tcr_set_t1sz, valreg, t1sz
342 bfi \valreg, \t1sz, #TCR_T1SZ_OFFSET, #TCR_TxSZ_WIDTH
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000343 .endm
344
345/*
346 * tcr_compute_pa_size - set TCR.(I)PS to the highest supported
347 * ID_AA64MMFR0_EL1.PARange value
348 *
349 * tcr: register with the TCR_ELx value to be updated
350 * pos: IPS or PS bitfield position
351 * tmp{0,1}: temporary registers
352 */
353 .macro tcr_compute_pa_size, tcr, pos, tmp0, tmp1
354 mrs \tmp0, ID_AA64MMFR0_EL1
355 // Narrow PARange to fit the PS field in TCR_ELx
356 ubfx \tmp0, \tmp0, #ID_AA64MMFR0_PARANGE_SHIFT, #3
357 mov \tmp1, #ID_AA64MMFR0_PARANGE_MAX
358 cmp \tmp0, \tmp1
359 csel \tmp0, \tmp1, \tmp0, hi
360 bfi \tcr, \tmp0, \pos, #3
361 .endm
362
363/*
364 * Macro to perform a data cache maintenance for the interval
365 * [kaddr, kaddr + size)
366 *
367 * op: operation passed to dc instruction
368 * domain: domain used in dsb instruciton
369 * kaddr: starting virtual address of the region
370 * size: size of the region
371 * Corrupts: kaddr, size, tmp1, tmp2
372 */
David Brazdil0f672f62019-12-10 10:32:29 +0000373 .macro __dcache_op_workaround_clean_cache, op, kaddr
374alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE
375 dc \op, \kaddr
376alternative_else
377 dc civac, \kaddr
378alternative_endif
379 .endm
380
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000381 .macro dcache_by_line_op op, domain, kaddr, size, tmp1, tmp2
382 dcache_line_size \tmp1, \tmp2
383 add \size, \kaddr, \size
384 sub \tmp2, \tmp1, #1
385 bic \kaddr, \kaddr, \tmp2
3869998:
David Brazdil0f672f62019-12-10 10:32:29 +0000387 .ifc \op, cvau
388 __dcache_op_workaround_clean_cache \op, \kaddr
389 .else
390 .ifc \op, cvac
391 __dcache_op_workaround_clean_cache \op, \kaddr
392 .else
393 .ifc \op, cvap
394 sys 3, c7, c12, 1, \kaddr // dc cvap
395 .else
396 .ifc \op, cvadp
397 sys 3, c7, c13, 1, \kaddr // dc cvadp
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000398 .else
399 dc \op, \kaddr
400 .endif
David Brazdil0f672f62019-12-10 10:32:29 +0000401 .endif
402 .endif
403 .endif
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000404 add \kaddr, \kaddr, \tmp1
405 cmp \kaddr, \size
406 b.lo 9998b
407 dsb \domain
408 .endm
409
410/*
411 * Macro to perform an instruction cache maintenance for the interval
412 * [start, end)
413 *
414 * start, end: virtual addresses describing the region
415 * label: A label to branch to on user fault.
416 * Corrupts: tmp1, tmp2
417 */
418 .macro invalidate_icache_by_line start, end, tmp1, tmp2, label
419 icache_line_size \tmp1, \tmp2
420 sub \tmp2, \tmp1, #1
421 bic \tmp2, \start, \tmp2
4229997:
423USER(\label, ic ivau, \tmp2) // invalidate I line PoU
424 add \tmp2, \tmp2, \tmp1
425 cmp \tmp2, \end
426 b.lo 9997b
427 dsb ish
428 isb
429 .endm
430
431/*
432 * reset_pmuserenr_el0 - reset PMUSERENR_EL0 if PMUv3 present
433 */
434 .macro reset_pmuserenr_el0, tmpreg
David Brazdil0f672f62019-12-10 10:32:29 +0000435 mrs \tmpreg, id_aa64dfr0_el1
436 sbfx \tmpreg, \tmpreg, #ID_AA64DFR0_PMUVER_SHIFT, #4
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000437 cmp \tmpreg, #1 // Skip if no PMU present
438 b.lt 9000f
439 msr pmuserenr_el0, xzr // Disable PMU access from EL0
4409000:
441 .endm
442
443/*
Olivier Deprez157378f2022-04-04 15:47:50 +0200444 * reset_amuserenr_el0 - reset AMUSERENR_EL0 if AMUv1 present
445 */
446 .macro reset_amuserenr_el0, tmpreg
447 mrs \tmpreg, id_aa64pfr0_el1 // Check ID_AA64PFR0_EL1
448 ubfx \tmpreg, \tmpreg, #ID_AA64PFR0_AMU_SHIFT, #4
449 cbz \tmpreg, .Lskip_\@ // Skip if no AMU present
450 msr_s SYS_AMUSERENR_EL0, xzr // Disable AMU access from EL0
451.Lskip_\@:
452 .endm
453/*
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000454 * copy_page - copy src to dest using temp registers t1-t8
455 */
456 .macro copy_page dest:req src:req t1:req t2:req t3:req t4:req t5:req t6:req t7:req t8:req
4579998: ldp \t1, \t2, [\src]
458 ldp \t3, \t4, [\src, #16]
459 ldp \t5, \t6, [\src, #32]
460 ldp \t7, \t8, [\src, #48]
461 add \src, \src, #64
462 stnp \t1, \t2, [\dest]
463 stnp \t3, \t4, [\dest, #16]
464 stnp \t5, \t6, [\dest, #32]
465 stnp \t7, \t8, [\dest, #48]
466 add \dest, \dest, #64
467 tst \src, #(PAGE_SIZE - 1)
468 b.ne 9998b
469 .endm
470
471/*
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000472 * Annotate a function as being unsuitable for kprobes.
473 */
474#ifdef CONFIG_KPROBES
475#define NOKPROBE(x) \
476 .pushsection "_kprobe_blacklist", "aw"; \
477 .quad x; \
478 .popsection;
479#else
480#define NOKPROBE(x)
481#endif
David Brazdil0f672f62019-12-10 10:32:29 +0000482
483#ifdef CONFIG_KASAN
484#define EXPORT_SYMBOL_NOKASAN(name)
485#else
486#define EXPORT_SYMBOL_NOKASAN(name) EXPORT_SYMBOL(name)
487#endif
488
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000489 /*
490 * Emit a 64-bit absolute little endian symbol reference in a way that
491 * ensures that it will be resolved at build time, even when building a
492 * PIE binary. This requires cooperation from the linker script, which
493 * must emit the lo32/hi32 halves individually.
494 */
495 .macro le64sym, sym
496 .long \sym\()_lo32
497 .long \sym\()_hi32
498 .endm
499
500 /*
501 * mov_q - move an immediate constant into a 64-bit register using
502 * between 2 and 4 movz/movk instructions (depending on the
503 * magnitude and sign of the operand)
504 */
505 .macro mov_q, reg, val
506 .if (((\val) >> 31) == 0 || ((\val) >> 31) == 0x1ffffffff)
507 movz \reg, :abs_g1_s:\val
508 .else
509 .if (((\val) >> 47) == 0 || ((\val) >> 47) == 0x1ffff)
510 movz \reg, :abs_g2_s:\val
511 .else
512 movz \reg, :abs_g3:\val
513 movk \reg, :abs_g2_nc:\val
514 .endif
515 movk \reg, :abs_g1_nc:\val
516 .endif
517 movk \reg, :abs_g0_nc:\val
518 .endm
519
520/*
David Brazdil0f672f62019-12-10 10:32:29 +0000521 * Return the current task_struct.
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000522 */
David Brazdil0f672f62019-12-10 10:32:29 +0000523 .macro get_current_task, rd
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000524 mrs \rd, sp_el0
525 .endm
526
527/*
David Brazdil0f672f62019-12-10 10:32:29 +0000528 * Offset ttbr1 to allow for 48-bit kernel VAs set with 52-bit PTRS_PER_PGD.
529 * orr is used as it can cover the immediate value (and is idempotent).
530 * In future this may be nop'ed out when dealing with 52-bit kernel VAs.
531 * ttbr: Value of ttbr to set, modified.
532 */
533 .macro offset_ttbr1, ttbr, tmp
534#ifdef CONFIG_ARM64_VA_BITS_52
535 mrs_s \tmp, SYS_ID_AA64MMFR2_EL1
536 and \tmp, \tmp, #(0xf << ID_AA64MMFR2_LVA_SHIFT)
537 cbnz \tmp, .Lskipoffs_\@
538 orr \ttbr, \ttbr, #TTBR1_BADDR_4852_OFFSET
539.Lskipoffs_\@ :
540#endif
541 .endm
542
543/*
544 * Perform the reverse of offset_ttbr1.
545 * bic is used as it can cover the immediate value and, in future, won't need
546 * to be nop'ed out when dealing with 52-bit kernel VAs.
547 */
548 .macro restore_ttbr1, ttbr
549#ifdef CONFIG_ARM64_VA_BITS_52
550 bic \ttbr, \ttbr, #TTBR1_BADDR_4852_OFFSET
551#endif
552 .endm
553
554/*
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000555 * Arrange a physical address in a TTBR register, taking care of 52-bit
556 * addresses.
557 *
558 * phys: physical address, preserved
559 * ttbr: returns the TTBR value
560 */
561 .macro phys_to_ttbr, ttbr, phys
562#ifdef CONFIG_ARM64_PA_BITS_52
563 orr \ttbr, \phys, \phys, lsr #46
564 and \ttbr, \ttbr, #TTBR_BADDR_MASK_52
565#else
566 mov \ttbr, \phys
567#endif
568 .endm
569
570 .macro phys_to_pte, pte, phys
571#ifdef CONFIG_ARM64_PA_BITS_52
572 /*
573 * We assume \phys is 64K aligned and this is guaranteed by only
574 * supporting this configuration with 64K pages.
575 */
576 orr \pte, \phys, \phys, lsr #36
577 and \pte, \pte, #PTE_ADDR_MASK
578#else
579 mov \pte, \phys
580#endif
581 .endm
582
583 .macro pte_to_phys, phys, pte
584#ifdef CONFIG_ARM64_PA_BITS_52
585 ubfiz \phys, \pte, #(48 - 16 - 12), #16
586 bfxil \phys, \pte, #16, #32
587 lsl \phys, \phys, #16
588#else
589 and \phys, \pte, #PTE_ADDR_MASK
590#endif
591 .endm
592
David Brazdil0f672f62019-12-10 10:32:29 +0000593/*
594 * tcr_clear_errata_bits - Clear TCR bits that trigger an errata on this CPU.
595 */
596 .macro tcr_clear_errata_bits, tcr, tmp1, tmp2
597#ifdef CONFIG_FUJITSU_ERRATUM_010001
598 mrs \tmp1, midr_el1
599
600 mov_q \tmp2, MIDR_FUJITSU_ERRATUM_010001_MASK
601 and \tmp1, \tmp1, \tmp2
602 mov_q \tmp2, MIDR_FUJITSU_ERRATUM_010001
603 cmp \tmp1, \tmp2
604 b.ne 10f
605
606 mov_q \tmp2, TCR_CLEAR_FUJITSU_ERRATUM_010001
607 bic \tcr, \tcr, \tmp2
60810:
609#endif /* CONFIG_FUJITSU_ERRATUM_010001 */
610 .endm
611
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000612/**
613 * Errata workaround prior to disable MMU. Insert an ISB immediately prior
614 * to executing the MSR that will change SCTLR_ELn[M] from a value of 1 to 0.
615 */
616 .macro pre_disable_mmu_workaround
617#ifdef CONFIG_QCOM_FALKOR_ERRATUM_E1041
618 isb
619#endif
620 .endm
621
622 /*
623 * frame_push - Push @regcount callee saved registers to the stack,
624 * starting at x19, as well as x29/x30, and set x29 to
625 * the new value of sp. Add @extra bytes of stack space
626 * for locals.
627 */
628 .macro frame_push, regcount:req, extra
629 __frame st, \regcount, \extra
630 .endm
631
632 /*
633 * frame_pop - Pop the callee saved registers from the stack that were
634 * pushed in the most recent call to frame_push, as well
635 * as x29/x30 and any extra stack space that may have been
636 * allocated.
637 */
638 .macro frame_pop
639 __frame ld
640 .endm
641
642 .macro __frame_regs, reg1, reg2, op, num
643 .if .Lframe_regcount == \num
644 \op\()r \reg1, [sp, #(\num + 1) * 8]
645 .elseif .Lframe_regcount > \num
646 \op\()p \reg1, \reg2, [sp, #(\num + 1) * 8]
647 .endif
648 .endm
649
650 .macro __frame, op, regcount, extra=0
651 .ifc \op, st
652 .if (\regcount) < 0 || (\regcount) > 10
653 .error "regcount should be in the range [0 ... 10]"
654 .endif
655 .if ((\extra) % 16) != 0
656 .error "extra should be a multiple of 16 bytes"
657 .endif
658 .ifdef .Lframe_regcount
659 .if .Lframe_regcount != -1
660 .error "frame_push/frame_pop may not be nested"
661 .endif
662 .endif
663 .set .Lframe_regcount, \regcount
664 .set .Lframe_extra, \extra
665 .set .Lframe_local_offset, ((\regcount + 3) / 2) * 16
666 stp x29, x30, [sp, #-.Lframe_local_offset - .Lframe_extra]!
667 mov x29, sp
668 .endif
669
670 __frame_regs x19, x20, \op, 1
671 __frame_regs x21, x22, \op, 3
672 __frame_regs x23, x24, \op, 5
673 __frame_regs x25, x26, \op, 7
674 __frame_regs x27, x28, \op, 9
675
676 .ifc \op, ld
677 .if .Lframe_regcount == -1
678 .error "frame_push/frame_pop may not be nested"
679 .endif
680 ldp x29, x30, [sp], #.Lframe_local_offset + .Lframe_extra
681 .set .Lframe_regcount, -1
682 .endif
683 .endm
684
685/*
686 * Check whether to yield to another runnable task from kernel mode NEON code
687 * (which runs with preemption disabled).
688 *
689 * if_will_cond_yield_neon
690 * // pre-yield patchup code
691 * do_cond_yield_neon
692 * // post-yield patchup code
693 * endif_yield_neon <label>
694 *
695 * where <label> is optional, and marks the point where execution will resume
696 * after a yield has been performed. If omitted, execution resumes right after
697 * the endif_yield_neon invocation. Note that the entire sequence, including
Olivier Deprez157378f2022-04-04 15:47:50 +0200698 * the provided patchup code, will be omitted from the image if
699 * CONFIG_PREEMPTION is not defined.
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000700 *
701 * As a convenience, in the case where no patchup code is required, the above
702 * sequence may be abbreviated to
703 *
704 * cond_yield_neon <label>
705 *
706 * Note that the patchup code does not support assembler directives that change
707 * the output section, any use of such directives is undefined.
708 *
709 * The yield itself consists of the following:
David Brazdil0f672f62019-12-10 10:32:29 +0000710 * - Check whether the preempt count is exactly 1 and a reschedule is also
711 * needed. If so, calling of preempt_enable() in kernel_neon_end() will
712 * trigger a reschedule. If it is not the case, yielding is pointless.
713 * - Disable and re-enable kernel mode NEON, and branch to the yield fixup
714 * code.
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000715 *
716 * This macro sequence may clobber all CPU state that is not guaranteed by the
717 * AAPCS to be preserved across an ordinary function call.
718 */
719
720 .macro cond_yield_neon, lbl
721 if_will_cond_yield_neon
722 do_cond_yield_neon
723 endif_yield_neon \lbl
724 .endm
725
726 .macro if_will_cond_yield_neon
Olivier Deprez157378f2022-04-04 15:47:50 +0200727#ifdef CONFIG_PREEMPTION
David Brazdil0f672f62019-12-10 10:32:29 +0000728 get_current_task x0
729 ldr x0, [x0, #TSK_TI_PREEMPT]
730 sub x0, x0, #PREEMPT_DISABLE_OFFSET
731 cbz x0, .Lyield_\@
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000732 /* fall through to endif_yield_neon */
733 .subsection 1
734.Lyield_\@ :
735#else
736 .section ".discard.cond_yield_neon", "ax"
737#endif
738 .endm
739
740 .macro do_cond_yield_neon
741 bl kernel_neon_end
742 bl kernel_neon_begin
743 .endm
744
745 .macro endif_yield_neon, lbl
746 .ifnb \lbl
747 b \lbl
748 .else
749 b .Lyield_out_\@
750 .endif
751 .previous
752.Lyield_out_\@ :
753 .endm
754
Olivier Deprez157378f2022-04-04 15:47:50 +0200755/*
756 * This macro emits a program property note section identifying
757 * architecture features which require special handling, mainly for
758 * use in assembly files included in the VDSO.
759 */
760
761#define NT_GNU_PROPERTY_TYPE_0 5
762#define GNU_PROPERTY_AARCH64_FEATURE_1_AND 0xc0000000
763
764#define GNU_PROPERTY_AARCH64_FEATURE_1_BTI (1U << 0)
765#define GNU_PROPERTY_AARCH64_FEATURE_1_PAC (1U << 1)
766
767#ifdef CONFIG_ARM64_BTI_KERNEL
768#define GNU_PROPERTY_AARCH64_FEATURE_1_DEFAULT \
769 ((GNU_PROPERTY_AARCH64_FEATURE_1_BTI | \
770 GNU_PROPERTY_AARCH64_FEATURE_1_PAC))
771#endif
772
773#ifdef GNU_PROPERTY_AARCH64_FEATURE_1_DEFAULT
774.macro emit_aarch64_feature_1_and, feat=GNU_PROPERTY_AARCH64_FEATURE_1_DEFAULT
775 .pushsection .note.gnu.property, "a"
776 .align 3
777 .long 2f - 1f
778 .long 6f - 3f
779 .long NT_GNU_PROPERTY_TYPE_0
7801: .string "GNU"
7812:
782 .align 3
7833: .long GNU_PROPERTY_AARCH64_FEATURE_1_AND
784 .long 5f - 4f
7854:
786 /*
787 * This is described with an array of char in the Linux API
788 * spec but the text and all other usage (including binutils,
789 * clang and GCC) treat this as a 32 bit value so no swizzling
790 * is required for big endian.
791 */
792 .long \feat
7935:
794 .align 3
7956:
796 .popsection
797.endm
798
799#else
800.macro emit_aarch64_feature_1_and, feat=0
801.endm
802
803#endif /* GNU_PROPERTY_AARCH64_FEATURE_1_DEFAULT */
804
805 .macro __mitigate_spectre_bhb_loop tmp
806#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
807alternative_cb spectre_bhb_patch_loop_iter
808 mov \tmp, #32 // Patched to correct the immediate
809alternative_cb_end
810.Lspectre_bhb_loop\@:
811 b . + 4
812 subs \tmp, \tmp, #1
813 b.ne .Lspectre_bhb_loop\@
814 sb
815#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
816 .endm
817
818 /* Save/restores x0-x3 to the stack */
819 .macro __mitigate_spectre_bhb_fw
820#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
821 stp x0, x1, [sp, #-16]!
822 stp x2, x3, [sp, #-16]!
823 mov w0, #ARM_SMCCC_ARCH_WORKAROUND_3
824alternative_cb smccc_patch_fw_mitigation_conduit
825 nop // Patched to SMC/HVC #0
826alternative_cb_end
827 ldp x2, x3, [sp], #16
828 ldp x0, x1, [sp], #16
829#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
830 .endm
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000831#endif /* __ASM_ASSEMBLER_H */