TrustedFirmware Git Browser
Code Review Sign In
review.trustedfirmware.org / hafnium / third_party / linux.git / 0f672f6c0b52b7b0700b0915c72b540721af4465 / . / arch / powerpc / lib / memcpy_power7.S
blob: 54f226333c9422f4f95f025ee00c18074cc81b55 [file] [log] [blame]
David Brazdil0f672f62019-12-10 10:32:29 +0000[diff] [blame^]1/* SPDX-License-Identifier: GPL-2.0-or-later */
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000[diff] [blame]2/*
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000[diff] [blame]3 *
4 * Copyright (C) IBM Corporation, 2012
5 *
6 * Author: Anton Blanchard <anton@au.ibm.com>
7 */
8#include <asm/ppc_asm.h>
9
10#ifndef SELFTEST_CASE
11/* 0 == don't use VMX, 1 == use VMX */
12#define SELFTEST_CASE 0
13#endif
14
15#ifdef __BIG_ENDIAN__
16#define LVS(VRT,RA,RB) lvsl VRT,RA,RB
17#define VPERM(VRT,VRA,VRB,VRC) vperm VRT,VRA,VRB,VRC
18#else
19#define LVS(VRT,RA,RB) lvsr VRT,RA,RB
20#define VPERM(VRT,VRA,VRB,VRC) vperm VRT,VRB,VRA,VRC
21#endif
22
23_GLOBAL(memcpy_power7)
24 cmpldi r5,16
25 cmpldi cr1,r5,4096
26 std r3,-STACKFRAMESIZE+STK_REG(R31)(r1)
27 blt .Lshort_copy
28
29#ifdef CONFIG_ALTIVEC
30test_feature = SELFTEST_CASE
31BEGIN_FTR_SECTION
32 bgt cr1, .Lvmx_copy
33END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
34#endif
35
36.Lnonvmx_copy:
37 /* Get the source 8B aligned */
38 neg r6,r4
39 mtocrf 0x01,r6
40 clrldi r6,r6,(64-3)
41
42 bf cr7*4+3,1f
43 lbz r0,0(r4)
44 addi r4,r4,1
45 stb r0,0(r3)
46 addi r3,r3,1
47
481: bf cr7*4+2,2f
49 lhz r0,0(r4)
50 addi r4,r4,2
51 sth r0,0(r3)
52 addi r3,r3,2
53
542: bf cr7*4+1,3f
55 lwz r0,0(r4)
56 addi r4,r4,4
57 stw r0,0(r3)
58 addi r3,r3,4
59
603: sub r5,r5,r6
61 cmpldi r5,128
62 blt 5f
63
64 mflr r0
65 stdu r1,-STACKFRAMESIZE(r1)
66 std r14,STK_REG(R14)(r1)
67 std r15,STK_REG(R15)(r1)
68 std r16,STK_REG(R16)(r1)
69 std r17,STK_REG(R17)(r1)
70 std r18,STK_REG(R18)(r1)
71 std r19,STK_REG(R19)(r1)
72 std r20,STK_REG(R20)(r1)
73 std r21,STK_REG(R21)(r1)
74 std r22,STK_REG(R22)(r1)
75 std r0,STACKFRAMESIZE+16(r1)
76
77 srdi r6,r5,7
78 mtctr r6
79
80 /* Now do cacheline (128B) sized loads and stores. */
81 .align 5
824:
83 ld r0,0(r4)
84 ld r6,8(r4)
85 ld r7,16(r4)
86 ld r8,24(r4)
87 ld r9,32(r4)
88 ld r10,40(r4)
89 ld r11,48(r4)
90 ld r12,56(r4)
91 ld r14,64(r4)
92 ld r15,72(r4)
93 ld r16,80(r4)
94 ld r17,88(r4)
95 ld r18,96(r4)
96 ld r19,104(r4)
97 ld r20,112(r4)
98 ld r21,120(r4)
99 addi r4,r4,128
100 std r0,0(r3)
101 std r6,8(r3)
102 std r7,16(r3)
103 std r8,24(r3)
104 std r9,32(r3)
105 std r10,40(r3)
106 std r11,48(r3)
107 std r12,56(r3)
108 std r14,64(r3)
109 std r15,72(r3)
110 std r16,80(r3)
111 std r17,88(r3)
112 std r18,96(r3)
113 std r19,104(r3)
114 std r20,112(r3)
115 std r21,120(r3)
116 addi r3,r3,128
117 bdnz 4b
118
119 clrldi r5,r5,(64-7)
120
121 ld r14,STK_REG(R14)(r1)
122 ld r15,STK_REG(R15)(r1)
123 ld r16,STK_REG(R16)(r1)
124 ld r17,STK_REG(R17)(r1)
125 ld r18,STK_REG(R18)(r1)
126 ld r19,STK_REG(R19)(r1)
127 ld r20,STK_REG(R20)(r1)
128 ld r21,STK_REG(R21)(r1)
129 ld r22,STK_REG(R22)(r1)
130 addi r1,r1,STACKFRAMESIZE
131
132 /* Up to 127B to go */
1335: srdi r6,r5,4
134 mtocrf 0x01,r6
135
1366: bf cr7*4+1,7f
137 ld r0,0(r4)
138 ld r6,8(r4)
139 ld r7,16(r4)
140 ld r8,24(r4)
141 ld r9,32(r4)
142 ld r10,40(r4)
143 ld r11,48(r4)
144 ld r12,56(r4)
145 addi r4,r4,64
146 std r0,0(r3)
147 std r6,8(r3)
148 std r7,16(r3)
149 std r8,24(r3)
150 std r9,32(r3)
151 std r10,40(r3)
152 std r11,48(r3)
153 std r12,56(r3)
154 addi r3,r3,64
155
156 /* Up to 63B to go */
1577: bf cr7*4+2,8f
158 ld r0,0(r4)
159 ld r6,8(r4)
160 ld r7,16(r4)
161 ld r8,24(r4)
162 addi r4,r4,32
163 std r0,0(r3)
164 std r6,8(r3)
165 std r7,16(r3)
166 std r8,24(r3)
167 addi r3,r3,32
168
169 /* Up to 31B to go */
1708: bf cr7*4+3,9f
171 ld r0,0(r4)
172 ld r6,8(r4)
173 addi r4,r4,16
174 std r0,0(r3)
175 std r6,8(r3)
176 addi r3,r3,16
177
1789: clrldi r5,r5,(64-4)
179
180 /* Up to 15B to go */
181.Lshort_copy:
182 mtocrf 0x01,r5
183 bf cr7*4+0,12f
184 lwz r0,0(r4) /* Less chance of a reject with word ops */
185 lwz r6,4(r4)
186 addi r4,r4,8
187 stw r0,0(r3)
188 stw r6,4(r3)
189 addi r3,r3,8
190
19112: bf cr7*4+1,13f
192 lwz r0,0(r4)
193 addi r4,r4,4
194 stw r0,0(r3)
195 addi r3,r3,4
196
19713: bf cr7*4+2,14f
198 lhz r0,0(r4)
199 addi r4,r4,2
200 sth r0,0(r3)
201 addi r3,r3,2
202
20314: bf cr7*4+3,15f
204 lbz r0,0(r4)
205 stb r0,0(r3)
206
20715: ld r3,-STACKFRAMESIZE+STK_REG(R31)(r1)
208 blr
209
210.Lunwind_stack_nonvmx_copy:
211 addi r1,r1,STACKFRAMESIZE
212 b .Lnonvmx_copy
213
214.Lvmx_copy:
215#ifdef CONFIG_ALTIVEC
216 mflr r0
217 std r4,-STACKFRAMESIZE+STK_REG(R30)(r1)
218 std r5,-STACKFRAMESIZE+STK_REG(R29)(r1)
219 std r0,16(r1)
220 stdu r1,-STACKFRAMESIZE(r1)
221 bl enter_vmx_ops
222 cmpwi cr1,r3,0
223 ld r0,STACKFRAMESIZE+16(r1)
224 ld r3,STK_REG(R31)(r1)
225 ld r4,STK_REG(R30)(r1)
226 ld r5,STK_REG(R29)(r1)
227 mtlr r0
228
229 /*
230 * We prefetch both the source and destination using enhanced touch
231 * instructions. We use a stream ID of 0 for the load side and
232 * 1 for the store side.
233 */
234 clrrdi r6,r4,7
235 clrrdi r9,r3,7
236 ori r9,r9,1 /* stream=1 */
237
238 srdi r7,r5,7 /* length in cachelines, capped at 0x3FF */
239 cmpldi r7,0x3FF
240 ble 1f
241 li r7,0x3FF
2421: lis r0,0x0E00 /* depth=7 */
243 sldi r7,r7,7
244 or r7,r7,r0
245 ori r10,r7,1 /* stream=1 */
246
247 lis r8,0x8000 /* GO=1 */
248 clrldi r8,r8,32
249
250 dcbt 0,r6,0b01000
251 dcbt 0,r7,0b01010
252 dcbtst 0,r9,0b01000
253 dcbtst 0,r10,0b01010
254 eieio
255 dcbt 0,r8,0b01010 /* GO */
256
257 beq cr1,.Lunwind_stack_nonvmx_copy
258
259 /*
260 * If source and destination are not relatively aligned we use a
261 * slower permute loop.
262 */
263 xor r6,r4,r3
264 rldicl. r6,r6,0,(64-4)
265 bne .Lvmx_unaligned_copy
266
267 /* Get the destination 16B aligned */
268 neg r6,r3
269 mtocrf 0x01,r6
270 clrldi r6,r6,(64-4)
271
272 bf cr7*4+3,1f
273 lbz r0,0(r4)
274 addi r4,r4,1
275 stb r0,0(r3)
276 addi r3,r3,1
277
2781: bf cr7*4+2,2f
279 lhz r0,0(r4)
280 addi r4,r4,2
281 sth r0,0(r3)
282 addi r3,r3,2
283
2842: bf cr7*4+1,3f
285 lwz r0,0(r4)
286 addi r4,r4,4
287 stw r0,0(r3)
288 addi r3,r3,4
289
2903: bf cr7*4+0,4f
291 ld r0,0(r4)
292 addi r4,r4,8
293 std r0,0(r3)
294 addi r3,r3,8
295
2964: sub r5,r5,r6
297
298 /* Get the desination 128B aligned */
299 neg r6,r3
300 srdi r7,r6,4
301 mtocrf 0x01,r7
302 clrldi r6,r6,(64-7)
303
304 li r9,16
305 li r10,32
306 li r11,48
307
308 bf cr7*4+3,5f
309 lvx v1,0,r4
310 addi r4,r4,16
311 stvx v1,0,r3
312 addi r3,r3,16
313
3145: bf cr7*4+2,6f
315 lvx v1,0,r4
316 lvx v0,r4,r9
317 addi r4,r4,32
318 stvx v1,0,r3
319 stvx v0,r3,r9
320 addi r3,r3,32
321
3226: bf cr7*4+1,7f
323 lvx v3,0,r4
324 lvx v2,r4,r9
325 lvx v1,r4,r10
326 lvx v0,r4,r11
327 addi r4,r4,64
328 stvx v3,0,r3
329 stvx v2,r3,r9
330 stvx v1,r3,r10
331 stvx v0,r3,r11
332 addi r3,r3,64
333
3347: sub r5,r5,r6
335 srdi r6,r5,7
336
337 std r14,STK_REG(R14)(r1)
338 std r15,STK_REG(R15)(r1)
339 std r16,STK_REG(R16)(r1)
340
341 li r12,64
342 li r14,80
343 li r15,96
344 li r16,112
345
346 mtctr r6
347
348 /*
349 * Now do cacheline sized loads and stores. By this stage the
350 * cacheline stores are also cacheline aligned.
351 */
352 .align 5
3538:
354 lvx v7,0,r4
355 lvx v6,r4,r9
356 lvx v5,r4,r10
357 lvx v4,r4,r11
358 lvx v3,r4,r12
359 lvx v2,r4,r14
360 lvx v1,r4,r15
361 lvx v0,r4,r16
362 addi r4,r4,128
363 stvx v7,0,r3
364 stvx v6,r3,r9
365 stvx v5,r3,r10
366 stvx v4,r3,r11
367 stvx v3,r3,r12
368 stvx v2,r3,r14
369 stvx v1,r3,r15
370 stvx v0,r3,r16
371 addi r3,r3,128
372 bdnz 8b
373
374 ld r14,STK_REG(R14)(r1)
375 ld r15,STK_REG(R15)(r1)
376 ld r16,STK_REG(R16)(r1)
377
378 /* Up to 127B to go */
379 clrldi r5,r5,(64-7)
380 srdi r6,r5,4
381 mtocrf 0x01,r6
382
383 bf cr7*4+1,9f
384 lvx v3,0,r4
385 lvx v2,r4,r9
386 lvx v1,r4,r10
387 lvx v0,r4,r11
388 addi r4,r4,64
389 stvx v3,0,r3
390 stvx v2,r3,r9
391 stvx v1,r3,r10
392 stvx v0,r3,r11
393 addi r3,r3,64
394
3959: bf cr7*4+2,10f
396 lvx v1,0,r4
397 lvx v0,r4,r9
398 addi r4,r4,32
399 stvx v1,0,r3
400 stvx v0,r3,r9
401 addi r3,r3,32
402
40310: bf cr7*4+3,11f
404 lvx v1,0,r4
405 addi r4,r4,16
406 stvx v1,0,r3
407 addi r3,r3,16
408
409 /* Up to 15B to go */
41011: clrldi r5,r5,(64-4)
411 mtocrf 0x01,r5
412 bf cr7*4+0,12f
413 ld r0,0(r4)
414 addi r4,r4,8
415 std r0,0(r3)
416 addi r3,r3,8
417
41812: bf cr7*4+1,13f
419 lwz r0,0(r4)
420 addi r4,r4,4
421 stw r0,0(r3)
422 addi r3,r3,4
423
42413: bf cr7*4+2,14f
425 lhz r0,0(r4)
426 addi r4,r4,2
427 sth r0,0(r3)
428 addi r3,r3,2
429
43014: bf cr7*4+3,15f
431 lbz r0,0(r4)
432 stb r0,0(r3)
433
43415: addi r1,r1,STACKFRAMESIZE
435 ld r3,-STACKFRAMESIZE+STK_REG(R31)(r1)
436 b exit_vmx_ops /* tail call optimise */
437
438.Lvmx_unaligned_copy:
439 /* Get the destination 16B aligned */
440 neg r6,r3
441 mtocrf 0x01,r6
442 clrldi r6,r6,(64-4)
443
444 bf cr7*4+3,1f
445 lbz r0,0(r4)
446 addi r4,r4,1
447 stb r0,0(r3)
448 addi r3,r3,1
449
4501: bf cr7*4+2,2f
451 lhz r0,0(r4)
452 addi r4,r4,2
453 sth r0,0(r3)
454 addi r3,r3,2
455
4562: bf cr7*4+1,3f
457 lwz r0,0(r4)
458 addi r4,r4,4
459 stw r0,0(r3)
460 addi r3,r3,4
461
4623: bf cr7*4+0,4f
463 lwz r0,0(r4) /* Less chance of a reject with word ops */
464 lwz r7,4(r4)
465 addi r4,r4,8
466 stw r0,0(r3)
467 stw r7,4(r3)
468 addi r3,r3,8
469
4704: sub r5,r5,r6
471
472 /* Get the desination 128B aligned */
473 neg r6,r3
474 srdi r7,r6,4
475 mtocrf 0x01,r7
476 clrldi r6,r6,(64-7)
477
478 li r9,16
479 li r10,32
480 li r11,48
481
482 LVS(v16,0,r4) /* Setup permute control vector */
483 lvx v0,0,r4
484 addi r4,r4,16
485
486 bf cr7*4+3,5f
487 lvx v1,0,r4
488 VPERM(v8,v0,v1,v16)
489 addi r4,r4,16
490 stvx v8,0,r3
491 addi r3,r3,16
492 vor v0,v1,v1
493
4945: bf cr7*4+2,6f
495 lvx v1,0,r4
496 VPERM(v8,v0,v1,v16)
497 lvx v0,r4,r9
498 VPERM(v9,v1,v0,v16)
499 addi r4,r4,32
500 stvx v8,0,r3
501 stvx v9,r3,r9
502 addi r3,r3,32
503
5046: bf cr7*4+1,7f
505 lvx v3,0,r4
506 VPERM(v8,v0,v3,v16)
507 lvx v2,r4,r9
508 VPERM(v9,v3,v2,v16)
509 lvx v1,r4,r10
510 VPERM(v10,v2,v1,v16)
511 lvx v0,r4,r11
512 VPERM(v11,v1,v0,v16)
513 addi r4,r4,64
514 stvx v8,0,r3
515 stvx v9,r3,r9
516 stvx v10,r3,r10
517 stvx v11,r3,r11
518 addi r3,r3,64
519
5207: sub r5,r5,r6
521 srdi r6,r5,7
522
523 std r14,STK_REG(R14)(r1)
524 std r15,STK_REG(R15)(r1)
525 std r16,STK_REG(R16)(r1)
526
527 li r12,64
528 li r14,80
529 li r15,96
530 li r16,112
531
532 mtctr r6
533
534 /*
535 * Now do cacheline sized loads and stores. By this stage the
536 * cacheline stores are also cacheline aligned.
537 */
538 .align 5
5398:
540 lvx v7,0,r4
541 VPERM(v8,v0,v7,v16)
542 lvx v6,r4,r9
543 VPERM(v9,v7,v6,v16)
544 lvx v5,r4,r10
545 VPERM(v10,v6,v5,v16)
546 lvx v4,r4,r11
547 VPERM(v11,v5,v4,v16)
548 lvx v3,r4,r12
549 VPERM(v12,v4,v3,v16)
550 lvx v2,r4,r14
551 VPERM(v13,v3,v2,v16)
552 lvx v1,r4,r15
553 VPERM(v14,v2,v1,v16)
554 lvx v0,r4,r16
555 VPERM(v15,v1,v0,v16)
556 addi r4,r4,128
557 stvx v8,0,r3
558 stvx v9,r3,r9
559 stvx v10,r3,r10
560 stvx v11,r3,r11
561 stvx v12,r3,r12
562 stvx v13,r3,r14
563 stvx v14,r3,r15
564 stvx v15,r3,r16
565 addi r3,r3,128
566 bdnz 8b
567
568 ld r14,STK_REG(R14)(r1)
569 ld r15,STK_REG(R15)(r1)
570 ld r16,STK_REG(R16)(r1)
571
572 /* Up to 127B to go */
573 clrldi r5,r5,(64-7)
574 srdi r6,r5,4
575 mtocrf 0x01,r6
576
577 bf cr7*4+1,9f
578 lvx v3,0,r4
579 VPERM(v8,v0,v3,v16)
580 lvx v2,r4,r9
581 VPERM(v9,v3,v2,v16)
582 lvx v1,r4,r10
583 VPERM(v10,v2,v1,v16)
584 lvx v0,r4,r11
585 VPERM(v11,v1,v0,v16)
586 addi r4,r4,64
587 stvx v8,0,r3
588 stvx v9,r3,r9
589 stvx v10,r3,r10
590 stvx v11,r3,r11
591 addi r3,r3,64
592
5939: bf cr7*4+2,10f
594 lvx v1,0,r4
595 VPERM(v8,v0,v1,v16)
596 lvx v0,r4,r9
597 VPERM(v9,v1,v0,v16)
598 addi r4,r4,32
599 stvx v8,0,r3
600 stvx v9,r3,r9
601 addi r3,r3,32
602
60310: bf cr7*4+3,11f
604 lvx v1,0,r4
605 VPERM(v8,v0,v1,v16)
606 addi r4,r4,16
607 stvx v8,0,r3
608 addi r3,r3,16
609
610 /* Up to 15B to go */
61111: clrldi r5,r5,(64-4)
612 addi r4,r4,-16 /* Unwind the +16 load offset */
613 mtocrf 0x01,r5
614 bf cr7*4+0,12f
615 lwz r0,0(r4) /* Less chance of a reject with word ops */
616 lwz r6,4(r4)
617 addi r4,r4,8
618 stw r0,0(r3)
619 stw r6,4(r3)
620 addi r3,r3,8
621
62212: bf cr7*4+1,13f
623 lwz r0,0(r4)
624 addi r4,r4,4
625 stw r0,0(r3)
626 addi r3,r3,4
627
62813: bf cr7*4+2,14f
629 lhz r0,0(r4)
630 addi r4,r4,2
631 sth r0,0(r3)
632 addi r3,r3,2
633
63414: bf cr7*4+3,15f
635 lbz r0,0(r4)
636 stb r0,0(r3)
637
63815: addi r1,r1,STACKFRAMESIZE
639 ld r3,-STACKFRAMESIZE+STK_REG(R31)(r1)
640 b exit_vmx_ops /* tail call optimise */
641#endif /* CONFIG_ALTIVEC */