3 # ====================================================================
4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
5 # project. The module is, however, dual licensed under OpenSSL and
6 # CRYPTOGAMS licenses depending on where you obtain it. For further
7 # details see http://www.openssl.org/~appro/cryptogams/.
8 # ====================================================================
10 # This module implements Poly1305 hash for ARMv8.
14 # Numbers are cycles per processed byte with poly1305_blocks alone.
18 # Apple A7 1.86/+5% 0.72
19 # Cortex-A53 2.69/+58% 1.47
20 # Cortex-A57 2.70/+7% 1.14
21 # Denver 1.64/+50% 1.18(*)
22 # X-Gene 2.13/+68% 2.27
24 # (*) estimate based on resources availability is less than 1.0,
25 # i.e. measured result is worse than expected, presumably binary
26 # translator is not almighty;
31 $0 =~ m/(.*[\/\\])[^\
/\\]+$/; $dir=$1;
32 ( $xlate="${dir}arm-xlate.pl" and -f
$xlate ) or
33 ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f
$xlate) or
34 die "can't locate arm-xlate.pl";
36 open OUT
,"| \"$^X\" $xlate $flavour $output";
39 my ($ctx,$inp,$len,$padbit) = map("x$_",(0..3));
40 my ($mac,$nonce)=($inp,$len);
42 my ($h0,$h1,$h2,$r0,$r1,$s1,$t0,$t1,$d0,$d1,$d2) = map("x$_",(4..14));
49 // forward
"declarations" are required
for Apple
50 .extern OPENSSL_armcap_P
51 .globl poly1305_blocks
55 .type poly1305_init
,%function
59 stp xzr
,xzr
,[$ctx] // zero hash value
60 stp xzr
,xzr
,[$ctx,#16] // [along with is_base2_26]
66 ldrsw
$t1,.LOPENSSL_armcap_P
68 ldr
$t1,.LOPENSSL_armcap_P
70 adr
$t0,.LOPENSSL_armcap_P
72 ldp
$r0,$r1,[$inp] // load key
73 mov
$s1,#0xfffffffc0fffffff
74 movk
$s1,#0x0fff,lsl#48
77 rev
$r0,$r0 // flip bytes
80 and $r0,$r0,$s1 // &=0ffffffc0fffffff
82 and $r1,$r1,$s1 // &=0ffffffc0ffffffc
83 stp
$r0,$r1,[$ctx,#32] // save key value
87 adr
$d0,poly1305_blocks
88 adr
$r0,poly1305_blocks_neon
90 adr
$r1,poly1305_emit_neon
100 .size poly1305_init
,.-poly1305_init
102 .type poly1305_blocks
,%function
108 ldp
$h0,$h1,[$ctx] // load hash value
109 ldp
$r0,$r1,[$ctx,#32] // load key value
111 add
$s1,$r1,$r1,lsr
#2 // s1 = r1 + (r1 >> 2)
116 ldp
$t0,$t1,[$inp],#16 // load input
122 adds
$h0,$h0,$t0 // accumulate input
125 mul
$d0,$h0,$r0 // h0
*r0
129 mul
$t0,$h1,$s1 // h1
*5*r1
133 mul
$t0,$h0,$r1 // h0
*r1
138 mul
$t0,$h1,$r0 // h1
*r0
143 mul
$t0,$h2,$s1 // h2
*5*r1
145 mul
$t1,$h2,$r0 // h2
*r0
150 and $t0,$d2,#-4 // final reduction
152 add
$t0,$t0,$d2,lsr
#2
159 stp
$h0,$h1,[$ctx] // store hash value
164 .size poly1305_blocks
,.-poly1305_blocks
166 .type poly1305_emit
,%function
169 ldp
$h0,$h1,[$ctx] // load hash base
2^64
171 ldp
$t0,$t1,[$nonce] // load nonce
173 adds
$d0,$h0,#5 // compare to modulus
177 tst
$d2,#-4 // see if it's carried/borrowed
183 ror
$t0,$t0,#32 // flip nonce words
186 adds
$h0,$h0,$t0 // accumulate nonce
189 rev
$h0,$h0 // flip output bytes
192 stp
$h0,$h1,[$mac] // write result
195 .size poly1305_emit
,.-poly1305_emit
197 my ($R0,$R1,$S1,$R2,$S2,$R3,$S3,$R4,$S4) = map("v$_.4s",(0..8));
198 my ($IN01_0,$IN01_1,$IN01_2,$IN01_3,$IN01_4) = map("v$_.2s",(9..13));
199 my ($IN23_0,$IN23_1,$IN23_2,$IN23_3,$IN23_4) = map("v$_.2s",(14..18));
200 my ($ACC0,$ACC1,$ACC2,$ACC3,$ACC4) = map("v$_.2d",(19..23));
201 my ($H0,$H1,$H2,$H3,$H4) = map("v$_.2s",(24..28));
202 my ($T0,$T1,$MASK) = map("v$_",(29..31));
204 my ($in2,$zeros)=("x16","x17");
205 my $is_base2_26 = $zeros; # borrow
208 .type poly1305_mult
,%function
211 mul
$d0,$h0,$r0 // h0
*r0
214 mul
$t0,$h1,$s1 // h1
*5*r1
218 mul
$t0,$h0,$r1 // h0
*r1
223 mul
$t0,$h1,$r0 // h1
*r0
228 mul
$t0,$h2,$s1 // h2
*5*r1
230 mul
$t1,$h2,$r0 // h2
*r0
235 and $t0,$d2,#-4 // final reduction
237 add
$t0,$t0,$d2,lsr
#2
243 .size poly1305_mult
,.-poly1305_mult
245 .type poly1305_splat
,%function
248 and x12
,$h0,#0x03ffffff // base 2^64 -> base 2^26
251 and x14
,x14
,#0x03ffffff
255 str w12
,[$ctx,#16*0] // r0
256 add w12
,w13
,w13
,lsl
#2 // r1*5
257 str w13
,[$ctx,#16*1] // r1
258 add w13
,w14
,w14
,lsl
#2 // r2*5
259 str w12
,[$ctx,#16*2] // s1
260 str w14
,[$ctx,#16*3] // r2
261 add w14
,w15
,w15
,lsl
#2 // r3*5
262 str w13
,[$ctx,#16*4] // s2
263 str w15
,[$ctx,#16*5] // r3
264 add w15
,w16
,w16
,lsl
#2 // r4*5
265 str w14
,[$ctx,#16*6] // s3
266 str w16
,[$ctx,#16*7] // r4
267 str w15
,[$ctx,#16*8] // s4
270 .size poly1305_splat
,.-poly1305_splat
272 .type poly1305_blocks_neon
,%function
274 poly1305_blocks_neon
:
275 ldr
$is_base2_26,[$ctx,#24]
278 cbz
$is_base2_26,poly1305_blocks
281 stp x29
,x30
,[sp
,#-80]!
287 cbz
$is_base2_26,.Lbase2_64_neon
289 ldp w10
,w11
,[$ctx] // load hash value base
2^26
290 ldp w12
,w13
,[$ctx,#8]
296 ldp
$r0,$r1,[$ctx,#32] // load key value
298 add
$h0,x10
,x11
,lsl
#26 // base 2^26 -> base 2^64
300 adds
$h0,$h0,x12
,lsl
#52
301 add
$h1,$h1,x13
,lsl
#14
304 adds
$h1,$h1,x14
,lsl
#40
305 adc
$d2,$h2,xzr
// can be partially reduced
...
307 ldp
$d0,$d1,[$inp],#16 // load input
309 add
$s1,$r1,$r1,lsr
#2 // s1 = r1 + (r1 >> 2)
311 and $t0,$d2,#-4 // ... so reduce
313 add
$t0,$t0,$d2,lsr
#2
322 adds
$h0,$h0,$d0 // accumulate input
329 cbz
$padbit,.Lstore_base2_64_neon
331 and x10
,$h0,#0x03ffffff // base 2^64 -> base 2^26
334 and x12
,x12
,#0x03ffffff
338 cbnz
$len,.Leven_neon
340 stp w10
,w11
,[$ctx] // store hash value base
2^26
341 stp w12
,w13
,[$ctx,#8]
346 .Lstore_base2_64_neon
:
347 stp
$h0,$h1,[$ctx] // store hash value base
2^64
348 stp
$h2,xzr
,[$ctx,#16] // note that is_base2_26 is zeroed
353 ldp
$r0,$r1,[$ctx,#32] // load key value
355 ldp
$h0,$h1,[$ctx] // load hash value base
2^64
361 ldp
$d0,$d1,[$inp],#16 // load input
363 add
$s1,$r1,$r1,lsr
#2 // s1 = r1 + (r1 >> 2)
368 adds
$h0,$h0,$d0 // accumulate input
375 and x10
,$h0,#0x03ffffff // base 2^64 -> base 2^26
378 and x12
,x12
,#0x03ffffff
382 stp d8
,d9
,[sp
,#16] // meet ABI requirements
393 ////////////////////////////////// initialize r
^n table
395 add
$s1,$r1,$r1,lsr
#2 // s1 = r1 + (r1 >> 2)
401 bl poly1305_mult
// r
^2
405 bl poly1305_mult
// r
^3
409 bl poly1305_mult
// r
^4
417 csel
$in2,$zeros,$in2,lo
420 str x4
,[$ctx,#-24] // set is_base2_26
421 sub $ctx,$ctx,#48 // restore original $ctx
429 csel
$in2,$zeros,$in2,lo
431 stp d8
,d9
,[sp
,#16] // meet ABI requirements
443 ldp x8
,x12
,[$in2],#16 // inp[2:3] (or zero)
444 ldp x9
,x13
,[$in2],#48
446 lsl
$padbit,$padbit,#24
455 and x4
,x8
,#0x03ffffff // base 2^64 -> base 2^26
456 and x5
,x9
,#0x03ffffff
459 add x4
,x4
,x5
,lsl
#32 // bfi x4,x5,#32,#32
462 add x6
,x6
,x7
,lsl
#32 // bfi x6,x7,#32,#32
464 and x8
,x8
,#0x03ffffff
465 and x9
,x9
,#0x03ffffff
468 add x12
,$padbit,x12
,lsr
#40
469 add x13
,$padbit,x13
,lsr
#40
470 add x8
,x8
,x9
,lsl
#32 // bfi x8,x9,#32,#32
472 add x10
,x10
,x11
,lsl
#32 // bfi x10,x11,#32,#32
473 add x12
,x12
,x13
,lsl
#32 // bfi x12,x13,#32,#32
478 ldp x8
,x12
,[$inp],#16 // inp[0:1]
479 ldp x9
,x13
,[$inp],#48
481 ld1
{$R0,$R1,$S1,$R2},[x15
],#64
482 ld1
{$S2,$R3,$S3,$R4},[x15
],#64
491 and x4
,x8
,#0x03ffffff // base 2^64 -> base 2^26
492 and x5
,x9
,#0x03ffffff
495 add x4
,x4
,x5
,lsl
#32 // bfi x4,x5,#32,#32
498 add x6
,x6
,x7
,lsl
#32 // bfi x6,x7,#32,#32
500 and x8
,x8
,#0x03ffffff
501 and x9
,x9
,#0x03ffffff
504 add x12
,$padbit,x12
,lsr
#40
505 add x13
,$padbit,x13
,lsr
#40
506 add x8
,x8
,x9
,lsl
#32 // bfi x8,x9,#32,#32
508 add x10
,x10
,x11
,lsl
#32 // bfi x10,x11,#32,#32
509 add x12
,x12
,x13
,lsl
#32 // bfi x12,x13,#32,#32
514 ushr
$MASK.2d
,$MASK.2d
,#38
520 ////////////////////////////////////////////////////////////////
521 // ((inp
[0]*r
^4+inp
[2]*r
^2+inp
[4])*r
^4+inp
[6]*r
^2
522 // ((inp
[1]*r
^4+inp
[3]*r
^2+inp
[5])*r
^3+inp
[7]*r
523 // \___________________
/
524 // ((inp
[0]*r
^4+inp
[2]*r
^2+inp
[4])*r
^4+inp
[6]*r
^2+inp
[8])*r
^2
525 // ((inp
[1]*r
^4+inp
[3]*r
^2+inp
[5])*r
^4+inp
[7]*r
^2+inp
[9])*r
526 // \___________________
/ \____________________/
528 // Note that we start with inp
[2:3]*r
^2. This is because it
529 // doesn
't depend on reduction in previous iteration.
530 ////////////////////////////////////////////////////////////////
531 // d4 = h0*r4 + h1*r3 + h2*r2 + h3*r1 + h4*r0
532 // d3 = h0*r3 + h1*r2 + h2*r1 + h3*r0 + h4*5*r4
533 // d2 = h0*r2 + h1*r1 + h2*r0 + h3*5*r4 + h4*5*r3
534 // d1 = h0*r1 + h1*r0 + h2*5*r4 + h3*5*r3 + h4*5*r2
535 // d0 = h0*r0 + h1*5*r4 + h2*5*r3 + h3*5*r2 + h4*5*r1
538 umull $ACC4,$IN23_0,${R4}[2]
539 csel $in2,$zeros,$in2,lo
540 umull $ACC3,$IN23_0,${R3}[2]
541 umull $ACC2,$IN23_0,${R2}[2]
542 ldp x8,x12,[$in2],#16 // inp[2:3] (or zero)
543 umull $ACC1,$IN23_0,${R1}[2]
544 ldp x9,x13,[$in2],#48
545 umull $ACC0,$IN23_0,${R0}[2]
553 umlal $ACC4,$IN23_1,${R3}[2]
554 and x4,x8,#0x03ffffff // base 2^64 -> base 2^26
555 umlal $ACC3,$IN23_1,${R2}[2]
556 and x5,x9,#0x03ffffff
557 umlal $ACC2,$IN23_1,${R1}[2]
559 umlal $ACC1,$IN23_1,${R0}[2]
561 umlal $ACC0,$IN23_1,${S4}[2]
562 add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32
564 umlal $ACC4,$IN23_2,${R2}[2]
566 umlal $ACC3,$IN23_2,${R1}[2]
568 umlal $ACC2,$IN23_2,${R0}[2]
569 add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32
570 umlal $ACC1,$IN23_2,${S4}[2]
572 umlal $ACC0,$IN23_2,${S3}[2]
573 and x8,x8,#0x03ffffff
575 umlal $ACC4,$IN23_3,${R1}[2]
576 and x9,x9,#0x03ffffff
577 umlal $ACC3,$IN23_3,${R0}[2]
579 umlal $ACC2,$IN23_3,${S4}[2]
581 umlal $ACC1,$IN23_3,${S3}[2]
582 add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32
583 umlal $ACC0,$IN23_3,${S2}[2]
586 add $IN01_2,$IN01_2,$H2
587 add x12,$padbit,x12,lsr#40
588 umlal $ACC4,$IN23_4,${R0}[2]
589 add x13,$padbit,x13,lsr#40
590 umlal $ACC3,$IN23_4,${S4}[2]
591 add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32
592 umlal $ACC2,$IN23_4,${S3}[2]
593 add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32
594 umlal $ACC1,$IN23_4,${S2}[2]
596 umlal $ACC0,$IN23_4,${S1}[2]
599 ////////////////////////////////////////////////////////////////
600 // (hash+inp[0:1])*r^4 and accumulate
602 add $IN01_0,$IN01_0,$H0
604 umlal $ACC3,$IN01_2,${R1}[0]
605 ldp x8,x12,[$inp],#16 // inp[0:1]
606 umlal $ACC0,$IN01_2,${S3}[0]
607 ldp x9,x13,[$inp],#48
608 umlal $ACC4,$IN01_2,${R2}[0]
609 umlal $ACC1,$IN01_2,${S4}[0]
610 umlal $ACC2,$IN01_2,${R0}[0]
618 add $IN01_1,$IN01_1,$H1
619 umlal $ACC3,$IN01_0,${R3}[0]
620 umlal $ACC4,$IN01_0,${R4}[0]
621 and x4,x8,#0x03ffffff // base 2^64 -> base 2^26
622 umlal $ACC2,$IN01_0,${R2}[0]
623 and x5,x9,#0x03ffffff
624 umlal $ACC0,$IN01_0,${R0}[0]
626 umlal $ACC1,$IN01_0,${R1}[0]
629 add $IN01_3,$IN01_3,$H3
630 add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32
631 umlal $ACC3,$IN01_1,${R2}[0]
633 umlal $ACC4,$IN01_1,${R3}[0]
635 umlal $ACC0,$IN01_1,${S4}[0]
636 add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32
637 umlal $ACC2,$IN01_1,${R1}[0]
639 umlal $ACC1,$IN01_1,${R0}[0]
640 and x8,x8,#0x03ffffff
642 add $IN01_4,$IN01_4,$H4
643 and x9,x9,#0x03ffffff
644 umlal $ACC3,$IN01_3,${R0}[0]
646 umlal $ACC0,$IN01_3,${S2}[0]
648 umlal $ACC4,$IN01_3,${R1}[0]
649 add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32
650 umlal $ACC1,$IN01_3,${S3}[0]
652 umlal $ACC2,$IN01_3,${S4}[0]
653 add x12,$padbit,x12,lsr#40
655 umlal $ACC3,$IN01_4,${S4}[0]
656 add x13,$padbit,x13,lsr#40
657 umlal $ACC0,$IN01_4,${S1}[0]
658 add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32
659 umlal $ACC4,$IN01_4,${R0}[0]
660 add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32
661 umlal $ACC1,$IN01_4,${S2}[0]
663 umlal $ACC2,$IN01_4,${S3}[0]
667 /////////////////////////////////////////////////////////////////
668 // lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
671 // [see discussion in poly1305-armv4 module]
673 ushr $T0.2d,$ACC3,#26
675 ushr $T1.2d,$ACC0,#26
676 and $ACC0,$ACC0,$MASK.2d
677 add $ACC4,$ACC4,$T0.2d // h3 -> h4
678 bic $H3,#0xfc,lsl#24 // &=0x03ffffff
679 add $ACC1,$ACC1,$T1.2d // h0 -> h1
681 ushr $T0.2d,$ACC4,#26
683 ushr $T1.2d,$ACC1,#26
686 add $ACC2,$ACC2,$T1.2d // h1 -> h2
688 add $ACC0,$ACC0,$T0.2d
690 shrn $T1.2s,$ACC2,#26
692 add $ACC0,$ACC0,$T0.2d // h4 -> h0
694 add $H3,$H3,$T1.2s // h2 -> h3
697 shrn $T0.2s,$ACC0,#26
702 add $H1,$H1,$T0.2s // h0 -> h1
703 add $H4,$H4,$T1.2s // h3 -> h4
708 dup $IN23_2,${IN23_2}[0]
709 add $IN01_2,$IN01_2,$H2
711 ////////////////////////////////////////////////////////////////
712 // multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1
717 dup $IN23_2,${IN01_2}[0]
718 add $IN23_0,$IN01_0,$H0
719 add $IN23_3,$IN01_3,$H3
720 add $IN23_1,$IN01_1,$H1
721 add $IN23_4,$IN01_4,$H4
724 dup $IN23_0,${IN23_0}[0]
725 umull2 $ACC0,$IN23_2,${S3}
726 umull2 $ACC3,$IN23_2,${R1}
727 umull2 $ACC4,$IN23_2,${R2}
728 umull2 $ACC2,$IN23_2,${R0}
729 umull2 $ACC1,$IN23_2,${S4}
731 dup $IN23_1,${IN23_1}[0]
732 umlal2 $ACC0,$IN23_0,${R0}
733 umlal2 $ACC2,$IN23_0,${R2}
734 umlal2 $ACC3,$IN23_0,${R3}
735 umlal2 $ACC4,$IN23_0,${R4}
736 umlal2 $ACC1,$IN23_0,${R1}
738 dup $IN23_3,${IN23_3}[0]
739 umlal2 $ACC0,$IN23_1,${S4}
740 umlal2 $ACC3,$IN23_1,${R2}
741 umlal2 $ACC2,$IN23_1,${R1}
742 umlal2 $ACC4,$IN23_1,${R3}
743 umlal2 $ACC1,$IN23_1,${R0}
745 dup $IN23_4,${IN23_4}[0]
746 umlal2 $ACC3,$IN23_3,${R0}
747 umlal2 $ACC4,$IN23_3,${R1}
748 umlal2 $ACC0,$IN23_3,${S2}
749 umlal2 $ACC1,$IN23_3,${S3}
750 umlal2 $ACC2,$IN23_3,${S4}
752 umlal2 $ACC3,$IN23_4,${S4}
753 umlal2 $ACC0,$IN23_4,${S1}
754 umlal2 $ACC4,$IN23_4,${R0}
755 umlal2 $ACC1,$IN23_4,${S2}
756 umlal2 $ACC2,$IN23_4,${S3}
760 ////////////////////////////////////////////////////////////////
761 // (hash+inp[0:1])*r^4:r^3 and accumulate
763 add $IN01_0,$IN01_0,$H0
764 umlal $ACC3,$IN01_2,${R1}
765 umlal $ACC0,$IN01_2,${S3}
766 umlal $ACC4,$IN01_2,${R2}
767 umlal $ACC1,$IN01_2,${S4}
768 umlal $ACC2,$IN01_2,${R0}
770 add $IN01_1,$IN01_1,$H1
771 umlal $ACC3,$IN01_0,${R3}
772 umlal $ACC0,$IN01_0,${R0}
773 umlal $ACC4,$IN01_0,${R4}
774 umlal $ACC1,$IN01_0,${R1}
775 umlal $ACC2,$IN01_0,${R2}
777 add $IN01_3,$IN01_3,$H3
778 umlal $ACC3,$IN01_1,${R2}
779 umlal $ACC0,$IN01_1,${S4}
780 umlal $ACC4,$IN01_1,${R3}
781 umlal $ACC1,$IN01_1,${R0}
782 umlal $ACC2,$IN01_1,${R1}
784 add $IN01_4,$IN01_4,$H4
785 umlal $ACC3,$IN01_3,${R0}
786 umlal $ACC0,$IN01_3,${S2}
787 umlal $ACC4,$IN01_3,${R1}
788 umlal $ACC1,$IN01_3,${S3}
789 umlal $ACC2,$IN01_3,${S4}
791 umlal $ACC3,$IN01_4,${S4}
792 umlal $ACC0,$IN01_4,${S1}
793 umlal $ACC4,$IN01_4,${R0}
794 umlal $ACC1,$IN01_4,${S2}
795 umlal $ACC2,$IN01_4,${S3}
798 ////////////////////////////////////////////////////////////////
801 addp $ACC3,$ACC3,$ACC3
802 ldp d8,d9,[sp,#16] // meet ABI requirements
803 addp $ACC0,$ACC0,$ACC0
805 addp $ACC4,$ACC4,$ACC4
807 addp $ACC1,$ACC1,$ACC1
809 addp $ACC2,$ACC2,$ACC2
811 ////////////////////////////////////////////////////////////////
812 // lazy reduction, but without narrowing
814 ushr $T0.2d,$ACC3,#26
815 and $ACC3,$ACC3,$MASK.2d
816 ushr $T1.2d,$ACC0,#26
817 and $ACC0,$ACC0,$MASK.2d
819 add $ACC4,$ACC4,$T0.2d // h3 -> h4
820 add $ACC1,$ACC1,$T1.2d // h0 -> h1
822 ushr $T0.2d,$ACC4,#26
823 and $ACC4,$ACC4,$MASK.2d
824 ushr $T1.2d,$ACC1,#26
825 and $ACC1,$ACC1,$MASK.2d
826 add $ACC2,$ACC2,$T1.2d // h1 -> h2
828 add $ACC0,$ACC0,$T0.2d
830 ushr $T1.2d,$ACC2,#26
831 and $ACC2,$ACC2,$MASK.2d
832 add $ACC0,$ACC0,$T0.2d // h4 -> h0
833 add $ACC3,$ACC3,$T1.2d // h2 -> h3
835 ushr $T0.2d,$ACC0,#26
836 and $ACC0,$ACC0,$MASK.2d
837 ushr $T1.2d,$ACC3,#26
838 and $ACC3,$ACC3,$MASK.2d
839 add $ACC1,$ACC1,$T0.2d // h0 -> h1
840 add $ACC4,$ACC4,$T1.2d // h3 -> h4
842 ////////////////////////////////////////////////////////////////
843 // write the result, can be partially reduced
845 st4 {$ACC0,$ACC1,$ACC2,$ACC3}[0],[$ctx],#16
846 st1 {$ACC4}[0],[$ctx]
851 .size poly1305_blocks_neon,.-poly1305_blocks_neon
853 .type poly1305_emit_neon,%function
856 ldr $is_base2_26,[$ctx,#24]
857 cbz $is_base2_26,poly1305_emit
859 ldp w10,w11,[$ctx] // load hash value base 2^26
860 ldp w12,w13,[$ctx,#8]
863 add $h0,x10,x11,lsl#26 // base 2^26 -> base 2^64
865 adds $h0,$h0,x12,lsl#52
866 add $h1,$h1,x13,lsl#14
869 adds $h1,$h1,x14,lsl#40
870 adc $h2,$h2,xzr // can be partially reduced...
872 ldp $t0,$t1,[$nonce] // load nonce
874 and $d0,$h2,#-4 // ... so reduce
875 add $d0,$d0,$h2,lsr#2
881 adds $d0,$h0,#5 // compare to modulus
885 tst $d2,#-4 // see if it's carried
/borrowed
891 ror
$t0,$t0,#32 // flip nonce words
894 adds
$h0,$h0,$t0 // accumulate nonce
897 rev
$h0,$h0 // flip output bytes
900 stp
$h0,$h1,[$mac] // write result
903 .size poly1305_emit_neon
,.-poly1305_emit_neon
907 .long
0,0,0,0,0,0,0,0
910 .long OPENSSL_armcap_P
-.
912 .quad OPENSSL_armcap_P
-.
914 .asciz
"Poly1305 for ARMv8, CRYPTOGAMS by <appro\@openssl.org>"
918 foreach (split("\n",$code)) {
919 s/\b(shrn\s+v[0-9]+)\.[24]d/$1.2s/ or
920 s/\b(fmov\s+)v([0-9]+)[^,]*,\s*x([0-9]+)/$1d$2,x$3/ or
921 (m/\bdup\b/ and (s/\.[24]s/.2d/g or 1)) or
922 (m/\b(eor|and)/ and (s/\.[248][sdh]/.16b/g or 1)) or
923 (m/\bum(ul|la)l\b/ and (s/\.4s/.2s/g or 1)) or
924 (m/\bum(ul|la)l2\b/ and (s/\.2s/.4s/g or 1)) or
925 (m/\bst[1-4]\s+{[^}]+}\[/ and (s/\.[24]d/.s/g or 1));
927 s/\.[124]([sd])\[/.$1\[/;