3 # ====================================================================
4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
5 # project. The module is, however, dual licensed under OpenSSL and
6 # CRYPTOGAMS licenses depending on where you obtain it. For further
7 # details see http://www.openssl.org/~appro/cryptogams/.
8 # ====================================================================
10 # IALU(*)/gcc-4.4 NEON
12 # ARM11xx(ARMv6) 7.78/+100% -
13 # Cortex-A5 6.35/+130% 2.96
14 # Cortex-A8 6.25/+115% 2.36
15 # Cortex-A9 5.10/+95% 2.55
16 # Cortex-A15 3.85/+85% 1.25(**)
17 # Snapdragon S4 5.70/+100% 1.48(**)
19 # (*) this is for -march=armv6, i.e. with bunch of ldrb loading data;
20 # (**) these are trade-off results, they can be improved by ~8% but at
21 # the cost of 15/12% regression on Cortex-A5/A7, it's even possible
22 # to improve Cortex-A9 result, but then A5/A7 loose more than 20%;
25 if ($flavour=~/\w[\w\-]*\.\w+$/) { $output=$flavour; undef $flavour; }
26 else { while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {} }
28 if ($flavour && $flavour ne "void") {
29 $0 =~ m/(.*[\/\\])[^\
/\\]+$/; $dir=$1;
30 ( $xlate="${dir}arm-xlate.pl" and -f
$xlate ) or
31 ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f
$xlate) or
32 die "can't locate arm-xlate.pl";
34 open STDOUT
,"| \"$^X\" $xlate $flavour $output";
36 open STDOUT
,">$output";
39 ($ctx,$inp,$len,$padbit)=map("r$_",(0..3));
45 #if defined(__thumb2__)
53 .globl poly1305_blocks
55 .type poly1305_init
,%function
63 str r3
,[$ctx,#0] @ zero hash value
68 str r3
,[$ctx,#36] @ is_base2_26
77 #if __ARM_MAX_ARCH__>=7
78 adr r11
,.Lpoly1305_init
79 ldr r12
,.LOPENSSL_armcap
84 and r3
,r10
,#-4 @ 0x0ffffffc
95 #if __ARM_MAX_ARCH__>=7
96 ldr r12
,[r11
,r12
] @ OPENSSL_armcap_P
110 #if __ARM_MAX_ARCH__>=7
111 tst r12
,#ARMV7_NEON @ check for NEON
113 adr r9
,poly1305_blocks_neon
114 adr r11
,poly1305_blocks
119 adr r12
,poly1305_emit
120 adr r10
,poly1305_emit_neon
129 addeq r12
,r11
,#(poly1305_emit-.Lpoly1305_init)
130 addne r12
,r11
,#(poly1305_emit_neon-.Lpoly1305_init)
131 addeq r11
,r11
,#(poly1305_blocks-.Lpoly1305_init)
132 addne r11
,r11
,#(poly1305_blocks_neon-.Lpoly1305_init)
135 orr r12
,r12
,#1 @ thumb-ify address
157 #if __ARM_MAX_ARCH__>=7
158 stmia r2
,{r11
,r12
} @ fill functions table
169 moveq pc
,lr @ be binary compatible with V4
, yet
170 bx lr @ interoperable with Thumb ISA
:-)
172 .size poly1305_init
,.-poly1305_init
175 my ($h0,$h1,$h2,$h3,$h4,$r0,$r1,$r2,$r3)=map("r$_",(4..12));
176 my ($s1,$s2,$s3)=($r1,$r2,$r3);
179 .type poly1305_blocks
,%function
182 stmdb sp
!,{r3
-r11
,lr
}
188 add
$len,$len,$inp @ end pointer
191 ldmia
$ctx,{$h0-$r3} @ load context
193 str
$ctx,[sp
,#12] @ offload stuff
203 ldrb r0
,[lr
],#16 @ load input
207 addhi
$h4,$h4,#1 @ 1<<128
217 adds
$h0,$h0,r3 @ accumulate input
239 str lr
,[sp
,#8] @ offload input pointer
241 add
$s1,$r1,$r1,lsr
#2
244 ldr r0
,[lr
],#16 @ load input
248 addhi
$h4,$h4,#1 @ padbit
258 adds
$h0,$h0,r0 @ accumulate input
259 str lr
,[sp
,#8] @ offload input pointer
261 add
$s1,$r1,$r1,lsr
#2
264 add
$s2,$r2,$r2,lsr
#2
266 add
$s3,$r3,$r3,lsr
#2
273 ldr
$r1,[sp
,#20] @ reload $r1
279 str r0
,[sp
,#0] @ future $h0
281 ldr
$r2,[sp
,#24] @ reload $r2
282 adds r2
,r2
,r1 @ d1
+=d0
>>32
284 adc lr
,r3
,#0 @ future $h2
285 str r2
,[sp
,#4] @ future $h1
290 ldr
$r3,[sp
,#28] @ reload $r3
302 adds
$h2,lr
,r0 @ d2
+=d1
>>32
303 ldr lr
,[sp
,#8] @ reload input pointer
305 adds
$h3,r2
,r1 @ d3
+=d2
>>32
306 ldr r0
,[sp
,#16] @ reload end pointer
308 add
$h4,$h4,r3 @ h4
+=d3
>>32
312 add r1
,r1
,r1
,lsr
#2 @ *=5
319 cmp r0
,lr @ done yet?
324 stmia
$ctx,{$h0-$h4} @ store the result
328 ldmia sp
!,{r3
-r11
,pc
}
330 ldmia sp
!,{r3
-r11
,lr
}
332 moveq pc
,lr @ be binary compatible with V4
, yet
333 bx lr @ interoperable with Thumb ISA
:-)
335 .size poly1305_blocks
,.-poly1305_blocks
339 my ($ctx,$mac,$nonce)=map("r$_",(0..2));
340 my ($h0,$h1,$h2,$h3,$h4,$g0,$g1,$g2,$g3)=map("r$_",(3..11));
344 .type poly1305_emit
,%function
348 .Lpoly1305_emit_enter
:
351 adds
$g0,$h0,#5 @ compare to modulus
356 tst
$g4,#4 @ did it carry/borrow?
433 moveq pc
,lr @ be binary compatible with V4
, yet
434 bx lr @ interoperable with Thumb ISA
:-)
436 .size poly1305_emit
,.-poly1305_emit
439 my ($R0,$R1,$S1,$R2,$S2,$R3,$S3,$R4,$S4) = map("d$_",(0..9));
440 my ($D0,$D1,$D2,$D3,$D4, $H0,$H1,$H2,$H3,$H4) = map("q$_",(5..14));
441 my ($T0,$T1,$MASK) = map("q$_",(15,4,0));
443 my ($in2,$zeros,$tbl0,$tbl1) = map("r$_",(4..7));
446 #if __ARM_MAX_ARCH__>=7
449 .type poly1305_init_neon
,%function
452 ldr r4
,[$ctx,#20] @ load key base 2^32
457 and r2
,r4
,#0x03ffffff @ base 2^32 -> base 2^26
465 and r3
,r3
,#0x03ffffff
466 and r4
,r4
,#0x03ffffff
467 and r5
,r5
,#0x03ffffff
469 vdup
.32 $R0,r2 @ r
^1 in both lanes
470 add r2
,r3
,r3
,lsl
#2 @ *5
483 mov
$zeros,#2 @ counter
486 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
487 @ d0
= h0
*r0
+ h4
*5*r1
+ h3
*5*r2
+ h2
*5*r3
+ h1
*5*r4
488 @ d1
= h1
*r0
+ h0
*r1
+ h4
*5*r2
+ h3
*5*r3
+ h2
*5*r4
489 @ d2
= h2
*r0
+ h1
*r1
+ h0
*r2
+ h4
*5*r3
+ h3
*5*r4
490 @ d3
= h3
*r0
+ h2
*r1
+ h1
*r2
+ h0
*r3
+ h4
*5*r4
491 @ d4
= h4
*r0
+ h3
*r1
+ h2
*r2
+ h1
*r3
+ h0
*r4
493 vmull
.u32
$D0,$R0,${R0
}[1]
494 vmull
.u32
$D1,$R1,${R0
}[1]
495 vmull
.u32
$D2,$R2,${R0
}[1]
496 vmull
.u32
$D3,$R3,${R0
}[1]
497 vmull
.u32
$D4,$R4,${R0
}[1]
499 vmlal
.u32
$D0,$R4,${S1
}[1]
500 vmlal
.u32
$D1,$R0,${R1
}[1]
501 vmlal
.u32
$D2,$R1,${R1
}[1]
502 vmlal
.u32
$D3,$R2,${R1
}[1]
503 vmlal
.u32
$D4,$R3,${R1
}[1]
505 vmlal
.u32
$D0,$R3,${S2
}[1]
506 vmlal
.u32
$D1,$R4,${S2
}[1]
507 vmlal
.u32
$D3,$R1,${R2
}[1]
508 vmlal
.u32
$D2,$R0,${R2
}[1]
509 vmlal
.u32
$D4,$R2,${R2
}[1]
511 vmlal
.u32
$D0,$R2,${S3
}[1]
512 vmlal
.u32
$D3,$R0,${R3
}[1]
513 vmlal
.u32
$D1,$R3,${S3
}[1]
514 vmlal
.u32
$D2,$R4,${S3
}[1]
515 vmlal
.u32
$D4,$R1,${R3
}[1]
517 vmlal
.u32
$D3,$R4,${S4
}[1]
518 vmlal
.u32
$D0,$R1,${S4
}[1]
519 vmlal
.u32
$D1,$R2,${S4
}[1]
520 vmlal
.u32
$D2,$R3,${S4
}[1]
521 vmlal
.u32
$D4,$R0,${R4
}[1]
523 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
524 @ lazy reduction as discussed
in "NEON crypto" by D
.J
. Bernstein
531 vadd
.i64
$D4,$D4,$T0 @ h3
-> h4
532 vbic
.i32
$D3#lo,#0xfc000000 @ &=0x03ffffff
533 vadd
.i64
$D1,$D1,$T1 @ h0
-> h1
534 vbic
.i32
$D0#lo,#0xfc000000
536 vshrn
.u64
$T0#lo,$D4,#26
540 vadd
.i64
$D2,$D2,$T1 @ h1
-> h2
541 vbic
.i32
$D4#lo,#0xfc000000
542 vbic
.i32
$D1#lo,#0xfc000000
544 vadd
.i32
$D0#lo,$D0#lo,$T0#lo
545 vshl
.u32
$T0#lo,$T0#lo,#2
546 vshrn
.u64
$T1#lo,$D2,#26
548 vadd
.i32
$D0#lo,$D0#lo,$T0#lo @ h4 -> h0
549 vadd
.i32
$D3#lo,$D3#lo,$T1#lo @ h2 -> h3
550 vbic
.i32
$D2#lo,#0xfc000000
552 vshr
.u32
$T0#lo,$D0#lo,#26
553 vbic
.i32
$D0#lo,#0xfc000000
554 vshr
.u32
$T1#lo,$D3#lo,#26
555 vbic
.i32
$D3#lo,#0xfc000000
556 vadd
.i32
$D1#lo,$D1#lo,$T0#lo @ h0 -> h1
557 vadd
.i32
$D4#lo,$D4#lo,$T1#lo @ h3 -> h4
559 subs
$zeros,$zeros,#1
560 beq
.Lsquare_break_neon
562 add
$tbl0,$ctx,#(48+0*9*4)
563 add
$tbl1,$ctx,#(48+1*9*4)
565 vtrn
.32 $R0,$D0#lo @ r^2:r^1
571 vshl
.u32
$S2,$R2,#2 @ *5
580 vst4
.32
{${R0
}[0],${R1
}[0],${S1
}[0],${R2
}[0]},[$tbl0]!
581 vst4
.32
{${R0
}[1],${R1
}[1],${S1
}[1],${R2
}[1]},[$tbl1]!
582 vst4
.32
{${S2
}[0],${R3
}[0],${S3
}[0],${R4
}[0]},[$tbl0]!
583 vst4
.32
{${S2
}[1],${R3
}[1],${S3
}[1],${R4
}[1]},[$tbl1]!
584 vst1
.32
{${S4
}[0]},[$tbl0,:32]
585 vst1
.32
{${S4
}[1]},[$tbl1,:32]
591 add
$tbl0,$ctx,#(48+2*4*9)
592 add
$tbl1,$ctx,#(48+3*4*9)
594 vmov
$R0,$D0#lo @ r^4:r^3
595 vshl
.u32
$S1,$D1#lo,#2 @ *5
597 vshl
.u32
$S2,$D2#lo,#2
599 vshl
.u32
$S3,$D3#lo,#2
601 vshl
.u32
$S4,$D4#lo,#2
603 vadd
.i32
$S1,$S1,$D1#lo
604 vadd
.i32
$S2,$S2,$D2#lo
605 vadd
.i32
$S3,$S3,$D3#lo
606 vadd
.i32
$S4,$S4,$D4#lo
608 vst4
.32
{${R0
}[0],${R1
}[0],${S1
}[0],${R2
}[0]},[$tbl0]!
609 vst4
.32
{${R0
}[1],${R1
}[1],${S1
}[1],${R2
}[1]},[$tbl1]!
610 vst4
.32
{${S2
}[0],${R3
}[0],${S3
}[0],${R4
}[0]},[$tbl0]!
611 vst4
.32
{${S2
}[1],${R3
}[1],${S3
}[1],${R4
}[1]},[$tbl1]!
612 vst1
.32
{${S4
}[0]},[$tbl0]
613 vst1
.32
{${S4
}[1]},[$tbl1]
616 .size poly1305_init_neon
,.-poly1305_init_neon
618 .type poly1305_blocks_neon
,%function
620 poly1305_blocks_neon
:
621 ldr ip
,[$ctx,#36] @ is_base2_26
627 tst ip
,ip @ is_base2_26?
632 vstmdb sp
!,{d8
-d15
} @ ABI specification says so
634 tst ip
,ip @ is_base2_26?
638 bl poly1305_init_neon
640 ldr r4
,[$ctx,#0] @ load hash value base 2^32
646 and r2
,r4
,#0x03ffffff @ base 2^32 -> base 2^26
648 veor
$D0#lo,$D0#lo,$D0#lo
651 veor
$D1#lo,$D1#lo,$D1#lo
654 veor
$D2#lo,$D2#lo,$D2#lo
657 veor
$D3#lo,$D3#lo,$D3#lo
658 and r3
,r3
,#0x03ffffff
660 veor
$D4#lo,$D4#lo,$D4#lo
661 and r4
,r4
,#0x03ffffff
663 and r5
,r5
,#0x03ffffff
664 str r1
,[$ctx,#36] @ is_base2_26
678 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
681 veor
$D0#lo,$D0#lo,$D0#lo
682 veor
$D1#lo,$D1#lo,$D1#lo
683 veor
$D2#lo,$D2#lo,$D2#lo
684 veor
$D3#lo,$D3#lo,$D3#lo
685 veor
$D4#lo,$D4#lo,$D4#lo
686 vld4
.32
{$D0#lo[0],$D1#lo[0],$D2#lo[0],$D3#lo[0]},[$ctx]!
688 vld1
.32
{$D4#lo[0]},[$ctx]
689 sub $ctx,$ctx,#16 @ rewind
693 mov
$padbit,$padbit,lsl
#24
697 vld4
.32
{$H0#lo[0],$H1#lo[0],$H2#lo[0],$H3#lo[0]},[$inp]!
698 vmov
.32 $H4#lo[0],$padbit
708 vsri
.u32
$H4#lo,$H3#lo,#8 @ base 2^32 -> base 2^26
709 vshl
.u32
$H3#lo,$H3#lo,#18
711 vsri
.u32
$H3#lo,$H2#lo,#14
712 vshl
.u32
$H2#lo,$H2#lo,#12
713 vadd
.i32
$H4#hi,$H4#lo,$D4#lo @ add hash value and move to #hi
715 vbic
.i32
$H3#lo,#0xfc000000
716 vsri
.u32
$H2#lo,$H1#lo,#20
717 vshl
.u32
$H1#lo,$H1#lo,#6
719 vbic
.i32
$H2#lo,#0xfc000000
720 vsri
.u32
$H1#lo,$H0#lo,#26
721 vadd
.i32
$H3#hi,$H3#lo,$D3#lo
723 vbic
.i32
$H0#lo,#0xfc000000
724 vbic
.i32
$H1#lo,#0xfc000000
725 vadd
.i32
$H2#hi,$H2#lo,$D2#lo
727 vadd
.i32
$H0#hi,$H0#lo,$D0#lo
728 vadd
.i32
$H1#hi,$H1#lo,$D1#lo
742 vmov
.i32
$H4,#1<<24 @ padbit, yes, always
743 vld4
.32
{$H0#lo,$H1#lo,$H2#lo,$H3#lo},[$inp] @ inp[0:1]
745 vld4
.32
{$H0#hi,$H1#hi,$H2#hi,$H3#hi},[$in2] @ inp[2:3] (or 0)
748 addhi
$tbl1,$ctx,#(48+1*9*4)
749 addhi
$tbl0,$ctx,#(48+3*9*4)
757 vsri
.u32
$H4,$H3,#8 @ base 2^32 -> base 2^26
763 vbic
.i32
$H3,#0xfc000000
767 vbic
.i32
$H2,#0xfc000000
770 vbic
.i32
$H0,#0xfc000000
771 vbic
.i32
$H1,#0xfc000000
775 vld4
.32
{${R0
}[1],${R1
}[1],${S1
}[1],${R2
}[1]},[$tbl1]! @ load r
^2
776 vld4
.32
{${R0
}[0],${R1
}[0],${S1
}[0],${R2
}[0]},[$tbl0]! @ load r
^4
777 vld4
.32
{${S2
}[1],${R3
}[1],${S3
}[1],${R4
}[1]},[$tbl1]!
778 vld4
.32
{${S2
}[0],${R3
}[0],${S3
}[0],${R4
}[0]},[$tbl0]!
783 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
784 @
((inp
[0]*r
^4+inp
[2]*r
^2+inp
[4])*r
^4+inp
[6]*r
^2
785 @
((inp
[1]*r
^4+inp
[3]*r
^2+inp
[5])*r
^3+inp
[7]*r
786 @ \___________________
/
787 @
((inp
[0]*r
^4+inp
[2]*r
^2+inp
[4])*r
^4+inp
[6]*r
^2+inp
[8])*r
^2
788 @
((inp
[1]*r
^4+inp
[3]*r
^2+inp
[5])*r
^4+inp
[7]*r
^2+inp
[9])*r
789 @ \___________________
/ \____________________/
791 @ Note that we start with inp
[2:3]*r
^2. This is because it
792 @ doesn
't depend on reduction in previous iteration.
793 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
794 @ d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
795 @ d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
796 @ d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
797 @ d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
798 @ d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
800 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
803 vadd.i32 $H2#lo,$H2#lo,$D2#lo @ accumulate inp[0:1]
804 vmull.u32 $D2,$H2#hi,${R0}[1]
805 vadd.i32 $H0#lo,$H0#lo,$D0#lo
806 vmull.u32 $D0,$H0#hi,${R0}[1]
807 vadd.i32 $H3#lo,$H3#lo,$D3#lo
808 vmull.u32 $D3,$H3#hi,${R0}[1]
809 vmlal.u32 $D2,$H1#hi,${R1}[1]
810 vadd.i32 $H1#lo,$H1#lo,$D1#lo
811 vmull.u32 $D1,$H1#hi,${R0}[1]
813 vadd.i32 $H4#lo,$H4#lo,$D4#lo
814 vmull.u32 $D4,$H4#hi,${R0}[1]
816 vmlal.u32 $D0,$H4#hi,${S1}[1]
819 vmlal.u32 $D3,$H2#hi,${R1}[1]
820 vld1.32 ${S4}[1],[$tbl1,:32]
821 vmlal.u32 $D1,$H0#hi,${R1}[1]
822 vmlal.u32 $D4,$H3#hi,${R1}[1]
824 vmlal.u32 $D0,$H3#hi,${S2}[1]
825 vmlal.u32 $D3,$H1#hi,${R2}[1]
826 vmlal.u32 $D4,$H2#hi,${R2}[1]
827 vmlal.u32 $D1,$H4#hi,${S2}[1]
828 vmlal.u32 $D2,$H0#hi,${R2}[1]
830 vmlal.u32 $D3,$H0#hi,${R3}[1]
831 vmlal.u32 $D0,$H2#hi,${S3}[1]
832 vmlal.u32 $D4,$H1#hi,${R3}[1]
833 vmlal.u32 $D1,$H3#hi,${S3}[1]
834 vmlal.u32 $D2,$H4#hi,${S3}[1]
836 vmlal.u32 $D3,$H4#hi,${S4}[1]
837 vmlal.u32 $D0,$H1#hi,${S4}[1]
838 vmlal.u32 $D4,$H0#hi,${R4}[1]
839 vmlal.u32 $D1,$H2#hi,${S4}[1]
840 vmlal.u32 $D2,$H3#hi,${S4}[1]
842 vld4.32 {$H0#hi,$H1#hi,$H2#hi,$H3#hi},[$in2] @ inp[2:3] (or 0)
845 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
846 @ (hash+inp[0:1])*r^4 and accumulate
848 vmlal.u32 $D3,$H3#lo,${R0}[0]
849 vmlal.u32 $D0,$H0#lo,${R0}[0]
850 vmlal.u32 $D4,$H4#lo,${R0}[0]
851 vmlal.u32 $D1,$H1#lo,${R0}[0]
852 vmlal.u32 $D2,$H2#lo,${R0}[0]
853 vld1.32 ${S4}[0],[$tbl0,:32]
855 vmlal.u32 $D3,$H2#lo,${R1}[0]
856 vmlal.u32 $D0,$H4#lo,${S1}[0]
857 vmlal.u32 $D4,$H3#lo,${R1}[0]
858 vmlal.u32 $D1,$H0#lo,${R1}[0]
859 vmlal.u32 $D2,$H1#lo,${R1}[0]
861 vmlal.u32 $D3,$H1#lo,${R2}[0]
862 vmlal.u32 $D0,$H3#lo,${S2}[0]
863 vmlal.u32 $D4,$H2#lo,${R2}[0]
864 vmlal.u32 $D1,$H4#lo,${S2}[0]
865 vmlal.u32 $D2,$H0#lo,${R2}[0]
867 vmlal.u32 $D3,$H0#lo,${R3}[0]
868 vmlal.u32 $D0,$H2#lo,${S3}[0]
869 vmlal.u32 $D4,$H1#lo,${R3}[0]
870 vmlal.u32 $D1,$H3#lo,${S3}[0]
871 vmlal.u32 $D3,$H4#lo,${S4}[0]
873 vmlal.u32 $D2,$H4#lo,${S3}[0]
874 vmlal.u32 $D0,$H1#lo,${S4}[0]
875 vmlal.u32 $D4,$H0#lo,${R4}[0]
876 vmov.i32 $H4,#1<<24 @ padbit, yes, always
877 vmlal.u32 $D1,$H2#lo,${S4}[0]
878 vmlal.u32 $D2,$H3#lo,${S4}[0]
880 vld4.32 {$H0#lo,$H1#lo,$H2#lo,$H3#lo},[$inp] @ inp[0:1]
889 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
890 @ lazy reduction interleaved with base 2^32 -> base 2^26
896 vadd.i64 $D4,$D4,$T0 @ h3 -> h4
897 vbic.i32 $D3#lo,#0xfc000000
898 vsri.u32 $H4,$H3,#8 @ base 2^32 -> base 2^26
899 vadd.i64 $D1,$D1,$T1 @ h0 -> h1
901 vbic.i32 $D0#lo,#0xfc000000
903 vshrn.u64 $T0#lo,$D4,#26
907 vadd.i64 $D2,$D2,$T1 @ h1 -> h2
909 vbic.i32 $D4#lo,#0xfc000000
911 vbic.i32 $D1#lo,#0xfc000000
913 vadd.i32 $D0#lo,$D0#lo,$T0#lo
914 vshl.u32 $T0#lo,$T0#lo,#2
915 vbic.i32 $H3,#0xfc000000
916 vshrn.u64 $T1#lo,$D2,#26
918 vadd.i32 $D0#lo,$D0#lo,$T0#lo @ h4 -> h0
920 vadd.i32 $D3#lo,$D3#lo,$T1#lo @ h2 -> h3
922 vbic.i32 $D2#lo,#0xfc000000
923 vbic.i32 $H2,#0xfc000000
925 vshr.u32 $T0#lo,$D0#lo,#26
926 vbic.i32 $D0#lo,#0xfc000000
928 vbic.i32 $H0,#0xfc000000
929 vshr.u32 $T1#lo,$D3#lo,#26
930 vbic.i32 $D3#lo,#0xfc000000
931 vadd.i32 $D1#lo,$D1#lo,$T0#lo @ h0 -> h1
932 vadd.i32 $D4#lo,$D4#lo,$T1#lo @ h3 -> h4
933 vbic.i32 $H1,#0xfc000000
938 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
939 @ multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1
941 add $tbl1,$ctx,#(48+0*9*4)
942 add $tbl0,$ctx,#(48+1*9*4)
948 vadd.i32 $H2#hi,$H2#lo,$D2#lo @ add hash value and move to #hi
949 vadd.i32 $H0#hi,$H0#lo,$D0#lo
950 vadd.i32 $H3#hi,$H3#lo,$D3#lo
951 vadd.i32 $H1#hi,$H1#lo,$D1#lo
952 vadd.i32 $H4#hi,$H4#lo,$D4#lo
955 vld4.32 {${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]! @ load r^1
956 vld4.32 {${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]! @ load r^2
958 vadd.i32 $H2#lo,$H2#lo,$D2#lo @ can be redundant
959 vmull.u32 $D2,$H2#hi,$R0
960 vadd.i32 $H0#lo,$H0#lo,$D0#lo
961 vmull.u32 $D0,$H0#hi,$R0
962 vadd.i32 $H3#lo,$H3#lo,$D3#lo
963 vmull.u32 $D3,$H3#hi,$R0
964 vadd.i32 $H1#lo,$H1#lo,$D1#lo
965 vmull.u32 $D1,$H1#hi,$R0
966 vadd.i32 $H4#lo,$H4#lo,$D4#lo
967 vmull.u32 $D4,$H4#hi,$R0
969 vmlal.u32 $D0,$H4#hi,$S1
970 vld4.32 {${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]!
971 vmlal.u32 $D3,$H2#hi,$R1
972 vld4.32 {${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]!
973 vmlal.u32 $D1,$H0#hi,$R1
974 vmlal.u32 $D4,$H3#hi,$R1
975 vmlal.u32 $D2,$H1#hi,$R1
977 vmlal.u32 $D3,$H1#hi,$R2
978 vld1.32 ${S4}[1],[$tbl1,:32]
979 vmlal.u32 $D0,$H3#hi,$S2
980 vld1.32 ${S4}[0],[$tbl0,:32]
981 vmlal.u32 $D4,$H2#hi,$R2
982 vmlal.u32 $D1,$H4#hi,$S2
983 vmlal.u32 $D2,$H0#hi,$R2
985 vmlal.u32 $D3,$H0#hi,$R3
987 addne $tbl1,$ctx,#(48+2*9*4)
988 vmlal.u32 $D0,$H2#hi,$S3
990 addne $tbl0,$ctx,#(48+3*9*4)
991 vmlal.u32 $D4,$H1#hi,$R3
992 vmlal.u32 $D1,$H3#hi,$S3
993 vmlal.u32 $D2,$H4#hi,$S3
995 vmlal.u32 $D3,$H4#hi,$S4
996 vorn $MASK,$MASK,$MASK @ all-ones, can be redundant
997 vmlal.u32 $D0,$H1#hi,$S4
998 vshr.u64 $MASK,$MASK,#38
999 vmlal.u32 $D4,$H0#hi,$R4
1000 vmlal.u32 $D1,$H2#hi,$S4
1001 vmlal.u32 $D2,$H3#hi,$S4
1005 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
1006 @ (hash+inp[0:1])*r^4:r^3 and accumulate
1008 vld4.32 {${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]! @ load r^3
1009 vld4.32 {${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]! @ load r^4
1011 vmlal.u32 $D2,$H2#lo,$R0
1012 vmlal.u32 $D0,$H0#lo,$R0
1013 vmlal.u32 $D3,$H3#lo,$R0
1014 vmlal.u32 $D1,$H1#lo,$R0
1015 vmlal.u32 $D4,$H4#lo,$R0
1017 vmlal.u32 $D0,$H4#lo,$S1
1018 vld4.32 {${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]!
1019 vmlal.u32 $D3,$H2#lo,$R1
1020 vld4.32 {${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]!
1021 vmlal.u32 $D1,$H0#lo,$R1
1022 vmlal.u32 $D4,$H3#lo,$R1
1023 vmlal.u32 $D2,$H1#lo,$R1
1025 vmlal.u32 $D3,$H1#lo,$R2
1026 vld1.32 ${S4}[1],[$tbl1,:32]
1027 vmlal.u32 $D0,$H3#lo,$S2
1028 vld1.32 ${S4}[0],[$tbl0,:32]
1029 vmlal.u32 $D4,$H2#lo,$R2
1030 vmlal.u32 $D1,$H4#lo,$S2
1031 vmlal.u32 $D2,$H0#lo,$R2
1033 vmlal.u32 $D3,$H0#lo,$R3
1034 vmlal.u32 $D0,$H2#lo,$S3
1035 vmlal.u32 $D4,$H1#lo,$R3
1036 vmlal.u32 $D1,$H3#lo,$S3
1037 vmlal.u32 $D2,$H4#lo,$S3
1039 vmlal.u32 $D3,$H4#lo,$S4
1040 vorn $MASK,$MASK,$MASK @ all-ones
1041 vmlal.u32 $D0,$H1#lo,$S4
1042 vshr.u64 $MASK,$MASK,#38
1043 vmlal.u32 $D4,$H0#lo,$R4
1044 vmlal.u32 $D1,$H2#lo,$S4
1045 vmlal.u32 $D2,$H3#lo,$S4
1048 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
1049 @ horizontal addition
1051 vadd.i64 $D3#lo,$D3#lo,$D3#hi
1052 vadd.i64 $D0#lo,$D0#lo,$D0#hi
1053 vadd.i64 $D4#lo,$D4#lo,$D4#hi
1054 vadd.i64 $D1#lo,$D1#lo,$D1#hi
1055 vadd.i64 $D2#lo,$D2#lo,$D2#hi
1057 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
1058 @ lazy reduction, but without narrowing
1060 vshr.u64 $T0,$D3,#26
1061 vand.i64 $D3,$D3,$MASK
1062 vshr.u64 $T1,$D0,#26
1063 vand.i64 $D0,$D0,$MASK
1064 vadd.i64 $D4,$D4,$T0 @ h3 -> h4
1065 vadd.i64 $D1,$D1,$T1 @ h0 -> h1
1067 vshr.u64 $T0,$D4,#26
1068 vand.i64 $D4,$D4,$MASK
1069 vshr.u64 $T1,$D1,#26
1070 vand.i64 $D1,$D1,$MASK
1071 vadd.i64 $D2,$D2,$T1 @ h1 -> h2
1073 vadd.i64 $D0,$D0,$T0
1075 vshr.u64 $T1,$D2,#26
1076 vand.i64 $D2,$D2,$MASK
1077 vadd.i64 $D0,$D0,$T0 @ h4 -> h0
1078 vadd.i64 $D3,$D3,$T1 @ h2 -> h3
1080 vshr.u64 $T0,$D0,#26
1081 vand.i64 $D0,$D0,$MASK
1082 vshr.u64 $T1,$D3,#26
1083 vand.i64 $D3,$D3,$MASK
1084 vadd.i64 $D1,$D1,$T0 @ h0 -> h1
1085 vadd.i64 $D4,$D4,$T1 @ h3 -> h4
1090 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
1093 vst4.32 {$D0#lo[0],$D1#lo[0],$D2#lo[0],$D3#lo[0]},[$ctx]!
1094 vst1.32 {$D4#lo[0]},[$ctx]
1096 vldmia sp!,{d8-d15} @ epilogue
1100 .size poly1305_blocks_neon,.-poly1305_blocks_neon
1102 .type poly1305_emit_neon,%function
1105 ldr ip,[$ctx,#36] @ is_base2_26
1110 beq .Lpoly1305_emit_enter
1112 ldmia $ctx,{$h0-$h4}
1115 adds $h0,$h0,$h1,lsl#26 @ base 2^26 -> base 2^32
1117 adcs $h1,$h1,$h2,lsl#20
1119 adcs $h2,$h2,$h3,lsl#14
1121 adcs $h3,$h3,$h4,lsl#8
1122 adc $h4,$g0,$h4,lsr#24 @ can be partially reduced ...
1124 and $g0,$h4,#-4 @ ... so reduce
1126 add $g0,$g0,$g0,lsr#2 @ *= 5
1133 adds $g0,$h0,#5 @ compare to modulus
1138 tst $g4,#4 @ did it carry/borrow?
1151 ldr $g3,[$nonce,#12]
1153 adds $h0,$h0,$g0 @ accumulate nonce
1164 str $h0,[$mac,#0] @ store the result
1171 .size poly1305_emit_neon,.-poly1305_emit_neon
1175 .long 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
1177 .word OPENSSL_armcap_P-.Lpoly1305_init
1182 .asciz "Poly1305 for ARMv4/NEON, CRYPTOGAMS by <appro\@openssl.org>"
1184 #if __ARM_MAX_ARCH__>=7
1185 .comm OPENSSL_armcap_P,4,4
1189 foreach (split("\n",$code)) {
1190 s/\`([^\`]*)\`/eval $1/geo;
1192 s/\bq([0-9]+)#(lo|hi)/sprintf "d%d",2*$1+($2 eq "hi")/geo or
1193 s/\bret\b/bx lr/go or
1194 s/\bbx\s+lr\b/.word\t0xe12fff1e/go; # make it possible to compile with -march=armv4
1198 close STDOUT; # enforce flush