2 # Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
4 # Licensed under the OpenSSL license (the "License"). You may not use
5 # this file except in compliance with the License. You can obtain a copy
6 # in the file LICENSE in the source distribution or at
7 # https://www.openssl.org/source/license.html
10 # ====================================================================
11 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
12 # project. The module is, however, dual licensed under OpenSSL and
13 # CRYPTOGAMS licenses depending on where you obtain it. For further
14 # details see http://www.openssl.org/~appro/cryptogams/.
15 # ====================================================================
21 # Performance in cycles per byte out of large buffer.
23 # IALU/gcc-4.4 1xNEON 3xNEON+1xIALU
25 # Cortex-A5 19.3(*)/+95% 21.8 14.1
26 # Cortex-A8 10.5(*)/+160% 13.9 6.35
27 # Cortex-A9 12.9(**)/+110% 14.3 6.50
28 # Cortex-A15 11.0/+40% 16.0 5.00
29 # Snapdragon S4 11.5/+125% 13.6 4.90
31 # (*) most "favourable" result for aligned data on little-endian
32 # processor, result for misaligned data is 10-15% lower;
33 # (**) this result is a trade-off: it can be improved by 20%,
34 # but then Snapdragon S4 and Cortex-A8 results get
38 if ($flavour=~/\w[\w\-]*\.\w+$/) { $output=$flavour; undef $flavour; }
39 else { while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {} }
41 if ($flavour && $flavour ne "void") {
42 $0 =~ m/(.*[\/\\])[^\
/\\]+$/; $dir=$1;
43 ( $xlate="${dir}arm-xlate.pl" and -f
$xlate ) or
44 ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f
$xlate) or
45 die "can't locate arm-xlate.pl";
47 open STDOUT
,"| \"$^X\" $xlate $flavour $output";
49 open STDOUT
,">$output";
52 sub AUTOLOAD
() # thunk [simplified] x86-style perlasm
53 { my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./;
55 $arg = "#$arg" if ($arg*1 eq $arg);
56 $code .= "\t$opcode\t".join(',',@_,$arg)."\n";
59 my @x=map("r$_",(0..7,"x","x","x","x",12,"x",14,"x"));
60 my @t=map("r$_",(8..11));
63 my ($a0,$b0,$c0,$d0)=@_;
64 my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
65 my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
66 my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
68 my ($xc,$xc_) = (@t[0..1]);
69 my ($xd,$xd_) = $odd ?
(@t[2],@x[$d1]) : (@x[$d0],@t[2]);
72 # Consider order in which variables are addressed by their
77 # 0 4 8 12 < even round
81 # 0 5 10 15 < odd round
86 # 'a', 'b' are permanently allocated in registers, @x[0..7],
87 # while 'c's and pair of 'd's are maintained in memory. If
88 # you observe 'c' column, you'll notice that pair of 'c's is
89 # invariant between rounds. This means that we have to reload
90 # them once per round, in the middle. This is why you'll see
91 # bunch of 'c' stores and loads in the middle, but none in
92 # the beginning or end. If you observe 'd' column, you'll
93 # notice that 15 and 13 are reused in next pair of rounds.
94 # This is why these two are chosen for offloading to memory,
95 # to make loads count more.
97 "&add (@x[$a0],@x[$a0],@x[$b0])",
98 "&mov ($xd,$xd,'ror#16')",
99 "&add (@x[$a1],@x[$a1],@x[$b1])",
100 "&mov ($xd_,$xd_,'ror#16')",
101 "&eor ($xd,$xd,@x[$a0],'ror#16')",
102 "&eor ($xd_,$xd_,@x[$a1],'ror#16')",
104 "&add ($xc,$xc,$xd)",
105 "&mov (@x[$b0],@x[$b0],'ror#20')",
106 "&add ($xc_,$xc_,$xd_)",
107 "&mov (@x[$b1],@x[$b1],'ror#20')",
108 "&eor (@x[$b0],@x[$b0],$xc,'ror#20')",
109 "&eor (@x[$b1],@x[$b1],$xc_,'ror#20')",
111 "&add (@x[$a0],@x[$a0],@x[$b0])",
112 "&mov ($xd,$xd,'ror#24')",
113 "&add (@x[$a1],@x[$a1],@x[$b1])",
114 "&mov ($xd_,$xd_,'ror#24')",
115 "&eor ($xd,$xd,@x[$a0],'ror#24')",
116 "&eor ($xd_,$xd_,@x[$a1],'ror#24')",
118 "&add ($xc,$xc,$xd)",
119 "&mov (@x[$b0],@x[$b0],'ror#25')" );
121 "&str ($xd,'[sp,#4*(16+$d0)]')",
122 "&ldr ($xd,'[sp,#4*(16+$d2)]')" ) if ($odd);
124 "&add ($xc_,$xc_,$xd_)",
125 "&mov (@x[$b1],@x[$b1],'ror#25')" );
127 "&str ($xd_,'[sp,#4*(16+$d1)]')",
128 "&ldr ($xd_,'[sp,#4*(16+$d3)]')" ) if (!$odd);
130 "&eor (@x[$b0],@x[$b0],$xc,'ror#25')",
131 "&eor (@x[$b1],@x[$b1],$xc_,'ror#25')" );
133 $xd=@x[$d2] if (!$odd);
134 $xd_=@x[$d3] if ($odd);
136 "&str ($xc,'[sp,#4*(16+$c0)]')",
137 "&ldr ($xc,'[sp,#4*(16+$c2)]')",
138 "&add (@x[$a2],@x[$a2],@x[$b2])",
139 "&mov ($xd,$xd,'ror#16')",
140 "&str ($xc_,'[sp,#4*(16+$c1)]')",
141 "&ldr ($xc_,'[sp,#4*(16+$c3)]')",
142 "&add (@x[$a3],@x[$a3],@x[$b3])",
143 "&mov ($xd_,$xd_,'ror#16')",
144 "&eor ($xd,$xd,@x[$a2],'ror#16')",
145 "&eor ($xd_,$xd_,@x[$a3],'ror#16')",
147 "&add ($xc,$xc,$xd)",
148 "&mov (@x[$b2],@x[$b2],'ror#20')",
149 "&add ($xc_,$xc_,$xd_)",
150 "&mov (@x[$b3],@x[$b3],'ror#20')",
151 "&eor (@x[$b2],@x[$b2],$xc,'ror#20')",
152 "&eor (@x[$b3],@x[$b3],$xc_,'ror#20')",
154 "&add (@x[$a2],@x[$a2],@x[$b2])",
155 "&mov ($xd,$xd,'ror#24')",
156 "&add (@x[$a3],@x[$a3],@x[$b3])",
157 "&mov ($xd_,$xd_,'ror#24')",
158 "&eor ($xd,$xd,@x[$a2],'ror#24')",
159 "&eor ($xd_,$xd_,@x[$a3],'ror#24')",
161 "&add ($xc,$xc,$xd)",
162 "&mov (@x[$b2],@x[$b2],'ror#25')",
163 "&add ($xc_,$xc_,$xd_)",
164 "&mov (@x[$b3],@x[$b3],'ror#25')",
165 "&eor (@x[$b2],@x[$b2],$xc,'ror#25')",
166 "&eor (@x[$b3],@x[$b3],$xc_,'ror#25')" );
172 #include "arm_arch.h"
175 #if defined(__thumb2__)
182 #if defined(__thumb2__) || defined(__clang__)
183 #define ldrhsb ldrbhs
188 .long
0x61707865,0x3320646e,0x79622d32,0x6b206574 @ endian
-neutral
191 #if __ARM_MAX_ARCH__>=7
193 .word OPENSSL_armcap_P
-.LChaCha20_ctr32
198 .globl ChaCha20_ctr32
199 .type ChaCha20_ctr32
,%function
203 ldr r12
,[sp
,#0] @ pull pointer to counter and nonce
204 stmdb sp
!,{r0
-r2
,r4
-r11
,lr
}
205 #if __ARM_ARCH__<7 && !defined(__thumb2__)
206 sub r14
,pc
,#16 @ ChaCha20_ctr32
208 adr r14
,.LChaCha20_ctr32
216 #if __ARM_MAX_ARCH__>=7
217 cmp r2
,#192 @ test len
228 ldmia r12
,{r4
-r7
} @ load counter
and nonce
229 sub sp
,sp
,#4*(16) @ off-load area
230 sub r14
,r14
,#64 @ .Lsigma
231 stmdb sp
!,{r4
-r7
} @ copy counter
and nonce
232 ldmia r3
,{r4
-r11
} @ load key
233 ldmia r14
,{r0
-r3
} @ load sigma
234 stmdb sp
!,{r4
-r11
} @ copy key
235 stmdb sp
!,{r0
-r3
} @ copy sigma
236 str r10
,[sp
,#4*(16+10)] @ off-load "@x[10]"
237 str r11
,[sp
,#4*(16+11)] @ off-load "@x[11]"
242 ldmia sp
,{r0
-r9
} @ load key material
243 str
@t[3],[sp
,#4*(32+2)] @ save len
244 str r12
, [sp
,#4*(32+1)] @ save inp
245 str r14
, [sp
,#4*(32+0)] @ save out
247 ldr
@t[3], [sp
,#4*(15)]
248 ldr
@x[12],[sp
,#4*(12)] @ modulo-scheduled load
249 ldr
@t[2], [sp
,#4*(13)]
250 ldr
@x[14],[sp
,#4*(14)]
251 str
@t[3], [sp
,#4*(16+15)]
259 foreach (&ROUND
(0, 4, 8,12)) { eval; }
260 foreach (&ROUND
(0, 5,10,15)) { eval; }
264 ldr
@t[3],[sp
,#4*(32+2)] @ load len
266 str
@t[0], [sp
,#4*(16+8)] @ modulo-scheduled store
267 str
@t[1], [sp
,#4*(16+9)]
268 str
@x[12],[sp
,#4*(16+12)]
269 str
@t[2], [sp
,#4*(16+13)]
270 str
@x[14],[sp
,#4*(16+14)]
272 @ at this point we have first half of
512-bit result
in
273 @
@x[0-7] and second half at sp
+4*(16+8)
275 cmp @t[3],#64 @ done yet?
279 addlo r12
,sp
,#4*(0) @ shortcut or ...
280 ldrhs r12
,[sp
,#4*(32+1)] @ ... load inp
281 addlo r14
,sp
,#4*(0) @ shortcut or ...
282 ldrhs r14
,[sp
,#4*(32+0)] @ ... load out
284 ldr
@t[0],[sp
,#4*(0)] @ load key material
285 ldr
@t[1],[sp
,#4*(1)]
287 #if __ARM_ARCH__>=6 || !defined(__ARMEB__)
290 tst
@t[2],#3 @ are input and output aligned?
291 ldr
@t[2],[sp
,#4*(2)]
293 cmp @t[3],#64 @ restore flags
295 ldr
@t[2],[sp
,#4*(2)]
297 ldr
@t[3],[sp
,#4*(3)]
299 add
@x[0],@x[0],@t[0] @ accumulate key material
300 add
@x[1],@x[1],@t[1]
304 ldrhs
@t[0],[r12
],#16 @ load input
305 ldrhs
@t[1],[r12
,#-12]
307 add
@x[2],@x[2],@t[2]
308 add
@x[3],@x[3],@t[3]
312 ldrhs
@t[2],[r12
,#-8]
313 ldrhs
@t[3],[r12
,#-4]
314 # if __ARM_ARCH__>=6 && defined(__ARMEB__)
323 eorhs
@x[0],@x[0],@t[0] @
xor with input
324 eorhs
@x[1],@x[1],@t[1]
326 str
@x[0],[r14
],#16 @ store output
330 eorhs
@x[2],@x[2],@t[2]
331 eorhs
@x[3],@x[3],@t[3]
332 ldmia
@t[0],{@t[0]-@t[3]} @ load key material
337 add
@x[4],@x[4],@t[0] @ accumulate key material
338 add
@x[5],@x[5],@t[1]
342 ldrhs
@t[0],[r12
],#16 @ load input
343 ldrhs
@t[1],[r12
,#-12]
344 add
@x[6],@x[6],@t[2]
345 add
@x[7],@x[7],@t[3]
349 ldrhs
@t[2],[r12
,#-8]
350 ldrhs
@t[3],[r12
,#-4]
351 # if __ARM_ARCH__>=6 && defined(__ARMEB__)
360 eorhs
@x[4],@x[4],@t[0]
361 eorhs
@x[5],@x[5],@t[1]
363 str
@x[4],[r14
],#16 @ store output
367 eorhs
@x[6],@x[6],@t[2]
368 eorhs
@x[7],@x[7],@t[3]
370 ldmia
@t[0],{@t[0]-@t[3]} @ load key material
372 add
@x[0],sp
,#4*(16+8)
375 ldmia
@x[0],{@x[0]-@x[7]} @ load second half
377 add
@x[0],@x[0],@t[0] @ accumulate key material
378 add
@x[1],@x[1],@t[1]
382 ldrhs
@t[0],[r12
],#16 @ load input
383 ldrhs
@t[1],[r12
,#-12]
387 strhi
@t[2],[sp
,#4*(16+10)] @ copy "@x[10]" while at it
388 strhi
@t[3],[sp
,#4*(16+11)] @ copy "@x[11]" while at it
389 add
@x[2],@x[2],@t[2]
390 add
@x[3],@x[3],@t[3]
394 ldrhs
@t[2],[r12
,#-8]
395 ldrhs
@t[3],[r12
,#-4]
396 # if __ARM_ARCH__>=6 && defined(__ARMEB__)
405 eorhs
@x[0],@x[0],@t[0]
406 eorhs
@x[1],@x[1],@t[1]
408 str
@x[0],[r14
],#16 @ store output
412 eorhs
@x[2],@x[2],@t[2]
413 eorhs
@x[3],@x[3],@t[3]
415 ldmia
@t[0],{@t[0]-@t[3]} @ load key material
419 add
@x[4],@x[4],@t[0] @ accumulate key material
420 add
@x[5],@x[5],@t[1]
424 addhi
@t[0],@t[0],#1 @ next counter value
425 strhi
@t[0],[sp
,#4*(12)] @ save next counter value
429 ldrhs
@t[0],[r12
],#16 @ load input
430 ldrhs
@t[1],[r12
,#-12]
431 add
@x[6],@x[6],@t[2]
432 add
@x[7],@x[7],@t[3]
436 ldrhs
@t[2],[r12
,#-8]
437 ldrhs
@t[3],[r12
,#-4]
438 # if __ARM_ARCH__>=6 && defined(__ARMEB__)
447 eorhs
@x[4],@x[4],@t[0]
448 eorhs
@x[5],@x[5],@t[1]
452 ldrne
@t[0],[sp
,#4*(32+2)] @ re-load len
456 eorhs
@x[6],@x[6],@t[2]
457 eorhs
@x[7],@x[7],@t[3]
458 str
@x[4],[r14
],#16 @ store output
463 subhs
@t[3],@t[0],#64 @ len-=64
473 .Lunaligned
: @ unaligned endian
-neutral path
474 cmp @t[3],#64 @ restore flags
478 ldr
@t[3],[sp
,#4*(3)]
480 for ($i=0;$i<16;$i+=4) {
483 $code.=<<___
if ($i==4);
484 add
@x[0],sp
,#4*(16+8)
486 $code.=<<___
if ($i==8);
487 ldmia
@x[0],{@x[0]-@x[7]} @ load second half
491 strhi
@t[2],[sp
,#4*(16+10)] @ copy "@x[10]"
492 strhi
@t[3],[sp
,#4*(16+11)] @ copy "@x[11]"
495 add
@x[$j+0],@x[$j+0],@t[0] @ accumulate key material
497 $code.=<<___
if ($i==12);
501 addhi
@t[0],@t[0],#1 @ next counter value
502 strhi
@t[0],[sp
,#4*(12)] @ save next counter value
505 add
@x[$j+1],@x[$j+1],@t[1]
506 add
@x[$j+2],@x[$j+2],@t[2]
510 eorlo
@t[0],@t[0],@t[0] @ zero
or ...
511 ldrhsb
@t[0],[r12
],#16 @ ... load input
512 eorlo
@t[1],@t[1],@t[1]
513 ldrhsb
@t[1],[r12
,#-12]
515 add
@x[$j+3],@x[$j+3],@t[3]
519 eorlo
@t[2],@t[2],@t[2]
520 ldrhsb
@t[2],[r12
,#-8]
521 eorlo
@t[3],@t[3],@t[3]
522 ldrhsb
@t[3],[r12
,#-4]
524 eor
@x[$j+0],@t[0],@x[$j+0] @
xor with input
(or zero
)
525 eor
@x[$j+1],@t[1],@x[$j+1]
529 ldrhsb
@t[0],[r12
,#-15] @ load more input
530 ldrhsb
@t[1],[r12
,#-11]
531 eor
@x[$j+2],@t[2],@x[$j+2]
532 strb
@x[$j+0],[r14
],#16 @ store output
533 eor
@x[$j+3],@t[3],@x[$j+3]
537 ldrhsb
@t[2],[r12
,#-7]
538 ldrhsb
@t[3],[r12
,#-3]
539 strb
@x[$j+1],[r14
,#-12]
540 eor
@x[$j+0],@t[0],@x[$j+0],lsr
#8
541 strb
@x[$j+2],[r14
,#-8]
542 eor
@x[$j+1],@t[1],@x[$j+1],lsr
#8
546 ldrhsb
@t[0],[r12
,#-14] @ load more input
547 ldrhsb
@t[1],[r12
,#-10]
548 strb
@x[$j+3],[r14
,#-4]
549 eor
@x[$j+2],@t[2],@x[$j+2],lsr
#8
550 strb
@x[$j+0],[r14
,#-15]
551 eor
@x[$j+3],@t[3],@x[$j+3],lsr
#8
555 ldrhsb
@t[2],[r12
,#-6]
556 ldrhsb
@t[3],[r12
,#-2]
557 strb
@x[$j+1],[r14
,#-11]
558 eor
@x[$j+0],@t[0],@x[$j+0],lsr
#8
559 strb
@x[$j+2],[r14
,#-7]
560 eor
@x[$j+1],@t[1],@x[$j+1],lsr
#8
564 ldrhsb
@t[0],[r12
,#-13] @ load more input
565 ldrhsb
@t[1],[r12
,#-9]
566 strb
@x[$j+3],[r14
,#-3]
567 eor
@x[$j+2],@t[2],@x[$j+2],lsr
#8
568 strb
@x[$j+0],[r14
,#-14]
569 eor
@x[$j+3],@t[3],@x[$j+3],lsr
#8
573 ldrhsb
@t[2],[r12
,#-5]
574 ldrhsb
@t[3],[r12
,#-1]
575 strb
@x[$j+1],[r14
,#-10]
576 strb
@x[$j+2],[r14
,#-6]
577 eor
@x[$j+0],@t[0],@x[$j+0],lsr
#8
578 strb
@x[$j+3],[r14
,#-2]
579 eor
@x[$j+1],@t[1],@x[$j+1],lsr
#8
580 strb
@x[$j+0],[r14
,#-13]
581 eor
@x[$j+2],@t[2],@x[$j+2],lsr
#8
582 strb
@x[$j+1],[r14
,#-9]
583 eor
@x[$j+3],@t[3],@x[$j+3],lsr
#8
584 strb
@x[$j+2],[r14
,#-5]
585 strb
@x[$j+3],[r14
,#-1]
587 $code.=<<___
if ($i<12);
588 add
@t[0],sp
,#4*(4+$i)
589 ldmia
@t[0],{@t[0]-@t[3]} @ load key material
596 ldrne
@t[0],[sp
,#4*(32+2)] @ re-load len
600 subhs
@t[3],@t[0],#64 @ len-=64
607 ldr r12
,[sp
,#4*(32+1)] @ load inp
609 ldr r14
,[sp
,#4*(32+0)] @ load out
612 ldrb
@t[2],[@t[1]],#1 @ read buffer on stack
613 ldrb
@t[3],[r12
],#1 @ read input
615 eor
@t[3],@t[3],@t[2]
616 strb
@t[3],[r14
],#1 @ store output
622 ldmia sp
!,{r4
-r11
,pc
}
623 .size ChaCha20_ctr32
,.-ChaCha20_ctr32
627 my ($a0,$b0,$c0,$d0,$a1,$b1,$c1,$d1,$a2,$b2,$c2,$d2,$t0,$t1,$t2,$t3) =
632 my ($a,$b,$c,$d,$t)=@_;
635 "&vadd_i32 ($a,$a,$b)",
637 "&vrev32_16 ($d,$d)", # vrot ($d,16)
639 "&vadd_i32 ($c,$c,$d)",
641 "&vshr_u32 ($b,$t,20)",
642 "&vsli_32 ($b,$t,12)",
644 "&vadd_i32 ($a,$a,$b)",
646 "&vshr_u32 ($d,$t,24)",
647 "&vsli_32 ($d,$t,8)",
649 "&vadd_i32 ($c,$c,$d)",
651 "&vshr_u32 ($b,$t,25)",
652 "&vsli_32 ($b,$t,7)",
654 "&vext_8 ($c,$c,$c,8)",
655 "&vext_8 ($b,$b,$b,$odd?12:4)",
656 "&vext_8 ($d,$d,$d,$odd?4:12)"
661 #if __ARM_MAX_ARCH__>=7
665 .type ChaCha20_neon
,%function
668 ldr r12
,[sp
,#0] @ pull pointer to counter and nonce
669 stmdb sp
!,{r0
-r2
,r4
-r11
,lr
}
672 vstmdb sp
!,{d8
-d15
} @ ABI spec says so
675 vld1
.32
{$b0-$c0},[r3
] @ load key
676 ldmia r3
,{r4
-r11
} @ load key
679 vld1
.32
{$d0},[r12
] @ load counter
and nonce
681 ldmia r14
,{r0
-r3
} @ load sigma
682 vld1
.32
{$a0},[r14
]! @ load sigma
683 vld1
.32
{$t0},[r14
] @ one
684 vst1
.32
{$c0-$d0},[r12
] @ copy
1/2key
|counter
|nonce
685 vst1
.32
{$a0-$b0},[sp
] @ copy sigma
|1/2key
687 str r10
,[sp
,#4*(16+10)] @ off-load "@x[10]"
688 str r11
,[sp
,#4*(16+11)] @ off-load "@x[11]"
689 vshl
.i32
$t1#lo,$t0#lo,#1 @ two
690 vstr
$t0#lo,[sp,#4*(16+0)]
691 vshl
.i32
$t2#lo,$t0#lo,#2 @ four
692 vstr
$t1#lo,[sp,#4*(16+2)]
694 vstr
$t2#lo,[sp,#4*(16+4)]
702 ldmia sp
,{r0
-r9
} @ load key material
703 cmp @t[3],#64*2 @ if len<=64*2
704 bls
.Lbreak_neon @ switch to integer
-only
706 str
@t[3],[sp
,#4*(32+2)] @ save len
708 str r12
, [sp
,#4*(32+1)] @ save inp
710 str r14
, [sp
,#4*(32+0)] @ save out
713 ldr
@t[3], [sp
,#4*(15)]
714 vadd
.i32
$d1,$d0,$t0 @ counter
+1
715 ldr
@x[12],[sp
,#4*(12)] @ modulo-scheduled load
717 ldr
@t[2], [sp
,#4*(13)]
719 ldr
@x[14],[sp
,#4*(14)]
720 vadd
.i32
$d2,$d1,$t0 @ counter
+2
721 str
@t[3], [sp
,#4*(16+15)]
723 add
@x[12],@x[12],#3 @ counter+3
730 my @thread0=&NEONROUND
($a0,$b0,$c0,$d0,$t0,0);
731 my @thread1=&NEONROUND
($a1,$b1,$c1,$d1,$t1,0);
732 my @thread2=&NEONROUND
($a2,$b2,$c2,$d2,$t2,0);
733 my @thread3=&ROUND
(0,4,8,12);
736 eval; eval(shift(@thread3));
737 eval(shift(@thread1)); eval(shift(@thread3));
738 eval(shift(@thread2)); eval(shift(@thread3));
741 @thread0=&NEONROUND
($a0,$b0,$c0,$d0,$t0,1);
742 @thread1=&NEONROUND
($a1,$b1,$c1,$d1,$t1,1);
743 @thread2=&NEONROUND
($a2,$b2,$c2,$d2,$t2,1);
744 @thread3=&ROUND
(0,5,10,15);
747 eval; eval(shift(@thread3));
748 eval(shift(@thread1)); eval(shift(@thread3));
749 eval(shift(@thread2)); eval(shift(@thread3));
755 vld1
.32
{$t0-$t1},[sp
] @ load key material
756 vld1
.32
{$t2-$t3},[@t[3]]
758 ldr
@t[3],[sp
,#4*(32+2)] @ load len
760 str
@t[0], [sp
,#4*(16+8)] @ modulo-scheduled store
761 str
@t[1], [sp
,#4*(16+9)]
762 str
@x[12],[sp
,#4*(16+12)]
763 str
@t[2], [sp
,#4*(16+13)]
764 str
@x[14],[sp
,#4*(16+14)]
766 @ at this point we have first half of
512-bit result
in
767 @
@x[0-7] and second half at sp
+4*(16+8)
769 ldr r12
,[sp
,#4*(32+1)] @ load inp
770 ldr r14
,[sp
,#4*(32+0)] @ load out
772 vadd
.i32
$a0,$a0,$t0 @ accumulate key material
775 vldr
$t0#lo,[sp,#4*(16+0)] @ one
780 vldr
$t1#lo,[sp,#4*(16+2)] @ two
785 vadd
.i32
$d1#lo,$d1#lo,$t0#lo @ counter+1
786 vadd
.i32
$d2#lo,$d2#lo,$t1#lo @ counter+2
795 vld1
.8
{$t0-$t1},[r12
]! @ load input
797 vld1
.8
{$t2-$t3},[r12
]!
798 veor
$a0,$a0,$t0 @
xor with input
800 vld1
.8
{$t0-$t1},[r12
]!
803 vld1
.8
{$t2-$t3},[r12
]!
806 vst1
.8
{$a0-$b0},[r14
]! @ store output
808 vld1
.8
{$t0-$t1},[r12
]!
810 vst1
.8
{$c0-$d0},[r14
]!
812 vld1
.8
{$t2-$t3},[r12
]!
815 vld1
.32
{$a0-$b0},[@t[3]]! @ load
for next iteration
816 veor
$t0#hi,$t0#hi,$t0#hi
817 vldr
$t0#lo,[sp,#4*(16+4)] @ four
819 vld1
.32
{$c0-$d0},[@t[3]]
821 vst1
.8
{$a1-$b1},[r14
]!
823 vst1
.8
{$c1-$d1},[r14
]!
825 vadd
.i32
$d0#lo,$d0#lo,$t0#lo @ next counter value
826 vldr
$t0#lo,[sp,#4*(16+0)] @ one
828 ldmia sp
,{@t[0]-@t[3]} @ load key material
829 add
@x[0],@x[0],@t[0] @ accumulate key material
830 ldr
@t[0],[r12
],#16 @ load input
831 vst1
.8
{$a2-$b2},[r14
]!
832 add
@x[1],@x[1],@t[1]
834 vst1
.8
{$c2-$d2},[r14
]!
835 add
@x[2],@x[2],@t[2]
837 add
@x[3],@x[3],@t[3]
845 eor
@x[0],@x[0],@t[0] @
xor with input
847 eor
@x[1],@x[1],@t[1]
848 str
@x[0],[r14
],#16 @ store output
849 eor
@x[2],@x[2],@t[2]
851 eor
@x[3],@x[3],@t[3]
852 ldmia
@t[0],{@t[0]-@t[3]} @ load key material
856 add
@x[4],@x[4],@t[0] @ accumulate key material
857 ldr
@t[0],[r12
],#16 @ load input
858 add
@x[5],@x[5],@t[1]
860 add
@x[6],@x[6],@t[2]
862 add
@x[7],@x[7],@t[3]
870 eor
@x[4],@x[4],@t[0]
872 eor
@x[5],@x[5],@t[1]
873 str
@x[4],[r14
],#16 @ store output
874 eor
@x[6],@x[6],@t[2]
876 eor
@x[7],@x[7],@t[3]
877 ldmia
@t[0],{@t[0]-@t[3]} @ load key material
879 add
@x[0],sp
,#4*(16+8)
882 ldmia
@x[0],{@x[0]-@x[7]} @ load second half
884 add
@x[0],@x[0],@t[0] @ accumulate key material
885 ldr
@t[0],[r12
],#16 @ load input
886 add
@x[1],@x[1],@t[1]
891 strhi
@t[2],[sp
,#4*(16+10)] @ copy "@x[10]" while at it
892 add
@x[2],@x[2],@t[2]
897 strhi
@t[3],[sp
,#4*(16+11)] @ copy "@x[11]" while at it
898 add
@x[3],@x[3],@t[3]
906 eor
@x[0],@x[0],@t[0]
908 eor
@x[1],@x[1],@t[1]
909 str
@x[0],[r14
],#16 @ store output
910 eor
@x[2],@x[2],@t[2]
912 eor
@x[3],@x[3],@t[3]
913 ldmia
@t[0],{@t[0]-@t[3]} @ load key material
917 add
@x[4],@x[4],@t[0] @ accumulate key material
918 add
@t[0],@t[0],#4 @ next counter value
919 add
@x[5],@x[5],@t[1]
920 str
@t[0],[sp
,#4*(12)] @ save next counter value
921 ldr
@t[0],[r12
],#16 @ load input
922 add
@x[6],@x[6],@t[2]
923 add
@x[4],@x[4],#3 @ counter+3
925 add
@x[7],@x[7],@t[3]
934 eor
@x[4],@x[4],@t[0]
938 ldrhi
@t[0],[sp
,#4*(32+2)] @ re-load len
939 eor
@x[5],@x[5],@t[1]
940 eor
@x[6],@x[6],@t[2]
941 str
@x[4],[r14
],#16 @ store output
942 eor
@x[7],@x[7],@t[3]
944 sub @t[3],@t[0],#64*4 @ len-=64*4
953 @ harmonize NEON
and integer
-only stack frames
: load data
954 @ from NEON frame
, but save to integer
-only one
; distance
955 @ between the two is
4*(32+4+16-32)=4*(20).
957 str
@t[3], [sp
,#4*(20+32+2)] @ save len
958 add
@t[3],sp
,#4*(32+4)
959 str r12
, [sp
,#4*(20+32+1)] @ save inp
960 str r14
, [sp
,#4*(20+32+0)] @ save out
962 ldr
@x[12],[sp
,#4*(16+10)]
963 ldr
@x[14],[sp
,#4*(16+11)]
964 vldmia
@t[3],{d8
-d15
} @ fulfill ABI requirement
965 str
@x[12],[sp
,#4*(20+16+10)] @ copy "@x[10]"
966 str
@x[14],[sp
,#4*(20+16+11)] @ copy "@x[11]"
968 ldr
@t[3], [sp
,#4*(15)]
969 ldr
@x[12],[sp
,#4*(12)] @ modulo-scheduled load
970 ldr
@t[2], [sp
,#4*(13)]
971 ldr
@x[14],[sp
,#4*(14)]
972 str
@t[3], [sp
,#4*(20+16+15)]
974 vst1
.32
{$a0-$b0},[@t[3]]! @ copy key
975 add sp
,sp
,#4*(20) @ switch frame
976 vst1
.32
{$c0-$d0},[@t[3]]
978 b
.Loop @ go integer
-only
983 bhs
.L192_or_more_neon
985 bhs
.L128_or_more_neon
987 bhs
.L64_or_more_neon
990 vst1
.8
{$a0-$b0},[sp
]
992 vst1
.8
{$c0-$d0},[@t[0]]
997 vld1
.8
{$t0-$t1},[r12
]!
998 vld1
.8
{$t2-$t3},[r12
]!
1003 vst1
.8
{$a0-$b0},[r14
]!
1004 vst1
.8
{$c0-$d0},[r14
]!
1009 vst1
.8
{$a1-$b1},[sp
]
1011 vst1
.8
{$c1-$d1},[@t[0]]
1012 sub @t[3],@t[3],#64*1 @ len-=64*1
1017 vld1
.8
{$t0-$t1},[r12
]!
1018 vld1
.8
{$t2-$t3},[r12
]!
1021 vld1
.8
{$t0-$t1},[r12
]!
1024 vld1
.8
{$t2-$t3},[r12
]!
1028 vst1
.8
{$a0-$b0},[r14
]!
1030 vst1
.8
{$c0-$d0},[r14
]!
1032 vst1
.8
{$a1-$b1},[r14
]!
1033 vst1
.8
{$c1-$d1},[r14
]!
1038 vst1
.8
{$a2-$b2},[sp
]
1040 vst1
.8
{$c2-$d2},[@t[0]]
1041 sub @t[3],@t[3],#64*2 @ len-=64*2
1046 vld1
.8
{$t0-$t1},[r12
]!
1047 vld1
.8
{$t2-$t3},[r12
]!
1050 vld1
.8
{$t0-$t1},[r12
]!
1053 vld1
.8
{$t2-$t3},[r12
]!
1057 vld1
.8
{$t0-$t1},[r12
]!
1059 vst1
.8
{$a0-$b0},[r14
]!
1061 vld1
.8
{$t2-$t3},[r12
]!
1064 vst1
.8
{$c0-$d0},[r14
]!
1066 vst1
.8
{$a1-$b1},[r14
]!
1068 vst1
.8
{$c1-$d1},[r14
]!
1070 vst1
.8
{$a2-$b2},[r14
]!
1071 vst1
.8
{$c2-$d2},[r14
]!
1075 ldmia sp
,{@t[0]-@t[3]} @ load key material
1076 add
@x[0],@x[0],@t[0] @ accumulate key material
1078 add
@x[1],@x[1],@t[1]
1079 add
@x[2],@x[2],@t[2]
1080 add
@x[3],@x[3],@t[3]
1081 ldmia
@t[0],{@t[0]-@t[3]} @ load key material
1083 add
@x[4],@x[4],@t[0] @ accumulate key material
1085 add
@x[5],@x[5],@t[1]
1086 add
@x[6],@x[6],@t[2]
1087 add
@x[7],@x[7],@t[3]
1088 ldmia
@t[0],{@t[0]-@t[3]} @ load key material
1099 stmia sp
,{@x[0]-@x[7]}
1100 add
@x[0],sp
,#4*(16+8)
1102 ldmia
@x[0],{@x[0]-@x[7]} @ load second half
1104 add
@x[0],@x[0],@t[0] @ accumulate key material
1105 add
@t[0],sp
,#4*(12)
1106 add
@x[1],@x[1],@t[1]
1107 add
@x[2],@x[2],@t[2]
1108 add
@x[3],@x[3],@t[3]
1109 ldmia
@t[0],{@t[0]-@t[3]} @ load key material
1111 add
@x[4],@x[4],@t[0] @ accumulate key material
1113 add
@x[5],@x[5],@t[1]
1114 add
@x[4],@x[4],#3 @ counter+3
1115 add
@x[6],@x[6],@t[2]
1116 add
@x[7],@x[7],@t[3]
1117 ldr
@t[3],[sp
,#4*(32+2)] @ re-load len
1128 stmia
@t[0],{@x[0]-@x[7]}
1130 sub @t[3],@t[3],#64*3 @ len-=64*3
1133 ldrb
@t[0],[@t[2]],#1 @ read buffer on stack
1134 ldrb
@t[1],[r12
],#1 @ read input
1136 eor
@t[0],@t[0],@t[1]
1137 strb
@t[0],[r14
],#1 @ store output
1144 ldmia sp
!,{r4
-r11
,pc
}
1145 .size ChaCha20_neon
,.-ChaCha20_neon
1146 .comm OPENSSL_armcap_P
,4,4
1151 foreach (split("\n",$code)) {
1152 s/\`([^\`]*)\`/eval $1/geo;
1154 s/\bq([0-9]+)#(lo|hi)/sprintf "d%d",2*$1+($2 eq "hi")/geo;