]> git.ipfire.org Git - thirdparty/openssl.git/blob - crypto/poly1305/asm/poly1305-armv4.pl
projects / thirdparty / openssl.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Copyright year updates
[thirdparty/openssl.git] / crypto / poly1305 / asm / poly1305-armv4.pl
1 #! /usr/bin/env perl
2 # Copyright 2016-2023 The OpenSSL Project Authors. All Rights Reserved.
3 #
4 # Licensed under the Apache License 2.0 (the "License"). You may not use
5 # this file except in compliance with the License. You can obtain a copy
6 # in the file LICENSE in the source distribution or at
7 # https://www.openssl.org/source/license.html
8
9 #
10 # ====================================================================
11 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
12 # project. The module is, however, dual licensed under OpenSSL and
13 # CRYPTOGAMS licenses depending on where you obtain it. For further
14 # details see http://www.openssl.org/~appro/cryptogams/.
15 # ====================================================================
16 #
17 # IALU(*)/gcc-4.4 NEON
18 #
19 # ARM11xx(ARMv6) 7.78/+100% -
20 # Cortex-A5 6.35/+130% 3.00
21 # Cortex-A8 6.25/+115% 2.36
22 # Cortex-A9 5.10/+95% 2.55
23 # Cortex-A15 3.85/+85% 1.25(**)
24 # Snapdragon S4 5.70/+100% 1.48(**)
25 #
26 # (*) this is for -march=armv6, i.e. with bunch of ldrb loading data;
27 # (**) these are trade-off results, they can be improved by ~8% but at
28 # the cost of 15/12% regression on Cortex-A5/A7, it's even possible
29 # to improve Cortex-A9 result, but then A5/A7 loose more than 20%;
30
31 # $output is the last argument if it looks like a file (it has an extension)
32 # $flavour is the first argument if it doesn't look like a file
33 $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
34 $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
35
36 if ($flavour && $flavour ne "void") {
37 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
38 ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
39 ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
40 die "can't locate arm-xlate.pl";
41
42 open STDOUT,"| \"$^X\" $xlate $flavour \"$output\""
43 or die "can't call $xlate: $!";
44 } else {
45 $output and open STDOUT,">$output";
46 }
47
48 ($ctx,$inp,$len,$padbit)=map("r$_",(0..3));
49
50 $code.=<<___;
51 #include "arm_arch.h"
52
53 #if defined(__thumb2__)
54 .syntax unified
55 .thumb
56 #else
57 .code 32
58 #endif
59
60 .text
61
62 .globl poly1305_emit
63 .globl poly1305_blocks
64 .globl poly1305_init
65 .type poly1305_init,%function
66 .align 5
67 poly1305_init:
68 .Lpoly1305_init:
69 stmdb sp!,{r4-r11}
70
71 eor r3,r3,r3
72 cmp $inp,#0
73 str r3,[$ctx,#0] @ zero hash value
74 str r3,[$ctx,#4]
75 str r3,[$ctx,#8]
76 str r3,[$ctx,#12]
77 str r3,[$ctx,#16]
78 str r3,[$ctx,#36] @ is_base2_26
79 add $ctx,$ctx,#20
80
81 #ifdef __thumb2__
82 it eq
83 #endif
84 moveq r0,#0
85 beq .Lno_key
86
87 #if __ARM_MAX_ARCH__>=7
88 adr r11,.Lpoly1305_init
89 ldr r12,.LOPENSSL_armcap
90 #endif
91 ldrb r4,[$inp,#0]
92 mov r10,#0x0fffffff
93 ldrb r5,[$inp,#1]
94 and r3,r10,#-4 @ 0x0ffffffc
95 ldrb r6,[$inp,#2]
96 ldrb r7,[$inp,#3]
97 orr r4,r4,r5,lsl#8
98 ldrb r5,[$inp,#4]
99 orr r4,r4,r6,lsl#16
100 ldrb r6,[$inp,#5]
101 orr r4,r4,r7,lsl#24
102 ldrb r7,[$inp,#6]
103 and r4,r4,r10
104
105 #if __ARM_MAX_ARCH__>=7
106 # if !defined(_WIN32)
107 ldr r12,[r11,r12] @ OPENSSL_armcap_P
108 # endif
109 # if defined(__APPLE__) || defined(_WIN32)
110 ldr r12,[r12]
111 # endif
112 #endif
113 ldrb r8,[$inp,#7]
114 orr r5,r5,r6,lsl#8
115 ldrb r6,[$inp,#8]
116 orr r5,r5,r7,lsl#16
117 ldrb r7,[$inp,#9]
118 orr r5,r5,r8,lsl#24
119 ldrb r8,[$inp,#10]
120 and r5,r5,r3
121
122 #if __ARM_MAX_ARCH__>=7
123 tst r12,#ARMV7_NEON @ check for NEON
124 # ifdef __thumb2__
125 adr r9,.Lpoly1305_blocks_neon
126 adr r11,.Lpoly1305_blocks
127 adr r12,.Lpoly1305_emit
128 adr r10,.Lpoly1305_emit_neon
129 itt ne
130 movne r11,r9
131 movne r12,r10
132 orr r11,r11,#1 @ thumb-ify address
133 orr r12,r12,#1
134 # else
135 addeq r12,r11,#(.Lpoly1305_emit-.Lpoly1305_init)
136 addne r12,r11,#(.Lpoly1305_emit_neon-.Lpoly1305_init)
137 addeq r11,r11,#(.Lpoly1305_blocks-.Lpoly1305_init)
138 addne r11,r11,#(.Lpoly1305_blocks_neon-.Lpoly1305_init)
139 # endif
140 #endif
141 ldrb r9,[$inp,#11]
142 orr r6,r6,r7,lsl#8
143 ldrb r7,[$inp,#12]
144 orr r6,r6,r8,lsl#16
145 ldrb r8,[$inp,#13]
146 orr r6,r6,r9,lsl#24
147 ldrb r9,[$inp,#14]
148 and r6,r6,r3
149
150 ldrb r10,[$inp,#15]
151 orr r7,r7,r8,lsl#8
152 str r4,[$ctx,#0]
153 orr r7,r7,r9,lsl#16
154 str r5,[$ctx,#4]
155 orr r7,r7,r10,lsl#24
156 str r6,[$ctx,#8]
157 and r7,r7,r3
158 str r7,[$ctx,#12]
159 #if __ARM_MAX_ARCH__>=7
160 stmia r2,{r11,r12} @ fill functions table
161 mov r0,#1
162 #else
163 mov r0,#0
164 #endif
165 .Lno_key:
166 ldmia sp!,{r4-r11}
167 #if __ARM_ARCH__>=5
168 ret @ bx lr
169 #else
170 tst lr,#1
171 moveq pc,lr @ be binary compatible with V4, yet
172 bx lr @ interoperable with Thumb ISA:-)
173 #endif
174 .size poly1305_init,.-poly1305_init
175 ___
176 {
177 my ($h0,$h1,$h2,$h3,$h4,$r0,$r1,$r2,$r3)=map("r$_",(4..12));
178 my ($s1,$s2,$s3)=($r1,$r2,$r3);
179
180 $code.=<<___;
181 .type poly1305_blocks,%function
182 .align 5
183 poly1305_blocks:
184 .Lpoly1305_blocks:
185 stmdb sp!,{r3-r11,lr}
186
187 ands $len,$len,#-16
188 beq .Lno_data
189
190 cmp $padbit,#0
191 add $len,$len,$inp @ end pointer
192 sub sp,sp,#32
193
194 ldmia $ctx,{$h0-$r3} @ load context
195
196 str $ctx,[sp,#12] @ offload stuff
197 mov lr,$inp
198 str $len,[sp,#16]
199 str $r1,[sp,#20]
200 str $r2,[sp,#24]
201 str $r3,[sp,#28]
202 b .Loop
203
204 .Loop:
205 #if __ARM_ARCH__<7
206 ldrb r0,[lr],#16 @ load input
207 # ifdef __thumb2__
208 it hi
209 # endif
210 addhi $h4,$h4,#1 @ 1<<128
211 ldrb r1,[lr,#-15]
212 ldrb r2,[lr,#-14]
213 ldrb r3,[lr,#-13]
214 orr r1,r0,r1,lsl#8
215 ldrb r0,[lr,#-12]
216 orr r2,r1,r2,lsl#16
217 ldrb r1,[lr,#-11]
218 orr r3,r2,r3,lsl#24
219 ldrb r2,[lr,#-10]
220 adds $h0,$h0,r3 @ accumulate input
221
222 ldrb r3,[lr,#-9]
223 orr r1,r0,r1,lsl#8
224 ldrb r0,[lr,#-8]
225 orr r2,r1,r2,lsl#16
226 ldrb r1,[lr,#-7]
227 orr r3,r2,r3,lsl#24
228 ldrb r2,[lr,#-6]
229 adcs $h1,$h1,r3
230
231 ldrb r3,[lr,#-5]
232 orr r1,r0,r1,lsl#8
233 ldrb r0,[lr,#-4]
234 orr r2,r1,r2,lsl#16
235 ldrb r1,[lr,#-3]
236 orr r3,r2,r3,lsl#24
237 ldrb r2,[lr,#-2]
238 adcs $h2,$h2,r3
239
240 ldrb r3,[lr,#-1]
241 orr r1,r0,r1,lsl#8
242 str lr,[sp,#8] @ offload input pointer
243 orr r2,r1,r2,lsl#16
244 add $s1,$r1,$r1,lsr#2
245 orr r3,r2,r3,lsl#24
246 #else
247 ldr r0,[lr],#16 @ load input
248 # ifdef __thumb2__
249 it hi
250 # endif
251 addhi $h4,$h4,#1 @ padbit
252 ldr r1,[lr,#-12]
253 ldr r2,[lr,#-8]
254 ldr r3,[lr,#-4]
255 # ifdef __ARMEB__
256 rev r0,r0
257 rev r1,r1
258 rev r2,r2
259 rev r3,r3
260 # endif
261 adds $h0,$h0,r0 @ accumulate input
262 str lr,[sp,#8] @ offload input pointer
263 adcs $h1,$h1,r1
264 add $s1,$r1,$r1,lsr#2
265 adcs $h2,$h2,r2
266 #endif
267 add $s2,$r2,$r2,lsr#2
268 adcs $h3,$h3,r3
269 add $s3,$r3,$r3,lsr#2
270
271 umull r2,r3,$h1,$r0
272 adc $h4,$h4,#0
273 umull r0,r1,$h0,$r0
274 umlal r2,r3,$h4,$s1
275 umlal r0,r1,$h3,$s1
276 ldr $r1,[sp,#20] @ reload $r1
277 umlal r2,r3,$h2,$s3
278 umlal r0,r1,$h1,$s3
279 umlal r2,r3,$h3,$s2
280 umlal r0,r1,$h2,$s2
281 umlal r2,r3,$h0,$r1
282 str r0,[sp,#0] @ future $h0
283 mul r0,$s2,$h4
284 ldr $r2,[sp,#24] @ reload $r2
285 adds r2,r2,r1 @ d1+=d0>>32
286 eor r1,r1,r1
287 adc lr,r3,#0 @ future $h2
288 str r2,[sp,#4] @ future $h1
289
290 mul r2,$s3,$h4
291 eor r3,r3,r3
292 umlal r0,r1,$h3,$s3
293 ldr $r3,[sp,#28] @ reload $r3
294 umlal r2,r3,$h3,$r0
295 umlal r0,r1,$h2,$r0
296 umlal r2,r3,$h2,$r1
297 umlal r0,r1,$h1,$r1
298 umlal r2,r3,$h1,$r2
299 umlal r0,r1,$h0,$r2
300 umlal r2,r3,$h0,$r3
301 ldr $h0,[sp,#0]
302 mul $h4,$r0,$h4
303 ldr $h1,[sp,#4]
304
305 adds $h2,lr,r0 @ d2+=d1>>32
306 ldr lr,[sp,#8] @ reload input pointer
307 adc r1,r1,#0
308 adds $h3,r2,r1 @ d3+=d2>>32
309 ldr r0,[sp,#16] @ reload end pointer
310 adc r3,r3,#0
311 add $h4,$h4,r3 @ h4+=d3>>32
312
313 and r1,$h4,#-4
314 and $h4,$h4,#3
315 add r1,r1,r1,lsr#2 @ *=5
316 adds $h0,$h0,r1
317 adcs $h1,$h1,#0
318 adcs $h2,$h2,#0
319 adcs $h3,$h3,#0
320 adc $h4,$h4,#0
321
322 cmp r0,lr @ done yet?
323 bhi .Loop
324
325 ldr $ctx,[sp,#12]
326 add sp,sp,#32
327 stmia $ctx,{$h0-$h4} @ store the result
328
329 .Lno_data:
330 #if __ARM_ARCH__>=5
331 ldmia sp!,{r3-r11,pc}
332 #else
333 ldmia sp!,{r3-r11,lr}
334 tst lr,#1
335 moveq pc,lr @ be binary compatible with V4, yet
336 bx lr @ interoperable with Thumb ISA:-)
337 #endif
338 .size poly1305_blocks,.-poly1305_blocks
339 ___
340 }
341 {
342 my ($ctx,$mac,$nonce)=map("r$_",(0..2));
343 my ($h0,$h1,$h2,$h3,$h4,$g0,$g1,$g2,$g3)=map("r$_",(3..11));
344 my $g4=$h4;
345
346 $code.=<<___;
347 .type poly1305_emit,%function
348 .align 5
349 poly1305_emit:
350 .Lpoly1305_emit:
351 stmdb sp!,{r4-r11}
352 .Lpoly1305_emit_enter:
353
354 ldmia $ctx,{$h0-$h4}
355 adds $g0,$h0,#5 @ compare to modulus
356 adcs $g1,$h1,#0
357 adcs $g2,$h2,#0
358 adcs $g3,$h3,#0
359 adc $g4,$h4,#0
360 tst $g4,#4 @ did it carry/borrow?
361
362 #ifdef __thumb2__
363 it ne
364 #endif
365 movne $h0,$g0
366 ldr $g0,[$nonce,#0]
367 #ifdef __thumb2__
368 it ne
369 #endif
370 movne $h1,$g1
371 ldr $g1,[$nonce,#4]
372 #ifdef __thumb2__
373 it ne
374 #endif
375 movne $h2,$g2
376 ldr $g2,[$nonce,#8]
377 #ifdef __thumb2__
378 it ne
379 #endif
380 movne $h3,$g3
381 ldr $g3,[$nonce,#12]
382
383 adds $h0,$h0,$g0
384 adcs $h1,$h1,$g1
385 adcs $h2,$h2,$g2
386 adc $h3,$h3,$g3
387
388 #if __ARM_ARCH__>=7
389 # ifdef __ARMEB__
390 rev $h0,$h0
391 rev $h1,$h1
392 rev $h2,$h2
393 rev $h3,$h3
394 # endif
395 str $h0,[$mac,#0]
396 str $h1,[$mac,#4]
397 str $h2,[$mac,#8]
398 str $h3,[$mac,#12]
399 #else
400 strb $h0,[$mac,#0]
401 mov $h0,$h0,lsr#8
402 strb $h1,[$mac,#4]
403 mov $h1,$h1,lsr#8
404 strb $h2,[$mac,#8]
405 mov $h2,$h2,lsr#8
406 strb $h3,[$mac,#12]
407 mov $h3,$h3,lsr#8
408
409 strb $h0,[$mac,#1]
410 mov $h0,$h0,lsr#8
411 strb $h1,[$mac,#5]
412 mov $h1,$h1,lsr#8
413 strb $h2,[$mac,#9]
414 mov $h2,$h2,lsr#8
415 strb $h3,[$mac,#13]
416 mov $h3,$h3,lsr#8
417
418 strb $h0,[$mac,#2]
419 mov $h0,$h0,lsr#8
420 strb $h1,[$mac,#6]
421 mov $h1,$h1,lsr#8
422 strb $h2,[$mac,#10]
423 mov $h2,$h2,lsr#8
424 strb $h3,[$mac,#14]
425 mov $h3,$h3,lsr#8
426
427 strb $h0,[$mac,#3]
428 strb $h1,[$mac,#7]
429 strb $h2,[$mac,#11]
430 strb $h3,[$mac,#15]
431 #endif
432 ldmia sp!,{r4-r11}
433 #if __ARM_ARCH__>=5
434 ret @ bx lr
435 #else
436 tst lr,#1
437 moveq pc,lr @ be binary compatible with V4, yet
438 bx lr @ interoperable with Thumb ISA:-)
439 #endif
440 .size poly1305_emit,.-poly1305_emit
441 ___
442 {
443 my ($R0,$R1,$S1,$R2,$S2,$R3,$S3,$R4,$S4) = map("d$_",(0..9));
444 my ($D0,$D1,$D2,$D3,$D4, $H0,$H1,$H2,$H3,$H4) = map("q$_",(5..14));
445 my ($T0,$T1,$MASK) = map("q$_",(15,4,0));
446
447 my ($in2,$zeros,$tbl0,$tbl1) = map("r$_",(4..7));
448
449 $code.=<<___;
450 #if __ARM_MAX_ARCH__>=7
451 .fpu neon
452
453 .type poly1305_init_neon,%function
454 .align 5
455 poly1305_init_neon:
456 ldr r4,[$ctx,#20] @ load key base 2^32
457 ldr r5,[$ctx,#24]
458 ldr r6,[$ctx,#28]
459 ldr r7,[$ctx,#32]
460
461 and r2,r4,#0x03ffffff @ base 2^32 -> base 2^26
462 mov r3,r4,lsr#26
463 mov r4,r5,lsr#20
464 orr r3,r3,r5,lsl#6
465 mov r5,r6,lsr#14
466 orr r4,r4,r6,lsl#12
467 mov r6,r7,lsr#8
468 orr r5,r5,r7,lsl#18
469 and r3,r3,#0x03ffffff
470 and r4,r4,#0x03ffffff
471 and r5,r5,#0x03ffffff
472
473 vdup.32 $R0,r2 @ r^1 in both lanes
474 add r2,r3,r3,lsl#2 @ *5
475 vdup.32 $R1,r3
476 add r3,r4,r4,lsl#2
477 vdup.32 $S1,r2
478 vdup.32 $R2,r4
479 add r4,r5,r5,lsl#2
480 vdup.32 $S2,r3
481 vdup.32 $R3,r5
482 add r5,r6,r6,lsl#2
483 vdup.32 $S3,r4
484 vdup.32 $R4,r6
485 vdup.32 $S4,r5
486
487 mov $zeros,#2 @ counter
488
489 .Lsquare_neon:
490 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
491 @ d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
492 @ d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
493 @ d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
494 @ d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
495 @ d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
496
497 vmull.u32 $D0,$R0,${R0}[1]
498 vmull.u32 $D1,$R1,${R0}[1]
499 vmull.u32 $D2,$R2,${R0}[1]
500 vmull.u32 $D3,$R3,${R0}[1]
501 vmull.u32 $D4,$R4,${R0}[1]
502
503 vmlal.u32 $D0,$R4,${S1}[1]
504 vmlal.u32 $D1,$R0,${R1}[1]
505 vmlal.u32 $D2,$R1,${R1}[1]
506 vmlal.u32 $D3,$R2,${R1}[1]
507 vmlal.u32 $D4,$R3,${R1}[1]
508
509 vmlal.u32 $D0,$R3,${S2}[1]
510 vmlal.u32 $D1,$R4,${S2}[1]
511 vmlal.u32 $D3,$R1,${R2}[1]
512 vmlal.u32 $D2,$R0,${R2}[1]
513 vmlal.u32 $D4,$R2,${R2}[1]
514
515 vmlal.u32 $D0,$R2,${S3}[1]
516 vmlal.u32 $D3,$R0,${R3}[1]
517 vmlal.u32 $D1,$R3,${S3}[1]
518 vmlal.u32 $D2,$R4,${S3}[1]
519 vmlal.u32 $D4,$R1,${R3}[1]
520
521 vmlal.u32 $D3,$R4,${S4}[1]
522 vmlal.u32 $D0,$R1,${S4}[1]
523 vmlal.u32 $D1,$R2,${S4}[1]
524 vmlal.u32 $D2,$R3,${S4}[1]
525 vmlal.u32 $D4,$R0,${R4}[1]
526
527 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
528 @ lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
529 @ and P. Schwabe
530 @
531 @ H0>>+H1>>+H2>>+H3>>+H4
532 @ H3>>+H4>>*5+H0>>+H1
533 @
534 @ Trivia.
535 @
536 @ Result of multiplication of n-bit number by m-bit number is
537 @ n+m bits wide. However! Even though 2^n is a n+1-bit number,
538 @ m-bit number multiplied by 2^n is still n+m bits wide.
539 @
540 @ Sum of two n-bit numbers is n+1 bits wide, sum of three - n+2,
541 @ and so is sum of four. Sum of 2^m n-m-bit numbers and n-bit
542 @ one is n+1 bits wide.
543 @
544 @ >>+ denotes Hnext += Hn>>26, Hn &= 0x3ffffff. This means that
545 @ H0, H2, H3 are guaranteed to be 26 bits wide, while H1 and H4
546 @ can be 27. However! In cases when their width exceeds 26 bits
547 @ they are limited by 2^26+2^6. This in turn means that *sum*
548 @ of the products with these values can still be viewed as sum
549 @ of 52-bit numbers as long as the amount of addends is not a
550 @ power of 2. For example,
551 @
552 @ H4 = H4*R0 + H3*R1 + H2*R2 + H1*R3 + H0 * R4,
553 @
554 @ which can't be larger than 5 * (2^26 + 2^6) * (2^26 + 2^6), or
555 @ 5 * (2^52 + 2*2^32 + 2^12), which in turn is smaller than
556 @ 8 * (2^52) or 2^55. However, the value is then multiplied by
557 @ by 5, so we should be looking at 5 * 5 * (2^52 + 2^33 + 2^12),
558 @ which is less than 32 * (2^52) or 2^57. And when processing
559 @ data we are looking at triple as many addends...
560 @
561 @ In key setup procedure pre-reduced H0 is limited by 5*4+1 and
562 @ 5*H4 - by 5*5 52-bit addends, or 57 bits. But when hashing the
563 @ input H0 is limited by (5*4+1)*3 addends, or 58 bits, while
564 @ 5*H4 by 5*5*3, or 59[!] bits. How is this relevant? vmlal.u32
565 @ instruction accepts 2x32-bit input and writes 2x64-bit result.
566 @ This means that result of reduction have to be compressed upon
567 @ loop wrap-around. This can be done in the process of reduction
568 @ to minimize amount of instructions [as well as amount of
569 @ 128-bit instructions, which benefits low-end processors], but
570 @ one has to watch for H2 (which is narrower than H0) and 5*H4
571 @ not being wider than 58 bits, so that result of right shift
572 @ by 26 bits fits in 32 bits. This is also useful on x86,
573 @ because it allows to use paddd in place for paddq, which
574 @ benefits Atom, where paddq is ridiculously slow.
575
576 vshr.u64 $T0,$D3,#26
577 vmovn.i64 $D3#lo,$D3
578 vshr.u64 $T1,$D0,#26
579 vmovn.i64 $D0#lo,$D0
580 vadd.i64 $D4,$D4,$T0 @ h3 -> h4
581 vbic.i32 $D3#lo,#0xfc000000 @ &=0x03ffffff
582 vadd.i64 $D1,$D1,$T1 @ h0 -> h1
583 vbic.i32 $D0#lo,#0xfc000000
584
585 vshrn.u64 $T0#lo,$D4,#26
586 vmovn.i64 $D4#lo,$D4
587 vshr.u64 $T1,$D1,#26
588 vmovn.i64 $D1#lo,$D1
589 vadd.i64 $D2,$D2,$T1 @ h1 -> h2
590 vbic.i32 $D4#lo,#0xfc000000
591 vbic.i32 $D1#lo,#0xfc000000
592
593 vadd.i32 $D0#lo,$D0#lo,$T0#lo
594 vshl.u32 $T0#lo,$T0#lo,#2
595 vshrn.u64 $T1#lo,$D2,#26
596 vmovn.i64 $D2#lo,$D2
597 vadd.i32 $D0#lo,$D0#lo,$T0#lo @ h4 -> h0
598 vadd.i32 $D3#lo,$D3#lo,$T1#lo @ h2 -> h3
599 vbic.i32 $D2#lo,#0xfc000000
600
601 vshr.u32 $T0#lo,$D0#lo,#26
602 vbic.i32 $D0#lo,#0xfc000000
603 vshr.u32 $T1#lo,$D3#lo,#26
604 vbic.i32 $D3#lo,#0xfc000000
605 vadd.i32 $D1#lo,$D1#lo,$T0#lo @ h0 -> h1
606 vadd.i32 $D4#lo,$D4#lo,$T1#lo @ h3 -> h4
607
608 subs $zeros,$zeros,#1
609 beq .Lsquare_break_neon
610
611 add $tbl0,$ctx,#(48+0*9*4)
612 add $tbl1,$ctx,#(48+1*9*4)
613
614 vtrn.32 $R0,$D0#lo @ r^2:r^1
615 vtrn.32 $R2,$D2#lo
616 vtrn.32 $R3,$D3#lo
617 vtrn.32 $R1,$D1#lo
618 vtrn.32 $R4,$D4#lo
619
620 vshl.u32 $S2,$R2,#2 @ *5
621 vshl.u32 $S3,$R3,#2
622 vshl.u32 $S1,$R1,#2
623 vshl.u32 $S4,$R4,#2
624 vadd.i32 $S2,$S2,$R2
625 vadd.i32 $S1,$S1,$R1
626 vadd.i32 $S3,$S3,$R3
627 vadd.i32 $S4,$S4,$R4
628
629 vst4.32 {${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]!
630 vst4.32 {${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]!
631 vst4.32 {${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]!
632 vst4.32 {${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]!
633 vst1.32 {${S4}[0]},[$tbl0,:32]
634 vst1.32 {${S4}[1]},[$tbl1,:32]
635
636 b .Lsquare_neon
637
638 .align 4
639 .Lsquare_break_neon:
640 add $tbl0,$ctx,#(48+2*4*9)
641 add $tbl1,$ctx,#(48+3*4*9)
642
643 vmov $R0,$D0#lo @ r^4:r^3
644 vshl.u32 $S1,$D1#lo,#2 @ *5
645 vmov $R1,$D1#lo
646 vshl.u32 $S2,$D2#lo,#2
647 vmov $R2,$D2#lo
648 vshl.u32 $S3,$D3#lo,#2
649 vmov $R3,$D3#lo
650 vshl.u32 $S4,$D4#lo,#2
651 vmov $R4,$D4#lo
652 vadd.i32 $S1,$S1,$D1#lo
653 vadd.i32 $S2,$S2,$D2#lo
654 vadd.i32 $S3,$S3,$D3#lo
655 vadd.i32 $S4,$S4,$D4#lo
656
657 vst4.32 {${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]!
658 vst4.32 {${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]!
659 vst4.32 {${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]!
660 vst4.32 {${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]!
661 vst1.32 {${S4}[0]},[$tbl0]
662 vst1.32 {${S4}[1]},[$tbl1]
663
664 ret @ bx lr
665 .size poly1305_init_neon,.-poly1305_init_neon
666
667 .type poly1305_blocks_neon,%function
668 .align 5
669 poly1305_blocks_neon:
670 .Lpoly1305_blocks_neon:
671 ldr ip,[$ctx,#36] @ is_base2_26
672 ands $len,$len,#-16
673 beq .Lno_data_neon
674
675 cmp $len,#64
676 bhs .Lenter_neon
677 tst ip,ip @ is_base2_26?
678 beq .Lpoly1305_blocks
679
680 .Lenter_neon:
681 stmdb sp!,{r4-r7}
682 vstmdb sp!,{d8-d15} @ ABI specification says so
683
684 tst ip,ip @ is_base2_26?
685 bne .Lbase2_26_neon
686
687 stmdb sp!,{r1-r3,lr}
688 bl poly1305_init_neon
689
690 ldr r4,[$ctx,#0] @ load hash value base 2^32
691 ldr r5,[$ctx,#4]
692 ldr r6,[$ctx,#8]
693 ldr r7,[$ctx,#12]
694 ldr ip,[$ctx,#16]
695
696 and r2,r4,#0x03ffffff @ base 2^32 -> base 2^26
697 mov r3,r4,lsr#26
698 veor $D0#lo,$D0#lo,$D0#lo
699 mov r4,r5,lsr#20
700 orr r3,r3,r5,lsl#6
701 veor $D1#lo,$D1#lo,$D1#lo
702 mov r5,r6,lsr#14
703 orr r4,r4,r6,lsl#12
704 veor $D2#lo,$D2#lo,$D2#lo
705 mov r6,r7,lsr#8
706 orr r5,r5,r7,lsl#18
707 veor $D3#lo,$D3#lo,$D3#lo
708 and r3,r3,#0x03ffffff
709 orr r6,r6,ip,lsl#24
710 veor $D4#lo,$D4#lo,$D4#lo
711 and r4,r4,#0x03ffffff
712 mov r1,#1
713 and r5,r5,#0x03ffffff
714 str r1,[$ctx,#36] @ is_base2_26
715
716 vmov.32 $D0#lo[0],r2
717 vmov.32 $D1#lo[0],r3
718 vmov.32 $D2#lo[0],r4
719 vmov.32 $D3#lo[0],r5
720 vmov.32 $D4#lo[0],r6
721 adr $zeros,.Lzeros
722
723 ldmia sp!,{r1-r3,lr}
724 b .Lbase2_32_neon
725
726 .align 4
727 .Lbase2_26_neon:
728 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
729 @ load hash value
730
731 veor $D0#lo,$D0#lo,$D0#lo
732 veor $D1#lo,$D1#lo,$D1#lo
733 veor $D2#lo,$D2#lo,$D2#lo
734 veor $D3#lo,$D3#lo,$D3#lo
735 veor $D4#lo,$D4#lo,$D4#lo
736 vld4.32 {$D0#lo[0],$D1#lo[0],$D2#lo[0],$D3#lo[0]},[$ctx]!
737 adr $zeros,.Lzeros
738 vld1.32 {$D4#lo[0]},[$ctx]
739 sub $ctx,$ctx,#16 @ rewind
740
741 .Lbase2_32_neon:
742 add $in2,$inp,#32
743 mov $padbit,$padbit,lsl#24
744 tst $len,#31
745 beq .Leven
746
747 vld4.32 {$H0#lo[0],$H1#lo[0],$H2#lo[0],$H3#lo[0]},[$inp]!
748 vmov.32 $H4#lo[0],$padbit
749 sub $len,$len,#16
750 add $in2,$inp,#32
751
752 # ifdef __ARMEB__
753 vrev32.8 $H0,$H0
754 vrev32.8 $H3,$H3
755 vrev32.8 $H1,$H1
756 vrev32.8 $H2,$H2
757 # endif
758 vsri.u32 $H4#lo,$H3#lo,#8 @ base 2^32 -> base 2^26
759 vshl.u32 $H3#lo,$H3#lo,#18
760
761 vsri.u32 $H3#lo,$H2#lo,#14
762 vshl.u32 $H2#lo,$H2#lo,#12
763 vadd.i32 $H4#hi,$H4#lo,$D4#lo @ add hash value and move to #hi
764
765 vbic.i32 $H3#lo,#0xfc000000
766 vsri.u32 $H2#lo,$H1#lo,#20
767 vshl.u32 $H1#lo,$H1#lo,#6
768
769 vbic.i32 $H2#lo,#0xfc000000
770 vsri.u32 $H1#lo,$H0#lo,#26
771 vadd.i32 $H3#hi,$H3#lo,$D3#lo
772
773 vbic.i32 $H0#lo,#0xfc000000
774 vbic.i32 $H1#lo,#0xfc000000
775 vadd.i32 $H2#hi,$H2#lo,$D2#lo
776
777 vadd.i32 $H0#hi,$H0#lo,$D0#lo
778 vadd.i32 $H1#hi,$H1#lo,$D1#lo
779
780 mov $tbl1,$zeros
781 add $tbl0,$ctx,#48
782
783 cmp $len,$len
784 b .Long_tail
785
786 .align 4
787 .Leven:
788 subs $len,$len,#64
789 it lo
790 movlo $in2,$zeros
791
792 vmov.i32 $H4,#1<<24 @ padbit, yes, always
793 vld4.32 {$H0#lo,$H1#lo,$H2#lo,$H3#lo},[$inp] @ inp[0:1]
794 add $inp,$inp,#64
795 vld4.32 {$H0#hi,$H1#hi,$H2#hi,$H3#hi},[$in2] @ inp[2:3] (or 0)
796 add $in2,$in2,#64
797 itt hi
798 addhi $tbl1,$ctx,#(48+1*9*4)
799 addhi $tbl0,$ctx,#(48+3*9*4)
800
801 # ifdef __ARMEB__
802 vrev32.8 $H0,$H0
803 vrev32.8 $H3,$H3
804 vrev32.8 $H1,$H1
805 vrev32.8 $H2,$H2
806 # endif
807 vsri.u32 $H4,$H3,#8 @ base 2^32 -> base 2^26
808 vshl.u32 $H3,$H3,#18
809
810 vsri.u32 $H3,$H2,#14
811 vshl.u32 $H2,$H2,#12
812
813 vbic.i32 $H3,#0xfc000000
814 vsri.u32 $H2,$H1,#20
815 vshl.u32 $H1,$H1,#6
816
817 vbic.i32 $H2,#0xfc000000
818 vsri.u32 $H1,$H0,#26
819
820 vbic.i32 $H0,#0xfc000000
821 vbic.i32 $H1,#0xfc000000
822
823 bls .Lskip_loop
824
825 vld4.32 {${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]! @ load r^2
826 vld4.32 {${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]! @ load r^4
827 vld4.32 {${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]!
828 vld4.32 {${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]!
829 b .Loop_neon
830
831 .align 5
832 .Loop_neon:
833 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
834 @ ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2
835 @ ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r
836 @ \___________________/
837 @ ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2
838 @ ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r
839 @ \___________________/ \____________________/
840 @
841 @ Note that we start with inp[2:3]*r^2. This is because it
842 @ doesn't depend on reduction in previous iteration.
843 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
844 @ d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
845 @ d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
846 @ d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
847 @ d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
848 @ d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
849
850 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
851 @ inp[2:3]*r^2
852
853 vadd.i32 $H2#lo,$H2#lo,$D2#lo @ accumulate inp[0:1]
854 vmull.u32 $D2,$H2#hi,${R0}[1]
855 vadd.i32 $H0#lo,$H0#lo,$D0#lo
856 vmull.u32 $D0,$H0#hi,${R0}[1]
857 vadd.i32 $H3#lo,$H3#lo,$D3#lo
858 vmull.u32 $D3,$H3#hi,${R0}[1]
859 vmlal.u32 $D2,$H1#hi,${R1}[1]
860 vadd.i32 $H1#lo,$H1#lo,$D1#lo
861 vmull.u32 $D1,$H1#hi,${R0}[1]
862
863 vadd.i32 $H4#lo,$H4#lo,$D4#lo
864 vmull.u32 $D4,$H4#hi,${R0}[1]
865 subs $len,$len,#64
866 vmlal.u32 $D0,$H4#hi,${S1}[1]
867 it lo
868 movlo $in2,$zeros
869 vmlal.u32 $D3,$H2#hi,${R1}[1]
870 vld1.32 ${S4}[1],[$tbl1,:32]
871 vmlal.u32 $D1,$H0#hi,${R1}[1]
872 vmlal.u32 $D4,$H3#hi,${R1}[1]
873
874 vmlal.u32 $D0,$H3#hi,${S2}[1]
875 vmlal.u32 $D3,$H1#hi,${R2}[1]
876 vmlal.u32 $D4,$H2#hi,${R2}[1]
877 vmlal.u32 $D1,$H4#hi,${S2}[1]
878 vmlal.u32 $D2,$H0#hi,${R2}[1]
879
880 vmlal.u32 $D3,$H0#hi,${R3}[1]
881 vmlal.u32 $D0,$H2#hi,${S3}[1]
882 vmlal.u32 $D4,$H1#hi,${R3}[1]
883 vmlal.u32 $D1,$H3#hi,${S3}[1]
884 vmlal.u32 $D2,$H4#hi,${S3}[1]
885
886 vmlal.u32 $D3,$H4#hi,${S4}[1]
887 vmlal.u32 $D0,$H1#hi,${S4}[1]
888 vmlal.u32 $D4,$H0#hi,${R4}[1]
889 vmlal.u32 $D1,$H2#hi,${S4}[1]
890 vmlal.u32 $D2,$H3#hi,${S4}[1]
891
892 vld4.32 {$H0#hi,$H1#hi,$H2#hi,$H3#hi},[$in2] @ inp[2:3] (or 0)
893 add $in2,$in2,#64
894
895 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
896 @ (hash+inp[0:1])*r^4 and accumulate
897
898 vmlal.u32 $D3,$H3#lo,${R0}[0]
899 vmlal.u32 $D0,$H0#lo,${R0}[0]
900 vmlal.u32 $D4,$H4#lo,${R0}[0]
901 vmlal.u32 $D1,$H1#lo,${R0}[0]
902 vmlal.u32 $D2,$H2#lo,${R0}[0]
903 vld1.32 ${S4}[0],[$tbl0,:32]
904
905 vmlal.u32 $D3,$H2#lo,${R1}[0]
906 vmlal.u32 $D0,$H4#lo,${S1}[0]
907 vmlal.u32 $D4,$H3#lo,${R1}[0]
908 vmlal.u32 $D1,$H0#lo,${R1}[0]
909 vmlal.u32 $D2,$H1#lo,${R1}[0]
910
911 vmlal.u32 $D3,$H1#lo,${R2}[0]
912 vmlal.u32 $D0,$H3#lo,${S2}[0]
913 vmlal.u32 $D4,$H2#lo,${R2}[0]
914 vmlal.u32 $D1,$H4#lo,${S2}[0]
915 vmlal.u32 $D2,$H0#lo,${R2}[0]
916
917 vmlal.u32 $D3,$H0#lo,${R3}[0]
918 vmlal.u32 $D0,$H2#lo,${S3}[0]
919 vmlal.u32 $D4,$H1#lo,${R3}[0]
920 vmlal.u32 $D1,$H3#lo,${S3}[0]
921 vmlal.u32 $D3,$H4#lo,${S4}[0]
922
923 vmlal.u32 $D2,$H4#lo,${S3}[0]
924 vmlal.u32 $D0,$H1#lo,${S4}[0]
925 vmlal.u32 $D4,$H0#lo,${R4}[0]
926 vmov.i32 $H4,#1<<24 @ padbit, yes, always
927 vmlal.u32 $D1,$H2#lo,${S4}[0]
928 vmlal.u32 $D2,$H3#lo,${S4}[0]
929
930 vld4.32 {$H0#lo,$H1#lo,$H2#lo,$H3#lo},[$inp] @ inp[0:1]
931 add $inp,$inp,#64
932 # ifdef __ARMEB__
933 vrev32.8 $H0,$H0
934 vrev32.8 $H1,$H1
935 vrev32.8 $H2,$H2
936 vrev32.8 $H3,$H3
937 # endif
938
939 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
940 @ lazy reduction interleaved with base 2^32 -> base 2^26 of
941 @ inp[0:3] previously loaded to $H0-$H3 and smashed to $H0-$H4.
942
943 vshr.u64 $T0,$D3,#26
944 vmovn.i64 $D3#lo,$D3
945 vshr.u64 $T1,$D0,#26
946 vmovn.i64 $D0#lo,$D0
947 vadd.i64 $D4,$D4,$T0 @ h3 -> h4
948 vbic.i32 $D3#lo,#0xfc000000
949 vsri.u32 $H4,$H3,#8 @ base 2^32 -> base 2^26
950 vadd.i64 $D1,$D1,$T1 @ h0 -> h1
951 vshl.u32 $H3,$H3,#18
952 vbic.i32 $D0#lo,#0xfc000000
953
954 vshrn.u64 $T0#lo,$D4,#26
955 vmovn.i64 $D4#lo,$D4
956 vshr.u64 $T1,$D1,#26
957 vmovn.i64 $D1#lo,$D1
958 vadd.i64 $D2,$D2,$T1 @ h1 -> h2
959 vsri.u32 $H3,$H2,#14
960 vbic.i32 $D4#lo,#0xfc000000
961 vshl.u32 $H2,$H2,#12
962 vbic.i32 $D1#lo,#0xfc000000
963
964 vadd.i32 $D0#lo,$D0#lo,$T0#lo
965 vshl.u32 $T0#lo,$T0#lo,#2
966 vbic.i32 $H3,#0xfc000000
967 vshrn.u64 $T1#lo,$D2,#26
968 vmovn.i64 $D2#lo,$D2
969 vaddl.u32 $D0,$D0#lo,$T0#lo @ h4 -> h0 [widen for a sec]
970 vsri.u32 $H2,$H1,#20
971 vadd.i32 $D3#lo,$D3#lo,$T1#lo @ h2 -> h3
972 vshl.u32 $H1,$H1,#6
973 vbic.i32 $D2#lo,#0xfc000000
974 vbic.i32 $H2,#0xfc000000
975
976 vshrn.u64 $T0#lo,$D0,#26 @ re-narrow
977 vmovn.i64 $D0#lo,$D0
978 vsri.u32 $H1,$H0,#26
979 vbic.i32 $H0,#0xfc000000
980 vshr.u32 $T1#lo,$D3#lo,#26
981 vbic.i32 $D3#lo,#0xfc000000
982 vbic.i32 $D0#lo,#0xfc000000
983 vadd.i32 $D1#lo,$D1#lo,$T0#lo @ h0 -> h1
984 vadd.i32 $D4#lo,$D4#lo,$T1#lo @ h3 -> h4
985 vbic.i32 $H1,#0xfc000000
986
987 bhi .Loop_neon
988
989 .Lskip_loop:
990 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
991 @ multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1
992
993 add $tbl1,$ctx,#(48+0*9*4)
994 add $tbl0,$ctx,#(48+1*9*4)
995 adds $len,$len,#32
996 it ne
997 movne $len,#0
998 bne .Long_tail
999
1000 vadd.i32 $H2#hi,$H2#lo,$D2#lo @ add hash value and move to #hi
1001 vadd.i32 $H0#hi,$H0#lo,$D0#lo
1002 vadd.i32 $H3#hi,$H3#lo,$D3#lo
1003 vadd.i32 $H1#hi,$H1#lo,$D1#lo
1004 vadd.i32 $H4#hi,$H4#lo,$D4#lo
1005
1006 .Long_tail:
1007 vld4.32 {${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]! @ load r^1
1008 vld4.32 {${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]! @ load r^2
1009
1010 vadd.i32 $H2#lo,$H2#lo,$D2#lo @ can be redundant
1011 vmull.u32 $D2,$H2#hi,$R0
1012 vadd.i32 $H0#lo,$H0#lo,$D0#lo
1013 vmull.u32 $D0,$H0#hi,$R0
1014 vadd.i32 $H3#lo,$H3#lo,$D3#lo
1015 vmull.u32 $D3,$H3#hi,$R0
1016 vadd.i32 $H1#lo,$H1#lo,$D1#lo
1017 vmull.u32 $D1,$H1#hi,$R0
1018 vadd.i32 $H4#lo,$H4#lo,$D4#lo
1019 vmull.u32 $D4,$H4#hi,$R0
1020
1021 vmlal.u32 $D0,$H4#hi,$S1
1022 vld4.32 {${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]!
1023 vmlal.u32 $D3,$H2#hi,$R1
1024 vld4.32 {${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]!
1025 vmlal.u32 $D1,$H0#hi,$R1
1026 vmlal.u32 $D4,$H3#hi,$R1
1027 vmlal.u32 $D2,$H1#hi,$R1
1028
1029 vmlal.u32 $D3,$H1#hi,$R2
1030 vld1.32 ${S4}[1],[$tbl1,:32]
1031 vmlal.u32 $D0,$H3#hi,$S2
1032 vld1.32 ${S4}[0],[$tbl0,:32]
1033 vmlal.u32 $D4,$H2#hi,$R2
1034 vmlal.u32 $D1,$H4#hi,$S2
1035 vmlal.u32 $D2,$H0#hi,$R2
1036
1037 vmlal.u32 $D3,$H0#hi,$R3
1038 it ne
1039 addne $tbl1,$ctx,#(48+2*9*4)
1040 vmlal.u32 $D0,$H2#hi,$S3
1041 it ne
1042 addne $tbl0,$ctx,#(48+3*9*4)
1043 vmlal.u32 $D4,$H1#hi,$R3
1044 vmlal.u32 $D1,$H3#hi,$S3
1045 vmlal.u32 $D2,$H4#hi,$S3
1046
1047 vmlal.u32 $D3,$H4#hi,$S4
1048 vorn $MASK,$MASK,$MASK @ all-ones, can be redundant
1049 vmlal.u32 $D0,$H1#hi,$S4
1050 vshr.u64 $MASK,$MASK,#38
1051 vmlal.u32 $D4,$H0#hi,$R4
1052 vmlal.u32 $D1,$H2#hi,$S4
1053 vmlal.u32 $D2,$H3#hi,$S4
1054
1055 beq .Lshort_tail
1056
1057 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
1058 @ (hash+inp[0:1])*r^4:r^3 and accumulate
1059
1060 vld4.32 {${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]! @ load r^3
1061 vld4.32 {${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]! @ load r^4
1062
1063 vmlal.u32 $D2,$H2#lo,$R0
1064 vmlal.u32 $D0,$H0#lo,$R0
1065 vmlal.u32 $D3,$H3#lo,$R0
1066 vmlal.u32 $D1,$H1#lo,$R0
1067 vmlal.u32 $D4,$H4#lo,$R0
1068
1069 vmlal.u32 $D0,$H4#lo,$S1
1070 vld4.32 {${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]!
1071 vmlal.u32 $D3,$H2#lo,$R1
1072 vld4.32 {${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]!
1073 vmlal.u32 $D1,$H0#lo,$R1
1074 vmlal.u32 $D4,$H3#lo,$R1
1075 vmlal.u32 $D2,$H1#lo,$R1
1076
1077 vmlal.u32 $D3,$H1#lo,$R2
1078 vld1.32 ${S4}[1],[$tbl1,:32]
1079 vmlal.u32 $D0,$H3#lo,$S2
1080 vld1.32 ${S4}[0],[$tbl0,:32]
1081 vmlal.u32 $D4,$H2#lo,$R2
1082 vmlal.u32 $D1,$H4#lo,$S2
1083 vmlal.u32 $D2,$H0#lo,$R2
1084
1085 vmlal.u32 $D3,$H0#lo,$R3
1086 vmlal.u32 $D0,$H2#lo,$S3
1087 vmlal.u32 $D4,$H1#lo,$R3
1088 vmlal.u32 $D1,$H3#lo,$S3
1089 vmlal.u32 $D2,$H4#lo,$S3
1090
1091 vmlal.u32 $D3,$H4#lo,$S4
1092 vorn $MASK,$MASK,$MASK @ all-ones
1093 vmlal.u32 $D0,$H1#lo,$S4
1094 vshr.u64 $MASK,$MASK,#38
1095 vmlal.u32 $D4,$H0#lo,$R4
1096 vmlal.u32 $D1,$H2#lo,$S4
1097 vmlal.u32 $D2,$H3#lo,$S4
1098
1099 .Lshort_tail:
1100 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
1101 @ horizontal addition
1102
1103 vadd.i64 $D3#lo,$D3#lo,$D3#hi
1104 vadd.i64 $D0#lo,$D0#lo,$D0#hi
1105 vadd.i64 $D4#lo,$D4#lo,$D4#hi
1106 vadd.i64 $D1#lo,$D1#lo,$D1#hi
1107 vadd.i64 $D2#lo,$D2#lo,$D2#hi
1108
1109 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
1110 @ lazy reduction, but without narrowing
1111
1112 vshr.u64 $T0,$D3,#26
1113 vand.i64 $D3,$D3,$MASK
1114 vshr.u64 $T1,$D0,#26
1115 vand.i64 $D0,$D0,$MASK
1116 vadd.i64 $D4,$D4,$T0 @ h3 -> h4
1117 vadd.i64 $D1,$D1,$T1 @ h0 -> h1
1118
1119 vshr.u64 $T0,$D4,#26
1120 vand.i64 $D4,$D4,$MASK
1121 vshr.u64 $T1,$D1,#26
1122 vand.i64 $D1,$D1,$MASK
1123 vadd.i64 $D2,$D2,$T1 @ h1 -> h2
1124
1125 vadd.i64 $D0,$D0,$T0
1126 vshl.u64 $T0,$T0,#2
1127 vshr.u64 $T1,$D2,#26
1128 vand.i64 $D2,$D2,$MASK
1129 vadd.i64 $D0,$D0,$T0 @ h4 -> h0
1130 vadd.i64 $D3,$D3,$T1 @ h2 -> h3
1131
1132 vshr.u64 $T0,$D0,#26
1133 vand.i64 $D0,$D0,$MASK
1134 vshr.u64 $T1,$D3,#26
1135 vand.i64 $D3,$D3,$MASK
1136 vadd.i64 $D1,$D1,$T0 @ h0 -> h1
1137 vadd.i64 $D4,$D4,$T1 @ h3 -> h4
1138
1139 cmp $len,#0
1140 bne .Leven
1141
1142 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
1143 @ store hash value
1144
1145 vst4.32 {$D0#lo[0],$D1#lo[0],$D2#lo[0],$D3#lo[0]},[$ctx]!
1146 vst1.32 {$D4#lo[0]},[$ctx]
1147
1148 vldmia sp!,{d8-d15} @ epilogue
1149 ldmia sp!,{r4-r7}
1150 .Lno_data_neon:
1151 ret @ bx lr
1152 .size poly1305_blocks_neon,.-poly1305_blocks_neon
1153
1154 .type poly1305_emit_neon,%function
1155 .align 5
1156 poly1305_emit_neon:
1157 .Lpoly1305_emit_neon:
1158 ldr ip,[$ctx,#36] @ is_base2_26
1159
1160 stmdb sp!,{r4-r11}
1161
1162 tst ip,ip
1163 beq .Lpoly1305_emit_enter
1164
1165 ldmia $ctx,{$h0-$h4}
1166 eor $g0,$g0,$g0
1167
1168 adds $h0,$h0,$h1,lsl#26 @ base 2^26 -> base 2^32
1169 mov $h1,$h1,lsr#6
1170 adcs $h1,$h1,$h2,lsl#20
1171 mov $h2,$h2,lsr#12
1172 adcs $h2,$h2,$h3,lsl#14
1173 mov $h3,$h3,lsr#18
1174 adcs $h3,$h3,$h4,lsl#8
1175 adc $h4,$g0,$h4,lsr#24 @ can be partially reduced ...
1176
1177 and $g0,$h4,#-4 @ ... so reduce
1178 and $h4,$h3,#3
1179 add $g0,$g0,$g0,lsr#2 @ *= 5
1180 adds $h0,$h0,$g0
1181 adcs $h1,$h1,#0
1182 adcs $h2,$h2,#0
1183 adcs $h3,$h3,#0
1184 adc $h4,$h4,#0
1185
1186 adds $g0,$h0,#5 @ compare to modulus
1187 adcs $g1,$h1,#0
1188 adcs $g2,$h2,#0
1189 adcs $g3,$h3,#0
1190 adc $g4,$h4,#0
1191 tst $g4,#4 @ did it carry/borrow?
1192
1193 it ne
1194 movne $h0,$g0
1195 ldr $g0,[$nonce,#0]
1196 it ne
1197 movne $h1,$g1
1198 ldr $g1,[$nonce,#4]
1199 it ne
1200 movne $h2,$g2
1201 ldr $g2,[$nonce,#8]
1202 it ne
1203 movne $h3,$g3
1204 ldr $g3,[$nonce,#12]
1205
1206 adds $h0,$h0,$g0 @ accumulate nonce
1207 adcs $h1,$h1,$g1
1208 adcs $h2,$h2,$g2
1209 adc $h3,$h3,$g3
1210
1211 # ifdef __ARMEB__
1212 rev $h0,$h0
1213 rev $h1,$h1
1214 rev $h2,$h2
1215 rev $h3,$h3
1216 # endif
1217 str $h0,[$mac,#0] @ store the result
1218 str $h1,[$mac,#4]
1219 str $h2,[$mac,#8]
1220 str $h3,[$mac,#12]
1221
1222 ldmia sp!,{r4-r11}
1223 ret @ bx lr
1224 .size poly1305_emit_neon,.-poly1305_emit_neon
1225
1226 .align 5
1227 .Lzeros:
1228 .long 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
1229 .LOPENSSL_armcap:
1230 # ifdef _WIN32
1231 .word OPENSSL_armcap_P
1232 # else
1233 .word OPENSSL_armcap_P-.Lpoly1305_init
1234 # endif
1235 #endif
1236 ___
1237 } }
1238 $code.=<<___;
1239 .asciz "Poly1305 for ARMv4/NEON, CRYPTOGAMS by <appro\@openssl.org>"
1240 .align 2
1241 #if __ARM_MAX_ARCH__>=7
1242 .extern OPENSSL_armcap_P
1243 #endif
1244 ___
1245
1246 foreach (split("\n",$code)) {
1247 s/\`([^\`]*)\`/eval $1/geo;
1248
1249 s/\bq([0-9]+)#(lo|hi)/sprintf "d%d",2*$1+($2 eq "hi")/geo or
1250 s/\bret\b/bx lr/go or
1251 s/\bbx\s+lr\b/.word\t0xe12fff1e/go; # make it possible to compile with -march=armv4
1252
1253 print $_,"\n";
1254 }
1255 close STDOUT or die "error closing STDOUT: $!"; # enforce flush
A mirror of the official openssl repository