]> git.ipfire.org Git - thirdparty/openssl.git/blob - crypto/poly1305/asm/poly1305-x86.pl
projects / thirdparty / openssl.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
crypto/poly1305/asm: chase overflow bit on x86 and ARM platforms.
[thirdparty/openssl.git] / crypto / poly1305 / asm / poly1305-x86.pl
1 #!/usr/bin/env perl
2 #
3 # ====================================================================
4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
5 # project. The module is, however, dual licensed under OpenSSL and
6 # CRYPTOGAMS licenses depending on where you obtain it. For further
7 # details see http://www.openssl.org/~appro/cryptogams/.
8 # ====================================================================
9 #
10 # This module implements Poly1305 hash for x86.
11 #
12 # April 2015
13 #
14 # Numbers are cycles per processed byte with poly1305_blocks alone,
15 # measured with rdtsc at fixed clock frequency.
16 #
17 # IALU/gcc-3.4(*) SSE2(**) AVX2
18 # Pentium 15.7/+80% -
19 # PIII 6.21/+90% -
20 # P4 19.8/+40% 3.24
21 # Core 2 4.85/+90% 1.80
22 # Westmere 4.58/+100% 1.43
23 # Sandy Bridge 3.90/+100% 1.36
24 # Haswell 3.88/+70% 1.18 0.72
25 # Silvermont 11.0/+40% 4.80
26 # VIA Nano 6.71/+90% 2.47
27 # Sledgehammer 3.51/+180% 4.27
28 # Bulldozer 4.53/+140% 1.31
29 #
30 # (*) gcc 4.8 for some reason generated worse code;
31 # (**) besides SSE2 there are floating-point and AVX options; FP
32 # is deemed unnecessary, because pre-SSE2 processor are too
33 # old to care about, while it's not the fastest option on
34 # SSE2-capable ones; AVX is omitted, because it doesn't give
35 # a lot of improvement, 5-10% depending on processor;
36
37 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
38 push(@INC,"${dir}","${dir}../../perlasm");
39 require "x86asm.pl";
40
41 $output=pop;
42 open STDOUT,">$output";
43
44 &asm_init($ARGV[0],"poly1305-x86.pl",$ARGV[$#ARGV] eq "386");
45
46 $sse2=$avx=0;
47 for (@ARGV) { $sse2=1 if (/-DOPENSSL_IA32_SSE2/); }
48
49 if ($sse2) {
50 &static_label("const_sse2");
51 &static_label("enter_blocks");
52 &static_label("enter_emit");
53 &external_label("OPENSSL_ia32cap_P");
54
55 if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
56 =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
57 $avx = ($1>=2.19) + ($1>=2.22);
58 }
59
60 if (!$avx && $ARGV[0] eq "win32n" &&
61 `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
62 $avx = ($1>=2.09) + ($1>=2.10);
63 }
64
65 if (!$avx && `$ENV{CC} -v 2>&1` =~ /(^clang version|based on LLVM) ([3-9]\.[0-9]+)/) {
66 $avx = ($2>=3.0) + ($2>3.0);
67 }
68 }
69
70 ########################################################################
71 # Layout of opaque area is following.
72 #
73 # unsigned __int32 h[5]; # current hash value base 2^32
74 # unsigned __int32 pad; # is_base2_26 in vector context
75 # unsigned __int32 r[4]; # key value base 2^32
76
77 &align(64);
78 &function_begin("poly1305_init");
79 &mov ("edi",&wparam(0)); # context
80 &mov ("esi",&wparam(1)); # key
81 &mov ("ebp",&wparam(2)); # function table
82
83 &xor ("eax","eax");
84 &mov (&DWP(4*0,"edi"),"eax"); # zero hash value
85 &mov (&DWP(4*1,"edi"),"eax");
86 &mov (&DWP(4*2,"edi"),"eax");
87 &mov (&DWP(4*3,"edi"),"eax");
88 &mov (&DWP(4*4,"edi"),"eax");
89 &mov (&DWP(4*5,"edi"),"eax"); # is_base2_26
90
91 &cmp ("esi",0);
92 &je (&label("nokey"));
93
94 if ($sse2) {
95 &call (&label("pic_point"));
96 &set_label("pic_point");
97 &blindpop("ebx");
98
99 &lea ("eax",&DWP("poly1305_blocks-".&label("pic_point"),"ebx"));
100 &lea ("edx",&DWP("poly1305_emit-".&label("pic_point"),"ebx"));
101
102 &picmeup("edi","OPENSSL_ia32cap_P","ebx",&label("pic_point"));
103 &mov ("ecx",&DWP(0,"edi"));
104 &and ("ecx",1<<26|1<<24);
105 &cmp ("ecx",1<<26|1<<24); # SSE2 and XMM?
106 &jne (&label("no_sse2"));
107
108 &lea ("eax",&DWP("_poly1305_blocks_sse2-".&label("pic_point"),"ebx"));
109 &lea ("edx",&DWP("_poly1305_emit_sse2-".&label("pic_point"),"ebx"));
110
111 if ($avx>1) {
112 &mov ("ecx",&DWP(8,"edi"));
113 &test ("ecx",1<<5); # AVX2?
114 &jz (&label("no_sse2"));
115
116 &lea ("eax",&DWP("_poly1305_blocks_avx2-".&label("pic_point"),"ebx"));
117 }
118 &set_label("no_sse2");
119 &mov ("edi",&wparam(0)); # reload context
120 &mov (&DWP(0,"ebp"),"eax"); # fill function table
121 &mov (&DWP(4,"ebp"),"edx");
122 }
123
124 &mov ("eax",&DWP(4*0,"esi")); # load input key
125 &mov ("ebx",&DWP(4*1,"esi"));
126 &mov ("ecx",&DWP(4*2,"esi"));
127 &mov ("edx",&DWP(4*3,"esi"));
128 &and ("eax",0x0fffffff);
129 &and ("ebx",0x0ffffffc);
130 &and ("ecx",0x0ffffffc);
131 &and ("edx",0x0ffffffc);
132 &mov (&DWP(4*6,"edi"),"eax");
133 &mov (&DWP(4*7,"edi"),"ebx");
134 &mov (&DWP(4*8,"edi"),"ecx");
135 &mov (&DWP(4*9,"edi"),"edx");
136
137 &mov ("eax",$sse2);
138 &set_label("nokey");
139 &function_end("poly1305_init");
140
141 ($h0,$h1,$h2,$h3,$h4,
142 $d0,$d1,$d2,$d3,
143 $r0,$r1,$r2,$r3,
144 $s1,$s2,$s3)=map(4*$_,(0..15));
145
146 &function_begin("poly1305_blocks");
147 &mov ("edi",&wparam(0)); # ctx
148 &mov ("esi",&wparam(1)); # inp
149 &mov ("ecx",&wparam(2)); # len
150 &set_label("enter_blocks");
151 &and ("ecx",-15);
152 &jz (&label("nodata"));
153
154 &stack_push(16);
155 &mov ("eax",&DWP(4*6,"edi")); # r0
156 &mov ("ebx",&DWP(4*7,"edi")); # r1
157 &lea ("ebp",&DWP(0,"esi","ecx")); # end of input
158 &mov ("ecx",&DWP(4*8,"edi")); # r2
159 &mov ("edx",&DWP(4*9,"edi")); # r3
160
161 &mov (&wparam(2),"ebp");
162 &mov ("ebp","esi");
163
164 &mov (&DWP($r0,"esp"),"eax"); # r0
165 &mov ("eax","ebx");
166 &shr ("eax",2);
167 &mov (&DWP($r1,"esp"),"ebx"); # r1
168 &add ("eax","ebx"); # s1
169 &mov ("ebx","ecx");
170 &shr ("ebx",2);
171 &mov (&DWP($r2,"esp"),"ecx"); # r2
172 &add ("ebx","ecx"); # s2
173 &mov ("ecx","edx");
174 &shr ("ecx",2);
175 &mov (&DWP($r3,"esp"),"edx"); # r3
176 &add ("ecx","edx"); # s3
177 &mov (&DWP($s1,"esp"),"eax"); # s1
178 &mov (&DWP($s2,"esp"),"ebx"); # s2
179 &mov (&DWP($s3,"esp"),"ecx"); # s3
180
181 &mov ("eax",&DWP(4*0,"edi")); # load hash value
182 &mov ("ebx",&DWP(4*1,"edi"));
183 &mov ("ecx",&DWP(4*2,"edi"));
184 &mov ("esi",&DWP(4*3,"edi"));
185 &mov ("edi",&DWP(4*4,"edi"));
186 &jmp (&label("loop"));
187
188 &set_label("loop",32);
189 &add ("eax",&DWP(4*0,"ebp")); # accumulate input
190 &adc ("ebx",&DWP(4*1,"ebp"));
191 &adc ("ecx",&DWP(4*2,"ebp"));
192 &adc ("esi",&DWP(4*3,"ebp"));
193 &lea ("ebp",&DWP(4*4,"ebp"));
194 &adc ("edi",&wparam(3)); # padbit
195
196 &mov (&DWP($h0,"esp"),"eax"); # put aside hash[+inp]
197 &mov (&DWP($h3,"esp"),"esi");
198
199 &mul (&DWP($r0,"esp")); # h0*r0
200 &mov (&DWP($h4,"esp"),"edi");
201 &mov ("edi","eax");
202 &mov ("eax","ebx"); # h1
203 &mov ("esi","edx");
204 &mul (&DWP($s3,"esp")); # h1*s3
205 &add ("edi","eax");
206 &mov ("eax","ecx"); # h2
207 &adc ("esi","edx");
208 &mul (&DWP($s2,"esp")); # h2*s2
209 &add ("edi","eax");
210 &mov ("eax",&DWP($h3,"esp"));
211 &adc ("esi","edx");
212 &mul (&DWP($s1,"esp")); # h3*s1
213 &add ("edi","eax");
214 &mov ("eax",&DWP($h0,"esp"));
215 &adc ("esi","edx");
216
217 &mul (&DWP($r1,"esp")); # h0*r1
218 &mov (&DWP($d0,"esp"),"edi");
219 &xor ("edi","edi");
220 &add ("esi","eax");
221 &mov ("eax","ebx"); # h1
222 &adc ("edi","edx");
223 &mul (&DWP($r0,"esp")); # h1*r0
224 &add ("esi","eax");
225 &mov ("eax","ecx"); # h2
226 &adc ("edi","edx");
227 &mul (&DWP($s3,"esp")); # h2*s3
228 &add ("esi","eax");
229 &mov ("eax",&DWP($h3,"esp"));
230 &adc ("edi","edx");
231 &mul (&DWP($s2,"esp")); # h3*s2
232 &add ("esi","eax");
233 &mov ("eax",&DWP($h4,"esp"));
234 &adc ("edi","edx");
235 &imul ("eax",&DWP($s1,"esp")); # h4*s1
236 &add ("esi","eax");
237 &mov ("eax",&DWP($h0,"esp"));
238 &adc ("edi",0);
239
240 &mul (&DWP($r2,"esp")); # h0*r2
241 &mov (&DWP($d1,"esp"),"esi");
242 &xor ("esi","esi");
243 &add ("edi","eax");
244 &mov ("eax","ebx"); # h1
245 &adc ("esi","edx");
246 &mul (&DWP($r1,"esp")); # h1*r1
247 &add ("edi","eax");
248 &mov ("eax","ecx"); # h2
249 &adc ("esi","edx");
250 &mul (&DWP($r0,"esp")); # h2*r0
251 &add ("edi","eax");
252 &mov ("eax",&DWP($h3,"esp"));
253 &adc ("esi","edx");
254 &mul (&DWP($s3,"esp")); # h3*s3
255 &add ("edi","eax");
256 &mov ("eax",&DWP($h4,"esp"));
257 &adc ("esi","edx");
258 &imul ("eax",&DWP($s2,"esp")); # h4*s2
259 &add ("edi","eax");
260 &mov ("eax",&DWP($h0,"esp"));
261 &adc ("esi",0);
262
263 &mul (&DWP($r3,"esp")); # h0*r3
264 &mov (&DWP($d2,"esp"),"edi");
265 &xor ("edi","edi");
266 &add ("esi","eax");
267 &mov ("eax","ebx"); # h1
268 &adc ("edi","edx");
269 &mul (&DWP($r2,"esp")); # h1*r2
270 &add ("esi","eax");
271 &mov ("eax","ecx"); # h2
272 &adc ("edi","edx");
273 &mul (&DWP($r1,"esp")); # h2*r1
274 &add ("esi","eax");
275 &mov ("eax",&DWP($h3,"esp"));
276 &adc ("edi","edx");
277 &mul (&DWP($r0,"esp")); # h3*r0
278 &add ("esi","eax");
279 &mov ("ecx",&DWP($h4,"esp"));
280 &adc ("edi","edx");
281
282 &mov ("edx","ecx");
283 &imul ("ecx",&DWP($s3,"esp")); # h4*s3
284 &add ("esi","ecx");
285 &mov ("eax",&DWP($d0,"esp"));
286 &adc ("edi",0);
287
288 &imul ("edx",&DWP($r0,"esp")); # h4*r0
289 &add ("edx","edi");
290
291 &mov ("ebx",&DWP($d1,"esp"));
292 &mov ("ecx",&DWP($d2,"esp"));
293
294 &mov ("edi","edx"); # last reduction step
295 &shr ("edx",2);
296 &and ("edi",3);
297 &lea ("edx",&DWP(0,"edx","edx",4)); # *5
298 &add ("eax","edx");
299 &adc ("ebx",0);
300 &adc ("ecx",0);
301 &adc ("esi",0);
302 &adc ("edi",0);
303
304 &cmp ("ebp",&wparam(2)); # done yet?
305 &jne (&label("loop"));
306
307 &mov ("edx",&wparam(0)); # ctx
308 &stack_pop(16);
309 &mov (&DWP(4*0,"edx"),"eax"); # store hash value
310 &mov (&DWP(4*1,"edx"),"ebx");
311 &mov (&DWP(4*2,"edx"),"ecx");
312 &mov (&DWP(4*3,"edx"),"esi");
313 &mov (&DWP(4*4,"edx"),"edi");
314 &set_label("nodata");
315 &function_end("poly1305_blocks");
316
317 &function_begin("poly1305_emit");
318 &mov ("ebp",&wparam(0)); # context
319 &set_label("enter_emit");
320 &mov ("edi",&wparam(1)); # output
321 &mov ("eax",&DWP(4*0,"ebp")); # load hash value
322 &mov ("ebx",&DWP(4*1,"ebp"));
323 &mov ("ecx",&DWP(4*2,"ebp"));
324 &mov ("edx",&DWP(4*3,"ebp"));
325 &mov ("esi",&DWP(4*4,"ebp"));
326
327 &add ("eax",5); # compare to modulus
328 &adc ("ebx",0);
329 &adc ("ecx",0);
330 &adc ("edx",0);
331 &adc ("esi",0);
332 &shr ("esi",2); # did it carry/borrow?
333 &neg ("esi"); # do we choose hash-modulus?
334
335 &and ("eax","esi");
336 &and ("ebx","esi");
337 &and ("ecx","esi");
338 &and ("edx","esi");
339 &mov (&DWP(4*0,"edi"),"eax");
340 &mov (&DWP(4*1,"edi"),"ebx");
341 &mov (&DWP(4*2,"edi"),"ecx");
342 &mov (&DWP(4*3,"edi"),"edx");
343
344 &not ("esi"); # or original hash value?
345 &mov ("eax",&DWP(4*0,"ebp"));
346 &mov ("ebx",&DWP(4*1,"ebp"));
347 &mov ("ecx",&DWP(4*2,"ebp"));
348 &mov ("edx",&DWP(4*3,"ebp"));
349 &mov ("ebp",&wparam(2));
350 &and ("eax","esi");
351 &and ("ebx","esi");
352 &and ("ecx","esi");
353 &and ("edx","esi");
354 &or ("eax",&DWP(4*0,"edi"));
355 &or ("ebx",&DWP(4*1,"edi"));
356 &or ("ecx",&DWP(4*2,"edi"));
357 &or ("edx",&DWP(4*3,"edi"));
358
359 &add ("eax",&DWP(4*0,"ebp")); # accumulate key
360 &adc ("ebx",&DWP(4*1,"ebp"));
361 &adc ("ecx",&DWP(4*2,"ebp"));
362 &adc ("edx",&DWP(4*3,"ebp"));
363
364 &mov (&DWP(4*0,"edi"),"eax");
365 &mov (&DWP(4*1,"edi"),"ebx");
366 &mov (&DWP(4*2,"edi"),"ecx");
367 &mov (&DWP(4*3,"edi"),"edx");
368 &function_end("poly1305_emit");
369
370 if ($sse2) {
371 ########################################################################
372 # Layout of opaque area is following.
373 #
374 # unsigned __int32 h[5]; # current hash value base 2^26
375 # unsigned __int32 is_base2_26;
376 # unsigned __int32 r[4]; # key value base 2^32
377 # unsigned __int32 pad[2];
378 # struct { unsigned __int32 r^4, r^3, r^2, r^1; } r[9];
379 #
380 # where r^n are base 2^26 digits of degrees of multiplier key. There are
381 # 5 digits, but last four are interleaved with multiples of 5, totalling
382 # in 9 elements: r0, r1, 5*r1, r2, 5*r2, r3, 5*r3, r4, 5*r4.
383
384 my ($D0,$D1,$D2,$D3,$D4,$T0,$T1,$T2)=map("xmm$_",(0..7));
385 my $MASK=$T2; # borrow and keep in mind
386
387 &align (32);
388 &function_begin_B("_poly1305_init_sse2");
389 &movdqu ($D4,&QWP(4*6,"edi")); # key base 2^32
390 &lea ("edi",&DWP(16*3,"edi")); # size optimization
391 &mov ("ebp","esp");
392 &sub ("esp",16*(9+5));
393 &and ("esp",-16);
394
395 #&pand ($D4,&QWP(96,"ebx")); # magic mask
396 &movq ($MASK,&QWP(64,"ebx"));
397
398 &movdqa ($D0,$D4);
399 &movdqa ($D1,$D4);
400 &movdqa ($D2,$D4);
401
402 &pand ($D0,$MASK); # -> base 2^26
403 &psrlq ($D1,26);
404 &psrldq ($D2,6);
405 &pand ($D1,$MASK);
406 &movdqa ($D3,$D2);
407 &psrlq ($D2,4)
408 &psrlq ($D3,30);
409 &pand ($D2,$MASK);
410 &pand ($D3,$MASK);
411 &psrldq ($D4,13);
412
413 &lea ("edx",&DWP(16*9,"esp")); # size optimization
414 &mov ("ecx",2);
415 &set_label("square");
416 &movdqa (&QWP(16*0,"esp"),$D0);
417 &movdqa (&QWP(16*1,"esp"),$D1);
418 &movdqa (&QWP(16*2,"esp"),$D2);
419 &movdqa (&QWP(16*3,"esp"),$D3);
420 &movdqa (&QWP(16*4,"esp"),$D4);
421
422 &movdqa ($T1,$D1);
423 &movdqa ($T0,$D2);
424 &pslld ($T1,2);
425 &pslld ($T0,2);
426 &paddd ($T1,$D1); # *5
427 &paddd ($T0,$D2); # *5
428 &movdqa (&QWP(16*5,"esp"),$T1);
429 &movdqa (&QWP(16*6,"esp"),$T0);
430 &movdqa ($T1,$D3);
431 &movdqa ($T0,$D4);
432 &pslld ($T1,2);
433 &pslld ($T0,2);
434 &paddd ($T1,$D3); # *5
435 &paddd ($T0,$D4); # *5
436 &movdqa (&QWP(16*7,"esp"),$T1);
437 &movdqa (&QWP(16*8,"esp"),$T0);
438
439 &pshufd ($T1,$D0,0b01000100);
440 &movdqa ($T0,$D1);
441 &pshufd ($D1,$D1,0b01000100);
442 &pshufd ($D2,$D2,0b01000100);
443 &pshufd ($D3,$D3,0b01000100);
444 &pshufd ($D4,$D4,0b01000100);
445 &movdqa (&QWP(16*0,"edx"),$T1);
446 &movdqa (&QWP(16*1,"edx"),$D1);
447 &movdqa (&QWP(16*2,"edx"),$D2);
448 &movdqa (&QWP(16*3,"edx"),$D3);
449 &movdqa (&QWP(16*4,"edx"),$D4);
450
451 ################################################################
452 # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
453 # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
454 # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
455 # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
456 # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
457
458 &pmuludq ($D4,$D0); # h4*r0
459 &pmuludq ($D3,$D0); # h3*r0
460 &pmuludq ($D2,$D0); # h2*r0
461 &pmuludq ($D1,$D0); # h1*r0
462 &pmuludq ($D0,$T1); # h0*r0
463
464 sub pmuladd {
465 my $load = shift;
466 my $base = shift; $base = "esp" if (!defined($base));
467
468 ################################################################
469 # As for choice to "rotate" $T0-$T2 in order to move paddq
470 # past next multiplication. While it makes code harder to read
471 # and doesn't have significant effect on most processors, it
472 # makes a lot of difference on Atom, up to 30% improvement.
473
474 &movdqa ($T1,$T0);
475 &pmuludq ($T0,&QWP(16*3,$base)); # r1*h3
476 &movdqa ($T2,$T1);
477 &pmuludq ($T1,&QWP(16*2,$base)); # r1*h2
478 &paddq ($D4,$T0);
479 &movdqa ($T0,$T2);
480 &pmuludq ($T2,&QWP(16*1,$base)); # r1*h1
481 &paddq ($D3,$T1);
482 &$load ($T1,5); # s1
483 &pmuludq ($T0,&QWP(16*0,$base)); # r1*h0
484 &paddq ($D2,$T2);
485 &pmuludq ($T1,&QWP(16*4,$base)); # s1*h4
486 &$load ($T2,2); # r2^n
487 &paddq ($D1,$T0);
488
489 &movdqa ($T0,$T2);
490 &pmuludq ($T2,&QWP(16*2,$base)); # r2*h2
491 &paddq ($D0,$T1);
492 &movdqa ($T1,$T0);
493 &pmuludq ($T0,&QWP(16*1,$base)); # r2*h1
494 &paddq ($D4,$T2);
495 &$load ($T2,6); # s2^n
496 &pmuludq ($T1,&QWP(16*0,$base)); # r2*h0
497 &paddq ($D3,$T0);
498 &movdqa ($T0,$T2);
499 &pmuludq ($T2,&QWP(16*4,$base)); # s2*h4
500 &paddq ($D2,$T1);
501 &pmuludq ($T0,&QWP(16*3,$base)); # s2*h3
502 &$load ($T1,3); # r3^n
503 &paddq ($D1,$T2);
504
505 &movdqa ($T2,$T1);
506 &pmuludq ($T1,&QWP(16*1,$base)); # r3*h1
507 &paddq ($D0,$T0);
508 &$load ($T0,7); # s3^n
509 &pmuludq ($T2,&QWP(16*0,$base)); # r3*h0
510 &paddq ($D4,$T1);
511 &movdqa ($T1,$T0);
512 &pmuludq ($T0,&QWP(16*4,$base)); # s3*h4
513 &paddq ($D3,$T2);
514 &movdqa ($T2,$T1);
515 &pmuludq ($T1,&QWP(16*3,$base)); # s3*h3
516 &paddq ($D2,$T0);
517 &pmuludq ($T2,&QWP(16*2,$base)); # s3*h2
518 &$load ($T0,4); # r4^n
519 &paddq ($D1,$T1);
520
521 &$load ($T1,8); # s4^n
522 &pmuludq ($T0,&QWP(16*0,$base)); # r4*h0
523 &paddq ($D0,$T2);
524 &movdqa ($T2,$T1);
525 &pmuludq ($T1,&QWP(16*4,$base)); # s4*h4
526 &paddq ($D4,$T0);
527 &movdqa ($T0,$T2);
528 &pmuludq ($T2,&QWP(16*1,$base)); # s4*h1
529 &paddq ($D3,$T1);
530 &movdqa ($T1,$T0);
531 &pmuludq ($T0,&QWP(16*2,$base)); # s4*h2
532 &paddq ($D0,$T2);
533 &pmuludq ($T1,&QWP(16*3,$base)); # s4*h3
534 &movdqa ($MASK,&QWP(64,"ebx"));
535 &paddq ($D1,$T0);
536 &paddq ($D2,$T1);
537 }
538 &pmuladd (sub { my ($reg,$i)=@_;
539 &movdqa ($reg,&QWP(16*$i,"esp"));
540 },"edx");
541
542 sub lazy_reduction {
543 my $extra = shift;
544
545 ################################################################
546 # lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
547 # and P. Schwabe
548 #
549 # [(*) see discussion in poly1305-armv4 module]
550
551 &movdqa ($T0,$D3);
552 &pand ($D3,$MASK);
553 &psrlq ($T0,26);
554 &$extra () if (defined($extra));
555 &paddq ($T0,$D4); # h3 -> h4
556 &movdqa ($T1,$D0);
557 &pand ($D0,$MASK);
558 &psrlq ($T1,26);
559 &movdqa ($D4,$T0);
560 &paddq ($T1,$D1); # h0 -> h1
561 &psrlq ($T0,26);
562 &pand ($D4,$MASK);
563 &movdqa ($D1,$T1);
564 &psrlq ($T1,26);
565 &paddd ($D0,$T0); # favour paddd when
566 # possible, because
567 # paddq is "broken"
568 # on Atom
569 &psllq ($T0,2);
570 &paddq ($T1,$D2); # h1 -> h2
571 &paddq ($T0,$D0); # h4 -> h0 (*)
572 &pand ($D1,$MASK);
573 &movdqa ($D2,$T1);
574 &psrlq ($T1,26);
575 &pand ($D2,$MASK);
576 &paddd ($T1,$D3); # h2 -> h3
577 &movdqa ($D0,$T0);
578 &psrlq ($T0,26);
579 &movdqa ($D3,$T1);
580 &psrlq ($T1,26);
581 &pand ($D0,$MASK);
582 &paddd ($D1,$T0); # h0 -> h1
583 &pand ($D3,$MASK);
584 &paddd ($D4,$T1); # h3 -> h4
585 }
586 &lazy_reduction ();
587
588 &dec ("ecx");
589 &jz (&label("square_break"));
590
591 &punpcklqdq ($D0,&QWP(16*0,"esp")); # 0:r^1:0:r^2
592 &punpcklqdq ($D1,&QWP(16*1,"esp"));
593 &punpcklqdq ($D2,&QWP(16*2,"esp"));
594 &punpcklqdq ($D3,&QWP(16*3,"esp"));
595 &punpcklqdq ($D4,&QWP(16*4,"esp"));
596 &jmp (&label("square"));
597
598 &set_label("square_break");
599 &psllq ($D0,32); # -> r^3:0:r^4:0
600 &psllq ($D1,32);
601 &psllq ($D2,32);
602 &psllq ($D3,32);
603 &psllq ($D4,32);
604 &por ($D0,&QWP(16*0,"esp")); # r^3:r^1:r^4:r^2
605 &por ($D1,&QWP(16*1,"esp"));
606 &por ($D2,&QWP(16*2,"esp"));
607 &por ($D3,&QWP(16*3,"esp"));
608 &por ($D4,&QWP(16*4,"esp"));
609
610 &pshufd ($D0,$D0,0b10001101); # -> r^1:r^2:r^3:r^4
611 &pshufd ($D1,$D1,0b10001101);
612 &pshufd ($D2,$D2,0b10001101);
613 &pshufd ($D3,$D3,0b10001101);
614 &pshufd ($D4,$D4,0b10001101);
615
616 &movdqu (&QWP(16*0,"edi"),$D0); # save the table
617 &movdqu (&QWP(16*1,"edi"),$D1);
618 &movdqu (&QWP(16*2,"edi"),$D2);
619 &movdqu (&QWP(16*3,"edi"),$D3);
620 &movdqu (&QWP(16*4,"edi"),$D4);
621
622 &movdqa ($T1,$D1);
623 &movdqa ($T0,$D2);
624 &pslld ($T1,2);
625 &pslld ($T0,2);
626 &paddd ($T1,$D1); # *5
627 &paddd ($T0,$D2); # *5
628 &movdqu (&QWP(16*5,"edi"),$T1);
629 &movdqu (&QWP(16*6,"edi"),$T0);
630 &movdqa ($T1,$D3);
631 &movdqa ($T0,$D4);
632 &pslld ($T1,2);
633 &pslld ($T0,2);
634 &paddd ($T1,$D3); # *5
635 &paddd ($T0,$D4); # *5
636 &movdqu (&QWP(16*7,"edi"),$T1);
637 &movdqu (&QWP(16*8,"edi"),$T0);
638
639 &mov ("esp","ebp");
640 &lea ("edi",&DWP(-16*3,"edi")); # size de-optimization
641 &ret ();
642 &function_end_B("_poly1305_init_sse2");
643
644 &align (32);
645 &function_begin("_poly1305_blocks_sse2");
646 &mov ("edi",&wparam(0)); # ctx
647 &mov ("esi",&wparam(1)); # inp
648 &mov ("ecx",&wparam(2)); # len
649
650 &mov ("eax",&DWP(4*5,"edi")); # is_base2_26
651 &and ("ecx",-16);
652 &jz (&label("nodata"));
653 &cmp ("ecx",64);
654 &jae (&label("enter_sse2"));
655 &test ("eax","eax"); # is_base2_26?
656 &jz (&label("enter_blocks"));
657
658 &set_label("enter_sse2",16);
659 &call (&label("pic_point"));
660 &set_label("pic_point");
661 &blindpop("ebx");
662 &lea ("ebx",&DWP(&label("const_sse2")."-".&label("pic_point"),"ebx"));
663
664 &test ("eax","eax"); # is_base2_26?
665 &jnz (&label("base2_26"));
666
667 &call ("_poly1305_init_sse2");
668
669 ################################################# base 2^32 -> base 2^26
670 &mov ("eax",&DWP(0,"edi"));
671 &mov ("ecx",&DWP(3,"edi"));
672 &mov ("edx",&DWP(6,"edi"));
673 &mov ("esi",&DWP(9,"edi"));
674 &mov ("ebp",&DWP(13,"edi"));
675 &mov (&DWP(4*5,"edi"),1); # is_base2_26
676
677 &shr ("ecx",2);
678 &and ("eax",0x3ffffff);
679 &shr ("edx",4);
680 &and ("ecx",0x3ffffff);
681 &shr ("esi",6);
682 &and ("edx",0x3ffffff);
683
684 &movd ($D0,"eax");
685 &movd ($D1,"ecx");
686 &movd ($D2,"edx");
687 &movd ($D3,"esi");
688 &movd ($D4,"ebp");
689
690 &mov ("esi",&wparam(1)); # [reload] inp
691 &mov ("ecx",&wparam(2)); # [reload] len
692 &jmp (&label("base2_32"));
693
694 &set_label("base2_26",16);
695 &movd ($D0,&DWP(4*0,"edi")); # load hash value
696 &movd ($D1,&DWP(4*1,"edi"));
697 &movd ($D2,&DWP(4*2,"edi"));
698 &movd ($D3,&DWP(4*3,"edi"));
699 &movd ($D4,&DWP(4*4,"edi"));
700 &movdqa ($MASK,&QWP(64,"ebx"));
701
702 &set_label("base2_32");
703 &mov ("eax",&wparam(3)); # padbit
704 &mov ("ebp","esp");
705
706 &sub ("esp",16*(5+5+5+9+9));
707 &and ("esp",-16);
708
709 &lea ("edi",&DWP(16*3,"edi")); # size optimization
710 &shl ("eax",24); # padbit
711
712 &test ("ecx",31);
713 &jz (&label("even"));
714
715 ################################################################
716 # process single block, with SSE2, because it's still faster
717 # even though half of result is discarded
718
719 &movdqu ($T1,&QWP(0,"esi")); # input
720 &lea ("esi",&DWP(16,"esi"));
721
722 &movdqa ($T0,$T1); # -> base 2^26 ...
723 &pand ($T1,$MASK);
724 &paddd ($D0,$T1); # ... and accumuate
725
726 &movdqa ($T1,$T0);
727 &psrlq ($T0,26);
728 &psrldq ($T1,6);
729 &pand ($T0,$MASK);
730 &paddd ($D1,$T0);
731
732 &movdqa ($T0,$T1);
733 &psrlq ($T1,4);
734 &pand ($T1,$MASK);
735 &paddd ($D2,$T1);
736
737 &movdqa ($T1,$T0);
738 &psrlq ($T0,30);
739 &pand ($T0,$MASK);
740 &psrldq ($T1,7);
741 &paddd ($D3,$T0);
742
743 &movd ($T0,"eax"); # padbit
744 &paddd ($D4,$T1);
745 &movd ($T1,&DWP(16*0+12,"edi")); # r0
746 &paddd ($D4,$T0);
747
748 &movdqa (&QWP(16*0,"esp"),$D0);
749 &movdqa (&QWP(16*1,"esp"),$D1);
750 &movdqa (&QWP(16*2,"esp"),$D2);
751 &movdqa (&QWP(16*3,"esp"),$D3);
752 &movdqa (&QWP(16*4,"esp"),$D4);
753
754 ################################################################
755 # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
756 # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
757 # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
758 # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
759 # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
760
761 &pmuludq ($D0,$T1); # h4*r0
762 &pmuludq ($D1,$T1); # h3*r0
763 &pmuludq ($D2,$T1); # h2*r0
764 &movd ($T0,&DWP(16*1+12,"edi")); # r1
765 &pmuludq ($D3,$T1); # h1*r0
766 &pmuludq ($D4,$T1); # h0*r0
767
768 &pmuladd (sub { my ($reg,$i)=@_;
769 &movd ($reg,&DWP(16*$i+12,"edi"));
770 });
771
772 &lazy_reduction ();
773
774 &sub ("ecx",16);
775 &jz (&label("done"));
776
777 &set_label("even");
778 &lea ("edx",&DWP(16*(5+5+5+9),"esp"));# size optimization
779 &lea ("eax",&DWP(-16*2,"esi"));
780 &sub ("ecx",64);
781
782 ################################################################
783 # expand and copy pre-calculated table to stack
784
785 &movdqu ($T0,&QWP(16*0,"edi")); # r^1:r^2:r^3:r^4
786 &pshufd ($T1,$T0,0b01000100); # duplicate r^3:r^4
787 &cmovb ("esi","eax");
788 &pshufd ($T0,$T0,0b11101110); # duplicate r^1:r^2
789 &movdqa (&QWP(16*0,"edx"),$T1);
790 &lea ("eax",&DWP(16*10,"esp"));
791 &movdqu ($T1,&QWP(16*1,"edi"));
792 &movdqa (&QWP(16*(0-9),"edx"),$T0);
793 &pshufd ($T0,$T1,0b01000100);
794 &pshufd ($T1,$T1,0b11101110);
795 &movdqa (&QWP(16*1,"edx"),$T0);
796 &movdqu ($T0,&QWP(16*2,"edi"));
797 &movdqa (&QWP(16*(1-9),"edx"),$T1);
798 &pshufd ($T1,$T0,0b01000100);
799 &pshufd ($T0,$T0,0b11101110);
800 &movdqa (&QWP(16*2,"edx"),$T1);
801 &movdqu ($T1,&QWP(16*3,"edi"));
802 &movdqa (&QWP(16*(2-9),"edx"),$T0);
803 &pshufd ($T0,$T1,0b01000100);
804 &pshufd ($T1,$T1,0b11101110);
805 &movdqa (&QWP(16*3,"edx"),$T0);
806 &movdqu ($T0,&QWP(16*4,"edi"));
807 &movdqa (&QWP(16*(3-9),"edx"),$T1);
808 &pshufd ($T1,$T0,0b01000100);
809 &pshufd ($T0,$T0,0b11101110);
810 &movdqa (&QWP(16*4,"edx"),$T1);
811 &movdqu ($T1,&QWP(16*5,"edi"));
812 &movdqa (&QWP(16*(4-9),"edx"),$T0);
813 &pshufd ($T0,$T1,0b01000100);
814 &pshufd ($T1,$T1,0b11101110);
815 &movdqa (&QWP(16*5,"edx"),$T0);
816 &movdqu ($T0,&QWP(16*6,"edi"));
817 &movdqa (&QWP(16*(5-9),"edx"),$T1);
818 &pshufd ($T1,$T0,0b01000100);
819 &pshufd ($T0,$T0,0b11101110);
820 &movdqa (&QWP(16*6,"edx"),$T1);
821 &movdqu ($T1,&QWP(16*7,"edi"));
822 &movdqa (&QWP(16*(6-9),"edx"),$T0);
823 &pshufd ($T0,$T1,0b01000100);
824 &pshufd ($T1,$T1,0b11101110);
825 &movdqa (&QWP(16*7,"edx"),$T0);
826 &movdqu ($T0,&QWP(16*8,"edi"));
827 &movdqa (&QWP(16*(7-9),"edx"),$T1);
828 &pshufd ($T1,$T0,0b01000100);
829 &pshufd ($T0,$T0,0b11101110);
830 &movdqa (&QWP(16*8,"edx"),$T1);
831 &movdqa (&QWP(16*(8-9),"edx"),$T0);
832
833 sub load_input {
834 my ($inpbase,$offbase)=@_;
835
836 &movdqu ($T0,&QWP($inpbase+0,"esi")); # load input
837 &movdqu ($T1,&QWP($inpbase+16,"esi"));
838 &lea ("esi",&DWP(16*2,"esi"));
839
840 &movdqa (&QWP($offbase+16*2,"esp"),$D2);
841 &movdqa (&QWP($offbase+16*3,"esp"),$D3);
842 &movdqa (&QWP($offbase+16*4,"esp"),$D4);
843
844 &movdqa ($D2,$T0); # splat input
845 &movdqa ($D3,$T1);
846 &psrldq ($D2,6);
847 &psrldq ($D3,6);
848 &movdqa ($D4,$T0);
849 &punpcklqdq ($D2,$D3); # 2:3
850 &punpckhqdq ($D4,$T1); # 4
851 &punpcklqdq ($T0,$T1); # 0:1
852
853 &movdqa ($D3,$D2);
854 &psrlq ($D2,4);
855 &psrlq ($D3,30);
856 &movdqa ($T1,$T0);
857 &psrlq ($D4,40); # 4
858 &psrlq ($T1,26);
859 &pand ($T0,$MASK); # 0
860 &pand ($T1,$MASK); # 1
861 &pand ($D2,$MASK); # 2
862 &pand ($D3,$MASK); # 3
863 &por ($D4,&QWP(0,"ebx")); # padbit, yes, always
864
865 &movdqa (&QWP($offbase+16*0,"esp"),$D0) if ($offbase);
866 &movdqa (&QWP($offbase+16*1,"esp"),$D1) if ($offbase);
867 }
868 &load_input (16*2,16*5);
869
870 &jbe (&label("skip_loop"));
871 &jmp (&label("loop"));
872
873 &set_label("loop",32);
874 ################################################################
875 # ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2
876 # ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r
877 # \___________________/
878 # ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2
879 # ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r
880 # \___________________/ \____________________/
881 ################################################################
882
883 &movdqa ($T2,&QWP(16*(0-9),"edx")); # r0^2
884 &movdqa (&QWP(16*1,"eax"),$T1);
885 &movdqa (&QWP(16*2,"eax"),$D2);
886 &movdqa (&QWP(16*3,"eax"),$D3);
887 &movdqa (&QWP(16*4,"eax"),$D4);
888
889 ################################################################
890 # d4 = h4*r0 + h0*r4 + h1*r3 + h2*r2 + h3*r1
891 # d3 = h3*r0 + h0*r3 + h1*r2 + h2*r1 + h4*5*r4
892 # d2 = h2*r0 + h0*r2 + h1*r1 + h3*5*r4 + h4*5*r3
893 # d1 = h1*r0 + h0*r1 + h2*5*r4 + h3*5*r3 + h4*5*r2
894 # d0 = h0*r0 + h1*5*r4 + h2*5*r3 + h3*5*r2 + h4*5*r1
895
896 &movdqa ($D1,$T0);
897 &pmuludq ($T0,$T2); # h0*r0
898 &movdqa ($D0,$T1);
899 &pmuludq ($T1,$T2); # h1*r0
900 &pmuludq ($D2,$T2); # h2*r0
901 &pmuludq ($D3,$T2); # h3*r0
902 &pmuludq ($D4,$T2); # h4*r0
903
904 sub pmuladd_alt {
905 my $addr = shift;
906
907 &pmuludq ($D0,&$addr(8)); # h1*s4
908 &movdqa ($T2,$D1);
909 &pmuludq ($D1,&$addr(1)); # h0*r1
910 &paddq ($D0,$T0);
911 &movdqa ($T0,$T2);
912 &pmuludq ($T2,&$addr(2)); # h0*r2
913 &paddq ($D1,$T1);
914 &movdqa ($T1,$T0);
915 &pmuludq ($T0,&$addr(3)); # h0*r3
916 &paddq ($D2,$T2);
917 &movdqa ($T2,&QWP(16*1,"eax")); # pull h1
918 &pmuludq ($T1,&$addr(4)); # h0*r4
919 &paddq ($D3,$T0);
920
921 &movdqa ($T0,$T2);
922 &pmuludq ($T2,&$addr(1)); # h1*r1
923 &paddq ($D4,$T1);
924 &movdqa ($T1,$T0);
925 &pmuludq ($T0,&$addr(2)); # h1*r2
926 &paddq ($D2,$T2);
927 &movdqa ($T2,&QWP(16*2,"eax")); # pull h2
928 &pmuludq ($T1,&$addr(3)); # h1*r3
929 &paddq ($D3,$T0);
930 &movdqa ($T0,$T2);
931 &pmuludq ($T2,&$addr(7)); # h2*s3
932 &paddq ($D4,$T1);
933 &movdqa ($T1,$T0);
934 &pmuludq ($T0,&$addr(8)); # h2*s4
935 &paddq ($D0,$T2);
936
937 &movdqa ($T2,$T1);
938 &pmuludq ($T1,&$addr(1)); # h2*r1
939 &paddq ($D1,$T0);
940 &movdqa ($T0,&QWP(16*3,"eax")); # pull h3
941 &pmuludq ($T2,&$addr(2)); # h2*r2
942 &paddq ($D3,$T1);
943 &movdqa ($T1,$T0);
944 &pmuludq ($T0,&$addr(6)); # h3*s2
945 &paddq ($D4,$T2);
946 &movdqa ($T2,$T1);
947 &pmuludq ($T1,&$addr(7)); # h3*s3
948 &paddq ($D0,$T0);
949 &movdqa ($T0,$T2);
950 &pmuludq ($T2,&$addr(8)); # h3*s4
951 &paddq ($D1,$T1);
952
953 &movdqa ($T1,&QWP(16*4,"eax")); # pull h4
954 &pmuludq ($T0,&$addr(1)); # h3*r1
955 &paddq ($D2,$T2);
956 &movdqa ($T2,$T1);
957 &pmuludq ($T1,&$addr(8)); # h4*s4
958 &paddq ($D4,$T0);
959 &movdqa ($T0,$T2);
960 &pmuludq ($T2,&$addr(5)); # h4*s1
961 &paddq ($D3,$T1);
962 &movdqa ($T1,$T0);
963 &pmuludq ($T0,&$addr(6)); # h4*s2
964 &paddq ($D0,$T2);
965 &movdqa ($MASK,&QWP(64,"ebx"));
966 &pmuludq ($T1,&$addr(7)); # h4*s3
967 &paddq ($D1,$T0);
968 &paddq ($D2,$T1);
969 }
970 &pmuladd_alt (sub { my $i=shift; &QWP(16*($i-9),"edx"); });
971
972 &load_input (-16*2,0);
973 &lea ("eax",&DWP(-16*2,"esi"));
974 &sub ("ecx",64);
975
976 &paddd ($T0,&QWP(16*(5+0),"esp")); # add hash value
977 &paddd ($T1,&QWP(16*(5+1),"esp"));
978 &paddd ($D2,&QWP(16*(5+2),"esp"));
979 &paddd ($D3,&QWP(16*(5+3),"esp"));
980 &paddd ($D4,&QWP(16*(5+4),"esp"));
981
982 &cmovb ("esi","eax");
983 &lea ("eax",&DWP(16*10,"esp"));
984
985 &movdqa ($T2,&QWP(16*0,"edx")); # r0^4
986 &movdqa (&QWP(16*1,"esp"),$D1);
987 &movdqa (&QWP(16*1,"eax"),$T1);
988 &movdqa (&QWP(16*2,"eax"),$D2);
989 &movdqa (&QWP(16*3,"eax"),$D3);
990 &movdqa (&QWP(16*4,"eax"),$D4);
991
992 ################################################################
993 # d4 += h4*r0 + h0*r4 + h1*r3 + h2*r2 + h3*r1
994 # d3 += h3*r0 + h0*r3 + h1*r2 + h2*r1 + h4*5*r4
995 # d2 += h2*r0 + h0*r2 + h1*r1 + h3*5*r4 + h4*5*r3
996 # d1 += h1*r0 + h0*r1 + h2*5*r4 + h3*5*r3 + h4*5*r2
997 # d0 += h0*r0 + h1*5*r4 + h2*5*r3 + h3*5*r2 + h4*5*r1
998
999 &movdqa ($D1,$T0);
1000 &pmuludq ($T0,$T2); # h0*r0
1001 &paddq ($T0,$D0);
1002 &movdqa ($D0,$T1);
1003 &pmuludq ($T1,$T2); # h1*r0
1004 &pmuludq ($D2,$T2); # h2*r0
1005 &pmuludq ($D3,$T2); # h3*r0
1006 &pmuludq ($D4,$T2); # h4*r0
1007
1008 &paddq ($T1,&QWP(16*1,"esp"));
1009 &paddq ($D2,&QWP(16*2,"esp"));
1010 &paddq ($D3,&QWP(16*3,"esp"));
1011 &paddq ($D4,&QWP(16*4,"esp"));
1012
1013 &pmuladd_alt (sub { my $i=shift; &QWP(16*$i,"edx"); });
1014
1015 &lazy_reduction ();
1016
1017 &load_input (16*2,16*5);
1018
1019 &ja (&label("loop"));
1020
1021 &set_label("skip_loop");
1022 ################################################################
1023 # multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1
1024
1025 &pshufd ($T2,&QWP(16*(0-9),"edx"),0x10);# r0^n
1026 &add ("ecx",32);
1027 &jnz (&label("long_tail"));
1028
1029 &paddd ($T0,$D0); # add hash value
1030 &paddd ($T1,$D1);
1031 &paddd ($D2,&QWP(16*7,"esp"));
1032 &paddd ($D3,&QWP(16*8,"esp"));
1033 &paddd ($D4,&QWP(16*9,"esp"));
1034
1035 &set_label("long_tail");
1036
1037 &movdqa (&QWP(16*0,"eax"),$T0);
1038 &movdqa (&QWP(16*1,"eax"),$T1);
1039 &movdqa (&QWP(16*2,"eax"),$D2);
1040 &movdqa (&QWP(16*3,"eax"),$D3);
1041 &movdqa (&QWP(16*4,"eax"),$D4);
1042
1043 ################################################################
1044 # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
1045 # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
1046 # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
1047 # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
1048 # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
1049
1050 &pmuludq ($T0,$T2); # h0*r0
1051 &pmuludq ($T1,$T2); # h1*r0
1052 &pmuludq ($D2,$T2); # h2*r0
1053 &movdqa ($D0,$T0);
1054 &pshufd ($T0,&QWP(16*(1-9),"edx"),0x10);# r1^n
1055 &pmuludq ($D3,$T2); # h3*r0
1056 &movdqa ($D1,$T1);
1057 &pmuludq ($D4,$T2); # h4*r0
1058
1059 &pmuladd (sub { my ($reg,$i)=@_;
1060 &pshufd ($reg,&QWP(16*($i-9),"edx"),0x10);
1061 },"eax");
1062
1063 &jz (&label("short_tail"));
1064
1065 &load_input (-16*2,0);
1066
1067 &pshufd ($T2,&QWP(16*0,"edx"),0x10); # r0^n
1068 &paddd ($T0,&QWP(16*5,"esp")); # add hash value
1069 &paddd ($T1,&QWP(16*6,"esp"));
1070 &paddd ($D2,&QWP(16*7,"esp"));
1071 &paddd ($D3,&QWP(16*8,"esp"));
1072 &paddd ($D4,&QWP(16*9,"esp"));
1073
1074 ################################################################
1075 # multiply inp[0:1] by r^4:r^3 and accumulate
1076
1077 &movdqa (&QWP(16*0,"esp"),$T0);
1078 &pmuludq ($T0,$T2); # h0*r0
1079 &movdqa (&QWP(16*1,"esp"),$T1);
1080 &pmuludq ($T1,$T2); # h1*r0
1081 &paddq ($D0,$T0);
1082 &movdqa ($T0,$D2);
1083 &pmuludq ($D2,$T2); # h2*r0
1084 &paddq ($D1,$T1);
1085 &movdqa ($T1,$D3);
1086 &pmuludq ($D3,$T2); # h3*r0
1087 &paddq ($D2,&QWP(16*2,"esp"));
1088 &movdqa (&QWP(16*2,"esp"),$T0);
1089 &pshufd ($T0,&QWP(16*1,"edx"),0x10); # r1^n
1090 &paddq ($D3,&QWP(16*3,"esp"));
1091 &movdqa (&QWP(16*3,"esp"),$T1);
1092 &movdqa ($T1,$D4);
1093 &pmuludq ($D4,$T2); # h4*r0
1094 &paddq ($D4,&QWP(16*4,"esp"));
1095 &movdqa (&QWP(16*4,"esp"),$T1);
1096
1097 &pmuladd (sub { my ($reg,$i)=@_;
1098 &pshufd ($reg,&QWP(16*$i,"edx"),0x10);
1099 });
1100
1101 &set_label("short_tail");
1102
1103 ################################################################
1104 # horizontal addition
1105
1106 &pshufd ($T1,$D4,0b01001110);
1107 &pshufd ($T0,$D3,0b01001110);
1108 &paddq ($D4,$T1);
1109 &paddq ($D3,$T0);
1110 &pshufd ($T1,$D0,0b01001110);
1111 &pshufd ($T0,$D1,0b01001110);
1112 &paddq ($D0,$T1);
1113 &paddq ($D1,$T0);
1114 &pshufd ($T1,$D2,0b01001110);
1115 #&paddq ($D2,$T1);
1116
1117 &lazy_reduction (sub { &paddq ($D2,$T1) });
1118
1119 &set_label("done");
1120 &movd (&DWP(-16*3+4*0,"edi"),$D0); # store hash value
1121 &movd (&DWP(-16*3+4*1,"edi"),$D1);
1122 &movd (&DWP(-16*3+4*2,"edi"),$D2);
1123 &movd (&DWP(-16*3+4*3,"edi"),$D3);
1124 &movd (&DWP(-16*3+4*4,"edi"),$D4);
1125 &mov ("esp","ebp");
1126 &set_label("nodata");
1127 &function_end("_poly1305_blocks_sse2");
1128
1129 &align (32);
1130 &function_begin("_poly1305_emit_sse2");
1131 &mov ("ebp",&wparam(0)); # context
1132
1133 &cmp (&DWP(4*5,"ebp"),0); # is_base2_26?
1134 &je (&label("enter_emit"));
1135
1136 &mov ("eax",&DWP(4*0,"ebp")); # load hash value
1137 &mov ("edi",&DWP(4*1,"ebp"));
1138 &mov ("ecx",&DWP(4*2,"ebp"));
1139 &mov ("edx",&DWP(4*3,"ebp"));
1140 &mov ("esi",&DWP(4*4,"ebp"));
1141
1142 &mov ("ebx","edi"); # base 2^26 -> base 2^32
1143 &shl ("edi",26);
1144 &shr ("ebx",6);
1145 &add ("eax","edi");
1146 &mov ("edi","ecx");
1147 &adc ("ebx",0);
1148
1149 &shl ("edi",20);
1150 &shr ("ecx",12);
1151 &add ("ebx","edi");
1152 &mov ("edi","edx");
1153 &adc ("ecx",0);
1154
1155 &shl ("edi",14);
1156 &shr ("edx",18);
1157 &add ("ecx","edi");
1158 &mov ("edi","esi");
1159 &adc ("edx",0);
1160
1161 &shl ("edi",8);
1162 &shr ("esi",24);
1163 &add ("edx","edi");
1164 &adc ("esi",0); # can be partially reduced
1165
1166 &mov ("edi","esi"); # final reduction
1167 &and ("esi",3);
1168 &shr ("edi",2);
1169 &lea ("ebp",&DWP(0,"edi","edi",4)); # *5
1170 &mov ("edi",&wparam(1)); # output
1171 &add ("eax","ebp");
1172 &mov ("ebp",&wparam(2)); # key
1173 &adc ("ebx",0);
1174 &adc ("ecx",0);
1175 &adc ("edx",0);
1176 &adc ("esi",0);
1177
1178 &movd ($D0,"eax"); # offload original hash value
1179 &add ("eax",5); # compare to modulus
1180 &movd ($D1,"ebx");
1181 &adc ("ebx",0);
1182 &movd ($D2,"ecx");
1183 &adc ("ecx",0);
1184 &movd ($D3,"edx");
1185 &adc ("edx",0);
1186 &adc ("esi",0);
1187 &shr ("esi",2); # did it carry/borrow?
1188
1189 &neg ("esi"); # do we choose (hash-modulus) ...
1190 &and ("eax","esi");
1191 &and ("ebx","esi");
1192 &and ("ecx","esi");
1193 &and ("edx","esi");
1194 &mov (&DWP(4*0,"edi"),"eax");
1195 &movd ("eax",$D0);
1196 &mov (&DWP(4*1,"edi"),"ebx");
1197 &movd ("ebx",$D1);
1198 &mov (&DWP(4*2,"edi"),"ecx");
1199 &movd ("ecx",$D2);
1200 &mov (&DWP(4*3,"edi"),"edx");
1201 &movd ("edx",$D3);
1202
1203 &not ("esi"); # ... or original hash value?
1204 &and ("eax","esi");
1205 &and ("ebx","esi");
1206 &or ("eax",&DWP(4*0,"edi"));
1207 &and ("ecx","esi");
1208 &or ("ebx",&DWP(4*1,"edi"));
1209 &and ("edx","esi");
1210 &or ("ecx",&DWP(4*2,"edi"));
1211 &or ("edx",&DWP(4*3,"edi"));
1212
1213 &add ("eax",&DWP(4*0,"ebp")); # accumulate key
1214 &adc ("ebx",&DWP(4*1,"ebp"));
1215 &mov (&DWP(4*0,"edi"),"eax");
1216 &adc ("ecx",&DWP(4*2,"ebp"));
1217 &mov (&DWP(4*1,"edi"),"ebx");
1218 &adc ("edx",&DWP(4*3,"ebp"));
1219 &mov (&DWP(4*2,"edi"),"ecx");
1220 &mov (&DWP(4*3,"edi"),"edx");
1221 &function_end("_poly1305_emit_sse2");
1222
1223 if ($avx>1) {
1224 ########################################################################
1225 # Note that poly1305_init_avx2 operates on %xmm, I could have used
1226 # poly1305_init_sse2...
1227
1228 &align (32);
1229 &function_begin_B("_poly1305_init_avx2");
1230 &vmovdqu ($D4,&QWP(4*6,"edi")); # key base 2^32
1231 &lea ("edi",&DWP(16*3,"edi")); # size optimization
1232 &mov ("ebp","esp");
1233 &sub ("esp",16*(9+5));
1234 &and ("esp",-16);
1235
1236 #&vpand ($D4,$D4,&QWP(96,"ebx")); # magic mask
1237 &vmovdqa ($MASK,&QWP(64,"ebx"));
1238
1239 &vpand ($D0,$D4,$MASK); # -> base 2^26
1240 &vpsrlq ($D1,$D4,26);
1241 &vpsrldq ($D3,$D4,6);
1242 &vpand ($D1,$D1,$MASK);
1243 &vpsrlq ($D2,$D3,4)
1244 &vpsrlq ($D3,$D3,30);
1245 &vpand ($D2,$D2,$MASK);
1246 &vpand ($D3,$D3,$MASK);
1247 &vpsrldq ($D4,$D4,13);
1248
1249 &lea ("edx",&DWP(16*9,"esp")); # size optimization
1250 &mov ("ecx",2);
1251 &set_label("square");
1252 &vmovdqa (&QWP(16*0,"esp"),$D0);
1253 &vmovdqa (&QWP(16*1,"esp"),$D1);
1254 &vmovdqa (&QWP(16*2,"esp"),$D2);
1255 &vmovdqa (&QWP(16*3,"esp"),$D3);
1256 &vmovdqa (&QWP(16*4,"esp"),$D4);
1257
1258 &vpslld ($T1,$D1,2);
1259 &vpslld ($T0,$D2,2);
1260 &vpaddd ($T1,$T1,$D1); # *5
1261 &vpaddd ($T0,$T0,$D2); # *5
1262 &vmovdqa (&QWP(16*5,"esp"),$T1);
1263 &vmovdqa (&QWP(16*6,"esp"),$T0);
1264 &vpslld ($T1,$D3,2);
1265 &vpslld ($T0,$D4,2);
1266 &vpaddd ($T1,$T1,$D3); # *5
1267 &vpaddd ($T0,$T0,$D4); # *5
1268 &vmovdqa (&QWP(16*7,"esp"),$T1);
1269 &vmovdqa (&QWP(16*8,"esp"),$T0);
1270
1271 &vpshufd ($T0,$D0,0b01000100);
1272 &vmovdqa ($T1,$D1);
1273 &vpshufd ($D1,$D1,0b01000100);
1274 &vpshufd ($D2,$D2,0b01000100);
1275 &vpshufd ($D3,$D3,0b01000100);
1276 &vpshufd ($D4,$D4,0b01000100);
1277 &vmovdqa (&QWP(16*0,"edx"),$T0);
1278 &vmovdqa (&QWP(16*1,"edx"),$D1);
1279 &vmovdqa (&QWP(16*2,"edx"),$D2);
1280 &vmovdqa (&QWP(16*3,"edx"),$D3);
1281 &vmovdqa (&QWP(16*4,"edx"),$D4);
1282
1283 ################################################################
1284 # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
1285 # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
1286 # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
1287 # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
1288 # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
1289
1290 &vpmuludq ($D4,$D4,$D0); # h4*r0
1291 &vpmuludq ($D3,$D3,$D0); # h3*r0
1292 &vpmuludq ($D2,$D2,$D0); # h2*r0
1293 &vpmuludq ($D1,$D1,$D0); # h1*r0
1294 &vpmuludq ($D0,$T0,$D0); # h0*r0
1295
1296 &vpmuludq ($T0,$T1,&QWP(16*3,"edx")); # r1*h3
1297 &vpaddq ($D4,$D4,$T0);
1298 &vpmuludq ($T2,$T1,&QWP(16*2,"edx")); # r1*h2
1299 &vpaddq ($D3,$D3,$T2);
1300 &vpmuludq ($T0,$T1,&QWP(16*1,"edx")); # r1*h1
1301 &vpaddq ($D2,$D2,$T0);
1302 &vmovdqa ($T2,&QWP(16*5,"esp")); # s1
1303 &vpmuludq ($T1,$T1,&QWP(16*0,"edx")); # r1*h0
1304 &vpaddq ($D1,$D1,$T1);
1305 &vmovdqa ($T0,&QWP(16*2,"esp")); # r2
1306 &vpmuludq ($T2,$T2,&QWP(16*4,"edx")); # s1*h4
1307 &vpaddq ($D0,$D0,$T2);
1308
1309 &vpmuludq ($T1,$T0,&QWP(16*2,"edx")); # r2*h2
1310 &vpaddq ($D4,$D4,$T1);
1311 &vpmuludq ($T2,$T0,&QWP(16*1,"edx")); # r2*h1
1312 &vpaddq ($D3,$D3,$T2);
1313 &vmovdqa ($T1,&QWP(16*6,"esp")); # s2
1314 &vpmuludq ($T0,$T0,&QWP(16*0,"edx")); # r2*h0
1315 &vpaddq ($D2,$D2,$T0);
1316 &vpmuludq ($T2,$T1,&QWP(16*4,"edx")); # s2*h4
1317 &vpaddq ($D1,$D1,$T2);
1318 &vmovdqa ($T0,&QWP(16*3,"esp")); # r3
1319 &vpmuludq ($T1,$T1,&QWP(16*3,"edx")); # s2*h3
1320 &vpaddq ($D0,$D0,$T1);
1321
1322 &vpmuludq ($T2,$T0,&QWP(16*1,"edx")); # r3*h1
1323 &vpaddq ($D4,$D4,$T2);
1324 &vmovdqa ($T1,&QWP(16*7,"esp")); # s3
1325 &vpmuludq ($T0,$T0,&QWP(16*0,"edx")); # r3*h0
1326 &vpaddq ($D3,$D3,$T0);
1327 &vpmuludq ($T2,$T1,&QWP(16*4,"edx")); # s3*h4
1328 &vpaddq ($D2,$D2,$T2);
1329 &vpmuludq ($T0,$T1,&QWP(16*3,"edx")); # s3*h3
1330 &vpaddq ($D1,$D1,$T0);
1331 &vmovdqa ($T2,&QWP(16*4,"esp")); # r4
1332 &vpmuludq ($T1,$T1,&QWP(16*2,"edx")); # s3*h2
1333 &vpaddq ($D0,$D0,$T1);
1334
1335 &vmovdqa ($T0,&QWP(16*8,"esp")); # s4
1336 &vpmuludq ($T2,$T2,&QWP(16*0,"edx")); # r4*h0
1337 &vpaddq ($D4,$D4,$T2);
1338 &vpmuludq ($T1,$T0,&QWP(16*4,"edx")); # s4*h4
1339 &vpaddq ($D3,$D3,$T1);
1340 &vpmuludq ($T2,$T0,&QWP(16*1,"edx")); # s4*h1
1341 &vpaddq ($D0,$D0,$T2);
1342 &vpmuludq ($T1,$T0,&QWP(16*2,"edx")); # s4*h2
1343 &vpaddq ($D1,$D1,$T1);
1344 &vmovdqa ($MASK,&QWP(64,"ebx"));
1345 &vpmuludq ($T0,$T0,&QWP(16*3,"edx")); # s4*h3
1346 &vpaddq ($D2,$D2,$T0);
1347
1348 ################################################################
1349 # lazy reduction
1350 &vpsrlq ($T0,$D3,26);
1351 &vpand ($D3,$D3,$MASK);
1352 &vpsrlq ($T1,$D0,26);
1353 &vpand ($D0,$D0,$MASK);
1354 &vpaddq ($D4,$D4,$T0); # h3 -> h4
1355 &vpaddq ($D1,$D1,$T1); # h0 -> h1
1356 &vpsrlq ($T0,$D4,26);
1357 &vpand ($D4,$D4,$MASK);
1358 &vpsrlq ($T1,$D1,26);
1359 &vpand ($D1,$D1,$MASK);
1360 &vpaddq ($D2,$D2,$T1); # h1 -> h2
1361 &vpaddd ($D0,$D0,$T0);
1362 &vpsllq ($T0,$T0,2);
1363 &vpsrlq ($T1,$D2,26);
1364 &vpand ($D2,$D2,$MASK);
1365 &vpaddd ($D0,$D0,$T0); # h4 -> h0
1366 &vpaddd ($D3,$D3,$T1); # h2 -> h3
1367 &vpsrlq ($T1,$D3,26);
1368 &vpsrlq ($T0,$D0,26);
1369 &vpand ($D0,$D0,$MASK);
1370 &vpand ($D3,$D3,$MASK);
1371 &vpaddd ($D1,$D1,$T0); # h0 -> h1
1372 &vpaddd ($D4,$D4,$T1); # h3 -> h4
1373
1374 &dec ("ecx");
1375 &jz (&label("square_break"));
1376
1377 &vpunpcklqdq ($D0,$D0,&QWP(16*0,"esp")); # 0:r^1:0:r^2
1378 &vpunpcklqdq ($D1,$D1,&QWP(16*1,"esp"));
1379 &vpunpcklqdq ($D2,$D2,&QWP(16*2,"esp"));
1380 &vpunpcklqdq ($D3,$D3,&QWP(16*3,"esp"));
1381 &vpunpcklqdq ($D4,$D4,&QWP(16*4,"esp"));
1382 &jmp (&label("square"));
1383
1384 &set_label("square_break");
1385 &vpsllq ($D0,$D0,32); # -> r^3:0:r^4:0
1386 &vpsllq ($D1,$D1,32);
1387 &vpsllq ($D2,$D2,32);
1388 &vpsllq ($D3,$D3,32);
1389 &vpsllq ($D4,$D4,32);
1390 &vpor ($D0,$D0,&QWP(16*0,"esp")); # r^3:r^1:r^4:r^2
1391 &vpor ($D1,$D1,&QWP(16*1,"esp"));
1392 &vpor ($D2,$D2,&QWP(16*2,"esp"));
1393 &vpor ($D3,$D3,&QWP(16*3,"esp"));
1394 &vpor ($D4,$D4,&QWP(16*4,"esp"));
1395
1396 &vpshufd ($D0,$D0,0b10001101); # -> r^1:r^2:r^3:r^4
1397 &vpshufd ($D1,$D1,0b10001101);
1398 &vpshufd ($D2,$D2,0b10001101);
1399 &vpshufd ($D3,$D3,0b10001101);
1400 &vpshufd ($D4,$D4,0b10001101);
1401
1402 &vmovdqu (&QWP(16*0,"edi"),$D0); # save the table
1403 &vmovdqu (&QWP(16*1,"edi"),$D1);
1404 &vmovdqu (&QWP(16*2,"edi"),$D2);
1405 &vmovdqu (&QWP(16*3,"edi"),$D3);
1406 &vmovdqu (&QWP(16*4,"edi"),$D4);
1407
1408 &vpslld ($T1,$D1,2);
1409 &vpslld ($T0,$D2,2);
1410 &vpaddd ($T1,$T1,$D1); # *5
1411 &vpaddd ($T0,$T0,$D2); # *5
1412 &vmovdqu (&QWP(16*5,"edi"),$T1);
1413 &vmovdqu (&QWP(16*6,"edi"),$T0);
1414 &vpslld ($T1,$D3,2);
1415 &vpslld ($T0,$D4,2);
1416 &vpaddd ($T1,$T1,$D3); # *5
1417 &vpaddd ($T0,$T0,$D4); # *5
1418 &vmovdqu (&QWP(16*7,"edi"),$T1);
1419 &vmovdqu (&QWP(16*8,"edi"),$T0);
1420
1421 &mov ("esp","ebp");
1422 &lea ("edi",&DWP(-16*3,"edi")); # size de-optimization
1423 &ret ();
1424 &function_end_B("_poly1305_init_avx2");
1425
1426 ########################################################################
1427 # now it's time to switch to %ymm
1428
1429 my ($D0,$D1,$D2,$D3,$D4,$T0,$T1,$T2)=map("ymm$_",(0..7));
1430 my $MASK=$T2;
1431
1432 sub X { my $reg=shift; $reg=~s/^ymm/xmm/; $reg; }
1433
1434 &align (32);
1435 &function_begin("_poly1305_blocks_avx2");
1436 &mov ("edi",&wparam(0)); # ctx
1437 &mov ("esi",&wparam(1)); # inp
1438 &mov ("ecx",&wparam(2)); # len
1439
1440 &mov ("eax",&DWP(4*5,"edi")); # is_base2_26
1441 &and ("ecx",-16);
1442 &jz (&label("nodata"));
1443 &cmp ("ecx",64);
1444 &jae (&label("enter_avx2"));
1445 &test ("eax","eax"); # is_base2_26?
1446 &jz (&label("enter_blocks"));
1447
1448 &set_label("enter_avx2");
1449 &vzeroupper ();
1450
1451 &call (&label("pic_point"));
1452 &set_label("pic_point");
1453 &blindpop("ebx");
1454 &lea ("ebx",&DWP(&label("const_sse2")."-".&label("pic_point"),"ebx"));
1455
1456 &test ("eax","eax"); # is_base2_26?
1457 &jnz (&label("base2_26"));
1458
1459 &call ("_poly1305_init_avx2");
1460
1461 ################################################# base 2^32 -> base 2^26
1462 &mov ("eax",&DWP(0,"edi"));
1463 &mov ("ecx",&DWP(3,"edi"));
1464 &mov ("edx",&DWP(6,"edi"));
1465 &mov ("esi",&DWP(9,"edi"));
1466 &mov ("ebp",&DWP(13,"edi"));
1467
1468 &shr ("ecx",2);
1469 &and ("eax",0x3ffffff);
1470 &shr ("edx",4);
1471 &and ("ecx",0x3ffffff);
1472 &shr ("esi",6);
1473 &and ("edx",0x3ffffff);
1474
1475 &mov (&DWP(4*0,"edi"),"eax");
1476 &mov (&DWP(4*1,"edi"),"ecx");
1477 &mov (&DWP(4*2,"edi"),"edx");
1478 &mov (&DWP(4*3,"edi"),"esi");
1479 &mov (&DWP(4*4,"edi"),"ebp");
1480 &mov (&DWP(4*5,"edi"),1); # is_base2_26
1481
1482 &mov ("esi",&wparam(1)); # [reload] inp
1483 &mov ("ecx",&wparam(2)); # [reload] len
1484
1485 &set_label("base2_26");
1486 &mov ("eax",&wparam(3)); # padbit
1487 &mov ("ebp","esp");
1488
1489 &sub ("esp",32*(5+9));
1490 &and ("esp",-512); # ensure that frame
1491 # doesn't cross page
1492 # boundary, which is
1493 # essential for
1494 # misaligned 32-byte
1495 # loads
1496
1497 ################################################################
1498 # expand and copy pre-calculated table to stack
1499
1500 &vmovdqu (&X($D0),&QWP(16*(3+0),"edi"));
1501 &lea ("edx",&DWP(32*5+128,"esp")); # +128 size optimization
1502 &vmovdqu (&X($D1),&QWP(16*(3+1),"edi"));
1503 &vmovdqu (&X($D2),&QWP(16*(3+2),"edi"));
1504 &vmovdqu (&X($D3),&QWP(16*(3+3),"edi"));
1505 &vmovdqu (&X($D4),&QWP(16*(3+4),"edi"));
1506 &lea ("edi",&DWP(16*3,"edi")); # size optimization
1507 &vpermq ($D0,$D0,0b01000000); # 00001234 -> 12343434
1508 &vpermq ($D1,$D1,0b01000000);
1509 &vpermq ($D2,$D2,0b01000000);
1510 &vpermq ($D3,$D3,0b01000000);
1511 &vpermq ($D4,$D4,0b01000000);
1512 &vpshufd ($D0,$D0,0b11001000); # 12343434 -> 14243444
1513 &vpshufd ($D1,$D1,0b11001000);
1514 &vpshufd ($D2,$D2,0b11001000);
1515 &vpshufd ($D3,$D3,0b11001000);
1516 &vpshufd ($D4,$D4,0b11001000);
1517 &vmovdqa (&QWP(32*0-128,"edx"),$D0);
1518 &vmovdqu (&X($D0),&QWP(16*5,"edi"));
1519 &vmovdqa (&QWP(32*1-128,"edx"),$D1);
1520 &vmovdqu (&X($D1),&QWP(16*6,"edi"));
1521 &vmovdqa (&QWP(32*2-128,"edx"),$D2);
1522 &vmovdqu (&X($D2),&QWP(16*7,"edi"));
1523 &vmovdqa (&QWP(32*3-128,"edx"),$D3);
1524 &vmovdqu (&X($D3),&QWP(16*8,"edi"));
1525 &vmovdqa (&QWP(32*4-128,"edx"),$D4);
1526 &vpermq ($D0,$D0,0b01000000);
1527 &vpermq ($D1,$D1,0b01000000);
1528 &vpermq ($D2,$D2,0b01000000);
1529 &vpermq ($D3,$D3,0b01000000);
1530 &vpshufd ($D0,$D0,0b11001000);
1531 &vpshufd ($D1,$D1,0b11001000);
1532 &vpshufd ($D2,$D2,0b11001000);
1533 &vpshufd ($D3,$D3,0b11001000);
1534 &vmovdqa (&QWP(32*5-128,"edx"),$D0);
1535 &vmovd (&X($D0),&DWP(-16*3+4*0,"edi"));# load hash value
1536 &vmovdqa (&QWP(32*6-128,"edx"),$D1);
1537 &vmovd (&X($D1),&DWP(-16*3+4*1,"edi"));
1538 &vmovdqa (&QWP(32*7-128,"edx"),$D2);
1539 &vmovd (&X($D2),&DWP(-16*3+4*2,"edi"));
1540 &vmovdqa (&QWP(32*8-128,"edx"),$D3);
1541 &vmovd (&X($D3),&DWP(-16*3+4*3,"edi"));
1542 &vmovd (&X($D4),&DWP(-16*3+4*4,"edi"));
1543 &vmovdqa ($MASK,&QWP(64,"ebx"));
1544 &neg ("eax"); # padbit
1545
1546 &test ("ecx",63);
1547 &jz (&label("even"));
1548
1549 &mov ("edx","ecx");
1550 &and ("ecx",-64);
1551 &and ("edx",63);
1552
1553 &vmovdqu (&X($T0),&QWP(16*0,"esi"));
1554 &cmp ("edx",32);
1555 &jb (&label("one"));
1556
1557 &vmovdqu (&X($T1),&QWP(16*1,"esi"));
1558 &je (&label("two"));
1559
1560 &vinserti128 ($T0,$T0,&QWP(16*2,"esi"),1);
1561 &lea ("esi",&DWP(16*3,"esi"));
1562 &lea ("ebx",&DWP(8,"ebx")); # three padbits
1563 &lea ("edx",&DWP(32*5+128+8,"esp")); # --:r^1:r^2:r^3 (*)
1564 &jmp (&label("tail"));
1565
1566 &set_label("two");
1567 &lea ("esi",&DWP(16*2,"esi"));
1568 &lea ("ebx",&DWP(16,"ebx")); # two padbits
1569 &lea ("edx",&DWP(32*5+128+16,"esp"));# --:--:r^1:r^2 (*)
1570 &jmp (&label("tail"));
1571
1572 &set_label("one");
1573 &lea ("esi",&DWP(16*1,"esi"));
1574 &vpxor ($T1,$T1,$T1);
1575 &lea ("ebx",&DWP(32,"ebx","eax",8)); # one or no padbits
1576 &lea ("edx",&DWP(32*5+128+24,"esp"));# --:--:--:r^1 (*)
1577 &jmp (&label("tail"));
1578
1579 # (*) spots marked with '--' are data from next table entry, but they
1580 # are multiplied by 0 and therefore rendered insignificant
1581
1582 &set_label("even",32);
1583 &vmovdqu (&X($T0),&QWP(16*0,"esi")); # load input
1584 &vmovdqu (&X($T1),&QWP(16*1,"esi"));
1585 &vinserti128 ($T0,$T0,&QWP(16*2,"esi"),1);
1586 &vinserti128 ($T1,$T1,&QWP(16*3,"esi"),1);
1587 &lea ("esi",&DWP(16*4,"esi"));
1588 &sub ("ecx",64);
1589 &jz (&label("tail"));
1590
1591 &set_label("loop");
1592 ################################################################
1593 # ((inp[0]*r^4+r[4])*r^4+r[8])*r^4
1594 # ((inp[1]*r^4+r[5])*r^4+r[9])*r^3
1595 # ((inp[2]*r^4+r[6])*r^4+r[10])*r^2
1596 # ((inp[3]*r^4+r[7])*r^4+r[11])*r^1
1597 # \________/ \_______/
1598 ################################################################
1599
1600 sub vsplat_input {
1601 &vmovdqa (&QWP(32*2,"esp"),$D2);
1602 &vpsrldq ($D2,$T0,6); # splat input
1603 &vmovdqa (&QWP(32*0,"esp"),$D0);
1604 &vpsrldq ($D0,$T1,6);
1605 &vmovdqa (&QWP(32*1,"esp"),$D1);
1606 &vpunpckhqdq ($D1,$T0,$T1); # 4
1607 &vpunpcklqdq ($T0,$T0,$T1); # 0:1
1608 &vpunpcklqdq ($D2,$D2,$D0); # 2:3
1609
1610 &vpsrlq ($D0,$D2,30);
1611 &vpsrlq ($D2,$D2,4);
1612 &vpsrlq ($T1,$T0,26);
1613 &vpsrlq ($D1,$D1,40); # 4
1614 &vpand ($D2,$D2,$MASK); # 2
1615 &vpand ($T0,$T0,$MASK); # 0
1616 &vpand ($T1,$T1,$MASK); # 1
1617 &vpand ($D0,$D0,$MASK); # 3 (*)
1618 &vpor ($D1,$D1,&QWP(0,"ebx")); # padbit, yes, always
1619
1620 # (*) note that output is counterintuitive, inp[3:4] is
1621 # returned in $D1-2, while $D3-4 are preserved;
1622 }
1623 &vsplat_input ();
1624
1625 sub vpmuladd {
1626 my $addr = shift;
1627
1628 &vpaddq ($D2,$D2,&QWP(32*2,"esp")); # add hash value
1629 &vpaddq ($T0,$T0,&QWP(32*0,"esp"));
1630 &vpaddq ($T1,$T1,&QWP(32*1,"esp"));
1631 &vpaddq ($D0,$D0,$D3);
1632 &vpaddq ($D1,$D1,$D4);
1633
1634 ################################################################
1635 # d3 = h2*r1 + h0*r3 + h1*r2 + h3*r0 + h4*5*r4
1636 # d4 = h2*r2 + h0*r4 + h1*r3 + h3*r1 + h4*r0
1637 # d0 = h2*5*r3 + h0*r0 + h1*5*r4 + h3*5*r2 + h4*5*r1
1638 # d1 = h2*5*r4 + h0*r1 + h1*r0 + h3*5*r3 + h4*5*r2
1639 # d2 = h2*r0 + h0*r2 + h1*r1 + h3*5*r4 + h4*5*r3
1640
1641 &vpmuludq ($D3,$D2,&$addr(1)); # d3 = h2*r1
1642 &vmovdqa (QWP(32*1,"esp"),$T1);
1643 &vpmuludq ($D4,$D2,&$addr(2)); # d4 = h2*r2
1644 &vmovdqa (QWP(32*3,"esp"),$D0);
1645 &vpmuludq ($D0,$D2,&$addr(7)); # d0 = h2*s3
1646 &vmovdqa (QWP(32*4,"esp"),$D1);
1647 &vpmuludq ($D1,$D2,&$addr(8)); # d1 = h2*s4
1648 &vpmuludq ($D2,$D2,&$addr(0)); # d2 = h2*r0
1649
1650 &vpmuludq ($T2,$T0,&$addr(3)); # h0*r3
1651 &vpaddq ($D3,$D3,$T2); # d3 += h0*r3
1652 &vpmuludq ($T1,$T0,&$addr(4)); # h0*r4
1653 &vpaddq ($D4,$D4,$T1); # d4 + h0*r4
1654 &vpmuludq ($T2,$T0,&$addr(0)); # h0*r0
1655 &vpaddq ($D0,$D0,$T2); # d0 + h0*r0
1656 &vmovdqa ($T2,&QWP(32*1,"esp")); # h1
1657 &vpmuludq ($T1,$T0,&$addr(1)); # h0*r1
1658 &vpaddq ($D1,$D1,$T1); # d1 += h0*r1
1659 &vpmuludq ($T0,$T0,&$addr(2)); # h0*r2
1660 &vpaddq ($D2,$D2,$T0); # d2 += h0*r2
1661
1662 &vpmuludq ($T1,$T2,&$addr(2)); # h1*r2
1663 &vpaddq ($D3,$D3,$T1); # d3 += h1*r2
1664 &vpmuludq ($T0,$T2,&$addr(3)); # h1*r3
1665 &vpaddq ($D4,$D4,$T0); # d4 += h1*r3
1666 &vpmuludq ($T1,$T2,&$addr(8)); # h1*s4
1667 &vpaddq ($D0,$D0,$T1); # d0 += h1*s4
1668 &vmovdqa ($T1,&QWP(32*3,"esp")); # h3
1669 &vpmuludq ($T0,$T2,&$addr(0)); # h1*r0
1670 &vpaddq ($D1,$D1,$T0); # d1 += h1*r0
1671 &vpmuludq ($T2,$T2,&$addr(1)); # h1*r1
1672 &vpaddq ($D2,$D2,$T2); # d2 += h1*r1
1673
1674 &vpmuludq ($T0,$T1,&$addr(0)); # h3*r0
1675 &vpaddq ($D3,$D3,$T0); # d3 += h3*r0
1676 &vpmuludq ($T2,$T1,&$addr(1)); # h3*r1
1677 &vpaddq ($D4,$D4,$T2); # d4 += h3*r1
1678 &vpmuludq ($T0,$T1,&$addr(6)); # h3*s2
1679 &vpaddq ($D0,$D0,$T0); # d0 += h3*s2
1680 &vmovdqa ($T0,&QWP(32*4,"esp")); # h4
1681 &vpmuludq ($T2,$T1,&$addr(7)); # h3*s3
1682 &vpaddq ($D1,$D1,$T2); # d1+= h3*s3
1683 &vpmuludq ($T1,$T1,&$addr(8)); # h3*s4
1684 &vpaddq ($D2,$D2,$T1); # d2 += h3*s4
1685
1686 &vpmuludq ($T2,$T0,&$addr(8)); # h4*s4
1687 &vpaddq ($D3,$D3,$T2); # d3 += h4*s4
1688 &vpmuludq ($T1,$T0,&$addr(5)); # h4*s1
1689 &vpaddq ($D0,$D0,$T1); # d0 += h4*s1
1690 &vpmuludq ($T2,$T0,&$addr(0)); # h4*r0
1691 &vpaddq ($D4,$D4,$T2); # d4 += h4*r0
1692 &vmovdqa ($MASK,&QWP(64,"ebx"));
1693 &vpmuludq ($T1,$T0,&$addr(6)); # h4*s2
1694 &vpaddq ($D1,$D1,$T1); # d1 += h4*s2
1695 &vpmuludq ($T0,$T0,&$addr(7)); # h4*s3
1696 &vpaddq ($D2,$D2,$T0); # d2 += h4*s3
1697 }
1698 &vpmuladd (sub { my $i=shift; &QWP(32*$i-128,"edx"); });
1699
1700 sub vlazy_reduction {
1701 ################################################################
1702 # lazy reduction
1703
1704 &vpsrlq ($T0,$D3,26);
1705 &vpand ($D3,$D3,$MASK);
1706 &vpsrlq ($T1,$D0,26);
1707 &vpand ($D0,$D0,$MASK);
1708 &vpaddq ($D4,$D4,$T0); # h3 -> h4
1709 &vpaddq ($D1,$D1,$T1); # h0 -> h1
1710 &vpsrlq ($T0,$D4,26);
1711 &vpand ($D4,$D4,$MASK);
1712 &vpsrlq ($T1,$D1,26);
1713 &vpand ($D1,$D1,$MASK);
1714 &vpaddq ($D2,$D2,$T1); # h1 -> h2
1715 &vpaddq ($D0,$D0,$T0);
1716 &vpsllq ($T0,$T0,2);
1717 &vpsrlq ($T1,$D2,26);
1718 &vpand ($D2,$D2,$MASK);
1719 &vpaddq ($D0,$D0,$T0); # h4 -> h0
1720 &vpaddq ($D3,$D3,$T1); # h2 -> h3
1721 &vpsrlq ($T1,$D3,26);
1722 &vpsrlq ($T0,$D0,26);
1723 &vpand ($D0,$D0,$MASK);
1724 &vpand ($D3,$D3,$MASK);
1725 &vpaddq ($D1,$D1,$T0); # h0 -> h1
1726 &vpaddq ($D4,$D4,$T1); # h3 -> h4
1727 }
1728 &vlazy_reduction();
1729
1730 &vmovdqu (&X($T0),&QWP(16*0,"esi")); # load input
1731 &vmovdqu (&X($T1),&QWP(16*1,"esi"));
1732 &vinserti128 ($T0,$T0,&QWP(16*2,"esi"),1);
1733 &vinserti128 ($T1,$T1,&QWP(16*3,"esi"),1);
1734 &lea ("esi",&DWP(16*4,"esi"));
1735 &sub ("ecx",64);
1736 &jnz (&label("loop"));
1737
1738 &set_label("tail");
1739 &vsplat_input ();
1740 &and ("ebx",-64); # restore pointer
1741
1742 &vpmuladd (sub { my $i=shift; &QWP(4+32*$i-128,"edx"); });
1743
1744 ################################################################
1745 # horizontal addition
1746
1747 &vpsrldq ($T0,$D4,8);
1748 &vpsrldq ($T1,$D3,8);
1749 &vpaddq ($D4,$D4,$T0);
1750 &vpsrldq ($T0,$D0,8);
1751 &vpaddq ($D3,$D3,$T1);
1752 &vpsrldq ($T1,$D1,8);
1753 &vpaddq ($D0,$D0,$T0);
1754 &vpsrldq ($T0,$D2,8);
1755 &vpaddq ($D1,$D1,$T1);
1756 &vpermq ($T1,$D4,2); # keep folding
1757 &vpaddq ($D2,$D2,$T0);
1758 &vpermq ($T0,$D3,2);
1759 &vpaddq ($D4,$D4,$T1);
1760 &vpermq ($T1,$D0,2);
1761 &vpaddq ($D3,$D3,$T0);
1762 &vpermq ($T0,$D1,2);
1763 &vpaddq ($D0,$D0,$T1);
1764 &vpermq ($T1,$D2,2);
1765 &vpaddq ($D1,$D1,$T0);
1766 &vpaddq ($D2,$D2,$T1);
1767
1768 &vlazy_reduction();
1769
1770 &cmp ("ecx",0);
1771 &je (&label("done"));
1772
1773 ################################################################
1774 # clear all but single word
1775
1776 &vpshufd (&X($D0),&X($D0),0b11111100);
1777 &lea ("edx",&DWP(32*5+128,"esp")); # restore pointer
1778 &vpshufd (&X($D1),&X($D1),0b11111100);
1779 &vpshufd (&X($D2),&X($D2),0b11111100);
1780 &vpshufd (&X($D3),&X($D3),0b11111100);
1781 &vpshufd (&X($D4),&X($D4),0b11111100);
1782 &jmp (&label("even"));
1783
1784 &set_label("done",16);
1785 &vmovd (&DWP(-16*3+4*0,"edi"),&X($D0));# store hash value
1786 &vmovd (&DWP(-16*3+4*1,"edi"),&X($D1));
1787 &vmovd (&DWP(-16*3+4*2,"edi"),&X($D2));
1788 &vmovd (&DWP(-16*3+4*3,"edi"),&X($D3));
1789 &vmovd (&DWP(-16*3+4*4,"edi"),&X($D4));
1790 &vzeroupper ();
1791 &mov ("esp","ebp");
1792 &set_label("nodata");
1793 &function_end("_poly1305_blocks_avx2");
1794 }
1795 &set_label("const_sse2",64);
1796 &data_word(1<<24,0, 1<<24,0, 1<<24,0, 1<<24,0);
1797 &data_word(0,0, 0,0, 0,0, 0,0);
1798 &data_word(0x03ffffff,0,0x03ffffff,0, 0x03ffffff,0, 0x03ffffff,0);
1799 &data_word(0x0fffffff,0x0ffffffc,0x0ffffffc,0x0ffffffc);
1800 }
1801 &asciz ("Poly1305 for x86, CRYPTOGAMS by <appro\@openssl.org>");
1802 &align (4);
1803
1804 &asm_finish();
1805
1806 close STDOUT;
A mirror of the official openssl repository