]> git.ipfire.org Git - thirdparty/openssl.git/blob - crypto/sha/asm/sha1-mb-x86_64.pl
projects / thirdparty / openssl.git / blob
? search:


ef1228786f73577d3696775cdc391f6516509b8e
[thirdparty/openssl.git] / crypto / sha / asm / sha1-mb-x86_64.pl
1 #! /usr/bin/env perl
2 # Copyright 2013-2016 The OpenSSL Project Authors. All Rights Reserved.
3 #
4 # Licensed under the Apache License 2.0 (the "License"). You may not use
5 # this file except in compliance with the License. You can obtain a copy
6 # in the file LICENSE in the source distribution or at
7 # https://www.openssl.org/source/license.html
8
9
10 # ====================================================================
11 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
12 # project. The module is, however, dual licensed under OpenSSL and
13 # CRYPTOGAMS licenses depending on where you obtain it. For further
14 # details see http://www.openssl.org/~appro/cryptogams/.
15 # ====================================================================
16
17 # Multi-buffer SHA1 procedure processes n buffers in parallel by
18 # placing buffer data to designated lane of SIMD register. n is
19 # naturally limited to 4 on pre-AVX2 processors and to 8 on
20 # AVX2-capable processors such as Haswell.
21 #
22 # this +aesni(i) sha1 aesni-sha1 gain(iv)
23 # -------------------------------------------------------------------
24 # Westmere(ii) 10.7/n +1.28=3.96(n=4) 5.30 6.66 +68%
25 # Atom(ii) 18.1/n +3.93=8.46(n=4) 9.37 12.8 +51%
26 # Sandy Bridge (8.16 +5.15=13.3)/n 4.99 5.98 +80%
27 # Ivy Bridge (8.08 +5.14=13.2)/n 4.60 5.54 +68%
28 # Haswell(iii) (8.96 +5.00=14.0)/n 3.57 4.55 +160%
29 # Skylake (8.70 +5.00=13.7)/n 3.64 4.20 +145%
30 # Bulldozer (9.76 +5.76=15.5)/n 5.95 6.37 +64%
31 #
32 # (i) multi-block CBC encrypt with 128-bit key;
33 # (ii) (HASH+AES)/n does not apply to Westmere for n>3 and Atom,
34 # because of lower AES-NI instruction throughput;
35 # (iii) "this" is for n=8, when we gather twice as much data, result
36 # for n=4 is 8.00+4.44=12.4;
37 # (iv) presented improvement coefficients are asymptotic limits and
38 # in real-life application are somewhat lower, e.g. for 2KB
39 # fragments they range from 30% to 100% (on Haswell);
40
41 # $output is the last argument if it looks like a file (it has an extension)
42 # $flavour is the first argument if it doesn't look like a file
43 $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
44 $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
45
46 $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
47
48 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
49 ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
50 ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
51 die "can't locate x86_64-xlate.pl";
52
53 push(@INC,"${dir}","${dir}../../perlasm");
54 require "x86_64-support.pl";
55
56 $ptr_size=&pointer_size($flavour);
57
58 $avx=0;
59
60 if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
61 =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
62 $avx = ($1>=2.19) + ($1>=2.22);
63 }
64
65 if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
66 `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
67 $avx = ($1>=2.09) + ($1>=2.10);
68 }
69
70 if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&
71 `ml64 2>&1` =~ /Version ([0-9]+)\./) {
72 $avx = ($1>=10) + ($1>=11);
73 }
74
75 if (!$avx && `$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|.*based on LLVM) ([0-9]+\.[0-9]+)/) {
76 $avx = ($2>=3.0) + ($2>3.0);
77 }
78
79 open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\""
80 or die "can't call $xlate: $!";
81 *STDOUT=*OUT;
82
83 # void sha1_multi_block (
84 # struct { unsigned int A[8];
85 # unsigned int B[8];
86 # unsigned int C[8];
87 # unsigned int D[8];
88 # unsigned int E[8]; } *ctx,
89 # struct { void *ptr; int blocks; } inp[8],
90 # int num); /* 1 or 2 */
91 #
92 $ctx="%rdi"; # 1st arg
93 $inp="%rsi"; # 2nd arg
94 $num="%edx";
95 @ptr=map("%r$_",(8..11));
96 $Tbl="%rbp";
97 $inp_elm_size=2*$ptr_size;
98
99 @V=($A,$B,$C,$D,$E)=map("%xmm$_",(0..4));
100 ($t0,$t1,$t2,$t3,$tx)=map("%xmm$_",(5..9));
101 @Xi=map("%xmm$_",(10..14));
102 $K="%xmm15";
103
104 if (1) {
105 # Atom-specific optimization aiming to eliminate pshufb with high
106 # registers [and thus get rid of 48 cycles accumulated penalty]
107 @Xi=map("%xmm$_",(0..4));
108 ($tx,$t0,$t1,$t2,$t3)=map("%xmm$_",(5..9));
109 @V=($A,$B,$C,$D,$E)=map("%xmm$_",(10..14));
110 }
111
112 $REG_SZ=16;
113
114 sub Xi_off {
115 my $off = shift;
116
117 $off %= 16; $off *= $REG_SZ;
118 $off<256 ? "$off-128(%rax)" : "$off-256-128(%rbx)";
119 }
120
121 sub BODY_00_19 {
122 my ($i,$a,$b,$c,$d,$e)=@_;
123 my $j=$i+1;
124 my $k=$i+2;
125
126 # Loads are performed 2+3/4 iterations in advance. 3/4 means that out
127 # of 4 words you would expect to be loaded per given iteration one is
128 # spilled to next iteration. In other words indices in four input
129 # streams are distributed as following:
130 #
131 # $i==0: 0,0,0,0,1,1,1,1,2,2,2,
132 # $i==1: 2,3,3,3,
133 # $i==2: 3,4,4,4,
134 # ...
135 # $i==13: 14,15,15,15,
136 # $i==14: 15
137 #
138 # Then at $i==15 Xupdate is applied one iteration in advance...
139 $code.=<<___ if ($i==0);
140 movd (@ptr[0]),@Xi[0]
141 lea `16*4`(@ptr[0]),@ptr[0]
142 movd (@ptr[1]),@Xi[2] # borrow @Xi[2]
143 lea `16*4`(@ptr[1]),@ptr[1]
144 movd (@ptr[2]),@Xi[3] # borrow @Xi[3]
145 lea `16*4`(@ptr[2]),@ptr[2]
146 movd (@ptr[3]),@Xi[4] # borrow @Xi[4]
147 lea `16*4`(@ptr[3]),@ptr[3]
148 punpckldq @Xi[3],@Xi[0]
149 movd `4*$j-16*4`(@ptr[0]),@Xi[1]
150 punpckldq @Xi[4],@Xi[2]
151 movd `4*$j-16*4`(@ptr[1]),$t3
152 punpckldq @Xi[2],@Xi[0]
153 movd `4*$j-16*4`(@ptr[2]),$t2
154 pshufb $tx,@Xi[0]
155 ___
156 $code.=<<___ if ($i<14); # just load input
157 movd `4*$j-16*4`(@ptr[3]),$t1
158 punpckldq $t2,@Xi[1]
159 movdqa $a,$t2
160 paddd $K,$e # e+=K_00_19
161 punpckldq $t1,$t3
162 movdqa $b,$t1
163 movdqa $b,$t0
164 pslld \$5,$t2
165 pandn $d,$t1
166 pand $c,$t0
167 punpckldq $t3,@Xi[1]
168 movdqa $a,$t3
169
170 movdqa @Xi[0],`&Xi_off($i)`
171 paddd @Xi[0],$e # e+=X[i]
172 movd `4*$k-16*4`(@ptr[0]),@Xi[2]
173 psrld \$27,$t3
174 pxor $t1,$t0 # Ch(b,c,d)
175 movdqa $b,$t1
176
177 por $t3,$t2 # rol(a,5)
178 movd `4*$k-16*4`(@ptr[1]),$t3
179 pslld \$30,$t1
180 paddd $t0,$e # e+=Ch(b,c,d)
181
182 psrld \$2,$b
183 paddd $t2,$e # e+=rol(a,5)
184 pshufb $tx,@Xi[1]
185 movd `4*$k-16*4`(@ptr[2]),$t2
186 por $t1,$b # b=rol(b,30)
187 ___
188 $code.=<<___ if ($i==14); # just load input
189 movd `4*$j-16*4`(@ptr[3]),$t1
190 punpckldq $t2,@Xi[1]
191 movdqa $a,$t2
192 paddd $K,$e # e+=K_00_19
193 punpckldq $t1,$t3
194 movdqa $b,$t1
195 movdqa $b,$t0
196 pslld \$5,$t2
197 prefetcht0 63(@ptr[0])
198 pandn $d,$t1
199 pand $c,$t0
200 punpckldq $t3,@Xi[1]
201 movdqa $a,$t3
202
203 movdqa @Xi[0],`&Xi_off($i)`
204 paddd @Xi[0],$e # e+=X[i]
205 psrld \$27,$t3
206 pxor $t1,$t0 # Ch(b,c,d)
207 movdqa $b,$t1
208 prefetcht0 63(@ptr[1])
209
210 por $t3,$t2 # rol(a,5)
211 pslld \$30,$t1
212 paddd $t0,$e # e+=Ch(b,c,d)
213 prefetcht0 63(@ptr[2])
214
215 psrld \$2,$b
216 paddd $t2,$e # e+=rol(a,5)
217 pshufb $tx,@Xi[1]
218 prefetcht0 63(@ptr[3])
219 por $t1,$b # b=rol(b,30)
220 ___
221 $code.=<<___ if ($i>=13 && $i<15);
222 movdqa `&Xi_off($j+2)`,@Xi[3] # preload "X[2]"
223 ___
224 $code.=<<___ if ($i>=15); # apply Xupdate
225 pxor @Xi[-2],@Xi[1] # "X[13]"
226 movdqa `&Xi_off($j+2)`,@Xi[3] # "X[2]"
227
228 movdqa $a,$t2
229 pxor `&Xi_off($j+8)`,@Xi[1]
230 paddd $K,$e # e+=K_00_19
231 movdqa $b,$t1
232 pslld \$5,$t2
233 pxor @Xi[3],@Xi[1]
234 movdqa $b,$t0
235 pandn $d,$t1
236 movdqa @Xi[1],$tx
237 pand $c,$t0
238 movdqa $a,$t3
239 psrld \$31,$tx
240 paddd @Xi[1],@Xi[1]
241
242 movdqa @Xi[0],`&Xi_off($i)`
243 paddd @Xi[0],$e # e+=X[i]
244 psrld \$27,$t3
245 pxor $t1,$t0 # Ch(b,c,d)
246
247 movdqa $b,$t1
248 por $t3,$t2 # rol(a,5)
249 pslld \$30,$t1
250 paddd $t0,$e # e+=Ch(b,c,d)
251
252 psrld \$2,$b
253 paddd $t2,$e # e+=rol(a,5)
254 por $tx,@Xi[1] # rol \$1,@Xi[1]
255 por $t1,$b # b=rol(b,30)
256 ___
257 push(@Xi,shift(@Xi));
258 }
259
260 sub BODY_20_39 {
261 my ($i,$a,$b,$c,$d,$e)=@_;
262 my $j=$i+1;
263
264 $code.=<<___ if ($i<79);
265 pxor @Xi[-2],@Xi[1] # "X[13]"
266 movdqa `&Xi_off($j+2)`,@Xi[3] # "X[2]"
267
268 movdqa $a,$t2
269 movdqa $d,$t0
270 pxor `&Xi_off($j+8)`,@Xi[1]
271 paddd $K,$e # e+=K_20_39
272 pslld \$5,$t2
273 pxor $b,$t0
274
275 movdqa $a,$t3
276 ___
277 $code.=<<___ if ($i<72);
278 movdqa @Xi[0],`&Xi_off($i)`
279 ___
280 $code.=<<___ if ($i<79);
281 paddd @Xi[0],$e # e+=X[i]
282 pxor @Xi[3],@Xi[1]
283 psrld \$27,$t3
284 pxor $c,$t0 # Parity(b,c,d)
285 movdqa $b,$t1
286
287 pslld \$30,$t1
288 movdqa @Xi[1],$tx
289 por $t3,$t2 # rol(a,5)
290 psrld \$31,$tx
291 paddd $t0,$e # e+=Parity(b,c,d)
292 paddd @Xi[1],@Xi[1]
293
294 psrld \$2,$b
295 paddd $t2,$e # e+=rol(a,5)
296 por $tx,@Xi[1] # rol(@Xi[1],1)
297 por $t1,$b # b=rol(b,30)
298 ___
299 $code.=<<___ if ($i==79);
300 movdqa $a,$t2
301 paddd $K,$e # e+=K_20_39
302 movdqa $d,$t0
303 pslld \$5,$t2
304 pxor $b,$t0
305
306 movdqa $a,$t3
307 paddd @Xi[0],$e # e+=X[i]
308 psrld \$27,$t3
309 movdqa $b,$t1
310 pxor $c,$t0 # Parity(b,c,d)
311
312 pslld \$30,$t1
313 por $t3,$t2 # rol(a,5)
314 paddd $t0,$e # e+=Parity(b,c,d)
315
316 psrld \$2,$b
317 paddd $t2,$e # e+=rol(a,5)
318 por $t1,$b # b=rol(b,30)
319 ___
320 push(@Xi,shift(@Xi));
321 }
322
323 sub BODY_40_59 {
324 my ($i,$a,$b,$c,$d,$e)=@_;
325 my $j=$i+1;
326
327 $code.=<<___;
328 pxor @Xi[-2],@Xi[1] # "X[13]"
329 movdqa `&Xi_off($j+2)`,@Xi[3] # "X[2]"
330
331 movdqa $a,$t2
332 movdqa $d,$t1
333 pxor `&Xi_off($j+8)`,@Xi[1]
334 pxor @Xi[3],@Xi[1]
335 paddd $K,$e # e+=K_40_59
336 pslld \$5,$t2
337 movdqa $a,$t3
338 pand $c,$t1
339
340 movdqa $d,$t0
341 movdqa @Xi[1],$tx
342 psrld \$27,$t3
343 paddd $t1,$e
344 pxor $c,$t0
345
346 movdqa @Xi[0],`&Xi_off($i)`
347 paddd @Xi[0],$e # e+=X[i]
348 por $t3,$t2 # rol(a,5)
349 psrld \$31,$tx
350 pand $b,$t0
351 movdqa $b,$t1
352
353 pslld \$30,$t1
354 paddd @Xi[1],@Xi[1]
355 paddd $t0,$e # e+=Maj(b,d,c)
356
357 psrld \$2,$b
358 paddd $t2,$e # e+=rol(a,5)
359 por $tx,@Xi[1] # rol(@X[1],1)
360 por $t1,$b # b=rol(b,30)
361 ___
362 push(@Xi,shift(@Xi));
363 }
364
365 $code.=<<___;
366 .text
367
368 .extern OPENSSL_ia32cap_P
369
370 .globl sha1_multi_block
371 .type sha1_multi_block,\@function,3
372 .align 32
373 sha1_multi_block:
374 .cfi_startproc
375 mov OPENSSL_ia32cap_P+4(%rip),%rcx
376 bt \$61,%rcx # check SHA bit
377 jc _shaext_shortcut
378 ___
379 $code.=<<___ if ($avx);
380 test \$`1<<28`,%ecx
381 jnz _avx_shortcut
382 ___
383 $code.=<<___;
384 mov %rsp,%rax
385 .cfi_def_cfa_register %rax
386 push %rbx
387 .cfi_push %rbx
388 push %rbp
389 .cfi_push %rbx
390 ___
391 $code.=<<___ if ($win64);
392 lea -0xa8(%rsp),%rsp
393 movaps %xmm6,(%rsp)
394 movaps %xmm7,0x10(%rsp)
395 movaps %xmm8,0x20(%rsp)
396 movaps %xmm9,0x30(%rsp)
397 movaps %xmm10,-0x78(%rax)
398 movaps %xmm11,-0x68(%rax)
399 movaps %xmm12,-0x58(%rax)
400 movaps %xmm13,-0x48(%rax)
401 movaps %xmm14,-0x38(%rax)
402 movaps %xmm15,-0x28(%rax)
403 ___
404 $code.=<<___;
405 sub \$`$REG_SZ*18`,%rsp
406 and \$-256,%rsp
407 mov %rax,`$REG_SZ*17`(%rsp) # original %rsp
408 .cfi_cfa_expression %rsp+`$REG_SZ*17`,deref,+8
409 .Lbody:
410 lea K_XX_XX(%rip),$Tbl
411 lea `$REG_SZ*16`(%rsp),%rbx
412
413 .Loop_grande:
414 mov $num,`$REG_SZ*17+8`(%rsp) # original $num
415 xor $num,$num
416 ___
417 for($i=0;$i<4;$i++) {
418 $ptr_reg=&pointer_register($flavour,@ptr[$i]);
419 $code.=<<___;
420 # input pointer
421 mov `$inp_elm_size*$i+0`($inp),$ptr_reg
422 # number of blocks
423 mov `$inp_elm_size*$i+$ptr_size`($inp),%ecx
424 cmp $num,%ecx
425 cmovg %ecx,$num # find maximum
426 test %ecx,%ecx
427 mov %ecx,`4*$i`(%rbx) # initialize counters
428 cmovle $Tbl,@ptr[$i] # cancel input
429 ___
430 }
431 $code.=<<___;
432 test $num,$num
433 jz .Ldone
434
435 movdqu 0x00($ctx),$A # load context
436 lea 128(%rsp),%rax
437 movdqu 0x20($ctx),$B
438 movdqu 0x40($ctx),$C
439 movdqu 0x60($ctx),$D
440 movdqu 0x80($ctx),$E
441 movdqa 0x60($Tbl),$tx # pbswap_mask
442 movdqa -0x20($Tbl),$K # K_00_19
443 jmp .Loop
444
445 .align 32
446 .Loop:
447 ___
448 for($i=0;$i<20;$i++) { &BODY_00_19($i,@V); unshift(@V,pop(@V)); }
449 $code.=" movdqa 0x00($Tbl),$K\n"; # K_20_39
450 for(;$i<40;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); }
451 $code.=" movdqa 0x20($Tbl),$K\n"; # K_40_59
452 for(;$i<60;$i++) { &BODY_40_59($i,@V); unshift(@V,pop(@V)); }
453 $code.=" movdqa 0x40($Tbl),$K\n"; # K_60_79
454 for(;$i<80;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); }
455 $code.=<<___;
456 movdqa (%rbx),@Xi[0] # pull counters
457 mov \$1,%ecx
458 cmp 4*0(%rbx),%ecx # examine counters
459 pxor $t2,$t2
460 cmovge $Tbl,@ptr[0] # cancel input
461 cmp 4*1(%rbx),%ecx
462 movdqa @Xi[0],@Xi[1]
463 cmovge $Tbl,@ptr[1]
464 cmp 4*2(%rbx),%ecx
465 pcmpgtd $t2,@Xi[1] # mask value
466 cmovge $Tbl,@ptr[2]
467 cmp 4*3(%rbx),%ecx
468 paddd @Xi[1],@Xi[0] # counters--
469 cmovge $Tbl,@ptr[3]
470
471 movdqu 0x00($ctx),$t0
472 pand @Xi[1],$A
473 movdqu 0x20($ctx),$t1
474 pand @Xi[1],$B
475 paddd $t0,$A
476 movdqu 0x40($ctx),$t2
477 pand @Xi[1],$C
478 paddd $t1,$B
479 movdqu 0x60($ctx),$t3
480 pand @Xi[1],$D
481 paddd $t2,$C
482 movdqu 0x80($ctx),$tx
483 pand @Xi[1],$E
484 movdqu $A,0x00($ctx)
485 paddd $t3,$D
486 movdqu $B,0x20($ctx)
487 paddd $tx,$E
488 movdqu $C,0x40($ctx)
489 movdqu $D,0x60($ctx)
490 movdqu $E,0x80($ctx)
491
492 movdqa @Xi[0],(%rbx) # save counters
493 movdqa 0x60($Tbl),$tx # pbswap_mask
494 movdqa -0x20($Tbl),$K # K_00_19
495 dec $num
496 jnz .Loop
497
498 mov `$REG_SZ*17+8`(%rsp),$num
499 lea $REG_SZ($ctx),$ctx
500 lea `$inp_elm_size*$REG_SZ/4`($inp),$inp
501 dec $num
502 jnz .Loop_grande
503
504 .Ldone:
505 mov `$REG_SZ*17`(%rsp),%rax # original %rsp
506 .cfi_def_cfa %rax,8
507 ___
508 $code.=<<___ if ($win64);
509 movaps -0xb8(%rax),%xmm6
510 movaps -0xa8(%rax),%xmm7
511 movaps -0x98(%rax),%xmm8
512 movaps -0x88(%rax),%xmm9
513 movaps -0x78(%rax),%xmm10
514 movaps -0x68(%rax),%xmm11
515 movaps -0x58(%rax),%xmm12
516 movaps -0x48(%rax),%xmm13
517 movaps -0x38(%rax),%xmm14
518 movaps -0x28(%rax),%xmm15
519 ___
520 $code.=<<___;
521 mov -16(%rax),%rbp
522 .cfi_restore %rbp
523 mov -8(%rax),%rbx
524 .cfi_restore %rbx
525 lea (%rax),%rsp
526 .cfi_def_cfa_register %rsp
527 .Lepilogue:
528 ret
529 .cfi_endproc
530 .size sha1_multi_block,.-sha1_multi_block
531 ___
532 {{{
533 my ($ABCD0,$E0,$E0_,$BSWAP,$ABCD1,$E1,$E1_)=map("%xmm$_",(0..3,8..10));
534 my @MSG0=map("%xmm$_",(4..7));
535 my @MSG1=map("%xmm$_",(11..14));
536
537 $code.=<<___;
538 .type sha1_multi_block_shaext,\@function,3
539 .align 32
540 sha1_multi_block_shaext:
541 .cfi_startproc
542 _shaext_shortcut:
543 mov %rsp,%rax
544 .cfi_def_cfa_register %rax
545 push %rbx
546 .cfi_push %rbx
547 push %rbp
548 .cfi_push %rbp
549 ___
550 $code.=<<___ if ($win64);
551 lea -0xa8(%rsp),%rsp
552 movaps %xmm6,(%rsp)
553 movaps %xmm7,0x10(%rsp)
554 movaps %xmm8,0x20(%rsp)
555 movaps %xmm9,0x30(%rsp)
556 movaps %xmm10,-0x78(%rax)
557 movaps %xmm11,-0x68(%rax)
558 movaps %xmm12,-0x58(%rax)
559 movaps %xmm13,-0x48(%rax)
560 movaps %xmm14,-0x38(%rax)
561 movaps %xmm15,-0x28(%rax)
562 ___
563 $code.=<<___;
564 sub \$`$REG_SZ*18`,%rsp
565 shl \$1,$num # we process pair at a time
566 and \$-256,%rsp
567 lea 0x40($ctx),$ctx # size optimization
568 mov %rax,`$REG_SZ*17`(%rsp) # original %rsp
569 .Lbody_shaext:
570 lea `$REG_SZ*16`(%rsp),%rbx
571 movdqa K_XX_XX+0x80(%rip),$BSWAP # byte-n-word swap
572
573 .Loop_grande_shaext:
574 mov $num,`$REG_SZ*17+8`(%rsp) # original $num
575 xor $num,$num
576 ___
577 for($i=0;$i<2;$i++) {
578 $ptr_reg=&pointer_register($flavour,@ptr[$i]);
579 $code.=<<___;
580 # input pointer
581 mov `$inp_elm_size*$i+0`($inp),$ptr_reg
582 # number of blocks
583 mov `$inp_elm_size*$i+$ptr_size`($inp),%ecx
584 cmp $num,%ecx
585 cmovg %ecx,$num # find maximum
586 test %ecx,%ecx
587 mov %ecx,`4*$i`(%rbx) # initialize counters
588 cmovle %rsp,@ptr[$i] # cancel input
589 ___
590 }
591 $code.=<<___;
592 test $num,$num
593 jz .Ldone_shaext
594
595 movq 0x00-0x40($ctx),$ABCD0 # a1.a0
596 movq 0x20-0x40($ctx),@MSG0[0]# b1.b0
597 movq 0x40-0x40($ctx),@MSG0[1]# c1.c0
598 movq 0x60-0x40($ctx),@MSG0[2]# d1.d0
599 movq 0x80-0x40($ctx),@MSG0[3]# e1.e0
600
601 punpckldq @MSG0[0],$ABCD0 # b1.a1.b0.a0
602 punpckldq @MSG0[2],@MSG0[1] # d1.c1.d0.c0
603
604 movdqa $ABCD0,$ABCD1
605 punpcklqdq @MSG0[1],$ABCD0 # d0.c0.b0.a0
606 punpckhqdq @MSG0[1],$ABCD1 # d1.c1.b1.a1
607
608 pshufd \$0b00111111,@MSG0[3],$E0
609 pshufd \$0b01111111,@MSG0[3],$E1
610 pshufd \$0b00011011,$ABCD0,$ABCD0
611 pshufd \$0b00011011,$ABCD1,$ABCD1
612 jmp .Loop_shaext
613
614 .align 32
615 .Loop_shaext:
616 movdqu 0x00(@ptr[0]),@MSG0[0]
617 movdqu 0x00(@ptr[1]),@MSG1[0]
618 movdqu 0x10(@ptr[0]),@MSG0[1]
619 movdqu 0x10(@ptr[1]),@MSG1[1]
620 movdqu 0x20(@ptr[0]),@MSG0[2]
621 pshufb $BSWAP,@MSG0[0]
622 movdqu 0x20(@ptr[1]),@MSG1[2]
623 pshufb $BSWAP,@MSG1[0]
624 movdqu 0x30(@ptr[0]),@MSG0[3]
625 lea 0x40(@ptr[0]),@ptr[0]
626 pshufb $BSWAP,@MSG0[1]
627 movdqu 0x30(@ptr[1]),@MSG1[3]
628 lea 0x40(@ptr[1]),@ptr[1]
629 pshufb $BSWAP,@MSG1[1]
630
631 movdqa $E0,0x50(%rsp) # offload
632 paddd @MSG0[0],$E0
633 movdqa $E1,0x70(%rsp)
634 paddd @MSG1[0],$E1
635 movdqa $ABCD0,0x40(%rsp) # offload
636 movdqa $ABCD0,$E0_
637 movdqa $ABCD1,0x60(%rsp)
638 movdqa $ABCD1,$E1_
639 sha1rnds4 \$0,$E0,$ABCD0 # 0-3
640 sha1nexte @MSG0[1],$E0_
641 sha1rnds4 \$0,$E1,$ABCD1 # 0-3
642 sha1nexte @MSG1[1],$E1_
643 pshufb $BSWAP,@MSG0[2]
644 prefetcht0 127(@ptr[0])
645 sha1msg1 @MSG0[1],@MSG0[0]
646 pshufb $BSWAP,@MSG1[2]
647 prefetcht0 127(@ptr[1])
648 sha1msg1 @MSG1[1],@MSG1[0]
649
650 pshufb $BSWAP,@MSG0[3]
651 movdqa $ABCD0,$E0
652 pshufb $BSWAP,@MSG1[3]
653 movdqa $ABCD1,$E1
654 sha1rnds4 \$0,$E0_,$ABCD0 # 4-7
655 sha1nexte @MSG0[2],$E0
656 sha1rnds4 \$0,$E1_,$ABCD1 # 4-7
657 sha1nexte @MSG1[2],$E1
658 pxor @MSG0[2],@MSG0[0]
659 sha1msg1 @MSG0[2],@MSG0[1]
660 pxor @MSG1[2],@MSG1[0]
661 sha1msg1 @MSG1[2],@MSG1[1]
662 ___
663 for($i=2;$i<20-4;$i++) {
664 $code.=<<___;
665 movdqa $ABCD0,$E0_
666 movdqa $ABCD1,$E1_
667 sha1rnds4 \$`int($i/5)`,$E0,$ABCD0 # 8-11
668 sha1nexte @MSG0[3],$E0_
669 sha1rnds4 \$`int($i/5)`,$E1,$ABCD1 # 8-11
670 sha1nexte @MSG1[3],$E1_
671 sha1msg2 @MSG0[3],@MSG0[0]
672 sha1msg2 @MSG1[3],@MSG1[0]
673 pxor @MSG0[3],@MSG0[1]
674 sha1msg1 @MSG0[3],@MSG0[2]
675 pxor @MSG1[3],@MSG1[1]
676 sha1msg1 @MSG1[3],@MSG1[2]
677 ___
678 ($E0,$E0_)=($E0_,$E0); ($E1,$E1_)=($E1_,$E1);
679 push(@MSG0,shift(@MSG0)); push(@MSG1,shift(@MSG1));
680 }
681 $code.=<<___;
682 movdqa $ABCD0,$E0_
683 movdqa $ABCD1,$E1_
684 sha1rnds4 \$3,$E0,$ABCD0 # 64-67
685 sha1nexte @MSG0[3],$E0_
686 sha1rnds4 \$3,$E1,$ABCD1 # 64-67
687 sha1nexte @MSG1[3],$E1_
688 sha1msg2 @MSG0[3],@MSG0[0]
689 sha1msg2 @MSG1[3],@MSG1[0]
690 pxor @MSG0[3],@MSG0[1]
691 pxor @MSG1[3],@MSG1[1]
692
693 mov \$1,%ecx
694 pxor @MSG0[2],@MSG0[2] # zero
695 cmp 4*0(%rbx),%ecx # examine counters
696 cmovge %rsp,@ptr[0] # cancel input
697
698 movdqa $ABCD0,$E0
699 movdqa $ABCD1,$E1
700 sha1rnds4 \$3,$E0_,$ABCD0 # 68-71
701 sha1nexte @MSG0[0],$E0
702 sha1rnds4 \$3,$E1_,$ABCD1 # 68-71
703 sha1nexte @MSG1[0],$E1
704 sha1msg2 @MSG0[0],@MSG0[1]
705 sha1msg2 @MSG1[0],@MSG1[1]
706
707 cmp 4*1(%rbx),%ecx
708 cmovge %rsp,@ptr[1]
709 movq (%rbx),@MSG0[0] # pull counters
710
711 movdqa $ABCD0,$E0_
712 movdqa $ABCD1,$E1_
713 sha1rnds4 \$3,$E0,$ABCD0 # 72-75
714 sha1nexte @MSG0[1],$E0_
715 sha1rnds4 \$3,$E1,$ABCD1 # 72-75
716 sha1nexte @MSG1[1],$E1_
717
718 pshufd \$0x00,@MSG0[0],@MSG1[2]
719 pshufd \$0x55,@MSG0[0],@MSG1[3]
720 movdqa @MSG0[0],@MSG0[1]
721 pcmpgtd @MSG0[2],@MSG1[2]
722 pcmpgtd @MSG0[2],@MSG1[3]
723
724 movdqa $ABCD0,$E0
725 movdqa $ABCD1,$E1
726 sha1rnds4 \$3,$E0_,$ABCD0 # 76-79
727 sha1nexte $MSG0[2],$E0
728 sha1rnds4 \$3,$E1_,$ABCD1 # 76-79
729 sha1nexte $MSG0[2],$E1
730
731 pcmpgtd @MSG0[2],@MSG0[1] # counter mask
732 pand @MSG1[2],$ABCD0
733 pand @MSG1[2],$E0
734 pand @MSG1[3],$ABCD1
735 pand @MSG1[3],$E1
736 paddd @MSG0[1],@MSG0[0] # counters--
737
738 paddd 0x40(%rsp),$ABCD0
739 paddd 0x50(%rsp),$E0
740 paddd 0x60(%rsp),$ABCD1
741 paddd 0x70(%rsp),$E1
742
743 movq @MSG0[0],(%rbx) # save counters
744 dec $num
745 jnz .Loop_shaext
746
747 mov `$REG_SZ*17+8`(%rsp),$num
748
749 pshufd \$0b00011011,$ABCD0,$ABCD0
750 pshufd \$0b00011011,$ABCD1,$ABCD1
751
752 movdqa $ABCD0,@MSG0[0]
753 punpckldq $ABCD1,$ABCD0 # b1.b0.a1.a0
754 punpckhdq $ABCD1,@MSG0[0] # d1.d0.c1.c0
755 punpckhdq $E1,$E0 # e1.e0.xx.xx
756 movq $ABCD0,0x00-0x40($ctx) # a1.a0
757 psrldq \$8,$ABCD0
758 movq @MSG0[0],0x40-0x40($ctx)# c1.c0
759 psrldq \$8,@MSG0[0]
760 movq $ABCD0,0x20-0x40($ctx) # b1.b0
761 psrldq \$8,$E0
762 movq @MSG0[0],0x60-0x40($ctx)# d1.d0
763 movq $E0,0x80-0x40($ctx) # e1.e0
764
765 lea `$REG_SZ/2`($ctx),$ctx
766 lea `$inp_elm_size*2`($inp),$inp
767 dec $num
768 jnz .Loop_grande_shaext
769
770 .Ldone_shaext:
771 #mov `$REG_SZ*17`(%rsp),%rax # original %rsp
772 ___
773 $code.=<<___ if ($win64);
774 movaps -0xb8(%rax),%xmm6
775 movaps -0xa8(%rax),%xmm7
776 movaps -0x98(%rax),%xmm8
777 movaps -0x88(%rax),%xmm9
778 movaps -0x78(%rax),%xmm10
779 movaps -0x68(%rax),%xmm11
780 movaps -0x58(%rax),%xmm12
781 movaps -0x48(%rax),%xmm13
782 movaps -0x38(%rax),%xmm14
783 movaps -0x28(%rax),%xmm15
784 ___
785 $code.=<<___;
786 mov -16(%rax),%rbp
787 .cfi_restore %rbp
788 mov -8(%rax),%rbx
789 .cfi_restore %rbx
790 lea (%rax),%rsp
791 .cfi_def_cfa_register %rsp
792 .Lepilogue_shaext:
793 ret
794 .cfi_endproc
795 .size sha1_multi_block_shaext,.-sha1_multi_block_shaext
796 ___
797 }}}
798
799 if ($avx) {{{
800 sub BODY_00_19_avx {
801 my ($i,$a,$b,$c,$d,$e)=@_;
802 my $j=$i+1;
803 my $k=$i+2;
804 my $vpack = $REG_SZ==16 ? "vpunpckldq" : "vinserti128";
805 my $ptr_n = $REG_SZ==16 ? @ptr[1] : @ptr[4];
806
807 $code.=<<___ if ($i==0 && $REG_SZ==16);
808 vmovd (@ptr[0]),@Xi[0]
809 lea `16*4`(@ptr[0]),@ptr[0]
810 vmovd (@ptr[1]),@Xi[2] # borrow Xi[2]
811 lea `16*4`(@ptr[1]),@ptr[1]
812 vpinsrd \$1,(@ptr[2]),@Xi[0],@Xi[0]
813 lea `16*4`(@ptr[2]),@ptr[2]
814 vpinsrd \$1,(@ptr[3]),@Xi[2],@Xi[2]
815 lea `16*4`(@ptr[3]),@ptr[3]
816 vmovd `4*$j-16*4`(@ptr[0]),@Xi[1]
817 vpunpckldq @Xi[2],@Xi[0],@Xi[0]
818 vmovd `4*$j-16*4`($ptr_n),$t3
819 vpshufb $tx,@Xi[0],@Xi[0]
820 ___
821 $code.=<<___ if ($i<15 && $REG_SZ==16); # just load input
822 vpinsrd \$1,`4*$j-16*4`(@ptr[2]),@Xi[1],@Xi[1]
823 vpinsrd \$1,`4*$j-16*4`(@ptr[3]),$t3,$t3
824 ___
825 $code.=<<___ if ($i==0 && $REG_SZ==32);
826 vmovd (@ptr[0]),@Xi[0]
827 lea `16*4`(@ptr[0]),@ptr[0]
828 vmovd (@ptr[4]),@Xi[2] # borrow Xi[2]
829 lea `16*4`(@ptr[4]),@ptr[4]
830 vmovd (@ptr[1]),$t2
831 lea `16*4`(@ptr[1]),@ptr[1]
832 vmovd (@ptr[5]),$t1
833 lea `16*4`(@ptr[5]),@ptr[5]
834 vpinsrd \$1,(@ptr[2]),@Xi[0],@Xi[0]
835 lea `16*4`(@ptr[2]),@ptr[2]
836 vpinsrd \$1,(@ptr[6]),@Xi[2],@Xi[2]
837 lea `16*4`(@ptr[6]),@ptr[6]
838 vpinsrd \$1,(@ptr[3]),$t2,$t2
839 lea `16*4`(@ptr[3]),@ptr[3]
840 vpunpckldq $t2,@Xi[0],@Xi[0]
841 vpinsrd \$1,(@ptr[7]),$t1,$t1
842 lea `16*4`(@ptr[7]),@ptr[7]
843 vpunpckldq $t1,@Xi[2],@Xi[2]
844 vmovd `4*$j-16*4`(@ptr[0]),@Xi[1]
845 vinserti128 @Xi[2],@Xi[0],@Xi[0]
846 vmovd `4*$j-16*4`($ptr_n),$t3
847 vpshufb $tx,@Xi[0],@Xi[0]
848 ___
849 $code.=<<___ if ($i<15 && $REG_SZ==32); # just load input
850 vmovd `4*$j-16*4`(@ptr[1]),$t2
851 vmovd `4*$j-16*4`(@ptr[5]),$t1
852 vpinsrd \$1,`4*$j-16*4`(@ptr[2]),@Xi[1],@Xi[1]
853 vpinsrd \$1,`4*$j-16*4`(@ptr[6]),$t3,$t3
854 vpinsrd \$1,`4*$j-16*4`(@ptr[3]),$t2,$t2
855 vpunpckldq $t2,@Xi[1],@Xi[1]
856 vpinsrd \$1,`4*$j-16*4`(@ptr[7]),$t1,$t1
857 vpunpckldq $t1,$t3,$t3
858 ___
859 $code.=<<___ if ($i<14);
860 vpaddd $K,$e,$e # e+=K_00_19
861 vpslld \$5,$a,$t2
862 vpandn $d,$b,$t1
863 vpand $c,$b,$t0
864
865 vmovdqa @Xi[0],`&Xi_off($i)`
866 vpaddd @Xi[0],$e,$e # e+=X[i]
867 $vpack $t3,@Xi[1],@Xi[1]
868 vpsrld \$27,$a,$t3
869 vpxor $t1,$t0,$t0 # Ch(b,c,d)
870 vmovd `4*$k-16*4`(@ptr[0]),@Xi[2]
871
872 vpslld \$30,$b,$t1
873 vpor $t3,$t2,$t2 # rol(a,5)
874 vmovd `4*$k-16*4`($ptr_n),$t3
875 vpaddd $t0,$e,$e # e+=Ch(b,c,d)
876
877 vpsrld \$2,$b,$b
878 vpaddd $t2,$e,$e # e+=rol(a,5)
879 vpshufb $tx,@Xi[1],@Xi[1]
880 vpor $t1,$b,$b # b=rol(b,30)
881 ___
882 $code.=<<___ if ($i==14);
883 vpaddd $K,$e,$e # e+=K_00_19
884 prefetcht0 63(@ptr[0])
885 vpslld \$5,$a,$t2
886 vpandn $d,$b,$t1
887 vpand $c,$b,$t0
888
889 vmovdqa @Xi[0],`&Xi_off($i)`
890 vpaddd @Xi[0],$e,$e # e+=X[i]
891 $vpack $t3,@Xi[1],@Xi[1]
892 vpsrld \$27,$a,$t3
893 prefetcht0 63(@ptr[1])
894 vpxor $t1,$t0,$t0 # Ch(b,c,d)
895
896 vpslld \$30,$b,$t1
897 vpor $t3,$t2,$t2 # rol(a,5)
898 prefetcht0 63(@ptr[2])
899 vpaddd $t0,$e,$e # e+=Ch(b,c,d)
900
901 vpsrld \$2,$b,$b
902 vpaddd $t2,$e,$e # e+=rol(a,5)
903 prefetcht0 63(@ptr[3])
904 vpshufb $tx,@Xi[1],@Xi[1]
905 vpor $t1,$b,$b # b=rol(b,30)
906 ___
907 $code.=<<___ if ($i>=13 && $i<15);
908 vmovdqa `&Xi_off($j+2)`,@Xi[3] # preload "X[2]"
909 ___
910 $code.=<<___ if ($i>=15); # apply Xupdate
911 vpxor @Xi[-2],@Xi[1],@Xi[1] # "X[13]"
912 vmovdqa `&Xi_off($j+2)`,@Xi[3] # "X[2]"
913
914 vpaddd $K,$e,$e # e+=K_00_19
915 vpslld \$5,$a,$t2
916 vpandn $d,$b,$t1
917 `"prefetcht0 63(@ptr[4])" if ($i==15 && $REG_SZ==32)`
918 vpand $c,$b,$t0
919
920 vmovdqa @Xi[0],`&Xi_off($i)`
921 vpaddd @Xi[0],$e,$e # e+=X[i]
922 vpxor `&Xi_off($j+8)`,@Xi[1],@Xi[1]
923 vpsrld \$27,$a,$t3
924 vpxor $t1,$t0,$t0 # Ch(b,c,d)
925 vpxor @Xi[3],@Xi[1],@Xi[1]
926 `"prefetcht0 63(@ptr[5])" if ($i==15 && $REG_SZ==32)`
927
928 vpslld \$30,$b,$t1
929 vpor $t3,$t2,$t2 # rol(a,5)
930 vpaddd $t0,$e,$e # e+=Ch(b,c,d)
931 `"prefetcht0 63(@ptr[6])" if ($i==15 && $REG_SZ==32)`
932 vpsrld \$31,@Xi[1],$tx
933 vpaddd @Xi[1],@Xi[1],@Xi[1]
934
935 vpsrld \$2,$b,$b
936 `"prefetcht0 63(@ptr[7])" if ($i==15 && $REG_SZ==32)`
937 vpaddd $t2,$e,$e # e+=rol(a,5)
938 vpor $tx,@Xi[1],@Xi[1] # rol \$1,@Xi[1]
939 vpor $t1,$b,$b # b=rol(b,30)
940 ___
941 push(@Xi,shift(@Xi));
942 }
943
944 sub BODY_20_39_avx {
945 my ($i,$a,$b,$c,$d,$e)=@_;
946 my $j=$i+1;
947
948 $code.=<<___ if ($i<79);
949 vpxor @Xi[-2],@Xi[1],@Xi[1] # "X[13]"
950 vmovdqa `&Xi_off($j+2)`,@Xi[3] # "X[2]"
951
952 vpslld \$5,$a,$t2
953 vpaddd $K,$e,$e # e+=K_20_39
954 vpxor $b,$d,$t0
955 ___
956 $code.=<<___ if ($i<72);
957 vmovdqa @Xi[0],`&Xi_off($i)`
958 ___
959 $code.=<<___ if ($i<79);
960 vpaddd @Xi[0],$e,$e # e+=X[i]
961 vpxor `&Xi_off($j+8)`,@Xi[1],@Xi[1]
962 vpsrld \$27,$a,$t3
963 vpxor $c,$t0,$t0 # Parity(b,c,d)
964 vpxor @Xi[3],@Xi[1],@Xi[1]
965
966 vpslld \$30,$b,$t1
967 vpor $t3,$t2,$t2 # rol(a,5)
968 vpaddd $t0,$e,$e # e+=Parity(b,c,d)
969 vpsrld \$31,@Xi[1],$tx
970 vpaddd @Xi[1],@Xi[1],@Xi[1]
971
972 vpsrld \$2,$b,$b
973 vpaddd $t2,$e,$e # e+=rol(a,5)
974 vpor $tx,@Xi[1],@Xi[1] # rol(@Xi[1],1)
975 vpor $t1,$b,$b # b=rol(b,30)
976 ___
977 $code.=<<___ if ($i==79);
978 vpslld \$5,$a,$t2
979 vpaddd $K,$e,$e # e+=K_20_39
980 vpxor $b,$d,$t0
981
982 vpsrld \$27,$a,$t3
983 vpaddd @Xi[0],$e,$e # e+=X[i]
984 vpxor $c,$t0,$t0 # Parity(b,c,d)
985
986 vpslld \$30,$b,$t1
987 vpor $t3,$t2,$t2 # rol(a,5)
988 vpaddd $t0,$e,$e # e+=Parity(b,c,d)
989
990 vpsrld \$2,$b,$b
991 vpaddd $t2,$e,$e # e+=rol(a,5)
992 vpor $t1,$b,$b # b=rol(b,30)
993 ___
994 push(@Xi,shift(@Xi));
995 }
996
997 sub BODY_40_59_avx {
998 my ($i,$a,$b,$c,$d,$e)=@_;
999 my $j=$i+1;
1000
1001 $code.=<<___;
1002 vpxor @Xi[-2],@Xi[1],@Xi[1] # "X[13]"
1003 vmovdqa `&Xi_off($j+2)`,@Xi[3] # "X[2]"
1004
1005 vpaddd $K,$e,$e # e+=K_40_59
1006 vpslld \$5,$a,$t2
1007 vpand $c,$d,$t1
1008 vpxor `&Xi_off($j+8)`,@Xi[1],@Xi[1]
1009
1010 vpaddd $t1,$e,$e
1011 vpsrld \$27,$a,$t3
1012 vpxor $c,$d,$t0
1013 vpxor @Xi[3],@Xi[1],@Xi[1]
1014
1015 vmovdqu @Xi[0],`&Xi_off($i)`
1016 vpaddd @Xi[0],$e,$e # e+=X[i]
1017 vpor $t3,$t2,$t2 # rol(a,5)
1018 vpsrld \$31,@Xi[1],$tx
1019 vpand $b,$t0,$t0
1020 vpaddd @Xi[1],@Xi[1],@Xi[1]
1021
1022 vpslld \$30,$b,$t1
1023 vpaddd $t0,$e,$e # e+=Maj(b,d,c)
1024
1025 vpsrld \$2,$b,$b
1026 vpaddd $t2,$e,$e # e+=rol(a,5)
1027 vpor $tx,@Xi[1],@Xi[1] # rol(@X[1],1)
1028 vpor $t1,$b,$b # b=rol(b,30)
1029 ___
1030 push(@Xi,shift(@Xi));
1031 }
1032
1033 $code.=<<___;
1034 .type sha1_multi_block_avx,\@function,3
1035 .align 32
1036 sha1_multi_block_avx:
1037 .cfi_startproc
1038 _avx_shortcut:
1039 ___
1040 $code.=<<___ if ($avx>1);
1041 shr \$32,%rcx
1042 cmp \$2,$num
1043 jb .Lavx
1044 test \$`1<<5`,%ecx
1045 jnz _avx2_shortcut
1046 jmp .Lavx
1047 .align 32
1048 .Lavx:
1049 ___
1050 $code.=<<___;
1051 mov %rsp,%rax
1052 .cfi_def_cfa_register %rax
1053 push %rbx
1054 .cfi_push %rbx
1055 push %rbp
1056 .cfi_push %rbp
1057 ___
1058 $code.=<<___ if ($win64);
1059 lea -0xa8(%rsp),%rsp
1060 movaps %xmm6,(%rsp)
1061 movaps %xmm7,0x10(%rsp)
1062 movaps %xmm8,0x20(%rsp)
1063 movaps %xmm9,0x30(%rsp)
1064 movaps %xmm10,-0x78(%rax)
1065 movaps %xmm11,-0x68(%rax)
1066 movaps %xmm12,-0x58(%rax)
1067 movaps %xmm13,-0x48(%rax)
1068 movaps %xmm14,-0x38(%rax)
1069 movaps %xmm15,-0x28(%rax)
1070 ___
1071 $code.=<<___;
1072 sub \$`$REG_SZ*18`, %rsp
1073 and \$-256,%rsp
1074 mov %rax,`$REG_SZ*17`(%rsp) # original %rsp
1075 .cfi_cfa_expression %rsp+`$REG_SZ*17`,deref,+8
1076 .Lbody_avx:
1077 lea K_XX_XX(%rip),$Tbl
1078 lea `$REG_SZ*16`(%rsp),%rbx
1079
1080 vzeroupper
1081 .Loop_grande_avx:
1082 mov $num,`$REG_SZ*17+8`(%rsp) # original $num
1083 xor $num,$num
1084 ___
1085 for($i=0;$i<4;$i++) {
1086 $ptr_reg=&pointer_register($flavour,@ptr[$i]);
1087 $code.=<<___;
1088 # input pointer
1089 mov `$inp_elm_size*$i+0`($inp),$ptr_reg
1090 # number of blocks
1091 mov `$inp_elm_size*$i+$ptr_size`($inp),%ecx
1092 cmp $num,%ecx
1093 cmovg %ecx,$num # find maximum
1094 test %ecx,%ecx
1095 mov %ecx,`4*$i`(%rbx) # initialize counters
1096 cmovle $Tbl,@ptr[$i] # cancel input
1097 ___
1098 }
1099 $code.=<<___;
1100 test $num,$num
1101 jz .Ldone_avx
1102
1103 vmovdqu 0x00($ctx),$A # load context
1104 lea 128(%rsp),%rax
1105 vmovdqu 0x20($ctx),$B
1106 vmovdqu 0x40($ctx),$C
1107 vmovdqu 0x60($ctx),$D
1108 vmovdqu 0x80($ctx),$E
1109 vmovdqu 0x60($Tbl),$tx # pbswap_mask
1110 jmp .Loop_avx
1111
1112 .align 32
1113 .Loop_avx:
1114 ___
1115 $code.=" vmovdqa -0x20($Tbl),$K\n"; # K_00_19
1116 for($i=0;$i<20;$i++) { &BODY_00_19_avx($i,@V); unshift(@V,pop(@V)); }
1117 $code.=" vmovdqa 0x00($Tbl),$K\n"; # K_20_39
1118 for(;$i<40;$i++) { &BODY_20_39_avx($i,@V); unshift(@V,pop(@V)); }
1119 $code.=" vmovdqa 0x20($Tbl),$K\n"; # K_40_59
1120 for(;$i<60;$i++) { &BODY_40_59_avx($i,@V); unshift(@V,pop(@V)); }
1121 $code.=" vmovdqa 0x40($Tbl),$K\n"; # K_60_79
1122 for(;$i<80;$i++) { &BODY_20_39_avx($i,@V); unshift(@V,pop(@V)); }
1123 $code.=<<___;
1124 mov \$1,%ecx
1125 ___
1126 for($i=0;$i<4;$i++) {
1127 $code.=<<___;
1128 cmp `4*$i`(%rbx),%ecx # examine counters
1129 cmovge $Tbl,@ptr[$i] # cancel input
1130 ___
1131 }
1132 $code.=<<___;
1133 vmovdqu (%rbx),$t0 # pull counters
1134 vpxor $t2,$t2,$t2
1135 vmovdqa $t0,$t1
1136 vpcmpgtd $t2,$t1,$t1 # mask value
1137 vpaddd $t1,$t0,$t0 # counters--
1138
1139 vpand $t1,$A,$A
1140 vpand $t1,$B,$B
1141 vpaddd 0x00($ctx),$A,$A
1142 vpand $t1,$C,$C
1143 vpaddd 0x20($ctx),$B,$B
1144 vpand $t1,$D,$D
1145 vpaddd 0x40($ctx),$C,$C
1146 vpand $t1,$E,$E
1147 vpaddd 0x60($ctx),$D,$D
1148 vpaddd 0x80($ctx),$E,$E
1149 vmovdqu $A,0x00($ctx)
1150 vmovdqu $B,0x20($ctx)
1151 vmovdqu $C,0x40($ctx)
1152 vmovdqu $D,0x60($ctx)
1153 vmovdqu $E,0x80($ctx)
1154
1155 vmovdqu $t0,(%rbx) # save counters
1156 vmovdqu 0x60($Tbl),$tx # pbswap_mask
1157 dec $num
1158 jnz .Loop_avx
1159
1160 mov `$REG_SZ*17+8`(%rsp),$num
1161 lea $REG_SZ($ctx),$ctx
1162 lea `$inp_elm_size*$REG_SZ/4`($inp),$inp
1163 dec $num
1164 jnz .Loop_grande_avx
1165
1166 .Ldone_avx:
1167 mov `$REG_SZ*17`(%rsp),%rax # original %rsp
1168 .cfi_def_cfa %rax,8
1169 vzeroupper
1170 ___
1171 $code.=<<___ if ($win64);
1172 movaps -0xb8(%rax),%xmm6
1173 movaps -0xa8(%rax),%xmm7
1174 movaps -0x98(%rax),%xmm8
1175 movaps -0x88(%rax),%xmm9
1176 movaps -0x78(%rax),%xmm10
1177 movaps -0x68(%rax),%xmm11
1178 movaps -0x58(%rax),%xmm12
1179 movaps -0x48(%rax),%xmm13
1180 movaps -0x38(%rax),%xmm14
1181 movaps -0x28(%rax),%xmm15
1182 ___
1183 $code.=<<___;
1184 mov -16(%rax),%rbp
1185 .cfi_restore %rbp
1186 mov -8(%rax),%rbx
1187 .cfi_restore %rbx
1188 lea (%rax),%rsp
1189 .cfi_def_cfa_register %rsp
1190 .Lepilogue_avx:
1191 ret
1192 .cfi_endproc
1193 .size sha1_multi_block_avx,.-sha1_multi_block_avx
1194 ___
1195
1196 if ($avx>1) {
1197 $code =~ s/\`([^\`]*)\`/eval $1/gem;
1198
1199 $REG_SZ=32;
1200
1201 @ptr=map("%r$_",(12..15,8..11));
1202
1203 @V=($A,$B,$C,$D,$E)=map("%ymm$_",(0..4));
1204 ($t0,$t1,$t2,$t3,$tx)=map("%ymm$_",(5..9));
1205 @Xi=map("%ymm$_",(10..14));
1206 $K="%ymm15";
1207
1208 $code.=<<___;
1209 .type sha1_multi_block_avx2,\@function,3
1210 .align 32
1211 sha1_multi_block_avx2:
1212 .cfi_startproc
1213 _avx2_shortcut:
1214 mov %rsp,%rax
1215 .cfi_def_cfa_register %rax
1216 push %rbx
1217 .cfi_push %rbx
1218 push %rbp
1219 .cfi_push %rbp
1220 push %r12
1221 .cfi_push %r12
1222 push %r13
1223 .cfi_push %r13
1224 push %r14
1225 .cfi_push %r14
1226 push %r15
1227 .cfi_push %r15
1228 ___
1229 $code.=<<___ if ($win64);
1230 lea -0xa8(%rsp),%rsp
1231 movaps %xmm6,(%rsp)
1232 movaps %xmm7,0x10(%rsp)
1233 movaps %xmm8,0x20(%rsp)
1234 movaps %xmm9,0x30(%rsp)
1235 movaps %xmm10,0x40(%rsp)
1236 movaps %xmm11,0x50(%rsp)
1237 movaps %xmm12,-0x78(%rax)
1238 movaps %xmm13,-0x68(%rax)
1239 movaps %xmm14,-0x58(%rax)
1240 movaps %xmm15,-0x48(%rax)
1241 ___
1242 $code.=<<___;
1243 sub \$`$REG_SZ*18`, %rsp
1244 and \$-256,%rsp
1245 mov %rax,`$REG_SZ*17`(%rsp) # original %rsp
1246 .cfi_cfa_expression %rsp+`$REG_SZ*17`,deref,+8
1247 .Lbody_avx2:
1248 lea K_XX_XX(%rip),$Tbl
1249 shr \$1,$num
1250
1251 vzeroupper
1252 .Loop_grande_avx2:
1253 mov $num,`$REG_SZ*17+8`(%rsp) # original $num
1254 xor $num,$num
1255 lea `$REG_SZ*16`(%rsp),%rbx
1256 ___
1257 for($i=0;$i<8;$i++) {
1258 $ptr_reg=&pointer_register($flavour,@ptr[$i]);
1259 $code.=<<___;
1260 # input pointer
1261 mov `$inp_elm_size*$i+0`($inp),$ptr_reg
1262 # number of blocks
1263 mov `$inp_elm_size*$i+$ptr_size`($inp),%ecx
1264 cmp $num,%ecx
1265 cmovg %ecx,$num # find maximum
1266 test %ecx,%ecx
1267 mov %ecx,`4*$i`(%rbx) # initialize counters
1268 cmovle $Tbl,@ptr[$i] # cancel input
1269 ___
1270 }
1271 $code.=<<___;
1272 vmovdqu 0x00($ctx),$A # load context
1273 lea 128(%rsp),%rax
1274 vmovdqu 0x20($ctx),$B
1275 lea 256+128(%rsp),%rbx
1276 vmovdqu 0x40($ctx),$C
1277 vmovdqu 0x60($ctx),$D
1278 vmovdqu 0x80($ctx),$E
1279 vmovdqu 0x60($Tbl),$tx # pbswap_mask
1280 jmp .Loop_avx2
1281
1282 .align 32
1283 .Loop_avx2:
1284 ___
1285 $code.=" vmovdqa -0x20($Tbl),$K\n"; # K_00_19
1286 for($i=0;$i<20;$i++) { &BODY_00_19_avx($i,@V); unshift(@V,pop(@V)); }
1287 $code.=" vmovdqa 0x00($Tbl),$K\n"; # K_20_39
1288 for(;$i<40;$i++) { &BODY_20_39_avx($i,@V); unshift(@V,pop(@V)); }
1289 $code.=" vmovdqa 0x20($Tbl),$K\n"; # K_40_59
1290 for(;$i<60;$i++) { &BODY_40_59_avx($i,@V); unshift(@V,pop(@V)); }
1291 $code.=" vmovdqa 0x40($Tbl),$K\n"; # K_60_79
1292 for(;$i<80;$i++) { &BODY_20_39_avx($i,@V); unshift(@V,pop(@V)); }
1293 $code.=<<___;
1294 mov \$1,%ecx
1295 lea `$REG_SZ*16`(%rsp),%rbx
1296 ___
1297 for($i=0;$i<8;$i++) {
1298 $code.=<<___;
1299 cmp `4*$i`(%rbx),%ecx # examine counters
1300 cmovge $Tbl,@ptr[$i] # cancel input
1301 ___
1302 }
1303 $code.=<<___;
1304 vmovdqu (%rbx),$t0 # pull counters
1305 vpxor $t2,$t2,$t2
1306 vmovdqa $t0,$t1
1307 vpcmpgtd $t2,$t1,$t1 # mask value
1308 vpaddd $t1,$t0,$t0 # counters--
1309
1310 vpand $t1,$A,$A
1311 vpand $t1,$B,$B
1312 vpaddd 0x00($ctx),$A,$A
1313 vpand $t1,$C,$C
1314 vpaddd 0x20($ctx),$B,$B
1315 vpand $t1,$D,$D
1316 vpaddd 0x40($ctx),$C,$C
1317 vpand $t1,$E,$E
1318 vpaddd 0x60($ctx),$D,$D
1319 vpaddd 0x80($ctx),$E,$E
1320 vmovdqu $A,0x00($ctx)
1321 vmovdqu $B,0x20($ctx)
1322 vmovdqu $C,0x40($ctx)
1323 vmovdqu $D,0x60($ctx)
1324 vmovdqu $E,0x80($ctx)
1325
1326 vmovdqu $t0,(%rbx) # save counters
1327 lea 256+128(%rsp),%rbx
1328 vmovdqu 0x60($Tbl),$tx # pbswap_mask
1329 dec $num
1330 jnz .Loop_avx2
1331
1332 #mov `$REG_SZ*17+8`(%rsp),$num
1333 #lea $REG_SZ($ctx),$ctx
1334 #lea `$inp_elm_size*$REG_SZ/4`($inp),$inp
1335 #dec $num
1336 #jnz .Loop_grande_avx2
1337
1338 .Ldone_avx2:
1339 mov `$REG_SZ*17`(%rsp),%rax # original %rsp
1340 .cfi_def_cfa %rax,8
1341 vzeroupper
1342 ___
1343 $code.=<<___ if ($win64);
1344 movaps -0xd8(%rax),%xmm6
1345 movaps -0xc8(%rax),%xmm7
1346 movaps -0xb8(%rax),%xmm8
1347 movaps -0xa8(%rax),%xmm9
1348 movaps -0x98(%rax),%xmm10
1349 movaps -0x88(%rax),%xmm11
1350 movaps -0x78(%rax),%xmm12
1351 movaps -0x68(%rax),%xmm13
1352 movaps -0x58(%rax),%xmm14
1353 movaps -0x48(%rax),%xmm15
1354 ___
1355 $code.=<<___;
1356 mov -48(%rax),%r15
1357 .cfi_restore %r15
1358 mov -40(%rax),%r14
1359 .cfi_restore %r14
1360 mov -32(%rax),%r13
1361 .cfi_restore %r13
1362 mov -24(%rax),%r12
1363 .cfi_restore %r12
1364 mov -16(%rax),%rbp
1365 .cfi_restore %rbp
1366 mov -8(%rax),%rbx
1367 .cfi_restore %rbx
1368 lea (%rax),%rsp
1369 .cfi_def_cfa_register %rsp
1370 .Lepilogue_avx2:
1371 ret
1372 .cfi_endproc
1373 .size sha1_multi_block_avx2,.-sha1_multi_block_avx2
1374 ___
1375 } }}}
1376 $code.=<<___;
1377
1378 .align 256
1379 .long 0x5a827999,0x5a827999,0x5a827999,0x5a827999 # K_00_19
1380 .long 0x5a827999,0x5a827999,0x5a827999,0x5a827999 # K_00_19
1381 K_XX_XX:
1382 .long 0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1 # K_20_39
1383 .long 0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1 # K_20_39
1384 .long 0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc # K_40_59
1385 .long 0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc # K_40_59
1386 .long 0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6 # K_60_79
1387 .long 0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6 # K_60_79
1388 .long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f # pbswap
1389 .long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f # pbswap
1390 .byte 0xf,0xe,0xd,0xc,0xb,0xa,0x9,0x8,0x7,0x6,0x5,0x4,0x3,0x2,0x1,0x0
1391 .asciz "SHA1 multi-block transform for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
1392 ___
1393
1394 if ($win64) {
1395 # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
1396 # CONTEXT *context,DISPATCHER_CONTEXT *disp)
1397 $rec="%rcx";
1398 $frame="%rdx";
1399 $context="%r8";
1400 $disp="%r9";
1401
1402 $code.=<<___;
1403 .extern __imp_RtlVirtualUnwind
1404 .type se_handler,\@abi-omnipotent
1405 .align 16
1406 se_handler:
1407 push %rsi
1408 push %rdi
1409 push %rbx
1410 push %rbp
1411 push %r12
1412 push %r13
1413 push %r14
1414 push %r15
1415 pushfq
1416 sub \$64,%rsp
1417
1418 mov 120($context),%rax # pull context->Rax
1419 mov 248($context),%rbx # pull context->Rip
1420
1421 mov 8($disp),%rsi # disp->ImageBase
1422 mov 56($disp),%r11 # disp->HandlerData
1423
1424 mov 0(%r11),%r10d # HandlerData[0]
1425 lea (%rsi,%r10),%r10 # end of prologue label
1426 cmp %r10,%rbx # context->Rip<.Lbody
1427 jb .Lin_prologue
1428
1429 mov 152($context),%rax # pull context->Rsp
1430
1431 mov 4(%r11),%r10d # HandlerData[1]
1432 lea (%rsi,%r10),%r10 # epilogue label
1433 cmp %r10,%rbx # context->Rip>=.Lepilogue
1434 jae .Lin_prologue
1435
1436 mov `16*17`(%rax),%rax # pull saved stack pointer
1437
1438 mov -8(%rax),%rbx
1439 mov -16(%rax),%rbp
1440 mov %rbx,144($context) # restore context->Rbx
1441 mov %rbp,160($context) # restore context->Rbp
1442
1443 lea -24-10*16(%rax),%rsi
1444 lea 512($context),%rdi # &context.Xmm6
1445 mov \$20,%ecx
1446 .long 0xa548f3fc # cld; rep movsq
1447
1448 .Lin_prologue:
1449 mov 8(%rax),%rdi
1450 mov 16(%rax),%rsi
1451 mov %rax,152($context) # restore context->Rsp
1452 mov %rsi,168($context) # restore context->Rsi
1453 mov %rdi,176($context) # restore context->Rdi
1454
1455 mov 40($disp),%rdi # disp->ContextRecord
1456 mov $context,%rsi # context
1457 mov \$154,%ecx # sizeof(CONTEXT)
1458 .long 0xa548f3fc # cld; rep movsq
1459
1460 mov $disp,%rsi
1461 xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER
1462 mov 8(%rsi),%rdx # arg2, disp->ImageBase
1463 mov 0(%rsi),%r8 # arg3, disp->ControlPc
1464 mov 16(%rsi),%r9 # arg4, disp->FunctionEntry
1465 mov 40(%rsi),%r10 # disp->ContextRecord
1466 lea 56(%rsi),%r11 # &disp->HandlerData
1467 lea 24(%rsi),%r12 # &disp->EstablisherFrame
1468 mov %r10,32(%rsp) # arg5
1469 mov %r11,40(%rsp) # arg6
1470 mov %r12,48(%rsp) # arg7
1471 mov %rcx,56(%rsp) # arg8, (NULL)
1472 call *__imp_RtlVirtualUnwind(%rip)
1473
1474 mov \$1,%eax # ExceptionContinueSearch
1475 add \$64,%rsp
1476 popfq
1477 pop %r15
1478 pop %r14
1479 pop %r13
1480 pop %r12
1481 pop %rbp
1482 pop %rbx
1483 pop %rdi
1484 pop %rsi
1485 ret
1486 .size se_handler,.-se_handler
1487 ___
1488 $code.=<<___ if ($avx>1);
1489 .type avx2_handler,\@abi-omnipotent
1490 .align 16
1491 avx2_handler:
1492 push %rsi
1493 push %rdi
1494 push %rbx
1495 push %rbp
1496 push %r12
1497 push %r13
1498 push %r14
1499 push %r15
1500 pushfq
1501 sub \$64,%rsp
1502
1503 mov 120($context),%rax # pull context->Rax
1504 mov 248($context),%rbx # pull context->Rip
1505
1506 mov 8($disp),%rsi # disp->ImageBase
1507 mov 56($disp),%r11 # disp->HandlerData
1508
1509 mov 0(%r11),%r10d # HandlerData[0]
1510 lea (%rsi,%r10),%r10 # end of prologue label
1511 cmp %r10,%rbx # context->Rip<body label
1512 jb .Lin_prologue
1513
1514 mov 152($context),%rax # pull context->Rsp
1515
1516 mov 4(%r11),%r10d # HandlerData[1]
1517 lea (%rsi,%r10),%r10 # epilogue label
1518 cmp %r10,%rbx # context->Rip>=epilogue label
1519 jae .Lin_prologue
1520
1521 mov `32*17`($context),%rax # pull saved stack pointer
1522
1523 mov -8(%rax),%rbx
1524 mov -16(%rax),%rbp
1525 mov -24(%rax),%r12
1526 mov -32(%rax),%r13
1527 mov -40(%rax),%r14
1528 mov -48(%rax),%r15
1529 mov %rbx,144($context) # restore context->Rbx
1530 mov %rbp,160($context) # restore context->Rbp
1531 mov %r12,216($context) # restore context->R12
1532 mov %r13,224($context) # restore context->R13
1533 mov %r14,232($context) # restore context->R14
1534 mov %r15,240($context) # restore context->R15
1535
1536 lea -56-10*16(%rax),%rsi
1537 lea 512($context),%rdi # &context.Xmm6
1538 mov \$20,%ecx
1539 .long 0xa548f3fc # cld; rep movsq
1540
1541 jmp .Lin_prologue
1542 .size avx2_handler,.-avx2_handler
1543 ___
1544 $code.=<<___;
1545 .section .pdata
1546 .align 4
1547 .rva .LSEH_begin_sha1_multi_block
1548 .rva .LSEH_end_sha1_multi_block
1549 .rva .LSEH_info_sha1_multi_block
1550 .rva .LSEH_begin_sha1_multi_block_shaext
1551 .rva .LSEH_end_sha1_multi_block_shaext
1552 .rva .LSEH_info_sha1_multi_block_shaext
1553 ___
1554 $code.=<<___ if ($avx);
1555 .rva .LSEH_begin_sha1_multi_block_avx
1556 .rva .LSEH_end_sha1_multi_block_avx
1557 .rva .LSEH_info_sha1_multi_block_avx
1558 ___
1559 $code.=<<___ if ($avx>1);
1560 .rva .LSEH_begin_sha1_multi_block_avx2
1561 .rva .LSEH_end_sha1_multi_block_avx2
1562 .rva .LSEH_info_sha1_multi_block_avx2
1563 ___
1564 $code.=<<___;
1565 .section .xdata
1566 .align 8
1567 .LSEH_info_sha1_multi_block:
1568 .byte 9,0,0,0
1569 .rva se_handler
1570 .rva .Lbody,.Lepilogue # HandlerData[]
1571 .LSEH_info_sha1_multi_block_shaext:
1572 .byte 9,0,0,0
1573 .rva se_handler
1574 .rva .Lbody_shaext,.Lepilogue_shaext # HandlerData[]
1575 ___
1576 $code.=<<___ if ($avx);
1577 .LSEH_info_sha1_multi_block_avx:
1578 .byte 9,0,0,0
1579 .rva se_handler
1580 .rva .Lbody_avx,.Lepilogue_avx # HandlerData[]
1581 ___
1582 $code.=<<___ if ($avx>1);
1583 .LSEH_info_sha1_multi_block_avx2:
1584 .byte 9,0,0,0
1585 .rva avx2_handler
1586 .rva .Lbody_avx2,.Lepilogue_avx2 # HandlerData[]
1587 ___
1588 }
1589 ####################################################################
1590
1591 sub rex {
1592 local *opcode=shift;
1593 my ($dst,$src)=@_;
1594 my $rex=0;
1595
1596 $rex|=0x04 if ($dst>=8);
1597 $rex|=0x01 if ($src>=8);
1598 unshift @opcode,$rex|0x40 if ($rex);
1599 }
1600
1601 sub sha1rnds4 {
1602 if (@_[0] =~ /\$([x0-9a-f]+),\s*%xmm([0-9]+),\s*%xmm([0-9]+)/) {
1603 my @opcode=(0x0f,0x3a,0xcc);
1604 rex(\@opcode,$3,$2);
1605 push @opcode,0xc0|($2&7)|(($3&7)<<3); # ModR/M
1606 my $c=$1;
1607 push @opcode,$c=~/^0/?oct($c):$c;
1608 return ".byte\t".join(',',@opcode);
1609 } else {
1610 return "sha1rnds4\t".@_[0];
1611 }
1612 }
1613
1614 sub sha1op38 {
1615 my $instr = shift;
1616 my %opcodelet = (
1617 "sha1nexte" => 0xc8,
1618 "sha1msg1" => 0xc9,
1619 "sha1msg2" => 0xca );
1620
1621 if (defined($opcodelet{$instr}) && @_[0] =~ /%xmm([0-9]+),\s*%xmm([0-9]+)/) {
1622 my @opcode=(0x0f,0x38);
1623 rex(\@opcode,$2,$1);
1624 push @opcode,$opcodelet{$instr};
1625 push @opcode,0xc0|($1&7)|(($2&7)<<3); # ModR/M
1626 return ".byte\t".join(',',@opcode);
1627 } else {
1628 return $instr."\t".@_[0];
1629 }
1630 }
1631
1632 foreach (split("\n",$code)) {
1633 s/\`([^\`]*)\`/eval($1)/ge;
1634
1635 s/\b(sha1rnds4)\s+(.*)/sha1rnds4($2)/geo or
1636 s/\b(sha1[^\s]*)\s+(.*)/sha1op38($1,$2)/geo or
1637
1638 s/\b(vmov[dq])\b(.+)%ymm([0-9]+)/$1$2%xmm$3/go or
1639 s/\b(vmovdqu)\b(.+)%x%ymm([0-9]+)/$1$2%xmm$3/go or
1640 s/\b(vpinsr[qd])\b(.+)%ymm([0-9]+),%ymm([0-9]+)/$1$2%xmm$3,%xmm$4/go or
1641 s/\b(vpextr[qd])\b(.+)%ymm([0-9]+)/$1$2%xmm$3/go or
1642 s/\b(vinserti128)\b(\s+)%ymm/$1$2\$1,%xmm/go or
1643 s/\b(vpbroadcast[qd]\s+)%ymm([0-9]+)/$1%xmm$2/go;
1644
1645 print $_,"\n";
1646 }
1647
1648 close STDOUT or die "error closing STDOUT: $!";
A mirror of the official openssl repository