]>
Commit | Line | Data |
---|---|---|
6aa36e8e RS |
1 | #! /usr/bin/env perl |
2 | # Copyright 2013-2016 The OpenSSL Project Authors. All Rights Reserved. | |
3 | # | |
a598ed0d | 4 | # Licensed under the Apache License 2.0 (the "License"). You may not use |
6aa36e8e RS |
5 | # this file except in compliance with the License. You can obtain a copy |
6 | # in the file LICENSE in the source distribution or at | |
7 | # https://www.openssl.org/source/license.html | |
8 | ||
b7838586 AP |
9 | |
10 | # ==================================================================== | |
11 | # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL | |
12 | # project. The module is, however, dual licensed under OpenSSL and | |
13 | # CRYPTOGAMS licenses depending on where you obtain it. For further | |
14 | # details see http://www.openssl.org/~appro/cryptogams/. | |
15 | # ==================================================================== | |
16 | ||
17 | # Multi-buffer SHA256 procedure processes n buffers in parallel by | |
18 | # placing buffer data to designated lane of SIMD register. n is | |
19 | # naturally limited to 4 on pre-AVX2 processors and to 8 on | |
20 | # AVX2-capable processors such as Haswell. | |
21 | # | |
61ba602a | 22 | # this +aesni(i) sha256 aesni-sha256 gain(iv) |
b7838586 | 23 | # ------------------------------------------------------------------- |
61ba602a | 24 | # Westmere(ii) 23.3/n +1.28=7.11(n=4) 12.3 +3.75=16.1 +126% |
619b9466 | 25 | # Atom(ii) 38.7/n +3.93=13.6(n=4) 20.8 +5.69=26.5 +95% |
b7838586 AP |
26 | # Sandy Bridge (20.5 +5.15=25.7)/n 11.6 13.0 +103% |
27 | # Ivy Bridge (20.4 +5.14=25.5)/n 10.3 11.6 +82% | |
61ba602a | 28 | # Haswell(iii) (21.0 +5.00=26.0)/n 7.80 8.79 +170% |
b7f5503f | 29 | # Skylake (18.9 +5.00=23.9)/n 7.70 8.17 +170% |
b7838586 AP |
30 | # Bulldozer (21.6 +5.76=27.4)/n 13.6 13.7 +100% |
31 | # | |
61ba602a AP |
32 | # (i) multi-block CBC encrypt with 128-bit key; |
33 | # (ii) (HASH+AES)/n does not apply to Westmere for n>3 and Atom, | |
b7838586 AP |
34 | # because of lower AES-NI instruction throughput, nor is there |
35 | # AES-NI-SHA256 stitch for these processors; | |
61ba602a | 36 | # (iii) "this" is for n=8, when we gather twice as much data, result |
b7838586 | 37 | # for n=4 is 20.3+4.44=24.7; |
3847d15d | 38 | # (iv) presented improvement coefficients are asymptotic limits and |
609b0852 | 39 | # in real-life application are somewhat lower, e.g. for 2KB |
619b9466 | 40 | # fragments they range from 75% to 130% (on Haswell); |
b7838586 AP |
41 | |
42 | $flavour = shift; | |
43 | $output = shift; | |
44 | if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } | |
45 | ||
46 | $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); | |
47 | ||
48 | $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; | |
49 | ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or | |
50 | ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or | |
51 | die "can't locate x86_64-xlate.pl"; | |
52 | ||
53 | $avx=0; | |
54 | ||
55 | if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` | |
56 | =~ /GNU assembler version ([2-9]\.[0-9]+)/) { | |
57 | $avx = ($1>=2.19) + ($1>=2.22); | |
58 | } | |
59 | ||
60 | if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && | |
61 | `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) { | |
62 | $avx = ($1>=2.09) + ($1>=2.10); | |
63 | } | |
64 | ||
65 | if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && | |
66 | `ml64 2>&1` =~ /Version ([0-9]+)\./) { | |
67 | $avx = ($1>=10) + ($1>=11); | |
68 | } | |
69 | ||
b9749432 | 70 | if (!$avx && `$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|.*based on LLVM) ([3-9]\.[0-9]+)/) { |
a356e488 | 71 | $avx = ($2>=3.0) + ($2>3.0); |
ac171925 AP |
72 | } |
73 | ||
cfe1d992 | 74 | open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\""; |
b7838586 AP |
75 | *STDOUT=*OUT; |
76 | ||
77 | # void sha256_multi_block ( | |
78 | # struct { unsigned int A[8]; | |
79 | # unsigned int B[8]; | |
80 | # unsigned int C[8]; | |
81 | # unsigned int D[8]; | |
82 | # unsigned int E[8]; | |
83 | # unsigned int F[8]; | |
84 | # unsigned int G[8]; | |
85 | # unsigned int H[8]; } *ctx, | |
86 | # struct { void *ptr; int blocks; } inp[8], | |
87 | # int num); /* 1 or 2 */ | |
88 | # | |
89 | $ctx="%rdi"; # 1st arg | |
90 | $inp="%rsi"; # 2nd arg | |
91 | $num="%edx"; # 3rd arg | |
92 | @ptr=map("%r$_",(8..11)); | |
93 | $Tbl="%rbp"; | |
94 | ||
95 | @V=($A,$B,$C,$D,$E,$F,$G,$H)=map("%xmm$_",(8..15)); | |
96 | ($t1,$t2,$t3,$axb,$bxc,$Xi,$Xn,$sigma)=map("%xmm$_",(0..7)); | |
97 | ||
98 | $REG_SZ=16; | |
99 | ||
100 | sub Xi_off { | |
101 | my $off = shift; | |
102 | ||
103 | $off %= 16; $off *= $REG_SZ; | |
104 | $off<256 ? "$off-128(%rax)" : "$off-256-128(%rbx)"; | |
105 | } | |
106 | ||
107 | sub ROUND_00_15 { | |
108 | my ($i,$a,$b,$c,$d,$e,$f,$g,$h)=@_; | |
109 | ||
110 | $code.=<<___ if ($i<15); | |
111 | movd `4*$i`(@ptr[0]),$Xi | |
112 | movd `4*$i`(@ptr[1]),$t1 | |
113 | movd `4*$i`(@ptr[2]),$t2 | |
114 | movd `4*$i`(@ptr[3]),$t3 | |
115 | punpckldq $t2,$Xi | |
116 | punpckldq $t3,$t1 | |
117 | punpckldq $t1,$Xi | |
b7838586 AP |
118 | ___ |
119 | $code.=<<___ if ($i==15); | |
120 | movd `4*$i`(@ptr[0]),$Xi | |
121 | lea `16*4`(@ptr[0]),@ptr[0] | |
122 | movd `4*$i`(@ptr[1]),$t1 | |
123 | lea `16*4`(@ptr[1]),@ptr[1] | |
124 | movd `4*$i`(@ptr[2]),$t2 | |
125 | lea `16*4`(@ptr[2]),@ptr[2] | |
126 | movd `4*$i`(@ptr[3]),$t3 | |
127 | lea `16*4`(@ptr[3]),@ptr[3] | |
128 | punpckldq $t2,$Xi | |
129 | punpckldq $t3,$t1 | |
130 | punpckldq $t1,$Xi | |
b7838586 AP |
131 | ___ |
132 | $code.=<<___; | |
133 | movdqa $e,$sigma | |
619b9466 | 134 | `"pshufb $Xn,$Xi" if ($i<=15 && ($i&1)==0)` |
b7838586 | 135 | movdqa $e,$t3 |
619b9466 | 136 | `"pshufb $Xn,$Xi" if ($i<=15 && ($i&1)==1)` |
b7838586 AP |
137 | psrld \$6,$sigma |
138 | movdqa $e,$t2 | |
139 | pslld \$7,$t3 | |
140 | movdqa $Xi,`&Xi_off($i)` | |
141 | paddd $h,$Xi # Xi+=h | |
142 | ||
143 | psrld \$11,$t2 | |
144 | pxor $t3,$sigma | |
145 | pslld \$21-7,$t3 | |
146 | paddd `32*($i%8)-128`($Tbl),$Xi # Xi+=K[round] | |
147 | pxor $t2,$sigma | |
148 | ||
149 | psrld \$25-11,$t2 | |
150 | movdqa $e,$t1 | |
619b9466 | 151 | `"prefetcht0 63(@ptr[0])" if ($i==15)` |
b7838586 AP |
152 | pxor $t3,$sigma |
153 | movdqa $e,$axb # borrow $axb | |
154 | pslld \$26-21,$t3 | |
155 | pandn $g,$t1 | |
156 | pand $f,$axb | |
157 | pxor $t2,$sigma | |
158 | ||
619b9466 | 159 | `"prefetcht0 63(@ptr[1])" if ($i==15)` |
b7838586 AP |
160 | movdqa $a,$t2 |
161 | pxor $t3,$sigma # Sigma1(e) | |
162 | movdqa $a,$t3 | |
163 | psrld \$2,$t2 | |
164 | paddd $sigma,$Xi # Xi+=Sigma1(e) | |
165 | pxor $axb,$t1 # Ch(e,f,g) | |
166 | movdqa $b,$axb | |
167 | movdqa $a,$sigma | |
168 | pslld \$10,$t3 | |
169 | pxor $a,$axb # a^b, b^c in next round | |
170 | ||
619b9466 | 171 | `"prefetcht0 63(@ptr[2])" if ($i==15)` |
b7838586 AP |
172 | psrld \$13,$sigma |
173 | pxor $t3,$t2 | |
174 | paddd $t1,$Xi # Xi+=Ch(e,f,g) | |
175 | pslld \$19-10,$t3 | |
176 | pand $axb,$bxc | |
177 | pxor $sigma,$t2 | |
178 | ||
619b9466 | 179 | `"prefetcht0 63(@ptr[3])" if ($i==15)` |
b7838586 AP |
180 | psrld \$22-13,$sigma |
181 | pxor $t3,$t2 | |
182 | movdqa $b,$h | |
183 | pslld \$30-19,$t3 | |
184 | pxor $t2,$sigma | |
185 | pxor $bxc,$h # h=Maj(a,b,c)=Ch(a^b,c,b) | |
186 | paddd $Xi,$d # d+=Xi | |
187 | pxor $t3,$sigma # Sigma0(a) | |
188 | ||
189 | paddd $Xi,$h # h+=Xi | |
190 | paddd $sigma,$h # h+=Sigma0(a) | |
191 | ___ | |
192 | $code.=<<___ if (($i%8)==7); | |
193 | lea `32*8`($Tbl),$Tbl | |
194 | ___ | |
195 | ($axb,$bxc)=($bxc,$axb); | |
196 | } | |
197 | ||
198 | sub ROUND_16_XX { | |
199 | my $i=shift; | |
200 | ||
201 | $code.=<<___; | |
202 | movdqa `&Xi_off($i+1)`,$Xn | |
203 | paddd `&Xi_off($i+9)`,$Xi # Xi+=X[i+9] | |
204 | ||
205 | movdqa $Xn,$sigma | |
206 | movdqa $Xn,$t2 | |
207 | psrld \$3,$sigma | |
208 | movdqa $Xn,$t3 | |
209 | ||
210 | psrld \$7,$t2 | |
211 | movdqa `&Xi_off($i+14)`,$t1 | |
212 | pslld \$14,$t3 | |
213 | pxor $t2,$sigma | |
214 | psrld \$18-7,$t2 | |
215 | movdqa $t1,$axb # borrow $axb | |
216 | pxor $t3,$sigma | |
217 | pslld \$25-14,$t3 | |
218 | pxor $t2,$sigma | |
219 | psrld \$10,$t1 | |
220 | movdqa $axb,$t2 | |
221 | ||
222 | psrld \$17,$axb | |
223 | pxor $t3,$sigma # sigma0(X[i+1]) | |
224 | pslld \$13,$t2 | |
225 | paddd $sigma,$Xi # Xi+=sigma0(e) | |
226 | pxor $axb,$t1 | |
227 | psrld \$19-17,$axb | |
228 | pxor $t2,$t1 | |
229 | pslld \$15-13,$t2 | |
230 | pxor $axb,$t1 | |
231 | pxor $t2,$t1 # sigma0(X[i+14]) | |
232 | paddd $t1,$Xi # Xi+=sigma1(X[i+14]) | |
233 | ___ | |
234 | &ROUND_00_15($i,@_); | |
235 | ($Xi,$Xn)=($Xn,$Xi); | |
236 | } | |
237 | ||
238 | $code.=<<___; | |
239 | .text | |
240 | ||
241 | .extern OPENSSL_ia32cap_P | |
242 | ||
243 | .globl sha256_multi_block | |
244 | .type sha256_multi_block,\@function,3 | |
245 | .align 32 | |
246 | sha256_multi_block: | |
399976c7 | 247 | .cfi_startproc |
619b9466 AP |
248 | mov OPENSSL_ia32cap_P+4(%rip),%rcx |
249 | bt \$61,%rcx # check SHA bit | |
250 | jc _shaext_shortcut | |
b7838586 AP |
251 | ___ |
252 | $code.=<<___ if ($avx); | |
b7838586 AP |
253 | test \$`1<<28`,%ecx |
254 | jnz _avx_shortcut | |
255 | ___ | |
256 | $code.=<<___; | |
257 | mov %rsp,%rax | |
399976c7 | 258 | .cfi_def_cfa_register %rax |
b7838586 | 259 | push %rbx |
399976c7 | 260 | .cfi_push %rbx |
b7838586 | 261 | push %rbp |
399976c7 | 262 | .cfi_push %rbp |
b7838586 AP |
263 | ___ |
264 | $code.=<<___ if ($win64); | |
265 | lea -0xa8(%rsp),%rsp | |
266 | movaps %xmm6,(%rsp) | |
267 | movaps %xmm7,0x10(%rsp) | |
268 | movaps %xmm8,0x20(%rsp) | |
269 | movaps %xmm9,0x30(%rsp) | |
270 | movaps %xmm10,-0x78(%rax) | |
271 | movaps %xmm11,-0x68(%rax) | |
272 | movaps %xmm12,-0x58(%rax) | |
273 | movaps %xmm13,-0x48(%rax) | |
274 | movaps %xmm14,-0x38(%rax) | |
275 | movaps %xmm15,-0x28(%rax) | |
276 | ___ | |
277 | $code.=<<___; | |
278 | sub \$`$REG_SZ*18`, %rsp | |
279 | and \$-256,%rsp | |
280 | mov %rax,`$REG_SZ*17`(%rsp) # original %rsp | |
399976c7 | 281 | .cfi_cfa_expression %rsp+`$REG_SZ*17`,deref,+8 |
619b9466 | 282 | .Lbody: |
b7838586 AP |
283 | lea K256+128(%rip),$Tbl |
284 | lea `$REG_SZ*16`(%rsp),%rbx | |
285 | lea 0x80($ctx),$ctx # size optimization | |
286 | ||
287 | .Loop_grande: | |
288 | mov $num,`$REG_SZ*17+8`(%rsp) # original $num | |
289 | xor $num,$num | |
290 | ___ | |
291 | for($i=0;$i<4;$i++) { | |
292 | $code.=<<___; | |
293 | mov `16*$i+0`($inp),@ptr[$i] # input pointer | |
294 | mov `16*$i+8`($inp),%ecx # number of blocks | |
295 | cmp $num,%ecx | |
296 | cmovg %ecx,$num # find maximum | |
297 | test %ecx,%ecx | |
298 | mov %ecx,`4*$i`(%rbx) # initialize counters | |
299 | cmovle $Tbl,@ptr[$i] # cancel input | |
300 | ___ | |
301 | } | |
302 | $code.=<<___; | |
303 | test $num,$num | |
304 | jz .Ldone | |
305 | ||
306 | movdqu 0x00-0x80($ctx),$A # load context | |
307 | lea 128(%rsp),%rax | |
308 | movdqu 0x20-0x80($ctx),$B | |
309 | movdqu 0x40-0x80($ctx),$C | |
310 | movdqu 0x60-0x80($ctx),$D | |
311 | movdqu 0x80-0x80($ctx),$E | |
312 | movdqu 0xa0-0x80($ctx),$F | |
313 | movdqu 0xc0-0x80($ctx),$G | |
314 | movdqu 0xe0-0x80($ctx),$H | |
315 | movdqu .Lpbswap(%rip),$Xn | |
316 | jmp .Loop | |
317 | ||
318 | .align 32 | |
319 | .Loop: | |
320 | movdqa $C,$bxc | |
321 | pxor $B,$bxc # magic seed | |
322 | ___ | |
323 | for($i=0;$i<16;$i++) { &ROUND_00_15($i,@V); unshift(@V,pop(@V)); } | |
324 | $code.=<<___; | |
325 | movdqu `&Xi_off($i)`,$Xi | |
326 | mov \$3,%ecx | |
327 | jmp .Loop_16_xx | |
328 | .align 32 | |
329 | .Loop_16_xx: | |
330 | ___ | |
331 | for(;$i<32;$i++) { &ROUND_16_XX($i,@V); unshift(@V,pop(@V)); } | |
332 | $code.=<<___; | |
333 | dec %ecx | |
334 | jnz .Loop_16_xx | |
335 | ||
336 | mov \$1,%ecx | |
337 | lea K256+128(%rip),$Tbl | |
338 | ||
339 | movdqa (%rbx),$sigma # pull counters | |
340 | cmp 4*0(%rbx),%ecx # examine counters | |
341 | pxor $t1,$t1 | |
342 | cmovge $Tbl,@ptr[0] # cancel input | |
343 | cmp 4*1(%rbx),%ecx | |
344 | movdqa $sigma,$Xn | |
345 | cmovge $Tbl,@ptr[1] | |
346 | cmp 4*2(%rbx),%ecx | |
347 | pcmpgtd $t1,$Xn # mask value | |
348 | cmovge $Tbl,@ptr[2] | |
349 | cmp 4*3(%rbx),%ecx | |
350 | paddd $Xn,$sigma # counters-- | |
351 | cmovge $Tbl,@ptr[3] | |
352 | ||
353 | movdqu 0x00-0x80($ctx),$t1 | |
354 | pand $Xn,$A | |
355 | movdqu 0x20-0x80($ctx),$t2 | |
356 | pand $Xn,$B | |
357 | movdqu 0x40-0x80($ctx),$t3 | |
358 | pand $Xn,$C | |
359 | movdqu 0x60-0x80($ctx),$Xi | |
360 | pand $Xn,$D | |
361 | paddd $t1,$A | |
362 | movdqu 0x80-0x80($ctx),$t1 | |
363 | pand $Xn,$E | |
364 | paddd $t2,$B | |
365 | movdqu 0xa0-0x80($ctx),$t2 | |
366 | pand $Xn,$F | |
367 | paddd $t3,$C | |
368 | movdqu 0xc0-0x80($ctx),$t3 | |
369 | pand $Xn,$G | |
370 | paddd $Xi,$D | |
371 | movdqu 0xe0-0x80($ctx),$Xi | |
372 | pand $Xn,$H | |
373 | paddd $t1,$E | |
374 | paddd $t2,$F | |
375 | movdqu $A,0x00-0x80($ctx) | |
376 | paddd $t3,$G | |
377 | movdqu $B,0x20-0x80($ctx) | |
378 | paddd $Xi,$H | |
379 | movdqu $C,0x40-0x80($ctx) | |
380 | movdqu $D,0x60-0x80($ctx) | |
381 | movdqu $E,0x80-0x80($ctx) | |
382 | movdqu $F,0xa0-0x80($ctx) | |
383 | movdqu $G,0xc0-0x80($ctx) | |
384 | movdqu $H,0xe0-0x80($ctx) | |
385 | ||
386 | movdqa $sigma,(%rbx) # save counters | |
387 | movdqa .Lpbswap(%rip),$Xn | |
388 | dec $num | |
389 | jnz .Loop | |
390 | ||
391 | mov `$REG_SZ*17+8`(%rsp),$num | |
392 | lea $REG_SZ($ctx),$ctx | |
393 | lea `16*$REG_SZ/4`($inp),$inp | |
394 | dec $num | |
395 | jnz .Loop_grande | |
396 | ||
397 | .Ldone: | |
0d4fb843 | 398 | mov `$REG_SZ*17`(%rsp),%rax # original %rsp |
399976c7 | 399 | .cfi_def_cfa %rax,8 |
b7838586 AP |
400 | ___ |
401 | $code.=<<___ if ($win64); | |
402 | movaps -0xb8(%rax),%xmm6 | |
403 | movaps -0xa8(%rax),%xmm7 | |
404 | movaps -0x98(%rax),%xmm8 | |
405 | movaps -0x88(%rax),%xmm9 | |
406 | movaps -0x78(%rax),%xmm10 | |
407 | movaps -0x68(%rax),%xmm11 | |
408 | movaps -0x58(%rax),%xmm12 | |
409 | movaps -0x48(%rax),%xmm13 | |
410 | movaps -0x38(%rax),%xmm14 | |
411 | movaps -0x28(%rax),%xmm15 | |
412 | ___ | |
413 | $code.=<<___; | |
414 | mov -16(%rax),%rbp | |
399976c7 | 415 | .cfi_restore %rbp |
b7838586 | 416 | mov -8(%rax),%rbx |
399976c7 | 417 | .cfi_restore %rbx |
b7838586 | 418 | lea (%rax),%rsp |
399976c7 | 419 | .cfi_def_cfa_register %rsp |
619b9466 | 420 | .Lepilogue: |
b7838586 | 421 | ret |
399976c7 | 422 | .cfi_endproc |
b7838586 AP |
423 | .size sha256_multi_block,.-sha256_multi_block |
424 | ___ | |
619b9466 AP |
425 | {{{ |
426 | my ($Wi,$TMP0,$TMP1,$TMPx,$ABEF0,$CDGH0,$ABEF1,$CDGH1)=map("%xmm$_",(0..3,12..15)); | |
427 | my @MSG0=map("%xmm$_",(4..7)); | |
428 | my @MSG1=map("%xmm$_",(8..11)); | |
429 | ||
430 | $code.=<<___; | |
431 | .type sha256_multi_block_shaext,\@function,3 | |
432 | .align 32 | |
433 | sha256_multi_block_shaext: | |
399976c7 | 434 | .cfi_startproc |
619b9466 AP |
435 | _shaext_shortcut: |
436 | mov %rsp,%rax | |
399976c7 | 437 | .cfi_def_cfa_register %rax |
619b9466 | 438 | push %rbx |
399976c7 | 439 | .cfi_push %rbx |
619b9466 | 440 | push %rbp |
399976c7 | 441 | .cfi_push %rbp |
619b9466 AP |
442 | ___ |
443 | $code.=<<___ if ($win64); | |
444 | lea -0xa8(%rsp),%rsp | |
445 | movaps %xmm6,(%rsp) | |
446 | movaps %xmm7,0x10(%rsp) | |
447 | movaps %xmm8,0x20(%rsp) | |
448 | movaps %xmm9,0x30(%rsp) | |
449 | movaps %xmm10,-0x78(%rax) | |
450 | movaps %xmm11,-0x68(%rax) | |
451 | movaps %xmm12,-0x58(%rax) | |
452 | movaps %xmm13,-0x48(%rax) | |
453 | movaps %xmm14,-0x38(%rax) | |
454 | movaps %xmm15,-0x28(%rax) | |
455 | ___ | |
456 | $code.=<<___; | |
457 | sub \$`$REG_SZ*18`,%rsp | |
458 | shl \$1,$num # we process pair at a time | |
459 | and \$-256,%rsp | |
460 | lea 0x80($ctx),$ctx # size optimization | |
461 | mov %rax,`$REG_SZ*17`(%rsp) # original %rsp | |
462 | .Lbody_shaext: | |
463 | lea `$REG_SZ*16`(%rsp),%rbx | |
464 | lea K256_shaext+0x80(%rip),$Tbl | |
465 | ||
466 | .Loop_grande_shaext: | |
0d4fb843 | 467 | mov $num,`$REG_SZ*17+8`(%rsp) # original $num |
619b9466 AP |
468 | xor $num,$num |
469 | ___ | |
470 | for($i=0;$i<2;$i++) { | |
471 | $code.=<<___; | |
472 | mov `16*$i+0`($inp),@ptr[$i] # input pointer | |
473 | mov `16*$i+8`($inp),%ecx # number of blocks | |
474 | cmp $num,%ecx | |
475 | cmovg %ecx,$num # find maximum | |
476 | test %ecx,%ecx | |
477 | mov %ecx,`4*$i`(%rbx) # initialize counters | |
478 | cmovle %rsp,@ptr[$i] # cancel input | |
479 | ___ | |
480 | } | |
481 | $code.=<<___; | |
482 | test $num,$num | |
483 | jz .Ldone_shaext | |
484 | ||
485 | movq 0x00-0x80($ctx),$ABEF0 # A1.A0 | |
486 | movq 0x20-0x80($ctx),@MSG0[0] # B1.B0 | |
487 | movq 0x40-0x80($ctx),$CDGH0 # C1.C0 | |
488 | movq 0x60-0x80($ctx),@MSG0[1] # D1.D0 | |
489 | movq 0x80-0x80($ctx),@MSG1[0] # E1.E0 | |
490 | movq 0xa0-0x80($ctx),@MSG1[1] # F1.F0 | |
491 | movq 0xc0-0x80($ctx),@MSG1[2] # G1.G0 | |
492 | movq 0xe0-0x80($ctx),@MSG1[3] # H1.H0 | |
493 | ||
494 | punpckldq @MSG0[0],$ABEF0 # B1.A1.B0.A0 | |
495 | punpckldq @MSG0[1],$CDGH0 # D1.C1.D0.C0 | |
496 | punpckldq @MSG1[1],@MSG1[0] # F1.E1.F0.E0 | |
497 | punpckldq @MSG1[3],@MSG1[2] # H1.G1.H0.G0 | |
498 | movdqa K256_shaext-0x10(%rip),$TMPx # byte swap | |
499 | ||
500 | movdqa $ABEF0,$ABEF1 | |
501 | movdqa $CDGH0,$CDGH1 | |
502 | punpcklqdq @MSG1[0],$ABEF0 # F0.E0.B0.A0 | |
503 | punpcklqdq @MSG1[2],$CDGH0 # H0.G0.D0.C0 | |
504 | punpckhqdq @MSG1[0],$ABEF1 # F1.E1.B1.A1 | |
505 | punpckhqdq @MSG1[2],$CDGH1 # H1.G1.D1.C1 | |
506 | ||
507 | pshufd \$0b00011011,$ABEF0,$ABEF0 | |
508 | pshufd \$0b00011011,$CDGH0,$CDGH0 | |
509 | pshufd \$0b00011011,$ABEF1,$ABEF1 | |
510 | pshufd \$0b00011011,$CDGH1,$CDGH1 | |
511 | jmp .Loop_shaext | |
512 | ||
513 | .align 32 | |
514 | .Loop_shaext: | |
515 | movdqu 0x00(@ptr[0]),@MSG0[0] | |
516 | movdqu 0x00(@ptr[1]),@MSG1[0] | |
517 | movdqu 0x10(@ptr[0]),@MSG0[1] | |
518 | movdqu 0x10(@ptr[1]),@MSG1[1] | |
519 | movdqu 0x20(@ptr[0]),@MSG0[2] | |
520 | pshufb $TMPx,@MSG0[0] | |
521 | movdqu 0x20(@ptr[1]),@MSG1[2] | |
522 | pshufb $TMPx,@MSG1[0] | |
523 | movdqu 0x30(@ptr[0]),@MSG0[3] | |
524 | lea 0x40(@ptr[0]),@ptr[0] | |
525 | movdqu 0x30(@ptr[1]),@MSG1[3] | |
526 | lea 0x40(@ptr[1]),@ptr[1] | |
527 | ||
528 | movdqa 0*16-0x80($Tbl),$Wi | |
529 | pshufb $TMPx,@MSG0[1] | |
530 | paddd @MSG0[0],$Wi | |
531 | pxor $ABEF0,@MSG0[0] # black magic | |
532 | movdqa $Wi,$TMP0 | |
533 | movdqa 0*16-0x80($Tbl),$TMP1 | |
534 | pshufb $TMPx,@MSG1[1] | |
535 | paddd @MSG1[0],$TMP1 | |
536 | movdqa $CDGH0,0x50(%rsp) # offload | |
537 | sha256rnds2 $ABEF0,$CDGH0 # 0-3 | |
538 | pxor $ABEF1,@MSG1[0] # black magic | |
539 | movdqa $TMP1,$Wi | |
540 | movdqa $CDGH1,0x70(%rsp) | |
541 | sha256rnds2 $ABEF1,$CDGH1 # 0-3 | |
542 | pshufd \$0x0e,$TMP0,$Wi | |
543 | pxor $ABEF0,@MSG0[0] # black magic | |
544 | movdqa $ABEF0,0x40(%rsp) # offload | |
545 | sha256rnds2 $CDGH0,$ABEF0 | |
546 | pshufd \$0x0e,$TMP1,$Wi | |
547 | pxor $ABEF1,@MSG1[0] # black magic | |
548 | movdqa $ABEF1,0x60(%rsp) | |
549 | movdqa 1*16-0x80($Tbl),$TMP0 | |
550 | paddd @MSG0[1],$TMP0 | |
551 | pshufb $TMPx,@MSG0[2] | |
552 | sha256rnds2 $CDGH1,$ABEF1 | |
553 | ||
554 | movdqa $TMP0,$Wi | |
555 | movdqa 1*16-0x80($Tbl),$TMP1 | |
556 | paddd @MSG1[1],$TMP1 | |
557 | sha256rnds2 $ABEF0,$CDGH0 # 4-7 | |
558 | movdqa $TMP1,$Wi | |
559 | prefetcht0 127(@ptr[0]) | |
560 | pshufb $TMPx,@MSG0[3] | |
561 | pshufb $TMPx,@MSG1[2] | |
562 | prefetcht0 127(@ptr[1]) | |
563 | sha256rnds2 $ABEF1,$CDGH1 # 4-7 | |
564 | pshufd \$0x0e,$TMP0,$Wi | |
565 | pshufb $TMPx,@MSG1[3] | |
566 | sha256msg1 @MSG0[1],@MSG0[0] | |
567 | sha256rnds2 $CDGH0,$ABEF0 | |
568 | pshufd \$0x0e,$TMP1,$Wi | |
569 | movdqa 2*16-0x80($Tbl),$TMP0 | |
570 | paddd @MSG0[2],$TMP0 | |
571 | sha256rnds2 $CDGH1,$ABEF1 | |
572 | ||
573 | movdqa $TMP0,$Wi | |
574 | movdqa 2*16-0x80($Tbl),$TMP1 | |
575 | paddd @MSG1[2],$TMP1 | |
576 | sha256rnds2 $ABEF0,$CDGH0 # 8-11 | |
577 | sha256msg1 @MSG1[1],@MSG1[0] | |
578 | movdqa $TMP1,$Wi | |
579 | movdqa @MSG0[3],$TMPx | |
580 | sha256rnds2 $ABEF1,$CDGH1 # 8-11 | |
581 | pshufd \$0x0e,$TMP0,$Wi | |
582 | palignr \$4,@MSG0[2],$TMPx | |
583 | paddd $TMPx,@MSG0[0] | |
584 | movdqa @MSG1[3],$TMPx | |
585 | palignr \$4,@MSG1[2],$TMPx | |
586 | sha256msg1 @MSG0[2],@MSG0[1] | |
587 | sha256rnds2 $CDGH0,$ABEF0 | |
588 | pshufd \$0x0e,$TMP1,$Wi | |
589 | movdqa 3*16-0x80($Tbl),$TMP0 | |
590 | paddd @MSG0[3],$TMP0 | |
591 | sha256rnds2 $CDGH1,$ABEF1 | |
592 | sha256msg1 @MSG1[2],@MSG1[1] | |
593 | ||
594 | movdqa $TMP0,$Wi | |
595 | movdqa 3*16-0x80($Tbl),$TMP1 | |
596 | paddd $TMPx,@MSG1[0] | |
597 | paddd @MSG1[3],$TMP1 | |
598 | sha256msg2 @MSG0[3],@MSG0[0] | |
599 | sha256rnds2 $ABEF0,$CDGH0 # 12-15 | |
600 | movdqa $TMP1,$Wi | |
601 | movdqa @MSG0[0],$TMPx | |
602 | palignr \$4,@MSG0[3],$TMPx | |
603 | sha256rnds2 $ABEF1,$CDGH1 # 12-15 | |
604 | sha256msg2 @MSG1[3],@MSG1[0] | |
605 | pshufd \$0x0e,$TMP0,$Wi | |
606 | paddd $TMPx,@MSG0[1] | |
607 | movdqa @MSG1[0],$TMPx | |
608 | palignr \$4,@MSG1[3],$TMPx | |
609 | sha256msg1 @MSG0[3],@MSG0[2] | |
610 | sha256rnds2 $CDGH0,$ABEF0 | |
611 | pshufd \$0x0e,$TMP1,$Wi | |
612 | movdqa 4*16-0x80($Tbl),$TMP0 | |
613 | paddd @MSG0[0],$TMP0 | |
614 | sha256rnds2 $CDGH1,$ABEF1 | |
615 | sha256msg1 @MSG1[3],@MSG1[2] | |
616 | ___ | |
617 | for($i=4;$i<16-3;$i++) { | |
618 | $code.=<<___; | |
619 | movdqa $TMP0,$Wi | |
620 | movdqa $i*16-0x80($Tbl),$TMP1 | |
621 | paddd $TMPx,@MSG1[1] | |
622 | paddd @MSG1[0],$TMP1 | |
623 | sha256msg2 @MSG0[0],@MSG0[1] | |
624 | sha256rnds2 $ABEF0,$CDGH0 # 16-19... | |
625 | movdqa $TMP1,$Wi | |
626 | movdqa @MSG0[1],$TMPx | |
627 | palignr \$4,@MSG0[0],$TMPx | |
628 | sha256rnds2 $ABEF1,$CDGH1 # 16-19... | |
629 | sha256msg2 @MSG1[0],@MSG1[1] | |
630 | pshufd \$0x0e,$TMP0,$Wi | |
631 | paddd $TMPx,@MSG0[2] | |
632 | movdqa @MSG1[1],$TMPx | |
633 | palignr \$4,@MSG1[0],$TMPx | |
634 | sha256msg1 @MSG0[0],@MSG0[3] | |
635 | sha256rnds2 $CDGH0,$ABEF0 | |
636 | pshufd \$0x0e,$TMP1,$Wi | |
637 | movdqa `($i+1)*16`-0x80($Tbl),$TMP0 | |
638 | paddd @MSG0[1],$TMP0 | |
639 | sha256rnds2 $CDGH1,$ABEF1 | |
640 | sha256msg1 @MSG1[0],@MSG1[3] | |
641 | ___ | |
642 | push(@MSG0,shift(@MSG0)); push(@MSG1,shift(@MSG1)); | |
643 | } | |
644 | $code.=<<___; | |
645 | movdqa $TMP0,$Wi | |
646 | movdqa 13*16-0x80($Tbl),$TMP1 | |
647 | paddd $TMPx,@MSG1[1] | |
648 | paddd @MSG1[0],$TMP1 | |
649 | sha256msg2 @MSG0[0],@MSG0[1] | |
650 | sha256rnds2 $ABEF0,$CDGH0 # 52-55 | |
651 | movdqa $TMP1,$Wi | |
652 | movdqa @MSG0[1],$TMPx | |
653 | palignr \$4,@MSG0[0],$TMPx | |
654 | sha256rnds2 $ABEF1,$CDGH1 # 52-55 | |
655 | sha256msg2 @MSG1[0],@MSG1[1] | |
656 | pshufd \$0x0e,$TMP0,$Wi | |
657 | paddd $TMPx,@MSG0[2] | |
658 | movdqa @MSG1[1],$TMPx | |
659 | palignr \$4,@MSG1[0],$TMPx | |
660 | nop | |
661 | sha256rnds2 $CDGH0,$ABEF0 | |
662 | pshufd \$0x0e,$TMP1,$Wi | |
663 | movdqa 14*16-0x80($Tbl),$TMP0 | |
664 | paddd @MSG0[1],$TMP0 | |
665 | sha256rnds2 $CDGH1,$ABEF1 | |
666 | ||
667 | movdqa $TMP0,$Wi | |
668 | movdqa 14*16-0x80($Tbl),$TMP1 | |
669 | paddd $TMPx,@MSG1[2] | |
670 | paddd @MSG1[1],$TMP1 | |
671 | sha256msg2 @MSG0[1],@MSG0[2] | |
672 | nop | |
673 | sha256rnds2 $ABEF0,$CDGH0 # 56-59 | |
674 | movdqa $TMP1,$Wi | |
675 | mov \$1,%ecx | |
676 | pxor @MSG0[1],@MSG0[1] # zero | |
677 | sha256rnds2 $ABEF1,$CDGH1 # 56-59 | |
678 | sha256msg2 @MSG1[1],@MSG1[2] | |
679 | pshufd \$0x0e,$TMP0,$Wi | |
680 | movdqa 15*16-0x80($Tbl),$TMP0 | |
681 | paddd @MSG0[2],$TMP0 | |
682 | movq (%rbx),@MSG0[2] # pull counters | |
683 | nop | |
684 | sha256rnds2 $CDGH0,$ABEF0 | |
685 | pshufd \$0x0e,$TMP1,$Wi | |
686 | movdqa 15*16-0x80($Tbl),$TMP1 | |
687 | paddd @MSG1[2],$TMP1 | |
688 | sha256rnds2 $CDGH1,$ABEF1 | |
689 | ||
690 | movdqa $TMP0,$Wi | |
691 | cmp 4*0(%rbx),%ecx # examine counters | |
692 | cmovge %rsp,@ptr[0] # cancel input | |
693 | cmp 4*1(%rbx),%ecx | |
694 | cmovge %rsp,@ptr[1] | |
695 | pshufd \$0x00,@MSG0[2],@MSG1[0] | |
696 | sha256rnds2 $ABEF0,$CDGH0 # 60-63 | |
697 | movdqa $TMP1,$Wi | |
698 | pshufd \$0x55,@MSG0[2],@MSG1[1] | |
699 | movdqa @MSG0[2],@MSG1[2] | |
700 | sha256rnds2 $ABEF1,$CDGH1 # 60-63 | |
701 | pshufd \$0x0e,$TMP0,$Wi | |
702 | pcmpgtd @MSG0[1],@MSG1[0] | |
703 | pcmpgtd @MSG0[1],@MSG1[1] | |
704 | sha256rnds2 $CDGH0,$ABEF0 | |
705 | pshufd \$0x0e,$TMP1,$Wi | |
706 | pcmpgtd @MSG0[1],@MSG1[2] # counter mask | |
707 | movdqa K256_shaext-0x10(%rip),$TMPx | |
708 | sha256rnds2 $CDGH1,$ABEF1 | |
709 | ||
710 | pand @MSG1[0],$CDGH0 | |
711 | pand @MSG1[1],$CDGH1 | |
712 | pand @MSG1[0],$ABEF0 | |
713 | pand @MSG1[1],$ABEF1 | |
714 | paddd @MSG0[2],@MSG1[2] # counters-- | |
715 | ||
716 | paddd 0x50(%rsp),$CDGH0 | |
717 | paddd 0x70(%rsp),$CDGH1 | |
718 | paddd 0x40(%rsp),$ABEF0 | |
719 | paddd 0x60(%rsp),$ABEF1 | |
720 | ||
721 | movq @MSG1[2],(%rbx) # save counters | |
722 | dec $num | |
723 | jnz .Loop_shaext | |
724 | ||
725 | mov `$REG_SZ*17+8`(%rsp),$num | |
726 | ||
727 | pshufd \$0b00011011,$ABEF0,$ABEF0 | |
728 | pshufd \$0b00011011,$CDGH0,$CDGH0 | |
729 | pshufd \$0b00011011,$ABEF1,$ABEF1 | |
730 | pshufd \$0b00011011,$CDGH1,$CDGH1 | |
731 | ||
732 | movdqa $ABEF0,@MSG0[0] | |
733 | movdqa $CDGH0,@MSG0[1] | |
734 | punpckldq $ABEF1,$ABEF0 # B1.B0.A1.A0 | |
735 | punpckhdq $ABEF1,@MSG0[0] # F1.F0.E1.E0 | |
736 | punpckldq $CDGH1,$CDGH0 # D1.D0.C1.C0 | |
737 | punpckhdq $CDGH1,@MSG0[1] # H1.H0.G1.G0 | |
738 | ||
739 | movq $ABEF0,0x00-0x80($ctx) # A1.A0 | |
740 | psrldq \$8,$ABEF0 | |
741 | movq @MSG0[0],0x80-0x80($ctx) # E1.E0 | |
742 | psrldq \$8,@MSG0[0] | |
743 | movq $ABEF0,0x20-0x80($ctx) # B1.B0 | |
744 | movq @MSG0[0],0xa0-0x80($ctx) # F1.F0 | |
745 | ||
746 | movq $CDGH0,0x40-0x80($ctx) # C1.C0 | |
747 | psrldq \$8,$CDGH0 | |
748 | movq @MSG0[1],0xc0-0x80($ctx) # G1.G0 | |
749 | psrldq \$8,@MSG0[1] | |
750 | movq $CDGH0,0x60-0x80($ctx) # D1.D0 | |
751 | movq @MSG0[1],0xe0-0x80($ctx) # H1.H0 | |
752 | ||
753 | lea `$REG_SZ/2`($ctx),$ctx | |
754 | lea `16*2`($inp),$inp | |
755 | dec $num | |
756 | jnz .Loop_grande_shaext | |
757 | ||
758 | .Ldone_shaext: | |
759 | #mov `$REG_SZ*17`(%rsp),%rax # original %rsp | |
760 | ___ | |
761 | $code.=<<___ if ($win64); | |
762 | movaps -0xb8(%rax),%xmm6 | |
763 | movaps -0xa8(%rax),%xmm7 | |
764 | movaps -0x98(%rax),%xmm8 | |
765 | movaps -0x88(%rax),%xmm9 | |
766 | movaps -0x78(%rax),%xmm10 | |
767 | movaps -0x68(%rax),%xmm11 | |
768 | movaps -0x58(%rax),%xmm12 | |
769 | movaps -0x48(%rax),%xmm13 | |
770 | movaps -0x38(%rax),%xmm14 | |
771 | movaps -0x28(%rax),%xmm15 | |
772 | ___ | |
773 | $code.=<<___; | |
774 | mov -16(%rax),%rbp | |
399976c7 | 775 | .cfi_restore %rbp |
619b9466 | 776 | mov -8(%rax),%rbx |
399976c7 | 777 | .cfi_restore %rbx |
619b9466 | 778 | lea (%rax),%rsp |
399976c7 | 779 | .cfi_def_cfa_register %rsp |
619b9466 AP |
780 | .Lepilogue_shaext: |
781 | ret | |
399976c7 | 782 | .cfi_endproc |
619b9466 AP |
783 | .size sha256_multi_block_shaext,.-sha256_multi_block_shaext |
784 | ___ | |
785 | }}} | |
b7838586 AP |
786 | if ($avx) {{{ |
787 | sub ROUND_00_15_avx { | |
788 | my ($i,$a,$b,$c,$d,$e,$f,$g,$h)=@_; | |
789 | ||
790 | $code.=<<___ if ($i<15 && $REG_SZ==16); | |
791 | vmovd `4*$i`(@ptr[0]),$Xi | |
792 | vmovd `4*$i`(@ptr[1]),$t1 | |
793 | vpinsrd \$1,`4*$i`(@ptr[2]),$Xi,$Xi | |
794 | vpinsrd \$1,`4*$i`(@ptr[3]),$t1,$t1 | |
795 | vpunpckldq $t1,$Xi,$Xi | |
796 | vpshufb $Xn,$Xi,$Xi | |
797 | ___ | |
798 | $code.=<<___ if ($i==15 && $REG_SZ==16); | |
799 | vmovd `4*$i`(@ptr[0]),$Xi | |
800 | lea `16*4`(@ptr[0]),@ptr[0] | |
801 | vmovd `4*$i`(@ptr[1]),$t1 | |
802 | lea `16*4`(@ptr[1]),@ptr[1] | |
803 | vpinsrd \$1,`4*$i`(@ptr[2]),$Xi,$Xi | |
804 | lea `16*4`(@ptr[2]),@ptr[2] | |
805 | vpinsrd \$1,`4*$i`(@ptr[3]),$t1,$t1 | |
806 | lea `16*4`(@ptr[3]),@ptr[3] | |
807 | vpunpckldq $t1,$Xi,$Xi | |
808 | vpshufb $Xn,$Xi,$Xi | |
809 | ___ | |
810 | $code.=<<___ if ($i<15 && $REG_SZ==32); | |
811 | vmovd `4*$i`(@ptr[0]),$Xi | |
812 | vmovd `4*$i`(@ptr[4]),$t1 | |
813 | vmovd `4*$i`(@ptr[1]),$t2 | |
814 | vmovd `4*$i`(@ptr[5]),$t3 | |
815 | vpinsrd \$1,`4*$i`(@ptr[2]),$Xi,$Xi | |
816 | vpinsrd \$1,`4*$i`(@ptr[6]),$t1,$t1 | |
817 | vpinsrd \$1,`4*$i`(@ptr[3]),$t2,$t2 | |
818 | vpunpckldq $t2,$Xi,$Xi | |
819 | vpinsrd \$1,`4*$i`(@ptr[7]),$t3,$t3 | |
820 | vpunpckldq $t3,$t1,$t1 | |
821 | vinserti128 $t1,$Xi,$Xi | |
822 | vpshufb $Xn,$Xi,$Xi | |
823 | ___ | |
824 | $code.=<<___ if ($i==15 && $REG_SZ==32); | |
825 | vmovd `4*$i`(@ptr[0]),$Xi | |
826 | lea `16*4`(@ptr[0]),@ptr[0] | |
827 | vmovd `4*$i`(@ptr[4]),$t1 | |
828 | lea `16*4`(@ptr[4]),@ptr[4] | |
829 | vmovd `4*$i`(@ptr[1]),$t2 | |
830 | lea `16*4`(@ptr[1]),@ptr[1] | |
831 | vmovd `4*$i`(@ptr[5]),$t3 | |
832 | lea `16*4`(@ptr[5]),@ptr[5] | |
833 | vpinsrd \$1,`4*$i`(@ptr[2]),$Xi,$Xi | |
834 | lea `16*4`(@ptr[2]),@ptr[2] | |
835 | vpinsrd \$1,`4*$i`(@ptr[6]),$t1,$t1 | |
836 | lea `16*4`(@ptr[6]),@ptr[6] | |
837 | vpinsrd \$1,`4*$i`(@ptr[3]),$t2,$t2 | |
838 | lea `16*4`(@ptr[3]),@ptr[3] | |
839 | vpunpckldq $t2,$Xi,$Xi | |
840 | vpinsrd \$1,`4*$i`(@ptr[7]),$t3,$t3 | |
841 | lea `16*4`(@ptr[7]),@ptr[7] | |
842 | vpunpckldq $t3,$t1,$t1 | |
843 | vinserti128 $t1,$Xi,$Xi | |
844 | vpshufb $Xn,$Xi,$Xi | |
845 | ___ | |
846 | $code.=<<___; | |
847 | vpsrld \$6,$e,$sigma | |
848 | vpslld \$26,$e,$t3 | |
849 | vmovdqu $Xi,`&Xi_off($i)` | |
850 | vpaddd $h,$Xi,$Xi # Xi+=h | |
851 | ||
852 | vpsrld \$11,$e,$t2 | |
853 | vpxor $t3,$sigma,$sigma | |
854 | vpslld \$21,$e,$t3 | |
855 | vpaddd `32*($i%8)-128`($Tbl),$Xi,$Xi # Xi+=K[round] | |
856 | vpxor $t2,$sigma,$sigma | |
857 | ||
858 | vpsrld \$25,$e,$t2 | |
859 | vpxor $t3,$sigma,$sigma | |
619b9466 | 860 | `"prefetcht0 63(@ptr[0])" if ($i==15)` |
b7838586 AP |
861 | vpslld \$7,$e,$t3 |
862 | vpandn $g,$e,$t1 | |
863 | vpand $f,$e,$axb # borrow $axb | |
619b9466 | 864 | `"prefetcht0 63(@ptr[1])" if ($i==15)` |
b7838586 AP |
865 | vpxor $t2,$sigma,$sigma |
866 | ||
867 | vpsrld \$2,$a,$h # borrow $h | |
868 | vpxor $t3,$sigma,$sigma # Sigma1(e) | |
619b9466 | 869 | `"prefetcht0 63(@ptr[2])" if ($i==15)` |
b7838586 AP |
870 | vpslld \$30,$a,$t2 |
871 | vpxor $axb,$t1,$t1 # Ch(e,f,g) | |
872 | vpxor $a,$b,$axb # a^b, b^c in next round | |
619b9466 | 873 | `"prefetcht0 63(@ptr[3])" if ($i==15)` |
b7838586 AP |
874 | vpxor $t2,$h,$h |
875 | vpaddd $sigma,$Xi,$Xi # Xi+=Sigma1(e) | |
876 | ||
877 | vpsrld \$13,$a,$t2 | |
619b9466 | 878 | `"prefetcht0 63(@ptr[4])" if ($i==15 && $REG_SZ==32)` |
b7838586 AP |
879 | vpslld \$19,$a,$t3 |
880 | vpaddd $t1,$Xi,$Xi # Xi+=Ch(e,f,g) | |
881 | vpand $axb,$bxc,$bxc | |
619b9466 | 882 | `"prefetcht0 63(@ptr[5])" if ($i==15 && $REG_SZ==32)` |
b7838586 AP |
883 | vpxor $t2,$h,$sigma |
884 | ||
885 | vpsrld \$22,$a,$t2 | |
886 | vpxor $t3,$sigma,$sigma | |
619b9466 | 887 | `"prefetcht0 63(@ptr[6])" if ($i==15 && $REG_SZ==32)` |
b7838586 AP |
888 | vpslld \$10,$a,$t3 |
889 | vpxor $bxc,$b,$h # h=Maj(a,b,c)=Ch(a^b,c,b) | |
890 | vpaddd $Xi,$d,$d # d+=Xi | |
619b9466 | 891 | `"prefetcht0 63(@ptr[7])" if ($i==15 && $REG_SZ==32)` |
b7838586 AP |
892 | vpxor $t2,$sigma,$sigma |
893 | vpxor $t3,$sigma,$sigma # Sigma0(a) | |
894 | ||
895 | vpaddd $Xi,$h,$h # h+=Xi | |
896 | vpaddd $sigma,$h,$h # h+=Sigma0(a) | |
897 | ___ | |
898 | $code.=<<___ if (($i%8)==7); | |
899 | add \$`32*8`,$Tbl | |
900 | ___ | |
901 | ($axb,$bxc)=($bxc,$axb); | |
902 | } | |
903 | ||
904 | sub ROUND_16_XX_avx { | |
905 | my $i=shift; | |
906 | ||
907 | $code.=<<___; | |
908 | vmovdqu `&Xi_off($i+1)`,$Xn | |
909 | vpaddd `&Xi_off($i+9)`,$Xi,$Xi # Xi+=X[i+9] | |
910 | ||
911 | vpsrld \$3,$Xn,$sigma | |
912 | vpsrld \$7,$Xn,$t2 | |
913 | vpslld \$25,$Xn,$t3 | |
914 | vpxor $t2,$sigma,$sigma | |
915 | vpsrld \$18,$Xn,$t2 | |
916 | vpxor $t3,$sigma,$sigma | |
917 | vpslld \$14,$Xn,$t3 | |
918 | vmovdqu `&Xi_off($i+14)`,$t1 | |
919 | vpsrld \$10,$t1,$axb # borrow $axb | |
920 | ||
921 | vpxor $t2,$sigma,$sigma | |
922 | vpsrld \$17,$t1,$t2 | |
923 | vpxor $t3,$sigma,$sigma # sigma0(X[i+1]) | |
924 | vpslld \$15,$t1,$t3 | |
925 | vpaddd $sigma,$Xi,$Xi # Xi+=sigma0(e) | |
926 | vpxor $t2,$axb,$sigma | |
927 | vpsrld \$19,$t1,$t2 | |
928 | vpxor $t3,$sigma,$sigma | |
929 | vpslld \$13,$t1,$t3 | |
930 | vpxor $t2,$sigma,$sigma | |
931 | vpxor $t3,$sigma,$sigma # sigma0(X[i+14]) | |
932 | vpaddd $sigma,$Xi,$Xi # Xi+=sigma1(X[i+14]) | |
933 | ___ | |
934 | &ROUND_00_15_avx($i,@_); | |
935 | ($Xi,$Xn)=($Xn,$Xi); | |
936 | } | |
937 | ||
938 | $code.=<<___; | |
939 | .type sha256_multi_block_avx,\@function,3 | |
940 | .align 32 | |
941 | sha256_multi_block_avx: | |
399976c7 | 942 | .cfi_startproc |
b7838586 AP |
943 | _avx_shortcut: |
944 | ___ | |
945 | $code.=<<___ if ($avx>1); | |
946 | shr \$32,%rcx | |
947 | cmp \$2,$num | |
948 | jb .Lavx | |
949 | test \$`1<<5`,%ecx | |
950 | jnz _avx2_shortcut | |
951 | jmp .Lavx | |
952 | .align 32 | |
953 | .Lavx: | |
954 | ___ | |
955 | $code.=<<___; | |
956 | mov %rsp,%rax | |
399976c7 | 957 | .cfi_def_cfa_register %rax |
b7838586 | 958 | push %rbx |
399976c7 | 959 | .cfi_push %rbx |
b7838586 | 960 | push %rbp |
399976c7 | 961 | .cfi_push %rbp |
b7838586 AP |
962 | ___ |
963 | $code.=<<___ if ($win64); | |
964 | lea -0xa8(%rsp),%rsp | |
965 | movaps %xmm6,(%rsp) | |
966 | movaps %xmm7,0x10(%rsp) | |
967 | movaps %xmm8,0x20(%rsp) | |
968 | movaps %xmm9,0x30(%rsp) | |
969 | movaps %xmm10,-0x78(%rax) | |
970 | movaps %xmm11,-0x68(%rax) | |
971 | movaps %xmm12,-0x58(%rax) | |
972 | movaps %xmm13,-0x48(%rax) | |
973 | movaps %xmm14,-0x38(%rax) | |
974 | movaps %xmm15,-0x28(%rax) | |
975 | ___ | |
976 | $code.=<<___; | |
977 | sub \$`$REG_SZ*18`, %rsp | |
978 | and \$-256,%rsp | |
979 | mov %rax,`$REG_SZ*17`(%rsp) # original %rsp | |
399976c7 | 980 | .cfi_cfa_expression %rsp+`$REG_SZ*17`,deref,+8 |
619b9466 | 981 | .Lbody_avx: |
b7838586 AP |
982 | lea K256+128(%rip),$Tbl |
983 | lea `$REG_SZ*16`(%rsp),%rbx | |
984 | lea 0x80($ctx),$ctx # size optimization | |
985 | ||
986 | .Loop_grande_avx: | |
987 | mov $num,`$REG_SZ*17+8`(%rsp) # original $num | |
988 | xor $num,$num | |
989 | ___ | |
990 | for($i=0;$i<4;$i++) { | |
991 | $code.=<<___; | |
992 | mov `16*$i+0`($inp),@ptr[$i] # input pointer | |
993 | mov `16*$i+8`($inp),%ecx # number of blocks | |
994 | cmp $num,%ecx | |
995 | cmovg %ecx,$num # find maximum | |
996 | test %ecx,%ecx | |
997 | mov %ecx,`4*$i`(%rbx) # initialize counters | |
998 | cmovle $Tbl,@ptr[$i] # cancel input | |
999 | ___ | |
1000 | } | |
1001 | $code.=<<___; | |
1002 | test $num,$num | |
1003 | jz .Ldone_avx | |
1004 | ||
1005 | vmovdqu 0x00-0x80($ctx),$A # load context | |
1006 | lea 128(%rsp),%rax | |
1007 | vmovdqu 0x20-0x80($ctx),$B | |
1008 | vmovdqu 0x40-0x80($ctx),$C | |
1009 | vmovdqu 0x60-0x80($ctx),$D | |
1010 | vmovdqu 0x80-0x80($ctx),$E | |
1011 | vmovdqu 0xa0-0x80($ctx),$F | |
1012 | vmovdqu 0xc0-0x80($ctx),$G | |
1013 | vmovdqu 0xe0-0x80($ctx),$H | |
1014 | vmovdqu .Lpbswap(%rip),$Xn | |
1015 | jmp .Loop_avx | |
1016 | ||
1017 | .align 32 | |
1018 | .Loop_avx: | |
1019 | vpxor $B,$C,$bxc # magic seed | |
1020 | ___ | |
1021 | for($i=0;$i<16;$i++) { &ROUND_00_15_avx($i,@V); unshift(@V,pop(@V)); } | |
1022 | $code.=<<___; | |
1023 | vmovdqu `&Xi_off($i)`,$Xi | |
1024 | mov \$3,%ecx | |
1025 | jmp .Loop_16_xx_avx | |
1026 | .align 32 | |
1027 | .Loop_16_xx_avx: | |
1028 | ___ | |
1029 | for(;$i<32;$i++) { &ROUND_16_XX_avx($i,@V); unshift(@V,pop(@V)); } | |
1030 | $code.=<<___; | |
1031 | dec %ecx | |
1032 | jnz .Loop_16_xx_avx | |
1033 | ||
1034 | mov \$1,%ecx | |
1035 | lea K256+128(%rip),$Tbl | |
1036 | ___ | |
1037 | for($i=0;$i<4;$i++) { | |
1038 | $code.=<<___; | |
1039 | cmp `4*$i`(%rbx),%ecx # examine counters | |
1040 | cmovge $Tbl,@ptr[$i] # cancel input | |
1041 | ___ | |
1042 | } | |
1043 | $code.=<<___; | |
1044 | vmovdqa (%rbx),$sigma # pull counters | |
1045 | vpxor $t1,$t1,$t1 | |
1046 | vmovdqa $sigma,$Xn | |
1047 | vpcmpgtd $t1,$Xn,$Xn # mask value | |
1048 | vpaddd $Xn,$sigma,$sigma # counters-- | |
1049 | ||
1050 | vmovdqu 0x00-0x80($ctx),$t1 | |
1051 | vpand $Xn,$A,$A | |
1052 | vmovdqu 0x20-0x80($ctx),$t2 | |
1053 | vpand $Xn,$B,$B | |
1054 | vmovdqu 0x40-0x80($ctx),$t3 | |
1055 | vpand $Xn,$C,$C | |
1056 | vmovdqu 0x60-0x80($ctx),$Xi | |
1057 | vpand $Xn,$D,$D | |
1058 | vpaddd $t1,$A,$A | |
1059 | vmovdqu 0x80-0x80($ctx),$t1 | |
1060 | vpand $Xn,$E,$E | |
1061 | vpaddd $t2,$B,$B | |
1062 | vmovdqu 0xa0-0x80($ctx),$t2 | |
1063 | vpand $Xn,$F,$F | |
1064 | vpaddd $t3,$C,$C | |
1065 | vmovdqu 0xc0-0x80($ctx),$t3 | |
1066 | vpand $Xn,$G,$G | |
1067 | vpaddd $Xi,$D,$D | |
1068 | vmovdqu 0xe0-0x80($ctx),$Xi | |
1069 | vpand $Xn,$H,$H | |
1070 | vpaddd $t1,$E,$E | |
1071 | vpaddd $t2,$F,$F | |
1072 | vmovdqu $A,0x00-0x80($ctx) | |
1073 | vpaddd $t3,$G,$G | |
1074 | vmovdqu $B,0x20-0x80($ctx) | |
1075 | vpaddd $Xi,$H,$H | |
1076 | vmovdqu $C,0x40-0x80($ctx) | |
1077 | vmovdqu $D,0x60-0x80($ctx) | |
1078 | vmovdqu $E,0x80-0x80($ctx) | |
1079 | vmovdqu $F,0xa0-0x80($ctx) | |
1080 | vmovdqu $G,0xc0-0x80($ctx) | |
1081 | vmovdqu $H,0xe0-0x80($ctx) | |
1082 | ||
1083 | vmovdqu $sigma,(%rbx) # save counters | |
1084 | vmovdqu .Lpbswap(%rip),$Xn | |
1085 | dec $num | |
1086 | jnz .Loop_avx | |
1087 | ||
1088 | mov `$REG_SZ*17+8`(%rsp),$num | |
1089 | lea $REG_SZ($ctx),$ctx | |
1090 | lea `16*$REG_SZ/4`($inp),$inp | |
1091 | dec $num | |
1092 | jnz .Loop_grande_avx | |
1093 | ||
1094 | .Ldone_avx: | |
0d4fb843 | 1095 | mov `$REG_SZ*17`(%rsp),%rax # original %rsp |
399976c7 | 1096 | .cfi_def_cfa %rax,8 |
b7838586 AP |
1097 | vzeroupper |
1098 | ___ | |
1099 | $code.=<<___ if ($win64); | |
1100 | movaps -0xb8(%rax),%xmm6 | |
1101 | movaps -0xa8(%rax),%xmm7 | |
1102 | movaps -0x98(%rax),%xmm8 | |
1103 | movaps -0x88(%rax),%xmm9 | |
1104 | movaps -0x78(%rax),%xmm10 | |
1105 | movaps -0x68(%rax),%xmm11 | |
1106 | movaps -0x58(%rax),%xmm12 | |
1107 | movaps -0x48(%rax),%xmm13 | |
1108 | movaps -0x38(%rax),%xmm14 | |
1109 | movaps -0x28(%rax),%xmm15 | |
1110 | ___ | |
1111 | $code.=<<___; | |
1112 | mov -16(%rax),%rbp | |
399976c7 | 1113 | .cfi_restore %rbp |
b7838586 | 1114 | mov -8(%rax),%rbx |
399976c7 | 1115 | .cfi_restore %rbx |
b7838586 | 1116 | lea (%rax),%rsp |
399976c7 | 1117 | .cfi_def_cfa_register %rsp |
619b9466 | 1118 | .Lepilogue_avx: |
b7838586 | 1119 | ret |
399976c7 | 1120 | .cfi_endproc |
b7838586 AP |
1121 | .size sha256_multi_block_avx,.-sha256_multi_block_avx |
1122 | ___ | |
1123 | if ($avx>1) { | |
1124 | $code =~ s/\`([^\`]*)\`/eval $1/gem; | |
1125 | ||
1126 | $REG_SZ=32; | |
1127 | @ptr=map("%r$_",(12..15,8..11)); | |
1128 | ||
1129 | @V=($A,$B,$C,$D,$E,$F,$G,$H)=map("%ymm$_",(8..15)); | |
1130 | ($t1,$t2,$t3,$axb,$bxc,$Xi,$Xn,$sigma)=map("%ymm$_",(0..7)); | |
1131 | ||
1132 | $code.=<<___; | |
1133 | .type sha256_multi_block_avx2,\@function,3 | |
1134 | .align 32 | |
1135 | sha256_multi_block_avx2: | |
399976c7 | 1136 | .cfi_startproc |
b7838586 AP |
1137 | _avx2_shortcut: |
1138 | mov %rsp,%rax | |
399976c7 | 1139 | .cfi_def_cfa_register %rax |
b7838586 | 1140 | push %rbx |
399976c7 | 1141 | .cfi_push %rbx |
b7838586 | 1142 | push %rbp |
399976c7 | 1143 | .cfi_push %rbp |
b7838586 | 1144 | push %r12 |
399976c7 | 1145 | .cfi_push %r12 |
b7838586 | 1146 | push %r13 |
399976c7 | 1147 | .cfi_push %r13 |
b7838586 | 1148 | push %r14 |
399976c7 | 1149 | .cfi_push %r14 |
b7838586 | 1150 | push %r15 |
399976c7 | 1151 | .cfi_push %r15 |
b7838586 AP |
1152 | ___ |
1153 | $code.=<<___ if ($win64); | |
1154 | lea -0xa8(%rsp),%rsp | |
1155 | movaps %xmm6,(%rsp) | |
1156 | movaps %xmm7,0x10(%rsp) | |
1157 | movaps %xmm8,0x20(%rsp) | |
1158 | movaps %xmm9,0x30(%rsp) | |
1159 | movaps %xmm10,0x40(%rsp) | |
1160 | movaps %xmm11,0x50(%rsp) | |
1161 | movaps %xmm12,-0x78(%rax) | |
1162 | movaps %xmm13,-0x68(%rax) | |
1163 | movaps %xmm14,-0x58(%rax) | |
1164 | movaps %xmm15,-0x48(%rax) | |
1165 | ___ | |
1166 | $code.=<<___; | |
1167 | sub \$`$REG_SZ*18`, %rsp | |
1168 | and \$-256,%rsp | |
1169 | mov %rax,`$REG_SZ*17`(%rsp) # original %rsp | |
399976c7 | 1170 | .cfi_cfa_expression %rsp+`$REG_SZ*17`,deref,+8 |
619b9466 | 1171 | .Lbody_avx2: |
b7838586 AP |
1172 | lea K256+128(%rip),$Tbl |
1173 | lea 0x80($ctx),$ctx # size optimization | |
1174 | ||
1175 | .Loop_grande_avx2: | |
1176 | mov $num,`$REG_SZ*17+8`(%rsp) # original $num | |
1177 | xor $num,$num | |
1178 | lea `$REG_SZ*16`(%rsp),%rbx | |
1179 | ___ | |
1180 | for($i=0;$i<8;$i++) { | |
1181 | $code.=<<___; | |
1182 | mov `16*$i+0`($inp),@ptr[$i] # input pointer | |
1183 | mov `16*$i+8`($inp),%ecx # number of blocks | |
1184 | cmp $num,%ecx | |
1185 | cmovg %ecx,$num # find maximum | |
1186 | test %ecx,%ecx | |
1187 | mov %ecx,`4*$i`(%rbx) # initialize counters | |
1188 | cmovle $Tbl,@ptr[$i] # cancel input | |
1189 | ___ | |
1190 | } | |
1191 | $code.=<<___; | |
1192 | vmovdqu 0x00-0x80($ctx),$A # load context | |
1193 | lea 128(%rsp),%rax | |
1194 | vmovdqu 0x20-0x80($ctx),$B | |
1195 | lea 256+128(%rsp),%rbx | |
1196 | vmovdqu 0x40-0x80($ctx),$C | |
1197 | vmovdqu 0x60-0x80($ctx),$D | |
1198 | vmovdqu 0x80-0x80($ctx),$E | |
1199 | vmovdqu 0xa0-0x80($ctx),$F | |
1200 | vmovdqu 0xc0-0x80($ctx),$G | |
1201 | vmovdqu 0xe0-0x80($ctx),$H | |
1202 | vmovdqu .Lpbswap(%rip),$Xn | |
1203 | jmp .Loop_avx2 | |
1204 | ||
1205 | .align 32 | |
1206 | .Loop_avx2: | |
1207 | vpxor $B,$C,$bxc # magic seed | |
1208 | ___ | |
1209 | for($i=0;$i<16;$i++) { &ROUND_00_15_avx($i,@V); unshift(@V,pop(@V)); } | |
1210 | $code.=<<___; | |
1211 | vmovdqu `&Xi_off($i)`,$Xi | |
1212 | mov \$3,%ecx | |
1213 | jmp .Loop_16_xx_avx2 | |
1214 | .align 32 | |
1215 | .Loop_16_xx_avx2: | |
1216 | ___ | |
1217 | for(;$i<32;$i++) { &ROUND_16_XX_avx($i,@V); unshift(@V,pop(@V)); } | |
1218 | $code.=<<___; | |
1219 | dec %ecx | |
1220 | jnz .Loop_16_xx_avx2 | |
1221 | ||
1222 | mov \$1,%ecx | |
1223 | lea `$REG_SZ*16`(%rsp),%rbx | |
1224 | lea K256+128(%rip),$Tbl | |
1225 | ___ | |
1226 | for($i=0;$i<8;$i++) { | |
1227 | $code.=<<___; | |
1228 | cmp `4*$i`(%rbx),%ecx # examine counters | |
1229 | cmovge $Tbl,@ptr[$i] # cancel input | |
1230 | ___ | |
1231 | } | |
1232 | $code.=<<___; | |
1233 | vmovdqa (%rbx),$sigma # pull counters | |
1234 | vpxor $t1,$t1,$t1 | |
1235 | vmovdqa $sigma,$Xn | |
1236 | vpcmpgtd $t1,$Xn,$Xn # mask value | |
1237 | vpaddd $Xn,$sigma,$sigma # counters-- | |
1238 | ||
1239 | vmovdqu 0x00-0x80($ctx),$t1 | |
1240 | vpand $Xn,$A,$A | |
1241 | vmovdqu 0x20-0x80($ctx),$t2 | |
1242 | vpand $Xn,$B,$B | |
1243 | vmovdqu 0x40-0x80($ctx),$t3 | |
1244 | vpand $Xn,$C,$C | |
1245 | vmovdqu 0x60-0x80($ctx),$Xi | |
1246 | vpand $Xn,$D,$D | |
1247 | vpaddd $t1,$A,$A | |
1248 | vmovdqu 0x80-0x80($ctx),$t1 | |
1249 | vpand $Xn,$E,$E | |
1250 | vpaddd $t2,$B,$B | |
1251 | vmovdqu 0xa0-0x80($ctx),$t2 | |
1252 | vpand $Xn,$F,$F | |
1253 | vpaddd $t3,$C,$C | |
1254 | vmovdqu 0xc0-0x80($ctx),$t3 | |
1255 | vpand $Xn,$G,$G | |
1256 | vpaddd $Xi,$D,$D | |
1257 | vmovdqu 0xe0-0x80($ctx),$Xi | |
1258 | vpand $Xn,$H,$H | |
1259 | vpaddd $t1,$E,$E | |
1260 | vpaddd $t2,$F,$F | |
1261 | vmovdqu $A,0x00-0x80($ctx) | |
1262 | vpaddd $t3,$G,$G | |
1263 | vmovdqu $B,0x20-0x80($ctx) | |
1264 | vpaddd $Xi,$H,$H | |
1265 | vmovdqu $C,0x40-0x80($ctx) | |
1266 | vmovdqu $D,0x60-0x80($ctx) | |
1267 | vmovdqu $E,0x80-0x80($ctx) | |
1268 | vmovdqu $F,0xa0-0x80($ctx) | |
1269 | vmovdqu $G,0xc0-0x80($ctx) | |
1270 | vmovdqu $H,0xe0-0x80($ctx) | |
1271 | ||
1272 | vmovdqu $sigma,(%rbx) # save counters | |
1273 | lea 256+128(%rsp),%rbx | |
1274 | vmovdqu .Lpbswap(%rip),$Xn | |
1275 | dec $num | |
1276 | jnz .Loop_avx2 | |
1277 | ||
1278 | #mov `$REG_SZ*17+8`(%rsp),$num | |
1279 | #lea $REG_SZ($ctx),$ctx | |
1280 | #lea `16*$REG_SZ/4`($inp),$inp | |
1281 | #dec $num | |
1282 | #jnz .Loop_grande_avx2 | |
1283 | ||
1284 | .Ldone_avx2: | |
0d4fb843 | 1285 | mov `$REG_SZ*17`(%rsp),%rax # original %rsp |
399976c7 | 1286 | .cfi_def_cfa %rax,8 |
b7838586 AP |
1287 | vzeroupper |
1288 | ___ | |
1289 | $code.=<<___ if ($win64); | |
1290 | movaps -0xd8(%rax),%xmm6 | |
1291 | movaps -0xc8(%rax),%xmm7 | |
1292 | movaps -0xb8(%rax),%xmm8 | |
1293 | movaps -0xa8(%rax),%xmm9 | |
1294 | movaps -0x98(%rax),%xmm10 | |
1295 | movaps -0x88(%rax),%xmm11 | |
1296 | movaps -0x78(%rax),%xmm12 | |
1297 | movaps -0x68(%rax),%xmm13 | |
1298 | movaps -0x58(%rax),%xmm14 | |
1299 | movaps -0x48(%rax),%xmm15 | |
1300 | ___ | |
1301 | $code.=<<___; | |
1302 | mov -48(%rax),%r15 | |
399976c7 | 1303 | .cfi_restore %r15 |
b7838586 | 1304 | mov -40(%rax),%r14 |
399976c7 | 1305 | .cfi_restore %r14 |
b7838586 | 1306 | mov -32(%rax),%r13 |
399976c7 | 1307 | .cfi_restore %r13 |
b7838586 | 1308 | mov -24(%rax),%r12 |
399976c7 | 1309 | .cfi_restore %r12 |
b7838586 | 1310 | mov -16(%rax),%rbp |
399976c7 | 1311 | .cfi_restore %rbp |
b7838586 | 1312 | mov -8(%rax),%rbx |
399976c7 | 1313 | .cfi_restore %rbx |
b7838586 | 1314 | lea (%rax),%rsp |
399976c7 | 1315 | .cfi_def_cfa_register %rsp |
619b9466 | 1316 | .Lepilogue_avx2: |
b7838586 | 1317 | ret |
399976c7 | 1318 | .cfi_endproc |
b7838586 AP |
1319 | .size sha256_multi_block_avx2,.-sha256_multi_block_avx2 |
1320 | ___ | |
1321 | } }}} | |
1322 | $code.=<<___; | |
1323 | .align 256 | |
1324 | K256: | |
1325 | ___ | |
1326 | sub TABLE { | |
1327 | foreach (@_) { | |
1328 | $code.=<<___; | |
1329 | .long $_,$_,$_,$_ | |
1330 | .long $_,$_,$_,$_ | |
1331 | ___ | |
1332 | } | |
1333 | } | |
1334 | &TABLE( 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5, | |
1335 | 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5, | |
1336 | 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3, | |
1337 | 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174, | |
1338 | 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc, | |
1339 | 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da, | |
1340 | 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7, | |
1341 | 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967, | |
1342 | 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13, | |
1343 | 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85, | |
1344 | 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3, | |
1345 | 0xd192e819,0xd6990624,0xf40e3585,0x106aa070, | |
1346 | 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5, | |
1347 | 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3, | |
1348 | 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208, | |
1349 | 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 ); | |
1350 | $code.=<<___; | |
1351 | .Lpbswap: | |
1352 | .long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f # pbswap | |
1353 | .long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f # pbswap | |
619b9466 AP |
1354 | K256_shaext: |
1355 | .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 | |
1356 | .long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 | |
1357 | .long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 | |
1358 | .long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 | |
1359 | .long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc | |
1360 | .long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da | |
1361 | .long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 | |
1362 | .long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 | |
1363 | .long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 | |
1364 | .long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 | |
1365 | .long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 | |
1366 | .long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 | |
1367 | .long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 | |
1368 | .long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 | |
1369 | .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 | |
1370 | .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 | |
1371 | .asciz "SHA256 multi-block transform for x86_64, CRYPTOGAMS by <appro\@openssl.org>" | |
b7838586 AP |
1372 | ___ |
1373 | ||
619b9466 AP |
1374 | if ($win64) { |
1375 | # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, | |
1376 | # CONTEXT *context,DISPATCHER_CONTEXT *disp) | |
1377 | $rec="%rcx"; | |
1378 | $frame="%rdx"; | |
1379 | $context="%r8"; | |
1380 | $disp="%r9"; | |
1381 | ||
1382 | $code.=<<___; | |
1383 | .extern __imp_RtlVirtualUnwind | |
1384 | .type se_handler,\@abi-omnipotent | |
1385 | .align 16 | |
1386 | se_handler: | |
1387 | push %rsi | |
1388 | push %rdi | |
1389 | push %rbx | |
1390 | push %rbp | |
1391 | push %r12 | |
1392 | push %r13 | |
1393 | push %r14 | |
1394 | push %r15 | |
1395 | pushfq | |
1396 | sub \$64,%rsp | |
1397 | ||
1398 | mov 120($context),%rax # pull context->Rax | |
1399 | mov 248($context),%rbx # pull context->Rip | |
1400 | ||
1401 | mov 8($disp),%rsi # disp->ImageBase | |
1402 | mov 56($disp),%r11 # disp->HandlerData | |
1403 | ||
1404 | mov 0(%r11),%r10d # HandlerData[0] | |
1405 | lea (%rsi,%r10),%r10 # end of prologue label | |
1406 | cmp %r10,%rbx # context->Rip<.Lbody | |
1407 | jb .Lin_prologue | |
1408 | ||
1409 | mov 152($context),%rax # pull context->Rsp | |
1410 | ||
1411 | mov 4(%r11),%r10d # HandlerData[1] | |
1412 | lea (%rsi,%r10),%r10 # epilogue label | |
1413 | cmp %r10,%rbx # context->Rip>=.Lepilogue | |
1414 | jae .Lin_prologue | |
1415 | ||
1416 | mov `16*17`(%rax),%rax # pull saved stack pointer | |
1417 | ||
1418 | mov -8(%rax),%rbx | |
1419 | mov -16(%rax),%rbp | |
1420 | mov %rbx,144($context) # restore context->Rbx | |
1421 | mov %rbp,160($context) # restore context->Rbp | |
1422 | ||
1423 | lea -24-10*16(%rax),%rsi | |
1424 | lea 512($context),%rdi # &context.Xmm6 | |
1425 | mov \$20,%ecx | |
1426 | .long 0xa548f3fc # cld; rep movsq | |
1427 | ||
1428 | .Lin_prologue: | |
1429 | mov 8(%rax),%rdi | |
1430 | mov 16(%rax),%rsi | |
1431 | mov %rax,152($context) # restore context->Rsp | |
1432 | mov %rsi,168($context) # restore context->Rsi | |
1433 | mov %rdi,176($context) # restore context->Rdi | |
1434 | ||
1435 | mov 40($disp),%rdi # disp->ContextRecord | |
1436 | mov $context,%rsi # context | |
1437 | mov \$154,%ecx # sizeof(CONTEXT) | |
1438 | .long 0xa548f3fc # cld; rep movsq | |
1439 | ||
1440 | mov $disp,%rsi | |
1441 | xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER | |
1442 | mov 8(%rsi),%rdx # arg2, disp->ImageBase | |
1443 | mov 0(%rsi),%r8 # arg3, disp->ControlPc | |
1444 | mov 16(%rsi),%r9 # arg4, disp->FunctionEntry | |
1445 | mov 40(%rsi),%r10 # disp->ContextRecord | |
1446 | lea 56(%rsi),%r11 # &disp->HandlerData | |
1447 | lea 24(%rsi),%r12 # &disp->EstablisherFrame | |
1448 | mov %r10,32(%rsp) # arg5 | |
1449 | mov %r11,40(%rsp) # arg6 | |
1450 | mov %r12,48(%rsp) # arg7 | |
1451 | mov %rcx,56(%rsp) # arg8, (NULL) | |
1452 | call *__imp_RtlVirtualUnwind(%rip) | |
1453 | ||
1454 | mov \$1,%eax # ExceptionContinueSearch | |
1455 | add \$64,%rsp | |
1456 | popfq | |
1457 | pop %r15 | |
1458 | pop %r14 | |
1459 | pop %r13 | |
1460 | pop %r12 | |
1461 | pop %rbp | |
1462 | pop %rbx | |
1463 | pop %rdi | |
1464 | pop %rsi | |
1465 | ret | |
1466 | .size se_handler,.-se_handler | |
1467 | ___ | |
1468 | $code.=<<___ if ($avx>1); | |
1469 | .type avx2_handler,\@abi-omnipotent | |
1470 | .align 16 | |
1471 | avx2_handler: | |
1472 | push %rsi | |
1473 | push %rdi | |
1474 | push %rbx | |
1475 | push %rbp | |
1476 | push %r12 | |
1477 | push %r13 | |
1478 | push %r14 | |
1479 | push %r15 | |
1480 | pushfq | |
1481 | sub \$64,%rsp | |
1482 | ||
1483 | mov 120($context),%rax # pull context->Rax | |
1484 | mov 248($context),%rbx # pull context->Rip | |
1485 | ||
1486 | mov 8($disp),%rsi # disp->ImageBase | |
1487 | mov 56($disp),%r11 # disp->HandlerData | |
1488 | ||
1489 | mov 0(%r11),%r10d # HandlerData[0] | |
1490 | lea (%rsi,%r10),%r10 # end of prologue label | |
1491 | cmp %r10,%rbx # context->Rip<body label | |
1492 | jb .Lin_prologue | |
1493 | ||
1494 | mov 152($context),%rax # pull context->Rsp | |
1495 | ||
1496 | mov 4(%r11),%r10d # HandlerData[1] | |
1497 | lea (%rsi,%r10),%r10 # epilogue label | |
1498 | cmp %r10,%rbx # context->Rip>=epilogue label | |
1499 | jae .Lin_prologue | |
1500 | ||
1501 | mov `32*17`($context),%rax # pull saved stack pointer | |
1502 | ||
1503 | mov -8(%rax),%rbx | |
1504 | mov -16(%rax),%rbp | |
1505 | mov -24(%rax),%r12 | |
1506 | mov -32(%rax),%r13 | |
1507 | mov -40(%rax),%r14 | |
1508 | mov -48(%rax),%r15 | |
1509 | mov %rbx,144($context) # restore context->Rbx | |
1510 | mov %rbp,160($context) # restore context->Rbp | |
46f4e1be JS |
1511 | mov %r12,216($context) # restore context->R12 |
1512 | mov %r13,224($context) # restore context->R13 | |
1513 | mov %r14,232($context) # restore context->R14 | |
1514 | mov %r15,240($context) # restore context->R15 | |
619b9466 AP |
1515 | |
1516 | lea -56-10*16(%rax),%rsi | |
1517 | lea 512($context),%rdi # &context.Xmm6 | |
1518 | mov \$20,%ecx | |
1519 | .long 0xa548f3fc # cld; rep movsq | |
1520 | ||
1521 | jmp .Lin_prologue | |
1522 | .size avx2_handler,.-avx2_handler | |
1523 | ___ | |
1524 | $code.=<<___; | |
1525 | .section .pdata | |
1526 | .align 4 | |
1527 | .rva .LSEH_begin_sha256_multi_block | |
1528 | .rva .LSEH_end_sha256_multi_block | |
1529 | .rva .LSEH_info_sha256_multi_block | |
1530 | .rva .LSEH_begin_sha256_multi_block_shaext | |
1531 | .rva .LSEH_end_sha256_multi_block_shaext | |
1532 | .rva .LSEH_info_sha256_multi_block_shaext | |
1533 | ___ | |
1534 | $code.=<<___ if ($avx); | |
1535 | .rva .LSEH_begin_sha256_multi_block_avx | |
1536 | .rva .LSEH_end_sha256_multi_block_avx | |
1537 | .rva .LSEH_info_sha256_multi_block_avx | |
1538 | ___ | |
1539 | $code.=<<___ if ($avx>1); | |
1540 | .rva .LSEH_begin_sha256_multi_block_avx2 | |
1541 | .rva .LSEH_end_sha256_multi_block_avx2 | |
1542 | .rva .LSEH_info_sha256_multi_block_avx2 | |
1543 | ___ | |
1544 | $code.=<<___; | |
1545 | .section .xdata | |
1546 | .align 8 | |
1547 | .LSEH_info_sha256_multi_block: | |
1548 | .byte 9,0,0,0 | |
1549 | .rva se_handler | |
1550 | .rva .Lbody,.Lepilogue # HandlerData[] | |
1551 | .LSEH_info_sha256_multi_block_shaext: | |
1552 | .byte 9,0,0,0 | |
1553 | .rva se_handler | |
1554 | .rva .Lbody_shaext,.Lepilogue_shaext # HandlerData[] | |
1555 | ___ | |
1556 | $code.=<<___ if ($avx); | |
1557 | .LSEH_info_sha256_multi_block_avx: | |
1558 | .byte 9,0,0,0 | |
1559 | .rva se_handler | |
1560 | .rva .Lbody_avx,.Lepilogue_avx # HandlerData[] | |
1561 | ___ | |
1562 | $code.=<<___ if ($avx>1); | |
1563 | .LSEH_info_sha256_multi_block_avx2: | |
1564 | .byte 9,0,0,0 | |
1565 | .rva avx2_handler | |
1566 | .rva .Lbody_avx2,.Lepilogue_avx2 # HandlerData[] | |
1567 | ___ | |
1568 | } | |
1569 | #################################################################### | |
1570 | ||
1571 | sub rex { | |
1572 | local *opcode=shift; | |
1573 | my ($dst,$src)=@_; | |
1574 | my $rex=0; | |
1575 | ||
1576 | $rex|=0x04 if ($dst>=8); | |
1577 | $rex|=0x01 if ($src>=8); | |
1578 | unshift @opcode,$rex|0x40 if ($rex); | |
1579 | } | |
1580 | ||
1581 | sub sha256op38 { | |
1582 | my $instr = shift; | |
1583 | my %opcodelet = ( | |
1584 | "sha256rnds2" => 0xcb, | |
1585 | "sha256msg1" => 0xcc, | |
1586 | "sha256msg2" => 0xcd ); | |
1587 | ||
1588 | if (defined($opcodelet{$instr}) && @_[0] =~ /%xmm([0-9]+),\s*%xmm([0-9]+)/) { | |
1589 | my @opcode=(0x0f,0x38); | |
1590 | rex(\@opcode,$2,$1); | |
1591 | push @opcode,$opcodelet{$instr}; | |
1592 | push @opcode,0xc0|($1&7)|(($2&7)<<3); # ModR/M | |
1593 | return ".byte\t".join(',',@opcode); | |
1594 | } else { | |
1595 | return $instr."\t".@_[0]; | |
1596 | } | |
1597 | } | |
1598 | ||
b7838586 AP |
1599 | foreach (split("\n",$code)) { |
1600 | s/\`([^\`]*)\`/eval($1)/ge; | |
1601 | ||
619b9466 AP |
1602 | s/\b(sha256[^\s]*)\s+(.*)/sha256op38($1,$2)/geo or |
1603 | ||
b7838586 AP |
1604 | s/\b(vmov[dq])\b(.+)%ymm([0-9]+)/$1$2%xmm$3/go or |
1605 | s/\b(vmovdqu)\b(.+)%x%ymm([0-9]+)/$1$2%xmm$3/go or | |
1606 | s/\b(vpinsr[qd])\b(.+)%ymm([0-9]+),%ymm([0-9]+)/$1$2%xmm$3,%xmm$4/go or | |
1607 | s/\b(vpextr[qd])\b(.+)%ymm([0-9]+)/$1$2%xmm$3/go or | |
1608 | s/\b(vinserti128)\b(\s+)%ymm/$1$2\$1,%xmm/go or | |
1609 | s/\b(vpbroadcast[qd]\s+)%ymm([0-9]+)/$1%xmm$2/go; | |
619b9466 | 1610 | |
b7838586 AP |
1611 | print $_,"\n"; |
1612 | } | |
1613 | ||
1614 | close STDOUT; |