]> git.ipfire.org Git - thirdparty/openssl.git/blame - crypto/chacha/asm/chacha-x86.pl
projects / thirdparty / openssl.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Fix spelling in pod files
[thirdparty/openssl.git] / crypto / chacha / asm / chacha-x86.pl
CommitLineData
a98c648e
AP		 1#!/usr/bin/env perl
2#
3# ====================================================================
4# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
5# project. The module is, however, dual licensed under OpenSSL and
6# CRYPTOGAMS licenses depending on where you obtain it. For further
7# details see http://www.openssl.org/~appro/cryptogams/.
8# ====================================================================
9#
10# January 2015
11#
12# ChaCha20 for x86.
13#
14# Performance in cycles per byte out of large buffer.
15#
16# 1xIALU/gcc 4xSSSE3
17# Pentium 17.5/+80%
18# PIII 14.2/+60%
19# P4 18.6/+84%
20# Core2 9.56/+89% 4.83
21# Westmere 9.50/+45% 3.35
22# Sandy Bridge 10.5/+47% 3.20
23# Haswell 8.15/+50% 2.83
24# Silvermont 17.4/+36% 8.35
25# Sledgehammer 10.2/+54%
26# Bulldozer 13.4/+50% 4.38(*)
27#
28# (*) Bulldozer actually executes 4xXOP code path that delivers 3.55;
29
30$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
31push(@INC,"${dir}","${dir}../../perlasm");
32require "x86asm.pl";
33
df0cb57c
RL		 34$output=pop;
35open STDOUT,">$output";
36
a98c648e
AP		 37&asm_init($ARGV[0],"chacha-x86.pl",$ARGV[$#ARGV] eq "386");
38
39$xmm=$ymm=0;
40for (@ARGV) { $xmm=1 if (/-DOPENSSL_IA32_SSE2/); }
41
42$ymm=1 if ($xmm &&
43 `$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
44 =~ /GNU assembler version ([2-9]\.[0-9]+)/ &&
45 $1>=2.19); # first version supporting AVX
46
47$ymm=1 if ($xmm && !$ymm && $ARGV[0] eq "win32n" &&
48 `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/ &&
49 $1>=2.03); # first version supporting AVX
50
51$ymm=1 if ($xmm && !$ymm && $ARGV[0] eq "win32" &&
52 `ml 2>&1` =~ /Version ([0-9]+)\./ &&
53 $1>=10); # first version supporting AVX
54
55$ymm=1 if ($xmm && !$ymm &&
56 `$ENV{CC} -v 2>&1` =~ /(^clang version|based on LLVM) ([3-9]\.[0-9]+)/ &&
57 $2>=3.0); # first version supporting AVX
58
59$a="eax";
60($b,$b_)=("ebx","ebp");
61($c,$c_)=("ecx","esi");
62($d,$d_)=("edx","edi");
63
64sub QUARTERROUND {
65my ($ai,$bi,$ci,$di,$i)=@_;
66my ($an,$bn,$cn,$dn)=map(($_&~3)+(($_+1)&3),($ai,$bi,$ci,$di)); # next
67my ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_-1)&3),($ai,$bi,$ci,$di)); # previous
68
69 # a b c d
70 #
71 # 0 4 8 12 < even round
72 # 1 5 9 13
73 # 2 6 10 14
74 # 3 7 11 15
75 # 0 5 10 15 < odd round
76 # 1 6 11 12
77 # 2 7 8 13
78 # 3 4 9 14
79
80 if ($i==0) {
81 my $j=4;
82 ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_-$j--)&3),($ap,$bp,$cp,$dp));
83 } elsif ($i==3) {
84 my $j=0;
85 ($an,$bn,$cn,$dn)=map(($_&~3)+(($_+$j++)&3),($an,$bn,$cn,$dn));
86 } elsif ($i==4) {
87 my $j=4;
88 ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_+$j--)&3),($ap,$bp,$cp,$dp));
89 } elsif ($i==7) {
90 my $j=0;
91 ($an,$bn,$cn,$dn)=map(($_&~3)+(($_-$j++)&3),($an,$bn,$cn,$dn));
92 }
93
94 #&add ($a,$b); # see elsewhere
95 &xor ($d,$a);
96 &mov (&DWP(4*$cp,"esp"),$c_) if ($ai>0 && $ai<3);
97 &rol ($d,16);
98 &mov (&DWP(4*$bp,"esp"),$b_) if ($i!=0);
99 &add ($c,$d);
100 &mov ($c_,&DWP(4*$cn,"esp")) if ($ai>0 && $ai<3);
101 &xor ($b,$c);
102 &mov ($d_,&DWP(4*$dn,"esp")) if ($di!=$dn);
103 &rol ($b,12);
104 &mov ($b_,&DWP(4*$bn,"esp")) if ($i<7);
105 &mov ($b_,&DWP(128,"esp")) if ($i==7); # loop counter
106 &add ($a,$b);
107 &xor ($d,$a);
108 &mov (&DWP(4*$ai,"esp"),$a);
109 &rol ($d,8);
110 &mov ($a,&DWP(4*$an,"esp"));
111 &add ($c,$d);
112 &mov (&DWP(4*$di,"esp"),$d) if ($di!=$dn);
113 &mov ($d_,$d) if ($di==$dn);
114 &xor ($b,$c);
115 &add ($a,$b_) if ($i<7); # elsewhere
116 &rol ($b,7);
117
118 ($b,$b_)=($b_,$b);
119 ($c,$c_)=($c_,$c);
120 ($d,$d_)=($d_,$d);
121}
122
123&static_label("ssse3_shortcut");
124&static_label("xop_shortcut");
125&static_label("ssse3_data");
126&static_label("pic_point");
127
128&function_begin("ChaCha20_ctr32");
622a531c
AP		 129 &xor ("eax","eax");
130 &cmp ("eax",&wparam(2)); # len==0?
131 &je (&label("no_data"));
a98c648e
AP		 132if ($xmm) {
133 &call (&label("pic_point"));
134&set_label("pic_point");
135 &blindpop("eax");
136 &picmeup("ebp","OPENSSL_ia32cap_P","eax",&label("pic_point"));
137 &test (&DWP(0,"ebp"),1<<24); # test FXSR bit
138 &jz (&label("x86"));
139 &test (&DWP(4,"ebp"),1<<9); # test SSSE3 bit
140 &jz (&label("x86"));
141 &jmp (&label("ssse3_shortcut"));
142&set_label("x86");
143}
144 &mov ("esi",&wparam(3)); # key
145 &mov ("edi",&wparam(4)); # counter and nonce
146
147 &stack_push(33);
148
149 &mov ("eax",&DWP(4*0,"esi")); # copy key
150 &mov ("ebx",&DWP(4*1,"esi"));
151 &mov ("ecx",&DWP(4*2,"esi"));
152 &mov ("edx",&DWP(4*3,"esi"));
153 &mov (&DWP(64+4*4,"esp"),"eax");
154 &mov (&DWP(64+4*5,"esp"),"ebx");
155 &mov (&DWP(64+4*6,"esp"),"ecx");
156 &mov (&DWP(64+4*7,"esp"),"edx");
157 &mov ("eax",&DWP(4*4,"esi"));
158 &mov ("ebx",&DWP(4*5,"esi"));
159 &mov ("ecx",&DWP(4*6,"esi"));
160 &mov ("edx",&DWP(4*7,"esi"));
161 &mov (&DWP(64+4*8,"esp"),"eax");
162 &mov (&DWP(64+4*9,"esp"),"ebx");
163 &mov (&DWP(64+4*10,"esp"),"ecx");
164 &mov (&DWP(64+4*11,"esp"),"edx");
165 &mov ("eax",&DWP(4*0,"edi")); # copy counter and nonce
166 &mov ("ebx",&DWP(4*1,"edi"));
167 &mov ("ecx",&DWP(4*2,"edi"));
168 &mov ("edx",&DWP(4*3,"edi"));
169 &sub ("eax",1);
170 &mov (&DWP(64+4*12,"esp"),"eax");
171 &mov (&DWP(64+4*13,"esp"),"ebx");
172 &mov (&DWP(64+4*14,"esp"),"ecx");
173 &mov (&DWP(64+4*15,"esp"),"edx");
174 &jmp (&label("entry"));
175
176&set_label("outer_loop",16);
177 &mov (&wparam(1),$b); # save input
178 &mov (&wparam(0),$a); # save output
179 &mov (&wparam(2),$c); # save len
180&set_label("entry");
181 &mov ($a,0x61707865);
182 &mov (&DWP(4*1,"esp"),0x3320646e);
183 &mov (&DWP(4*2,"esp"),0x79622d32);
184 &mov (&DWP(4*3,"esp"),0x6b206574);
185
186 &mov ($b, &DWP(64+4*5,"esp")); # copy key material
187 &mov ($b_,&DWP(64+4*6,"esp"));
188 &mov ($c, &DWP(64+4*10,"esp"));
189 &mov ($c_,&DWP(64+4*11,"esp"));
190 &mov ($d, &DWP(64+4*13,"esp"));
191 &mov ($d_,&DWP(64+4*14,"esp"));
192 &mov (&DWP(4*5,"esp"),$b);
193 &mov (&DWP(4*6,"esp"),$b_);
194 &mov (&DWP(4*10,"esp"),$c);
195 &mov (&DWP(4*11,"esp"),$c_);
196 &mov (&DWP(4*13,"esp"),$d);
197 &mov (&DWP(4*14,"esp"),$d_);
198
199 &mov ($b, &DWP(64+4*7,"esp"));
200 &mov ($d_,&DWP(64+4*15,"esp"));
201 &mov ($d, &DWP(64+4*12,"esp"));
202 &mov ($b_,&DWP(64+4*4,"esp"));
203 &mov ($c, &DWP(64+4*8,"esp"));
204 &mov ($c_,&DWP(64+4*9,"esp"));
205 &add ($d,1); # counter value
206 &mov (&DWP(4*7,"esp"),$b);
207 &mov (&DWP(4*15,"esp"),$d_);
208 &mov (&DWP(64+4*12,"esp"),$d); # save counter value
209
210 &mov ($b,10); # loop counter
211 &jmp (&label("loop"));
212
213&set_label("loop",16);
214 &add ($a,$b_); # elsewhere
215 &mov (&DWP(128,"esp"),$b); # save loop counter
216 &mov ($b,$b_);
217 &QUARTERROUND(0, 4, 8, 12, 0);
218 &QUARTERROUND(1, 5, 9, 13, 1);
219 &QUARTERROUND(2, 6,10, 14, 2);
220 &QUARTERROUND(3, 7,11, 15, 3);
221 &QUARTERROUND(0, 5,10, 15, 4);
222 &QUARTERROUND(1, 6,11, 12, 5);
223 &QUARTERROUND(2, 7, 8, 13, 6);
224 &QUARTERROUND(3, 4, 9, 14, 7);
225 &dec ($b);
226 &jnz (&label("loop"));
227
29880e97 228 &mov ($b,&wparam(2)); # load len
a98c648e
AP		 229
230 &add ($a,0x61707865); # accumulate key material
231 &add ($b_,&DWP(64+4*4,"esp"));
232 &add ($c, &DWP(64+4*8,"esp"));
233 &add ($c_,&DWP(64+4*9,"esp"));
234
235 &cmp ($b,64);
236 &jb (&label("tail"));
237
238 &mov ($b,&wparam(1)); # load input pointer
239 &add ($d, &DWP(64+4*12,"esp"));
240 &add ($d_,&DWP(64+4*14,"esp"));
241
242 &xor ($a, &DWP(4*0,$b)); # xor with input
243 &xor ($b_,&DWP(4*4,$b));
244 &mov (&DWP(4*0,"esp"),$a);
245 &mov ($a,&wparam(0)); # load output pointer
246 &xor ($c, &DWP(4*8,$b));
247 &xor ($c_,&DWP(4*9,$b));
248 &xor ($d, &DWP(4*12,$b));
249 &xor ($d_,&DWP(4*14,$b));
250 &mov (&DWP(4*4,$a),$b_); # write output
251 &mov (&DWP(4*8,$a),$c);
252 &mov (&DWP(4*9,$a),$c_);
253 &mov (&DWP(4*12,$a),$d);
254 &mov (&DWP(4*14,$a),$d_);
255
256 &mov ($b_,&DWP(4*1,"esp"));
257 &mov ($c, &DWP(4*2,"esp"));
258 &mov ($c_,&DWP(4*3,"esp"));
259 &mov ($d, &DWP(4*5,"esp"));
260 &mov ($d_,&DWP(4*6,"esp"));
261 &add ($b_,0x3320646e); # accumulate key material
262 &add ($c, 0x79622d32);
263 &add ($c_,0x6b206574);
264 &add ($d, &DWP(64+4*5,"esp"));
265 &add ($d_,&DWP(64+4*6,"esp"));
266 &xor ($b_,&DWP(4*1,$b));
267 &xor ($c, &DWP(4*2,$b));
268 &xor ($c_,&DWP(4*3,$b));
269 &xor ($d, &DWP(4*5,$b));
270 &xor ($d_,&DWP(4*6,$b));
271 &mov (&DWP(4*1,$a),$b_);
272 &mov (&DWP(4*2,$a),$c);
273 &mov (&DWP(4*3,$a),$c_);
274 &mov (&DWP(4*5,$a),$d);
275 &mov (&DWP(4*6,$a),$d_);
276
277 &mov ($b_,&DWP(4*7,"esp"));
278 &mov ($c, &DWP(4*10,"esp"));
279 &mov ($c_,&DWP(4*11,"esp"));
280 &mov ($d, &DWP(4*13,"esp"));
281 &mov ($d_,&DWP(4*15,"esp"));
282 &add ($b_,&DWP(64+4*7,"esp"));
283 &add ($c, &DWP(64+4*10,"esp"));
284 &add ($c_,&DWP(64+4*11,"esp"));
285 &add ($d, &DWP(64+4*13,"esp"));
286 &add ($d_,&DWP(64+4*15,"esp"));
287 &xor ($b_,&DWP(4*7,$b));
288 &xor ($c, &DWP(4*10,$b));
289 &xor ($c_,&DWP(4*11,$b));
290 &xor ($d, &DWP(4*13,$b));
291 &xor ($d_,&DWP(4*15,$b));
292 &lea ($b,&DWP(4*16,$b));
293 &mov (&DWP(4*7,$a),$b_);
294 &mov ($b_,&DWP(4*0,"esp"));
295 &mov (&DWP(4*10,$a),$c);
296 &mov ($c,&wparam(2)); # len
297 &mov (&DWP(4*11,$a),$c_);
298 &mov (&DWP(4*13,$a),$d);
299 &mov (&DWP(4*15,$a),$d_);
300 &mov (&DWP(4*0,$a),$b_);
301 &lea ($a,&DWP(4*16,$a));
302 &sub ($c,64);
303 &jnz (&label("outer_loop"));
304
305 &jmp (&label("done"));
306
307&set_label("tail");
308 &add ($d, &DWP(64+4*12,"esp"));
309 &add ($d_,&DWP(64+4*14,"esp"));
310 &mov (&DWP(4*0,"esp"),$a);
311 &mov (&DWP(4*4,"esp"),$b_);
312 &mov (&DWP(4*8,"esp"),$c);
313 &mov (&DWP(4*9,"esp"),$c_);
314 &mov (&DWP(4*12,"esp"),$d);
315 &mov (&DWP(4*14,"esp"),$d_);
316
317 &mov ($b_,&DWP(4*1,"esp"));
318 &mov ($c, &DWP(4*2,"esp"));
319 &mov ($c_,&DWP(4*3,"esp"));
320 &mov ($d, &DWP(4*5,"esp"));
321 &mov ($d_,&DWP(4*6,"esp"));
322 &add ($b_,0x3320646e); # accumulate key material
323 &add ($c, 0x79622d32);
324 &add ($c_,0x6b206574);
325 &add ($d, &DWP(64+4*5,"esp"));
326 &add ($d_,&DWP(64+4*6,"esp"));
327 &mov (&DWP(4*1,"esp"),$b_);
328 &mov (&DWP(4*2,"esp"),$c);
329 &mov (&DWP(4*3,"esp"),$c_);
330 &mov (&DWP(4*5,"esp"),$d);
331 &mov (&DWP(4*6,"esp"),$d_);
332
333 &mov ($b_,&DWP(4*7,"esp"));
334 &mov ($c, &DWP(4*10,"esp"));
335 &mov ($c_,&DWP(4*11,"esp"));
336 &mov ($d, &DWP(4*13,"esp"));
337 &mov ($d_,&DWP(4*15,"esp"));
338 &add ($b_,&DWP(64+4*7,"esp"));
339 &add ($c, &DWP(64+4*10,"esp"));
340 &add ($c_,&DWP(64+4*11,"esp"));
341 &add ($d, &DWP(64+4*13,"esp"));
342 &add ($d_,&DWP(64+4*15,"esp"));
343 &mov (&DWP(4*7,"esp"),$b_);
344 &mov ($b_,&wparam(1)); # load input
345 &mov (&DWP(4*10,"esp"),$c);
346 &mov ($c,&wparam(0)); # load output
347 &mov (&DWP(4*11,"esp"),$c_);
348 &xor ($c_,$c_);
349 &mov (&DWP(4*13,"esp"),$d);
350 &mov (&DWP(4*15,"esp"),$d_);
351
352 &xor ("eax","eax");
353 &xor ("edx","edx");
354&set_label("tail_loop");
b44a9641
AP		 355 &movb ("al",&BP(0,$c_,$b_));
356 &movb ("dl",&BP(0,"esp",$c_));
a98c648e
AP		 357 &lea ($c_,&DWP(1,$c_));
358 &xor ("al","dl");
b44a9641 359 &mov (&BP(-1,$c,$c_),"al");
a98c648e
AP		 360 &dec ($b);
361 &jnz (&label("tail_loop"));
362
363&set_label("done");
364 &stack_pop(33);
622a531c 365&set_label("no_data");
a98c648e
AP		 366&function_end("ChaCha20_ctr32");
367
368if ($xmm) {
369my ($xa,$xa_,$xb,$xb_,$xc,$xc_,$xd,$xd_)=map("xmm$_",(0..7));
370my ($out,$inp,$len)=("edi","esi","ecx");
371
372sub QUARTERROUND_SSSE3 {
373my ($ai,$bi,$ci,$di,$i)=@_;
374my ($an,$bn,$cn,$dn)=map(($_&~3)+(($_+1)&3),($ai,$bi,$ci,$di)); # next
375my ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_-1)&3),($ai,$bi,$ci,$di)); # previous
376
377 # a b c d
378 #
379 # 0 4 8 12 < even round
380 # 1 5 9 13
381 # 2 6 10 14
382 # 3 7 11 15
383 # 0 5 10 15 < odd round
384 # 1 6 11 12
385 # 2 7 8 13
386 # 3 4 9 14
387
388 if ($i==0) {
389 my $j=4;
390 ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_-$j--)&3),($ap,$bp,$cp,$dp));
391 } elsif ($i==3) {
392 my $j=0;
393 ($an,$bn,$cn,$dn)=map(($_&~3)+(($_+$j++)&3),($an,$bn,$cn,$dn));
394 } elsif ($i==4) {
395 my $j=4;
396 ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_+$j--)&3),($ap,$bp,$cp,$dp));
397 } elsif ($i==7) {
398 my $j=0;
399 ($an,$bn,$cn,$dn)=map(($_&~3)+(($_-$j++)&3),($an,$bn,$cn,$dn));
400 }
401
402 #&paddd ($xa,$xb); # see elsewhere
403 #&pxor ($xd,$xa); # see elsewhere
404 &movdqa(&QWP(16*$cp-128,"ebx"),$xc_) if ($ai>0 && $ai<3);
405 &pshufb ($xd,&QWP(0,"eax")); # rot16
406 &movdqa(&QWP(16*$bp-128,"ebx"),$xb_) if ($i!=0);
407 &paddd ($xc,$xd);
408 &movdqa($xc_,&QWP(16*$cn-128,"ebx")) if ($ai>0 && $ai<3);
409 &pxor ($xb,$xc);
410 &movdqa($xb_,&QWP(16*$bn-128,"ebx")) if ($i<7);
411 &movdqa ($xa_,$xb); # borrow as temporary
412 &pslld ($xb,12);
413 &psrld ($xa_,20);
414 &por ($xb,$xa_);
415 &movdqa($xa_,&QWP(16*$an-128,"ebx"));
416 &paddd ($xa,$xb);
417 &movdqa($xd_,&QWP(16*$dn-128,"ebx")) if ($di!=$dn);
418 &pxor ($xd,$xa);
419 &movdqa (&QWP(16*$ai-128,"ebx"),$xa);
420 &pshufb ($xd,&QWP(16,"eax")); # rot8
421 &paddd ($xc,$xd);
422 &movdqa (&QWP(16*$di-128,"ebx"),$xd) if ($di!=$dn);
423 &movdqa ($xd_,$xd) if ($di==$dn);
424 &pxor ($xb,$xc);
425 &paddd ($xa_,$xb_) if ($i<7); # elsewhere
426 &movdqa ($xa,$xb); # borrow as temporary
427 &pslld ($xb,7);
428 &psrld ($xa,25);
429 &pxor ($xd_,$xa_) if ($i<7); # elsewhere
430 &por ($xb,$xa);
431
432 ($xa,$xa_)=($xa_,$xa);
433 ($xb,$xb_)=($xb_,$xb);
434 ($xc,$xc_)=($xc_,$xc);
435 ($xd,$xd_)=($xd_,$xd);
436}
437
438&function_begin("ChaCha20_ssse3");
439&set_label("ssse3_shortcut");
440 &test (&DWP(4,"ebp"),1<<11); # test XOP bit
441 &jnz (&label("xop_shortcut"));
442
443 &mov ($out,&wparam(0));
444 &mov ($inp,&wparam(1));
445 &mov ($len,&wparam(2));
446 &mov ("edx",&wparam(3)); # key
447 &mov ("ebx",&wparam(4)); # counter and nonce
448
449 &mov ("ebp","esp");
450 &stack_push (131);
451 &and ("esp",-64);
452 &mov (&DWP(512,"esp"),"ebp");
453
454 &lea ("eax",&DWP(&label("ssse3_data")."-".
455 &label("pic_point"),"eax"));
456 &movdqu ("xmm3",&QWP(0,"ebx")); # counter and nonce
457
458 &cmp ($len,64*4);
459 &jb (&label("1x"));
460
461 &mov (&DWP(512+4,"esp"),"edx"); # offload pointers
462 &mov (&DWP(512+8,"esp"),"ebx");
463 &sub ($len,64*4); # bias len
464 &lea ("ebp",&DWP(256+128,"esp")); # size optimization
465
b44a9641 466 &movdqu ("xmm7",&QWP(0,"edx")); # key
a98c648e
AP		 467 &pshufd ("xmm0","xmm3",0x00);
468 &pshufd ("xmm1","xmm3",0x55);
469 &pshufd ("xmm2","xmm3",0xaa);
470 &pshufd ("xmm3","xmm3",0xff);
471 &paddd ("xmm0",&QWP(16*3,"eax")); # fix counters
472 &pshufd ("xmm4","xmm7",0x00);
473 &pshufd ("xmm5","xmm7",0x55);
474 &psubd ("xmm0",&QWP(16*4,"eax"));
475 &pshufd ("xmm6","xmm7",0xaa);
476 &pshufd ("xmm7","xmm7",0xff);
477 &movdqa (&QWP(16*12-128,"ebp"),"xmm0");
478 &movdqa (&QWP(16*13-128,"ebp"),"xmm1");
479 &movdqa (&QWP(16*14-128,"ebp"),"xmm2");
480 &movdqa (&QWP(16*15-128,"ebp"),"xmm3");
b44a9641 481 &movdqu ("xmm3",&QWP(16,"edx")); # key
a98c648e
AP		 482 &movdqa (&QWP(16*4-128,"ebp"),"xmm4");
483 &movdqa (&QWP(16*5-128,"ebp"),"xmm5");
484 &movdqa (&QWP(16*6-128,"ebp"),"xmm6");
485 &movdqa (&QWP(16*7-128,"ebp"),"xmm7");
b44a9641 486 &movdqa ("xmm7",&QWP(16*2,"eax")); # sigma
a98c648e
AP		 487 &lea ("ebx",&DWP(128,"esp")); # size optimization
488
489 &pshufd ("xmm0","xmm3",0x00);
490 &pshufd ("xmm1","xmm3",0x55);
491 &pshufd ("xmm2","xmm3",0xaa);
492 &pshufd ("xmm3","xmm3",0xff);
493 &pshufd ("xmm4","xmm7",0x00);
494 &pshufd ("xmm5","xmm7",0x55);
495 &pshufd ("xmm6","xmm7",0xaa);
496 &pshufd ("xmm7","xmm7",0xff);
497 &movdqa (&QWP(16*8-128,"ebp"),"xmm0");
498 &movdqa (&QWP(16*9-128,"ebp"),"xmm1");
499 &movdqa (&QWP(16*10-128,"ebp"),"xmm2");
500 &movdqa (&QWP(16*11-128,"ebp"),"xmm3");
501 &movdqa (&QWP(16*0-128,"ebp"),"xmm4");
502 &movdqa (&QWP(16*1-128,"ebp"),"xmm5");
503 &movdqa (&QWP(16*2-128,"ebp"),"xmm6");
504 &movdqa (&QWP(16*3-128,"ebp"),"xmm7");
505
506 &lea ($inp,&DWP(128,$inp)); # size optimization
507 &lea ($out,&DWP(128,$out)); # size optimization
508 &jmp (&label("outer_loop"));
509
510&set_label("outer_loop",16);
511 #&movdqa ("xmm0",&QWP(16*0-128,"ebp")); # copy key material
512 &movdqa ("xmm1",&QWP(16*1-128,"ebp"));
513 &movdqa ("xmm2",&QWP(16*2-128,"ebp"));
514 &movdqa ("xmm3",&QWP(16*3-128,"ebp"));
515 #&movdqa ("xmm4",&QWP(16*4-128,"ebp"));
516 &movdqa ("xmm5",&QWP(16*5-128,"ebp"));
517 &movdqa ("xmm6",&QWP(16*6-128,"ebp"));
518 &movdqa ("xmm7",&QWP(16*7-128,"ebp"));
519 #&movdqa (&QWP(16*0-128,"ebx"),"xmm0");
520 &movdqa (&QWP(16*1-128,"ebx"),"xmm1");
521 &movdqa (&QWP(16*2-128,"ebx"),"xmm2");
522 &movdqa (&QWP(16*3-128,"ebx"),"xmm3");
523 #&movdqa (&QWP(16*4-128,"ebx"),"xmm4");
524 &movdqa (&QWP(16*5-128,"ebx"),"xmm5");
525 &movdqa (&QWP(16*6-128,"ebx"),"xmm6");
526 &movdqa (&QWP(16*7-128,"ebx"),"xmm7");
527 #&movdqa ("xmm0",&QWP(16*8-128,"ebp"));
528 #&movdqa ("xmm1",&QWP(16*9-128,"ebp"));
529 &movdqa ("xmm2",&QWP(16*10-128,"ebp"));
530 &movdqa ("xmm3",&QWP(16*11-128,"ebp"));
531 &movdqa ("xmm4",&QWP(16*12-128,"ebp"));
532 &movdqa ("xmm5",&QWP(16*13-128,"ebp"));
533 &movdqa ("xmm6",&QWP(16*14-128,"ebp"));
534 &movdqa ("xmm7",&QWP(16*15-128,"ebp"));
535 &paddd ("xmm4",&QWP(16*4,"eax")); # counter value
536 #&movdqa (&QWP(16*8-128,"ebx"),"xmm0");
537 #&movdqa (&QWP(16*9-128,"ebx"),"xmm1");
538 &movdqa (&QWP(16*10-128,"ebx"),"xmm2");
539 &movdqa (&QWP(16*11-128,"ebx"),"xmm3");
540 &movdqa (&QWP(16*12-128,"ebx"),"xmm4");
541 &movdqa (&QWP(16*13-128,"ebx"),"xmm5");
542 &movdqa (&QWP(16*14-128,"ebx"),"xmm6");
543 &movdqa (&QWP(16*15-128,"ebx"),"xmm7");
544 &movdqa (&QWP(16*12-128,"ebp"),"xmm4"); # save counter value
545
546 &movdqa ($xa, &QWP(16*0-128,"ebp"));
547 &movdqa ($xd, "xmm4");
548 &movdqa ($xb_,&QWP(16*4-128,"ebp"));
549 &movdqa ($xc, &QWP(16*8-128,"ebp"));
550 &movdqa ($xc_,&QWP(16*9-128,"ebp"));
551
552 &mov ("edx",10); # loop counter
553 &nop ();
554
555&set_label("loop",16);
556 &paddd ($xa,$xb_); # elsewhere
557 &movdqa ($xb,$xb_);
558 &pxor ($xd,$xa); # elsewhere
559 &QUARTERROUND_SSSE3(0, 4, 8, 12, 0);
560 &QUARTERROUND_SSSE3(1, 5, 9, 13, 1);
561 &QUARTERROUND_SSSE3(2, 6,10, 14, 2);
562 &QUARTERROUND_SSSE3(3, 7,11, 15, 3);
563 &QUARTERROUND_SSSE3(0, 5,10, 15, 4);
564 &QUARTERROUND_SSSE3(1, 6,11, 12, 5);
565 &QUARTERROUND_SSSE3(2, 7, 8, 13, 6);
566 &QUARTERROUND_SSSE3(3, 4, 9, 14, 7);
567 &dec ("edx");
568 &jnz (&label("loop"));
569
570 &movdqa (&QWP(16*4-128,"ebx"),$xb_);
571 &movdqa (&QWP(16*8-128,"ebx"),$xc);
572 &movdqa (&QWP(16*9-128,"ebx"),$xc_);
573 &movdqa (&QWP(16*12-128,"ebx"),$xd);
574 &movdqa (&QWP(16*14-128,"ebx"),$xd_);
575
576 my ($xa0,$xa1,$xa2,$xa3,$xt0,$xt1,$xt2,$xt3)=map("xmm$_",(0..7));
577
578 #&movdqa ($xa0,&QWP(16*0-128,"ebx")); # it's there
579 &movdqa ($xa1,&QWP(16*1-128,"ebx"));
580 &movdqa ($xa2,&QWP(16*2-128,"ebx"));
581 &movdqa ($xa3,&QWP(16*3-128,"ebx"));
582
583 for($i=0;$i<256;$i+=64) {
584 &paddd ($xa0,&QWP($i+16*0-128,"ebp")); # accumulate key material
585 &paddd ($xa1,&QWP($i+16*1-128,"ebp"));
586 &paddd ($xa2,&QWP($i+16*2-128,"ebp"));
587 &paddd ($xa3,&QWP($i+16*3-128,"ebp"));
588
589 &movdqa ($xt2,$xa0); # "de-interlace" data
590 &punpckldq ($xa0,$xa1);
591 &movdqa ($xt3,$xa2);
592 &punpckldq ($xa2,$xa3);
593 &punpckhdq ($xt2,$xa1);
594 &punpckhdq ($xt3,$xa3);
595 &movdqa ($xa1,$xa0);
596 &punpcklqdq ($xa0,$xa2); # "a0"
597 &movdqa ($xa3,$xt2);
598 &punpcklqdq ($xt2,$xt3); # "a2"
599 &punpckhqdq ($xa1,$xa2); # "a1"
600 &punpckhqdq ($xa3,$xt3); # "a3"
601
602 #($xa2,$xt2)=($xt2,$xa2);
603
604 &movdqu ($xt0,&QWP(64*0-128,$inp)); # load input
605 &movdqu ($xt1,&QWP(64*1-128,$inp));
606 &movdqu ($xa2,&QWP(64*2-128,$inp));
607 &movdqu ($xt3,&QWP(64*3-128,$inp));
608 &lea ($inp,&QWP($i<192?16:(64*4-16*3),$inp));
609 &pxor ($xt0,$xa0);
610 &movdqa ($xa0,&QWP($i+16*4-128,"ebx")) if ($i<192);
611 &pxor ($xt1,$xa1);
612 &movdqa ($xa1,&QWP($i+16*5-128,"ebx")) if ($i<192);
613 &pxor ($xt2,$xa2);
614 &movdqa ($xa2,&QWP($i+16*6-128,"ebx")) if ($i<192);
615 &pxor ($xt3,$xa3);
616 &movdqa ($xa3,&QWP($i+16*7-128,"ebx")) if ($i<192);
617 &movdqu (&QWP(64*0-128,$out),$xt0); # store output
618 &movdqu (&QWP(64*1-128,$out),$xt1);
619 &movdqu (&QWP(64*2-128,$out),$xt2);
620 &movdqu (&QWP(64*3-128,$out),$xt3);
621 &lea ($out,&QWP($i<192?16:(64*4-16*3),$out));
622 }
623 &sub ($len,64*4);
624 &jnc (&label("outer_loop"));
625
626 &add ($len,64*4);
627 &jz (&label("done"));
628
629 &mov ("ebx",&DWP(512+8,"esp")); # restore pointers
630 &lea ($inp,&DWP(-128,$inp));
631 &mov ("edx",&DWP(512+4,"esp"));
632 &lea ($out,&DWP(-128,$out));
633
634 &movd ("xmm2",&DWP(16*12-128,"ebp")); # counter value
635 &movdqu ("xmm3",&QWP(0,"ebx"));
636 &paddd ("xmm2",&QWP(16*6,"eax")); # +four
637 &pand ("xmm3",&QWP(16*7,"eax"));
638 &por ("xmm3","xmm2"); # counter value
639{
b44a9641 640my ($a,$b,$c,$d,$t,$t1,$rot16,$rot24)=map("xmm$_",(0..7));
a98c648e
AP		 641
642sub SSSE3ROUND { # critical path is 20 "SIMD ticks" per round
643 &paddd ($a,$b);
644 &pxor ($d,$a);
645 &pshufb ($d,$rot16);
646
647 &paddd ($c,$d);
648 &pxor ($b,$c);
649 &movdqa ($t,$b);
650 &psrld ($b,20);
651 &pslld ($t,12);
652 &por ($b,$t);
653
654 &paddd ($a,$b);
655 &pxor ($d,$a);
656 &pshufb ($d,$rot24);
657
658 &paddd ($c,$d);
659 &pxor ($b,$c);
660 &movdqa ($t,$b);
661 &psrld ($b,25);
662 &pslld ($t,7);
663 &por ($b,$t);
664}
665
666&set_label("1x");
667 &movdqa ($a,&QWP(16*2,"eax")); # sigma
668 &movdqu ($b,&QWP(0,"edx"));
669 &movdqu ($c,&QWP(16,"edx"));
670 #&movdqu ($d,&QWP(0,"ebx")); # already loaded
671 &movdqa ($rot16,&QWP(0,"eax"));
672 &movdqa ($rot24,&QWP(16,"eax"));
673 &mov (&DWP(16*3,"esp"),"ebp");
674
675 &movdqa (&QWP(16*0,"esp"),$a);
676 &movdqa (&QWP(16*1,"esp"),$b);
677 &movdqa (&QWP(16*2,"esp"),$c);
678 &movdqa (&QWP(16*3,"esp"),$d);
679 &mov ("edx",10);
680 &jmp (&label("loop1x"));
681
682&set_label("outer1x",16);
683 &movdqa ($d,&QWP(16*5,"eax")); # one
684 &movdqa ($a,&QWP(16*0,"esp"));
685 &movdqa ($b,&QWP(16*1,"esp"));
686 &movdqa ($c,&QWP(16*2,"esp"));
687 &paddd ($d,&QWP(16*3,"esp"));
688 &mov ("edx",10);
689 &movdqa (&QWP(16*3,"esp"),$d);
690 &jmp (&label("loop1x"));
691
692&set_label("loop1x",16);
693 &SSSE3ROUND();
694 &pshufd ($c,$c,0b01001110);
695 &pshufd ($b,$b,0b00111001);
696 &pshufd ($d,$d,0b10010011);
697 &nop ();
698
699 &SSSE3ROUND();
700 &pshufd ($c,$c,0b01001110);
701 &pshufd ($b,$b,0b10010011);
702 &pshufd ($d,$d,0b00111001);
703
704 &dec ("edx");
705 &jnz (&label("loop1x"));
706
707 &paddd ($a,&QWP(16*0,"esp"));
708 &paddd ($b,&QWP(16*1,"esp"));
709 &paddd ($c,&QWP(16*2,"esp"));
710 &paddd ($d,&QWP(16*3,"esp"));
711
712 &cmp ($len,64);
713 &jb (&label("tail"));
714
715 &movdqu ($t,&QWP(16*0,$inp));
716 &movdqu ($t1,&QWP(16*1,$inp));
717 &pxor ($a,$t); # xor with input
718 &movdqu ($t,&QWP(16*2,$inp));
719 &pxor ($b,$t1);
720 &movdqu ($t1,&QWP(16*3,$inp));
721 &pxor ($c,$t);
722 &pxor ($d,$t1);
723 &lea ($inp,&DWP(16*4,$inp)); # inp+=64
724
725 &movdqu (&QWP(16*0,$out),$a); # write output
726 &movdqu (&QWP(16*1,$out),$b);
727 &movdqu (&QWP(16*2,$out),$c);
728 &movdqu (&QWP(16*3,$out),$d);
729 &lea ($out,&DWP(16*4,$out)); # inp+=64
730
731 &sub ($len,64);
732 &jnz (&label("outer1x"));
733
734 &jmp (&label("done"));
735
736&set_label("tail");
737 &movdqa (&QWP(16*0,"esp"),$a);
738 &movdqa (&QWP(16*1,"esp"),$b);
739 &movdqa (&QWP(16*2,"esp"),$c);
740 &movdqa (&QWP(16*3,"esp"),$d);
741
742 &xor ("eax","eax");
743 &xor ("edx","edx");
744 &xor ("ebp","ebp");
745
746&set_label("tail_loop");
747 &movb ("al",&BP(0,"esp","ebp"));
748 &movb ("dl",&BP(0,$inp,"ebp"));
749 &lea ("ebp",&DWP(1,"ebp"));
750 &xor ("al","dl");
751 &movb (&BP(-1,$out,"ebp"),"al");
752 &dec ($len);
753 &jnz (&label("tail_loop"));
754}
755&set_label("done");
756 &mov ("esp",&DWP(512,"esp"));
757&function_end("ChaCha20_ssse3");
758
759&align (64);
760&set_label("ssse3_data");
761&data_byte(0x2,0x3,0x0,0x1, 0x6,0x7,0x4,0x5, 0xa,0xb,0x8,0x9, 0xe,0xf,0xc,0xd);
762&data_byte(0x3,0x0,0x1,0x2, 0x7,0x4,0x5,0x6, 0xb,0x8,0x9,0xa, 0xf,0xc,0xd,0xe);
763&data_word(0x61707865,0x3320646e,0x79622d32,0x6b206574);
764&data_word(0,1,2,3);
765&data_word(4,4,4,4);
766&data_word(1,0,0,0);
767&data_word(4,0,0,0);
768&data_word(0,-1,-1,-1);
769&align (64);
770}
771&asciz ("ChaCha20 for x86, CRYPTOGAMS by <appro\@openssl.org>");
772
773if ($xmm) {
774my ($xa,$xa_,$xb,$xb_,$xc,$xc_,$xd,$xd_)=map("xmm$_",(0..7));
775my ($out,$inp,$len)=("edi","esi","ecx");
776
777sub QUARTERROUND_XOP {
778my ($ai,$bi,$ci,$di,$i)=@_;
779my ($an,$bn,$cn,$dn)=map(($_&~3)+(($_+1)&3),($ai,$bi,$ci,$di)); # next
780my ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_-1)&3),($ai,$bi,$ci,$di)); # previous
781
782 # a b c d
783 #
784 # 0 4 8 12 < even round
785 # 1 5 9 13
786 # 2 6 10 14
787 # 3 7 11 15
788 # 0 5 10 15 < odd round
789 # 1 6 11 12
790 # 2 7 8 13
791 # 3 4 9 14
792
793 if ($i==0) {
794 my $j=4;
795 ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_-$j--)&3),($ap,$bp,$cp,$dp));
796 } elsif ($i==3) {
797 my $j=0;
798 ($an,$bn,$cn,$dn)=map(($_&~3)+(($_+$j++)&3),($an,$bn,$cn,$dn));
799 } elsif ($i==4) {
800 my $j=4;
801 ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_+$j--)&3),($ap,$bp,$cp,$dp));
802 } elsif ($i==7) {
803 my $j=0;
804 ($an,$bn,$cn,$dn)=map(($_&~3)+(($_-$j++)&3),($an,$bn,$cn,$dn));
805 }
806
807 #&vpaddd ($xa,$xa,$xb); # see elsewhere
808 #&vpxor ($xd,$xd,$xa); # see elsewhere
809 &vmovdqa (&QWP(16*$cp-128,"ebx"),$xc_) if ($ai>0 && $ai<3);
810 &vprotd ($xd,$xd,16);
811 &vmovdqa (&QWP(16*$bp-128,"ebx"),$xb_) if ($i!=0);
812 &vpaddd ($xc,$xc,$xd);
813 &vmovdqa ($xc_,&QWP(16*$cn-128,"ebx")) if ($ai>0 && $ai<3);
814 &vpxor ($xb,$i!=0?$xb:$xb_,$xc);
815 &vmovdqa ($xa_,&QWP(16*$an-128,"ebx"));
816 &vprotd ($xb,$xb,12);
817 &vmovdqa ($xb_,&QWP(16*$bn-128,"ebx")) if ($i<7);
818 &vpaddd ($xa,$xa,$xb);
819 &vmovdqa ($xd_,&QWP(16*$dn-128,"ebx")) if ($di!=$dn);
820 &vpxor ($xd,$xd,$xa);
821 &vpaddd ($xa_,$xa_,$xb_) if ($i<7); # elsewhere
822 &vprotd ($xd,$xd,8);
823 &vmovdqa (&QWP(16*$ai-128,"ebx"),$xa);
824 &vpaddd ($xc,$xc,$xd);
825 &vmovdqa (&QWP(16*$di-128,"ebx"),$xd) if ($di!=$dn);
826 &vpxor ($xb,$xb,$xc);
827 &vpxor ($xd_,$di==$dn?$xd:$xd_,$xa_) if ($i<7); # elsewhere
828 &vprotd ($xb,$xb,7);
829
830 ($xa,$xa_)=($xa_,$xa);
831 ($xb,$xb_)=($xb_,$xb);
832 ($xc,$xc_)=($xc_,$xc);
833 ($xd,$xd_)=($xd_,$xd);
834}
835
836&function_begin("ChaCha20_xop");
837&set_label("xop_shortcut");
838 &mov ($out,&wparam(0));
839 &mov ($inp,&wparam(1));
840 &mov ($len,&wparam(2));
841 &mov ("edx",&wparam(3)); # key
842 &mov ("ebx",&wparam(4)); # counter and nonce
843 &vzeroupper ();
844
845 &mov ("ebp","esp");
846 &stack_push (131);
847 &and ("esp",-64);
848 &mov (&DWP(512,"esp"),"ebp");
849
850 &lea ("eax",&DWP(&label("ssse3_data")."-".
851 &label("pic_point"),"eax"));
852 &vmovdqu ("xmm3",&QWP(0,"ebx")); # counter and nonce
853
854 &cmp ($len,64*4);
855 &jb (&label("1x"));
856
857 &mov (&DWP(512+4,"esp"),"edx"); # offload pointers
858 &mov (&DWP(512+8,"esp"),"ebx");
859 &sub ($len,64*4); # bias len
860 &lea ("ebp",&DWP(256+128,"esp")); # size optimization
861
b44a9641 862 &vmovdqu ("xmm7",&QWP(0,"edx")); # key
a98c648e
AP		 863 &vpshufd ("xmm0","xmm3",0x00);
864 &vpshufd ("xmm1","xmm3",0x55);
865 &vpshufd ("xmm2","xmm3",0xaa);
866 &vpshufd ("xmm3","xmm3",0xff);
867 &vpaddd ("xmm0","xmm0",&QWP(16*3,"eax")); # fix counters
868 &vpshufd ("xmm4","xmm7",0x00);
869 &vpshufd ("xmm5","xmm7",0x55);
870 &vpsubd ("xmm0","xmm0",&QWP(16*4,"eax"));
871 &vpshufd ("xmm6","xmm7",0xaa);
872 &vpshufd ("xmm7","xmm7",0xff);
873 &vmovdqa (&QWP(16*12-128,"ebp"),"xmm0");
874 &vmovdqa (&QWP(16*13-128,"ebp"),"xmm1");
875 &vmovdqa (&QWP(16*14-128,"ebp"),"xmm2");
876 &vmovdqa (&QWP(16*15-128,"ebp"),"xmm3");
b44a9641 877 &vmovdqu ("xmm3",&QWP(16,"edx")); # key
a98c648e
AP		 878 &vmovdqa (&QWP(16*4-128,"ebp"),"xmm4");
879 &vmovdqa (&QWP(16*5-128,"ebp"),"xmm5");
880 &vmovdqa (&QWP(16*6-128,"ebp"),"xmm6");
881 &vmovdqa (&QWP(16*7-128,"ebp"),"xmm7");
b44a9641 882 &vmovdqa ("xmm7",&QWP(16*2,"eax")); # sigma
a98c648e
AP		 883 &lea ("ebx",&DWP(128,"esp")); # size optimization
884
885 &vpshufd ("xmm0","xmm3",0x00);
886 &vpshufd ("xmm1","xmm3",0x55);
887 &vpshufd ("xmm2","xmm3",0xaa);
888 &vpshufd ("xmm3","xmm3",0xff);
889 &vpshufd ("xmm4","xmm7",0x00);
890 &vpshufd ("xmm5","xmm7",0x55);
891 &vpshufd ("xmm6","xmm7",0xaa);
892 &vpshufd ("xmm7","xmm7",0xff);
893 &vmovdqa (&QWP(16*8-128,"ebp"),"xmm0");
894 &vmovdqa (&QWP(16*9-128,"ebp"),"xmm1");
895 &vmovdqa (&QWP(16*10-128,"ebp"),"xmm2");
896 &vmovdqa (&QWP(16*11-128,"ebp"),"xmm3");
897 &vmovdqa (&QWP(16*0-128,"ebp"),"xmm4");
898 &vmovdqa (&QWP(16*1-128,"ebp"),"xmm5");
899 &vmovdqa (&QWP(16*2-128,"ebp"),"xmm6");
900 &vmovdqa (&QWP(16*3-128,"ebp"),"xmm7");
901
902 &lea ($inp,&DWP(128,$inp)); # size optimization
903 &lea ($out,&DWP(128,$out)); # size optimization
904 &jmp (&label("outer_loop"));
905
906&set_label("outer_loop",32);
907 #&vmovdqa ("xmm0",&QWP(16*0-128,"ebp")); # copy key material
908 &vmovdqa ("xmm1",&QWP(16*1-128,"ebp"));
909 &vmovdqa ("xmm2",&QWP(16*2-128,"ebp"));
910 &vmovdqa ("xmm3",&QWP(16*3-128,"ebp"));
911 #&vmovdqa ("xmm4",&QWP(16*4-128,"ebp"));
912 &vmovdqa ("xmm5",&QWP(16*5-128,"ebp"));
913 &vmovdqa ("xmm6",&QWP(16*6-128,"ebp"));
914 &vmovdqa ("xmm7",&QWP(16*7-128,"ebp"));
915 #&vmovdqa (&QWP(16*0-128,"ebx"),"xmm0");
916 &vmovdqa (&QWP(16*1-128,"ebx"),"xmm1");
917 &vmovdqa (&QWP(16*2-128,"ebx"),"xmm2");
918 &vmovdqa (&QWP(16*3-128,"ebx"),"xmm3");
919 #&vmovdqa (&QWP(16*4-128,"ebx"),"xmm4");
920 &vmovdqa (&QWP(16*5-128,"ebx"),"xmm5");
921 &vmovdqa (&QWP(16*6-128,"ebx"),"xmm6");
922 &vmovdqa (&QWP(16*7-128,"ebx"),"xmm7");
923 #&vmovdqa ("xmm0",&QWP(16*8-128,"ebp"));
924 #&vmovdqa ("xmm1",&QWP(16*9-128,"ebp"));
925 &vmovdqa ("xmm2",&QWP(16*10-128,"ebp"));
926 &vmovdqa ("xmm3",&QWP(16*11-128,"ebp"));
927 &vmovdqa ("xmm4",&QWP(16*12-128,"ebp"));
928 &vmovdqa ("xmm5",&QWP(16*13-128,"ebp"));
929 &vmovdqa ("xmm6",&QWP(16*14-128,"ebp"));
930 &vmovdqa ("xmm7",&QWP(16*15-128,"ebp"));
931 &vpaddd ("xmm4","xmm4",&QWP(16*4,"eax")); # counter value
932 #&vmovdqa (&QWP(16*8-128,"ebx"),"xmm0");
933 #&vmovdqa (&QWP(16*9-128,"ebx"),"xmm1");
934 &vmovdqa (&QWP(16*10-128,"ebx"),"xmm2");
935 &vmovdqa (&QWP(16*11-128,"ebx"),"xmm3");
936 &vmovdqa (&QWP(16*12-128,"ebx"),"xmm4");
937 &vmovdqa (&QWP(16*13-128,"ebx"),"xmm5");
938 &vmovdqa (&QWP(16*14-128,"ebx"),"xmm6");
939 &vmovdqa (&QWP(16*15-128,"ebx"),"xmm7");
940 &vmovdqa (&QWP(16*12-128,"ebp"),"xmm4"); # save counter value
941
942 &vmovdqa ($xa, &QWP(16*0-128,"ebp"));
943 &vmovdqa ($xd, "xmm4");
944 &vmovdqa ($xb_,&QWP(16*4-128,"ebp"));
945 &vmovdqa ($xc, &QWP(16*8-128,"ebp"));
946 &vmovdqa ($xc_,&QWP(16*9-128,"ebp"));
947
948 &mov ("edx",10); # loop counter
949 &nop ();
950
951&set_label("loop",32);
952 &vpaddd ($xa,$xa,$xb_); # elsewhere
953 &vpxor ($xd,$xd,$xa); # elsewhere
954 &QUARTERROUND_XOP(0, 4, 8, 12, 0);
955 &QUARTERROUND_XOP(1, 5, 9, 13, 1);
956 &QUARTERROUND_XOP(2, 6,10, 14, 2);
957 &QUARTERROUND_XOP(3, 7,11, 15, 3);
958 &QUARTERROUND_XOP(0, 5,10, 15, 4);
959 &QUARTERROUND_XOP(1, 6,11, 12, 5);
960 &QUARTERROUND_XOP(2, 7, 8, 13, 6);
961 &QUARTERROUND_XOP(3, 4, 9, 14, 7);
962 &dec ("edx");
963 &jnz (&label("loop"));
964
965 &vmovdqa (&QWP(16*4-128,"ebx"),$xb_);
966 &vmovdqa (&QWP(16*8-128,"ebx"),$xc);
967 &vmovdqa (&QWP(16*9-128,"ebx"),$xc_);
968 &vmovdqa (&QWP(16*12-128,"ebx"),$xd);
969 &vmovdqa (&QWP(16*14-128,"ebx"),$xd_);
970
971 my ($xa0,$xa1,$xa2,$xa3,$xt0,$xt1,$xt2,$xt3)=map("xmm$_",(0..7));
972
973 #&vmovdqa ($xa0,&QWP(16*0-128,"ebx")); # it's there
974 &vmovdqa ($xa1,&QWP(16*1-128,"ebx"));
975 &vmovdqa ($xa2,&QWP(16*2-128,"ebx"));
976 &vmovdqa ($xa3,&QWP(16*3-128,"ebx"));
977
978 for($i=0;$i<256;$i+=64) {
979 &vpaddd ($xa0,$xa0,&QWP($i+16*0-128,"ebp")); # accumulate key material
980 &vpaddd ($xa1,$xa1,&QWP($i+16*1-128,"ebp"));
981 &vpaddd ($xa2,$xa2,&QWP($i+16*2-128,"ebp"));
982 &vpaddd ($xa3,$xa3,&QWP($i+16*3-128,"ebp"));
983
984 &vpunpckldq ($xt2,$xa0,$xa1); # "de-interlace" data
985 &vpunpckldq ($xt3,$xa2,$xa3);
986 &vpunpckhdq ($xa0,$xa0,$xa1);
987 &vpunpckhdq ($xa2,$xa2,$xa3);
988 &vpunpcklqdq ($xa1,$xt2,$xt3); # "a0"
989 &vpunpckhqdq ($xt2,$xt2,$xt3); # "a1"
990 &vpunpcklqdq ($xt3,$xa0,$xa2); # "a2"
991 &vpunpckhqdq ($xa3,$xa0,$xa2); # "a3"
992
993 &vpxor ($xt0,$xa1,&QWP(64*0-128,$inp));
994 &vpxor ($xt1,$xt2,&QWP(64*1-128,$inp));
995 &vpxor ($xt2,$xt3,&QWP(64*2-128,$inp));
996 &vpxor ($xt3,$xa3,&QWP(64*3-128,$inp));
997 &lea ($inp,&QWP($i<192?16:(64*4-16*3),$inp));
998 &vmovdqa ($xa0,&QWP($i+16*4-128,"ebx")) if ($i<192);
999 &vmovdqa ($xa1,&QWP($i+16*5-128,"ebx")) if ($i<192);
1000 &vmovdqa ($xa2,&QWP($i+16*6-128,"ebx")) if ($i<192);
1001 &vmovdqa ($xa3,&QWP($i+16*7-128,"ebx")) if ($i<192);
1002 &vmovdqu (&QWP(64*0-128,$out),$xt0); # store output
1003 &vmovdqu (&QWP(64*1-128,$out),$xt1);
1004 &vmovdqu (&QWP(64*2-128,$out),$xt2);
1005 &vmovdqu (&QWP(64*3-128,$out),$xt3);
1006 &lea ($out,&QWP($i<192?16:(64*4-16*3),$out));
1007 }
1008 &sub ($len,64*4);
1009 &jnc (&label("outer_loop"));
1010
1011 &add ($len,64*4);
1012 &jz (&label("done"));
1013
1014 &mov ("ebx",&DWP(512+8,"esp")); # restore pointers
1015 &lea ($inp,&DWP(-128,$inp));
1016 &mov ("edx",&DWP(512+4,"esp"));
1017 &lea ($out,&DWP(-128,$out));
1018
1019 &vmovd ("xmm2",&DWP(16*12-128,"ebp")); # counter value
1020 &vmovdqu ("xmm3",&QWP(0,"ebx"));
1021 &vpaddd ("xmm2","xmm2",&QWP(16*6,"eax"));# +four
1022 &vpand ("xmm3","xmm3",&QWP(16*7,"eax"));
1023 &vpor ("xmm3","xmm3","xmm2"); # counter value
1024{
b44a9641 1025my ($a,$b,$c,$d,$t,$t1,$rot16,$rot24)=map("xmm$_",(0..7));
a98c648e
AP		 1026
1027sub XOPROUND {
1028 &vpaddd ($a,$a,$b);
1029 &vpxor ($d,$d,$a);
1030 &vprotd ($d,$d,16);
1031
1032 &vpaddd ($c,$c,$d);
1033 &vpxor ($b,$b,$c);
1034 &vprotd ($b,$b,12);
1035
1036 &vpaddd ($a,$a,$b);
1037 &vpxor ($d,$d,$a);
1038 &vprotd ($d,$d,8);
1039
1040 &vpaddd ($c,$c,$d);
1041 &vpxor ($b,$b,$c);
1042 &vprotd ($b,$b,7);
1043}
1044
1045&set_label("1x");
1046 &vmovdqa ($a,&QWP(16*2,"eax")); # sigma
1047 &vmovdqu ($b,&QWP(0,"edx"));
1048 &vmovdqu ($c,&QWP(16,"edx"));
1049 #&vmovdqu ($d,&QWP(0,"ebx")); # already loaded
1050 &vmovdqa ($rot16,&QWP(0,"eax"));
1051 &vmovdqa ($rot24,&QWP(16,"eax"));
1052 &mov (&DWP(16*3,"esp"),"ebp");
1053
1054 &vmovdqa (&QWP(16*0,"esp"),$a);
1055 &vmovdqa (&QWP(16*1,"esp"),$b);
1056 &vmovdqa (&QWP(16*2,"esp"),$c);
1057 &vmovdqa (&QWP(16*3,"esp"),$d);
1058 &mov ("edx",10);
1059 &jmp (&label("loop1x"));
1060
1061&set_label("outer1x",16);
1062 &vmovdqa ($d,&QWP(16*5,"eax")); # one
1063 &vmovdqa ($a,&QWP(16*0,"esp"));
1064 &vmovdqa ($b,&QWP(16*1,"esp"));
1065 &vmovdqa ($c,&QWP(16*2,"esp"));
1066 &vpaddd ($d,$d,&QWP(16*3,"esp"));
1067 &mov ("edx",10);
1068 &vmovdqa (&QWP(16*3,"esp"),$d);
1069 &jmp (&label("loop1x"));
1070
1071&set_label("loop1x",16);
1072 &XOPROUND();
1073 &vpshufd ($c,$c,0b01001110);
1074 &vpshufd ($b,$b,0b00111001);
1075 &vpshufd ($d,$d,0b10010011);
1076
1077 &XOPROUND();
1078 &vpshufd ($c,$c,0b01001110);
1079 &vpshufd ($b,$b,0b10010011);
1080 &vpshufd ($d,$d,0b00111001);
1081
1082 &dec ("edx");
1083 &jnz (&label("loop1x"));
1084
1085 &vpaddd ($a,$a,&QWP(16*0,"esp"));
1086 &vpaddd ($b,$b,&QWP(16*1,"esp"));
1087 &vpaddd ($c,$c,&QWP(16*2,"esp"));
1088 &vpaddd ($d,$d,&QWP(16*3,"esp"));
1089
1090 &cmp ($len,64);
1091 &jb (&label("tail"));
1092
1093 &vpxor ($a,$a,&QWP(16*0,$inp)); # xor with input
1094 &vpxor ($b,$b,&QWP(16*1,$inp));
1095 &vpxor ($c,$c,&QWP(16*2,$inp));
1096 &vpxor ($d,$d,&QWP(16*3,$inp));
1097 &lea ($inp,&DWP(16*4,$inp)); # inp+=64
1098
1099 &vmovdqu (&QWP(16*0,$out),$a); # write output
1100 &vmovdqu (&QWP(16*1,$out),$b);
1101 &vmovdqu (&QWP(16*2,$out),$c);
1102 &vmovdqu (&QWP(16*3,$out),$d);
1103 &lea ($out,&DWP(16*4,$out)); # inp+=64
1104
1105 &sub ($len,64);
1106 &jnz (&label("outer1x"));
1107
1108 &jmp (&label("done"));
1109
1110&set_label("tail");
1111 &vmovdqa (&QWP(16*0,"esp"),$a);
1112 &vmovdqa (&QWP(16*1,"esp"),$b);
1113 &vmovdqa (&QWP(16*2,"esp"),$c);
1114 &vmovdqa (&QWP(16*3,"esp"),$d);
1115
1116 &xor ("eax","eax");
1117 &xor ("edx","edx");
1118 &xor ("ebp","ebp");
1119
1120&set_label("tail_loop");
1121 &movb ("al",&BP(0,"esp","ebp"));
1122 &movb ("dl",&BP(0,$inp,"ebp"));
1123 &lea ("ebp",&DWP(1,"ebp"));
1124 &xor ("al","dl");
1125 &movb (&BP(-1,$out,"ebp"),"al");
1126 &dec ($len);
1127 &jnz (&label("tail_loop"));
1128}
1129&set_label("done");
1130 &vzeroupper ();
1131 &mov ("esp",&DWP(512,"esp"));
1132&function_end("ChaCha20_xop");
1133}
1134
1135&asm_finish();
df0cb57c
RL		 1136
1137close STDOUT;
A mirror of the official openssl repository