]> git.ipfire.org Git - thirdparty/openssl.git/blame - crypto/aes/asm/aesni-x86.pl
projects / thirdparty / openssl.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Doc nits cleanup, round 2
[thirdparty/openssl.git] / crypto / aes / asm / aesni-x86.pl
CommitLineData
d64a7232
AP		 1#!/usr/bin/env perl
2
3# ====================================================================
d8ba0dc9 4# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
d64a7232
AP		 5# project. The module is, however, dual licensed under OpenSSL and
6# CRYPTOGAMS licenses depending on where you obtain it. For further
7# details see http://www.openssl.org/~appro/cryptogams/.
8# ====================================================================
9#
10# This module implements support for Intel AES-NI extension. In
11# OpenSSL context it's used with Intel engine, but can also be used as
12# drop-in replacement for crypto/aes/asm/aes-586.pl [see below for
13# details].
d7d119a3
AP		 14#
15# Performance.
16#
17# To start with see corresponding paragraph in aesni-x86_64.pl...
18# Instead of filling table similar to one found there I've chosen to
19# summarize *comparison* results for raw ECB, CTR and CBC benchmarks.
20# The simplified table below represents 32-bit performance relative
21# to 64-bit one in every given point. Ratios vary for different
22# encryption modes, therefore interval values.
23#
24# 16-byte 64-byte 256-byte 1-KB 8-KB
25# 53-67% 67-84% 91-94% 95-98% 97-99.5%
26#
27# Lower ratios for smaller block sizes are perfectly understandable,
28# because function call overhead is higher in 32-bit mode. Largest
29# 8-KB block performance is virtually same: 32-bit code is less than
f8501464
AP		 30# 1% slower for ECB, CBC and CCM, and ~3% slower otherwise.
31
32# January 2011
33#
34# See aesni-x86_64.pl for details. Unlike x86_64 version this module
35# interleaves at most 6 aes[enc|dec] instructions, because there are
36# not enough registers for 8x interleave [which should be optimal for
37# Sandy Bridge]. Actually, performance results for 6x interleave
38# factor presented in aesni-x86_64.pl (except for CTR) are for this
39# module.
40
41# April 2011
42#
43# Add aesni_xts_[en|de]crypt. Westmere spends 1.50 cycles processing
44# one byte out of 8KB with 128-bit key, Sandy Bridge - 1.09.
d64a7232 45
bd30091c
AP		 46# November 2015
47#
48# Add aesni_ocb_[en|de]crypt.
49
5599c733
AP		 50######################################################################
51# Current large-block performance in cycles per byte processed with
52# 128-bit key (less is better).
53#
bd30091c 54# CBC en-/decrypt CTR XTS ECB OCB
5599c733 55# Westmere 3.77/1.37 1.37 1.52 1.27
bd30091c
AP		 56# * Bridge 5.07/0.98 0.99 1.09 0.91 1.10
57# Haswell 4.44/0.80 0.97 1.03 0.72 0.76
58# Silvermont 5.77/3.56 3.67 4.03 3.46 4.03
59# Bulldozer 5.80/0.98 1.05 1.24 0.93 1.23
5599c733 60
d64a7232
AP		 61$PREFIX="aesni"; # if $PREFIX is set to "AES", the script
62 # generates drop-in replacement for
63 # crypto/aes/asm/aes-586.pl:-)
6f766a41 64$inline=1; # inline _aesni_[en|de]crypt
d64a7232
AP		 65
66$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
67push(@INC,"${dir}","${dir}../../perlasm");
68require "x86asm.pl";
69
184bc45f
RL		 70$output = pop;
71open OUT,">$output";
72*STDOUT=*OUT;
73
d64a7232
AP		 74&asm_init($ARGV[0],$0);
75
23f6eec7
AP		 76&external_label("OPENSSL_ia32cap_P");
77&static_label("key_const");
78
f9c5e5d9
AP		 79if ($PREFIX eq "aesni") { $movekey=\&movups; }
80else { $movekey=\&movups; }
d64a7232
AP		 81
82$len="eax";
83$rounds="ecx";
84$key="edx";
85$inp="esi";
86$out="edi";
d608b4d6
AP		 87$rounds_="ebx"; # backup copy for $rounds
88$key_="ebp"; # backup copy for $key
d64a7232 89
f8501464
AP		 90$rndkey0="xmm0";
91$rndkey1="xmm1";
92$inout0="xmm2";
93$inout1="xmm3";
94$inout2="xmm4";
95$inout3="xmm5"; $in1="xmm5";
96$inout4="xmm6"; $in0="xmm6";
97$inout5="xmm7"; $ivec="xmm7";
133a7f9a 98
d900a015 99# AESNI extension
133a7f9a
AP		 100sub aeskeygenassist
101{ my($dst,$src,$imm)=@_;
102 if ("$dst:$src" =~ /xmm([0-7]):xmm([0-7])/)
103 { &data_byte(0x66,0x0f,0x3a,0xdf,0xc0|($1<<3)|$2,$imm); }
104}
105sub aescommon
106{ my($opcodelet,$dst,$src)=@_;
107 if ("$dst:$src" =~ /xmm([0-7]):xmm([0-7])/)
108 { &data_byte(0x66,0x0f,0x38,$opcodelet,0xc0|($1<<3)|$2);}
109}
110sub aesimc { aescommon(0xdb,@_); }
111sub aesenc { aescommon(0xdc,@_); }
112sub aesenclast { aescommon(0xdd,@_); }
113sub aesdec { aescommon(0xde,@_); }
114sub aesdeclast { aescommon(0xdf,@_); }
6c83629b 115\f
d608b4d6 116# Inline version of internal aesni_[en|de]crypt1
d7d119a3 117{ my $sn;
d608b4d6 118sub aesni_inline_generate1
f8501464 119{ my ($p,$inout,$ivec)=@_; $inout=$inout0 if (!defined($inout));
d7d119a3 120 $sn++;
d64a7232 121
f8501464 122 &$movekey ($rndkey0,&QWP(0,$key));
d608b4d6 123 &$movekey ($rndkey1,&QWP(16,$key));
f8501464 124 &xorps ($ivec,$rndkey0) if (defined($ivec));
d608b4d6 125 &lea ($key,&DWP(32,$key));
f8501464
AP		 126 &xorps ($inout,$ivec) if (defined($ivec));
127 &xorps ($inout,$rndkey0) if (!defined($ivec));
d7d119a3
AP		 128 &set_label("${p}1_loop_$sn");
129 eval"&aes${p} ($inout,$rndkey1)";
d64a7232 130 &dec ($rounds);
d64a7232 131 &$movekey ($rndkey1,&QWP(0,$key));
d608b4d6 132 &lea ($key,&DWP(16,$key));
d7d119a3
AP		 133 &jnz (&label("${p}1_loop_$sn"));
134 eval"&aes${p}last ($inout,$rndkey1)";
135}}
d64a7232
AP		 136
137sub aesni_generate1 # fully unrolled loop
d7d119a3 138{ my ($p,$inout)=@_; $inout=$inout0 if (!defined($inout));
d64a7232
AP		 139
140 &function_begin_B("_aesni_${p}rypt1");
f8501464 141 &movups ($rndkey0,&QWP(0,$key));
d64a7232 142 &$movekey ($rndkey1,&QWP(0x10,$key));
f8501464 143 &xorps ($inout,$rndkey0);
d64a7232
AP		 144 &$movekey ($rndkey0,&QWP(0x20,$key));
145 &lea ($key,&DWP(0x30,$key));
d7d119a3 146 &cmp ($rounds,11);
d64a7232
AP		 147 &jb (&label("${p}128"));
148 &lea ($key,&DWP(0x20,$key));
149 &je (&label("${p}192"));
150 &lea ($key,&DWP(0x20,$key));
d7d119a3 151 eval"&aes${p} ($inout,$rndkey1)";
d64a7232 152 &$movekey ($rndkey1,&QWP(-0x40,$key));
d7d119a3 153 eval"&aes${p} ($inout,$rndkey0)";
d64a7232
AP		 154 &$movekey ($rndkey0,&QWP(-0x30,$key));
155 &set_label("${p}192");
d7d119a3 156 eval"&aes${p} ($inout,$rndkey1)";
d64a7232 157 &$movekey ($rndkey1,&QWP(-0x20,$key));
d7d119a3 158 eval"&aes${p} ($inout,$rndkey0)";
d64a7232
AP		 159 &$movekey ($rndkey0,&QWP(-0x10,$key));
160 &set_label("${p}128");
d7d119a3 161 eval"&aes${p} ($inout,$rndkey1)";
d64a7232 162 &$movekey ($rndkey1,&QWP(0,$key));
d7d119a3 163 eval"&aes${p} ($inout,$rndkey0)";
d64a7232 164 &$movekey ($rndkey0,&QWP(0x10,$key));
d7d119a3 165 eval"&aes${p} ($inout,$rndkey1)";
d64a7232 166 &$movekey ($rndkey1,&QWP(0x20,$key));
d7d119a3 167 eval"&aes${p} ($inout,$rndkey0)";
d64a7232 168 &$movekey ($rndkey0,&QWP(0x30,$key));
d7d119a3 169 eval"&aes${p} ($inout,$rndkey1)";
d64a7232 170 &$movekey ($rndkey1,&QWP(0x40,$key));
d7d119a3 171 eval"&aes${p} ($inout,$rndkey0)";
d64a7232 172 &$movekey ($rndkey0,&QWP(0x50,$key));
d7d119a3 173 eval"&aes${p} ($inout,$rndkey1)";
d64a7232 174 &$movekey ($rndkey1,&QWP(0x60,$key));
d7d119a3 175 eval"&aes${p} ($inout,$rndkey0)";
d64a7232 176 &$movekey ($rndkey0,&QWP(0x70,$key));
d7d119a3
AP		 177 eval"&aes${p} ($inout,$rndkey1)";
178 eval"&aes${p}last ($inout,$rndkey0)";
d64a7232
AP		 179 &ret();
180 &function_end_B("_aesni_${p}rypt1");
181}
6c83629b 182\f
d64a7232 183# void $PREFIX_encrypt (const void *inp,void *out,const AES_KEY *key);
6f766a41 184&aesni_generate1("enc") if (!$inline);
d64a7232
AP		 185&function_begin_B("${PREFIX}_encrypt");
186 &mov ("eax",&wparam(0));
187 &mov ($key,&wparam(2));
f8501464 188 &movups ($inout0,&QWP(0,"eax"));
d64a7232
AP		 189 &mov ($rounds,&DWP(240,$key));
190 &mov ("eax",&wparam(1));
6f766a41
AP		 191 if ($inline)
192 { &aesni_inline_generate1("enc"); }
193 else
194 { &call ("_aesni_encrypt1"); }
23f6eec7
AP		 195 &pxor ($rndkey0,$rndkey0); # clear register bank
196 &pxor ($rndkey1,$rndkey1);
d64a7232 197 &movups (&QWP(0,"eax"),$inout0);
23f6eec7 198 &pxor ($inout0,$inout0);
d64a7232
AP		 199 &ret ();
200&function_end_B("${PREFIX}_encrypt");
201
d64a7232 202# void $PREFIX_decrypt (const void *inp,void *out,const AES_KEY *key);
6f766a41 203&aesni_generate1("dec") if(!$inline);
d64a7232
AP		 204&function_begin_B("${PREFIX}_decrypt");
205 &mov ("eax",&wparam(0));
206 &mov ($key,&wparam(2));
f8501464 207 &movups ($inout0,&QWP(0,"eax"));
d64a7232
AP		 208 &mov ($rounds,&DWP(240,$key));
209 &mov ("eax",&wparam(1));
6f766a41
AP		 210 if ($inline)
211 { &aesni_inline_generate1("dec"); }
212 else
213 { &call ("_aesni_decrypt1"); }
23f6eec7
AP		 214 &pxor ($rndkey0,$rndkey0); # clear register bank
215 &pxor ($rndkey1,$rndkey1);
d64a7232 216 &movups (&QWP(0,"eax"),$inout0);
23f6eec7 217 &pxor ($inout0,$inout0);
d64a7232
AP		 218 &ret ();
219&function_end_B("${PREFIX}_decrypt");
6c83629b 220
f8501464
AP		 221# _aesni_[en|de]cryptN are private interfaces, N denotes interleave
222# factor. Why 3x subroutine were originally used in loops? Even though
223# aes[enc|dec] latency was originally 6, it could be scheduled only
224# every *2nd* cycle. Thus 3x interleave was the one providing optimal
d608b4d6
AP		 225# utilization, i.e. when subroutine's throughput is virtually same as
226# of non-interleaved subroutine [for number of input blocks up to 3].
214368ff
AP		 227# This is why it originally made no sense to implement 2x subroutine.
228# But times change and it became appropriate to spend extra 192 bytes
229# on 2x subroutine on Atom Silvermont account. For processors that
230# can schedule aes[enc|dec] every cycle optimal interleave factor
231# equals to corresponding instructions latency. 8x is optimal for
232# * Bridge, but it's unfeasible to accommodate such implementation
233# in XMM registers addreassable in 32-bit mode and therefore maximum
234# of 6x is used instead...
235
236sub aesni_generate2
237{ my $p=shift;
238
239 &function_begin_B("_aesni_${p}rypt2");
240 &$movekey ($rndkey0,&QWP(0,$key));
241 &shl ($rounds,4);
242 &$movekey ($rndkey1,&QWP(16,$key));
243 &xorps ($inout0,$rndkey0);
244 &pxor ($inout1,$rndkey0);
245 &$movekey ($rndkey0,&QWP(32,$key));
246 &lea ($key,&DWP(32,$key,$rounds));
247 &neg ($rounds);
248 &add ($rounds,16);
249
250 &set_label("${p}2_loop");
251 eval"&aes${p} ($inout0,$rndkey1)";
252 eval"&aes${p} ($inout1,$rndkey1)";
253 &$movekey ($rndkey1,&QWP(0,$key,$rounds));
254 &add ($rounds,32);
255 eval"&aes${p} ($inout0,$rndkey0)";
256 eval"&aes${p} ($inout1,$rndkey0)";
257 &$movekey ($rndkey0,&QWP(-16,$key,$rounds));
258 &jnz (&label("${p}2_loop"));
259 eval"&aes${p} ($inout0,$rndkey1)";
260 eval"&aes${p} ($inout1,$rndkey1)";
261 eval"&aes${p}last ($inout0,$rndkey0)";
262 eval"&aes${p}last ($inout1,$rndkey0)";
263 &ret();
264 &function_end_B("_aesni_${p}rypt2");
265}
f8501464 266
d64a7232
AP		 267sub aesni_generate3
268{ my $p=shift;
269
270 &function_begin_B("_aesni_${p}rypt3");
271 &$movekey ($rndkey0,&QWP(0,$key));
d8ba0dc9 272 &shl ($rounds,4);
d608b4d6 273 &$movekey ($rndkey1,&QWP(16,$key));
f8501464 274 &xorps ($inout0,$rndkey0);
d64a7232 275 &pxor ($inout1,$rndkey0);
d64a7232 276 &pxor ($inout2,$rndkey0);
d8ba0dc9
AP		 277 &$movekey ($rndkey0,&QWP(32,$key));
278 &lea ($key,&DWP(32,$key,$rounds));
279 &neg ($rounds);
280 &add ($rounds,16);
d7d119a3
AP		 281
282 &set_label("${p}3_loop");
283 eval"&aes${p} ($inout0,$rndkey1)";
d64a7232 284 eval"&aes${p} ($inout1,$rndkey1)";
d64a7232 285 eval"&aes${p} ($inout2,$rndkey1)";
d8ba0dc9
AP		 286 &$movekey ($rndkey1,&QWP(0,$key,$rounds));
287 &add ($rounds,32);
d64a7232 288 eval"&aes${p} ($inout0,$rndkey0)";
d64a7232
AP		 289 eval"&aes${p} ($inout1,$rndkey0)";
290 eval"&aes${p} ($inout2,$rndkey0)";
d8ba0dc9 291 &$movekey ($rndkey0,&QWP(-16,$key,$rounds));
d64a7232
AP		 292 &jnz (&label("${p}3_loop"));
293 eval"&aes${p} ($inout0,$rndkey1)";
d64a7232
AP		 294 eval"&aes${p} ($inout1,$rndkey1)";
295 eval"&aes${p} ($inout2,$rndkey1)";
296 eval"&aes${p}last ($inout0,$rndkey0)";
297 eval"&aes${p}last ($inout1,$rndkey0)";
298 eval"&aes${p}last ($inout2,$rndkey0)";
299 &ret();
300 &function_end_B("_aesni_${p}rypt3");
301}
d608b4d6
AP		 302
303# 4x interleave is implemented to improve small block performance,
304# most notably [and naturally] 4 block by ~30%. One can argue that one
305# should have implemented 5x as well, but improvement would be <20%,
306# so it's not worth it...
307sub aesni_generate4
308{ my $p=shift;
309
310 &function_begin_B("_aesni_${p}rypt4");
311 &$movekey ($rndkey0,&QWP(0,$key));
312 &$movekey ($rndkey1,&QWP(16,$key));
d8ba0dc9 313 &shl ($rounds,4);
f8501464 314 &xorps ($inout0,$rndkey0);
d608b4d6
AP		 315 &pxor ($inout1,$rndkey0);
316 &pxor ($inout2,$rndkey0);
317 &pxor ($inout3,$rndkey0);
d8ba0dc9
AP		 318 &$movekey ($rndkey0,&QWP(32,$key));
319 &lea ($key,&DWP(32,$key,$rounds));
320 &neg ($rounds);
321 &data_byte (0x0f,0x1f,0x40,0x00);
322 &add ($rounds,16);
d7d119a3 323
f8501464 324 &set_label("${p}4_loop");
d7d119a3 325 eval"&aes${p} ($inout0,$rndkey1)";
d608b4d6 326 eval"&aes${p} ($inout1,$rndkey1)";
d608b4d6
AP		 327 eval"&aes${p} ($inout2,$rndkey1)";
328 eval"&aes${p} ($inout3,$rndkey1)";
d8ba0dc9
AP		 329 &$movekey ($rndkey1,&QWP(0,$key,$rounds));
330 &add ($rounds,32);
d608b4d6 331 eval"&aes${p} ($inout0,$rndkey0)";
d608b4d6
AP		 332 eval"&aes${p} ($inout1,$rndkey0)";
333 eval"&aes${p} ($inout2,$rndkey0)";
334 eval"&aes${p} ($inout3,$rndkey0)";
d8ba0dc9 335 &$movekey ($rndkey0,&QWP(-16,$key,$rounds));
f8501464 336 &jnz (&label("${p}4_loop"));
d7d119a3 337
d608b4d6 338 eval"&aes${p} ($inout0,$rndkey1)";
d608b4d6
AP		 339 eval"&aes${p} ($inout1,$rndkey1)";
340 eval"&aes${p} ($inout2,$rndkey1)";
341 eval"&aes${p} ($inout3,$rndkey1)";
342 eval"&aes${p}last ($inout0,$rndkey0)";
343 eval"&aes${p}last ($inout1,$rndkey0)";
344 eval"&aes${p}last ($inout2,$rndkey0)";
345 eval"&aes${p}last ($inout3,$rndkey0)";
346 &ret();
347 &function_end_B("_aesni_${p}rypt4");
348}
f8501464
AP		 349
350sub aesni_generate6
351{ my $p=shift;
352
353 &function_begin_B("_aesni_${p}rypt6");
354 &static_label("_aesni_${p}rypt6_enter");
355 &$movekey ($rndkey0,&QWP(0,$key));
d8ba0dc9 356 &shl ($rounds,4);
f8501464 357 &$movekey ($rndkey1,&QWP(16,$key));
f8501464
AP		 358 &xorps ($inout0,$rndkey0);
359 &pxor ($inout1,$rndkey0); # pxor does better here
f8501464 360 &pxor ($inout2,$rndkey0);
d8ba0dc9 361 eval"&aes${p} ($inout0,$rndkey1)";
f8501464 362 &pxor ($inout3,$rndkey0);
f8501464 363 &pxor ($inout4,$rndkey0);
d8ba0dc9
AP		 364 eval"&aes${p} ($inout1,$rndkey1)";
365 &lea ($key,&DWP(32,$key,$rounds));
366 &neg ($rounds);
367 eval"&aes${p} ($inout2,$rndkey1)";
f8501464 368 &pxor ($inout5,$rndkey0);
23f6eec7 369 &$movekey ($rndkey0,&QWP(0,$key,$rounds));
d8ba0dc9 370 &add ($rounds,16);
23f6eec7 371 &jmp (&label("_aesni_${p}rypt6_inner"));
f8501464
AP		 372
373 &set_label("${p}6_loop",16);
374 eval"&aes${p} ($inout0,$rndkey1)";
375 eval"&aes${p} ($inout1,$rndkey1)";
f8501464 376 eval"&aes${p} ($inout2,$rndkey1)";
23f6eec7 377 &set_label("_aesni_${p}rypt6_inner");
f8501464
AP		 378 eval"&aes${p} ($inout3,$rndkey1)";
379 eval"&aes${p} ($inout4,$rndkey1)";
380 eval"&aes${p} ($inout5,$rndkey1)";
d8ba0dc9
AP		 381 &set_label("_aesni_${p}rypt6_enter");
382 &$movekey ($rndkey1,&QWP(0,$key,$rounds));
383 &add ($rounds,32);
f8501464
AP		 384 eval"&aes${p} ($inout0,$rndkey0)";
385 eval"&aes${p} ($inout1,$rndkey0)";
f8501464
AP		 386 eval"&aes${p} ($inout2,$rndkey0)";
387 eval"&aes${p} ($inout3,$rndkey0)";
388 eval"&aes${p} ($inout4,$rndkey0)";
389 eval"&aes${p} ($inout5,$rndkey0)";
d8ba0dc9 390 &$movekey ($rndkey0,&QWP(-16,$key,$rounds));
f8501464
AP		 391 &jnz (&label("${p}6_loop"));
392
393 eval"&aes${p} ($inout0,$rndkey1)";
394 eval"&aes${p} ($inout1,$rndkey1)";
395 eval"&aes${p} ($inout2,$rndkey1)";
396 eval"&aes${p} ($inout3,$rndkey1)";
397 eval"&aes${p} ($inout4,$rndkey1)";
398 eval"&aes${p} ($inout5,$rndkey1)";
399 eval"&aes${p}last ($inout0,$rndkey0)";
400 eval"&aes${p}last ($inout1,$rndkey0)";
401 eval"&aes${p}last ($inout2,$rndkey0)";
402 eval"&aes${p}last ($inout3,$rndkey0)";
403 eval"&aes${p}last ($inout4,$rndkey0)";
404 eval"&aes${p}last ($inout5,$rndkey0)";
405 &ret();
406 &function_end_B("_aesni_${p}rypt6");
407}
214368ff
AP		 408&aesni_generate2("enc") if ($PREFIX eq "aesni");
409&aesni_generate2("dec");
d64a7232
AP		 410&aesni_generate3("enc") if ($PREFIX eq "aesni");
411&aesni_generate3("dec");
d608b4d6
AP		 412&aesni_generate4("enc") if ($PREFIX eq "aesni");
413&aesni_generate4("dec");
f8501464
AP		 414&aesni_generate6("enc") if ($PREFIX eq "aesni");
415&aesni_generate6("dec");
6c83629b 416\f
d64a7232 417if ($PREFIX eq "aesni") {
6c83629b 418######################################################################
d64a7232
AP		 419# void aesni_ecb_encrypt (const void *in, void *out,
420# size_t length, const AES_KEY *key,
421# int enc);
d64a7232
AP		 422&function_begin("aesni_ecb_encrypt");
423 &mov ($inp,&wparam(0));
424 &mov ($out,&wparam(1));
425 &mov ($len,&wparam(2));
426 &mov ($key,&wparam(3));
f8501464 427 &mov ($rounds_,&wparam(4));
d64a7232 428 &and ($len,-16);
f8501464 429 &jz (&label("ecb_ret"));
d64a7232 430 &mov ($rounds,&DWP(240,$key));
f8501464
AP		 431 &test ($rounds_,$rounds_);
432 &jz (&label("ecb_decrypt"));
433
d64a7232
AP		 434 &mov ($key_,$key); # backup $key
435 &mov ($rounds_,$rounds); # backup $rounds
f8501464
AP		 436 &cmp ($len,0x60);
437 &jb (&label("ecb_enc_tail"));
438
439 &movdqu ($inout0,&QWP(0,$inp));
440 &movdqu ($inout1,&QWP(0x10,$inp));
441 &movdqu ($inout2,&QWP(0x20,$inp));
442 &movdqu ($inout3,&QWP(0x30,$inp));
443 &movdqu ($inout4,&QWP(0x40,$inp));
444 &movdqu ($inout5,&QWP(0x50,$inp));
445 &lea ($inp,&DWP(0x60,$inp));
446 &sub ($len,0x60);
447 &jmp (&label("ecb_enc_loop6_enter"));
448
449&set_label("ecb_enc_loop6",16);
450 &movups (&QWP(0,$out),$inout0);
451 &movdqu ($inout0,&QWP(0,$inp));
452 &movups (&QWP(0x10,$out),$inout1);
453 &movdqu ($inout1,&QWP(0x10,$inp));
454 &movups (&QWP(0x20,$out),$inout2);
455 &movdqu ($inout2,&QWP(0x20,$inp));
456 &movups (&QWP(0x30,$out),$inout3);
457 &movdqu ($inout3,&QWP(0x30,$inp));
458 &movups (&QWP(0x40,$out),$inout4);
459 &movdqu ($inout4,&QWP(0x40,$inp));
460 &movups (&QWP(0x50,$out),$inout5);
461 &lea ($out,&DWP(0x60,$out));
462 &movdqu ($inout5,&QWP(0x50,$inp));
463 &lea ($inp,&DWP(0x60,$inp));
464&set_label("ecb_enc_loop6_enter");
d64a7232 465
f8501464 466 &call ("_aesni_encrypt6");
d64a7232 467
d64a7232 468 &mov ($key,$key_); # restore $key
d64a7232 469 &mov ($rounds,$rounds_); # restore $rounds
f8501464
AP		 470 &sub ($len,0x60);
471 &jnc (&label("ecb_enc_loop6"));
472
473 &movups (&QWP(0,$out),$inout0);
474 &movups (&QWP(0x10,$out),$inout1);
d7d119a3 475 &movups (&QWP(0x20,$out),$inout2);
f8501464
AP		 476 &movups (&QWP(0x30,$out),$inout3);
477 &movups (&QWP(0x40,$out),$inout4);
478 &movups (&QWP(0x50,$out),$inout5);
479 &lea ($out,&DWP(0x60,$out));
480 &add ($len,0x60);
481 &jz (&label("ecb_ret"));
d64a7232 482
6c83629b 483&set_label("ecb_enc_tail");
6c83629b 484 &movups ($inout0,&QWP(0,$inp));
d7d119a3 485 &cmp ($len,0x20);
6c83629b 486 &jb (&label("ecb_enc_one"));
d64a7232 487 &movups ($inout1,&QWP(0x10,$inp));
d608b4d6 488 &je (&label("ecb_enc_two"));
d608b4d6 489 &movups ($inout2,&QWP(0x20,$inp));
f8501464
AP		 490 &cmp ($len,0x40);
491 &jb (&label("ecb_enc_three"));
d608b4d6 492 &movups ($inout3,&QWP(0x30,$inp));
f8501464
AP		 493 &je (&label("ecb_enc_four"));
494 &movups ($inout4,&QWP(0x40,$inp));
495 &xorps ($inout5,$inout5);
496 &call ("_aesni_encrypt6");
d64a7232
AP		 497 &movups (&QWP(0,$out),$inout0);
498 &movups (&QWP(0x10,$out),$inout1);
d608b4d6
AP		 499 &movups (&QWP(0x20,$out),$inout2);
500 &movups (&QWP(0x30,$out),$inout3);
f8501464 501 &movups (&QWP(0x40,$out),$inout4);
d64a7232
AP		 502 jmp (&label("ecb_ret"));
503
504&set_label("ecb_enc_one",16);
6f766a41
AP		 505 if ($inline)
506 { &aesni_inline_generate1("enc"); }
507 else
508 { &call ("_aesni_encrypt1"); }
d64a7232
AP		 509 &movups (&QWP(0,$out),$inout0);
510 &jmp (&label("ecb_ret"));
511
d608b4d6 512&set_label("ecb_enc_two",16);
214368ff 513 &call ("_aesni_encrypt2");
d608b4d6
AP		 514 &movups (&QWP(0,$out),$inout0);
515 &movups (&QWP(0x10,$out),$inout1);
516 &jmp (&label("ecb_ret"));
517
518&set_label("ecb_enc_three",16);
519 &call ("_aesni_encrypt3");
520 &movups (&QWP(0,$out),$inout0);
521 &movups (&QWP(0x10,$out),$inout1);
522 &movups (&QWP(0x20,$out),$inout2);
523 &jmp (&label("ecb_ret"));
f8501464
AP		 524
525&set_label("ecb_enc_four",16);
526 &call ("_aesni_encrypt4");
527 &movups (&QWP(0,$out),$inout0);
528 &movups (&QWP(0x10,$out),$inout1);
529 &movups (&QWP(0x20,$out),$inout2);
530 &movups (&QWP(0x30,$out),$inout3);
531 &jmp (&label("ecb_ret"));
6c83629b 532######################################################################
d64a7232 533&set_label("ecb_decrypt",16);
f8501464
AP		 534 &mov ($key_,$key); # backup $key
535 &mov ($rounds_,$rounds); # backup $rounds
536 &cmp ($len,0x60);
537 &jb (&label("ecb_dec_tail"));
538
539 &movdqu ($inout0,&QWP(0,$inp));
540 &movdqu ($inout1,&QWP(0x10,$inp));
541 &movdqu ($inout2,&QWP(0x20,$inp));
542 &movdqu ($inout3,&QWP(0x30,$inp));
543 &movdqu ($inout4,&QWP(0x40,$inp));
544 &movdqu ($inout5,&QWP(0x50,$inp));
545 &lea ($inp,&DWP(0x60,$inp));
546 &sub ($len,0x60);
547 &jmp (&label("ecb_dec_loop6_enter"));
548
549&set_label("ecb_dec_loop6",16);
d7d119a3 550 &movups (&QWP(0,$out),$inout0);
f8501464 551 &movdqu ($inout0,&QWP(0,$inp));
d7d119a3 552 &movups (&QWP(0x10,$out),$inout1);
f8501464
AP		 553 &movdqu ($inout1,&QWP(0x10,$inp));
554 &movups (&QWP(0x20,$out),$inout2);
555 &movdqu ($inout2,&QWP(0x20,$inp));
556 &movups (&QWP(0x30,$out),$inout3);
557 &movdqu ($inout3,&QWP(0x30,$inp));
558 &movups (&QWP(0x40,$out),$inout4);
559 &movdqu ($inout4,&QWP(0x40,$inp));
560 &movups (&QWP(0x50,$out),$inout5);
561 &lea ($out,&DWP(0x60,$out));
562 &movdqu ($inout5,&QWP(0x50,$inp));
563 &lea ($inp,&DWP(0x60,$inp));
564&set_label("ecb_dec_loop6_enter");
565
566 &call ("_aesni_decrypt6");
567
568 &mov ($key,$key_); # restore $key
d64a7232 569 &mov ($rounds,$rounds_); # restore $rounds
f8501464
AP		 570 &sub ($len,0x60);
571 &jnc (&label("ecb_dec_loop6"));
572
573 &movups (&QWP(0,$out),$inout0);
574 &movups (&QWP(0x10,$out),$inout1);
d7d119a3 575 &movups (&QWP(0x20,$out),$inout2);
f8501464
AP		 576 &movups (&QWP(0x30,$out),$inout3);
577 &movups (&QWP(0x40,$out),$inout4);
578 &movups (&QWP(0x50,$out),$inout5);
579 &lea ($out,&DWP(0x60,$out));
580 &add ($len,0x60);
581 &jz (&label("ecb_ret"));
d64a7232 582
6c83629b 583&set_label("ecb_dec_tail");
6c83629b 584 &movups ($inout0,&QWP(0,$inp));
d7d119a3 585 &cmp ($len,0x20);
6c83629b 586 &jb (&label("ecb_dec_one"));
d64a7232 587 &movups ($inout1,&QWP(0x10,$inp));
d608b4d6 588 &je (&label("ecb_dec_two"));
d608b4d6 589 &movups ($inout2,&QWP(0x20,$inp));
f8501464
AP		 590 &cmp ($len,0x40);
591 &jb (&label("ecb_dec_three"));
d608b4d6 592 &movups ($inout3,&QWP(0x30,$inp));
f8501464
AP		 593 &je (&label("ecb_dec_four"));
594 &movups ($inout4,&QWP(0x40,$inp));
595 &xorps ($inout5,$inout5);
596 &call ("_aesni_decrypt6");
d64a7232
AP		 597 &movups (&QWP(0,$out),$inout0);
598 &movups (&QWP(0x10,$out),$inout1);
d608b4d6
AP		 599 &movups (&QWP(0x20,$out),$inout2);
600 &movups (&QWP(0x30,$out),$inout3);
f8501464 601 &movups (&QWP(0x40,$out),$inout4);
d608b4d6 602 &jmp (&label("ecb_ret"));
d64a7232
AP		 603
604&set_label("ecb_dec_one",16);
6f766a41
AP		 605 if ($inline)
606 { &aesni_inline_generate1("dec"); }
607 else
608 { &call ("_aesni_decrypt1"); }
d64a7232 609 &movups (&QWP(0,$out),$inout0);
d608b4d6
AP		 610 &jmp (&label("ecb_ret"));
611
612&set_label("ecb_dec_two",16);
214368ff 613 &call ("_aesni_decrypt2");
d608b4d6
AP		 614 &movups (&QWP(0,$out),$inout0);
615 &movups (&QWP(0x10,$out),$inout1);
616 &jmp (&label("ecb_ret"));
617
618&set_label("ecb_dec_three",16);
619 &call ("_aesni_decrypt3");
620 &movups (&QWP(0,$out),$inout0);
621 &movups (&QWP(0x10,$out),$inout1);
622 &movups (&QWP(0x20,$out),$inout2);
f8501464
AP		 623 &jmp (&label("ecb_ret"));
624
625&set_label("ecb_dec_four",16);
626 &call ("_aesni_decrypt4");
627 &movups (&QWP(0,$out),$inout0);
628 &movups (&QWP(0x10,$out),$inout1);
629 &movups (&QWP(0x20,$out),$inout2);
630 &movups (&QWP(0x30,$out),$inout3);
d64a7232
AP		 631
632&set_label("ecb_ret");
23f6eec7
AP		 633 &pxor ("xmm0","xmm0"); # clear register bank
634 &pxor ("xmm1","xmm1");
635 &pxor ("xmm2","xmm2");
636 &pxor ("xmm3","xmm3");
637 &pxor ("xmm4","xmm4");
638 &pxor ("xmm5","xmm5");
639 &pxor ("xmm6","xmm6");
640 &pxor ("xmm7","xmm7");
d64a7232 641&function_end("aesni_ecb_encrypt");
6c83629b
AP		 642\f
643######################################################################
d7d119a3
AP		 644# void aesni_ccm64_[en|de]crypt_blocks (const void *in, void *out,
645# size_t blocks, const AES_KEY *key,
646# const char *ivec,char *cmac);
647#
648# Handles only complete blocks, operates on 64-bit counter and
649# does not update *ivec! Nor does it finalize CMAC value
650# (see engine/eng_aesni.c for details)
6c83629b 651#
f8501464 652{ my $cmac=$inout1;
d7d119a3
AP		 653&function_begin("aesni_ccm64_encrypt_blocks");
654 &mov ($inp,&wparam(0));
655 &mov ($out,&wparam(1));
656 &mov ($len,&wparam(2));
657 &mov ($key,&wparam(3));
658 &mov ($rounds_,&wparam(4));
659 &mov ($rounds,&wparam(5));
660 &mov ($key_,"esp");
661 &sub ("esp",60);
662 &and ("esp",-16); # align stack
663 &mov (&DWP(48,"esp"),$key_);
664
665 &movdqu ($ivec,&QWP(0,$rounds_)); # load ivec
f8501464 666 &movdqu ($cmac,&QWP(0,$rounds)); # load cmac
267b481c 667 &mov ($rounds,&DWP(240,$key));
d7d119a3
AP		 668
669 # compose byte-swap control mask for pshufb on stack
670 &mov (&DWP(0,"esp"),0x0c0d0e0f);
671 &mov (&DWP(4,"esp"),0x08090a0b);
672 &mov (&DWP(8,"esp"),0x04050607);
673 &mov (&DWP(12,"esp"),0x00010203);
674
675 # compose counter increment vector on stack
267b481c 676 &mov ($rounds_,1);
d7d119a3 677 &xor ($key_,$key_);
267b481c 678 &mov (&DWP(16,"esp"),$rounds_);
d7d119a3
AP		 679 &mov (&DWP(20,"esp"),$key_);
680 &mov (&DWP(24,"esp"),$key_);
681 &mov (&DWP(28,"esp"),$key_);
682
d8ba0dc9
AP		 683 &shl ($rounds,4);
684 &mov ($rounds_,16);
267b481c 685 &lea ($key_,&DWP(0,$key));
9ee5916d 686 &movdqa ($inout3,&QWP(0,"esp"));
d7d119a3 687 &movdqa ($inout0,$ivec);
d8ba0dc9
AP		 688 &lea ($key,&DWP(32,$key,$rounds));
689 &sub ($rounds_,$rounds);
9ee5916d 690 &pshufb ($ivec,$inout3);
d7d119a3
AP		 691
692&set_label("ccm64_enc_outer");
267b481c 693 &$movekey ($rndkey0,&QWP(0,$key_));
f8501464 694 &mov ($rounds,$rounds_);
267b481c 695 &movups ($in0,&QWP(0,$inp));
d7d119a3 696
f8501464 697 &xorps ($inout0,$rndkey0);
267b481c
AP		 698 &$movekey ($rndkey1,&QWP(16,$key_));
699 &xorps ($rndkey0,$in0);
267b481c 700 &xorps ($cmac,$rndkey0); # cmac^=inp
d8ba0dc9 701 &$movekey ($rndkey0,&QWP(32,$key_));
f8501464
AP		 702
703&set_label("ccm64_enc2_loop");
704 &aesenc ($inout0,$rndkey1);
f8501464 705 &aesenc ($cmac,$rndkey1);
d8ba0dc9
AP		 706 &$movekey ($rndkey1,&QWP(0,$key,$rounds));
707 &add ($rounds,32);
f8501464 708 &aesenc ($inout0,$rndkey0);
f8501464 709 &aesenc ($cmac,$rndkey0);
d8ba0dc9 710 &$movekey ($rndkey0,&QWP(-16,$key,$rounds));
f8501464
AP		 711 &jnz (&label("ccm64_enc2_loop"));
712 &aesenc ($inout0,$rndkey1);
713 &aesenc ($cmac,$rndkey1);
267b481c 714 &paddq ($ivec,&QWP(16,"esp"));
d8ba0dc9 715 &dec ($len);
f8501464
AP		 716 &aesenclast ($inout0,$rndkey0);
717 &aesenclast ($cmac,$rndkey0);
d7d119a3 718
d7d119a3 719 &lea ($inp,&DWP(16,$inp));
f8501464 720 &xorps ($in0,$inout0); # inp^=E(ivec)
d7d119a3 721 &movdqa ($inout0,$ivec);
267b481c 722 &movups (&QWP(0,$out),$in0); # save output
9ee5916d 723 &pshufb ($inout0,$inout3);
d8ba0dc9 724 &lea ($out,&DWP(16,$out));
d7d119a3
AP		 725 &jnz (&label("ccm64_enc_outer"));
726
727 &mov ("esp",&DWP(48,"esp"));
728 &mov ($out,&wparam(5));
f8501464 729 &movups (&QWP(0,$out),$cmac);
23f6eec7
AP		 730
731 &pxor ("xmm0","xmm0"); # clear register bank
732 &pxor ("xmm1","xmm1");
733 &pxor ("xmm2","xmm2");
734 &pxor ("xmm3","xmm3");
735 &pxor ("xmm4","xmm4");
736 &pxor ("xmm5","xmm5");
737 &pxor ("xmm6","xmm6");
738 &pxor ("xmm7","xmm7");
d7d119a3
AP		 739&function_end("aesni_ccm64_encrypt_blocks");
740
741&function_begin("aesni_ccm64_decrypt_blocks");
742 &mov ($inp,&wparam(0));
743 &mov ($out,&wparam(1));
744 &mov ($len,&wparam(2));
745 &mov ($key,&wparam(3));
746 &mov ($rounds_,&wparam(4));
747 &mov ($rounds,&wparam(5));
748 &mov ($key_,"esp");
749 &sub ("esp",60);
750 &and ("esp",-16); # align stack
751 &mov (&DWP(48,"esp"),$key_);
752
753 &movdqu ($ivec,&QWP(0,$rounds_)); # load ivec
f8501464 754 &movdqu ($cmac,&QWP(0,$rounds)); # load cmac
267b481c 755 &mov ($rounds,&DWP(240,$key));
d7d119a3
AP		 756
757 # compose byte-swap control mask for pshufb on stack
758 &mov (&DWP(0,"esp"),0x0c0d0e0f);
759 &mov (&DWP(4,"esp"),0x08090a0b);
760 &mov (&DWP(8,"esp"),0x04050607);
761 &mov (&DWP(12,"esp"),0x00010203);
762
763 # compose counter increment vector on stack
267b481c 764 &mov ($rounds_,1);
d7d119a3 765 &xor ($key_,$key_);
267b481c 766 &mov (&DWP(16,"esp"),$rounds_);
d7d119a3
AP		 767 &mov (&DWP(20,"esp"),$key_);
768 &mov (&DWP(24,"esp"),$key_);
769 &mov (&DWP(28,"esp"),$key_);
770
771 &movdqa ($inout3,&QWP(0,"esp")); # bswap mask
772 &movdqa ($inout0,$ivec);
d7d119a3 773
d7d119a3
AP		 774 &mov ($key_,$key);
775 &mov ($rounds_,$rounds);
776
267b481c 777 &pshufb ($ivec,$inout3);
d7d119a3
AP		 778 if ($inline)
779 { &aesni_inline_generate1("enc"); }
780 else
781 { &call ("_aesni_encrypt1"); }
d8ba0dc9
AP		 782 &shl ($rounds_,4);
783 &mov ($rounds,16);
f8501464 784 &movups ($in0,&QWP(0,$inp)); # load inp
267b481c 785 &paddq ($ivec,&QWP(16,"esp"));
f8501464 786 &lea ($inp,&QWP(16,$inp));
d8ba0dc9
AP		 787 &sub ($rounds,$rounds_);
788 &lea ($key,&DWP(32,$key_,$rounds_));
789 &mov ($rounds_,$rounds);
267b481c
AP		 790 &jmp (&label("ccm64_dec_outer"));
791
792&set_label("ccm64_dec_outer",16);
793 &xorps ($in0,$inout0); # inp ^= E(ivec)
794 &movdqa ($inout0,$ivec);
267b481c 795 &movups (&QWP(0,$out),$in0); # save output
d7d119a3 796 &lea ($out,&DWP(16,$out));
9ee5916d 797 &pshufb ($inout0,$inout3);
d7d119a3 798
f8501464 799 &sub ($len,1);
d7d119a3
AP		 800 &jz (&label("ccm64_dec_break"));
801
267b481c 802 &$movekey ($rndkey0,&QWP(0,$key_));
d8ba0dc9 803 &mov ($rounds,$rounds_);
267b481c 804 &$movekey ($rndkey1,&QWP(16,$key_));
f8501464 805 &xorps ($in0,$rndkey0);
f8501464
AP		 806 &xorps ($inout0,$rndkey0);
807 &xorps ($cmac,$in0); # cmac^=out
d8ba0dc9 808 &$movekey ($rndkey0,&QWP(32,$key_));
d7d119a3 809
f8501464
AP		 810&set_label("ccm64_dec2_loop");
811 &aesenc ($inout0,$rndkey1);
f8501464 812 &aesenc ($cmac,$rndkey1);
d8ba0dc9
AP		 813 &$movekey ($rndkey1,&QWP(0,$key,$rounds));
814 &add ($rounds,32);
f8501464 815 &aesenc ($inout0,$rndkey0);
f8501464 816 &aesenc ($cmac,$rndkey0);
d8ba0dc9 817 &$movekey ($rndkey0,&QWP(-16,$key,$rounds));
f8501464 818 &jnz (&label("ccm64_dec2_loop"));
267b481c
AP		 819 &movups ($in0,&QWP(0,$inp)); # load inp
820 &paddq ($ivec,&QWP(16,"esp"));
f8501464
AP		 821 &aesenc ($inout0,$rndkey1);
822 &aesenc ($cmac,$rndkey1);
823 &aesenclast ($inout0,$rndkey0);
824 &aesenclast ($cmac,$rndkey0);
d8ba0dc9 825 &lea ($inp,&QWP(16,$inp));
d7d119a3
AP		 826 &jmp (&label("ccm64_dec_outer"));
827
828&set_label("ccm64_dec_break",16);
d8ba0dc9 829 &mov ($rounds,&DWP(240,$key_));
267b481c 830 &mov ($key,$key_);
d7d119a3 831 if ($inline)
f8501464 832 { &aesni_inline_generate1("enc",$cmac,$in0); }
d7d119a3 833 else
f8501464 834 { &call ("_aesni_encrypt1",$cmac); }
d7d119a3
AP		 835
836 &mov ("esp",&DWP(48,"esp"));
837 &mov ($out,&wparam(5));
f8501464 838 &movups (&QWP(0,$out),$cmac);
23f6eec7
AP		 839
840 &pxor ("xmm0","xmm0"); # clear register bank
841 &pxor ("xmm1","xmm1");
842 &pxor ("xmm2","xmm2");
843 &pxor ("xmm3","xmm3");
844 &pxor ("xmm4","xmm4");
845 &pxor ("xmm5","xmm5");
846 &pxor ("xmm6","xmm6");
847 &pxor ("xmm7","xmm7");
d7d119a3 848&function_end("aesni_ccm64_decrypt_blocks");
f8501464 849}
d7d119a3
AP		 850\f
851######################################################################
6c83629b
AP		 852# void aesni_ctr32_encrypt_blocks (const void *in, void *out,
853# size_t blocks, const AES_KEY *key,
854# const char *ivec);
d7d119a3
AP		 855#
856# Handles only complete blocks, operates on 32-bit counter and
d8ba0dc9 857# does not update *ivec! (see crypto/modes/ctr128.c for details)
d7d119a3 858#
f8501464
AP		 859# stack layout:
860# 0 pshufb mask
861# 16 vector addend: 0,6,6,6
862# 32 counter-less ivec
863# 48 1st triplet of counter vector
864# 64 2nd triplet of counter vector
865# 80 saved %esp
866
6c83629b
AP		 867&function_begin("aesni_ctr32_encrypt_blocks");
868 &mov ($inp,&wparam(0));
869 &mov ($out,&wparam(1));
870 &mov ($len,&wparam(2));
871 &mov ($key,&wparam(3));
872 &mov ($rounds_,&wparam(4));
873 &mov ($key_,"esp");
f8501464 874 &sub ("esp",88);
6c83629b 875 &and ("esp",-16); # align stack
f8501464 876 &mov (&DWP(80,"esp"),$key_);
6c83629b 877
d7d119a3
AP		 878 &cmp ($len,1);
879 &je (&label("ctr32_one_shortcut"));
880
f8501464 881 &movdqu ($inout5,&QWP(0,$rounds_)); # load ivec
6c83629b
AP		 882
883 # compose byte-swap control mask for pshufb on stack
884 &mov (&DWP(0,"esp"),0x0c0d0e0f);
885 &mov (&DWP(4,"esp"),0x08090a0b);
886 &mov (&DWP(8,"esp"),0x04050607);
887 &mov (&DWP(12,"esp"),0x00010203);
888
889 # compose counter increment vector on stack
f8501464 890 &mov ($rounds,6);
6c83629b
AP		 891 &xor ($key_,$key_);
892 &mov (&DWP(16,"esp"),$rounds);
893 &mov (&DWP(20,"esp"),$rounds);
894 &mov (&DWP(24,"esp"),$rounds);
895 &mov (&DWP(28,"esp"),$key_);
896
f8501464
AP		 897 &pextrd ($rounds_,$inout5,3); # pull 32-bit counter
898 &pinsrd ($inout5,$key_,3); # wipe 32-bit counter
6c83629b
AP		 899
900 &mov ($rounds,&DWP(240,$key)); # key->rounds
6c83629b 901
f8501464 902 # compose 2 vectors of 3x32-bit counters
6c83629b 903 &bswap ($rounds_);
f8501464 904 &pxor ($rndkey0,$rndkey0);
d8ba0dc9 905 &pxor ($rndkey1,$rndkey1);
f8501464 906 &movdqa ($inout0,&QWP(0,"esp")); # load byte-swap mask
d8ba0dc9 907 &pinsrd ($rndkey0,$rounds_,0);
f8501464 908 &lea ($key_,&DWP(3,$rounds_));
d8ba0dc9 909 &pinsrd ($rndkey1,$key_,0);
6c83629b 910 &inc ($rounds_);
d8ba0dc9 911 &pinsrd ($rndkey0,$rounds_,1);
f8501464 912 &inc ($key_);
d8ba0dc9 913 &pinsrd ($rndkey1,$key_,1);
6c83629b 914 &inc ($rounds_);
d8ba0dc9 915 &pinsrd ($rndkey0,$rounds_,2);
f8501464 916 &inc ($key_);
d8ba0dc9
AP		 917 &pinsrd ($rndkey1,$key_,2);
918 &movdqa (&QWP(48,"esp"),$rndkey0); # save 1st triplet
f8501464 919 &pshufb ($rndkey0,$inout0); # byte swap
d8ba0dc9
AP		 920 &movdqu ($inout4,&QWP(0,$key)); # key[0]
921 &movdqa (&QWP(64,"esp"),$rndkey1); # save 2nd triplet
922 &pshufb ($rndkey1,$inout0); # byte swap
f8501464 923
d8ba0dc9
AP		 924 &pshufd ($inout0,$rndkey0,3<<6); # place counter to upper dword
925 &pshufd ($inout1,$rndkey0,2<<6);
f8501464
AP		 926 &cmp ($len,6);
927 &jb (&label("ctr32_tail"));
d8ba0dc9
AP		 928 &pxor ($inout5,$inout4); # counter-less ivec^key[0]
929 &shl ($rounds,4);
930 &mov ($rounds_,16);
931 &movdqa (&QWP(32,"esp"),$inout5); # save counter-less ivec^key[0]
f8501464 932 &mov ($key_,$key); # backup $key
d8ba0dc9
AP		 933 &sub ($rounds_,$rounds); # backup twisted $rounds
934 &lea ($key,&DWP(32,$key,$rounds));
f8501464
AP		 935 &sub ($len,6);
936 &jmp (&label("ctr32_loop6"));
937
938&set_label("ctr32_loop6",16);
d8ba0dc9
AP		 939 # inlining _aesni_encrypt6's prologue gives ~6% improvement...
940 &pshufd ($inout2,$rndkey0,1<<6);
941 &movdqa ($rndkey0,&QWP(32,"esp")); # pull counter-less ivec
942 &pshufd ($inout3,$rndkey1,3<<6);
943 &pxor ($inout0,$rndkey0); # merge counter-less ivec
944 &pshufd ($inout4,$rndkey1,2<<6);
d7d119a3 945 &pxor ($inout1,$rndkey0);
d8ba0dc9
AP		 946 &pshufd ($inout5,$rndkey1,1<<6);
947 &$movekey ($rndkey1,&QWP(16,$key_));
f8501464 948 &pxor ($inout2,$rndkey0);
f8501464 949 &pxor ($inout3,$rndkey0);
d8ba0dc9 950 &aesenc ($inout0,$rndkey1);
f8501464 951 &pxor ($inout4,$rndkey0);
f8501464 952 &pxor ($inout5,$rndkey0);
d8ba0dc9
AP		 953 &aesenc ($inout1,$rndkey1);
954 &$movekey ($rndkey0,&QWP(32,$key_));
955 &mov ($rounds,$rounds_);
956 &aesenc ($inout2,$rndkey1);
957 &aesenc ($inout3,$rndkey1);
f8501464 958 &aesenc ($inout4,$rndkey1);
f8501464 959 &aesenc ($inout5,$rndkey1);
d7d119a3 960
f8501464
AP		 961 &call (&label("_aesni_encrypt6_enter"));
962
963 &movups ($rndkey1,&QWP(0,$inp));
964 &movups ($rndkey0,&QWP(0x10,$inp));
965 &xorps ($inout0,$rndkey1);
966 &movups ($rndkey1,&QWP(0x20,$inp));
967 &xorps ($inout1,$rndkey0);
968 &movups (&QWP(0,$out),$inout0);
969 &movdqa ($rndkey0,&QWP(16,"esp")); # load increment
970 &xorps ($inout2,$rndkey1);
d8ba0dc9 971 &movdqa ($rndkey1,&QWP(64,"esp")); # load 2nd triplet
f8501464
AP		 972 &movups (&QWP(0x10,$out),$inout1);
973 &movups (&QWP(0x20,$out),$inout2);
974
d8ba0dc9
AP		 975 &paddd ($rndkey1,$rndkey0); # 2nd triplet increment
976 &paddd ($rndkey0,&QWP(48,"esp")); # 1st triplet increment
f8501464
AP		 977 &movdqa ($inout0,&QWP(0,"esp")); # load byte swap mask
978
979 &movups ($inout1,&QWP(0x30,$inp));
980 &movups ($inout2,&QWP(0x40,$inp));
981 &xorps ($inout3,$inout1);
982 &movups ($inout1,&QWP(0x50,$inp));
983 &lea ($inp,&DWP(0x60,$inp));
d8ba0dc9
AP		 984 &movdqa (&QWP(48,"esp"),$rndkey0); # save 1st triplet
985 &pshufb ($rndkey0,$inout0); # byte swap
f8501464
AP		 986 &xorps ($inout4,$inout2);
987 &movups (&QWP(0x30,$out),$inout3);
988 &xorps ($inout5,$inout1);
d8ba0dc9
AP		 989 &movdqa (&QWP(64,"esp"),$rndkey1); # save 2nd triplet
990 &pshufb ($rndkey1,$inout0); # byte swap
f8501464 991 &movups (&QWP(0x40,$out),$inout4);
d8ba0dc9 992 &pshufd ($inout0,$rndkey0,3<<6);
f8501464
AP		 993 &movups (&QWP(0x50,$out),$inout5);
994 &lea ($out,&DWP(0x60,$out));
d7d119a3 995
d8ba0dc9 996 &pshufd ($inout1,$rndkey0,2<<6);
f8501464
AP		 997 &sub ($len,6);
998 &jnc (&label("ctr32_loop6"));
6c83629b 999
f8501464
AP		 1000 &add ($len,6);
1001 &jz (&label("ctr32_ret"));
d8ba0dc9 1002 &movdqu ($inout5,&QWP(0,$key_));
f8501464 1003 &mov ($key,$key_);
d8ba0dc9
AP		 1004 &pxor ($inout5,&QWP(32,"esp")); # restore count-less ivec
1005 &mov ($rounds,&DWP(240,$key_)); # restore $rounds
6c83629b
AP		 1006
1007&set_label("ctr32_tail");
f8501464 1008 &por ($inout0,$inout5);
d7d119a3 1009 &cmp ($len,2);
6c83629b 1010 &jb (&label("ctr32_one"));
6c83629b 1011
d8ba0dc9 1012 &pshufd ($inout2,$rndkey0,1<<6);
f8501464
AP		 1013 &por ($inout1,$inout5);
1014 &je (&label("ctr32_two"));
6c83629b 1015
d8ba0dc9 1016 &pshufd ($inout3,$rndkey1,3<<6);
f8501464
AP		 1017 &por ($inout2,$inout5);
1018 &cmp ($len,4);
1019 &jb (&label("ctr32_three"));
1020
d8ba0dc9 1021 &pshufd ($inout4,$rndkey1,2<<6);
f8501464
AP		 1022 &por ($inout3,$inout5);
1023 &je (&label("ctr32_four"));
1024
1025 &por ($inout4,$inout5);
1026 &call ("_aesni_encrypt6");
1027 &movups ($rndkey1,&QWP(0,$inp));
1028 &movups ($rndkey0,&QWP(0x10,$inp));
1029 &xorps ($inout0,$rndkey1);
1030 &movups ($rndkey1,&QWP(0x20,$inp));
1031 &xorps ($inout1,$rndkey0);
1032 &movups ($rndkey0,&QWP(0x30,$inp));
1033 &xorps ($inout2,$rndkey1);
1034 &movups ($rndkey1,&QWP(0x40,$inp));
1035 &xorps ($inout3,$rndkey0);
1036 &movups (&QWP(0,$out),$inout0);
1037 &xorps ($inout4,$rndkey1);
1038 &movups (&QWP(0x10,$out),$inout1);
1039 &movups (&QWP(0x20,$out),$inout2);
1040 &movups (&QWP(0x30,$out),$inout3);
1041 &movups (&QWP(0x40,$out),$inout4);
6c83629b
AP		 1042 &jmp (&label("ctr32_ret"));
1043
d7d119a3 1044&set_label("ctr32_one_shortcut",16);
f8501464 1045 &movups ($inout0,&QWP(0,$rounds_)); # load ivec
d7d119a3
AP		 1046 &mov ($rounds,&DWP(240,$key));
1047
1048&set_label("ctr32_one");
6c83629b
AP		 1049 if ($inline)
1050 { &aesni_inline_generate1("enc"); }
1051 else
1052 { &call ("_aesni_encrypt1"); }
f8501464
AP		 1053 &movups ($in0,&QWP(0,$inp));
1054 &xorps ($in0,$inout0);
1055 &movups (&QWP(0,$out),$in0);
6c83629b 1056 &jmp (&label("ctr32_ret"));
d64a7232 1057
6c83629b 1058&set_label("ctr32_two",16);
214368ff 1059 &call ("_aesni_encrypt2");
f8501464
AP		 1060 &movups ($inout3,&QWP(0,$inp));
1061 &movups ($inout4,&QWP(0x10,$inp));
1062 &xorps ($inout0,$inout3);
1063 &xorps ($inout1,$inout4);
1064 &movups (&QWP(0,$out),$inout0);
1065 &movups (&QWP(0x10,$out),$inout1);
6c83629b
AP		 1066 &jmp (&label("ctr32_ret"));
1067
1068&set_label("ctr32_three",16);
1069 &call ("_aesni_encrypt3");
f8501464
AP		 1070 &movups ($inout3,&QWP(0,$inp));
1071 &movups ($inout4,&QWP(0x10,$inp));
1072 &xorps ($inout0,$inout3);
1073 &movups ($inout5,&QWP(0x20,$inp));
1074 &xorps ($inout1,$inout4);
1075 &movups (&QWP(0,$out),$inout0);
1076 &xorps ($inout2,$inout5);
1077 &movups (&QWP(0x10,$out),$inout1);
1078 &movups (&QWP(0x20,$out),$inout2);
1079 &jmp (&label("ctr32_ret"));
1080
1081&set_label("ctr32_four",16);
1082 &call ("_aesni_encrypt4");
1083 &movups ($inout4,&QWP(0,$inp));
1084 &movups ($inout5,&QWP(0x10,$inp));
1085 &movups ($rndkey1,&QWP(0x20,$inp));
1086 &xorps ($inout0,$inout4);
1087 &movups ($rndkey0,&QWP(0x30,$inp));
1088 &xorps ($inout1,$inout5);
1089 &movups (&QWP(0,$out),$inout0);
1090 &xorps ($inout2,$rndkey1);
1091 &movups (&QWP(0x10,$out),$inout1);
1092 &xorps ($inout3,$rndkey0);
1093 &movups (&QWP(0x20,$out),$inout2);
1094 &movups (&QWP(0x30,$out),$inout3);
6c83629b
AP		 1095
1096&set_label("ctr32_ret");
23f6eec7
AP		 1097 &pxor ("xmm0","xmm0"); # clear register bank
1098 &pxor ("xmm1","xmm1");
1099 &pxor ("xmm2","xmm2");
1100 &pxor ("xmm3","xmm3");
1101 &pxor ("xmm4","xmm4");
1102 &movdqa (&QWP(32,"esp"),"xmm0"); # clear stack
1103 &pxor ("xmm5","xmm5");
1104 &movdqa (&QWP(48,"esp"),"xmm0");
1105 &pxor ("xmm6","xmm6");
1106 &movdqa (&QWP(64,"esp"),"xmm0");
1107 &pxor ("xmm7","xmm7");
f8501464 1108 &mov ("esp",&DWP(80,"esp"));
6c83629b 1109&function_end("aesni_ctr32_encrypt_blocks");
f8501464
AP		 1110\f
1111######################################################################
1112# void aesni_xts_[en|de]crypt(const char *inp,char *out,size_t len,
1113# const AES_KEY *key1, const AES_KEY *key2
1114# const unsigned char iv[16]);
1115#
1116{ my ($tweak,$twtmp,$twres,$twmask)=($rndkey1,$rndkey0,$inout0,$inout1);
1117
1118&function_begin("aesni_xts_encrypt");
1119 &mov ($key,&wparam(4)); # key2
1120 &mov ($inp,&wparam(5)); # clear-text tweak
1121
1122 &mov ($rounds,&DWP(240,$key)); # key2->rounds
1123 &movups ($inout0,&QWP(0,$inp));
1124 if ($inline)
1125 { &aesni_inline_generate1("enc"); }
1126 else
1127 { &call ("_aesni_encrypt1"); }
1128
1129 &mov ($inp,&wparam(0));
1130 &mov ($out,&wparam(1));
1131 &mov ($len,&wparam(2));
1132 &mov ($key,&wparam(3)); # key1
1133
1134 &mov ($key_,"esp");
1135 &sub ("esp",16*7+8);
1136 &mov ($rounds,&DWP(240,$key)); # key1->rounds
1137 &and ("esp",-16); # align stack
1138
1139 &mov (&DWP(16*6+0,"esp"),0x87); # compose the magic constant
1140 &mov (&DWP(16*6+4,"esp"),0);
1141 &mov (&DWP(16*6+8,"esp"),1);
1142 &mov (&DWP(16*6+12,"esp"),0);
1143 &mov (&DWP(16*7+0,"esp"),$len); # save original $len
1144 &mov (&DWP(16*7+4,"esp"),$key_); # save original %esp
1145
1146 &movdqa ($tweak,$inout0);
1147 &pxor ($twtmp,$twtmp);
1148 &movdqa ($twmask,&QWP(6*16,"esp")); # 0x0...010...87
1149 &pcmpgtd($twtmp,$tweak); # broadcast upper bits
1150
1151 &and ($len,-16);
1152 &mov ($key_,$key); # backup $key
1153 &mov ($rounds_,$rounds); # backup $rounds
1154 &sub ($len,16*6);
1155 &jc (&label("xts_enc_short"));
1156
d8ba0dc9
AP		 1157 &shl ($rounds,4);
1158 &mov ($rounds_,16);
1159 &sub ($rounds_,$rounds);
1160 &lea ($key,&DWP(32,$key,$rounds));
f8501464
AP		 1161 &jmp (&label("xts_enc_loop6"));
1162
1163&set_label("xts_enc_loop6",16);
1164 for ($i=0;$i<4;$i++) {
1165 &pshufd ($twres,$twtmp,0x13);
1166 &pxor ($twtmp,$twtmp);
1167 &movdqa (&QWP(16*$i,"esp"),$tweak);
1168 &paddq ($tweak,$tweak); # &psllq($tweak,1);
1169 &pand ($twres,$twmask); # isolate carry and residue
1170 &pcmpgtd ($twtmp,$tweak); # broadcast upper bits
1171 &pxor ($tweak,$twres);
1172 }
1173 &pshufd ($inout5,$twtmp,0x13);
1174 &movdqa (&QWP(16*$i++,"esp"),$tweak);
1175 &paddq ($tweak,$tweak); # &psllq($tweak,1);
1176 &$movekey ($rndkey0,&QWP(0,$key_));
1177 &pand ($inout5,$twmask); # isolate carry and residue
1178 &movups ($inout0,&QWP(0,$inp)); # load input
1179 &pxor ($inout5,$tweak);
1180
1181 # inline _aesni_encrypt6 prologue and flip xor with tweak and key[0]
d8ba0dc9 1182 &mov ($rounds,$rounds_); # restore $rounds
f8501464
AP		 1183 &movdqu ($inout1,&QWP(16*1,$inp));
1184 &xorps ($inout0,$rndkey0); # input^=rndkey[0]
1185 &movdqu ($inout2,&QWP(16*2,$inp));
1186 &pxor ($inout1,$rndkey0);
1187 &movdqu ($inout3,&QWP(16*3,$inp));
1188 &pxor ($inout2,$rndkey0);
1189 &movdqu ($inout4,&QWP(16*4,$inp));
1190 &pxor ($inout3,$rndkey0);
1191 &movdqu ($rndkey1,&QWP(16*5,$inp));
1192 &pxor ($inout4,$rndkey0);
1193 &lea ($inp,&DWP(16*6,$inp));
1194 &pxor ($inout0,&QWP(16*0,"esp")); # input^=tweak
1195 &movdqa (&QWP(16*$i,"esp"),$inout5); # save last tweak
1196 &pxor ($inout5,$rndkey1);
1197
1198 &$movekey ($rndkey1,&QWP(16,$key_));
f8501464 1199 &pxor ($inout1,&QWP(16*1,"esp"));
f8501464 1200 &pxor ($inout2,&QWP(16*2,"esp"));
d8ba0dc9 1201 &aesenc ($inout0,$rndkey1);
f8501464 1202 &pxor ($inout3,&QWP(16*3,"esp"));
f8501464 1203 &pxor ($inout4,&QWP(16*4,"esp"));
d8ba0dc9 1204 &aesenc ($inout1,$rndkey1);
f8501464 1205 &pxor ($inout5,$rndkey0);
d8ba0dc9
AP		 1206 &$movekey ($rndkey0,&QWP(32,$key_));
1207 &aesenc ($inout2,$rndkey1);
1208 &aesenc ($inout3,$rndkey1);
f8501464 1209 &aesenc ($inout4,$rndkey1);
f8501464
AP		 1210 &aesenc ($inout5,$rndkey1);
1211 &call (&label("_aesni_encrypt6_enter"));
1212
1213 &movdqa ($tweak,&QWP(16*5,"esp")); # last tweak
1214 &pxor ($twtmp,$twtmp);
1215 &xorps ($inout0,&QWP(16*0,"esp")); # output^=tweak
1216 &pcmpgtd ($twtmp,$tweak); # broadcast upper bits
1217 &xorps ($inout1,&QWP(16*1,"esp"));
1218 &movups (&QWP(16*0,$out),$inout0); # write output
1219 &xorps ($inout2,&QWP(16*2,"esp"));
1220 &movups (&QWP(16*1,$out),$inout1);
1221 &xorps ($inout3,&QWP(16*3,"esp"));
1222 &movups (&QWP(16*2,$out),$inout2);
1223 &xorps ($inout4,&QWP(16*4,"esp"));
1224 &movups (&QWP(16*3,$out),$inout3);
1225 &xorps ($inout5,$tweak);
1226 &movups (&QWP(16*4,$out),$inout4);
1227 &pshufd ($twres,$twtmp,0x13);
1228 &movups (&QWP(16*5,$out),$inout5);
1229 &lea ($out,&DWP(16*6,$out));
1230 &movdqa ($twmask,&QWP(16*6,"esp")); # 0x0...010...87
1231
1232 &pxor ($twtmp,$twtmp);
1233 &paddq ($tweak,$tweak); # &psllq($tweak,1);
1234 &pand ($twres,$twmask); # isolate carry and residue
1235 &pcmpgtd($twtmp,$tweak); # broadcast upper bits
f8501464
AP		 1236 &pxor ($tweak,$twres);
1237
1238 &sub ($len,16*6);
1239 &jnc (&label("xts_enc_loop6"));
1240
d8ba0dc9 1241 &mov ($rounds,&DWP(240,$key_)); # restore $rounds
f8501464
AP		 1242 &mov ($key,$key_); # restore $key
1243 &mov ($rounds_,$rounds);
1244
1245&set_label("xts_enc_short");
1246 &add ($len,16*6);
1247 &jz (&label("xts_enc_done6x"));
1248
1249 &movdqa ($inout3,$tweak); # put aside previous tweak
1250 &cmp ($len,0x20);
1251 &jb (&label("xts_enc_one"));
1252
1253 &pshufd ($twres,$twtmp,0x13);
1254 &pxor ($twtmp,$twtmp);
1255 &paddq ($tweak,$tweak); # &psllq($tweak,1);
1256 &pand ($twres,$twmask); # isolate carry and residue
1257 &pcmpgtd($twtmp,$tweak); # broadcast upper bits
1258 &pxor ($tweak,$twres);
1259 &je (&label("xts_enc_two"));
1260
1261 &pshufd ($twres,$twtmp,0x13);
1262 &pxor ($twtmp,$twtmp);
1263 &movdqa ($inout4,$tweak); # put aside previous tweak
1264 &paddq ($tweak,$tweak); # &psllq($tweak,1);
1265 &pand ($twres,$twmask); # isolate carry and residue
1266 &pcmpgtd($twtmp,$tweak); # broadcast upper bits
1267 &pxor ($tweak,$twres);
1268 &cmp ($len,0x40);
1269 &jb (&label("xts_enc_three"));
1270
1271 &pshufd ($twres,$twtmp,0x13);
1272 &pxor ($twtmp,$twtmp);
1273 &movdqa ($inout5,$tweak); # put aside previous tweak
1274 &paddq ($tweak,$tweak); # &psllq($tweak,1);
1275 &pand ($twres,$twmask); # isolate carry and residue
1276 &pcmpgtd($twtmp,$tweak); # broadcast upper bits
1277 &pxor ($tweak,$twres);
1278 &movdqa (&QWP(16*0,"esp"),$inout3);
1279 &movdqa (&QWP(16*1,"esp"),$inout4);
1280 &je (&label("xts_enc_four"));
1281
1282 &movdqa (&QWP(16*2,"esp"),$inout5);
1283 &pshufd ($inout5,$twtmp,0x13);
1284 &movdqa (&QWP(16*3,"esp"),$tweak);
1285 &paddq ($tweak,$tweak); # &psllq($inout0,1);
1286 &pand ($inout5,$twmask); # isolate carry and residue
1287 &pxor ($inout5,$tweak);
1288
1289 &movdqu ($inout0,&QWP(16*0,$inp)); # load input
1290 &movdqu ($inout1,&QWP(16*1,$inp));
1291 &movdqu ($inout2,&QWP(16*2,$inp));
1292 &pxor ($inout0,&QWP(16*0,"esp")); # input^=tweak
1293 &movdqu ($inout3,&QWP(16*3,$inp));
1294 &pxor ($inout1,&QWP(16*1,"esp"));
1295 &movdqu ($inout4,&QWP(16*4,$inp));
1296 &pxor ($inout2,&QWP(16*2,"esp"));
1297 &lea ($inp,&DWP(16*5,$inp));
1298 &pxor ($inout3,&QWP(16*3,"esp"));
1299 &movdqa (&QWP(16*4,"esp"),$inout5); # save last tweak
1300 &pxor ($inout4,$inout5);
1301
1302 &call ("_aesni_encrypt6");
1303
1304 &movaps ($tweak,&QWP(16*4,"esp")); # last tweak
1305 &xorps ($inout0,&QWP(16*0,"esp")); # output^=tweak
1306 &xorps ($inout1,&QWP(16*1,"esp"));
1307 &xorps ($inout2,&QWP(16*2,"esp"));
1308 &movups (&QWP(16*0,$out),$inout0); # write output
1309 &xorps ($inout3,&QWP(16*3,"esp"));
1310 &movups (&QWP(16*1,$out),$inout1);
1311 &xorps ($inout4,$tweak);
1312 &movups (&QWP(16*2,$out),$inout2);
1313 &movups (&QWP(16*3,$out),$inout3);
1314 &movups (&QWP(16*4,$out),$inout4);
1315 &lea ($out,&DWP(16*5,$out));
1316 &jmp (&label("xts_enc_done"));
1317
1318&set_label("xts_enc_one",16);
1319 &movups ($inout0,&QWP(16*0,$inp)); # load input
1320 &lea ($inp,&DWP(16*1,$inp));
1321 &xorps ($inout0,$inout3); # input^=tweak
1322 if ($inline)
1323 { &aesni_inline_generate1("enc"); }
1324 else
1325 { &call ("_aesni_encrypt1"); }
1326 &xorps ($inout0,$inout3); # output^=tweak
1327 &movups (&QWP(16*0,$out),$inout0); # write output
1328 &lea ($out,&DWP(16*1,$out));
1329
1330 &movdqa ($tweak,$inout3); # last tweak
1331 &jmp (&label("xts_enc_done"));
1332
1333&set_label("xts_enc_two",16);
1334 &movaps ($inout4,$tweak); # put aside last tweak
1335
1336 &movups ($inout0,&QWP(16*0,$inp)); # load input
1337 &movups ($inout1,&QWP(16*1,$inp));
1338 &lea ($inp,&DWP(16*2,$inp));
1339 &xorps ($inout0,$inout3); # input^=tweak
1340 &xorps ($inout1,$inout4);
f8501464 1341
214368ff 1342 &call ("_aesni_encrypt2");
f8501464
AP		 1343
1344 &xorps ($inout0,$inout3); # output^=tweak
1345 &xorps ($inout1,$inout4);
1346 &movups (&QWP(16*0,$out),$inout0); # write output
1347 &movups (&QWP(16*1,$out),$inout1);
1348 &lea ($out,&DWP(16*2,$out));
1349
1350 &movdqa ($tweak,$inout4); # last tweak
1351 &jmp (&label("xts_enc_done"));
1352
1353&set_label("xts_enc_three",16);
1354 &movaps ($inout5,$tweak); # put aside last tweak
1355 &movups ($inout0,&QWP(16*0,$inp)); # load input
1356 &movups ($inout1,&QWP(16*1,$inp));
1357 &movups ($inout2,&QWP(16*2,$inp));
1358 &lea ($inp,&DWP(16*3,$inp));
1359 &xorps ($inout0,$inout3); # input^=tweak
1360 &xorps ($inout1,$inout4);
1361 &xorps ($inout2,$inout5);
1362
1363 &call ("_aesni_encrypt3");
1364
1365 &xorps ($inout0,$inout3); # output^=tweak
1366 &xorps ($inout1,$inout4);
1367 &xorps ($inout2,$inout5);
1368 &movups (&QWP(16*0,$out),$inout0); # write output
1369 &movups (&QWP(16*1,$out),$inout1);
1370 &movups (&QWP(16*2,$out),$inout2);
1371 &lea ($out,&DWP(16*3,$out));
1372
1373 &movdqa ($tweak,$inout5); # last tweak
1374 &jmp (&label("xts_enc_done"));
1375
1376&set_label("xts_enc_four",16);
1377 &movaps ($inout4,$tweak); # put aside last tweak
1378
1379 &movups ($inout0,&QWP(16*0,$inp)); # load input
1380 &movups ($inout1,&QWP(16*1,$inp));
1381 &movups ($inout2,&QWP(16*2,$inp));
1382 &xorps ($inout0,&QWP(16*0,"esp")); # input^=tweak
1383 &movups ($inout3,&QWP(16*3,$inp));
1384 &lea ($inp,&DWP(16*4,$inp));
1385 &xorps ($inout1,&QWP(16*1,"esp"));
1386 &xorps ($inout2,$inout5);
1387 &xorps ($inout3,$inout4);
1388
1389 &call ("_aesni_encrypt4");
1390
1391 &xorps ($inout0,&QWP(16*0,"esp")); # output^=tweak
1392 &xorps ($inout1,&QWP(16*1,"esp"));
1393 &xorps ($inout2,$inout5);
1394 &movups (&QWP(16*0,$out),$inout0); # write output
1395 &xorps ($inout3,$inout4);
1396 &movups (&QWP(16*1,$out),$inout1);
1397 &movups (&QWP(16*2,$out),$inout2);
1398 &movups (&QWP(16*3,$out),$inout3);
1399 &lea ($out,&DWP(16*4,$out));
1400
1401 &movdqa ($tweak,$inout4); # last tweak
1402 &jmp (&label("xts_enc_done"));
1403
1404&set_label("xts_enc_done6x",16); # $tweak is pre-calculated
1405 &mov ($len,&DWP(16*7+0,"esp")); # restore original $len
1406 &and ($len,15);
1407 &jz (&label("xts_enc_ret"));
1408 &movdqa ($inout3,$tweak);
1409 &mov (&DWP(16*7+0,"esp"),$len); # save $len%16
1410 &jmp (&label("xts_enc_steal"));
1411
1412&set_label("xts_enc_done",16);
1413 &mov ($len,&DWP(16*7+0,"esp")); # restore original $len
1414 &pxor ($twtmp,$twtmp);
1415 &and ($len,15);
1416 &jz (&label("xts_enc_ret"));
1417
1418 &pcmpgtd($twtmp,$tweak); # broadcast upper bits
1419 &mov (&DWP(16*7+0,"esp"),$len); # save $len%16
1420 &pshufd ($inout3,$twtmp,0x13);
1421 &paddq ($tweak,$tweak); # &psllq($tweak,1);
1422 &pand ($inout3,&QWP(16*6,"esp")); # isolate carry and residue
1423 &pxor ($inout3,$tweak);
1424
1425&set_label("xts_enc_steal");
1426 &movz ($rounds,&BP(0,$inp));
1427 &movz ($key,&BP(-16,$out));
1428 &lea ($inp,&DWP(1,$inp));
1429 &mov (&BP(-16,$out),&LB($rounds));
1430 &mov (&BP(0,$out),&LB($key));
1431 &lea ($out,&DWP(1,$out));
1432 &sub ($len,1);
1433 &jnz (&label("xts_enc_steal"));
1434
1435 &sub ($out,&DWP(16*7+0,"esp")); # rewind $out
1436 &mov ($key,$key_); # restore $key
1437 &mov ($rounds,$rounds_); # restore $rounds
1438
1439 &movups ($inout0,&QWP(-16,$out)); # load input
1440 &xorps ($inout0,$inout3); # input^=tweak
1441 if ($inline)
1442 { &aesni_inline_generate1("enc"); }
1443 else
1444 { &call ("_aesni_encrypt1"); }
1445 &xorps ($inout0,$inout3); # output^=tweak
1446 &movups (&QWP(-16,$out),$inout0); # write output
1447
1448&set_label("xts_enc_ret");
23f6eec7
AP		 1449 &pxor ("xmm0","xmm0"); # clear register bank
1450 &pxor ("xmm1","xmm1");
1451 &pxor ("xmm2","xmm2");
1452 &movdqa (&QWP(16*0,"esp"),"xmm0"); # clear stack
1453 &pxor ("xmm3","xmm3");
1454 &movdqa (&QWP(16*1,"esp"),"xmm0");
1455 &pxor ("xmm4","xmm4");
1456 &movdqa (&QWP(16*2,"esp"),"xmm0");
1457 &pxor ("xmm5","xmm5");
1458 &movdqa (&QWP(16*3,"esp"),"xmm0");
1459 &pxor ("xmm6","xmm6");
1460 &movdqa (&QWP(16*4,"esp"),"xmm0");
1461 &pxor ("xmm7","xmm7");
1462 &movdqa (&QWP(16*5,"esp"),"xmm0");
f8501464
AP		 1463 &mov ("esp",&DWP(16*7+4,"esp")); # restore %esp
1464&function_end("aesni_xts_encrypt");
1465
1466&function_begin("aesni_xts_decrypt");
1467 &mov ($key,&wparam(4)); # key2
1468 &mov ($inp,&wparam(5)); # clear-text tweak
1469
1470 &mov ($rounds,&DWP(240,$key)); # key2->rounds
1471 &movups ($inout0,&QWP(0,$inp));
1472 if ($inline)
1473 { &aesni_inline_generate1("enc"); }
1474 else
1475 { &call ("_aesni_encrypt1"); }
1476
1477 &mov ($inp,&wparam(0));
1478 &mov ($out,&wparam(1));
1479 &mov ($len,&wparam(2));
1480 &mov ($key,&wparam(3)); # key1
1481
1482 &mov ($key_,"esp");
1483 &sub ("esp",16*7+8);
1484 &and ("esp",-16); # align stack
1485
1486 &xor ($rounds_,$rounds_); # if(len%16) len-=16;
1487 &test ($len,15);
1488 &setnz (&LB($rounds_));
1489 &shl ($rounds_,4);
1490 &sub ($len,$rounds_);
1491
1492 &mov (&DWP(16*6+0,"esp"),0x87); # compose the magic constant
1493 &mov (&DWP(16*6+4,"esp"),0);
1494 &mov (&DWP(16*6+8,"esp"),1);
1495 &mov (&DWP(16*6+12,"esp"),0);
1496 &mov (&DWP(16*7+0,"esp"),$len); # save original $len
1497 &mov (&DWP(16*7+4,"esp"),$key_); # save original %esp
1498
1499 &mov ($rounds,&DWP(240,$key)); # key1->rounds
1500 &mov ($key_,$key); # backup $key
1501 &mov ($rounds_,$rounds); # backup $rounds
1502
1503 &movdqa ($tweak,$inout0);
1504 &pxor ($twtmp,$twtmp);
1505 &movdqa ($twmask,&QWP(6*16,"esp")); # 0x0...010...87
1506 &pcmpgtd($twtmp,$tweak); # broadcast upper bits
1507
1508 &and ($len,-16);
1509 &sub ($len,16*6);
1510 &jc (&label("xts_dec_short"));
1511
d8ba0dc9
AP		 1512 &shl ($rounds,4);
1513 &mov ($rounds_,16);
1514 &sub ($rounds_,$rounds);
1515 &lea ($key,&DWP(32,$key,$rounds));
f8501464
AP		 1516 &jmp (&label("xts_dec_loop6"));
1517
1518&set_label("xts_dec_loop6",16);
1519 for ($i=0;$i<4;$i++) {
1520 &pshufd ($twres,$twtmp,0x13);
1521 &pxor ($twtmp,$twtmp);
1522 &movdqa (&QWP(16*$i,"esp"),$tweak);
1523 &paddq ($tweak,$tweak); # &psllq($tweak,1);
1524 &pand ($twres,$twmask); # isolate carry and residue
1525 &pcmpgtd ($twtmp,$tweak); # broadcast upper bits
1526 &pxor ($tweak,$twres);
1527 }
1528 &pshufd ($inout5,$twtmp,0x13);
1529 &movdqa (&QWP(16*$i++,"esp"),$tweak);
1530 &paddq ($tweak,$tweak); # &psllq($tweak,1);
1531 &$movekey ($rndkey0,&QWP(0,$key_));
1532 &pand ($inout5,$twmask); # isolate carry and residue
1533 &movups ($inout0,&QWP(0,$inp)); # load input
1534 &pxor ($inout5,$tweak);
1535
1536 # inline _aesni_encrypt6 prologue and flip xor with tweak and key[0]
d8ba0dc9 1537 &mov ($rounds,$rounds_);
f8501464
AP		 1538 &movdqu ($inout1,&QWP(16*1,$inp));
1539 &xorps ($inout0,$rndkey0); # input^=rndkey[0]
1540 &movdqu ($inout2,&QWP(16*2,$inp));
1541 &pxor ($inout1,$rndkey0);
1542 &movdqu ($inout3,&QWP(16*3,$inp));
1543 &pxor ($inout2,$rndkey0);
1544 &movdqu ($inout4,&QWP(16*4,$inp));
1545 &pxor ($inout3,$rndkey0);
1546 &movdqu ($rndkey1,&QWP(16*5,$inp));
1547 &pxor ($inout4,$rndkey0);
1548 &lea ($inp,&DWP(16*6,$inp));
1549 &pxor ($inout0,&QWP(16*0,"esp")); # input^=tweak
1550 &movdqa (&QWP(16*$i,"esp"),$inout5); # save last tweak
1551 &pxor ($inout5,$rndkey1);
1552
1553 &$movekey ($rndkey1,&QWP(16,$key_));
f8501464 1554 &pxor ($inout1,&QWP(16*1,"esp"));
f8501464 1555 &pxor ($inout2,&QWP(16*2,"esp"));
d8ba0dc9 1556 &aesdec ($inout0,$rndkey1);
f8501464 1557 &pxor ($inout3,&QWP(16*3,"esp"));
f8501464 1558 &pxor ($inout4,&QWP(16*4,"esp"));
d8ba0dc9 1559 &aesdec ($inout1,$rndkey1);
f8501464 1560 &pxor ($inout5,$rndkey0);
d8ba0dc9
AP		 1561 &$movekey ($rndkey0,&QWP(32,$key_));
1562 &aesdec ($inout2,$rndkey1);
1563 &aesdec ($inout3,$rndkey1);
f8501464 1564 &aesdec ($inout4,$rndkey1);
f8501464
AP		 1565 &aesdec ($inout5,$rndkey1);
1566 &call (&label("_aesni_decrypt6_enter"));
1567
1568 &movdqa ($tweak,&QWP(16*5,"esp")); # last tweak
1569 &pxor ($twtmp,$twtmp);
1570 &xorps ($inout0,&QWP(16*0,"esp")); # output^=tweak
1571 &pcmpgtd ($twtmp,$tweak); # broadcast upper bits
1572 &xorps ($inout1,&QWP(16*1,"esp"));
1573 &movups (&QWP(16*0,$out),$inout0); # write output
1574 &xorps ($inout2,&QWP(16*2,"esp"));
1575 &movups (&QWP(16*1,$out),$inout1);
1576 &xorps ($inout3,&QWP(16*3,"esp"));
1577 &movups (&QWP(16*2,$out),$inout2);
1578 &xorps ($inout4,&QWP(16*4,"esp"));
1579 &movups (&QWP(16*3,$out),$inout3);
1580 &xorps ($inout5,$tweak);
1581 &movups (&QWP(16*4,$out),$inout4);
1582 &pshufd ($twres,$twtmp,0x13);
1583 &movups (&QWP(16*5,$out),$inout5);
1584 &lea ($out,&DWP(16*6,$out));
1585 &movdqa ($twmask,&QWP(16*6,"esp")); # 0x0...010...87
1586
1587 &pxor ($twtmp,$twtmp);
1588 &paddq ($tweak,$tweak); # &psllq($tweak,1);
1589 &pand ($twres,$twmask); # isolate carry and residue
1590 &pcmpgtd($twtmp,$tweak); # broadcast upper bits
f8501464
AP		 1591 &pxor ($tweak,$twres);
1592
1593 &sub ($len,16*6);
1594 &jnc (&label("xts_dec_loop6"));
1595
d8ba0dc9 1596 &mov ($rounds,&DWP(240,$key_)); # restore $rounds
f8501464
AP		 1597 &mov ($key,$key_); # restore $key
1598 &mov ($rounds_,$rounds);
1599
1600&set_label("xts_dec_short");
1601 &add ($len,16*6);
1602 &jz (&label("xts_dec_done6x"));
1603
1604 &movdqa ($inout3,$tweak); # put aside previous tweak
1605 &cmp ($len,0x20);
1606 &jb (&label("xts_dec_one"));
1607
1608 &pshufd ($twres,$twtmp,0x13);
1609 &pxor ($twtmp,$twtmp);
1610 &paddq ($tweak,$tweak); # &psllq($tweak,1);
1611 &pand ($twres,$twmask); # isolate carry and residue
1612 &pcmpgtd($twtmp,$tweak); # broadcast upper bits
1613 &pxor ($tweak,$twres);
1614 &je (&label("xts_dec_two"));
1615
1616 &pshufd ($twres,$twtmp,0x13);
1617 &pxor ($twtmp,$twtmp);
1618 &movdqa ($inout4,$tweak); # put aside previous tweak
1619 &paddq ($tweak,$tweak); # &psllq($tweak,1);
1620 &pand ($twres,$twmask); # isolate carry and residue
1621 &pcmpgtd($twtmp,$tweak); # broadcast upper bits
1622 &pxor ($tweak,$twres);
1623 &cmp ($len,0x40);
1624 &jb (&label("xts_dec_three"));
1625
1626 &pshufd ($twres,$twtmp,0x13);
1627 &pxor ($twtmp,$twtmp);
1628 &movdqa ($inout5,$tweak); # put aside previous tweak
1629 &paddq ($tweak,$tweak); # &psllq($tweak,1);
1630 &pand ($twres,$twmask); # isolate carry and residue
1631 &pcmpgtd($twtmp,$tweak); # broadcast upper bits
1632 &pxor ($tweak,$twres);
1633 &movdqa (&QWP(16*0,"esp"),$inout3);
1634 &movdqa (&QWP(16*1,"esp"),$inout4);
1635 &je (&label("xts_dec_four"));
1636
1637 &movdqa (&QWP(16*2,"esp"),$inout5);
1638 &pshufd ($inout5,$twtmp,0x13);
1639 &movdqa (&QWP(16*3,"esp"),$tweak);
1640 &paddq ($tweak,$tweak); # &psllq($inout0,1);
1641 &pand ($inout5,$twmask); # isolate carry and residue
1642 &pxor ($inout5,$tweak);
1643
1644 &movdqu ($inout0,&QWP(16*0,$inp)); # load input
1645 &movdqu ($inout1,&QWP(16*1,$inp));
1646 &movdqu ($inout2,&QWP(16*2,$inp));
1647 &pxor ($inout0,&QWP(16*0,"esp")); # input^=tweak
1648 &movdqu ($inout3,&QWP(16*3,$inp));
1649 &pxor ($inout1,&QWP(16*1,"esp"));
1650 &movdqu ($inout4,&QWP(16*4,$inp));
1651 &pxor ($inout2,&QWP(16*2,"esp"));
1652 &lea ($inp,&DWP(16*5,$inp));
1653 &pxor ($inout3,&QWP(16*3,"esp"));
1654 &movdqa (&QWP(16*4,"esp"),$inout5); # save last tweak
1655 &pxor ($inout4,$inout5);
1656
1657 &call ("_aesni_decrypt6");
1658
1659 &movaps ($tweak,&QWP(16*4,"esp")); # last tweak
1660 &xorps ($inout0,&QWP(16*0,"esp")); # output^=tweak
1661 &xorps ($inout1,&QWP(16*1,"esp"));
1662 &xorps ($inout2,&QWP(16*2,"esp"));
1663 &movups (&QWP(16*0,$out),$inout0); # write output
1664 &xorps ($inout3,&QWP(16*3,"esp"));
1665 &movups (&QWP(16*1,$out),$inout1);
1666 &xorps ($inout4,$tweak);
1667 &movups (&QWP(16*2,$out),$inout2);
1668 &movups (&QWP(16*3,$out),$inout3);
1669 &movups (&QWP(16*4,$out),$inout4);
1670 &lea ($out,&DWP(16*5,$out));
1671 &jmp (&label("xts_dec_done"));
1672
1673&set_label("xts_dec_one",16);
1674 &movups ($inout0,&QWP(16*0,$inp)); # load input
1675 &lea ($inp,&DWP(16*1,$inp));
1676 &xorps ($inout0,$inout3); # input^=tweak
1677 if ($inline)
1678 { &aesni_inline_generate1("dec"); }
1679 else
1680 { &call ("_aesni_decrypt1"); }
1681 &xorps ($inout0,$inout3); # output^=tweak
1682 &movups (&QWP(16*0,$out),$inout0); # write output
1683 &lea ($out,&DWP(16*1,$out));
1684
1685 &movdqa ($tweak,$inout3); # last tweak
1686 &jmp (&label("xts_dec_done"));
1687
1688&set_label("xts_dec_two",16);
1689 &movaps ($inout4,$tweak); # put aside last tweak
1690
1691 &movups ($inout0,&QWP(16*0,$inp)); # load input
1692 &movups ($inout1,&QWP(16*1,$inp));
1693 &lea ($inp,&DWP(16*2,$inp));
1694 &xorps ($inout0,$inout3); # input^=tweak
1695 &xorps ($inout1,$inout4);
1696
214368ff 1697 &call ("_aesni_decrypt2");
f8501464
AP		 1698
1699 &xorps ($inout0,$inout3); # output^=tweak
1700 &xorps ($inout1,$inout4);
1701 &movups (&QWP(16*0,$out),$inout0); # write output
1702 &movups (&QWP(16*1,$out),$inout1);
1703 &lea ($out,&DWP(16*2,$out));
1704
1705 &movdqa ($tweak,$inout4); # last tweak
1706 &jmp (&label("xts_dec_done"));
1707
1708&set_label("xts_dec_three",16);
1709 &movaps ($inout5,$tweak); # put aside last tweak
1710 &movups ($inout0,&QWP(16*0,$inp)); # load input
1711 &movups ($inout1,&QWP(16*1,$inp));
1712 &movups ($inout2,&QWP(16*2,$inp));
1713 &lea ($inp,&DWP(16*3,$inp));
1714 &xorps ($inout0,$inout3); # input^=tweak
1715 &xorps ($inout1,$inout4);
1716 &xorps ($inout2,$inout5);
1717
1718 &call ("_aesni_decrypt3");
1719
1720 &xorps ($inout0,$inout3); # output^=tweak
1721 &xorps ($inout1,$inout4);
1722 &xorps ($inout2,$inout5);
1723 &movups (&QWP(16*0,$out),$inout0); # write output
1724 &movups (&QWP(16*1,$out),$inout1);
1725 &movups (&QWP(16*2,$out),$inout2);
1726 &lea ($out,&DWP(16*3,$out));
1727
1728 &movdqa ($tweak,$inout5); # last tweak
1729 &jmp (&label("xts_dec_done"));
1730
1731&set_label("xts_dec_four",16);
1732 &movaps ($inout4,$tweak); # put aside last tweak
1733
1734 &movups ($inout0,&QWP(16*0,$inp)); # load input
1735 &movups ($inout1,&QWP(16*1,$inp));
1736 &movups ($inout2,&QWP(16*2,$inp));
1737 &xorps ($inout0,&QWP(16*0,"esp")); # input^=tweak
1738 &movups ($inout3,&QWP(16*3,$inp));
1739 &lea ($inp,&DWP(16*4,$inp));
1740 &xorps ($inout1,&QWP(16*1,"esp"));
1741 &xorps ($inout2,$inout5);
1742 &xorps ($inout3,$inout4);
1743
1744 &call ("_aesni_decrypt4");
1745
1746 &xorps ($inout0,&QWP(16*0,"esp")); # output^=tweak
1747 &xorps ($inout1,&QWP(16*1,"esp"));
1748 &xorps ($inout2,$inout5);
1749 &movups (&QWP(16*0,$out),$inout0); # write output
1750 &xorps ($inout3,$inout4);
1751 &movups (&QWP(16*1,$out),$inout1);
1752 &movups (&QWP(16*2,$out),$inout2);
1753 &movups (&QWP(16*3,$out),$inout3);
1754 &lea ($out,&DWP(16*4,$out));
1755
1756 &movdqa ($tweak,$inout4); # last tweak
1757 &jmp (&label("xts_dec_done"));
1758
1759&set_label("xts_dec_done6x",16); # $tweak is pre-calculated
1760 &mov ($len,&DWP(16*7+0,"esp")); # restore original $len
1761 &and ($len,15);
1762 &jz (&label("xts_dec_ret"));
1763 &mov (&DWP(16*7+0,"esp"),$len); # save $len%16
1764 &jmp (&label("xts_dec_only_one_more"));
1765
1766&set_label("xts_dec_done",16);
1767 &mov ($len,&DWP(16*7+0,"esp")); # restore original $len
1768 &pxor ($twtmp,$twtmp);
1769 &and ($len,15);
1770 &jz (&label("xts_dec_ret"));
1771
1772 &pcmpgtd($twtmp,$tweak); # broadcast upper bits
1773 &mov (&DWP(16*7+0,"esp"),$len); # save $len%16
1774 &pshufd ($twres,$twtmp,0x13);
1775 &pxor ($twtmp,$twtmp);
1776 &movdqa ($twmask,&QWP(16*6,"esp"));
1777 &paddq ($tweak,$tweak); # &psllq($tweak,1);
1778 &pand ($twres,$twmask); # isolate carry and residue
1779 &pcmpgtd($twtmp,$tweak); # broadcast upper bits
1780 &pxor ($tweak,$twres);
1781
1782&set_label("xts_dec_only_one_more");
1783 &pshufd ($inout3,$twtmp,0x13);
1784 &movdqa ($inout4,$tweak); # put aside previous tweak
1785 &paddq ($tweak,$tweak); # &psllq($tweak,1);
1786 &pand ($inout3,$twmask); # isolate carry and residue
1787 &pxor ($inout3,$tweak);
1788
1789 &mov ($key,$key_); # restore $key
1790 &mov ($rounds,$rounds_); # restore $rounds
1791
1792 &movups ($inout0,&QWP(0,$inp)); # load input
1793 &xorps ($inout0,$inout3); # input^=tweak
1794 if ($inline)
1795 { &aesni_inline_generate1("dec"); }
1796 else
1797 { &call ("_aesni_decrypt1"); }
1798 &xorps ($inout0,$inout3); # output^=tweak
1799 &movups (&QWP(0,$out),$inout0); # write output
1800
1801&set_label("xts_dec_steal");
1802 &movz ($rounds,&BP(16,$inp));
1803 &movz ($key,&BP(0,$out));
1804 &lea ($inp,&DWP(1,$inp));
1805 &mov (&BP(0,$out),&LB($rounds));
1806 &mov (&BP(16,$out),&LB($key));
1807 &lea ($out,&DWP(1,$out));
1808 &sub ($len,1);
1809 &jnz (&label("xts_dec_steal"));
1810
1811 &sub ($out,&DWP(16*7+0,"esp")); # rewind $out
1812 &mov ($key,$key_); # restore $key
1813 &mov ($rounds,$rounds_); # restore $rounds
1814
1815 &movups ($inout0,&QWP(0,$out)); # load input
1816 &xorps ($inout0,$inout4); # input^=tweak
1817 if ($inline)
1818 { &aesni_inline_generate1("dec"); }
1819 else
1820 { &call ("_aesni_decrypt1"); }
1821 &xorps ($inout0,$inout4); # output^=tweak
1822 &movups (&QWP(0,$out),$inout0); # write output
1823
1824&set_label("xts_dec_ret");
23f6eec7
AP		 1825 &pxor ("xmm0","xmm0"); # clear register bank
1826 &pxor ("xmm1","xmm1");
1827 &pxor ("xmm2","xmm2");
1828 &movdqa (&QWP(16*0,"esp"),"xmm0"); # clear stack
1829 &pxor ("xmm3","xmm3");
1830 &movdqa (&QWP(16*1,"esp"),"xmm0");
1831 &pxor ("xmm4","xmm4");
1832 &movdqa (&QWP(16*2,"esp"),"xmm0");
1833 &pxor ("xmm5","xmm5");
1834 &movdqa (&QWP(16*3,"esp"),"xmm0");
1835 &pxor ("xmm6","xmm6");
1836 &movdqa (&QWP(16*4,"esp"),"xmm0");
1837 &pxor ("xmm7","xmm7");
1838 &movdqa (&QWP(16*5,"esp"),"xmm0");
f8501464
AP		 1839 &mov ("esp",&DWP(16*7+4,"esp")); # restore %esp
1840&function_end("aesni_xts_decrypt");
1841}
bd30091c
AP		 1842\f
1843######################################################################
1844# void aesni_ocb_[en|de]crypt(const char *inp, char *out, size_t blocks,
1845# const AES_KEY *key, unsigned int start_block_num,
1846# unsigned char offset_i[16], const unsigned char L_[][16],
1847# unsigned char checksum[16]);
1848#
1849{
1850# offsets within stack frame
1851my $checksum = 16*6;
1852my ($key_off,$rounds_off,$out_off,$end_off,$esp_off)=map(16*7+4*$_,(0..4));
1853
1854# reassigned registers
1855my ($l_,$block,$i1,$i3,$i5) = ($rounds_,$key_,$rounds,$len,$out);
1856# $l_, $blocks, $inp, $key are permanently allocated in registers;
1857# remaining non-volatile ones are offloaded to stack, which even
1858# stay invariant after written to stack.
1859
1860&function_begin("aesni_ocb_encrypt");
1861 &mov ($rounds,&wparam(5)); # &offset_i
1862 &mov ($rounds_,&wparam(7)); # &checksum
1863
1864 &mov ($inp,&wparam(0));
1865 &mov ($out,&wparam(1));
1866 &mov ($len,&wparam(2));
1867 &mov ($key,&wparam(3));
1868 &movdqu ($rndkey0,&QWP(0,$rounds)); # load offset_i
1869 &mov ($block,&wparam(4)); # start_block_num
1870 &movdqu ($rndkey1,&QWP(0,$rounds_)); # load checksum
1871 &mov ($l_,&wparam(6)); # L_
1872
1873 &mov ($rounds,"esp");
1874 &sub ("esp",$esp_off+4); # alloca
1875 &and ("esp",-16); # align stack
1876
1877 &sub ($out,$inp);
1878 &shl ($len,4);
1879 &lea ($len,&DWP(-16*6,$inp,$len)); # end of input - 16*6
1880 &mov (&DWP($out_off,"esp"),$out);
1881 &mov (&DWP($end_off,"esp"),$len);
1882 &mov (&DWP($esp_off,"esp"),$rounds);
1883
1884 &mov ($rounds,&DWP(240,$key));
1885
1886 &test ($block,1);
1887 &jnz (&label("odd"));
1888
1889 &bsf ($i3,$block);
1890 &add ($block,1);
1891 &shl ($i3,4);
1892 &movdqu ($inout5,&QWP(0,$l_,$i3));
1893 &mov ($i3,$key); # put aside key
1894
1895 &movdqu ($inout0,&QWP(16*0,$inp)); # load input
1896 &lea ($inp,&DWP(16,$inp));
1897
1898 &pxor ($inout5,$rndkey0); # ^ last offset_i
1899 &pxor ($rndkey1,$inout0); # checksum
1900 &pxor ($inout0,$inout5); # ^ offset_i
1901
1902 &movdqa ($inout4,$rndkey1);
1903 if ($inline)
1904 { &aesni_inline_generate1("enc"); }
1905 else
1906 { &call ("_aesni_encrypt1"); }
1907
1908 &xorps ($inout0,$inout5); # ^ offset_i
1909 &movdqa ($rndkey0,$inout5); # pass last offset_i
1910 &movdqa ($rndkey1,$inout4); # pass the checksum
1911
1912 &movups (&QWP(-16,$out,$inp),$inout0); # store output
1913
1914 &mov ($rounds,&DWP(240,$i3));
1915 &mov ($key,$i3); # restore key
1916 &mov ($len,&DWP($end_off,"esp"));
1917
1918&set_label("odd");
1919 &shl ($rounds,4);
1920 &mov ($out,16);
1921 &sub ($out,$rounds); # twisted rounds
1922 &mov (&DWP($key_off,"esp"),$key);
1923 &lea ($key,&DWP(32,$key,$rounds)); # end of key schedule
1924 &mov (&DWP($rounds_off,"esp"),$out);
1925
1926 &cmp ($inp,$len);
1927 &ja (&label("short"));
1928 &jmp (&label("grandloop"));
1929
1930&set_label("grandloop",32);
1931 &lea ($i1,&DWP(1,$block));
1932 &lea ($i3,&DWP(3,$block));
1933 &lea ($i5,&DWP(5,$block));
1934 &add ($block,6);
1935 &bsf ($i1,$i1);
1936 &bsf ($i3,$i3);
1937 &bsf ($i5,$i5);
1938 &shl ($i1,4);
1939 &shl ($i3,4);
1940 &shl ($i5,4);
1941 &movdqu ($inout0,&QWP(0,$l_));
1942 &movdqu ($inout1,&QWP(0,$l_,$i1));
1943 &mov ($rounds,&DWP($rounds_off,"esp"));
1944 &movdqa ($inout2,$inout0);
1945 &movdqu ($inout3,&QWP(0,$l_,$i3));
1946 &movdqa ($inout4,$inout0);
1947 &movdqu ($inout5,&QWP(0,$l_,$i5));
1948
1949 &pxor ($inout0,$rndkey0); # ^ last offset_i
1950 &pxor ($inout1,$inout0);
1951 &movdqa (&QWP(16*0,"esp"),$inout0);
1952 &pxor ($inout2,$inout1);
1953 &movdqa (&QWP(16*1,"esp"),$inout1);
1954 &pxor ($inout3,$inout2);
1955 &movdqa (&QWP(16*2,"esp"),$inout2);
1956 &pxor ($inout4,$inout3);
1957 &movdqa (&QWP(16*3,"esp"),$inout3);
1958 &pxor ($inout5,$inout4);
1959 &movdqa (&QWP(16*4,"esp"),$inout4);
1960 &movdqa (&QWP(16*5,"esp"),$inout5);
1961
1962 &$movekey ($rndkey0,&QWP(-48,$key,$rounds));
1963 &movdqu ($inout0,&QWP(16*0,$inp)); # load input
1964 &movdqu ($inout1,&QWP(16*1,$inp));
1965 &movdqu ($inout2,&QWP(16*2,$inp));
1966 &movdqu ($inout3,&QWP(16*3,$inp));
1967 &movdqu ($inout4,&QWP(16*4,$inp));
1968 &movdqu ($inout5,&QWP(16*5,$inp));
1969 &lea ($inp,&DWP(16*6,$inp));
1970
1971 &pxor ($rndkey1,$inout0); # checksum
1972 &pxor ($inout0,$rndkey0); # ^ roundkey[0]
1973 &pxor ($rndkey1,$inout1);
1974 &pxor ($inout1,$rndkey0);
1975 &pxor ($rndkey1,$inout2);
1976 &pxor ($inout2,$rndkey0);
1977 &pxor ($rndkey1,$inout3);
1978 &pxor ($inout3,$rndkey0);
1979 &pxor ($rndkey1,$inout4);
1980 &pxor ($inout4,$rndkey0);
1981 &pxor ($rndkey1,$inout5);
1982 &pxor ($inout5,$rndkey0);
1983 &movdqa (&QWP($checksum,"esp"),$rndkey1);
1984
1985 &$movekey ($rndkey1,&QWP(-32,$key,$rounds));
1986 &pxor ($inout0,&QWP(16*0,"esp")); # ^ offset_i
1987 &pxor ($inout1,&QWP(16*1,"esp"));
1988 &pxor ($inout2,&QWP(16*2,"esp"));
1989 &pxor ($inout3,&QWP(16*3,"esp"));
1990 &pxor ($inout4,&QWP(16*4,"esp"));
1991 &pxor ($inout5,&QWP(16*5,"esp"));
1992
1993 &$movekey ($rndkey0,&QWP(-16,$key,$rounds));
1994 &aesenc ($inout0,$rndkey1);
1995 &aesenc ($inout1,$rndkey1);
1996 &aesenc ($inout2,$rndkey1);
1997 &aesenc ($inout3,$rndkey1);
1998 &aesenc ($inout4,$rndkey1);
1999 &aesenc ($inout5,$rndkey1);
2000
2001 &mov ($out,&DWP($out_off,"esp"));
2002 &mov ($len,&DWP($end_off,"esp"));
2003 &call ("_aesni_encrypt6_enter");
2004
2005 &movdqa ($rndkey0,&QWP(16*5,"esp")); # pass last offset_i
2006 &pxor ($inout0,&QWP(16*0,"esp")); # ^ offset_i
2007 &pxor ($inout1,&QWP(16*1,"esp"));
2008 &pxor ($inout2,&QWP(16*2,"esp"));
2009 &pxor ($inout3,&QWP(16*3,"esp"));
2010 &pxor ($inout4,&QWP(16*4,"esp"));
2011 &pxor ($inout5,$rndkey0);
2012 &movdqa ($rndkey1,&QWP($checksum,"esp"));# pass the checksum
2013
2014 &movdqu (&QWP(-16*6,$out,$inp),$inout0);# store output
2015 &movdqu (&QWP(-16*5,$out,$inp),$inout1);
2016 &movdqu (&QWP(-16*4,$out,$inp),$inout2);
2017 &movdqu (&QWP(-16*3,$out,$inp),$inout3);
2018 &movdqu (&QWP(-16*2,$out,$inp),$inout4);
2019 &movdqu (&QWP(-16*1,$out,$inp),$inout5);
2020 &cmp ($inp,$len); # done yet?
2021 &jb (&label("grandloop"));
2022
2023&set_label("short");
2024 &add ($len,16*6);
2025 &sub ($len,$inp);
2026 &jz (&label("done"));
2027
2028 &cmp ($len,16*2);
2029 &jb (&label("one"));
2030 &je (&label("two"));
2031
2032 &cmp ($len,16*4);
2033 &jb (&label("three"));
2034 &je (&label("four"));
2035
2036 &lea ($i1,&DWP(1,$block));
2037 &lea ($i3,&DWP(3,$block));
2038 &bsf ($i1,$i1);
2039 &bsf ($i3,$i3);
2040 &shl ($i1,4);
2041 &shl ($i3,4);
2042 &movdqu ($inout0,&QWP(0,$l_));
2043 &movdqu ($inout1,&QWP(0,$l_,$i1));
2044 &mov ($rounds,&DWP($rounds_off,"esp"));
2045 &movdqa ($inout2,$inout0);
2046 &movdqu ($inout3,&QWP(0,$l_,$i3));
2047 &movdqa ($inout4,$inout0);
2048
2049 &pxor ($inout0,$rndkey0); # ^ last offset_i
2050 &pxor ($inout1,$inout0);
2051 &movdqa (&QWP(16*0,"esp"),$inout0);
2052 &pxor ($inout2,$inout1);
2053 &movdqa (&QWP(16*1,"esp"),$inout1);
2054 &pxor ($inout3,$inout2);
2055 &movdqa (&QWP(16*2,"esp"),$inout2);
2056 &pxor ($inout4,$inout3);
2057 &movdqa (&QWP(16*3,"esp"),$inout3);
2058 &pxor ($inout5,$inout4);
2059 &movdqa (&QWP(16*4,"esp"),$inout4);
2060
2061 &$movekey ($rndkey0,&QWP(-48,$key,$rounds));
2062 &movdqu ($inout0,&QWP(16*0,$inp)); # load input
2063 &movdqu ($inout1,&QWP(16*1,$inp));
2064 &movdqu ($inout2,&QWP(16*2,$inp));
2065 &movdqu ($inout3,&QWP(16*3,$inp));
2066 &movdqu ($inout4,&QWP(16*4,$inp));
2067 &pxor ($inout5,$inout5);
2068
2069 &pxor ($rndkey1,$inout0); # checksum
2070 &pxor ($inout0,$rndkey0); # ^ roundkey[0]
2071 &pxor ($rndkey1,$inout1);
2072 &pxor ($inout1,$rndkey0);
2073 &pxor ($rndkey1,$inout2);
2074 &pxor ($inout2,$rndkey0);
2075 &pxor ($rndkey1,$inout3);
2076 &pxor ($inout3,$rndkey0);
2077 &pxor ($rndkey1,$inout4);
2078 &pxor ($inout4,$rndkey0);
2079 &movdqa (&QWP($checksum,"esp"),$rndkey1);
2080
2081 &$movekey ($rndkey1,&QWP(-32,$key,$rounds));
2082 &pxor ($inout0,&QWP(16*0,"esp")); # ^ offset_i
2083 &pxor ($inout1,&QWP(16*1,"esp"));
2084 &pxor ($inout2,&QWP(16*2,"esp"));
2085 &pxor ($inout3,&QWP(16*3,"esp"));
2086 &pxor ($inout4,&QWP(16*4,"esp"));
2087
2088 &$movekey ($rndkey0,&QWP(-16,$key,$rounds));
2089 &aesenc ($inout0,$rndkey1);
2090 &aesenc ($inout1,$rndkey1);
2091 &aesenc ($inout2,$rndkey1);
2092 &aesenc ($inout3,$rndkey1);
2093 &aesenc ($inout4,$rndkey1);
2094 &aesenc ($inout5,$rndkey1);
2095
2096 &mov ($out,&DWP($out_off,"esp"));
2097 &call ("_aesni_encrypt6_enter");
2098
2099 &movdqa ($rndkey0,&QWP(16*4,"esp")); # pass last offset_i
2100 &pxor ($inout0,&QWP(16*0,"esp")); # ^ offset_i
2101 &pxor ($inout1,&QWP(16*1,"esp"));
2102 &pxor ($inout2,&QWP(16*2,"esp"));
2103 &pxor ($inout3,&QWP(16*3,"esp"));
2104 &pxor ($inout4,$rndkey0);
2105 &movdqa ($rndkey1,&QWP($checksum,"esp"));# pass the checksum
2106
2107 &movdqu (&QWP(16*0,$out,$inp),$inout0); # store output
2108 &movdqu (&QWP(16*1,$out,$inp),$inout1);
2109 &movdqu (&QWP(16*2,$out,$inp),$inout2);
2110 &movdqu (&QWP(16*3,$out,$inp),$inout3);
2111 &movdqu (&QWP(16*4,$out,$inp),$inout4);
2112
2113 &jmp (&label("done"));
2114
2115&set_label("one",16);
2116 &movdqu ($inout5,&QWP(0,$l_));
2117 &mov ($key,&DWP($key_off,"esp")); # restore key
2118
2119 &movdqu ($inout0,&QWP(16*0,$inp)); # load input
2120 &mov ($rounds,&DWP(240,$key));
2121
2122 &pxor ($inout5,$rndkey0); # ^ last offset_i
2123 &pxor ($rndkey1,$inout0); # checksum
2124 &pxor ($inout0,$inout5); # ^ offset_i
2125
2126 &movdqa ($inout4,$rndkey1);
2127 &mov ($out,&DWP($out_off,"esp"));
2128 if ($inline)
2129 { &aesni_inline_generate1("enc"); }
2130 else
2131 { &call ("_aesni_encrypt1"); }
2132
2133 &xorps ($inout0,$inout5); # ^ offset_i
2134 &movdqa ($rndkey0,$inout5); # pass last offset_i
2135 &movdqa ($rndkey1,$inout4); # pass the checksum
2136 &movups (&QWP(0,$out,$inp),$inout0);
2137
2138 &jmp (&label("done"));
2139
2140&set_label("two",16);
2141 &lea ($i1,&DWP(1,$block));
2142 &mov ($key,&DWP($key_off,"esp")); # restore key
2143 &bsf ($i1,$i1);
2144 &shl ($i1,4);
2145 &movdqu ($inout4,&QWP(0,$l_));
2146 &movdqu ($inout5,&QWP(0,$l_,$i1));
2147
2148 &movdqu ($inout0,&QWP(16*0,$inp)); # load input
2149 &movdqu ($inout1,&QWP(16*1,$inp));
2150 &mov ($rounds,&DWP(240,$key));
2151
2152 &pxor ($inout4,$rndkey0); # ^ last offset_i
2153 &pxor ($inout5,$inout4);
2154
2155 &pxor ($rndkey1,$inout0); # checksum
2156 &pxor ($inout0,$inout4); # ^ offset_i
2157 &pxor ($rndkey1,$inout1);
2158 &pxor ($inout1,$inout5);
2159
2160 &movdqa ($inout3,$rndkey1)
2161 &mov ($out,&DWP($out_off,"esp"));
2162 &call ("_aesni_encrypt2");
2163
2164 &xorps ($inout0,$inout4); # ^ offset_i
2165 &xorps ($inout1,$inout5);
2166 &movdqa ($rndkey0,$inout5); # pass last offset_i
2167 &movdqa ($rndkey1,$inout3); # pass the checksum
2168 &movups (&QWP(16*0,$out,$inp),$inout0); # store output
2169 &movups (&QWP(16*1,$out,$inp),$inout1);
2170
2171 &jmp (&label("done"));
2172
2173&set_label("three",16);
2174 &lea ($i1,&DWP(1,$block));
2175 &mov ($key,&DWP($key_off,"esp")); # restore key
2176 &bsf ($i1,$i1);
2177 &shl ($i1,4);
2178 &movdqu ($inout3,&QWP(0,$l_));
2179 &movdqu ($inout4,&QWP(0,$l_,$i1));
2180 &movdqa ($inout5,$inout3);
2181
2182 &movdqu ($inout0,&QWP(16*0,$inp)); # load input
2183 &movdqu ($inout1,&QWP(16*1,$inp));
2184 &movdqu ($inout2,&QWP(16*2,$inp));
2185 &mov ($rounds,&DWP(240,$key));
2186
2187 &pxor ($inout3,$rndkey0); # ^ last offset_i
2188 &pxor ($inout4,$inout3);
2189 &pxor ($inout5,$inout4);
2190
2191 &pxor ($rndkey1,$inout0); # checksum
2192 &pxor ($inout0,$inout3); # ^ offset_i
2193 &pxor ($rndkey1,$inout1);
2194 &pxor ($inout1,$inout4);
2195 &pxor ($rndkey1,$inout2);
2196 &pxor ($inout2,$inout5);
2197
2198 &movdqa (&QWP($checksum,"esp"),$rndkey1);
2199 &mov ($out,&DWP($out_off,"esp"));
2200 &call ("_aesni_encrypt3");
2201
2202 &xorps ($inout0,$inout3); # ^ offset_i
2203 &xorps ($inout1,$inout4);
2204 &xorps ($inout2,$inout5);
2205 &movdqa ($rndkey0,$inout5); # pass last offset_i
2206 &movdqa ($rndkey1,&QWP($checksum,"esp"));# pass the checksum
2207 &movups (&QWP(16*0,$out,$inp),$inout0); # store output
2208 &movups (&QWP(16*1,$out,$inp),$inout1);
2209 &movups (&QWP(16*2,$out,$inp),$inout2);
2210
2211 &jmp (&label("done"));
2212
2213&set_label("four",16);
2214 &lea ($i1,&DWP(1,$block));
2215 &lea ($i3,&DWP(3,$block));
2216 &bsf ($i1,$i1);
2217 &bsf ($i3,$i3);
2218 &mov ($key,&DWP($key_off,"esp")); # restore key
2219 &shl ($i1,4);
2220 &shl ($i3,4);
2221 &movdqu ($inout2,&QWP(0,$l_));
2222 &movdqu ($inout3,&QWP(0,$l_,$i1));
2223 &movdqa ($inout4,$inout2);
2224 &movdqu ($inout5,&QWP(0,$l_,$i3));
2225
2226 &pxor ($inout2,$rndkey0); # ^ last offset_i
2227 &movdqu ($inout0,&QWP(16*0,$inp)); # load input
2228 &pxor ($inout3,$inout2);
2229 &movdqu ($inout1,&QWP(16*1,$inp));
2230 &pxor ($inout4,$inout3);
2231 &movdqa (&QWP(16*0,"esp"),$inout2);
2232 &pxor ($inout5,$inout4);
2233 &movdqa (&QWP(16*1,"esp"),$inout3);
2234 &movdqu ($inout2,&QWP(16*2,$inp));
2235 &movdqu ($inout3,&QWP(16*3,$inp));
2236 &mov ($rounds,&DWP(240,$key));
2237
2238 &pxor ($rndkey1,$inout0); # checksum
2239 &pxor ($inout0,&QWP(16*0,"esp")); # ^ offset_i
2240 &pxor ($rndkey1,$inout1);
2241 &pxor ($inout1,&QWP(16*1,"esp"));
2242 &pxor ($rndkey1,$inout2);
2243 &pxor ($inout2,$inout4);
2244 &pxor ($rndkey1,$inout3);
2245 &pxor ($inout3,$inout5);
2246
2247 &movdqa (&QWP($checksum,"esp"),$rndkey1)
2248 &mov ($out,&DWP($out_off,"esp"));
2249 &call ("_aesni_encrypt4");
2250
2251 &xorps ($inout0,&QWP(16*0,"esp")); # ^ offset_i
2252 &xorps ($inout1,&QWP(16*1,"esp"));
2253 &xorps ($inout2,$inout4);
2254 &movups (&QWP(16*0,$out,$inp),$inout0); # store output
2255 &xorps ($inout3,$inout5);
2256 &movups (&QWP(16*1,$out,$inp),$inout1);
2257 &movdqa ($rndkey0,$inout5); # pass last offset_i
2258 &movups (&QWP(16*2,$out,$inp),$inout2);
2259 &movdqa ($rndkey1,&QWP($checksum,"esp"));# pass the checksum
2260 &movups (&QWP(16*3,$out,$inp),$inout3);
2261
2262&set_label("done");
2263 &mov ($key,&DWP($esp_off,"esp"));
2264 &pxor ($inout0,$inout0); # clear register bank
2265 &pxor ($inout1,$inout1);
2266 &movdqa (&QWP(16*0,"esp"),$inout0); # clear stack
2267 &pxor ($inout2,$inout2);
2268 &movdqa (&QWP(16*1,"esp"),$inout0);
2269 &pxor ($inout3,$inout3);
2270 &movdqa (&QWP(16*2,"esp"),$inout0);
2271 &pxor ($inout4,$inout4);
2272 &movdqa (&QWP(16*3,"esp"),$inout0);
2273 &pxor ($inout5,$inout5);
2274 &movdqa (&QWP(16*4,"esp"),$inout0);
2275 &movdqa (&QWP(16*5,"esp"),$inout0);
2276 &movdqa (&QWP(16*6,"esp"),$inout0);
2277
2278 &lea ("esp",&DWP(0,$key));
2279 &mov ($rounds,&wparam(5)); # &offset_i
2280 &mov ($rounds_,&wparam(7)); # &checksum
2281 &movdqu (&QWP(0,$rounds),$rndkey0);
2282 &pxor ($rndkey0,$rndkey0);
2283 &movdqu (&QWP(0,$rounds_),$rndkey1);
2284 &pxor ($rndkey1,$rndkey1);
2285&function_end("aesni_ocb_encrypt");
2286
2287&function_begin("aesni_ocb_decrypt");
2288 &mov ($rounds,&wparam(5)); # &offset_i
2289 &mov ($rounds_,&wparam(7)); # &checksum
2290
2291 &mov ($inp,&wparam(0));
2292 &mov ($out,&wparam(1));
2293 &mov ($len,&wparam(2));
2294 &mov ($key,&wparam(3));
2295 &movdqu ($rndkey0,&QWP(0,$rounds)); # load offset_i
2296 &mov ($block,&wparam(4)); # start_block_num
2297 &movdqu ($rndkey1,&QWP(0,$rounds_)); # load checksum
2298 &mov ($l_,&wparam(6)); # L_
2299
2300 &mov ($rounds,"esp");
2301 &sub ("esp",$esp_off+4); # alloca
2302 &and ("esp",-16); # align stack
2303
2304 &sub ($out,$inp);
2305 &shl ($len,4);
2306 &lea ($len,&DWP(-16*6,$inp,$len)); # end of input - 16*6
2307 &mov (&DWP($out_off,"esp"),$out);
2308 &mov (&DWP($end_off,"esp"),$len);
2309 &mov (&DWP($esp_off,"esp"),$rounds);
2310
2311 &mov ($rounds,&DWP(240,$key));
2312
2313 &test ($block,1);
2314 &jnz (&label("odd"));
2315
2316 &bsf ($i3,$block);
2317 &add ($block,1);
2318 &shl ($i3,4);
2319 &movdqu ($inout5,&QWP(0,$l_,$i3));
2320 &mov ($i3,$key); # put aside key
2321
2322 &movdqu ($inout0,&QWP(16*0,$inp)); # load input
2323 &lea ($inp,&DWP(16,$inp));
2324
2325 &pxor ($inout5,$rndkey0); # ^ last offset_i
2326 &pxor ($inout0,$inout5); # ^ offset_i
2327
2328 &movdqa ($inout4,$rndkey1);
2329 if ($inline)
2330 { &aesni_inline_generate1("dec"); }
2331 else
2332 { &call ("_aesni_decrypt1"); }
2333
2334 &xorps ($inout0,$inout5); # ^ offset_i
2335 &movaps ($rndkey1,$inout4); # pass the checksum
2336 &movdqa ($rndkey0,$inout5); # pass last offset_i
2337 &xorps ($rndkey1,$inout0); # checksum
2338 &movups (&QWP(-16,$out,$inp),$inout0); # store output
2339
2340 &mov ($rounds,&DWP(240,$i3));
2341 &mov ($key,$i3); # restore key
2342 &mov ($len,&DWP($end_off,"esp"));
2343
2344&set_label("odd");
2345 &shl ($rounds,4);
2346 &mov ($out,16);
2347 &sub ($out,$rounds); # twisted rounds
2348 &mov (&DWP($key_off,"esp"),$key);
2349 &lea ($key,&DWP(32,$key,$rounds)); # end of key schedule
2350 &mov (&DWP($rounds_off,"esp"),$out);
2351
2352 &cmp ($inp,$len);
2353 &ja (&label("short"));
2354 &jmp (&label("grandloop"));
2355
2356&set_label("grandloop",32);
2357 &lea ($i1,&DWP(1,$block));
2358 &lea ($i3,&DWP(3,$block));
2359 &lea ($i5,&DWP(5,$block));
2360 &add ($block,6);
2361 &bsf ($i1,$i1);
2362 &bsf ($i3,$i3);
2363 &bsf ($i5,$i5);
2364 &shl ($i1,4);
2365 &shl ($i3,4);
2366 &shl ($i5,4);
2367 &movdqu ($inout0,&QWP(0,$l_));
2368 &movdqu ($inout1,&QWP(0,$l_,$i1));
2369 &mov ($rounds,&DWP($rounds_off,"esp"));
2370 &movdqa ($inout2,$inout0);
2371 &movdqu ($inout3,&QWP(0,$l_,$i3));
2372 &movdqa ($inout4,$inout0);
2373 &movdqu ($inout5,&QWP(0,$l_,$i5));
2374
2375 &pxor ($inout0,$rndkey0); # ^ last offset_i
2376 &pxor ($inout1,$inout0);
2377 &movdqa (&QWP(16*0,"esp"),$inout0);
2378 &pxor ($inout2,$inout1);
2379 &movdqa (&QWP(16*1,"esp"),$inout1);
2380 &pxor ($inout3,$inout2);
2381 &movdqa (&QWP(16*2,"esp"),$inout2);
2382 &pxor ($inout4,$inout3);
2383 &movdqa (&QWP(16*3,"esp"),$inout3);
2384 &pxor ($inout5,$inout4);
2385 &movdqa (&QWP(16*4,"esp"),$inout4);
2386 &movdqa (&QWP(16*5,"esp"),$inout5);
2387
2388 &$movekey ($rndkey0,&QWP(-48,$key,$rounds));
2389 &movdqu ($inout0,&QWP(16*0,$inp)); # load input
2390 &movdqu ($inout1,&QWP(16*1,$inp));
2391 &movdqu ($inout2,&QWP(16*2,$inp));
2392 &movdqu ($inout3,&QWP(16*3,$inp));
2393 &movdqu ($inout4,&QWP(16*4,$inp));
2394 &movdqu ($inout5,&QWP(16*5,$inp));
2395 &lea ($inp,&DWP(16*6,$inp));
2396
2397 &movdqa (&QWP($checksum,"esp"),$rndkey1);
2398 &pxor ($inout0,$rndkey0); # ^ roundkey[0]
2399 &pxor ($inout1,$rndkey0);
2400 &pxor ($inout2,$rndkey0);
2401 &pxor ($inout3,$rndkey0);
2402 &pxor ($inout4,$rndkey0);
2403 &pxor ($inout5,$rndkey0);
2404
2405 &$movekey ($rndkey1,&QWP(-32,$key,$rounds));
2406 &pxor ($inout0,&QWP(16*0,"esp")); # ^ offset_i
2407 &pxor ($inout1,&QWP(16*1,"esp"));
2408 &pxor ($inout2,&QWP(16*2,"esp"));
2409 &pxor ($inout3,&QWP(16*3,"esp"));
2410 &pxor ($inout4,&QWP(16*4,"esp"));
2411 &pxor ($inout5,&QWP(16*5,"esp"));
2412
2413 &$movekey ($rndkey0,&QWP(-16,$key,$rounds));
2414 &aesdec ($inout0,$rndkey1);
2415 &aesdec ($inout1,$rndkey1);
2416 &aesdec ($inout2,$rndkey1);
2417 &aesdec ($inout3,$rndkey1);
2418 &aesdec ($inout4,$rndkey1);
2419 &aesdec ($inout5,$rndkey1);
2420
2421 &mov ($out,&DWP($out_off,"esp"));
2422 &mov ($len,&DWP($end_off,"esp"));
2423 &call ("_aesni_decrypt6_enter");
2424
2425 &movdqa ($rndkey0,&QWP(16*5,"esp")); # pass last offset_i
2426 &pxor ($inout0,&QWP(16*0,"esp")); # ^ offset_i
2427 &movdqa ($rndkey1,&QWP($checksum,"esp"));
2428 &pxor ($inout1,&QWP(16*1,"esp"));
2429 &pxor ($inout2,&QWP(16*2,"esp"));
2430 &pxor ($inout3,&QWP(16*3,"esp"));
2431 &pxor ($inout4,&QWP(16*4,"esp"));
2432 &pxor ($inout5,$rndkey0);
2433
2434 &pxor ($rndkey1,$inout0); # checksum
2435 &movdqu (&QWP(-16*6,$out,$inp),$inout0);# store output
2436 &pxor ($rndkey1,$inout1);
2437 &movdqu (&QWP(-16*5,$out,$inp),$inout1);
2438 &pxor ($rndkey1,$inout2);
2439 &movdqu (&QWP(-16*4,$out,$inp),$inout2);
2440 &pxor ($rndkey1,$inout3);
2441 &movdqu (&QWP(-16*3,$out,$inp),$inout3);
2442 &pxor ($rndkey1,$inout4);
2443 &movdqu (&QWP(-16*2,$out,$inp),$inout4);
2444 &pxor ($rndkey1,$inout5);
2445 &movdqu (&QWP(-16*1,$out,$inp),$inout5);
2446 &cmp ($inp,$len); # done yet?
2447 &jb (&label("grandloop"));
2448
2449&set_label("short");
2450 &add ($len,16*6);
2451 &sub ($len,$inp);
2452 &jz (&label("done"));
2453
2454 &cmp ($len,16*2);
2455 &jb (&label("one"));
2456 &je (&label("two"));
2457
2458 &cmp ($len,16*4);
2459 &jb (&label("three"));
2460 &je (&label("four"));
2461
2462 &lea ($i1,&DWP(1,$block));
2463 &lea ($i3,&DWP(3,$block));
2464 &bsf ($i1,$i1);
2465 &bsf ($i3,$i3);
2466 &shl ($i1,4);
2467 &shl ($i3,4);
2468 &movdqu ($inout0,&QWP(0,$l_));
2469 &movdqu ($inout1,&QWP(0,$l_,$i1));
2470 &mov ($rounds,&DWP($rounds_off,"esp"));
2471 &movdqa ($inout2,$inout0);
2472 &movdqu ($inout3,&QWP(0,$l_,$i3));
2473 &movdqa ($inout4,$inout0);
2474
2475 &pxor ($inout0,$rndkey0); # ^ last offset_i
2476 &pxor ($inout1,$inout0);
2477 &movdqa (&QWP(16*0,"esp"),$inout0);
2478 &pxor ($inout2,$inout1);
2479 &movdqa (&QWP(16*1,"esp"),$inout1);
2480 &pxor ($inout3,$inout2);
2481 &movdqa (&QWP(16*2,"esp"),$inout2);
2482 &pxor ($inout4,$inout3);
2483 &movdqa (&QWP(16*3,"esp"),$inout3);
2484 &pxor ($inout5,$inout4);
2485 &movdqa (&QWP(16*4,"esp"),$inout4);
2486
2487 &$movekey ($rndkey0,&QWP(-48,$key,$rounds));
2488 &movdqu ($inout0,&QWP(16*0,$inp)); # load input
2489 &movdqu ($inout1,&QWP(16*1,$inp));
2490 &movdqu ($inout2,&QWP(16*2,$inp));
2491 &movdqu ($inout3,&QWP(16*3,$inp));
2492 &movdqu ($inout4,&QWP(16*4,$inp));
2493 &pxor ($inout5,$inout5);
2494
2495 &movdqa (&QWP($checksum,"esp"),$rndkey1);
2496 &pxor ($inout0,$rndkey0); # ^ roundkey[0]
2497 &pxor ($inout1,$rndkey0);
2498 &pxor ($inout2,$rndkey0);
2499 &pxor ($inout3,$rndkey0);
2500 &pxor ($inout4,$rndkey0);
2501
2502 &$movekey ($rndkey1,&QWP(-32,$key,$rounds));
2503 &pxor ($inout0,&QWP(16*0,"esp")); # ^ offset_i
2504 &pxor ($inout1,&QWP(16*1,"esp"));
2505 &pxor ($inout2,&QWP(16*2,"esp"));
2506 &pxor ($inout3,&QWP(16*3,"esp"));
2507 &pxor ($inout4,&QWP(16*4,"esp"));
2508
2509 &$movekey ($rndkey0,&QWP(-16,$key,$rounds));
2510 &aesdec ($inout0,$rndkey1);
2511 &aesdec ($inout1,$rndkey1);
2512 &aesdec ($inout2,$rndkey1);
2513 &aesdec ($inout3,$rndkey1);
2514 &aesdec ($inout4,$rndkey1);
2515 &aesdec ($inout5,$rndkey1);
2516
2517 &mov ($out,&DWP($out_off,"esp"));
2518 &call ("_aesni_decrypt6_enter");
2519
2520 &movdqa ($rndkey0,&QWP(16*4,"esp")); # pass last offset_i
2521 &pxor ($inout0,&QWP(16*0,"esp")); # ^ offset_i
2522 &movdqa ($rndkey1,&QWP($checksum,"esp"));
2523 &pxor ($inout1,&QWP(16*1,"esp"));
2524 &pxor ($inout2,&QWP(16*2,"esp"));
2525 &pxor ($inout3,&QWP(16*3,"esp"));
2526 &pxor ($inout4,$rndkey0);
2527
2528 &pxor ($rndkey1,$inout0); # checksum
2529 &movdqu (&QWP(16*0,$out,$inp),$inout0); # store output
2530 &pxor ($rndkey1,$inout1);
2531 &movdqu (&QWP(16*1,$out,$inp),$inout1);
2532 &pxor ($rndkey1,$inout2);
2533 &movdqu (&QWP(16*2,$out,$inp),$inout2);
2534 &pxor ($rndkey1,$inout3);
2535 &movdqu (&QWP(16*3,$out,$inp),$inout3);
2536 &pxor ($rndkey1,$inout4);
2537 &movdqu (&QWP(16*4,$out,$inp),$inout4);
2538
2539 &jmp (&label("done"));
2540
2541&set_label("one",16);
2542 &movdqu ($inout5,&QWP(0,$l_));
2543 &mov ($key,&DWP($key_off,"esp")); # restore key
2544
2545 &movdqu ($inout0,&QWP(16*0,$inp)); # load input
2546 &mov ($rounds,&DWP(240,$key));
2547
2548 &pxor ($inout5,$rndkey0); # ^ last offset_i
2549 &pxor ($inout0,$inout5); # ^ offset_i
2550
2551 &movdqa ($inout4,$rndkey1);
2552 &mov ($out,&DWP($out_off,"esp"));
2553 if ($inline)
2554 { &aesni_inline_generate1("dec"); }
2555 else
2556 { &call ("_aesni_decrypt1"); }
2557
2558 &xorps ($inout0,$inout5); # ^ offset_i
2559 &movaps ($rndkey1,$inout4); # pass the checksum
2560 &movdqa ($rndkey0,$inout5); # pass last offset_i
2561 &xorps ($rndkey1,$inout0); # checksum
2562 &movups (&QWP(0,$out,$inp),$inout0);
2563
2564 &jmp (&label("done"));
2565
2566&set_label("two",16);
2567 &lea ($i1,&DWP(1,$block));
2568 &mov ($key,&DWP($key_off,"esp")); # restore key
2569 &bsf ($i1,$i1);
2570 &shl ($i1,4);
2571 &movdqu ($inout4,&QWP(0,$l_));
2572 &movdqu ($inout5,&QWP(0,$l_,$i1));
2573
2574 &movdqu ($inout0,&QWP(16*0,$inp)); # load input
2575 &movdqu ($inout1,&QWP(16*1,$inp));
2576 &mov ($rounds,&DWP(240,$key));
2577
2578 &movdqa ($inout3,$rndkey1);
2579 &pxor ($inout4,$rndkey0); # ^ last offset_i
2580 &pxor ($inout5,$inout4);
2581
2582 &pxor ($inout0,$inout4); # ^ offset_i
2583 &pxor ($inout1,$inout5);
2584
2585 &mov ($out,&DWP($out_off,"esp"));
2586 &call ("_aesni_decrypt2");
2587
2588 &xorps ($inout0,$inout4); # ^ offset_i
2589 &xorps ($inout1,$inout5);
2590 &movdqa ($rndkey0,$inout5); # pass last offset_i
2591 &xorps ($inout3,$inout0); # checksum
2592 &movups (&QWP(16*0,$out,$inp),$inout0); # store output
2593 &xorps ($inout3,$inout1);
2594 &movups (&QWP(16*1,$out,$inp),$inout1);
2595 &movaps ($rndkey1,$inout3); # pass the checksum
2596
2597 &jmp (&label("done"));
2598
2599&set_label("three",16);
2600 &lea ($i1,&DWP(1,$block));
2601 &mov ($key,&DWP($key_off,"esp")); # restore key
2602 &bsf ($i1,$i1);
2603 &shl ($i1,4);
2604 &movdqu ($inout3,&QWP(0,$l_));
2605 &movdqu ($inout4,&QWP(0,$l_,$i1));
2606 &movdqa ($inout5,$inout3);
2607
2608 &movdqu ($inout0,&QWP(16*0,$inp)); # load input
2609 &movdqu ($inout1,&QWP(16*1,$inp));
2610 &movdqu ($inout2,&QWP(16*2,$inp));
2611 &mov ($rounds,&DWP(240,$key));
2612
2613 &movdqa (&QWP($checksum,"esp"),$rndkey1);
2614 &pxor ($inout3,$rndkey0); # ^ last offset_i
2615 &pxor ($inout4,$inout3);
2616 &pxor ($inout5,$inout4);
2617
2618 &pxor ($inout0,$inout3); # ^ offset_i
2619 &pxor ($inout1,$inout4);
2620 &pxor ($inout2,$inout5);
2621
2622 &mov ($out,&DWP($out_off,"esp"));
2623 &call ("_aesni_decrypt3");
2624
2625 &movdqa ($rndkey1,&QWP($checksum,"esp"));# pass the checksum
2626 &xorps ($inout0,$inout3); # ^ offset_i
2627 &xorps ($inout1,$inout4);
2628 &xorps ($inout2,$inout5);
2629 &movups (&QWP(16*0,$out,$inp),$inout0); # store output
2630 &pxor ($rndkey1,$inout0); # checksum
2631 &movdqa ($rndkey0,$inout5); # pass last offset_i
2632 &movups (&QWP(16*1,$out,$inp),$inout1);
2633 &pxor ($rndkey1,$inout1);
2634 &movups (&QWP(16*2,$out,$inp),$inout2);
2635 &pxor ($rndkey1,$inout2);
2636
2637 &jmp (&label("done"));
2638
2639&set_label("four",16);
2640 &lea ($i1,&DWP(1,$block));
2641 &lea ($i3,&DWP(3,$block));
2642 &bsf ($i1,$i1);
2643 &bsf ($i3,$i3);
2644 &mov ($key,&DWP($key_off,"esp")); # restore key
2645 &shl ($i1,4);
2646 &shl ($i3,4);
2647 &movdqu ($inout2,&QWP(0,$l_));
2648 &movdqu ($inout3,&QWP(0,$l_,$i1));
2649 &movdqa ($inout4,$inout2);
2650 &movdqu ($inout5,&QWP(0,$l_,$i3));
2651
2652 &pxor ($inout2,$rndkey0); # ^ last offset_i
2653 &movdqu ($inout0,&QWP(16*0,$inp)); # load input
2654 &pxor ($inout3,$inout2);
2655 &movdqu ($inout1,&QWP(16*1,$inp));
2656 &pxor ($inout4,$inout3);
2657 &movdqa (&QWP(16*0,"esp"),$inout2);
2658 &pxor ($inout5,$inout4);
2659 &movdqa (&QWP(16*1,"esp"),$inout3);
2660 &movdqu ($inout2,&QWP(16*2,$inp));
2661 &movdqu ($inout3,&QWP(16*3,$inp));
2662 &mov ($rounds,&DWP(240,$key));
2663
2664 &movdqa (&QWP($checksum,"esp"),$rndkey1);
2665 &pxor ($inout0,&QWP(16*0,"esp")); # ^ offset_i
2666 &pxor ($inout1,&QWP(16*1,"esp"));
2667 &pxor ($inout2,$inout4);
2668 &pxor ($inout3,$inout5);
2669
2670 &mov ($out,&DWP($out_off,"esp"));
2671 &call ("_aesni_decrypt4");
2672
2673 &movdqa ($rndkey1,&QWP($checksum,"esp"));# pass the checksum
2674 &xorps ($inout0,&QWP(16*0,"esp")); # ^ offset_i
2675 &xorps ($inout1,&QWP(16*1,"esp"));
2676 &xorps ($inout2,$inout4);
2677 &movups (&QWP(16*0,$out,$inp),$inout0); # store output
2678 &pxor ($rndkey1,$inout0); # checksum
2679 &xorps ($inout3,$inout5);
2680 &movups (&QWP(16*1,$out,$inp),$inout1);
2681 &pxor ($rndkey1,$inout1);
2682 &movdqa ($rndkey0,$inout5); # pass last offset_i
2683 &movups (&QWP(16*2,$out,$inp),$inout2);
2684 &pxor ($rndkey1,$inout2);
2685 &movups (&QWP(16*3,$out,$inp),$inout3);
2686 &pxor ($rndkey1,$inout3);
2687
2688&set_label("done");
2689 &mov ($key,&DWP($esp_off,"esp"));
2690 &pxor ($inout0,$inout0); # clear register bank
2691 &pxor ($inout1,$inout1);
2692 &movdqa (&QWP(16*0,"esp"),$inout0); # clear stack
2693 &pxor ($inout2,$inout2);
2694 &movdqa (&QWP(16*1,"esp"),$inout0);
2695 &pxor ($inout3,$inout3);
2696 &movdqa (&QWP(16*2,"esp"),$inout0);
2697 &pxor ($inout4,$inout4);
2698 &movdqa (&QWP(16*3,"esp"),$inout0);
2699 &pxor ($inout5,$inout5);
2700 &movdqa (&QWP(16*4,"esp"),$inout0);
2701 &movdqa (&QWP(16*5,"esp"),$inout0);
2702 &movdqa (&QWP(16*6,"esp"),$inout0);
2703
2704 &lea ("esp",&DWP(0,$key));
2705 &mov ($rounds,&wparam(5)); # &offset_i
2706 &mov ($rounds_,&wparam(7)); # &checksum
2707 &movdqu (&QWP(0,$rounds),$rndkey0);
2708 &pxor ($rndkey0,$rndkey0);
2709 &movdqu (&QWP(0,$rounds_),$rndkey1);
2710 &pxor ($rndkey1,$rndkey1);
2711&function_end("aesni_ocb_decrypt");
2712}
6c83629b
AP		 2713}
2714\f
2715######################################################################
d64a7232
AP		 2716# void $PREFIX_cbc_encrypt (const void *inp, void *out,
2717# size_t length, const AES_KEY *key,
2718# unsigned char *ivp,const int enc);
2719&function_begin("${PREFIX}_cbc_encrypt");
2720 &mov ($inp,&wparam(0));
f8501464 2721 &mov ($rounds_,"esp");
d64a7232 2722 &mov ($out,&wparam(1));
f8501464 2723 &sub ($rounds_,24);
d64a7232 2724 &mov ($len,&wparam(2));
f8501464 2725 &and ($rounds_,-16);
d64a7232 2726 &mov ($key,&wparam(3));
d64a7232 2727 &mov ($key_,&wparam(4));
d7d119a3 2728 &test ($len,$len);
f8501464 2729 &jz (&label("cbc_abort"));
d64a7232
AP		 2730
2731 &cmp (&wparam(5),0);
f8501464
AP		 2732 &xchg ($rounds_,"esp"); # alloca
2733 &movups ($ivec,&QWP(0,$key_)); # load IV
d64a7232 2734 &mov ($rounds,&DWP(240,$key));
f8501464
AP		 2735 &mov ($key_,$key); # backup $key
2736 &mov (&DWP(16,"esp"),$rounds_); # save original %esp
2737 &mov ($rounds_,$rounds); # backup $rounds
d64a7232
AP		 2738 &je (&label("cbc_decrypt"));
2739
f8501464 2740 &movaps ($inout0,$ivec);
d64a7232
AP		 2741 &cmp ($len,16);
2742 &jb (&label("cbc_enc_tail"));
2743 &sub ($len,16);
2744 &jmp (&label("cbc_enc_loop"));
2745
2746&set_label("cbc_enc_loop",16);
f8501464 2747 &movups ($ivec,&QWP(0,$inp)); # input actually
d64a7232 2748 &lea ($inp,&DWP(16,$inp));
6f766a41 2749 if ($inline)
f8501464 2750 { &aesni_inline_generate1("enc",$inout0,$ivec); }
6f766a41 2751 else
f8501464 2752 { &xorps($inout0,$ivec); &call("_aesni_encrypt1"); }
d64a7232
AP		 2753 &mov ($rounds,$rounds_); # restore $rounds
2754 &mov ($key,$key_); # restore $key
d7d119a3
AP		 2755 &movups (&QWP(0,$out),$inout0); # store output
2756 &lea ($out,&DWP(16,$out));
2757 &sub ($len,16);
d64a7232
AP		 2758 &jnc (&label("cbc_enc_loop"));
2759 &add ($len,16);
2760 &jnz (&label("cbc_enc_tail"));
2761 &movaps ($ivec,$inout0);
23f6eec7 2762 &pxor ($inout0,$inout0);
d64a7232
AP		 2763 &jmp (&label("cbc_ret"));
2764
2765&set_label("cbc_enc_tail");
2766 &mov ("ecx",$len); # zaps $rounds
2767 &data_word(0xA4F3F689); # rep movsb
2768 &mov ("ecx",16); # zero tail
2769 &sub ("ecx",$len);
2770 &xor ("eax","eax"); # zaps $len
2771 &data_word(0xAAF3F689); # rep stosb
2772 &lea ($out,&DWP(-16,$out)); # rewind $out by 1 block
2773 &mov ($rounds,$rounds_); # restore $rounds
2774 &mov ($inp,$out); # $inp and $out are the same
2775 &mov ($key,$key_); # restore $key
2776 &jmp (&label("cbc_enc_loop"));
6c83629b 2777######################################################################
d64a7232 2778&set_label("cbc_decrypt",16);
f8501464 2779 &cmp ($len,0x50);
d608b4d6 2780 &jbe (&label("cbc_dec_tail"));
f8501464
AP		 2781 &movaps (&QWP(0,"esp"),$ivec); # save IV
2782 &sub ($len,0x50);
2783 &jmp (&label("cbc_dec_loop6_enter"));
d64a7232 2784
f8501464
AP		 2785&set_label("cbc_dec_loop6",16);
2786 &movaps (&QWP(0,"esp"),$rndkey0); # save IV
2787 &movups (&QWP(0,$out),$inout5);
2788 &lea ($out,&DWP(0x10,$out));
2789&set_label("cbc_dec_loop6_enter");
2790 &movdqu ($inout0,&QWP(0,$inp));
2791 &movdqu ($inout1,&QWP(0x10,$inp));
2792 &movdqu ($inout2,&QWP(0x20,$inp));
2793 &movdqu ($inout3,&QWP(0x30,$inp));
2794 &movdqu ($inout4,&QWP(0x40,$inp));
2795 &movdqu ($inout5,&QWP(0x50,$inp));
2796
2797 &call ("_aesni_decrypt6");
2798
2799 &movups ($rndkey1,&QWP(0,$inp));
2800 &movups ($rndkey0,&QWP(0x10,$inp));
2801 &xorps ($inout0,&QWP(0,"esp")); # ^=IV
2802 &xorps ($inout1,$rndkey1);
2803 &movups ($rndkey1,&QWP(0x20,$inp));
2804 &xorps ($inout2,$rndkey0);
2805 &movups ($rndkey0,&QWP(0x30,$inp));
2806 &xorps ($inout3,$rndkey1);
2807 &movups ($rndkey1,&QWP(0x40,$inp));
2808 &xorps ($inout4,$rndkey0);
2809 &movups ($rndkey0,&QWP(0x50,$inp)); # IV
2810 &xorps ($inout5,$rndkey1);
2811 &movups (&QWP(0,$out),$inout0);
2812 &movups (&QWP(0x10,$out),$inout1);
2813 &lea ($inp,&DWP(0x60,$inp));
2814 &movups (&QWP(0x20,$out),$inout2);
f9c5e5d9 2815 &mov ($rounds,$rounds_); # restore $rounds
f8501464
AP		 2816 &movups (&QWP(0x30,$out),$inout3);
2817 &mov ($key,$key_); # restore $key
2818 &movups (&QWP(0x40,$out),$inout4);
2819 &lea ($out,&DWP(0x50,$out));
2820 &sub ($len,0x60);
2821 &ja (&label("cbc_dec_loop6"));
2822
2823 &movaps ($inout0,$inout5);
2824 &movaps ($ivec,$rndkey0);
2825 &add ($len,0x50);
23f6eec7 2826 &jle (&label("cbc_dec_clear_tail_collected"));
f8501464
AP		 2827 &movups (&QWP(0,$out),$inout0);
2828 &lea ($out,&DWP(0x10,$out));
6c83629b 2829&set_label("cbc_dec_tail");
d64a7232 2830 &movups ($inout0,&QWP(0,$inp));
d64a7232 2831 &movaps ($in0,$inout0);
d7d119a3 2832 &cmp ($len,0x10);
d64a7232 2833 &jbe (&label("cbc_dec_one"));
f8501464 2834
d64a7232 2835 &movups ($inout1,&QWP(0x10,$inp));
d64a7232 2836 &movaps ($in1,$inout1);
d7d119a3 2837 &cmp ($len,0x20);
d64a7232 2838 &jbe (&label("cbc_dec_two"));
f8501464 2839
d64a7232 2840 &movups ($inout2,&QWP(0x20,$inp));
d608b4d6
AP		 2841 &cmp ($len,0x30);
2842 &jbe (&label("cbc_dec_three"));
f8501464 2843
d608b4d6 2844 &movups ($inout3,&QWP(0x30,$inp));
f8501464
AP		 2845 &cmp ($len,0x40);
2846 &jbe (&label("cbc_dec_four"));
2847
2848 &movups ($inout4,&QWP(0x40,$inp));
2849 &movaps (&QWP(0,"esp"),$ivec); # save IV
2850 &movups ($inout0,&QWP(0,$inp));
2851 &xorps ($inout5,$inout5);
2852 &call ("_aesni_decrypt6");
2853 &movups ($rndkey1,&QWP(0,$inp));
2854 &movups ($rndkey0,&QWP(0x10,$inp));
2855 &xorps ($inout0,&QWP(0,"esp")); # ^= IV
2856 &xorps ($inout1,$rndkey1);
2857 &movups ($rndkey1,&QWP(0x20,$inp));
2858 &xorps ($inout2,$rndkey0);
2859 &movups ($rndkey0,&QWP(0x30,$inp));
2860 &xorps ($inout3,$rndkey1);
2861 &movups ($ivec,&QWP(0x40,$inp)); # IV
2862 &xorps ($inout4,$rndkey0);
2863 &movups (&QWP(0,$out),$inout0);
2864 &movups (&QWP(0x10,$out),$inout1);
23f6eec7 2865 &pxor ($inout1,$inout1);
f8501464 2866 &movups (&QWP(0x20,$out),$inout2);
23f6eec7 2867 &pxor ($inout2,$inout2);
f8501464 2868 &movups (&QWP(0x30,$out),$inout3);
23f6eec7 2869 &pxor ($inout3,$inout3);
f8501464
AP		 2870 &lea ($out,&DWP(0x40,$out));
2871 &movaps ($inout0,$inout4);
23f6eec7 2872 &pxor ($inout4,$inout4);
f8501464 2873 &sub ($len,0x50);
d64a7232
AP		 2874 &jmp (&label("cbc_dec_tail_collected"));
2875
d7d119a3 2876&set_label("cbc_dec_one",16);
6f766a41
AP		 2877 if ($inline)
2878 { &aesni_inline_generate1("dec"); }
2879 else
2880 { &call ("_aesni_decrypt1"); }
f8501464
AP		 2881 &xorps ($inout0,$ivec);
2882 &movaps ($ivec,$in0);
2883 &sub ($len,0x10);
d64a7232
AP		 2884 &jmp (&label("cbc_dec_tail_collected"));
2885
d7d119a3 2886&set_label("cbc_dec_two",16);
214368ff 2887 &call ("_aesni_decrypt2");
f8501464
AP		 2888 &xorps ($inout0,$ivec);
2889 &xorps ($inout1,$in0);
2890 &movups (&QWP(0,$out),$inout0);
2891 &movaps ($inout0,$inout1);
23f6eec7 2892 &pxor ($inout1,$inout1);
d64a7232 2893 &lea ($out,&DWP(0x10,$out));
f8501464
AP		 2894 &movaps ($ivec,$in1);
2895 &sub ($len,0x20);
d608b4d6
AP		 2896 &jmp (&label("cbc_dec_tail_collected"));
2897
d7d119a3 2898&set_label("cbc_dec_three",16);
d608b4d6 2899 &call ("_aesni_decrypt3");
f8501464
AP		 2900 &xorps ($inout0,$ivec);
2901 &xorps ($inout1,$in0);
2902 &xorps ($inout2,$in1);
2903 &movups (&QWP(0,$out),$inout0);
2904 &movaps ($inout0,$inout2);
23f6eec7 2905 &pxor ($inout2,$inout2);
f8501464 2906 &movups (&QWP(0x10,$out),$inout1);
23f6eec7 2907 &pxor ($inout1,$inout1);
d608b4d6 2908 &lea ($out,&DWP(0x20,$out));
f8501464
AP		 2909 &movups ($ivec,&QWP(0x20,$inp));
2910 &sub ($len,0x30);
2911 &jmp (&label("cbc_dec_tail_collected"));
2912
2913&set_label("cbc_dec_four",16);
2914 &call ("_aesni_decrypt4");
2915 &movups ($rndkey1,&QWP(0x10,$inp));
2916 &movups ($rndkey0,&QWP(0x20,$inp));
2917 &xorps ($inout0,$ivec);
2918 &movups ($ivec,&QWP(0x30,$inp));
2919 &xorps ($inout1,$in0);
2920 &movups (&QWP(0,$out),$inout0);
2921 &xorps ($inout2,$rndkey1);
2922 &movups (&QWP(0x10,$out),$inout1);
23f6eec7 2923 &pxor ($inout1,$inout1);
f8501464
AP		 2924 &xorps ($inout3,$rndkey0);
2925 &movups (&QWP(0x20,$out),$inout2);
23f6eec7 2926 &pxor ($inout2,$inout2);
f8501464
AP		 2927 &lea ($out,&DWP(0x30,$out));
2928 &movaps ($inout0,$inout3);
23f6eec7 2929 &pxor ($inout3,$inout3);
f8501464 2930 &sub ($len,0x40);
23f6eec7 2931 &jmp (&label("cbc_dec_tail_collected"));
d64a7232 2932
23f6eec7
AP		 2933&set_label("cbc_dec_clear_tail_collected",16);
2934 &pxor ($inout1,$inout1);
2935 &pxor ($inout2,$inout2);
2936 &pxor ($inout3,$inout3);
2937 &pxor ($inout4,$inout4);
d64a7232
AP		 2938&set_label("cbc_dec_tail_collected");
2939 &and ($len,15);
2940 &jnz (&label("cbc_dec_tail_partial"));
f8501464 2941 &movups (&QWP(0,$out),$inout0);
23f6eec7 2942 &pxor ($rndkey0,$rndkey0);
d64a7232
AP		 2943 &jmp (&label("cbc_ret"));
2944
d7d119a3 2945&set_label("cbc_dec_tail_partial",16);
f8501464 2946 &movaps (&QWP(0,"esp"),$inout0);
23f6eec7 2947 &pxor ($rndkey0,$rndkey0);
f8501464 2948 &mov ("ecx",16);
d64a7232 2949 &mov ($inp,"esp");
f8501464 2950 &sub ("ecx",$len);
d64a7232 2951 &data_word(0xA4F3F689); # rep movsb
23f6eec7 2952 &movdqa (&QWP(0,"esp"),$inout0);
d64a7232
AP		 2953
2954&set_label("cbc_ret");
f8501464 2955 &mov ("esp",&DWP(16,"esp")); # pull original %esp
d64a7232 2956 &mov ($key_,&wparam(4));
23f6eec7
AP		 2957 &pxor ($inout0,$inout0);
2958 &pxor ($rndkey1,$rndkey1);
d64a7232 2959 &movups (&QWP(0,$key_),$ivec); # output IV
23f6eec7 2960 &pxor ($ivec,$ivec);
f8501464 2961&set_label("cbc_abort");
d64a7232 2962&function_end("${PREFIX}_cbc_encrypt");
6c83629b
AP		 2963\f
2964######################################################################
d64a7232
AP		 2965# Mechanical port from aesni-x86_64.pl.
2966#
2967# _aesni_set_encrypt_key is private interface,
2968# input:
2969# "eax" const unsigned char *userKey
2970# $rounds int bits
2971# $key AES_KEY *key
2972# output:
2973# "eax" return code
2974# $round rounds
2975
2976&function_begin_B("_aesni_set_encrypt_key");
23f6eec7
AP		 2977 &push ("ebp");
2978 &push ("ebx");
d64a7232
AP		 2979 &test ("eax","eax");
2980 &jz (&label("bad_pointer"));
2981 &test ($key,$key);
2982 &jz (&label("bad_pointer"));
2983
23f6eec7
AP		 2984 &call (&label("pic"));
2985&set_label("pic");
2986 &blindpop("ebx");
2987 &lea ("ebx",&DWP(&label("key_const")."-".&label("pic"),"ebx"));
2988
2989 &picmeup("ebp","OPENSSL_ia32cap_P","ebx",&label("key_const"));
d64a7232 2990 &movups ("xmm0",&QWP(0,"eax")); # pull first 128 bits of *userKey
f8501464 2991 &xorps ("xmm4","xmm4"); # low dword of xmm4 is assumed 0
23f6eec7 2992 &mov ("ebp",&DWP(4,"ebp"));
d64a7232 2993 &lea ($key,&DWP(16,$key));
23f6eec7 2994 &and ("ebp",1<<28|1<<11); # AVX and XOP bits
d64a7232
AP		 2995 &cmp ($rounds,256);
2996 &je (&label("14rounds"));
2997 &cmp ($rounds,192);
2998 &je (&label("12rounds"));
2999 &cmp ($rounds,128);
3000 &jne (&label("bad_keybits"));
3001
3002&set_label("10rounds",16);
23f6eec7
AP		 3003 &cmp ("ebp",1<<28);
3004 &je (&label("10rounds_alt"));
3005
d608b4d6 3006 &mov ($rounds,9);
d64a7232
AP		 3007 &$movekey (&QWP(-16,$key),"xmm0"); # round 0
3008 &aeskeygenassist("xmm1","xmm0",0x01); # round 1
3009 &call (&label("key_128_cold"));
3010 &aeskeygenassist("xmm1","xmm0",0x2); # round 2
3011 &call (&label("key_128"));
3012 &aeskeygenassist("xmm1","xmm0",0x04); # round 3
3013 &call (&label("key_128"));
3014 &aeskeygenassist("xmm1","xmm0",0x08); # round 4
3015 &call (&label("key_128"));
3016 &aeskeygenassist("xmm1","xmm0",0x10); # round 5
3017 &call (&label("key_128"));
3018 &aeskeygenassist("xmm1","xmm0",0x20); # round 6
3019 &call (&label("key_128"));
3020 &aeskeygenassist("xmm1","xmm0",0x40); # round 7
3021 &call (&label("key_128"));
3022 &aeskeygenassist("xmm1","xmm0",0x80); # round 8
3023 &call (&label("key_128"));
3024 &aeskeygenassist("xmm1","xmm0",0x1b); # round 9
3025 &call (&label("key_128"));
3026 &aeskeygenassist("xmm1","xmm0",0x36); # round 10
3027 &call (&label("key_128"));
3028 &$movekey (&QWP(0,$key),"xmm0");
3029 &mov (&DWP(80,$key),$rounds);
23f6eec7
AP		 3030
3031 &jmp (&label("good_key"));
d64a7232
AP		 3032
3033&set_label("key_128",16);
3034 &$movekey (&QWP(0,$key),"xmm0");
3035 &lea ($key,&DWP(16,$key));
3036&set_label("key_128_cold");
3037 &shufps ("xmm4","xmm0",0b00010000);
f8501464
AP		 3038 &xorps ("xmm0","xmm4");
3039 &shufps ("xmm4","xmm0",0b10001100);
3040 &xorps ("xmm0","xmm4");
3041 &shufps ("xmm1","xmm1",0b11111111); # critical path
3042 &xorps ("xmm0","xmm1");
d64a7232
AP		 3043 &ret();
3044
23f6eec7
AP		 3045&set_label("10rounds_alt",16);
3046 &movdqa ("xmm5",&QWP(0x00,"ebx"));
3047 &mov ($rounds,8);
3048 &movdqa ("xmm4",&QWP(0x20,"ebx"));
3049 &movdqa ("xmm2","xmm0");
7be6bc68 3050 &movdqu (&QWP(-16,$key),"xmm0");
23f6eec7
AP		 3051
3052&set_label("loop_key128");
3053 &pshufb ("xmm0","xmm5");
3054 &aesenclast ("xmm0","xmm4");
3055 &pslld ("xmm4",1);
3056 &lea ($key,&DWP(16,$key));
3057
3058 &movdqa ("xmm3","xmm2");
3059 &pslldq ("xmm2",4);
3060 &pxor ("xmm3","xmm2");
3061 &pslldq ("xmm2",4);
3062 &pxor ("xmm3","xmm2");
3063 &pslldq ("xmm2",4);
3064 &pxor ("xmm2","xmm3");
3065
3066 &pxor ("xmm0","xmm2");
3067 &movdqu (&QWP(-16,$key),"xmm0");
3068 &movdqa ("xmm2","xmm0");
3069
3070 &dec ($rounds);
3071 &jnz (&label("loop_key128"));
3072
3073 &movdqa ("xmm4",&QWP(0x30,"ebx"));
3074
3075 &pshufb ("xmm0","xmm5");
3076 &aesenclast ("xmm0","xmm4");
3077 &pslld ("xmm4",1);
3078
3079 &movdqa ("xmm3","xmm2");
3080 &pslldq ("xmm2",4);
3081 &pxor ("xmm3","xmm2");
3082 &pslldq ("xmm2",4);
3083 &pxor ("xmm3","xmm2");
3084 &pslldq ("xmm2",4);
3085 &pxor ("xmm2","xmm3");
3086
3087 &pxor ("xmm0","xmm2");
3088 &movdqu (&QWP(0,$key),"xmm0");
3089
3090 &movdqa ("xmm2","xmm0");
3091 &pshufb ("xmm0","xmm5");
3092 &aesenclast ("xmm0","xmm4");
3093
3094 &movdqa ("xmm3","xmm2");
3095 &pslldq ("xmm2",4);
3096 &pxor ("xmm3","xmm2");
3097 &pslldq ("xmm2",4);
3098 &pxor ("xmm3","xmm2");
3099 &pslldq ("xmm2",4);
3100 &pxor ("xmm2","xmm3");
3101
3102 &pxor ("xmm0","xmm2");
3103 &movdqu (&QWP(16,$key),"xmm0");
3104
3105 &mov ($rounds,9);
3106 &mov (&DWP(96,$key),$rounds);
3107
3108 &jmp (&label("good_key"));
3109
d64a7232
AP		 3110&set_label("12rounds",16);
3111 &movq ("xmm2",&QWP(16,"eax")); # remaining 1/3 of *userKey
23f6eec7
AP		 3112 &cmp ("ebp",1<<28);
3113 &je (&label("12rounds_alt"));
3114
d608b4d6 3115 &mov ($rounds,11);
f9c5e5d9 3116 &$movekey (&QWP(-16,$key),"xmm0"); # round 0
d64a7232
AP		 3117 &aeskeygenassist("xmm1","xmm2",0x01); # round 1,2
3118 &call (&label("key_192a_cold"));
3119 &aeskeygenassist("xmm1","xmm2",0x02); # round 2,3
3120 &call (&label("key_192b"));
3121 &aeskeygenassist("xmm1","xmm2",0x04); # round 4,5
3122 &call (&label("key_192a"));
3123 &aeskeygenassist("xmm1","xmm2",0x08); # round 5,6
3124 &call (&label("key_192b"));
3125 &aeskeygenassist("xmm1","xmm2",0x10); # round 7,8
3126 &call (&label("key_192a"));
3127 &aeskeygenassist("xmm1","xmm2",0x20); # round 8,9
3128 &call (&label("key_192b"));
3129 &aeskeygenassist("xmm1","xmm2",0x40); # round 10,11
3130 &call (&label("key_192a"));
3131 &aeskeygenassist("xmm1","xmm2",0x80); # round 11,12
3132 &call (&label("key_192b"));
3133 &$movekey (&QWP(0,$key),"xmm0");
3134 &mov (&DWP(48,$key),$rounds);
23f6eec7
AP		 3135
3136 &jmp (&label("good_key"));
d64a7232
AP		 3137
3138&set_label("key_192a",16);
3139 &$movekey (&QWP(0,$key),"xmm0");
3140 &lea ($key,&DWP(16,$key));
3141&set_label("key_192a_cold",16);
3142 &movaps ("xmm5","xmm2");
3143&set_label("key_192b_warm");
3144 &shufps ("xmm4","xmm0",0b00010000);
f8501464
AP		 3145 &movdqa ("xmm3","xmm2");
3146 &xorps ("xmm0","xmm4");
d64a7232
AP		 3147 &shufps ("xmm4","xmm0",0b10001100);
3148 &pslldq ("xmm3",4);
f8501464 3149 &xorps ("xmm0","xmm4");
d64a7232
AP		 3150 &pshufd ("xmm1","xmm1",0b01010101); # critical path
3151 &pxor ("xmm2","xmm3");
3152 &pxor ("xmm0","xmm1");
3153 &pshufd ("xmm3","xmm0",0b11111111);
3154 &pxor ("xmm2","xmm3");
3155 &ret();
3156
3157&set_label("key_192b",16);
3158 &movaps ("xmm3","xmm0");
3159 &shufps ("xmm5","xmm0",0b01000100);
3160 &$movekey (&QWP(0,$key),"xmm5");
3161 &shufps ("xmm3","xmm2",0b01001110);
3162 &$movekey (&QWP(16,$key),"xmm3");
3163 &lea ($key,&DWP(32,$key));
3164 &jmp (&label("key_192b_warm"));
3165
23f6eec7
AP		 3166&set_label("12rounds_alt",16);
3167 &movdqa ("xmm5",&QWP(0x10,"ebx"));
3168 &movdqa ("xmm4",&QWP(0x20,"ebx"));
3169 &mov ($rounds,8);
3170 &movdqu (&QWP(-16,$key),"xmm0");
3171
3172&set_label("loop_key192");
3173 &movq (&QWP(0,$key),"xmm2");
3174 &movdqa ("xmm1","xmm2");
3175 &pshufb ("xmm2","xmm5");
3176 &aesenclast ("xmm2","xmm4");
3177 &pslld ("xmm4",1);
3178 &lea ($key,&DWP(24,$key));
3179
3180 &movdqa ("xmm3","xmm0");
3181 &pslldq ("xmm0",4);
3182 &pxor ("xmm3","xmm0");
3183 &pslldq ("xmm0",4);
3184 &pxor ("xmm3","xmm0");
3185 &pslldq ("xmm0",4);
3186 &pxor ("xmm0","xmm3");
3187
3188 &pshufd ("xmm3","xmm0",0xff);
3189 &pxor ("xmm3","xmm1");
3190 &pslldq ("xmm1",4);
3191 &pxor ("xmm3","xmm1");
3192
3193 &pxor ("xmm0","xmm2");
3194 &pxor ("xmm2","xmm3");
3195 &movdqu (&QWP(-16,$key),"xmm0");
3196
3197 &dec ($rounds);
3198 &jnz (&label("loop_key192"));
3199
3200 &mov ($rounds,11);
3201 &mov (&DWP(32,$key),$rounds);
3202
3203 &jmp (&label("good_key"));
3204
d64a7232
AP		 3205&set_label("14rounds",16);
3206 &movups ("xmm2",&QWP(16,"eax")); # remaining half of *userKey
d64a7232 3207 &lea ($key,&DWP(16,$key));
23f6eec7
AP		 3208 &cmp ("ebp",1<<28);
3209 &je (&label("14rounds_alt"));
3210
3211 &mov ($rounds,13);
d64a7232
AP		 3212 &$movekey (&QWP(-32,$key),"xmm0"); # round 0
3213 &$movekey (&QWP(-16,$key),"xmm2"); # round 1
3214 &aeskeygenassist("xmm1","xmm2",0x01); # round 2
3215 &call (&label("key_256a_cold"));
3216 &aeskeygenassist("xmm1","xmm0",0x01); # round 3
3217 &call (&label("key_256b"));
3218 &aeskeygenassist("xmm1","xmm2",0x02); # round 4
3219 &call (&label("key_256a"));
3220 &aeskeygenassist("xmm1","xmm0",0x02); # round 5
3221 &call (&label("key_256b"));
3222 &aeskeygenassist("xmm1","xmm2",0x04); # round 6
3223 &call (&label("key_256a"));
3224 &aeskeygenassist("xmm1","xmm0",0x04); # round 7
3225 &call (&label("key_256b"));
3226 &aeskeygenassist("xmm1","xmm2",0x08); # round 8
3227 &call (&label("key_256a"));
3228 &aeskeygenassist("xmm1","xmm0",0x08); # round 9
3229 &call (&label("key_256b"));
3230 &aeskeygenassist("xmm1","xmm2",0x10); # round 10
3231 &call (&label("key_256a"));
3232 &aeskeygenassist("xmm1","xmm0",0x10); # round 11
3233 &call (&label("key_256b"));
3234 &aeskeygenassist("xmm1","xmm2",0x20); # round 12
3235 &call (&label("key_256a"));
3236 &aeskeygenassist("xmm1","xmm0",0x20); # round 13
3237 &call (&label("key_256b"));
3238 &aeskeygenassist("xmm1","xmm2",0x40); # round 14
3239 &call (&label("key_256a"));
3240 &$movekey (&QWP(0,$key),"xmm0");
3241 &mov (&DWP(16,$key),$rounds);
3242 &xor ("eax","eax");
23f6eec7
AP		 3243
3244 &jmp (&label("good_key"));
d64a7232
AP		 3245
3246&set_label("key_256a",16);
3247 &$movekey (&QWP(0,$key),"xmm2");
3248 &lea ($key,&DWP(16,$key));
3249&set_label("key_256a_cold");
3250 &shufps ("xmm4","xmm0",0b00010000);
f8501464 3251 &xorps ("xmm0","xmm4");
d64a7232 3252 &shufps ("xmm4","xmm0",0b10001100);
f8501464
AP		 3253 &xorps ("xmm0","xmm4");
3254 &shufps ("xmm1","xmm1",0b11111111); # critical path
3255 &xorps ("xmm0","xmm1");
d64a7232
AP		 3256 &ret();
3257
3258&set_label("key_256b",16);
3259 &$movekey (&QWP(0,$key),"xmm0");
3260 &lea ($key,&DWP(16,$key));
3261
3262 &shufps ("xmm4","xmm2",0b00010000);
f8501464 3263 &xorps ("xmm2","xmm4");
d64a7232 3264 &shufps ("xmm4","xmm2",0b10001100);
f8501464
AP		 3265 &xorps ("xmm2","xmm4");
3266 &shufps ("xmm1","xmm1",0b10101010); # critical path
3267 &xorps ("xmm2","xmm1");
d64a7232
AP		 3268 &ret();
3269
23f6eec7
AP		 3270&set_label("14rounds_alt",16);
3271 &movdqa ("xmm5",&QWP(0x00,"ebx"));
3272 &movdqa ("xmm4",&QWP(0x20,"ebx"));
3273 &mov ($rounds,7);
3274 &movdqu (&QWP(-32,$key),"xmm0");
3275 &movdqa ("xmm1","xmm2");
3276 &movdqu (&QWP(-16,$key),"xmm2");
3277
3278&set_label("loop_key256");
3279 &pshufb ("xmm2","xmm5");
3280 &aesenclast ("xmm2","xmm4");
3281
3282 &movdqa ("xmm3","xmm0");
3283 &pslldq ("xmm0",4);
3284 &pxor ("xmm3","xmm0");
3285 &pslldq ("xmm0",4);
3286 &pxor ("xmm3","xmm0");
3287 &pslldq ("xmm0",4);
3288 &pxor ("xmm0","xmm3");
3289 &pslld ("xmm4",1);
3290
3291 &pxor ("xmm0","xmm2");
3292 &movdqu (&QWP(0,$key),"xmm0");
3293
3294 &dec ($rounds);
3295 &jz (&label("done_key256"));
3296
3297 &pshufd ("xmm2","xmm0",0xff);
3298 &pxor ("xmm3","xmm3");
3299 &aesenclast ("xmm2","xmm3");
3300
bd30091c 3301 &movdqa ("xmm3","xmm1");
23f6eec7
AP		 3302 &pslldq ("xmm1",4);
3303 &pxor ("xmm3","xmm1");
3304 &pslldq ("xmm1",4);
3305 &pxor ("xmm3","xmm1");
3306 &pslldq ("xmm1",4);
3307 &pxor ("xmm1","xmm3");
3308
3309 &pxor ("xmm2","xmm1");
3310 &movdqu (&QWP(16,$key),"xmm2");
3311 &lea ($key,&DWP(32,$key));
3312 &movdqa ("xmm1","xmm2");
3313 &jmp (&label("loop_key256"));
3314
3315&set_label("done_key256");
3316 &mov ($rounds,13);
3317 &mov (&DWP(16,$key),$rounds);
3318
3319&set_label("good_key");
3320 &pxor ("xmm0","xmm0");
3321 &pxor ("xmm1","xmm1");
3322 &pxor ("xmm2","xmm2");
3323 &pxor ("xmm3","xmm3");
3324 &pxor ("xmm4","xmm4");
3325 &pxor ("xmm5","xmm5");
3326 &xor ("eax","eax");
3327 &pop ("ebx");
3328 &pop ("ebp");
3329 &ret ();
3330
d64a7232
AP		 3331&set_label("bad_pointer",4);
3332 &mov ("eax",-1);
23f6eec7
AP		 3333 &pop ("ebx");
3334 &pop ("ebp");
d64a7232
AP		 3335 &ret ();
3336&set_label("bad_keybits",4);
23f6eec7 3337 &pxor ("xmm0","xmm0");
d64a7232 3338 &mov ("eax",-2);
23f6eec7
AP		 3339 &pop ("ebx");
3340 &pop ("ebp");
d64a7232
AP		 3341 &ret ();
3342&function_end_B("_aesni_set_encrypt_key");
3343
3344# int $PREFIX_set_encrypt_key (const unsigned char *userKey, int bits,
3345# AES_KEY *key)
3346&function_begin_B("${PREFIX}_set_encrypt_key");
3347 &mov ("eax",&wparam(0));
3348 &mov ($rounds,&wparam(1));
3349 &mov ($key,&wparam(2));
3350 &call ("_aesni_set_encrypt_key");
3351 &ret ();
3352&function_end_B("${PREFIX}_set_encrypt_key");
3353
3354# int $PREFIX_set_decrypt_key (const unsigned char *userKey, int bits,
3355# AES_KEY *key)
3356&function_begin_B("${PREFIX}_set_decrypt_key");
3357 &mov ("eax",&wparam(0));
3358 &mov ($rounds,&wparam(1));
3359 &mov ($key,&wparam(2));
3360 &call ("_aesni_set_encrypt_key");
3361 &mov ($key,&wparam(2));
f9c5e5d9 3362 &shl ($rounds,4); # rounds-1 after _aesni_set_encrypt_key
d64a7232
AP		 3363 &test ("eax","eax");
3364 &jnz (&label("dec_key_ret"));
d608b4d6 3365 &lea ("eax",&DWP(16,$key,$rounds)); # end of key schedule
d64a7232
AP		 3366
3367 &$movekey ("xmm0",&QWP(0,$key)); # just swap
3368 &$movekey ("xmm1",&QWP(0,"eax"));
3369 &$movekey (&QWP(0,"eax"),"xmm0");
3370 &$movekey (&QWP(0,$key),"xmm1");
3371 &lea ($key,&DWP(16,$key));
3372 &lea ("eax",&DWP(-16,"eax"));
d64a7232 3373
d608b4d6 3374&set_label("dec_key_inverse");
d64a7232
AP		 3375 &$movekey ("xmm0",&QWP(0,$key)); # swap and inverse
3376 &$movekey ("xmm1",&QWP(0,"eax"));
3377 &aesimc ("xmm0","xmm0");
3378 &aesimc ("xmm1","xmm1");
3379 &lea ($key,&DWP(16,$key));
3380 &lea ("eax",&DWP(-16,"eax"));
d64a7232
AP		 3381 &$movekey (&QWP(16,"eax"),"xmm0");
3382 &$movekey (&QWP(-16,$key),"xmm1");
d7d119a3 3383 &cmp ("eax",$key);
d64a7232
AP		 3384 &ja (&label("dec_key_inverse"));
3385
3386 &$movekey ("xmm0",&QWP(0,$key)); # inverse middle
3387 &aesimc ("xmm0","xmm0");
3388 &$movekey (&QWP(0,$key),"xmm0");
3389
23f6eec7
AP		 3390 &pxor ("xmm0","xmm0");
3391 &pxor ("xmm1","xmm1");
d64a7232
AP		 3392 &xor ("eax","eax"); # return success
3393&set_label("dec_key_ret");
3394 &ret ();
3395&function_end_B("${PREFIX}_set_decrypt_key");
23f6eec7
AP		 3396
3397&set_label("key_const",64);
3398&data_word(0x0c0f0e0d,0x0c0f0e0d,0x0c0f0e0d,0x0c0f0e0d);
3399&data_word(0x04070605,0x04070605,0x04070605,0x04070605);
3400&data_word(1,1,1,1);
3401&data_word(0x1b,0x1b,0x1b,0x1b);
d64a7232
AP		 3402&asciz("AES for Intel AES-NI, CRYPTOGAMS by <appro\@openssl.org>");
3403
3404&asm_finish();
184bc45f
RL		 3405
3406close STDOUT;
A mirror of the official openssl repository