]> git.ipfire.org Git - thirdparty/openssl.git/blame - crypto/aes/asm/aesni-x86.pl
projects / thirdparty / openssl.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
x86[_64] assembly pack: add optimized AES-NI OCB subroutines.
[thirdparty/openssl.git] / crypto / aes / asm / aesni-x86.pl
CommitLineData
d64a7232
AP		 1#!/usr/bin/env perl
2
3# ====================================================================
d8ba0dc9 4# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
d64a7232
AP		 5# project. The module is, however, dual licensed under OpenSSL and
6# CRYPTOGAMS licenses depending on where you obtain it. For further
7# details see http://www.openssl.org/~appro/cryptogams/.
8# ====================================================================
9#
10# This module implements support for Intel AES-NI extension. In
11# OpenSSL context it's used with Intel engine, but can also be used as
12# drop-in replacement for crypto/aes/asm/aes-586.pl [see below for
13# details].
d7d119a3
AP		 14#
15# Performance.
16#
17# To start with see corresponding paragraph in aesni-x86_64.pl...
18# Instead of filling table similar to one found there I've chosen to
19# summarize *comparison* results for raw ECB, CTR and CBC benchmarks.
20# The simplified table below represents 32-bit performance relative
21# to 64-bit one in every given point. Ratios vary for different
22# encryption modes, therefore interval values.
23#
24# 16-byte 64-byte 256-byte 1-KB 8-KB
25# 53-67% 67-84% 91-94% 95-98% 97-99.5%
26#
27# Lower ratios for smaller block sizes are perfectly understandable,
28# because function call overhead is higher in 32-bit mode. Largest
29# 8-KB block performance is virtually same: 32-bit code is less than
f8501464
AP		 30# 1% slower for ECB, CBC and CCM, and ~3% slower otherwise.
31
32# January 2011
33#
34# See aesni-x86_64.pl for details. Unlike x86_64 version this module
35# interleaves at most 6 aes[enc|dec] instructions, because there are
36# not enough registers for 8x interleave [which should be optimal for
37# Sandy Bridge]. Actually, performance results for 6x interleave
38# factor presented in aesni-x86_64.pl (except for CTR) are for this
39# module.
40
41# April 2011
42#
43# Add aesni_xts_[en|de]crypt. Westmere spends 1.50 cycles processing
44# one byte out of 8KB with 128-bit key, Sandy Bridge - 1.09.
d64a7232 45
bd30091c
AP		 46# November 2015
47#
48# Add aesni_ocb_[en|de]crypt.
49
5599c733
AP		 50######################################################################
51# Current large-block performance in cycles per byte processed with
52# 128-bit key (less is better).
53#
bd30091c 54# CBC en-/decrypt CTR XTS ECB OCB
5599c733 55# Westmere 3.77/1.37 1.37 1.52 1.27
bd30091c
AP		 56# * Bridge 5.07/0.98 0.99 1.09 0.91 1.10
57# Haswell 4.44/0.80 0.97 1.03 0.72 0.76
58# Silvermont 5.77/3.56 3.67 4.03 3.46 4.03
59# Bulldozer 5.80/0.98 1.05 1.24 0.93 1.23
5599c733 60
d64a7232
AP		 61$PREFIX="aesni"; # if $PREFIX is set to "AES", the script
62 # generates drop-in replacement for
63 # crypto/aes/asm/aes-586.pl:-)
6f766a41 64$inline=1; # inline _aesni_[en|de]crypt
d64a7232
AP		 65
66$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
67push(@INC,"${dir}","${dir}../../perlasm");
68require "x86asm.pl";
69
70&asm_init($ARGV[0],$0);
71
23f6eec7
AP		 72&external_label("OPENSSL_ia32cap_P");
73&static_label("key_const");
74
f9c5e5d9
AP		 75if ($PREFIX eq "aesni") { $movekey=\&movups; }
76else { $movekey=\&movups; }
d64a7232
AP		 77
78$len="eax";
79$rounds="ecx";
80$key="edx";
81$inp="esi";
82$out="edi";
d608b4d6
AP		 83$rounds_="ebx"; # backup copy for $rounds
84$key_="ebp"; # backup copy for $key
d64a7232 85
f8501464
AP		 86$rndkey0="xmm0";
87$rndkey1="xmm1";
88$inout0="xmm2";
89$inout1="xmm3";
90$inout2="xmm4";
91$inout3="xmm5"; $in1="xmm5";
92$inout4="xmm6"; $in0="xmm6";
93$inout5="xmm7"; $ivec="xmm7";
133a7f9a 94
d900a015 95# AESNI extension
133a7f9a
AP		 96sub aeskeygenassist
97{ my($dst,$src,$imm)=@_;
98 if ("$dst:$src" =~ /xmm([0-7]):xmm([0-7])/)
99 { &data_byte(0x66,0x0f,0x3a,0xdf,0xc0|($1<<3)|$2,$imm); }
100}
101sub aescommon
102{ my($opcodelet,$dst,$src)=@_;
103 if ("$dst:$src" =~ /xmm([0-7]):xmm([0-7])/)
104 { &data_byte(0x66,0x0f,0x38,$opcodelet,0xc0|($1<<3)|$2);}
105}
106sub aesimc { aescommon(0xdb,@_); }
107sub aesenc { aescommon(0xdc,@_); }
108sub aesenclast { aescommon(0xdd,@_); }
109sub aesdec { aescommon(0xde,@_); }
110sub aesdeclast { aescommon(0xdf,@_); }
6c83629b 111\f
d608b4d6 112# Inline version of internal aesni_[en|de]crypt1
d7d119a3 113{ my $sn;
d608b4d6 114sub aesni_inline_generate1
f8501464 115{ my ($p,$inout,$ivec)=@_; $inout=$inout0 if (!defined($inout));
d7d119a3 116 $sn++;
d64a7232 117
f8501464 118 &$movekey ($rndkey0,&QWP(0,$key));
d608b4d6 119 &$movekey ($rndkey1,&QWP(16,$key));
f8501464 120 &xorps ($ivec,$rndkey0) if (defined($ivec));
d608b4d6 121 &lea ($key,&DWP(32,$key));
f8501464
AP		 122 &xorps ($inout,$ivec) if (defined($ivec));
123 &xorps ($inout,$rndkey0) if (!defined($ivec));
d7d119a3
AP		 124 &set_label("${p}1_loop_$sn");
125 eval"&aes${p} ($inout,$rndkey1)";
d64a7232 126 &dec ($rounds);
d64a7232 127 &$movekey ($rndkey1,&QWP(0,$key));
d608b4d6 128 &lea ($key,&DWP(16,$key));
d7d119a3
AP		 129 &jnz (&label("${p}1_loop_$sn"));
130 eval"&aes${p}last ($inout,$rndkey1)";
131}}
d64a7232
AP		 132
133sub aesni_generate1 # fully unrolled loop
d7d119a3 134{ my ($p,$inout)=@_; $inout=$inout0 if (!defined($inout));
d64a7232
AP		 135
136 &function_begin_B("_aesni_${p}rypt1");
f8501464 137 &movups ($rndkey0,&QWP(0,$key));
d64a7232 138 &$movekey ($rndkey1,&QWP(0x10,$key));
f8501464 139 &xorps ($inout,$rndkey0);
d64a7232
AP		 140 &$movekey ($rndkey0,&QWP(0x20,$key));
141 &lea ($key,&DWP(0x30,$key));
d7d119a3 142 &cmp ($rounds,11);
d64a7232
AP		 143 &jb (&label("${p}128"));
144 &lea ($key,&DWP(0x20,$key));
145 &je (&label("${p}192"));
146 &lea ($key,&DWP(0x20,$key));
d7d119a3 147 eval"&aes${p} ($inout,$rndkey1)";
d64a7232 148 &$movekey ($rndkey1,&QWP(-0x40,$key));
d7d119a3 149 eval"&aes${p} ($inout,$rndkey0)";
d64a7232
AP		 150 &$movekey ($rndkey0,&QWP(-0x30,$key));
151 &set_label("${p}192");
d7d119a3 152 eval"&aes${p} ($inout,$rndkey1)";
d64a7232 153 &$movekey ($rndkey1,&QWP(-0x20,$key));
d7d119a3 154 eval"&aes${p} ($inout,$rndkey0)";
d64a7232
AP		 155 &$movekey ($rndkey0,&QWP(-0x10,$key));
156 &set_label("${p}128");
d7d119a3 157 eval"&aes${p} ($inout,$rndkey1)";
d64a7232 158 &$movekey ($rndkey1,&QWP(0,$key));
d7d119a3 159 eval"&aes${p} ($inout,$rndkey0)";
d64a7232 160 &$movekey ($rndkey0,&QWP(0x10,$key));
d7d119a3 161 eval"&aes${p} ($inout,$rndkey1)";
d64a7232 162 &$movekey ($rndkey1,&QWP(0x20,$key));
d7d119a3 163 eval"&aes${p} ($inout,$rndkey0)";
d64a7232 164 &$movekey ($rndkey0,&QWP(0x30,$key));
d7d119a3 165 eval"&aes${p} ($inout,$rndkey1)";
d64a7232 166 &$movekey ($rndkey1,&QWP(0x40,$key));
d7d119a3 167 eval"&aes${p} ($inout,$rndkey0)";
d64a7232 168 &$movekey ($rndkey0,&QWP(0x50,$key));
d7d119a3 169 eval"&aes${p} ($inout,$rndkey1)";
d64a7232 170 &$movekey ($rndkey1,&QWP(0x60,$key));
d7d119a3 171 eval"&aes${p} ($inout,$rndkey0)";
d64a7232 172 &$movekey ($rndkey0,&QWP(0x70,$key));
d7d119a3
AP		 173 eval"&aes${p} ($inout,$rndkey1)";
174 eval"&aes${p}last ($inout,$rndkey0)";
d64a7232
AP		 175 &ret();
176 &function_end_B("_aesni_${p}rypt1");
177}
6c83629b 178\f
d64a7232 179# void $PREFIX_encrypt (const void *inp,void *out,const AES_KEY *key);
6f766a41 180&aesni_generate1("enc") if (!$inline);
d64a7232
AP		 181&function_begin_B("${PREFIX}_encrypt");
182 &mov ("eax",&wparam(0));
183 &mov ($key,&wparam(2));
f8501464 184 &movups ($inout0,&QWP(0,"eax"));
d64a7232
AP		 185 &mov ($rounds,&DWP(240,$key));
186 &mov ("eax",&wparam(1));
6f766a41
AP		 187 if ($inline)
188 { &aesni_inline_generate1("enc"); }
189 else
190 { &call ("_aesni_encrypt1"); }
23f6eec7
AP		 191 &pxor ($rndkey0,$rndkey0); # clear register bank
192 &pxor ($rndkey1,$rndkey1);
d64a7232 193 &movups (&QWP(0,"eax"),$inout0);
23f6eec7 194 &pxor ($inout0,$inout0);
d64a7232
AP		 195 &ret ();
196&function_end_B("${PREFIX}_encrypt");
197
d64a7232 198# void $PREFIX_decrypt (const void *inp,void *out,const AES_KEY *key);
6f766a41 199&aesni_generate1("dec") if(!$inline);
d64a7232
AP		 200&function_begin_B("${PREFIX}_decrypt");
201 &mov ("eax",&wparam(0));
202 &mov ($key,&wparam(2));
f8501464 203 &movups ($inout0,&QWP(0,"eax"));
d64a7232
AP		 204 &mov ($rounds,&DWP(240,$key));
205 &mov ("eax",&wparam(1));
6f766a41
AP		 206 if ($inline)
207 { &aesni_inline_generate1("dec"); }
208 else
209 { &call ("_aesni_decrypt1"); }
23f6eec7
AP		 210 &pxor ($rndkey0,$rndkey0); # clear register bank
211 &pxor ($rndkey1,$rndkey1);
d64a7232 212 &movups (&QWP(0,"eax"),$inout0);
23f6eec7 213 &pxor ($inout0,$inout0);
d64a7232
AP		 214 &ret ();
215&function_end_B("${PREFIX}_decrypt");
6c83629b 216
f8501464
AP		 217# _aesni_[en|de]cryptN are private interfaces, N denotes interleave
218# factor. Why 3x subroutine were originally used in loops? Even though
219# aes[enc|dec] latency was originally 6, it could be scheduled only
220# every *2nd* cycle. Thus 3x interleave was the one providing optimal
d608b4d6
AP		 221# utilization, i.e. when subroutine's throughput is virtually same as
222# of non-interleaved subroutine [for number of input blocks up to 3].
214368ff
AP		 223# This is why it originally made no sense to implement 2x subroutine.
224# But times change and it became appropriate to spend extra 192 bytes
225# on 2x subroutine on Atom Silvermont account. For processors that
226# can schedule aes[enc|dec] every cycle optimal interleave factor
227# equals to corresponding instructions latency. 8x is optimal for
228# * Bridge, but it's unfeasible to accommodate such implementation
229# in XMM registers addreassable in 32-bit mode and therefore maximum
230# of 6x is used instead...
231
232sub aesni_generate2
233{ my $p=shift;
234
235 &function_begin_B("_aesni_${p}rypt2");
236 &$movekey ($rndkey0,&QWP(0,$key));
237 &shl ($rounds,4);
238 &$movekey ($rndkey1,&QWP(16,$key));
239 &xorps ($inout0,$rndkey0);
240 &pxor ($inout1,$rndkey0);
241 &$movekey ($rndkey0,&QWP(32,$key));
242 &lea ($key,&DWP(32,$key,$rounds));
243 &neg ($rounds);
244 &add ($rounds,16);
245
246 &set_label("${p}2_loop");
247 eval"&aes${p} ($inout0,$rndkey1)";
248 eval"&aes${p} ($inout1,$rndkey1)";
249 &$movekey ($rndkey1,&QWP(0,$key,$rounds));
250 &add ($rounds,32);
251 eval"&aes${p} ($inout0,$rndkey0)";
252 eval"&aes${p} ($inout1,$rndkey0)";
253 &$movekey ($rndkey0,&QWP(-16,$key,$rounds));
254 &jnz (&label("${p}2_loop"));
255 eval"&aes${p} ($inout0,$rndkey1)";
256 eval"&aes${p} ($inout1,$rndkey1)";
257 eval"&aes${p}last ($inout0,$rndkey0)";
258 eval"&aes${p}last ($inout1,$rndkey0)";
259 &ret();
260 &function_end_B("_aesni_${p}rypt2");
261}
f8501464 262
d64a7232
AP		 263sub aesni_generate3
264{ my $p=shift;
265
266 &function_begin_B("_aesni_${p}rypt3");
267 &$movekey ($rndkey0,&QWP(0,$key));
d8ba0dc9 268 &shl ($rounds,4);
d608b4d6 269 &$movekey ($rndkey1,&QWP(16,$key));
f8501464 270 &xorps ($inout0,$rndkey0);
d64a7232 271 &pxor ($inout1,$rndkey0);
d64a7232 272 &pxor ($inout2,$rndkey0);
d8ba0dc9
AP		 273 &$movekey ($rndkey0,&QWP(32,$key));
274 &lea ($key,&DWP(32,$key,$rounds));
275 &neg ($rounds);
276 &add ($rounds,16);
d7d119a3
AP		 277
278 &set_label("${p}3_loop");
279 eval"&aes${p} ($inout0,$rndkey1)";
d64a7232 280 eval"&aes${p} ($inout1,$rndkey1)";
d64a7232 281 eval"&aes${p} ($inout2,$rndkey1)";
d8ba0dc9
AP		 282 &$movekey ($rndkey1,&QWP(0,$key,$rounds));
283 &add ($rounds,32);
d64a7232 284 eval"&aes${p} ($inout0,$rndkey0)";
d64a7232
AP		 285 eval"&aes${p} ($inout1,$rndkey0)";
286 eval"&aes${p} ($inout2,$rndkey0)";
d8ba0dc9 287 &$movekey ($rndkey0,&QWP(-16,$key,$rounds));
d64a7232
AP		 288 &jnz (&label("${p}3_loop"));
289 eval"&aes${p} ($inout0,$rndkey1)";
d64a7232
AP		 290 eval"&aes${p} ($inout1,$rndkey1)";
291 eval"&aes${p} ($inout2,$rndkey1)";
292 eval"&aes${p}last ($inout0,$rndkey0)";
293 eval"&aes${p}last ($inout1,$rndkey0)";
294 eval"&aes${p}last ($inout2,$rndkey0)";
295 &ret();
296 &function_end_B("_aesni_${p}rypt3");
297}
d608b4d6
AP		 298
299# 4x interleave is implemented to improve small block performance,
300# most notably [and naturally] 4 block by ~30%. One can argue that one
301# should have implemented 5x as well, but improvement would be <20%,
302# so it's not worth it...
303sub aesni_generate4
304{ my $p=shift;
305
306 &function_begin_B("_aesni_${p}rypt4");
307 &$movekey ($rndkey0,&QWP(0,$key));
308 &$movekey ($rndkey1,&QWP(16,$key));
d8ba0dc9 309 &shl ($rounds,4);
f8501464 310 &xorps ($inout0,$rndkey0);
d608b4d6
AP		 311 &pxor ($inout1,$rndkey0);
312 &pxor ($inout2,$rndkey0);
313 &pxor ($inout3,$rndkey0);
d8ba0dc9
AP		 314 &$movekey ($rndkey0,&QWP(32,$key));
315 &lea ($key,&DWP(32,$key,$rounds));
316 &neg ($rounds);
317 &data_byte (0x0f,0x1f,0x40,0x00);
318 &add ($rounds,16);
d7d119a3 319
f8501464 320 &set_label("${p}4_loop");
d7d119a3 321 eval"&aes${p} ($inout0,$rndkey1)";
d608b4d6 322 eval"&aes${p} ($inout1,$rndkey1)";
d608b4d6
AP		 323 eval"&aes${p} ($inout2,$rndkey1)";
324 eval"&aes${p} ($inout3,$rndkey1)";
d8ba0dc9
AP		 325 &$movekey ($rndkey1,&QWP(0,$key,$rounds));
326 &add ($rounds,32);
d608b4d6 327 eval"&aes${p} ($inout0,$rndkey0)";
d608b4d6
AP		 328 eval"&aes${p} ($inout1,$rndkey0)";
329 eval"&aes${p} ($inout2,$rndkey0)";
330 eval"&aes${p} ($inout3,$rndkey0)";
d8ba0dc9 331 &$movekey ($rndkey0,&QWP(-16,$key,$rounds));
f8501464 332 &jnz (&label("${p}4_loop"));
d7d119a3 333
d608b4d6 334 eval"&aes${p} ($inout0,$rndkey1)";
d608b4d6
AP		 335 eval"&aes${p} ($inout1,$rndkey1)";
336 eval"&aes${p} ($inout2,$rndkey1)";
337 eval"&aes${p} ($inout3,$rndkey1)";
338 eval"&aes${p}last ($inout0,$rndkey0)";
339 eval"&aes${p}last ($inout1,$rndkey0)";
340 eval"&aes${p}last ($inout2,$rndkey0)";
341 eval"&aes${p}last ($inout3,$rndkey0)";
342 &ret();
343 &function_end_B("_aesni_${p}rypt4");
344}
f8501464
AP		 345
346sub aesni_generate6
347{ my $p=shift;
348
349 &function_begin_B("_aesni_${p}rypt6");
350 &static_label("_aesni_${p}rypt6_enter");
351 &$movekey ($rndkey0,&QWP(0,$key));
d8ba0dc9 352 &shl ($rounds,4);
f8501464 353 &$movekey ($rndkey1,&QWP(16,$key));
f8501464
AP		 354 &xorps ($inout0,$rndkey0);
355 &pxor ($inout1,$rndkey0); # pxor does better here
f8501464 356 &pxor ($inout2,$rndkey0);
d8ba0dc9 357 eval"&aes${p} ($inout0,$rndkey1)";
f8501464 358 &pxor ($inout3,$rndkey0);
f8501464 359 &pxor ($inout4,$rndkey0);
d8ba0dc9
AP		 360 eval"&aes${p} ($inout1,$rndkey1)";
361 &lea ($key,&DWP(32,$key,$rounds));
362 &neg ($rounds);
363 eval"&aes${p} ($inout2,$rndkey1)";
f8501464 364 &pxor ($inout5,$rndkey0);
23f6eec7 365 &$movekey ($rndkey0,&QWP(0,$key,$rounds));
d8ba0dc9 366 &add ($rounds,16);
23f6eec7 367 &jmp (&label("_aesni_${p}rypt6_inner"));
f8501464
AP		 368
369 &set_label("${p}6_loop",16);
370 eval"&aes${p} ($inout0,$rndkey1)";
371 eval"&aes${p} ($inout1,$rndkey1)";
f8501464 372 eval"&aes${p} ($inout2,$rndkey1)";
23f6eec7 373 &set_label("_aesni_${p}rypt6_inner");
f8501464
AP		 374 eval"&aes${p} ($inout3,$rndkey1)";
375 eval"&aes${p} ($inout4,$rndkey1)";
376 eval"&aes${p} ($inout5,$rndkey1)";
d8ba0dc9
AP		 377 &set_label("_aesni_${p}rypt6_enter");
378 &$movekey ($rndkey1,&QWP(0,$key,$rounds));
379 &add ($rounds,32);
f8501464
AP		 380 eval"&aes${p} ($inout0,$rndkey0)";
381 eval"&aes${p} ($inout1,$rndkey0)";
f8501464
AP		 382 eval"&aes${p} ($inout2,$rndkey0)";
383 eval"&aes${p} ($inout3,$rndkey0)";
384 eval"&aes${p} ($inout4,$rndkey0)";
385 eval"&aes${p} ($inout5,$rndkey0)";
d8ba0dc9 386 &$movekey ($rndkey0,&QWP(-16,$key,$rounds));
f8501464
AP		 387 &jnz (&label("${p}6_loop"));
388
389 eval"&aes${p} ($inout0,$rndkey1)";
390 eval"&aes${p} ($inout1,$rndkey1)";
391 eval"&aes${p} ($inout2,$rndkey1)";
392 eval"&aes${p} ($inout3,$rndkey1)";
393 eval"&aes${p} ($inout4,$rndkey1)";
394 eval"&aes${p} ($inout5,$rndkey1)";
395 eval"&aes${p}last ($inout0,$rndkey0)";
396 eval"&aes${p}last ($inout1,$rndkey0)";
397 eval"&aes${p}last ($inout2,$rndkey0)";
398 eval"&aes${p}last ($inout3,$rndkey0)";
399 eval"&aes${p}last ($inout4,$rndkey0)";
400 eval"&aes${p}last ($inout5,$rndkey0)";
401 &ret();
402 &function_end_B("_aesni_${p}rypt6");
403}
214368ff
AP		 404&aesni_generate2("enc") if ($PREFIX eq "aesni");
405&aesni_generate2("dec");
d64a7232
AP		 406&aesni_generate3("enc") if ($PREFIX eq "aesni");
407&aesni_generate3("dec");
d608b4d6
AP		 408&aesni_generate4("enc") if ($PREFIX eq "aesni");
409&aesni_generate4("dec");
f8501464
AP		 410&aesni_generate6("enc") if ($PREFIX eq "aesni");
411&aesni_generate6("dec");
6c83629b 412\f
d64a7232 413if ($PREFIX eq "aesni") {
6c83629b 414######################################################################
d64a7232
AP		 415# void aesni_ecb_encrypt (const void *in, void *out,
416# size_t length, const AES_KEY *key,
417# int enc);
d64a7232
AP		 418&function_begin("aesni_ecb_encrypt");
419 &mov ($inp,&wparam(0));
420 &mov ($out,&wparam(1));
421 &mov ($len,&wparam(2));
422 &mov ($key,&wparam(3));
f8501464 423 &mov ($rounds_,&wparam(4));
d64a7232 424 &and ($len,-16);
f8501464 425 &jz (&label("ecb_ret"));
d64a7232 426 &mov ($rounds,&DWP(240,$key));
f8501464
AP		 427 &test ($rounds_,$rounds_);
428 &jz (&label("ecb_decrypt"));
429
d64a7232
AP		 430 &mov ($key_,$key); # backup $key
431 &mov ($rounds_,$rounds); # backup $rounds
f8501464
AP		 432 &cmp ($len,0x60);
433 &jb (&label("ecb_enc_tail"));
434
435 &movdqu ($inout0,&QWP(0,$inp));
436 &movdqu ($inout1,&QWP(0x10,$inp));
437 &movdqu ($inout2,&QWP(0x20,$inp));
438 &movdqu ($inout3,&QWP(0x30,$inp));
439 &movdqu ($inout4,&QWP(0x40,$inp));
440 &movdqu ($inout5,&QWP(0x50,$inp));
441 &lea ($inp,&DWP(0x60,$inp));
442 &sub ($len,0x60);
443 &jmp (&label("ecb_enc_loop6_enter"));
444
445&set_label("ecb_enc_loop6",16);
446 &movups (&QWP(0,$out),$inout0);
447 &movdqu ($inout0,&QWP(0,$inp));
448 &movups (&QWP(0x10,$out),$inout1);
449 &movdqu ($inout1,&QWP(0x10,$inp));
450 &movups (&QWP(0x20,$out),$inout2);
451 &movdqu ($inout2,&QWP(0x20,$inp));
452 &movups (&QWP(0x30,$out),$inout3);
453 &movdqu ($inout3,&QWP(0x30,$inp));
454 &movups (&QWP(0x40,$out),$inout4);
455 &movdqu ($inout4,&QWP(0x40,$inp));
456 &movups (&QWP(0x50,$out),$inout5);
457 &lea ($out,&DWP(0x60,$out));
458 &movdqu ($inout5,&QWP(0x50,$inp));
459 &lea ($inp,&DWP(0x60,$inp));
460&set_label("ecb_enc_loop6_enter");
d64a7232 461
f8501464 462 &call ("_aesni_encrypt6");
d64a7232 463
d64a7232 464 &mov ($key,$key_); # restore $key
d64a7232 465 &mov ($rounds,$rounds_); # restore $rounds
f8501464
AP		 466 &sub ($len,0x60);
467 &jnc (&label("ecb_enc_loop6"));
468
469 &movups (&QWP(0,$out),$inout0);
470 &movups (&QWP(0x10,$out),$inout1);
d7d119a3 471 &movups (&QWP(0x20,$out),$inout2);
f8501464
AP		 472 &movups (&QWP(0x30,$out),$inout3);
473 &movups (&QWP(0x40,$out),$inout4);
474 &movups (&QWP(0x50,$out),$inout5);
475 &lea ($out,&DWP(0x60,$out));
476 &add ($len,0x60);
477 &jz (&label("ecb_ret"));
d64a7232 478
6c83629b 479&set_label("ecb_enc_tail");
6c83629b 480 &movups ($inout0,&QWP(0,$inp));
d7d119a3 481 &cmp ($len,0x20);
6c83629b 482 &jb (&label("ecb_enc_one"));
d64a7232 483 &movups ($inout1,&QWP(0x10,$inp));
d608b4d6 484 &je (&label("ecb_enc_two"));
d608b4d6 485 &movups ($inout2,&QWP(0x20,$inp));
f8501464
AP		 486 &cmp ($len,0x40);
487 &jb (&label("ecb_enc_three"));
d608b4d6 488 &movups ($inout3,&QWP(0x30,$inp));
f8501464
AP		 489 &je (&label("ecb_enc_four"));
490 &movups ($inout4,&QWP(0x40,$inp));
491 &xorps ($inout5,$inout5);
492 &call ("_aesni_encrypt6");
d64a7232
AP		 493 &movups (&QWP(0,$out),$inout0);
494 &movups (&QWP(0x10,$out),$inout1);
d608b4d6
AP		 495 &movups (&QWP(0x20,$out),$inout2);
496 &movups (&QWP(0x30,$out),$inout3);
f8501464 497 &movups (&QWP(0x40,$out),$inout4);
d64a7232
AP		 498 jmp (&label("ecb_ret"));
499
500&set_label("ecb_enc_one",16);
6f766a41
AP		 501 if ($inline)
502 { &aesni_inline_generate1("enc"); }
503 else
504 { &call ("_aesni_encrypt1"); }
d64a7232
AP		 505 &movups (&QWP(0,$out),$inout0);
506 &jmp (&label("ecb_ret"));
507
d608b4d6 508&set_label("ecb_enc_two",16);
214368ff 509 &call ("_aesni_encrypt2");
d608b4d6
AP		 510 &movups (&QWP(0,$out),$inout0);
511 &movups (&QWP(0x10,$out),$inout1);
512 &jmp (&label("ecb_ret"));
513
514&set_label("ecb_enc_three",16);
515 &call ("_aesni_encrypt3");
516 &movups (&QWP(0,$out),$inout0);
517 &movups (&QWP(0x10,$out),$inout1);
518 &movups (&QWP(0x20,$out),$inout2);
519 &jmp (&label("ecb_ret"));
f8501464
AP		 520
521&set_label("ecb_enc_four",16);
522 &call ("_aesni_encrypt4");
523 &movups (&QWP(0,$out),$inout0);
524 &movups (&QWP(0x10,$out),$inout1);
525 &movups (&QWP(0x20,$out),$inout2);
526 &movups (&QWP(0x30,$out),$inout3);
527 &jmp (&label("ecb_ret"));
6c83629b 528######################################################################
d64a7232 529&set_label("ecb_decrypt",16);
f8501464
AP		 530 &mov ($key_,$key); # backup $key
531 &mov ($rounds_,$rounds); # backup $rounds
532 &cmp ($len,0x60);
533 &jb (&label("ecb_dec_tail"));
534
535 &movdqu ($inout0,&QWP(0,$inp));
536 &movdqu ($inout1,&QWP(0x10,$inp));
537 &movdqu ($inout2,&QWP(0x20,$inp));
538 &movdqu ($inout3,&QWP(0x30,$inp));
539 &movdqu ($inout4,&QWP(0x40,$inp));
540 &movdqu ($inout5,&QWP(0x50,$inp));
541 &lea ($inp,&DWP(0x60,$inp));
542 &sub ($len,0x60);
543 &jmp (&label("ecb_dec_loop6_enter"));
544
545&set_label("ecb_dec_loop6",16);
d7d119a3 546 &movups (&QWP(0,$out),$inout0);
f8501464 547 &movdqu ($inout0,&QWP(0,$inp));
d7d119a3 548 &movups (&QWP(0x10,$out),$inout1);
f8501464
AP		 549 &movdqu ($inout1,&QWP(0x10,$inp));
550 &movups (&QWP(0x20,$out),$inout2);
551 &movdqu ($inout2,&QWP(0x20,$inp));
552 &movups (&QWP(0x30,$out),$inout3);
553 &movdqu ($inout3,&QWP(0x30,$inp));
554 &movups (&QWP(0x40,$out),$inout4);
555 &movdqu ($inout4,&QWP(0x40,$inp));
556 &movups (&QWP(0x50,$out),$inout5);
557 &lea ($out,&DWP(0x60,$out));
558 &movdqu ($inout5,&QWP(0x50,$inp));
559 &lea ($inp,&DWP(0x60,$inp));
560&set_label("ecb_dec_loop6_enter");
561
562 &call ("_aesni_decrypt6");
563
564 &mov ($key,$key_); # restore $key
d64a7232 565 &mov ($rounds,$rounds_); # restore $rounds
f8501464
AP		 566 &sub ($len,0x60);
567 &jnc (&label("ecb_dec_loop6"));
568
569 &movups (&QWP(0,$out),$inout0);
570 &movups (&QWP(0x10,$out),$inout1);
d7d119a3 571 &movups (&QWP(0x20,$out),$inout2);
f8501464
AP		 572 &movups (&QWP(0x30,$out),$inout3);
573 &movups (&QWP(0x40,$out),$inout4);
574 &movups (&QWP(0x50,$out),$inout5);
575 &lea ($out,&DWP(0x60,$out));
576 &add ($len,0x60);
577 &jz (&label("ecb_ret"));
d64a7232 578
6c83629b 579&set_label("ecb_dec_tail");
6c83629b 580 &movups ($inout0,&QWP(0,$inp));
d7d119a3 581 &cmp ($len,0x20);
6c83629b 582 &jb (&label("ecb_dec_one"));
d64a7232 583 &movups ($inout1,&QWP(0x10,$inp));
d608b4d6 584 &je (&label("ecb_dec_two"));
d608b4d6 585 &movups ($inout2,&QWP(0x20,$inp));
f8501464
AP		 586 &cmp ($len,0x40);
587 &jb (&label("ecb_dec_three"));
d608b4d6 588 &movups ($inout3,&QWP(0x30,$inp));
f8501464
AP		 589 &je (&label("ecb_dec_four"));
590 &movups ($inout4,&QWP(0x40,$inp));
591 &xorps ($inout5,$inout5);
592 &call ("_aesni_decrypt6");
d64a7232
AP		 593 &movups (&QWP(0,$out),$inout0);
594 &movups (&QWP(0x10,$out),$inout1);
d608b4d6
AP		 595 &movups (&QWP(0x20,$out),$inout2);
596 &movups (&QWP(0x30,$out),$inout3);
f8501464 597 &movups (&QWP(0x40,$out),$inout4);
d608b4d6 598 &jmp (&label("ecb_ret"));
d64a7232
AP		 599
600&set_label("ecb_dec_one",16);
6f766a41
AP		 601 if ($inline)
602 { &aesni_inline_generate1("dec"); }
603 else
604 { &call ("_aesni_decrypt1"); }
d64a7232 605 &movups (&QWP(0,$out),$inout0);
d608b4d6
AP		 606 &jmp (&label("ecb_ret"));
607
608&set_label("ecb_dec_two",16);
214368ff 609 &call ("_aesni_decrypt2");
d608b4d6
AP		 610 &movups (&QWP(0,$out),$inout0);
611 &movups (&QWP(0x10,$out),$inout1);
612 &jmp (&label("ecb_ret"));
613
614&set_label("ecb_dec_three",16);
615 &call ("_aesni_decrypt3");
616 &movups (&QWP(0,$out),$inout0);
617 &movups (&QWP(0x10,$out),$inout1);
618 &movups (&QWP(0x20,$out),$inout2);
f8501464
AP		 619 &jmp (&label("ecb_ret"));
620
621&set_label("ecb_dec_four",16);
622 &call ("_aesni_decrypt4");
623 &movups (&QWP(0,$out),$inout0);
624 &movups (&QWP(0x10,$out),$inout1);
625 &movups (&QWP(0x20,$out),$inout2);
626 &movups (&QWP(0x30,$out),$inout3);
d64a7232
AP		 627
628&set_label("ecb_ret");
23f6eec7
AP		 629 &pxor ("xmm0","xmm0"); # clear register bank
630 &pxor ("xmm1","xmm1");
631 &pxor ("xmm2","xmm2");
632 &pxor ("xmm3","xmm3");
633 &pxor ("xmm4","xmm4");
634 &pxor ("xmm5","xmm5");
635 &pxor ("xmm6","xmm6");
636 &pxor ("xmm7","xmm7");
d64a7232 637&function_end("aesni_ecb_encrypt");
6c83629b
AP		 638\f
639######################################################################
d7d119a3
AP		 640# void aesni_ccm64_[en|de]crypt_blocks (const void *in, void *out,
641# size_t blocks, const AES_KEY *key,
642# const char *ivec,char *cmac);
643#
644# Handles only complete blocks, operates on 64-bit counter and
645# does not update *ivec! Nor does it finalize CMAC value
646# (see engine/eng_aesni.c for details)
6c83629b 647#
f8501464 648{ my $cmac=$inout1;
d7d119a3
AP		 649&function_begin("aesni_ccm64_encrypt_blocks");
650 &mov ($inp,&wparam(0));
651 &mov ($out,&wparam(1));
652 &mov ($len,&wparam(2));
653 &mov ($key,&wparam(3));
654 &mov ($rounds_,&wparam(4));
655 &mov ($rounds,&wparam(5));
656 &mov ($key_,"esp");
657 &sub ("esp",60);
658 &and ("esp",-16); # align stack
659 &mov (&DWP(48,"esp"),$key_);
660
661 &movdqu ($ivec,&QWP(0,$rounds_)); # load ivec
f8501464 662 &movdqu ($cmac,&QWP(0,$rounds)); # load cmac
267b481c 663 &mov ($rounds,&DWP(240,$key));
d7d119a3
AP		 664
665 # compose byte-swap control mask for pshufb on stack
666 &mov (&DWP(0,"esp"),0x0c0d0e0f);
667 &mov (&DWP(4,"esp"),0x08090a0b);
668 &mov (&DWP(8,"esp"),0x04050607);
669 &mov (&DWP(12,"esp"),0x00010203);
670
671 # compose counter increment vector on stack
267b481c 672 &mov ($rounds_,1);
d7d119a3 673 &xor ($key_,$key_);
267b481c 674 &mov (&DWP(16,"esp"),$rounds_);
d7d119a3
AP		 675 &mov (&DWP(20,"esp"),$key_);
676 &mov (&DWP(24,"esp"),$key_);
677 &mov (&DWP(28,"esp"),$key_);
678
d8ba0dc9
AP		 679 &shl ($rounds,4);
680 &mov ($rounds_,16);
267b481c 681 &lea ($key_,&DWP(0,$key));
9ee5916d 682 &movdqa ($inout3,&QWP(0,"esp"));
d7d119a3 683 &movdqa ($inout0,$ivec);
d8ba0dc9
AP		 684 &lea ($key,&DWP(32,$key,$rounds));
685 &sub ($rounds_,$rounds);
9ee5916d 686 &pshufb ($ivec,$inout3);
d7d119a3
AP		 687
688&set_label("ccm64_enc_outer");
267b481c 689 &$movekey ($rndkey0,&QWP(0,$key_));
f8501464 690 &mov ($rounds,$rounds_);
267b481c 691 &movups ($in0,&QWP(0,$inp));
d7d119a3 692
f8501464 693 &xorps ($inout0,$rndkey0);
267b481c
AP		 694 &$movekey ($rndkey1,&QWP(16,$key_));
695 &xorps ($rndkey0,$in0);
267b481c 696 &xorps ($cmac,$rndkey0); # cmac^=inp
d8ba0dc9 697 &$movekey ($rndkey0,&QWP(32,$key_));
f8501464
AP		 698
699&set_label("ccm64_enc2_loop");
700 &aesenc ($inout0,$rndkey1);
f8501464 701 &aesenc ($cmac,$rndkey1);
d8ba0dc9
AP		 702 &$movekey ($rndkey1,&QWP(0,$key,$rounds));
703 &add ($rounds,32);
f8501464 704 &aesenc ($inout0,$rndkey0);
f8501464 705 &aesenc ($cmac,$rndkey0);
d8ba0dc9 706 &$movekey ($rndkey0,&QWP(-16,$key,$rounds));
f8501464
AP		 707 &jnz (&label("ccm64_enc2_loop"));
708 &aesenc ($inout0,$rndkey1);
709 &aesenc ($cmac,$rndkey1);
267b481c 710 &paddq ($ivec,&QWP(16,"esp"));
d8ba0dc9 711 &dec ($len);
f8501464
AP		 712 &aesenclast ($inout0,$rndkey0);
713 &aesenclast ($cmac,$rndkey0);
d7d119a3 714
d7d119a3 715 &lea ($inp,&DWP(16,$inp));
f8501464 716 &xorps ($in0,$inout0); # inp^=E(ivec)
d7d119a3 717 &movdqa ($inout0,$ivec);
267b481c 718 &movups (&QWP(0,$out),$in0); # save output
9ee5916d 719 &pshufb ($inout0,$inout3);
d8ba0dc9 720 &lea ($out,&DWP(16,$out));
d7d119a3
AP		 721 &jnz (&label("ccm64_enc_outer"));
722
723 &mov ("esp",&DWP(48,"esp"));
724 &mov ($out,&wparam(5));
f8501464 725 &movups (&QWP(0,$out),$cmac);
23f6eec7
AP		 726
727 &pxor ("xmm0","xmm0"); # clear register bank
728 &pxor ("xmm1","xmm1");
729 &pxor ("xmm2","xmm2");
730 &pxor ("xmm3","xmm3");
731 &pxor ("xmm4","xmm4");
732 &pxor ("xmm5","xmm5");
733 &pxor ("xmm6","xmm6");
734 &pxor ("xmm7","xmm7");
d7d119a3
AP		 735&function_end("aesni_ccm64_encrypt_blocks");
736
737&function_begin("aesni_ccm64_decrypt_blocks");
738 &mov ($inp,&wparam(0));
739 &mov ($out,&wparam(1));
740 &mov ($len,&wparam(2));
741 &mov ($key,&wparam(3));
742 &mov ($rounds_,&wparam(4));
743 &mov ($rounds,&wparam(5));
744 &mov ($key_,"esp");
745 &sub ("esp",60);
746 &and ("esp",-16); # align stack
747 &mov (&DWP(48,"esp"),$key_);
748
749 &movdqu ($ivec,&QWP(0,$rounds_)); # load ivec
f8501464 750 &movdqu ($cmac,&QWP(0,$rounds)); # load cmac
267b481c 751 &mov ($rounds,&DWP(240,$key));
d7d119a3
AP		 752
753 # compose byte-swap control mask for pshufb on stack
754 &mov (&DWP(0,"esp"),0x0c0d0e0f);
755 &mov (&DWP(4,"esp"),0x08090a0b);
756 &mov (&DWP(8,"esp"),0x04050607);
757 &mov (&DWP(12,"esp"),0x00010203);
758
759 # compose counter increment vector on stack
267b481c 760 &mov ($rounds_,1);
d7d119a3 761 &xor ($key_,$key_);
267b481c 762 &mov (&DWP(16,"esp"),$rounds_);
d7d119a3
AP		 763 &mov (&DWP(20,"esp"),$key_);
764 &mov (&DWP(24,"esp"),$key_);
765 &mov (&DWP(28,"esp"),$key_);
766
767 &movdqa ($inout3,&QWP(0,"esp")); # bswap mask
768 &movdqa ($inout0,$ivec);
d7d119a3 769
d7d119a3
AP		 770 &mov ($key_,$key);
771 &mov ($rounds_,$rounds);
772
267b481c 773 &pshufb ($ivec,$inout3);
d7d119a3
AP		 774 if ($inline)
775 { &aesni_inline_generate1("enc"); }
776 else
777 { &call ("_aesni_encrypt1"); }
d8ba0dc9
AP		 778 &shl ($rounds_,4);
779 &mov ($rounds,16);
f8501464 780 &movups ($in0,&QWP(0,$inp)); # load inp
267b481c 781 &paddq ($ivec,&QWP(16,"esp"));
f8501464 782 &lea ($inp,&QWP(16,$inp));
d8ba0dc9
AP		 783 &sub ($rounds,$rounds_);
784 &lea ($key,&DWP(32,$key_,$rounds_));
785 &mov ($rounds_,$rounds);
267b481c
AP		 786 &jmp (&label("ccm64_dec_outer"));
787
788&set_label("ccm64_dec_outer",16);
789 &xorps ($in0,$inout0); # inp ^= E(ivec)
790 &movdqa ($inout0,$ivec);
267b481c 791 &movups (&QWP(0,$out),$in0); # save output
d7d119a3 792 &lea ($out,&DWP(16,$out));
9ee5916d 793 &pshufb ($inout0,$inout3);
d7d119a3 794
f8501464 795 &sub ($len,1);
d7d119a3
AP		 796 &jz (&label("ccm64_dec_break"));
797
267b481c 798 &$movekey ($rndkey0,&QWP(0,$key_));
d8ba0dc9 799 &mov ($rounds,$rounds_);
267b481c 800 &$movekey ($rndkey1,&QWP(16,$key_));
f8501464 801 &xorps ($in0,$rndkey0);
f8501464
AP		 802 &xorps ($inout0,$rndkey0);
803 &xorps ($cmac,$in0); # cmac^=out
d8ba0dc9 804 &$movekey ($rndkey0,&QWP(32,$key_));
d7d119a3 805
f8501464
AP		 806&set_label("ccm64_dec2_loop");
807 &aesenc ($inout0,$rndkey1);
f8501464 808 &aesenc ($cmac,$rndkey1);
d8ba0dc9
AP		 809 &$movekey ($rndkey1,&QWP(0,$key,$rounds));
810 &add ($rounds,32);
f8501464 811 &aesenc ($inout0,$rndkey0);
f8501464 812 &aesenc ($cmac,$rndkey0);
d8ba0dc9 813 &$movekey ($rndkey0,&QWP(-16,$key,$rounds));
f8501464 814 &jnz (&label("ccm64_dec2_loop"));
267b481c
AP		 815 &movups ($in0,&QWP(0,$inp)); # load inp
816 &paddq ($ivec,&QWP(16,"esp"));
f8501464
AP		 817 &aesenc ($inout0,$rndkey1);
818 &aesenc ($cmac,$rndkey1);
819 &aesenclast ($inout0,$rndkey0);
820 &aesenclast ($cmac,$rndkey0);
d8ba0dc9 821 &lea ($inp,&QWP(16,$inp));
d7d119a3
AP		 822 &jmp (&label("ccm64_dec_outer"));
823
824&set_label("ccm64_dec_break",16);
d8ba0dc9 825 &mov ($rounds,&DWP(240,$key_));
267b481c 826 &mov ($key,$key_);
d7d119a3 827 if ($inline)
f8501464 828 { &aesni_inline_generate1("enc",$cmac,$in0); }
d7d119a3 829 else
f8501464 830 { &call ("_aesni_encrypt1",$cmac); }
d7d119a3
AP		 831
832 &mov ("esp",&DWP(48,"esp"));
833 &mov ($out,&wparam(5));
f8501464 834 &movups (&QWP(0,$out),$cmac);
23f6eec7
AP		 835
836 &pxor ("xmm0","xmm0"); # clear register bank
837 &pxor ("xmm1","xmm1");
838 &pxor ("xmm2","xmm2");
839 &pxor ("xmm3","xmm3");
840 &pxor ("xmm4","xmm4");
841 &pxor ("xmm5","xmm5");
842 &pxor ("xmm6","xmm6");
843 &pxor ("xmm7","xmm7");
d7d119a3 844&function_end("aesni_ccm64_decrypt_blocks");
f8501464 845}
d7d119a3
AP		 846\f
847######################################################################
6c83629b
AP		 848# void aesni_ctr32_encrypt_blocks (const void *in, void *out,
849# size_t blocks, const AES_KEY *key,
850# const char *ivec);
d7d119a3
AP		 851#
852# Handles only complete blocks, operates on 32-bit counter and
d8ba0dc9 853# does not update *ivec! (see crypto/modes/ctr128.c for details)
d7d119a3 854#
f8501464
AP		 855# stack layout:
856# 0 pshufb mask
857# 16 vector addend: 0,6,6,6
858# 32 counter-less ivec
859# 48 1st triplet of counter vector
860# 64 2nd triplet of counter vector
861# 80 saved %esp
862
6c83629b
AP		 863&function_begin("aesni_ctr32_encrypt_blocks");
864 &mov ($inp,&wparam(0));
865 &mov ($out,&wparam(1));
866 &mov ($len,&wparam(2));
867 &mov ($key,&wparam(3));
868 &mov ($rounds_,&wparam(4));
869 &mov ($key_,"esp");
f8501464 870 &sub ("esp",88);
6c83629b 871 &and ("esp",-16); # align stack
f8501464 872 &mov (&DWP(80,"esp"),$key_);
6c83629b 873
d7d119a3
AP		 874 &cmp ($len,1);
875 &je (&label("ctr32_one_shortcut"));
876
f8501464 877 &movdqu ($inout5,&QWP(0,$rounds_)); # load ivec
6c83629b
AP		 878
879 # compose byte-swap control mask for pshufb on stack
880 &mov (&DWP(0,"esp"),0x0c0d0e0f);
881 &mov (&DWP(4,"esp"),0x08090a0b);
882 &mov (&DWP(8,"esp"),0x04050607);
883 &mov (&DWP(12,"esp"),0x00010203);
884
885 # compose counter increment vector on stack
f8501464 886 &mov ($rounds,6);
6c83629b
AP		 887 &xor ($key_,$key_);
888 &mov (&DWP(16,"esp"),$rounds);
889 &mov (&DWP(20,"esp"),$rounds);
890 &mov (&DWP(24,"esp"),$rounds);
891 &mov (&DWP(28,"esp"),$key_);
892
f8501464
AP		 893 &pextrd ($rounds_,$inout5,3); # pull 32-bit counter
894 &pinsrd ($inout5,$key_,3); # wipe 32-bit counter
6c83629b
AP		 895
896 &mov ($rounds,&DWP(240,$key)); # key->rounds
6c83629b 897
f8501464 898 # compose 2 vectors of 3x32-bit counters
6c83629b 899 &bswap ($rounds_);
f8501464 900 &pxor ($rndkey0,$rndkey0);
d8ba0dc9 901 &pxor ($rndkey1,$rndkey1);
f8501464 902 &movdqa ($inout0,&QWP(0,"esp")); # load byte-swap mask
d8ba0dc9 903 &pinsrd ($rndkey0,$rounds_,0);
f8501464 904 &lea ($key_,&DWP(3,$rounds_));
d8ba0dc9 905 &pinsrd ($rndkey1,$key_,0);
6c83629b 906 &inc ($rounds_);
d8ba0dc9 907 &pinsrd ($rndkey0,$rounds_,1);
f8501464 908 &inc ($key_);
d8ba0dc9 909 &pinsrd ($rndkey1,$key_,1);
6c83629b 910 &inc ($rounds_);
d8ba0dc9 911 &pinsrd ($rndkey0,$rounds_,2);
f8501464 912 &inc ($key_);
d8ba0dc9
AP		 913 &pinsrd ($rndkey1,$key_,2);
914 &movdqa (&QWP(48,"esp"),$rndkey0); # save 1st triplet
f8501464 915 &pshufb ($rndkey0,$inout0); # byte swap
d8ba0dc9
AP		 916 &movdqu ($inout4,&QWP(0,$key)); # key[0]
917 &movdqa (&QWP(64,"esp"),$rndkey1); # save 2nd triplet
918 &pshufb ($rndkey1,$inout0); # byte swap
f8501464 919
d8ba0dc9
AP		 920 &pshufd ($inout0,$rndkey0,3<<6); # place counter to upper dword
921 &pshufd ($inout1,$rndkey0,2<<6);
f8501464
AP		 922 &cmp ($len,6);
923 &jb (&label("ctr32_tail"));
d8ba0dc9
AP		 924 &pxor ($inout5,$inout4); # counter-less ivec^key[0]
925 &shl ($rounds,4);
926 &mov ($rounds_,16);
927 &movdqa (&QWP(32,"esp"),$inout5); # save counter-less ivec^key[0]
f8501464 928 &mov ($key_,$key); # backup $key
d8ba0dc9
AP		 929 &sub ($rounds_,$rounds); # backup twisted $rounds
930 &lea ($key,&DWP(32,$key,$rounds));
f8501464
AP		 931 &sub ($len,6);
932 &jmp (&label("ctr32_loop6"));
933
934&set_label("ctr32_loop6",16);
d8ba0dc9
AP		 935 # inlining _aesni_encrypt6's prologue gives ~6% improvement...
936 &pshufd ($inout2,$rndkey0,1<<6);
937 &movdqa ($rndkey0,&QWP(32,"esp")); # pull counter-less ivec
938 &pshufd ($inout3,$rndkey1,3<<6);
939 &pxor ($inout0,$rndkey0); # merge counter-less ivec
940 &pshufd ($inout4,$rndkey1,2<<6);
d7d119a3 941 &pxor ($inout1,$rndkey0);
d8ba0dc9
AP		 942 &pshufd ($inout5,$rndkey1,1<<6);
943 &$movekey ($rndkey1,&QWP(16,$key_));
f8501464 944 &pxor ($inout2,$rndkey0);
f8501464 945 &pxor ($inout3,$rndkey0);
d8ba0dc9 946 &aesenc ($inout0,$rndkey1);
f8501464 947 &pxor ($inout4,$rndkey0);
f8501464 948 &pxor ($inout5,$rndkey0);
d8ba0dc9
AP		 949 &aesenc ($inout1,$rndkey1);
950 &$movekey ($rndkey0,&QWP(32,$key_));
951 &mov ($rounds,$rounds_);
952 &aesenc ($inout2,$rndkey1);
953 &aesenc ($inout3,$rndkey1);
f8501464 954 &aesenc ($inout4,$rndkey1);
f8501464 955 &aesenc ($inout5,$rndkey1);
d7d119a3 956
f8501464
AP		 957 &call (&label("_aesni_encrypt6_enter"));
958
959 &movups ($rndkey1,&QWP(0,$inp));
960 &movups ($rndkey0,&QWP(0x10,$inp));
961 &xorps ($inout0,$rndkey1);
962 &movups ($rndkey1,&QWP(0x20,$inp));
963 &xorps ($inout1,$rndkey0);
964 &movups (&QWP(0,$out),$inout0);
965 &movdqa ($rndkey0,&QWP(16,"esp")); # load increment
966 &xorps ($inout2,$rndkey1);
d8ba0dc9 967 &movdqa ($rndkey1,&QWP(64,"esp")); # load 2nd triplet
f8501464
AP		 968 &movups (&QWP(0x10,$out),$inout1);
969 &movups (&QWP(0x20,$out),$inout2);
970
d8ba0dc9
AP		 971 &paddd ($rndkey1,$rndkey0); # 2nd triplet increment
972 &paddd ($rndkey0,&QWP(48,"esp")); # 1st triplet increment
f8501464
AP		 973 &movdqa ($inout0,&QWP(0,"esp")); # load byte swap mask
974
975 &movups ($inout1,&QWP(0x30,$inp));
976 &movups ($inout2,&QWP(0x40,$inp));
977 &xorps ($inout3,$inout1);
978 &movups ($inout1,&QWP(0x50,$inp));
979 &lea ($inp,&DWP(0x60,$inp));
d8ba0dc9
AP		 980 &movdqa (&QWP(48,"esp"),$rndkey0); # save 1st triplet
981 &pshufb ($rndkey0,$inout0); # byte swap
f8501464
AP		 982 &xorps ($inout4,$inout2);
983 &movups (&QWP(0x30,$out),$inout3);
984 &xorps ($inout5,$inout1);
d8ba0dc9
AP		 985 &movdqa (&QWP(64,"esp"),$rndkey1); # save 2nd triplet
986 &pshufb ($rndkey1,$inout0); # byte swap
f8501464 987 &movups (&QWP(0x40,$out),$inout4);
d8ba0dc9 988 &pshufd ($inout0,$rndkey0,3<<6);
f8501464
AP		 989 &movups (&QWP(0x50,$out),$inout5);
990 &lea ($out,&DWP(0x60,$out));
d7d119a3 991
d8ba0dc9 992 &pshufd ($inout1,$rndkey0,2<<6);
f8501464
AP		 993 &sub ($len,6);
994 &jnc (&label("ctr32_loop6"));
6c83629b 995
f8501464
AP		 996 &add ($len,6);
997 &jz (&label("ctr32_ret"));
d8ba0dc9 998 &movdqu ($inout5,&QWP(0,$key_));
f8501464 999 &mov ($key,$key_);
d8ba0dc9
AP		 1000 &pxor ($inout5,&QWP(32,"esp")); # restore count-less ivec
1001 &mov ($rounds,&DWP(240,$key_)); # restore $rounds
6c83629b
AP		 1002
1003&set_label("ctr32_tail");
f8501464 1004 &por ($inout0,$inout5);
d7d119a3 1005 &cmp ($len,2);
6c83629b 1006 &jb (&label("ctr32_one"));
6c83629b 1007
d8ba0dc9 1008 &pshufd ($inout2,$rndkey0,1<<6);
f8501464
AP		 1009 &por ($inout1,$inout5);
1010 &je (&label("ctr32_two"));
6c83629b 1011
d8ba0dc9 1012 &pshufd ($inout3,$rndkey1,3<<6);
f8501464
AP		 1013 &por ($inout2,$inout5);
1014 &cmp ($len,4);
1015 &jb (&label("ctr32_three"));
1016
d8ba0dc9 1017 &pshufd ($inout4,$rndkey1,2<<6);
f8501464
AP		 1018 &por ($inout3,$inout5);
1019 &je (&label("ctr32_four"));
1020
1021 &por ($inout4,$inout5);
1022 &call ("_aesni_encrypt6");
1023 &movups ($rndkey1,&QWP(0,$inp));
1024 &movups ($rndkey0,&QWP(0x10,$inp));
1025 &xorps ($inout0,$rndkey1);
1026 &movups ($rndkey1,&QWP(0x20,$inp));
1027 &xorps ($inout1,$rndkey0);
1028 &movups ($rndkey0,&QWP(0x30,$inp));
1029 &xorps ($inout2,$rndkey1);
1030 &movups ($rndkey1,&QWP(0x40,$inp));
1031 &xorps ($inout3,$rndkey0);
1032 &movups (&QWP(0,$out),$inout0);
1033 &xorps ($inout4,$rndkey1);
1034 &movups (&QWP(0x10,$out),$inout1);
1035 &movups (&QWP(0x20,$out),$inout2);
1036 &movups (&QWP(0x30,$out),$inout3);
1037 &movups (&QWP(0x40,$out),$inout4);
6c83629b
AP		 1038 &jmp (&label("ctr32_ret"));
1039
d7d119a3 1040&set_label("ctr32_one_shortcut",16);
f8501464 1041 &movups ($inout0,&QWP(0,$rounds_)); # load ivec
d7d119a3
AP		 1042 &mov ($rounds,&DWP(240,$key));
1043
1044&set_label("ctr32_one");
6c83629b
AP		 1045 if ($inline)
1046 { &aesni_inline_generate1("enc"); }
1047 else
1048 { &call ("_aesni_encrypt1"); }
f8501464
AP		 1049 &movups ($in0,&QWP(0,$inp));
1050 &xorps ($in0,$inout0);
1051 &movups (&QWP(0,$out),$in0);
6c83629b 1052 &jmp (&label("ctr32_ret"));
d64a7232 1053
6c83629b 1054&set_label("ctr32_two",16);
214368ff 1055 &call ("_aesni_encrypt2");
f8501464
AP		 1056 &movups ($inout3,&QWP(0,$inp));
1057 &movups ($inout4,&QWP(0x10,$inp));
1058 &xorps ($inout0,$inout3);
1059 &xorps ($inout1,$inout4);
1060 &movups (&QWP(0,$out),$inout0);
1061 &movups (&QWP(0x10,$out),$inout1);
6c83629b
AP		 1062 &jmp (&label("ctr32_ret"));
1063
1064&set_label("ctr32_three",16);
1065 &call ("_aesni_encrypt3");
f8501464
AP		 1066 &movups ($inout3,&QWP(0,$inp));
1067 &movups ($inout4,&QWP(0x10,$inp));
1068 &xorps ($inout0,$inout3);
1069 &movups ($inout5,&QWP(0x20,$inp));
1070 &xorps ($inout1,$inout4);
1071 &movups (&QWP(0,$out),$inout0);
1072 &xorps ($inout2,$inout5);
1073 &movups (&QWP(0x10,$out),$inout1);
1074 &movups (&QWP(0x20,$out),$inout2);
1075 &jmp (&label("ctr32_ret"));
1076
1077&set_label("ctr32_four",16);
1078 &call ("_aesni_encrypt4");
1079 &movups ($inout4,&QWP(0,$inp));
1080 &movups ($inout5,&QWP(0x10,$inp));
1081 &movups ($rndkey1,&QWP(0x20,$inp));
1082 &xorps ($inout0,$inout4);
1083 &movups ($rndkey0,&QWP(0x30,$inp));
1084 &xorps ($inout1,$inout5);
1085 &movups (&QWP(0,$out),$inout0);
1086 &xorps ($inout2,$rndkey1);
1087 &movups (&QWP(0x10,$out),$inout1);
1088 &xorps ($inout3,$rndkey0);
1089 &movups (&QWP(0x20,$out),$inout2);
1090 &movups (&QWP(0x30,$out),$inout3);
6c83629b
AP		 1091
1092&set_label("ctr32_ret");
23f6eec7
AP		 1093 &pxor ("xmm0","xmm0"); # clear register bank
1094 &pxor ("xmm1","xmm1");
1095 &pxor ("xmm2","xmm2");
1096 &pxor ("xmm3","xmm3");
1097 &pxor ("xmm4","xmm4");
1098 &movdqa (&QWP(32,"esp"),"xmm0"); # clear stack
1099 &pxor ("xmm5","xmm5");
1100 &movdqa (&QWP(48,"esp"),"xmm0");
1101 &pxor ("xmm6","xmm6");
1102 &movdqa (&QWP(64,"esp"),"xmm0");
1103 &pxor ("xmm7","xmm7");
f8501464 1104 &mov ("esp",&DWP(80,"esp"));
6c83629b 1105&function_end("aesni_ctr32_encrypt_blocks");
f8501464
AP		 1106\f
1107######################################################################
1108# void aesni_xts_[en|de]crypt(const char *inp,char *out,size_t len,
1109# const AES_KEY *key1, const AES_KEY *key2
1110# const unsigned char iv[16]);
1111#
1112{ my ($tweak,$twtmp,$twres,$twmask)=($rndkey1,$rndkey0,$inout0,$inout1);
1113
1114&function_begin("aesni_xts_encrypt");
1115 &mov ($key,&wparam(4)); # key2
1116 &mov ($inp,&wparam(5)); # clear-text tweak
1117
1118 &mov ($rounds,&DWP(240,$key)); # key2->rounds
1119 &movups ($inout0,&QWP(0,$inp));
1120 if ($inline)
1121 { &aesni_inline_generate1("enc"); }
1122 else
1123 { &call ("_aesni_encrypt1"); }
1124
1125 &mov ($inp,&wparam(0));
1126 &mov ($out,&wparam(1));
1127 &mov ($len,&wparam(2));
1128 &mov ($key,&wparam(3)); # key1
1129
1130 &mov ($key_,"esp");
1131 &sub ("esp",16*7+8);
1132 &mov ($rounds,&DWP(240,$key)); # key1->rounds
1133 &and ("esp",-16); # align stack
1134
1135 &mov (&DWP(16*6+0,"esp"),0x87); # compose the magic constant
1136 &mov (&DWP(16*6+4,"esp"),0);
1137 &mov (&DWP(16*6+8,"esp"),1);
1138 &mov (&DWP(16*6+12,"esp"),0);
1139 &mov (&DWP(16*7+0,"esp"),$len); # save original $len
1140 &mov (&DWP(16*7+4,"esp"),$key_); # save original %esp
1141
1142 &movdqa ($tweak,$inout0);
1143 &pxor ($twtmp,$twtmp);
1144 &movdqa ($twmask,&QWP(6*16,"esp")); # 0x0...010...87
1145 &pcmpgtd($twtmp,$tweak); # broadcast upper bits
1146
1147 &and ($len,-16);
1148 &mov ($key_,$key); # backup $key
1149 &mov ($rounds_,$rounds); # backup $rounds
1150 &sub ($len,16*6);
1151 &jc (&label("xts_enc_short"));
1152
d8ba0dc9
AP		 1153 &shl ($rounds,4);
1154 &mov ($rounds_,16);
1155 &sub ($rounds_,$rounds);
1156 &lea ($key,&DWP(32,$key,$rounds));
f8501464
AP		 1157 &jmp (&label("xts_enc_loop6"));
1158
1159&set_label("xts_enc_loop6",16);
1160 for ($i=0;$i<4;$i++) {
1161 &pshufd ($twres,$twtmp,0x13);
1162 &pxor ($twtmp,$twtmp);
1163 &movdqa (&QWP(16*$i,"esp"),$tweak);
1164 &paddq ($tweak,$tweak); # &psllq($tweak,1);
1165 &pand ($twres,$twmask); # isolate carry and residue
1166 &pcmpgtd ($twtmp,$tweak); # broadcast upper bits
1167 &pxor ($tweak,$twres);
1168 }
1169 &pshufd ($inout5,$twtmp,0x13);
1170 &movdqa (&QWP(16*$i++,"esp"),$tweak);
1171 &paddq ($tweak,$tweak); # &psllq($tweak,1);
1172 &$movekey ($rndkey0,&QWP(0,$key_));
1173 &pand ($inout5,$twmask); # isolate carry and residue
1174 &movups ($inout0,&QWP(0,$inp)); # load input
1175 &pxor ($inout5,$tweak);
1176
1177 # inline _aesni_encrypt6 prologue and flip xor with tweak and key[0]
d8ba0dc9 1178 &mov ($rounds,$rounds_); # restore $rounds
f8501464
AP		 1179 &movdqu ($inout1,&QWP(16*1,$inp));
1180 &xorps ($inout0,$rndkey0); # input^=rndkey[0]
1181 &movdqu ($inout2,&QWP(16*2,$inp));
1182 &pxor ($inout1,$rndkey0);
1183 &movdqu ($inout3,&QWP(16*3,$inp));
1184 &pxor ($inout2,$rndkey0);
1185 &movdqu ($inout4,&QWP(16*4,$inp));
1186 &pxor ($inout3,$rndkey0);
1187 &movdqu ($rndkey1,&QWP(16*5,$inp));
1188 &pxor ($inout4,$rndkey0);
1189 &lea ($inp,&DWP(16*6,$inp));
1190 &pxor ($inout0,&QWP(16*0,"esp")); # input^=tweak
1191 &movdqa (&QWP(16*$i,"esp"),$inout5); # save last tweak
1192 &pxor ($inout5,$rndkey1);
1193
1194 &$movekey ($rndkey1,&QWP(16,$key_));
f8501464 1195 &pxor ($inout1,&QWP(16*1,"esp"));
f8501464 1196 &pxor ($inout2,&QWP(16*2,"esp"));
d8ba0dc9 1197 &aesenc ($inout0,$rndkey1);
f8501464 1198 &pxor ($inout3,&QWP(16*3,"esp"));
f8501464 1199 &pxor ($inout4,&QWP(16*4,"esp"));
d8ba0dc9 1200 &aesenc ($inout1,$rndkey1);
f8501464 1201 &pxor ($inout5,$rndkey0);
d8ba0dc9
AP		 1202 &$movekey ($rndkey0,&QWP(32,$key_));
1203 &aesenc ($inout2,$rndkey1);
1204 &aesenc ($inout3,$rndkey1);
f8501464 1205 &aesenc ($inout4,$rndkey1);
f8501464
AP		 1206 &aesenc ($inout5,$rndkey1);
1207 &call (&label("_aesni_encrypt6_enter"));
1208
1209 &movdqa ($tweak,&QWP(16*5,"esp")); # last tweak
1210 &pxor ($twtmp,$twtmp);
1211 &xorps ($inout0,&QWP(16*0,"esp")); # output^=tweak
1212 &pcmpgtd ($twtmp,$tweak); # broadcast upper bits
1213 &xorps ($inout1,&QWP(16*1,"esp"));
1214 &movups (&QWP(16*0,$out),$inout0); # write output
1215 &xorps ($inout2,&QWP(16*2,"esp"));
1216 &movups (&QWP(16*1,$out),$inout1);
1217 &xorps ($inout3,&QWP(16*3,"esp"));
1218 &movups (&QWP(16*2,$out),$inout2);
1219 &xorps ($inout4,&QWP(16*4,"esp"));
1220 &movups (&QWP(16*3,$out),$inout3);
1221 &xorps ($inout5,$tweak);
1222 &movups (&QWP(16*4,$out),$inout4);
1223 &pshufd ($twres,$twtmp,0x13);
1224 &movups (&QWP(16*5,$out),$inout5);
1225 &lea ($out,&DWP(16*6,$out));
1226 &movdqa ($twmask,&QWP(16*6,"esp")); # 0x0...010...87
1227
1228 &pxor ($twtmp,$twtmp);
1229 &paddq ($tweak,$tweak); # &psllq($tweak,1);
1230 &pand ($twres,$twmask); # isolate carry and residue
1231 &pcmpgtd($twtmp,$tweak); # broadcast upper bits
f8501464
AP		 1232 &pxor ($tweak,$twres);
1233
1234 &sub ($len,16*6);
1235 &jnc (&label("xts_enc_loop6"));
1236
d8ba0dc9 1237 &mov ($rounds,&DWP(240,$key_)); # restore $rounds
f8501464
AP		 1238 &mov ($key,$key_); # restore $key
1239 &mov ($rounds_,$rounds);
1240
1241&set_label("xts_enc_short");
1242 &add ($len,16*6);
1243 &jz (&label("xts_enc_done6x"));
1244
1245 &movdqa ($inout3,$tweak); # put aside previous tweak
1246 &cmp ($len,0x20);
1247 &jb (&label("xts_enc_one"));
1248
1249 &pshufd ($twres,$twtmp,0x13);
1250 &pxor ($twtmp,$twtmp);
1251 &paddq ($tweak,$tweak); # &psllq($tweak,1);
1252 &pand ($twres,$twmask); # isolate carry and residue
1253 &pcmpgtd($twtmp,$tweak); # broadcast upper bits
1254 &pxor ($tweak,$twres);
1255 &je (&label("xts_enc_two"));
1256
1257 &pshufd ($twres,$twtmp,0x13);
1258 &pxor ($twtmp,$twtmp);
1259 &movdqa ($inout4,$tweak); # put aside previous tweak
1260 &paddq ($tweak,$tweak); # &psllq($tweak,1);
1261 &pand ($twres,$twmask); # isolate carry and residue
1262 &pcmpgtd($twtmp,$tweak); # broadcast upper bits
1263 &pxor ($tweak,$twres);
1264 &cmp ($len,0x40);
1265 &jb (&label("xts_enc_three"));
1266
1267 &pshufd ($twres,$twtmp,0x13);
1268 &pxor ($twtmp,$twtmp);
1269 &movdqa ($inout5,$tweak); # put aside previous tweak
1270 &paddq ($tweak,$tweak); # &psllq($tweak,1);
1271 &pand ($twres,$twmask); # isolate carry and residue
1272 &pcmpgtd($twtmp,$tweak); # broadcast upper bits
1273 &pxor ($tweak,$twres);
1274 &movdqa (&QWP(16*0,"esp"),$inout3);
1275 &movdqa (&QWP(16*1,"esp"),$inout4);
1276 &je (&label("xts_enc_four"));
1277
1278 &movdqa (&QWP(16*2,"esp"),$inout5);
1279 &pshufd ($inout5,$twtmp,0x13);
1280 &movdqa (&QWP(16*3,"esp"),$tweak);
1281 &paddq ($tweak,$tweak); # &psllq($inout0,1);
1282 &pand ($inout5,$twmask); # isolate carry and residue
1283 &pxor ($inout5,$tweak);
1284
1285 &movdqu ($inout0,&QWP(16*0,$inp)); # load input
1286 &movdqu ($inout1,&QWP(16*1,$inp));
1287 &movdqu ($inout2,&QWP(16*2,$inp));
1288 &pxor ($inout0,&QWP(16*0,"esp")); # input^=tweak
1289 &movdqu ($inout3,&QWP(16*3,$inp));
1290 &pxor ($inout1,&QWP(16*1,"esp"));
1291 &movdqu ($inout4,&QWP(16*4,$inp));
1292 &pxor ($inout2,&QWP(16*2,"esp"));
1293 &lea ($inp,&DWP(16*5,$inp));
1294 &pxor ($inout3,&QWP(16*3,"esp"));
1295 &movdqa (&QWP(16*4,"esp"),$inout5); # save last tweak
1296 &pxor ($inout4,$inout5);
1297
1298 &call ("_aesni_encrypt6");
1299
1300 &movaps ($tweak,&QWP(16*4,"esp")); # last tweak
1301 &xorps ($inout0,&QWP(16*0,"esp")); # output^=tweak
1302 &xorps ($inout1,&QWP(16*1,"esp"));
1303 &xorps ($inout2,&QWP(16*2,"esp"));
1304 &movups (&QWP(16*0,$out),$inout0); # write output
1305 &xorps ($inout3,&QWP(16*3,"esp"));
1306 &movups (&QWP(16*1,$out),$inout1);
1307 &xorps ($inout4,$tweak);
1308 &movups (&QWP(16*2,$out),$inout2);
1309 &movups (&QWP(16*3,$out),$inout3);
1310 &movups (&QWP(16*4,$out),$inout4);
1311 &lea ($out,&DWP(16*5,$out));
1312 &jmp (&label("xts_enc_done"));
1313
1314&set_label("xts_enc_one",16);
1315 &movups ($inout0,&QWP(16*0,$inp)); # load input
1316 &lea ($inp,&DWP(16*1,$inp));
1317 &xorps ($inout0,$inout3); # input^=tweak
1318 if ($inline)
1319 { &aesni_inline_generate1("enc"); }
1320 else
1321 { &call ("_aesni_encrypt1"); }
1322 &xorps ($inout0,$inout3); # output^=tweak
1323 &movups (&QWP(16*0,$out),$inout0); # write output
1324 &lea ($out,&DWP(16*1,$out));
1325
1326 &movdqa ($tweak,$inout3); # last tweak
1327 &jmp (&label("xts_enc_done"));
1328
1329&set_label("xts_enc_two",16);
1330 &movaps ($inout4,$tweak); # put aside last tweak
1331
1332 &movups ($inout0,&QWP(16*0,$inp)); # load input
1333 &movups ($inout1,&QWP(16*1,$inp));
1334 &lea ($inp,&DWP(16*2,$inp));
1335 &xorps ($inout0,$inout3); # input^=tweak
1336 &xorps ($inout1,$inout4);
f8501464 1337
214368ff 1338 &call ("_aesni_encrypt2");
f8501464
AP		 1339
1340 &xorps ($inout0,$inout3); # output^=tweak
1341 &xorps ($inout1,$inout4);
1342 &movups (&QWP(16*0,$out),$inout0); # write output
1343 &movups (&QWP(16*1,$out),$inout1);
1344 &lea ($out,&DWP(16*2,$out));
1345
1346 &movdqa ($tweak,$inout4); # last tweak
1347 &jmp (&label("xts_enc_done"));
1348
1349&set_label("xts_enc_three",16);
1350 &movaps ($inout5,$tweak); # put aside last tweak
1351 &movups ($inout0,&QWP(16*0,$inp)); # load input
1352 &movups ($inout1,&QWP(16*1,$inp));
1353 &movups ($inout2,&QWP(16*2,$inp));
1354 &lea ($inp,&DWP(16*3,$inp));
1355 &xorps ($inout0,$inout3); # input^=tweak
1356 &xorps ($inout1,$inout4);
1357 &xorps ($inout2,$inout5);
1358
1359 &call ("_aesni_encrypt3");
1360
1361 &xorps ($inout0,$inout3); # output^=tweak
1362 &xorps ($inout1,$inout4);
1363 &xorps ($inout2,$inout5);
1364 &movups (&QWP(16*0,$out),$inout0); # write output
1365 &movups (&QWP(16*1,$out),$inout1);
1366 &movups (&QWP(16*2,$out),$inout2);
1367 &lea ($out,&DWP(16*3,$out));
1368
1369 &movdqa ($tweak,$inout5); # last tweak
1370 &jmp (&label("xts_enc_done"));
1371
1372&set_label("xts_enc_four",16);
1373 &movaps ($inout4,$tweak); # put aside last tweak
1374
1375 &movups ($inout0,&QWP(16*0,$inp)); # load input
1376 &movups ($inout1,&QWP(16*1,$inp));
1377 &movups ($inout2,&QWP(16*2,$inp));
1378 &xorps ($inout0,&QWP(16*0,"esp")); # input^=tweak
1379 &movups ($inout3,&QWP(16*3,$inp));
1380 &lea ($inp,&DWP(16*4,$inp));
1381 &xorps ($inout1,&QWP(16*1,"esp"));
1382 &xorps ($inout2,$inout5);
1383 &xorps ($inout3,$inout4);
1384
1385 &call ("_aesni_encrypt4");
1386
1387 &xorps ($inout0,&QWP(16*0,"esp")); # output^=tweak
1388 &xorps ($inout1,&QWP(16*1,"esp"));
1389 &xorps ($inout2,$inout5);
1390 &movups (&QWP(16*0,$out),$inout0); # write output
1391 &xorps ($inout3,$inout4);
1392 &movups (&QWP(16*1,$out),$inout1);
1393 &movups (&QWP(16*2,$out),$inout2);
1394 &movups (&QWP(16*3,$out),$inout3);
1395 &lea ($out,&DWP(16*4,$out));
1396
1397 &movdqa ($tweak,$inout4); # last tweak
1398 &jmp (&label("xts_enc_done"));
1399
1400&set_label("xts_enc_done6x",16); # $tweak is pre-calculated
1401 &mov ($len,&DWP(16*7+0,"esp")); # restore original $len
1402 &and ($len,15);
1403 &jz (&label("xts_enc_ret"));
1404 &movdqa ($inout3,$tweak);
1405 &mov (&DWP(16*7+0,"esp"),$len); # save $len%16
1406 &jmp (&label("xts_enc_steal"));
1407
1408&set_label("xts_enc_done",16);
1409 &mov ($len,&DWP(16*7+0,"esp")); # restore original $len
1410 &pxor ($twtmp,$twtmp);
1411 &and ($len,15);
1412 &jz (&label("xts_enc_ret"));
1413
1414 &pcmpgtd($twtmp,$tweak); # broadcast upper bits
1415 &mov (&DWP(16*7+0,"esp"),$len); # save $len%16
1416 &pshufd ($inout3,$twtmp,0x13);
1417 &paddq ($tweak,$tweak); # &psllq($tweak,1);
1418 &pand ($inout3,&QWP(16*6,"esp")); # isolate carry and residue
1419 &pxor ($inout3,$tweak);
1420
1421&set_label("xts_enc_steal");
1422 &movz ($rounds,&BP(0,$inp));
1423 &movz ($key,&BP(-16,$out));
1424 &lea ($inp,&DWP(1,$inp));
1425 &mov (&BP(-16,$out),&LB($rounds));
1426 &mov (&BP(0,$out),&LB($key));
1427 &lea ($out,&DWP(1,$out));
1428 &sub ($len,1);
1429 &jnz (&label("xts_enc_steal"));
1430
1431 &sub ($out,&DWP(16*7+0,"esp")); # rewind $out
1432 &mov ($key,$key_); # restore $key
1433 &mov ($rounds,$rounds_); # restore $rounds
1434
1435 &movups ($inout0,&QWP(-16,$out)); # load input
1436 &xorps ($inout0,$inout3); # input^=tweak
1437 if ($inline)
1438 { &aesni_inline_generate1("enc"); }
1439 else
1440 { &call ("_aesni_encrypt1"); }
1441 &xorps ($inout0,$inout3); # output^=tweak
1442 &movups (&QWP(-16,$out),$inout0); # write output
1443
1444&set_label("xts_enc_ret");
23f6eec7
AP		 1445 &pxor ("xmm0","xmm0"); # clear register bank
1446 &pxor ("xmm1","xmm1");
1447 &pxor ("xmm2","xmm2");
1448 &movdqa (&QWP(16*0,"esp"),"xmm0"); # clear stack
1449 &pxor ("xmm3","xmm3");
1450 &movdqa (&QWP(16*1,"esp"),"xmm0");
1451 &pxor ("xmm4","xmm4");
1452 &movdqa (&QWP(16*2,"esp"),"xmm0");
1453 &pxor ("xmm5","xmm5");
1454 &movdqa (&QWP(16*3,"esp"),"xmm0");
1455 &pxor ("xmm6","xmm6");
1456 &movdqa (&QWP(16*4,"esp"),"xmm0");
1457 &pxor ("xmm7","xmm7");
1458 &movdqa (&QWP(16*5,"esp"),"xmm0");
f8501464
AP		 1459 &mov ("esp",&DWP(16*7+4,"esp")); # restore %esp
1460&function_end("aesni_xts_encrypt");
1461
1462&function_begin("aesni_xts_decrypt");
1463 &mov ($key,&wparam(4)); # key2
1464 &mov ($inp,&wparam(5)); # clear-text tweak
1465
1466 &mov ($rounds,&DWP(240,$key)); # key2->rounds
1467 &movups ($inout0,&QWP(0,$inp));
1468 if ($inline)
1469 { &aesni_inline_generate1("enc"); }
1470 else
1471 { &call ("_aesni_encrypt1"); }
1472
1473 &mov ($inp,&wparam(0));
1474 &mov ($out,&wparam(1));
1475 &mov ($len,&wparam(2));
1476 &mov ($key,&wparam(3)); # key1
1477
1478 &mov ($key_,"esp");
1479 &sub ("esp",16*7+8);
1480 &and ("esp",-16); # align stack
1481
1482 &xor ($rounds_,$rounds_); # if(len%16) len-=16;
1483 &test ($len,15);
1484 &setnz (&LB($rounds_));
1485 &shl ($rounds_,4);
1486 &sub ($len,$rounds_);
1487
1488 &mov (&DWP(16*6+0,"esp"),0x87); # compose the magic constant
1489 &mov (&DWP(16*6+4,"esp"),0);
1490 &mov (&DWP(16*6+8,"esp"),1);
1491 &mov (&DWP(16*6+12,"esp"),0);
1492 &mov (&DWP(16*7+0,"esp"),$len); # save original $len
1493 &mov (&DWP(16*7+4,"esp"),$key_); # save original %esp
1494
1495 &mov ($rounds,&DWP(240,$key)); # key1->rounds
1496 &mov ($key_,$key); # backup $key
1497 &mov ($rounds_,$rounds); # backup $rounds
1498
1499 &movdqa ($tweak,$inout0);
1500 &pxor ($twtmp,$twtmp);
1501 &movdqa ($twmask,&QWP(6*16,"esp")); # 0x0...010...87
1502 &pcmpgtd($twtmp,$tweak); # broadcast upper bits
1503
1504 &and ($len,-16);
1505 &sub ($len,16*6);
1506 &jc (&label("xts_dec_short"));
1507
d8ba0dc9
AP		 1508 &shl ($rounds,4);
1509 &mov ($rounds_,16);
1510 &sub ($rounds_,$rounds);
1511 &lea ($key,&DWP(32,$key,$rounds));
f8501464
AP		 1512 &jmp (&label("xts_dec_loop6"));
1513
1514&set_label("xts_dec_loop6",16);
1515 for ($i=0;$i<4;$i++) {
1516 &pshufd ($twres,$twtmp,0x13);
1517 &pxor ($twtmp,$twtmp);
1518 &movdqa (&QWP(16*$i,"esp"),$tweak);
1519 &paddq ($tweak,$tweak); # &psllq($tweak,1);
1520 &pand ($twres,$twmask); # isolate carry and residue
1521 &pcmpgtd ($twtmp,$tweak); # broadcast upper bits
1522 &pxor ($tweak,$twres);
1523 }
1524 &pshufd ($inout5,$twtmp,0x13);
1525 &movdqa (&QWP(16*$i++,"esp"),$tweak);
1526 &paddq ($tweak,$tweak); # &psllq($tweak,1);
1527 &$movekey ($rndkey0,&QWP(0,$key_));
1528 &pand ($inout5,$twmask); # isolate carry and residue
1529 &movups ($inout0,&QWP(0,$inp)); # load input
1530 &pxor ($inout5,$tweak);
1531
1532 # inline _aesni_encrypt6 prologue and flip xor with tweak and key[0]
d8ba0dc9 1533 &mov ($rounds,$rounds_);
f8501464
AP		 1534 &movdqu ($inout1,&QWP(16*1,$inp));
1535 &xorps ($inout0,$rndkey0); # input^=rndkey[0]
1536 &movdqu ($inout2,&QWP(16*2,$inp));
1537 &pxor ($inout1,$rndkey0);
1538 &movdqu ($inout3,&QWP(16*3,$inp));
1539 &pxor ($inout2,$rndkey0);
1540 &movdqu ($inout4,&QWP(16*4,$inp));
1541 &pxor ($inout3,$rndkey0);
1542 &movdqu ($rndkey1,&QWP(16*5,$inp));
1543 &pxor ($inout4,$rndkey0);
1544 &lea ($inp,&DWP(16*6,$inp));
1545 &pxor ($inout0,&QWP(16*0,"esp")); # input^=tweak
1546 &movdqa (&QWP(16*$i,"esp"),$inout5); # save last tweak
1547 &pxor ($inout5,$rndkey1);
1548
1549 &$movekey ($rndkey1,&QWP(16,$key_));
f8501464 1550 &pxor ($inout1,&QWP(16*1,"esp"));
f8501464 1551 &pxor ($inout2,&QWP(16*2,"esp"));
d8ba0dc9 1552 &aesdec ($inout0,$rndkey1);
f8501464 1553 &pxor ($inout3,&QWP(16*3,"esp"));
f8501464 1554 &pxor ($inout4,&QWP(16*4,"esp"));
d8ba0dc9 1555 &aesdec ($inout1,$rndkey1);
f8501464 1556 &pxor ($inout5,$rndkey0);
d8ba0dc9
AP		 1557 &$movekey ($rndkey0,&QWP(32,$key_));
1558 &aesdec ($inout2,$rndkey1);
1559 &aesdec ($inout3,$rndkey1);
f8501464 1560 &aesdec ($inout4,$rndkey1);
f8501464
AP		 1561 &aesdec ($inout5,$rndkey1);
1562 &call (&label("_aesni_decrypt6_enter"));
1563
1564 &movdqa ($tweak,&QWP(16*5,"esp")); # last tweak
1565 &pxor ($twtmp,$twtmp);
1566 &xorps ($inout0,&QWP(16*0,"esp")); # output^=tweak
1567 &pcmpgtd ($twtmp,$tweak); # broadcast upper bits
1568 &xorps ($inout1,&QWP(16*1,"esp"));
1569 &movups (&QWP(16*0,$out),$inout0); # write output
1570 &xorps ($inout2,&QWP(16*2,"esp"));
1571 &movups (&QWP(16*1,$out),$inout1);
1572 &xorps ($inout3,&QWP(16*3,"esp"));
1573 &movups (&QWP(16*2,$out),$inout2);
1574 &xorps ($inout4,&QWP(16*4,"esp"));
1575 &movups (&QWP(16*3,$out),$inout3);
1576 &xorps ($inout5,$tweak);
1577 &movups (&QWP(16*4,$out),$inout4);
1578 &pshufd ($twres,$twtmp,0x13);
1579 &movups (&QWP(16*5,$out),$inout5);
1580 &lea ($out,&DWP(16*6,$out));
1581 &movdqa ($twmask,&QWP(16*6,"esp")); # 0x0...010...87
1582
1583 &pxor ($twtmp,$twtmp);
1584 &paddq ($tweak,$tweak); # &psllq($tweak,1);
1585 &pand ($twres,$twmask); # isolate carry and residue
1586 &pcmpgtd($twtmp,$tweak); # broadcast upper bits
f8501464
AP		 1587 &pxor ($tweak,$twres);
1588
1589 &sub ($len,16*6);
1590 &jnc (&label("xts_dec_loop6"));
1591
d8ba0dc9 1592 &mov ($rounds,&DWP(240,$key_)); # restore $rounds
f8501464
AP		 1593 &mov ($key,$key_); # restore $key
1594 &mov ($rounds_,$rounds);
1595
1596&set_label("xts_dec_short");
1597 &add ($len,16*6);
1598 &jz (&label("xts_dec_done6x"));
1599
1600 &movdqa ($inout3,$tweak); # put aside previous tweak
1601 &cmp ($len,0x20);
1602 &jb (&label("xts_dec_one"));
1603
1604 &pshufd ($twres,$twtmp,0x13);
1605 &pxor ($twtmp,$twtmp);
1606 &paddq ($tweak,$tweak); # &psllq($tweak,1);
1607 &pand ($twres,$twmask); # isolate carry and residue
1608 &pcmpgtd($twtmp,$tweak); # broadcast upper bits
1609 &pxor ($tweak,$twres);
1610 &je (&label("xts_dec_two"));
1611
1612 &pshufd ($twres,$twtmp,0x13);
1613 &pxor ($twtmp,$twtmp);
1614 &movdqa ($inout4,$tweak); # put aside previous tweak
1615 &paddq ($tweak,$tweak); # &psllq($tweak,1);
1616 &pand ($twres,$twmask); # isolate carry and residue
1617 &pcmpgtd($twtmp,$tweak); # broadcast upper bits
1618 &pxor ($tweak,$twres);
1619 &cmp ($len,0x40);
1620 &jb (&label("xts_dec_three"));
1621
1622 &pshufd ($twres,$twtmp,0x13);
1623 &pxor ($twtmp,$twtmp);
1624 &movdqa ($inout5,$tweak); # put aside previous tweak
1625 &paddq ($tweak,$tweak); # &psllq($tweak,1);
1626 &pand ($twres,$twmask); # isolate carry and residue
1627 &pcmpgtd($twtmp,$tweak); # broadcast upper bits
1628 &pxor ($tweak,$twres);
1629 &movdqa (&QWP(16*0,"esp"),$inout3);
1630 &movdqa (&QWP(16*1,"esp"),$inout4);
1631 &je (&label("xts_dec_four"));
1632
1633 &movdqa (&QWP(16*2,"esp"),$inout5);
1634 &pshufd ($inout5,$twtmp,0x13);
1635 &movdqa (&QWP(16*3,"esp"),$tweak);
1636 &paddq ($tweak,$tweak); # &psllq($inout0,1);
1637 &pand ($inout5,$twmask); # isolate carry and residue
1638 &pxor ($inout5,$tweak);
1639
1640 &movdqu ($inout0,&QWP(16*0,$inp)); # load input
1641 &movdqu ($inout1,&QWP(16*1,$inp));
1642 &movdqu ($inout2,&QWP(16*2,$inp));
1643 &pxor ($inout0,&QWP(16*0,"esp")); # input^=tweak
1644 &movdqu ($inout3,&QWP(16*3,$inp));
1645 &pxor ($inout1,&QWP(16*1,"esp"));
1646 &movdqu ($inout4,&QWP(16*4,$inp));
1647 &pxor ($inout2,&QWP(16*2,"esp"));
1648 &lea ($inp,&DWP(16*5,$inp));
1649 &pxor ($inout3,&QWP(16*3,"esp"));
1650 &movdqa (&QWP(16*4,"esp"),$inout5); # save last tweak
1651 &pxor ($inout4,$inout5);
1652
1653 &call ("_aesni_decrypt6");
1654
1655 &movaps ($tweak,&QWP(16*4,"esp")); # last tweak
1656 &xorps ($inout0,&QWP(16*0,"esp")); # output^=tweak
1657 &xorps ($inout1,&QWP(16*1,"esp"));
1658 &xorps ($inout2,&QWP(16*2,"esp"));
1659 &movups (&QWP(16*0,$out),$inout0); # write output
1660 &xorps ($inout3,&QWP(16*3,"esp"));
1661 &movups (&QWP(16*1,$out),$inout1);
1662 &xorps ($inout4,$tweak);
1663 &movups (&QWP(16*2,$out),$inout2);
1664 &movups (&QWP(16*3,$out),$inout3);
1665 &movups (&QWP(16*4,$out),$inout4);
1666 &lea ($out,&DWP(16*5,$out));
1667 &jmp (&label("xts_dec_done"));
1668
1669&set_label("xts_dec_one",16);
1670 &movups ($inout0,&QWP(16*0,$inp)); # load input
1671 &lea ($inp,&DWP(16*1,$inp));
1672 &xorps ($inout0,$inout3); # input^=tweak
1673 if ($inline)
1674 { &aesni_inline_generate1("dec"); }
1675 else
1676 { &call ("_aesni_decrypt1"); }
1677 &xorps ($inout0,$inout3); # output^=tweak
1678 &movups (&QWP(16*0,$out),$inout0); # write output
1679 &lea ($out,&DWP(16*1,$out));
1680
1681 &movdqa ($tweak,$inout3); # last tweak
1682 &jmp (&label("xts_dec_done"));
1683
1684&set_label("xts_dec_two",16);
1685 &movaps ($inout4,$tweak); # put aside last tweak
1686
1687 &movups ($inout0,&QWP(16*0,$inp)); # load input
1688 &movups ($inout1,&QWP(16*1,$inp));
1689 &lea ($inp,&DWP(16*2,$inp));
1690 &xorps ($inout0,$inout3); # input^=tweak
1691 &xorps ($inout1,$inout4);
1692
214368ff 1693 &call ("_aesni_decrypt2");
f8501464
AP		 1694
1695 &xorps ($inout0,$inout3); # output^=tweak
1696 &xorps ($inout1,$inout4);
1697 &movups (&QWP(16*0,$out),$inout0); # write output
1698 &movups (&QWP(16*1,$out),$inout1);
1699 &lea ($out,&DWP(16*2,$out));
1700
1701 &movdqa ($tweak,$inout4); # last tweak
1702 &jmp (&label("xts_dec_done"));
1703
1704&set_label("xts_dec_three",16);
1705 &movaps ($inout5,$tweak); # put aside last tweak
1706 &movups ($inout0,&QWP(16*0,$inp)); # load input
1707 &movups ($inout1,&QWP(16*1,$inp));
1708 &movups ($inout2,&QWP(16*2,$inp));
1709 &lea ($inp,&DWP(16*3,$inp));
1710 &xorps ($inout0,$inout3); # input^=tweak
1711 &xorps ($inout1,$inout4);
1712 &xorps ($inout2,$inout5);
1713
1714 &call ("_aesni_decrypt3");
1715
1716 &xorps ($inout0,$inout3); # output^=tweak
1717 &xorps ($inout1,$inout4);
1718 &xorps ($inout2,$inout5);
1719 &movups (&QWP(16*0,$out),$inout0); # write output
1720 &movups (&QWP(16*1,$out),$inout1);
1721 &movups (&QWP(16*2,$out),$inout2);
1722 &lea ($out,&DWP(16*3,$out));
1723
1724 &movdqa ($tweak,$inout5); # last tweak
1725 &jmp (&label("xts_dec_done"));
1726
1727&set_label("xts_dec_four",16);
1728 &movaps ($inout4,$tweak); # put aside last tweak
1729
1730 &movups ($inout0,&QWP(16*0,$inp)); # load input
1731 &movups ($inout1,&QWP(16*1,$inp));
1732 &movups ($inout2,&QWP(16*2,$inp));
1733 &xorps ($inout0,&QWP(16*0,"esp")); # input^=tweak
1734 &movups ($inout3,&QWP(16*3,$inp));
1735 &lea ($inp,&DWP(16*4,$inp));
1736 &xorps ($inout1,&QWP(16*1,"esp"));
1737 &xorps ($inout2,$inout5);
1738 &xorps ($inout3,$inout4);
1739
1740 &call ("_aesni_decrypt4");
1741
1742 &xorps ($inout0,&QWP(16*0,"esp")); # output^=tweak
1743 &xorps ($inout1,&QWP(16*1,"esp"));
1744 &xorps ($inout2,$inout5);
1745 &movups (&QWP(16*0,$out),$inout0); # write output
1746 &xorps ($inout3,$inout4);
1747 &movups (&QWP(16*1,$out),$inout1);
1748 &movups (&QWP(16*2,$out),$inout2);
1749 &movups (&QWP(16*3,$out),$inout3);
1750 &lea ($out,&DWP(16*4,$out));
1751
1752 &movdqa ($tweak,$inout4); # last tweak
1753 &jmp (&label("xts_dec_done"));
1754
1755&set_label("xts_dec_done6x",16); # $tweak is pre-calculated
1756 &mov ($len,&DWP(16*7+0,"esp")); # restore original $len
1757 &and ($len,15);
1758 &jz (&label("xts_dec_ret"));
1759 &mov (&DWP(16*7+0,"esp"),$len); # save $len%16
1760 &jmp (&label("xts_dec_only_one_more"));
1761
1762&set_label("xts_dec_done",16);
1763 &mov ($len,&DWP(16*7+0,"esp")); # restore original $len
1764 &pxor ($twtmp,$twtmp);
1765 &and ($len,15);
1766 &jz (&label("xts_dec_ret"));
1767
1768 &pcmpgtd($twtmp,$tweak); # broadcast upper bits
1769 &mov (&DWP(16*7+0,"esp"),$len); # save $len%16
1770 &pshufd ($twres,$twtmp,0x13);
1771 &pxor ($twtmp,$twtmp);
1772 &movdqa ($twmask,&QWP(16*6,"esp"));
1773 &paddq ($tweak,$tweak); # &psllq($tweak,1);
1774 &pand ($twres,$twmask); # isolate carry and residue
1775 &pcmpgtd($twtmp,$tweak); # broadcast upper bits
1776 &pxor ($tweak,$twres);
1777
1778&set_label("xts_dec_only_one_more");
1779 &pshufd ($inout3,$twtmp,0x13);
1780 &movdqa ($inout4,$tweak); # put aside previous tweak
1781 &paddq ($tweak,$tweak); # &psllq($tweak,1);
1782 &pand ($inout3,$twmask); # isolate carry and residue
1783 &pxor ($inout3,$tweak);
1784
1785 &mov ($key,$key_); # restore $key
1786 &mov ($rounds,$rounds_); # restore $rounds
1787
1788 &movups ($inout0,&QWP(0,$inp)); # load input
1789 &xorps ($inout0,$inout3); # input^=tweak
1790 if ($inline)
1791 { &aesni_inline_generate1("dec"); }
1792 else
1793 { &call ("_aesni_decrypt1"); }
1794 &xorps ($inout0,$inout3); # output^=tweak
1795 &movups (&QWP(0,$out),$inout0); # write output
1796
1797&set_label("xts_dec_steal");
1798 &movz ($rounds,&BP(16,$inp));
1799 &movz ($key,&BP(0,$out));
1800 &lea ($inp,&DWP(1,$inp));
1801 &mov (&BP(0,$out),&LB($rounds));
1802 &mov (&BP(16,$out),&LB($key));
1803 &lea ($out,&DWP(1,$out));
1804 &sub ($len,1);
1805 &jnz (&label("xts_dec_steal"));
1806
1807 &sub ($out,&DWP(16*7+0,"esp")); # rewind $out
1808 &mov ($key,$key_); # restore $key
1809 &mov ($rounds,$rounds_); # restore $rounds
1810
1811 &movups ($inout0,&QWP(0,$out)); # load input
1812 &xorps ($inout0,$inout4); # input^=tweak
1813 if ($inline)
1814 { &aesni_inline_generate1("dec"); }
1815 else
1816 { &call ("_aesni_decrypt1"); }
1817 &xorps ($inout0,$inout4); # output^=tweak
1818 &movups (&QWP(0,$out),$inout0); # write output
1819
1820&set_label("xts_dec_ret");
23f6eec7
AP		 1821 &pxor ("xmm0","xmm0"); # clear register bank
1822 &pxor ("xmm1","xmm1");
1823 &pxor ("xmm2","xmm2");
1824 &movdqa (&QWP(16*0,"esp"),"xmm0"); # clear stack
1825 &pxor ("xmm3","xmm3");
1826 &movdqa (&QWP(16*1,"esp"),"xmm0");
1827 &pxor ("xmm4","xmm4");
1828 &movdqa (&QWP(16*2,"esp"),"xmm0");
1829 &pxor ("xmm5","xmm5");
1830 &movdqa (&QWP(16*3,"esp"),"xmm0");
1831 &pxor ("xmm6","xmm6");
1832 &movdqa (&QWP(16*4,"esp"),"xmm0");
1833 &pxor ("xmm7","xmm7");
1834 &movdqa (&QWP(16*5,"esp"),"xmm0");
f8501464
AP		 1835 &mov ("esp",&DWP(16*7+4,"esp")); # restore %esp
1836&function_end("aesni_xts_decrypt");
1837}
bd30091c
AP		 1838\f
1839######################################################################
1840# void aesni_ocb_[en|de]crypt(const char *inp, char *out, size_t blocks,
1841# const AES_KEY *key, unsigned int start_block_num,
1842# unsigned char offset_i[16], const unsigned char L_[][16],
1843# unsigned char checksum[16]);
1844#
1845{
1846# offsets within stack frame
1847my $checksum = 16*6;
1848my ($key_off,$rounds_off,$out_off,$end_off,$esp_off)=map(16*7+4*$_,(0..4));
1849
1850# reassigned registers
1851my ($l_,$block,$i1,$i3,$i5) = ($rounds_,$key_,$rounds,$len,$out);
1852# $l_, $blocks, $inp, $key are permanently allocated in registers;
1853# remaining non-volatile ones are offloaded to stack, which even
1854# stay invariant after written to stack.
1855
1856&function_begin("aesni_ocb_encrypt");
1857 &mov ($rounds,&wparam(5)); # &offset_i
1858 &mov ($rounds_,&wparam(7)); # &checksum
1859
1860 &mov ($inp,&wparam(0));
1861 &mov ($out,&wparam(1));
1862 &mov ($len,&wparam(2));
1863 &mov ($key,&wparam(3));
1864 &movdqu ($rndkey0,&QWP(0,$rounds)); # load offset_i
1865 &mov ($block,&wparam(4)); # start_block_num
1866 &movdqu ($rndkey1,&QWP(0,$rounds_)); # load checksum
1867 &mov ($l_,&wparam(6)); # L_
1868
1869 &mov ($rounds,"esp");
1870 &sub ("esp",$esp_off+4); # alloca
1871 &and ("esp",-16); # align stack
1872
1873 &sub ($out,$inp);
1874 &shl ($len,4);
1875 &lea ($len,&DWP(-16*6,$inp,$len)); # end of input - 16*6
1876 &mov (&DWP($out_off,"esp"),$out);
1877 &mov (&DWP($end_off,"esp"),$len);
1878 &mov (&DWP($esp_off,"esp"),$rounds);
1879
1880 &mov ($rounds,&DWP(240,$key));
1881
1882 &test ($block,1);
1883 &jnz (&label("odd"));
1884
1885 &bsf ($i3,$block);
1886 &add ($block,1);
1887 &shl ($i3,4);
1888 &movdqu ($inout5,&QWP(0,$l_,$i3));
1889 &mov ($i3,$key); # put aside key
1890
1891 &movdqu ($inout0,&QWP(16*0,$inp)); # load input
1892 &lea ($inp,&DWP(16,$inp));
1893
1894 &pxor ($inout5,$rndkey0); # ^ last offset_i
1895 &pxor ($rndkey1,$inout0); # checksum
1896 &pxor ($inout0,$inout5); # ^ offset_i
1897
1898 &movdqa ($inout4,$rndkey1);
1899 if ($inline)
1900 { &aesni_inline_generate1("enc"); }
1901 else
1902 { &call ("_aesni_encrypt1"); }
1903
1904 &xorps ($inout0,$inout5); # ^ offset_i
1905 &movdqa ($rndkey0,$inout5); # pass last offset_i
1906 &movdqa ($rndkey1,$inout4); # pass the checksum
1907
1908 &movups (&QWP(-16,$out,$inp),$inout0); # store output
1909
1910 &mov ($rounds,&DWP(240,$i3));
1911 &mov ($key,$i3); # restore key
1912 &mov ($len,&DWP($end_off,"esp"));
1913
1914&set_label("odd");
1915 &shl ($rounds,4);
1916 &mov ($out,16);
1917 &sub ($out,$rounds); # twisted rounds
1918 &mov (&DWP($key_off,"esp"),$key);
1919 &lea ($key,&DWP(32,$key,$rounds)); # end of key schedule
1920 &mov (&DWP($rounds_off,"esp"),$out);
1921
1922 &cmp ($inp,$len);
1923 &ja (&label("short"));
1924 &jmp (&label("grandloop"));
1925
1926&set_label("grandloop",32);
1927 &lea ($i1,&DWP(1,$block));
1928 &lea ($i3,&DWP(3,$block));
1929 &lea ($i5,&DWP(5,$block));
1930 &add ($block,6);
1931 &bsf ($i1,$i1);
1932 &bsf ($i3,$i3);
1933 &bsf ($i5,$i5);
1934 &shl ($i1,4);
1935 &shl ($i3,4);
1936 &shl ($i5,4);
1937 &movdqu ($inout0,&QWP(0,$l_));
1938 &movdqu ($inout1,&QWP(0,$l_,$i1));
1939 &mov ($rounds,&DWP($rounds_off,"esp"));
1940 &movdqa ($inout2,$inout0);
1941 &movdqu ($inout3,&QWP(0,$l_,$i3));
1942 &movdqa ($inout4,$inout0);
1943 &movdqu ($inout5,&QWP(0,$l_,$i5));
1944
1945 &pxor ($inout0,$rndkey0); # ^ last offset_i
1946 &pxor ($inout1,$inout0);
1947 &movdqa (&QWP(16*0,"esp"),$inout0);
1948 &pxor ($inout2,$inout1);
1949 &movdqa (&QWP(16*1,"esp"),$inout1);
1950 &pxor ($inout3,$inout2);
1951 &movdqa (&QWP(16*2,"esp"),$inout2);
1952 &pxor ($inout4,$inout3);
1953 &movdqa (&QWP(16*3,"esp"),$inout3);
1954 &pxor ($inout5,$inout4);
1955 &movdqa (&QWP(16*4,"esp"),$inout4);
1956 &movdqa (&QWP(16*5,"esp"),$inout5);
1957
1958 &$movekey ($rndkey0,&QWP(-48,$key,$rounds));
1959 &movdqu ($inout0,&QWP(16*0,$inp)); # load input
1960 &movdqu ($inout1,&QWP(16*1,$inp));
1961 &movdqu ($inout2,&QWP(16*2,$inp));
1962 &movdqu ($inout3,&QWP(16*3,$inp));
1963 &movdqu ($inout4,&QWP(16*4,$inp));
1964 &movdqu ($inout5,&QWP(16*5,$inp));
1965 &lea ($inp,&DWP(16*6,$inp));
1966
1967 &pxor ($rndkey1,$inout0); # checksum
1968 &pxor ($inout0,$rndkey0); # ^ roundkey[0]
1969 &pxor ($rndkey1,$inout1);
1970 &pxor ($inout1,$rndkey0);
1971 &pxor ($rndkey1,$inout2);
1972 &pxor ($inout2,$rndkey0);
1973 &pxor ($rndkey1,$inout3);
1974 &pxor ($inout3,$rndkey0);
1975 &pxor ($rndkey1,$inout4);
1976 &pxor ($inout4,$rndkey0);
1977 &pxor ($rndkey1,$inout5);
1978 &pxor ($inout5,$rndkey0);
1979 &movdqa (&QWP($checksum,"esp"),$rndkey1);
1980
1981 &$movekey ($rndkey1,&QWP(-32,$key,$rounds));
1982 &pxor ($inout0,&QWP(16*0,"esp")); # ^ offset_i
1983 &pxor ($inout1,&QWP(16*1,"esp"));
1984 &pxor ($inout2,&QWP(16*2,"esp"));
1985 &pxor ($inout3,&QWP(16*3,"esp"));
1986 &pxor ($inout4,&QWP(16*4,"esp"));
1987 &pxor ($inout5,&QWP(16*5,"esp"));
1988
1989 &$movekey ($rndkey0,&QWP(-16,$key,$rounds));
1990 &aesenc ($inout0,$rndkey1);
1991 &aesenc ($inout1,$rndkey1);
1992 &aesenc ($inout2,$rndkey1);
1993 &aesenc ($inout3,$rndkey1);
1994 &aesenc ($inout4,$rndkey1);
1995 &aesenc ($inout5,$rndkey1);
1996
1997 &mov ($out,&DWP($out_off,"esp"));
1998 &mov ($len,&DWP($end_off,"esp"));
1999 &call ("_aesni_encrypt6_enter");
2000
2001 &movdqa ($rndkey0,&QWP(16*5,"esp")); # pass last offset_i
2002 &pxor ($inout0,&QWP(16*0,"esp")); # ^ offset_i
2003 &pxor ($inout1,&QWP(16*1,"esp"));
2004 &pxor ($inout2,&QWP(16*2,"esp"));
2005 &pxor ($inout3,&QWP(16*3,"esp"));
2006 &pxor ($inout4,&QWP(16*4,"esp"));
2007 &pxor ($inout5,$rndkey0);
2008 &movdqa ($rndkey1,&QWP($checksum,"esp"));# pass the checksum
2009
2010 &movdqu (&QWP(-16*6,$out,$inp),$inout0);# store output
2011 &movdqu (&QWP(-16*5,$out,$inp),$inout1);
2012 &movdqu (&QWP(-16*4,$out,$inp),$inout2);
2013 &movdqu (&QWP(-16*3,$out,$inp),$inout3);
2014 &movdqu (&QWP(-16*2,$out,$inp),$inout4);
2015 &movdqu (&QWP(-16*1,$out,$inp),$inout5);
2016 &cmp ($inp,$len); # done yet?
2017 &jb (&label("grandloop"));
2018
2019&set_label("short");
2020 &add ($len,16*6);
2021 &sub ($len,$inp);
2022 &jz (&label("done"));
2023
2024 &cmp ($len,16*2);
2025 &jb (&label("one"));
2026 &je (&label("two"));
2027
2028 &cmp ($len,16*4);
2029 &jb (&label("three"));
2030 &je (&label("four"));
2031
2032 &lea ($i1,&DWP(1,$block));
2033 &lea ($i3,&DWP(3,$block));
2034 &bsf ($i1,$i1);
2035 &bsf ($i3,$i3);
2036 &shl ($i1,4);
2037 &shl ($i3,4);
2038 &movdqu ($inout0,&QWP(0,$l_));
2039 &movdqu ($inout1,&QWP(0,$l_,$i1));
2040 &mov ($rounds,&DWP($rounds_off,"esp"));
2041 &movdqa ($inout2,$inout0);
2042 &movdqu ($inout3,&QWP(0,$l_,$i3));
2043 &movdqa ($inout4,$inout0);
2044
2045 &pxor ($inout0,$rndkey0); # ^ last offset_i
2046 &pxor ($inout1,$inout0);
2047 &movdqa (&QWP(16*0,"esp"),$inout0);
2048 &pxor ($inout2,$inout1);
2049 &movdqa (&QWP(16*1,"esp"),$inout1);
2050 &pxor ($inout3,$inout2);
2051 &movdqa (&QWP(16*2,"esp"),$inout2);
2052 &pxor ($inout4,$inout3);
2053 &movdqa (&QWP(16*3,"esp"),$inout3);
2054 &pxor ($inout5,$inout4);
2055 &movdqa (&QWP(16*4,"esp"),$inout4);
2056
2057 &$movekey ($rndkey0,&QWP(-48,$key,$rounds));
2058 &movdqu ($inout0,&QWP(16*0,$inp)); # load input
2059 &movdqu ($inout1,&QWP(16*1,$inp));
2060 &movdqu ($inout2,&QWP(16*2,$inp));
2061 &movdqu ($inout3,&QWP(16*3,$inp));
2062 &movdqu ($inout4,&QWP(16*4,$inp));
2063 &pxor ($inout5,$inout5);
2064
2065 &pxor ($rndkey1,$inout0); # checksum
2066 &pxor ($inout0,$rndkey0); # ^ roundkey[0]
2067 &pxor ($rndkey1,$inout1);
2068 &pxor ($inout1,$rndkey0);
2069 &pxor ($rndkey1,$inout2);
2070 &pxor ($inout2,$rndkey0);
2071 &pxor ($rndkey1,$inout3);
2072 &pxor ($inout3,$rndkey0);
2073 &pxor ($rndkey1,$inout4);
2074 &pxor ($inout4,$rndkey0);
2075 &movdqa (&QWP($checksum,"esp"),$rndkey1);
2076
2077 &$movekey ($rndkey1,&QWP(-32,$key,$rounds));
2078 &pxor ($inout0,&QWP(16*0,"esp")); # ^ offset_i
2079 &pxor ($inout1,&QWP(16*1,"esp"));
2080 &pxor ($inout2,&QWP(16*2,"esp"));
2081 &pxor ($inout3,&QWP(16*3,"esp"));
2082 &pxor ($inout4,&QWP(16*4,"esp"));
2083
2084 &$movekey ($rndkey0,&QWP(-16,$key,$rounds));
2085 &aesenc ($inout0,$rndkey1);
2086 &aesenc ($inout1,$rndkey1);
2087 &aesenc ($inout2,$rndkey1);
2088 &aesenc ($inout3,$rndkey1);
2089 &aesenc ($inout4,$rndkey1);
2090 &aesenc ($inout5,$rndkey1);
2091
2092 &mov ($out,&DWP($out_off,"esp"));
2093 &call ("_aesni_encrypt6_enter");
2094
2095 &movdqa ($rndkey0,&QWP(16*4,"esp")); # pass last offset_i
2096 &pxor ($inout0,&QWP(16*0,"esp")); # ^ offset_i
2097 &pxor ($inout1,&QWP(16*1,"esp"));
2098 &pxor ($inout2,&QWP(16*2,"esp"));
2099 &pxor ($inout3,&QWP(16*3,"esp"));
2100 &pxor ($inout4,$rndkey0);
2101 &movdqa ($rndkey1,&QWP($checksum,"esp"));# pass the checksum
2102
2103 &movdqu (&QWP(16*0,$out,$inp),$inout0); # store output
2104 &movdqu (&QWP(16*1,$out,$inp),$inout1);
2105 &movdqu (&QWP(16*2,$out,$inp),$inout2);
2106 &movdqu (&QWP(16*3,$out,$inp),$inout3);
2107 &movdqu (&QWP(16*4,$out,$inp),$inout4);
2108
2109 &jmp (&label("done"));
2110
2111&set_label("one",16);
2112 &movdqu ($inout5,&QWP(0,$l_));
2113 &mov ($key,&DWP($key_off,"esp")); # restore key
2114
2115 &movdqu ($inout0,&QWP(16*0,$inp)); # load input
2116 &mov ($rounds,&DWP(240,$key));
2117
2118 &pxor ($inout5,$rndkey0); # ^ last offset_i
2119 &pxor ($rndkey1,$inout0); # checksum
2120 &pxor ($inout0,$inout5); # ^ offset_i
2121
2122 &movdqa ($inout4,$rndkey1);
2123 &mov ($out,&DWP($out_off,"esp"));
2124 if ($inline)
2125 { &aesni_inline_generate1("enc"); }
2126 else
2127 { &call ("_aesni_encrypt1"); }
2128
2129 &xorps ($inout0,$inout5); # ^ offset_i
2130 &movdqa ($rndkey0,$inout5); # pass last offset_i
2131 &movdqa ($rndkey1,$inout4); # pass the checksum
2132 &movups (&QWP(0,$out,$inp),$inout0);
2133
2134 &jmp (&label("done"));
2135
2136&set_label("two",16);
2137 &lea ($i1,&DWP(1,$block));
2138 &mov ($key,&DWP($key_off,"esp")); # restore key
2139 &bsf ($i1,$i1);
2140 &shl ($i1,4);
2141 &movdqu ($inout4,&QWP(0,$l_));
2142 &movdqu ($inout5,&QWP(0,$l_,$i1));
2143
2144 &movdqu ($inout0,&QWP(16*0,$inp)); # load input
2145 &movdqu ($inout1,&QWP(16*1,$inp));
2146 &mov ($rounds,&DWP(240,$key));
2147
2148 &pxor ($inout4,$rndkey0); # ^ last offset_i
2149 &pxor ($inout5,$inout4);
2150
2151 &pxor ($rndkey1,$inout0); # checksum
2152 &pxor ($inout0,$inout4); # ^ offset_i
2153 &pxor ($rndkey1,$inout1);
2154 &pxor ($inout1,$inout5);
2155
2156 &movdqa ($inout3,$rndkey1)
2157 &mov ($out,&DWP($out_off,"esp"));
2158 &call ("_aesni_encrypt2");
2159
2160 &xorps ($inout0,$inout4); # ^ offset_i
2161 &xorps ($inout1,$inout5);
2162 &movdqa ($rndkey0,$inout5); # pass last offset_i
2163 &movdqa ($rndkey1,$inout3); # pass the checksum
2164 &movups (&QWP(16*0,$out,$inp),$inout0); # store output
2165 &movups (&QWP(16*1,$out,$inp),$inout1);
2166
2167 &jmp (&label("done"));
2168
2169&set_label("three",16);
2170 &lea ($i1,&DWP(1,$block));
2171 &mov ($key,&DWP($key_off,"esp")); # restore key
2172 &bsf ($i1,$i1);
2173 &shl ($i1,4);
2174 &movdqu ($inout3,&QWP(0,$l_));
2175 &movdqu ($inout4,&QWP(0,$l_,$i1));
2176 &movdqa ($inout5,$inout3);
2177
2178 &movdqu ($inout0,&QWP(16*0,$inp)); # load input
2179 &movdqu ($inout1,&QWP(16*1,$inp));
2180 &movdqu ($inout2,&QWP(16*2,$inp));
2181 &mov ($rounds,&DWP(240,$key));
2182
2183 &pxor ($inout3,$rndkey0); # ^ last offset_i
2184 &pxor ($inout4,$inout3);
2185 &pxor ($inout5,$inout4);
2186
2187 &pxor ($rndkey1,$inout0); # checksum
2188 &pxor ($inout0,$inout3); # ^ offset_i
2189 &pxor ($rndkey1,$inout1);
2190 &pxor ($inout1,$inout4);
2191 &pxor ($rndkey1,$inout2);
2192 &pxor ($inout2,$inout5);
2193
2194 &movdqa (&QWP($checksum,"esp"),$rndkey1);
2195 &mov ($out,&DWP($out_off,"esp"));
2196 &call ("_aesni_encrypt3");
2197
2198 &xorps ($inout0,$inout3); # ^ offset_i
2199 &xorps ($inout1,$inout4);
2200 &xorps ($inout2,$inout5);
2201 &movdqa ($rndkey0,$inout5); # pass last offset_i
2202 &movdqa ($rndkey1,&QWP($checksum,"esp"));# pass the checksum
2203 &movups (&QWP(16*0,$out,$inp),$inout0); # store output
2204 &movups (&QWP(16*1,$out,$inp),$inout1);
2205 &movups (&QWP(16*2,$out,$inp),$inout2);
2206
2207 &jmp (&label("done"));
2208
2209&set_label("four",16);
2210 &lea ($i1,&DWP(1,$block));
2211 &lea ($i3,&DWP(3,$block));
2212 &bsf ($i1,$i1);
2213 &bsf ($i3,$i3);
2214 &mov ($key,&DWP($key_off,"esp")); # restore key
2215 &shl ($i1,4);
2216 &shl ($i3,4);
2217 &movdqu ($inout2,&QWP(0,$l_));
2218 &movdqu ($inout3,&QWP(0,$l_,$i1));
2219 &movdqa ($inout4,$inout2);
2220 &movdqu ($inout5,&QWP(0,$l_,$i3));
2221
2222 &pxor ($inout2,$rndkey0); # ^ last offset_i
2223 &movdqu ($inout0,&QWP(16*0,$inp)); # load input
2224 &pxor ($inout3,$inout2);
2225 &movdqu ($inout1,&QWP(16*1,$inp));
2226 &pxor ($inout4,$inout3);
2227 &movdqa (&QWP(16*0,"esp"),$inout2);
2228 &pxor ($inout5,$inout4);
2229 &movdqa (&QWP(16*1,"esp"),$inout3);
2230 &movdqu ($inout2,&QWP(16*2,$inp));
2231 &movdqu ($inout3,&QWP(16*3,$inp));
2232 &mov ($rounds,&DWP(240,$key));
2233
2234 &pxor ($rndkey1,$inout0); # checksum
2235 &pxor ($inout0,&QWP(16*0,"esp")); # ^ offset_i
2236 &pxor ($rndkey1,$inout1);
2237 &pxor ($inout1,&QWP(16*1,"esp"));
2238 &pxor ($rndkey1,$inout2);
2239 &pxor ($inout2,$inout4);
2240 &pxor ($rndkey1,$inout3);
2241 &pxor ($inout3,$inout5);
2242
2243 &movdqa (&QWP($checksum,"esp"),$rndkey1)
2244 &mov ($out,&DWP($out_off,"esp"));
2245 &call ("_aesni_encrypt4");
2246
2247 &xorps ($inout0,&QWP(16*0,"esp")); # ^ offset_i
2248 &xorps ($inout1,&QWP(16*1,"esp"));
2249 &xorps ($inout2,$inout4);
2250 &movups (&QWP(16*0,$out,$inp),$inout0); # store output
2251 &xorps ($inout3,$inout5);
2252 &movups (&QWP(16*1,$out,$inp),$inout1);
2253 &movdqa ($rndkey0,$inout5); # pass last offset_i
2254 &movups (&QWP(16*2,$out,$inp),$inout2);
2255 &movdqa ($rndkey1,&QWP($checksum,"esp"));# pass the checksum
2256 &movups (&QWP(16*3,$out,$inp),$inout3);
2257
2258&set_label("done");
2259 &mov ($key,&DWP($esp_off,"esp"));
2260 &pxor ($inout0,$inout0); # clear register bank
2261 &pxor ($inout1,$inout1);
2262 &movdqa (&QWP(16*0,"esp"),$inout0); # clear stack
2263 &pxor ($inout2,$inout2);
2264 &movdqa (&QWP(16*1,"esp"),$inout0);
2265 &pxor ($inout3,$inout3);
2266 &movdqa (&QWP(16*2,"esp"),$inout0);
2267 &pxor ($inout4,$inout4);
2268 &movdqa (&QWP(16*3,"esp"),$inout0);
2269 &pxor ($inout5,$inout5);
2270 &movdqa (&QWP(16*4,"esp"),$inout0);
2271 &movdqa (&QWP(16*5,"esp"),$inout0);
2272 &movdqa (&QWP(16*6,"esp"),$inout0);
2273
2274 &lea ("esp",&DWP(0,$key));
2275 &mov ($rounds,&wparam(5)); # &offset_i
2276 &mov ($rounds_,&wparam(7)); # &checksum
2277 &movdqu (&QWP(0,$rounds),$rndkey0);
2278 &pxor ($rndkey0,$rndkey0);
2279 &movdqu (&QWP(0,$rounds_),$rndkey1);
2280 &pxor ($rndkey1,$rndkey1);
2281&function_end("aesni_ocb_encrypt");
2282
2283&function_begin("aesni_ocb_decrypt");
2284 &mov ($rounds,&wparam(5)); # &offset_i
2285 &mov ($rounds_,&wparam(7)); # &checksum
2286
2287 &mov ($inp,&wparam(0));
2288 &mov ($out,&wparam(1));
2289 &mov ($len,&wparam(2));
2290 &mov ($key,&wparam(3));
2291 &movdqu ($rndkey0,&QWP(0,$rounds)); # load offset_i
2292 &mov ($block,&wparam(4)); # start_block_num
2293 &movdqu ($rndkey1,&QWP(0,$rounds_)); # load checksum
2294 &mov ($l_,&wparam(6)); # L_
2295
2296 &mov ($rounds,"esp");
2297 &sub ("esp",$esp_off+4); # alloca
2298 &and ("esp",-16); # align stack
2299
2300 &sub ($out,$inp);
2301 &shl ($len,4);
2302 &lea ($len,&DWP(-16*6,$inp,$len)); # end of input - 16*6
2303 &mov (&DWP($out_off,"esp"),$out);
2304 &mov (&DWP($end_off,"esp"),$len);
2305 &mov (&DWP($esp_off,"esp"),$rounds);
2306
2307 &mov ($rounds,&DWP(240,$key));
2308
2309 &test ($block,1);
2310 &jnz (&label("odd"));
2311
2312 &bsf ($i3,$block);
2313 &add ($block,1);
2314 &shl ($i3,4);
2315 &movdqu ($inout5,&QWP(0,$l_,$i3));
2316 &mov ($i3,$key); # put aside key
2317
2318 &movdqu ($inout0,&QWP(16*0,$inp)); # load input
2319 &lea ($inp,&DWP(16,$inp));
2320
2321 &pxor ($inout5,$rndkey0); # ^ last offset_i
2322 &pxor ($inout0,$inout5); # ^ offset_i
2323
2324 &movdqa ($inout4,$rndkey1);
2325 if ($inline)
2326 { &aesni_inline_generate1("dec"); }
2327 else
2328 { &call ("_aesni_decrypt1"); }
2329
2330 &xorps ($inout0,$inout5); # ^ offset_i
2331 &movaps ($rndkey1,$inout4); # pass the checksum
2332 &movdqa ($rndkey0,$inout5); # pass last offset_i
2333 &xorps ($rndkey1,$inout0); # checksum
2334 &movups (&QWP(-16,$out,$inp),$inout0); # store output
2335
2336 &mov ($rounds,&DWP(240,$i3));
2337 &mov ($key,$i3); # restore key
2338 &mov ($len,&DWP($end_off,"esp"));
2339
2340&set_label("odd");
2341 &shl ($rounds,4);
2342 &mov ($out,16);
2343 &sub ($out,$rounds); # twisted rounds
2344 &mov (&DWP($key_off,"esp"),$key);
2345 &lea ($key,&DWP(32,$key,$rounds)); # end of key schedule
2346 &mov (&DWP($rounds_off,"esp"),$out);
2347
2348 &cmp ($inp,$len);
2349 &ja (&label("short"));
2350 &jmp (&label("grandloop"));
2351
2352&set_label("grandloop",32);
2353 &lea ($i1,&DWP(1,$block));
2354 &lea ($i3,&DWP(3,$block));
2355 &lea ($i5,&DWP(5,$block));
2356 &add ($block,6);
2357 &bsf ($i1,$i1);
2358 &bsf ($i3,$i3);
2359 &bsf ($i5,$i5);
2360 &shl ($i1,4);
2361 &shl ($i3,4);
2362 &shl ($i5,4);
2363 &movdqu ($inout0,&QWP(0,$l_));
2364 &movdqu ($inout1,&QWP(0,$l_,$i1));
2365 &mov ($rounds,&DWP($rounds_off,"esp"));
2366 &movdqa ($inout2,$inout0);
2367 &movdqu ($inout3,&QWP(0,$l_,$i3));
2368 &movdqa ($inout4,$inout0);
2369 &movdqu ($inout5,&QWP(0,$l_,$i5));
2370
2371 &pxor ($inout0,$rndkey0); # ^ last offset_i
2372 &pxor ($inout1,$inout0);
2373 &movdqa (&QWP(16*0,"esp"),$inout0);
2374 &pxor ($inout2,$inout1);
2375 &movdqa (&QWP(16*1,"esp"),$inout1);
2376 &pxor ($inout3,$inout2);
2377 &movdqa (&QWP(16*2,"esp"),$inout2);
2378 &pxor ($inout4,$inout3);
2379 &movdqa (&QWP(16*3,"esp"),$inout3);
2380 &pxor ($inout5,$inout4);
2381 &movdqa (&QWP(16*4,"esp"),$inout4);
2382 &movdqa (&QWP(16*5,"esp"),$inout5);
2383
2384 &$movekey ($rndkey0,&QWP(-48,$key,$rounds));
2385 &movdqu ($inout0,&QWP(16*0,$inp)); # load input
2386 &movdqu ($inout1,&QWP(16*1,$inp));
2387 &movdqu ($inout2,&QWP(16*2,$inp));
2388 &movdqu ($inout3,&QWP(16*3,$inp));
2389 &movdqu ($inout4,&QWP(16*4,$inp));
2390 &movdqu ($inout5,&QWP(16*5,$inp));
2391 &lea ($inp,&DWP(16*6,$inp));
2392
2393 &movdqa (&QWP($checksum,"esp"),$rndkey1);
2394 &pxor ($inout0,$rndkey0); # ^ roundkey[0]
2395 &pxor ($inout1,$rndkey0);
2396 &pxor ($inout2,$rndkey0);
2397 &pxor ($inout3,$rndkey0);
2398 &pxor ($inout4,$rndkey0);
2399 &pxor ($inout5,$rndkey0);
2400
2401 &$movekey ($rndkey1,&QWP(-32,$key,$rounds));
2402 &pxor ($inout0,&QWP(16*0,"esp")); # ^ offset_i
2403 &pxor ($inout1,&QWP(16*1,"esp"));
2404 &pxor ($inout2,&QWP(16*2,"esp"));
2405 &pxor ($inout3,&QWP(16*3,"esp"));
2406 &pxor ($inout4,&QWP(16*4,"esp"));
2407 &pxor ($inout5,&QWP(16*5,"esp"));
2408
2409 &$movekey ($rndkey0,&QWP(-16,$key,$rounds));
2410 &aesdec ($inout0,$rndkey1);
2411 &aesdec ($inout1,$rndkey1);
2412 &aesdec ($inout2,$rndkey1);
2413 &aesdec ($inout3,$rndkey1);
2414 &aesdec ($inout4,$rndkey1);
2415 &aesdec ($inout5,$rndkey1);
2416
2417 &mov ($out,&DWP($out_off,"esp"));
2418 &mov ($len,&DWP($end_off,"esp"));
2419 &call ("_aesni_decrypt6_enter");
2420
2421 &movdqa ($rndkey0,&QWP(16*5,"esp")); # pass last offset_i
2422 &pxor ($inout0,&QWP(16*0,"esp")); # ^ offset_i
2423 &movdqa ($rndkey1,&QWP($checksum,"esp"));
2424 &pxor ($inout1,&QWP(16*1,"esp"));
2425 &pxor ($inout2,&QWP(16*2,"esp"));
2426 &pxor ($inout3,&QWP(16*3,"esp"));
2427 &pxor ($inout4,&QWP(16*4,"esp"));
2428 &pxor ($inout5,$rndkey0);
2429
2430 &pxor ($rndkey1,$inout0); # checksum
2431 &movdqu (&QWP(-16*6,$out,$inp),$inout0);# store output
2432 &pxor ($rndkey1,$inout1);
2433 &movdqu (&QWP(-16*5,$out,$inp),$inout1);
2434 &pxor ($rndkey1,$inout2);
2435 &movdqu (&QWP(-16*4,$out,$inp),$inout2);
2436 &pxor ($rndkey1,$inout3);
2437 &movdqu (&QWP(-16*3,$out,$inp),$inout3);
2438 &pxor ($rndkey1,$inout4);
2439 &movdqu (&QWP(-16*2,$out,$inp),$inout4);
2440 &pxor ($rndkey1,$inout5);
2441 &movdqu (&QWP(-16*1,$out,$inp),$inout5);
2442 &cmp ($inp,$len); # done yet?
2443 &jb (&label("grandloop"));
2444
2445&set_label("short");
2446 &add ($len,16*6);
2447 &sub ($len,$inp);
2448 &jz (&label("done"));
2449
2450 &cmp ($len,16*2);
2451 &jb (&label("one"));
2452 &je (&label("two"));
2453
2454 &cmp ($len,16*4);
2455 &jb (&label("three"));
2456 &je (&label("four"));
2457
2458 &lea ($i1,&DWP(1,$block));
2459 &lea ($i3,&DWP(3,$block));
2460 &bsf ($i1,$i1);
2461 &bsf ($i3,$i3);
2462 &shl ($i1,4);
2463 &shl ($i3,4);
2464 &movdqu ($inout0,&QWP(0,$l_));
2465 &movdqu ($inout1,&QWP(0,$l_,$i1));
2466 &mov ($rounds,&DWP($rounds_off,"esp"));
2467 &movdqa ($inout2,$inout0);
2468 &movdqu ($inout3,&QWP(0,$l_,$i3));
2469 &movdqa ($inout4,$inout0);
2470
2471 &pxor ($inout0,$rndkey0); # ^ last offset_i
2472 &pxor ($inout1,$inout0);
2473 &movdqa (&QWP(16*0,"esp"),$inout0);
2474 &pxor ($inout2,$inout1);
2475 &movdqa (&QWP(16*1,"esp"),$inout1);
2476 &pxor ($inout3,$inout2);
2477 &movdqa (&QWP(16*2,"esp"),$inout2);
2478 &pxor ($inout4,$inout3);
2479 &movdqa (&QWP(16*3,"esp"),$inout3);
2480 &pxor ($inout5,$inout4);
2481 &movdqa (&QWP(16*4,"esp"),$inout4);
2482
2483 &$movekey ($rndkey0,&QWP(-48,$key,$rounds));
2484 &movdqu ($inout0,&QWP(16*0,$inp)); # load input
2485 &movdqu ($inout1,&QWP(16*1,$inp));
2486 &movdqu ($inout2,&QWP(16*2,$inp));
2487 &movdqu ($inout3,&QWP(16*3,$inp));
2488 &movdqu ($inout4,&QWP(16*4,$inp));
2489 &pxor ($inout5,$inout5);
2490
2491 &movdqa (&QWP($checksum,"esp"),$rndkey1);
2492 &pxor ($inout0,$rndkey0); # ^ roundkey[0]
2493 &pxor ($inout1,$rndkey0);
2494 &pxor ($inout2,$rndkey0);
2495 &pxor ($inout3,$rndkey0);
2496 &pxor ($inout4,$rndkey0);
2497
2498 &$movekey ($rndkey1,&QWP(-32,$key,$rounds));
2499 &pxor ($inout0,&QWP(16*0,"esp")); # ^ offset_i
2500 &pxor ($inout1,&QWP(16*1,"esp"));
2501 &pxor ($inout2,&QWP(16*2,"esp"));
2502 &pxor ($inout3,&QWP(16*3,"esp"));
2503 &pxor ($inout4,&QWP(16*4,"esp"));
2504
2505 &$movekey ($rndkey0,&QWP(-16,$key,$rounds));
2506 &aesdec ($inout0,$rndkey1);
2507 &aesdec ($inout1,$rndkey1);
2508 &aesdec ($inout2,$rndkey1);
2509 &aesdec ($inout3,$rndkey1);
2510 &aesdec ($inout4,$rndkey1);
2511 &aesdec ($inout5,$rndkey1);
2512
2513 &mov ($out,&DWP($out_off,"esp"));
2514 &call ("_aesni_decrypt6_enter");
2515
2516 &movdqa ($rndkey0,&QWP(16*4,"esp")); # pass last offset_i
2517 &pxor ($inout0,&QWP(16*0,"esp")); # ^ offset_i
2518 &movdqa ($rndkey1,&QWP($checksum,"esp"));
2519 &pxor ($inout1,&QWP(16*1,"esp"));
2520 &pxor ($inout2,&QWP(16*2,"esp"));
2521 &pxor ($inout3,&QWP(16*3,"esp"));
2522 &pxor ($inout4,$rndkey0);
2523
2524 &pxor ($rndkey1,$inout0); # checksum
2525 &movdqu (&QWP(16*0,$out,$inp),$inout0); # store output
2526 &pxor ($rndkey1,$inout1);
2527 &movdqu (&QWP(16*1,$out,$inp),$inout1);
2528 &pxor ($rndkey1,$inout2);
2529 &movdqu (&QWP(16*2,$out,$inp),$inout2);
2530 &pxor ($rndkey1,$inout3);
2531 &movdqu (&QWP(16*3,$out,$inp),$inout3);
2532 &pxor ($rndkey1,$inout4);
2533 &movdqu (&QWP(16*4,$out,$inp),$inout4);
2534
2535 &jmp (&label("done"));
2536
2537&set_label("one",16);
2538 &movdqu ($inout5,&QWP(0,$l_));
2539 &mov ($key,&DWP($key_off,"esp")); # restore key
2540
2541 &movdqu ($inout0,&QWP(16*0,$inp)); # load input
2542 &mov ($rounds,&DWP(240,$key));
2543
2544 &pxor ($inout5,$rndkey0); # ^ last offset_i
2545 &pxor ($inout0,$inout5); # ^ offset_i
2546
2547 &movdqa ($inout4,$rndkey1);
2548 &mov ($out,&DWP($out_off,"esp"));
2549 if ($inline)
2550 { &aesni_inline_generate1("dec"); }
2551 else
2552 { &call ("_aesni_decrypt1"); }
2553
2554 &xorps ($inout0,$inout5); # ^ offset_i
2555 &movaps ($rndkey1,$inout4); # pass the checksum
2556 &movdqa ($rndkey0,$inout5); # pass last offset_i
2557 &xorps ($rndkey1,$inout0); # checksum
2558 &movups (&QWP(0,$out,$inp),$inout0);
2559
2560 &jmp (&label("done"));
2561
2562&set_label("two",16);
2563 &lea ($i1,&DWP(1,$block));
2564 &mov ($key,&DWP($key_off,"esp")); # restore key
2565 &bsf ($i1,$i1);
2566 &shl ($i1,4);
2567 &movdqu ($inout4,&QWP(0,$l_));
2568 &movdqu ($inout5,&QWP(0,$l_,$i1));
2569
2570 &movdqu ($inout0,&QWP(16*0,$inp)); # load input
2571 &movdqu ($inout1,&QWP(16*1,$inp));
2572 &mov ($rounds,&DWP(240,$key));
2573
2574 &movdqa ($inout3,$rndkey1);
2575 &pxor ($inout4,$rndkey0); # ^ last offset_i
2576 &pxor ($inout5,$inout4);
2577
2578 &pxor ($inout0,$inout4); # ^ offset_i
2579 &pxor ($inout1,$inout5);
2580
2581 &mov ($out,&DWP($out_off,"esp"));
2582 &call ("_aesni_decrypt2");
2583
2584 &xorps ($inout0,$inout4); # ^ offset_i
2585 &xorps ($inout1,$inout5);
2586 &movdqa ($rndkey0,$inout5); # pass last offset_i
2587 &xorps ($inout3,$inout0); # checksum
2588 &movups (&QWP(16*0,$out,$inp),$inout0); # store output
2589 &xorps ($inout3,$inout1);
2590 &movups (&QWP(16*1,$out,$inp),$inout1);
2591 &movaps ($rndkey1,$inout3); # pass the checksum
2592
2593 &jmp (&label("done"));
2594
2595&set_label("three",16);
2596 &lea ($i1,&DWP(1,$block));
2597 &mov ($key,&DWP($key_off,"esp")); # restore key
2598 &bsf ($i1,$i1);
2599 &shl ($i1,4);
2600 &movdqu ($inout3,&QWP(0,$l_));
2601 &movdqu ($inout4,&QWP(0,$l_,$i1));
2602 &movdqa ($inout5,$inout3);
2603
2604 &movdqu ($inout0,&QWP(16*0,$inp)); # load input
2605 &movdqu ($inout1,&QWP(16*1,$inp));
2606 &movdqu ($inout2,&QWP(16*2,$inp));
2607 &mov ($rounds,&DWP(240,$key));
2608
2609 &movdqa (&QWP($checksum,"esp"),$rndkey1);
2610 &pxor ($inout3,$rndkey0); # ^ last offset_i
2611 &pxor ($inout4,$inout3);
2612 &pxor ($inout5,$inout4);
2613
2614 &pxor ($inout0,$inout3); # ^ offset_i
2615 &pxor ($inout1,$inout4);
2616 &pxor ($inout2,$inout5);
2617
2618 &mov ($out,&DWP($out_off,"esp"));
2619 &call ("_aesni_decrypt3");
2620
2621 &movdqa ($rndkey1,&QWP($checksum,"esp"));# pass the checksum
2622 &xorps ($inout0,$inout3); # ^ offset_i
2623 &xorps ($inout1,$inout4);
2624 &xorps ($inout2,$inout5);
2625 &movups (&QWP(16*0,$out,$inp),$inout0); # store output
2626 &pxor ($rndkey1,$inout0); # checksum
2627 &movdqa ($rndkey0,$inout5); # pass last offset_i
2628 &movups (&QWP(16*1,$out,$inp),$inout1);
2629 &pxor ($rndkey1,$inout1);
2630 &movups (&QWP(16*2,$out,$inp),$inout2);
2631 &pxor ($rndkey1,$inout2);
2632
2633 &jmp (&label("done"));
2634
2635&set_label("four",16);
2636 &lea ($i1,&DWP(1,$block));
2637 &lea ($i3,&DWP(3,$block));
2638 &bsf ($i1,$i1);
2639 &bsf ($i3,$i3);
2640 &mov ($key,&DWP($key_off,"esp")); # restore key
2641 &shl ($i1,4);
2642 &shl ($i3,4);
2643 &movdqu ($inout2,&QWP(0,$l_));
2644 &movdqu ($inout3,&QWP(0,$l_,$i1));
2645 &movdqa ($inout4,$inout2);
2646 &movdqu ($inout5,&QWP(0,$l_,$i3));
2647
2648 &pxor ($inout2,$rndkey0); # ^ last offset_i
2649 &movdqu ($inout0,&QWP(16*0,$inp)); # load input
2650 &pxor ($inout3,$inout2);
2651 &movdqu ($inout1,&QWP(16*1,$inp));
2652 &pxor ($inout4,$inout3);
2653 &movdqa (&QWP(16*0,"esp"),$inout2);
2654 &pxor ($inout5,$inout4);
2655 &movdqa (&QWP(16*1,"esp"),$inout3);
2656 &movdqu ($inout2,&QWP(16*2,$inp));
2657 &movdqu ($inout3,&QWP(16*3,$inp));
2658 &mov ($rounds,&DWP(240,$key));
2659
2660 &movdqa (&QWP($checksum,"esp"),$rndkey1);
2661 &pxor ($inout0,&QWP(16*0,"esp")); # ^ offset_i
2662 &pxor ($inout1,&QWP(16*1,"esp"));
2663 &pxor ($inout2,$inout4);
2664 &pxor ($inout3,$inout5);
2665
2666 &mov ($out,&DWP($out_off,"esp"));
2667 &call ("_aesni_decrypt4");
2668
2669 &movdqa ($rndkey1,&QWP($checksum,"esp"));# pass the checksum
2670 &xorps ($inout0,&QWP(16*0,"esp")); # ^ offset_i
2671 &xorps ($inout1,&QWP(16*1,"esp"));
2672 &xorps ($inout2,$inout4);
2673 &movups (&QWP(16*0,$out,$inp),$inout0); # store output
2674 &pxor ($rndkey1,$inout0); # checksum
2675 &xorps ($inout3,$inout5);
2676 &movups (&QWP(16*1,$out,$inp),$inout1);
2677 &pxor ($rndkey1,$inout1);
2678 &movdqa ($rndkey0,$inout5); # pass last offset_i
2679 &movups (&QWP(16*2,$out,$inp),$inout2);
2680 &pxor ($rndkey1,$inout2);
2681 &movups (&QWP(16*3,$out,$inp),$inout3);
2682 &pxor ($rndkey1,$inout3);
2683
2684&set_label("done");
2685 &mov ($key,&DWP($esp_off,"esp"));
2686 &pxor ($inout0,$inout0); # clear register bank
2687 &pxor ($inout1,$inout1);
2688 &movdqa (&QWP(16*0,"esp"),$inout0); # clear stack
2689 &pxor ($inout2,$inout2);
2690 &movdqa (&QWP(16*1,"esp"),$inout0);
2691 &pxor ($inout3,$inout3);
2692 &movdqa (&QWP(16*2,"esp"),$inout0);
2693 &pxor ($inout4,$inout4);
2694 &movdqa (&QWP(16*3,"esp"),$inout0);
2695 &pxor ($inout5,$inout5);
2696 &movdqa (&QWP(16*4,"esp"),$inout0);
2697 &movdqa (&QWP(16*5,"esp"),$inout0);
2698 &movdqa (&QWP(16*6,"esp"),$inout0);
2699
2700 &lea ("esp",&DWP(0,$key));
2701 &mov ($rounds,&wparam(5)); # &offset_i
2702 &mov ($rounds_,&wparam(7)); # &checksum
2703 &movdqu (&QWP(0,$rounds),$rndkey0);
2704 &pxor ($rndkey0,$rndkey0);
2705 &movdqu (&QWP(0,$rounds_),$rndkey1);
2706 &pxor ($rndkey1,$rndkey1);
2707&function_end("aesni_ocb_decrypt");
2708}
6c83629b
AP		 2709}
2710\f
2711######################################################################
d64a7232
AP		 2712# void $PREFIX_cbc_encrypt (const void *inp, void *out,
2713# size_t length, const AES_KEY *key,
2714# unsigned char *ivp,const int enc);
2715&function_begin("${PREFIX}_cbc_encrypt");
2716 &mov ($inp,&wparam(0));
f8501464 2717 &mov ($rounds_,"esp");
d64a7232 2718 &mov ($out,&wparam(1));
f8501464 2719 &sub ($rounds_,24);
d64a7232 2720 &mov ($len,&wparam(2));
f8501464 2721 &and ($rounds_,-16);
d64a7232 2722 &mov ($key,&wparam(3));
d64a7232 2723 &mov ($key_,&wparam(4));
d7d119a3 2724 &test ($len,$len);
f8501464 2725 &jz (&label("cbc_abort"));
d64a7232
AP		 2726
2727 &cmp (&wparam(5),0);
f8501464
AP		 2728 &xchg ($rounds_,"esp"); # alloca
2729 &movups ($ivec,&QWP(0,$key_)); # load IV
d64a7232 2730 &mov ($rounds,&DWP(240,$key));
f8501464
AP		 2731 &mov ($key_,$key); # backup $key
2732 &mov (&DWP(16,"esp"),$rounds_); # save original %esp
2733 &mov ($rounds_,$rounds); # backup $rounds
d64a7232
AP		 2734 &je (&label("cbc_decrypt"));
2735
f8501464 2736 &movaps ($inout0,$ivec);
d64a7232
AP		 2737 &cmp ($len,16);
2738 &jb (&label("cbc_enc_tail"));
2739 &sub ($len,16);
2740 &jmp (&label("cbc_enc_loop"));
2741
2742&set_label("cbc_enc_loop",16);
f8501464 2743 &movups ($ivec,&QWP(0,$inp)); # input actually
d64a7232 2744 &lea ($inp,&DWP(16,$inp));
6f766a41 2745 if ($inline)
f8501464 2746 { &aesni_inline_generate1("enc",$inout0,$ivec); }
6f766a41 2747 else
f8501464 2748 { &xorps($inout0,$ivec); &call("_aesni_encrypt1"); }
d64a7232
AP		 2749 &mov ($rounds,$rounds_); # restore $rounds
2750 &mov ($key,$key_); # restore $key
d7d119a3
AP		 2751 &movups (&QWP(0,$out),$inout0); # store output
2752 &lea ($out,&DWP(16,$out));
2753 &sub ($len,16);
d64a7232
AP		 2754 &jnc (&label("cbc_enc_loop"));
2755 &add ($len,16);
2756 &jnz (&label("cbc_enc_tail"));
2757 &movaps ($ivec,$inout0);
23f6eec7 2758 &pxor ($inout0,$inout0);
d64a7232
AP		 2759 &jmp (&label("cbc_ret"));
2760
2761&set_label("cbc_enc_tail");
2762 &mov ("ecx",$len); # zaps $rounds
2763 &data_word(0xA4F3F689); # rep movsb
2764 &mov ("ecx",16); # zero tail
2765 &sub ("ecx",$len);
2766 &xor ("eax","eax"); # zaps $len
2767 &data_word(0xAAF3F689); # rep stosb
2768 &lea ($out,&DWP(-16,$out)); # rewind $out by 1 block
2769 &mov ($rounds,$rounds_); # restore $rounds
2770 &mov ($inp,$out); # $inp and $out are the same
2771 &mov ($key,$key_); # restore $key
2772 &jmp (&label("cbc_enc_loop"));
6c83629b 2773######################################################################
d64a7232 2774&set_label("cbc_decrypt",16);
f8501464 2775 &cmp ($len,0x50);
d608b4d6 2776 &jbe (&label("cbc_dec_tail"));
f8501464
AP		 2777 &movaps (&QWP(0,"esp"),$ivec); # save IV
2778 &sub ($len,0x50);
2779 &jmp (&label("cbc_dec_loop6_enter"));
d64a7232 2780
f8501464
AP		 2781&set_label("cbc_dec_loop6",16);
2782 &movaps (&QWP(0,"esp"),$rndkey0); # save IV
2783 &movups (&QWP(0,$out),$inout5);
2784 &lea ($out,&DWP(0x10,$out));
2785&set_label("cbc_dec_loop6_enter");
2786 &movdqu ($inout0,&QWP(0,$inp));
2787 &movdqu ($inout1,&QWP(0x10,$inp));
2788 &movdqu ($inout2,&QWP(0x20,$inp));
2789 &movdqu ($inout3,&QWP(0x30,$inp));
2790 &movdqu ($inout4,&QWP(0x40,$inp));
2791 &movdqu ($inout5,&QWP(0x50,$inp));
2792
2793 &call ("_aesni_decrypt6");
2794
2795 &movups ($rndkey1,&QWP(0,$inp));
2796 &movups ($rndkey0,&QWP(0x10,$inp));
2797 &xorps ($inout0,&QWP(0,"esp")); # ^=IV
2798 &xorps ($inout1,$rndkey1);
2799 &movups ($rndkey1,&QWP(0x20,$inp));
2800 &xorps ($inout2,$rndkey0);
2801 &movups ($rndkey0,&QWP(0x30,$inp));
2802 &xorps ($inout3,$rndkey1);
2803 &movups ($rndkey1,&QWP(0x40,$inp));
2804 &xorps ($inout4,$rndkey0);
2805 &movups ($rndkey0,&QWP(0x50,$inp)); # IV
2806 &xorps ($inout5,$rndkey1);
2807 &movups (&QWP(0,$out),$inout0);
2808 &movups (&QWP(0x10,$out),$inout1);
2809 &lea ($inp,&DWP(0x60,$inp));
2810 &movups (&QWP(0x20,$out),$inout2);
f9c5e5d9 2811 &mov ($rounds,$rounds_); # restore $rounds
f8501464
AP		 2812 &movups (&QWP(0x30,$out),$inout3);
2813 &mov ($key,$key_); # restore $key
2814 &movups (&QWP(0x40,$out),$inout4);
2815 &lea ($out,&DWP(0x50,$out));
2816 &sub ($len,0x60);
2817 &ja (&label("cbc_dec_loop6"));
2818
2819 &movaps ($inout0,$inout5);
2820 &movaps ($ivec,$rndkey0);
2821 &add ($len,0x50);
23f6eec7 2822 &jle (&label("cbc_dec_clear_tail_collected"));
f8501464
AP		 2823 &movups (&QWP(0,$out),$inout0);
2824 &lea ($out,&DWP(0x10,$out));
6c83629b 2825&set_label("cbc_dec_tail");
d64a7232 2826 &movups ($inout0,&QWP(0,$inp));
d64a7232 2827 &movaps ($in0,$inout0);
d7d119a3 2828 &cmp ($len,0x10);
d64a7232 2829 &jbe (&label("cbc_dec_one"));
f8501464 2830
d64a7232 2831 &movups ($inout1,&QWP(0x10,$inp));
d64a7232 2832 &movaps ($in1,$inout1);
d7d119a3 2833 &cmp ($len,0x20);
d64a7232 2834 &jbe (&label("cbc_dec_two"));
f8501464 2835
d64a7232 2836 &movups ($inout2,&QWP(0x20,$inp));
d608b4d6
AP		 2837 &cmp ($len,0x30);
2838 &jbe (&label("cbc_dec_three"));
f8501464 2839
d608b4d6 2840 &movups ($inout3,&QWP(0x30,$inp));
f8501464
AP		 2841 &cmp ($len,0x40);
2842 &jbe (&label("cbc_dec_four"));
2843
2844 &movups ($inout4,&QWP(0x40,$inp));
2845 &movaps (&QWP(0,"esp"),$ivec); # save IV
2846 &movups ($inout0,&QWP(0,$inp));
2847 &xorps ($inout5,$inout5);
2848 &call ("_aesni_decrypt6");
2849 &movups ($rndkey1,&QWP(0,$inp));
2850 &movups ($rndkey0,&QWP(0x10,$inp));
2851 &xorps ($inout0,&QWP(0,"esp")); # ^= IV
2852 &xorps ($inout1,$rndkey1);
2853 &movups ($rndkey1,&QWP(0x20,$inp));
2854 &xorps ($inout2,$rndkey0);
2855 &movups ($rndkey0,&QWP(0x30,$inp));
2856 &xorps ($inout3,$rndkey1);
2857 &movups ($ivec,&QWP(0x40,$inp)); # IV
2858 &xorps ($inout4,$rndkey0);
2859 &movups (&QWP(0,$out),$inout0);
2860 &movups (&QWP(0x10,$out),$inout1);
23f6eec7 2861 &pxor ($inout1,$inout1);
f8501464 2862 &movups (&QWP(0x20,$out),$inout2);
23f6eec7 2863 &pxor ($inout2,$inout2);
f8501464 2864 &movups (&QWP(0x30,$out),$inout3);
23f6eec7 2865 &pxor ($inout3,$inout3);
f8501464
AP		 2866 &lea ($out,&DWP(0x40,$out));
2867 &movaps ($inout0,$inout4);
23f6eec7 2868 &pxor ($inout4,$inout4);
f8501464 2869 &sub ($len,0x50);
d64a7232
AP		 2870 &jmp (&label("cbc_dec_tail_collected"));
2871
d7d119a3 2872&set_label("cbc_dec_one",16);
6f766a41
AP		 2873 if ($inline)
2874 { &aesni_inline_generate1("dec"); }
2875 else
2876 { &call ("_aesni_decrypt1"); }
f8501464
AP		 2877 &xorps ($inout0,$ivec);
2878 &movaps ($ivec,$in0);
2879 &sub ($len,0x10);
d64a7232
AP		 2880 &jmp (&label("cbc_dec_tail_collected"));
2881
d7d119a3 2882&set_label("cbc_dec_two",16);
214368ff 2883 &call ("_aesni_decrypt2");
f8501464
AP		 2884 &xorps ($inout0,$ivec);
2885 &xorps ($inout1,$in0);
2886 &movups (&QWP(0,$out),$inout0);
2887 &movaps ($inout0,$inout1);
23f6eec7 2888 &pxor ($inout1,$inout1);
d64a7232 2889 &lea ($out,&DWP(0x10,$out));
f8501464
AP		 2890 &movaps ($ivec,$in1);
2891 &sub ($len,0x20);
d608b4d6
AP		 2892 &jmp (&label("cbc_dec_tail_collected"));
2893
d7d119a3 2894&set_label("cbc_dec_three",16);
d608b4d6 2895 &call ("_aesni_decrypt3");
f8501464
AP		 2896 &xorps ($inout0,$ivec);
2897 &xorps ($inout1,$in0);
2898 &xorps ($inout2,$in1);
2899 &movups (&QWP(0,$out),$inout0);
2900 &movaps ($inout0,$inout2);
23f6eec7 2901 &pxor ($inout2,$inout2);
f8501464 2902 &movups (&QWP(0x10,$out),$inout1);
23f6eec7 2903 &pxor ($inout1,$inout1);
d608b4d6 2904 &lea ($out,&DWP(0x20,$out));
f8501464
AP		 2905 &movups ($ivec,&QWP(0x20,$inp));
2906 &sub ($len,0x30);
2907 &jmp (&label("cbc_dec_tail_collected"));
2908
2909&set_label("cbc_dec_four",16);
2910 &call ("_aesni_decrypt4");
2911 &movups ($rndkey1,&QWP(0x10,$inp));
2912 &movups ($rndkey0,&QWP(0x20,$inp));
2913 &xorps ($inout0,$ivec);
2914 &movups ($ivec,&QWP(0x30,$inp));
2915 &xorps ($inout1,$in0);
2916 &movups (&QWP(0,$out),$inout0);
2917 &xorps ($inout2,$rndkey1);
2918 &movups (&QWP(0x10,$out),$inout1);
23f6eec7 2919 &pxor ($inout1,$inout1);
f8501464
AP		 2920 &xorps ($inout3,$rndkey0);
2921 &movups (&QWP(0x20,$out),$inout2);
23f6eec7 2922 &pxor ($inout2,$inout2);
f8501464
AP		 2923 &lea ($out,&DWP(0x30,$out));
2924 &movaps ($inout0,$inout3);
23f6eec7 2925 &pxor ($inout3,$inout3);
f8501464 2926 &sub ($len,0x40);
23f6eec7 2927 &jmp (&label("cbc_dec_tail_collected"));
d64a7232 2928
23f6eec7
AP		 2929&set_label("cbc_dec_clear_tail_collected",16);
2930 &pxor ($inout1,$inout1);
2931 &pxor ($inout2,$inout2);
2932 &pxor ($inout3,$inout3);
2933 &pxor ($inout4,$inout4);
d64a7232
AP		 2934&set_label("cbc_dec_tail_collected");
2935 &and ($len,15);
2936 &jnz (&label("cbc_dec_tail_partial"));
f8501464 2937 &movups (&QWP(0,$out),$inout0);
23f6eec7 2938 &pxor ($rndkey0,$rndkey0);
d64a7232
AP		 2939 &jmp (&label("cbc_ret"));
2940
d7d119a3 2941&set_label("cbc_dec_tail_partial",16);
f8501464 2942 &movaps (&QWP(0,"esp"),$inout0);
23f6eec7 2943 &pxor ($rndkey0,$rndkey0);
f8501464 2944 &mov ("ecx",16);
d64a7232 2945 &mov ($inp,"esp");
f8501464 2946 &sub ("ecx",$len);
d64a7232 2947 &data_word(0xA4F3F689); # rep movsb
23f6eec7 2948 &movdqa (&QWP(0,"esp"),$inout0);
d64a7232
AP		 2949
2950&set_label("cbc_ret");
f8501464 2951 &mov ("esp",&DWP(16,"esp")); # pull original %esp
d64a7232 2952 &mov ($key_,&wparam(4));
23f6eec7
AP		 2953 &pxor ($inout0,$inout0);
2954 &pxor ($rndkey1,$rndkey1);
d64a7232 2955 &movups (&QWP(0,$key_),$ivec); # output IV
23f6eec7 2956 &pxor ($ivec,$ivec);
f8501464 2957&set_label("cbc_abort");
d64a7232 2958&function_end("${PREFIX}_cbc_encrypt");
6c83629b
AP		 2959\f
2960######################################################################
d64a7232
AP		 2961# Mechanical port from aesni-x86_64.pl.
2962#
2963# _aesni_set_encrypt_key is private interface,
2964# input:
2965# "eax" const unsigned char *userKey
2966# $rounds int bits
2967# $key AES_KEY *key
2968# output:
2969# "eax" return code
2970# $round rounds
2971
2972&function_begin_B("_aesni_set_encrypt_key");
23f6eec7
AP		 2973 &push ("ebp");
2974 &push ("ebx");
d64a7232
AP		 2975 &test ("eax","eax");
2976 &jz (&label("bad_pointer"));
2977 &test ($key,$key);
2978 &jz (&label("bad_pointer"));
2979
23f6eec7
AP		 2980 &call (&label("pic"));
2981&set_label("pic");
2982 &blindpop("ebx");
2983 &lea ("ebx",&DWP(&label("key_const")."-".&label("pic"),"ebx"));
2984
2985 &picmeup("ebp","OPENSSL_ia32cap_P","ebx",&label("key_const"));
d64a7232 2986 &movups ("xmm0",&QWP(0,"eax")); # pull first 128 bits of *userKey
f8501464 2987 &xorps ("xmm4","xmm4"); # low dword of xmm4 is assumed 0
23f6eec7 2988 &mov ("ebp",&DWP(4,"ebp"));
d64a7232 2989 &lea ($key,&DWP(16,$key));
23f6eec7 2990 &and ("ebp",1<<28|1<<11); # AVX and XOP bits
d64a7232
AP		 2991 &cmp ($rounds,256);
2992 &je (&label("14rounds"));
2993 &cmp ($rounds,192);
2994 &je (&label("12rounds"));
2995 &cmp ($rounds,128);
2996 &jne (&label("bad_keybits"));
2997
2998&set_label("10rounds",16);
23f6eec7
AP		 2999 &cmp ("ebp",1<<28);
3000 &je (&label("10rounds_alt"));
3001
d608b4d6 3002 &mov ($rounds,9);
d64a7232
AP		 3003 &$movekey (&QWP(-16,$key),"xmm0"); # round 0
3004 &aeskeygenassist("xmm1","xmm0",0x01); # round 1
3005 &call (&label("key_128_cold"));
3006 &aeskeygenassist("xmm1","xmm0",0x2); # round 2
3007 &call (&label("key_128"));
3008 &aeskeygenassist("xmm1","xmm0",0x04); # round 3
3009 &call (&label("key_128"));
3010 &aeskeygenassist("xmm1","xmm0",0x08); # round 4
3011 &call (&label("key_128"));
3012 &aeskeygenassist("xmm1","xmm0",0x10); # round 5
3013 &call (&label("key_128"));
3014 &aeskeygenassist("xmm1","xmm0",0x20); # round 6
3015 &call (&label("key_128"));
3016 &aeskeygenassist("xmm1","xmm0",0x40); # round 7
3017 &call (&label("key_128"));
3018 &aeskeygenassist("xmm1","xmm0",0x80); # round 8
3019 &call (&label("key_128"));
3020 &aeskeygenassist("xmm1","xmm0",0x1b); # round 9
3021 &call (&label("key_128"));
3022 &aeskeygenassist("xmm1","xmm0",0x36); # round 10
3023 &call (&label("key_128"));
3024 &$movekey (&QWP(0,$key),"xmm0");
3025 &mov (&DWP(80,$key),$rounds);
23f6eec7
AP		 3026
3027 &jmp (&label("good_key"));
d64a7232
AP		 3028
3029&set_label("key_128",16);
3030 &$movekey (&QWP(0,$key),"xmm0");
3031 &lea ($key,&DWP(16,$key));
3032&set_label("key_128_cold");
3033 &shufps ("xmm4","xmm0",0b00010000);
f8501464
AP		 3034 &xorps ("xmm0","xmm4");
3035 &shufps ("xmm4","xmm0",0b10001100);
3036 &xorps ("xmm0","xmm4");
3037 &shufps ("xmm1","xmm1",0b11111111); # critical path
3038 &xorps ("xmm0","xmm1");
d64a7232
AP		 3039 &ret();
3040
23f6eec7
AP		 3041&set_label("10rounds_alt",16);
3042 &movdqa ("xmm5",&QWP(0x00,"ebx"));
3043 &mov ($rounds,8);
3044 &movdqa ("xmm4",&QWP(0x20,"ebx"));
3045 &movdqa ("xmm2","xmm0");
7be6bc68 3046 &movdqu (&QWP(-16,$key),"xmm0");
23f6eec7
AP		 3047
3048&set_label("loop_key128");
3049 &pshufb ("xmm0","xmm5");
3050 &aesenclast ("xmm0","xmm4");
3051 &pslld ("xmm4",1);
3052 &lea ($key,&DWP(16,$key));
3053
3054 &movdqa ("xmm3","xmm2");
3055 &pslldq ("xmm2",4);
3056 &pxor ("xmm3","xmm2");
3057 &pslldq ("xmm2",4);
3058 &pxor ("xmm3","xmm2");
3059 &pslldq ("xmm2",4);
3060 &pxor ("xmm2","xmm3");
3061
3062 &pxor ("xmm0","xmm2");
3063 &movdqu (&QWP(-16,$key),"xmm0");
3064 &movdqa ("xmm2","xmm0");
3065
3066 &dec ($rounds);
3067 &jnz (&label("loop_key128"));
3068
3069 &movdqa ("xmm4",&QWP(0x30,"ebx"));
3070
3071 &pshufb ("xmm0","xmm5");
3072 &aesenclast ("xmm0","xmm4");
3073 &pslld ("xmm4",1);
3074
3075 &movdqa ("xmm3","xmm2");
3076 &pslldq ("xmm2",4);
3077 &pxor ("xmm3","xmm2");
3078 &pslldq ("xmm2",4);
3079 &pxor ("xmm3","xmm2");
3080 &pslldq ("xmm2",4);
3081 &pxor ("xmm2","xmm3");
3082
3083 &pxor ("xmm0","xmm2");
3084 &movdqu (&QWP(0,$key),"xmm0");
3085
3086 &movdqa ("xmm2","xmm0");
3087 &pshufb ("xmm0","xmm5");
3088 &aesenclast ("xmm0","xmm4");
3089
3090 &movdqa ("xmm3","xmm2");
3091 &pslldq ("xmm2",4);
3092 &pxor ("xmm3","xmm2");
3093 &pslldq ("xmm2",4);
3094 &pxor ("xmm3","xmm2");
3095 &pslldq ("xmm2",4);
3096 &pxor ("xmm2","xmm3");
3097
3098 &pxor ("xmm0","xmm2");
3099 &movdqu (&QWP(16,$key),"xmm0");
3100
3101 &mov ($rounds,9);
3102 &mov (&DWP(96,$key),$rounds);
3103
3104 &jmp (&label("good_key"));
3105
d64a7232
AP		 3106&set_label("12rounds",16);
3107 &movq ("xmm2",&QWP(16,"eax")); # remaining 1/3 of *userKey
23f6eec7
AP		 3108 &cmp ("ebp",1<<28);
3109 &je (&label("12rounds_alt"));
3110
d608b4d6 3111 &mov ($rounds,11);
f9c5e5d9 3112 &$movekey (&QWP(-16,$key),"xmm0"); # round 0
d64a7232
AP		 3113 &aeskeygenassist("xmm1","xmm2",0x01); # round 1,2
3114 &call (&label("key_192a_cold"));
3115 &aeskeygenassist("xmm1","xmm2",0x02); # round 2,3
3116 &call (&label("key_192b"));
3117 &aeskeygenassist("xmm1","xmm2",0x04); # round 4,5
3118 &call (&label("key_192a"));
3119 &aeskeygenassist("xmm1","xmm2",0x08); # round 5,6
3120 &call (&label("key_192b"));
3121 &aeskeygenassist("xmm1","xmm2",0x10); # round 7,8
3122 &call (&label("key_192a"));
3123 &aeskeygenassist("xmm1","xmm2",0x20); # round 8,9
3124 &call (&label("key_192b"));
3125 &aeskeygenassist("xmm1","xmm2",0x40); # round 10,11
3126 &call (&label("key_192a"));
3127 &aeskeygenassist("xmm1","xmm2",0x80); # round 11,12
3128 &call (&label("key_192b"));
3129 &$movekey (&QWP(0,$key),"xmm0");
3130 &mov (&DWP(48,$key),$rounds);
23f6eec7
AP		 3131
3132 &jmp (&label("good_key"));
d64a7232
AP		 3133
3134&set_label("key_192a",16);
3135 &$movekey (&QWP(0,$key),"xmm0");
3136 &lea ($key,&DWP(16,$key));
3137&set_label("key_192a_cold",16);
3138 &movaps ("xmm5","xmm2");
3139&set_label("key_192b_warm");
3140 &shufps ("xmm4","xmm0",0b00010000);
f8501464
AP		 3141 &movdqa ("xmm3","xmm2");
3142 &xorps ("xmm0","xmm4");
d64a7232
AP		 3143 &shufps ("xmm4","xmm0",0b10001100);
3144 &pslldq ("xmm3",4);
f8501464 3145 &xorps ("xmm0","xmm4");
d64a7232
AP		 3146 &pshufd ("xmm1","xmm1",0b01010101); # critical path
3147 &pxor ("xmm2","xmm3");
3148 &pxor ("xmm0","xmm1");
3149 &pshufd ("xmm3","xmm0",0b11111111);
3150 &pxor ("xmm2","xmm3");
3151 &ret();
3152
3153&set_label("key_192b",16);
3154 &movaps ("xmm3","xmm0");
3155 &shufps ("xmm5","xmm0",0b01000100);
3156 &$movekey (&QWP(0,$key),"xmm5");
3157 &shufps ("xmm3","xmm2",0b01001110);
3158 &$movekey (&QWP(16,$key),"xmm3");
3159 &lea ($key,&DWP(32,$key));
3160 &jmp (&label("key_192b_warm"));
3161
23f6eec7
AP		 3162&set_label("12rounds_alt",16);
3163 &movdqa ("xmm5",&QWP(0x10,"ebx"));
3164 &movdqa ("xmm4",&QWP(0x20,"ebx"));
3165 &mov ($rounds,8);
3166 &movdqu (&QWP(-16,$key),"xmm0");
3167
3168&set_label("loop_key192");
3169 &movq (&QWP(0,$key),"xmm2");
3170 &movdqa ("xmm1","xmm2");
3171 &pshufb ("xmm2","xmm5");
3172 &aesenclast ("xmm2","xmm4");
3173 &pslld ("xmm4",1);
3174 &lea ($key,&DWP(24,$key));
3175
3176 &movdqa ("xmm3","xmm0");
3177 &pslldq ("xmm0",4);
3178 &pxor ("xmm3","xmm0");
3179 &pslldq ("xmm0",4);
3180 &pxor ("xmm3","xmm0");
3181 &pslldq ("xmm0",4);
3182 &pxor ("xmm0","xmm3");
3183
3184 &pshufd ("xmm3","xmm0",0xff);
3185 &pxor ("xmm3","xmm1");
3186 &pslldq ("xmm1",4);
3187 &pxor ("xmm3","xmm1");
3188
3189 &pxor ("xmm0","xmm2");
3190 &pxor ("xmm2","xmm3");
3191 &movdqu (&QWP(-16,$key),"xmm0");
3192
3193 &dec ($rounds);
3194 &jnz (&label("loop_key192"));
3195
3196 &mov ($rounds,11);
3197 &mov (&DWP(32,$key),$rounds);
3198
3199 &jmp (&label("good_key"));
3200
d64a7232
AP		 3201&set_label("14rounds",16);
3202 &movups ("xmm2",&QWP(16,"eax")); # remaining half of *userKey
d64a7232 3203 &lea ($key,&DWP(16,$key));
23f6eec7
AP		 3204 &cmp ("ebp",1<<28);
3205 &je (&label("14rounds_alt"));
3206
3207 &mov ($rounds,13);
d64a7232
AP		 3208 &$movekey (&QWP(-32,$key),"xmm0"); # round 0
3209 &$movekey (&QWP(-16,$key),"xmm2"); # round 1
3210 &aeskeygenassist("xmm1","xmm2",0x01); # round 2
3211 &call (&label("key_256a_cold"));
3212 &aeskeygenassist("xmm1","xmm0",0x01); # round 3
3213 &call (&label("key_256b"));
3214 &aeskeygenassist("xmm1","xmm2",0x02); # round 4
3215 &call (&label("key_256a"));
3216 &aeskeygenassist("xmm1","xmm0",0x02); # round 5
3217 &call (&label("key_256b"));
3218 &aeskeygenassist("xmm1","xmm2",0x04); # round 6
3219 &call (&label("key_256a"));
3220 &aeskeygenassist("xmm1","xmm0",0x04); # round 7
3221 &call (&label("key_256b"));
3222 &aeskeygenassist("xmm1","xmm2",0x08); # round 8
3223 &call (&label("key_256a"));
3224 &aeskeygenassist("xmm1","xmm0",0x08); # round 9
3225 &call (&label("key_256b"));
3226 &aeskeygenassist("xmm1","xmm2",0x10); # round 10
3227 &call (&label("key_256a"));
3228 &aeskeygenassist("xmm1","xmm0",0x10); # round 11
3229 &call (&label("key_256b"));
3230 &aeskeygenassist("xmm1","xmm2",0x20); # round 12
3231 &call (&label("key_256a"));
3232 &aeskeygenassist("xmm1","xmm0",0x20); # round 13
3233 &call (&label("key_256b"));
3234 &aeskeygenassist("xmm1","xmm2",0x40); # round 14
3235 &call (&label("key_256a"));
3236 &$movekey (&QWP(0,$key),"xmm0");
3237 &mov (&DWP(16,$key),$rounds);
3238 &xor ("eax","eax");
23f6eec7
AP		 3239
3240 &jmp (&label("good_key"));
d64a7232
AP		 3241
3242&set_label("key_256a",16);
3243 &$movekey (&QWP(0,$key),"xmm2");
3244 &lea ($key,&DWP(16,$key));
3245&set_label("key_256a_cold");
3246 &shufps ("xmm4","xmm0",0b00010000);
f8501464 3247 &xorps ("xmm0","xmm4");
d64a7232 3248 &shufps ("xmm4","xmm0",0b10001100);
f8501464
AP		 3249 &xorps ("xmm0","xmm4");
3250 &shufps ("xmm1","xmm1",0b11111111); # critical path
3251 &xorps ("xmm0","xmm1");
d64a7232
AP		 3252 &ret();
3253
3254&set_label("key_256b",16);
3255 &$movekey (&QWP(0,$key),"xmm0");
3256 &lea ($key,&DWP(16,$key));
3257
3258 &shufps ("xmm4","xmm2",0b00010000);
f8501464 3259 &xorps ("xmm2","xmm4");
d64a7232 3260 &shufps ("xmm4","xmm2",0b10001100);
f8501464
AP		 3261 &xorps ("xmm2","xmm4");
3262 &shufps ("xmm1","xmm1",0b10101010); # critical path
3263 &xorps ("xmm2","xmm1");
d64a7232
AP		 3264 &ret();
3265
23f6eec7
AP		 3266&set_label("14rounds_alt",16);
3267 &movdqa ("xmm5",&QWP(0x00,"ebx"));
3268 &movdqa ("xmm4",&QWP(0x20,"ebx"));
3269 &mov ($rounds,7);
3270 &movdqu (&QWP(-32,$key),"xmm0");
3271 &movdqa ("xmm1","xmm2");
3272 &movdqu (&QWP(-16,$key),"xmm2");
3273
3274&set_label("loop_key256");
3275 &pshufb ("xmm2","xmm5");
3276 &aesenclast ("xmm2","xmm4");
3277
3278 &movdqa ("xmm3","xmm0");
3279 &pslldq ("xmm0",4);
3280 &pxor ("xmm3","xmm0");
3281 &pslldq ("xmm0",4);
3282 &pxor ("xmm3","xmm0");
3283 &pslldq ("xmm0",4);
3284 &pxor ("xmm0","xmm3");
3285 &pslld ("xmm4",1);
3286
3287 &pxor ("xmm0","xmm2");
3288 &movdqu (&QWP(0,$key),"xmm0");
3289
3290 &dec ($rounds);
3291 &jz (&label("done_key256"));
3292
3293 &pshufd ("xmm2","xmm0",0xff);
3294 &pxor ("xmm3","xmm3");
3295 &aesenclast ("xmm2","xmm3");
3296
bd30091c 3297 &movdqa ("xmm3","xmm1");
23f6eec7
AP		 3298 &pslldq ("xmm1",4);
3299 &pxor ("xmm3","xmm1");
3300 &pslldq ("xmm1",4);
3301 &pxor ("xmm3","xmm1");
3302 &pslldq ("xmm1",4);
3303 &pxor ("xmm1","xmm3");
3304
3305 &pxor ("xmm2","xmm1");
3306 &movdqu (&QWP(16,$key),"xmm2");
3307 &lea ($key,&DWP(32,$key));
3308 &movdqa ("xmm1","xmm2");
3309 &jmp (&label("loop_key256"));
3310
3311&set_label("done_key256");
3312 &mov ($rounds,13);
3313 &mov (&DWP(16,$key),$rounds);
3314
3315&set_label("good_key");
3316 &pxor ("xmm0","xmm0");
3317 &pxor ("xmm1","xmm1");
3318 &pxor ("xmm2","xmm2");
3319 &pxor ("xmm3","xmm3");
3320 &pxor ("xmm4","xmm4");
3321 &pxor ("xmm5","xmm5");
3322 &xor ("eax","eax");
3323 &pop ("ebx");
3324 &pop ("ebp");
3325 &ret ();
3326
d64a7232
AP		 3327&set_label("bad_pointer",4);
3328 &mov ("eax",-1);
23f6eec7
AP		 3329 &pop ("ebx");
3330 &pop ("ebp");
d64a7232
AP		 3331 &ret ();
3332&set_label("bad_keybits",4);
23f6eec7 3333 &pxor ("xmm0","xmm0");
d64a7232 3334 &mov ("eax",-2);
23f6eec7
AP		 3335 &pop ("ebx");
3336 &pop ("ebp");
d64a7232
AP		 3337 &ret ();
3338&function_end_B("_aesni_set_encrypt_key");
3339
3340# int $PREFIX_set_encrypt_key (const unsigned char *userKey, int bits,
3341# AES_KEY *key)
3342&function_begin_B("${PREFIX}_set_encrypt_key");
3343 &mov ("eax",&wparam(0));
3344 &mov ($rounds,&wparam(1));
3345 &mov ($key,&wparam(2));
3346 &call ("_aesni_set_encrypt_key");
3347 &ret ();
3348&function_end_B("${PREFIX}_set_encrypt_key");
3349
3350# int $PREFIX_set_decrypt_key (const unsigned char *userKey, int bits,
3351# AES_KEY *key)
3352&function_begin_B("${PREFIX}_set_decrypt_key");
3353 &mov ("eax",&wparam(0));
3354 &mov ($rounds,&wparam(1));
3355 &mov ($key,&wparam(2));
3356 &call ("_aesni_set_encrypt_key");
3357 &mov ($key,&wparam(2));
f9c5e5d9 3358 &shl ($rounds,4); # rounds-1 after _aesni_set_encrypt_key
d64a7232
AP		 3359 &test ("eax","eax");
3360 &jnz (&label("dec_key_ret"));
d608b4d6 3361 &lea ("eax",&DWP(16,$key,$rounds)); # end of key schedule
d64a7232
AP		 3362
3363 &$movekey ("xmm0",&QWP(0,$key)); # just swap
3364 &$movekey ("xmm1",&QWP(0,"eax"));
3365 &$movekey (&QWP(0,"eax"),"xmm0");
3366 &$movekey (&QWP(0,$key),"xmm1");
3367 &lea ($key,&DWP(16,$key));
3368 &lea ("eax",&DWP(-16,"eax"));
d64a7232 3369
d608b4d6 3370&set_label("dec_key_inverse");
d64a7232
AP		 3371 &$movekey ("xmm0",&QWP(0,$key)); # swap and inverse
3372 &$movekey ("xmm1",&QWP(0,"eax"));
3373 &aesimc ("xmm0","xmm0");
3374 &aesimc ("xmm1","xmm1");
3375 &lea ($key,&DWP(16,$key));
3376 &lea ("eax",&DWP(-16,"eax"));
d64a7232
AP		 3377 &$movekey (&QWP(16,"eax"),"xmm0");
3378 &$movekey (&QWP(-16,$key),"xmm1");
d7d119a3 3379 &cmp ("eax",$key);
d64a7232
AP		 3380 &ja (&label("dec_key_inverse"));
3381
3382 &$movekey ("xmm0",&QWP(0,$key)); # inverse middle
3383 &aesimc ("xmm0","xmm0");
3384 &$movekey (&QWP(0,$key),"xmm0");
3385
23f6eec7
AP		 3386 &pxor ("xmm0","xmm0");
3387 &pxor ("xmm1","xmm1");
d64a7232
AP		 3388 &xor ("eax","eax"); # return success
3389&set_label("dec_key_ret");
3390 &ret ();
3391&function_end_B("${PREFIX}_set_decrypt_key");
23f6eec7
AP		 3392
3393&set_label("key_const",64);
3394&data_word(0x0c0f0e0d,0x0c0f0e0d,0x0c0f0e0d,0x0c0f0e0d);
3395&data_word(0x04070605,0x04070605,0x04070605,0x04070605);
3396&data_word(1,1,1,1);
3397&data_word(0x1b,0x1b,0x1b,0x1b);
d64a7232
AP		 3398&asciz("AES for Intel AES-NI, CRYPTOGAMS by <appro\@openssl.org>");
3399
3400&asm_finish();
A mirror of the official openssl repository