2 # Copyright 2014-2018 The OpenSSL Project Authors. All Rights Reserved.
3 # Copyright (c) 2014, Intel Corporation. All Rights Reserved.
4 # Copyright (c) 2015 CloudFlare, Inc.
6 # Licensed under the Apache License 2.0 (the "License"). You may not use
7 # this file except in compliance with the License. You can obtain a copy
8 # in the file LICENSE in the source distribution or at
9 # https://www.openssl.org/source/license.html
11 # Originally written by Shay Gueron (1, 2), and Vlad Krasnov (1, 3)
12 # (1) Intel Corporation, Israel Development Center, Haifa, Israel
13 # (2) University of Haifa, Israel
14 # (3) CloudFlare, Inc.
17 # S.Gueron and V.Krasnov, "Fast Prime Field Elliptic Curve Cryptography with
20 # Further optimization by <appro@openssl.org>:
22 # this/original with/without -DECP_NISTZ256_ASM(*)
23 # Opteron +15-49% +150-195%
24 # Bulldozer +18-45% +175-240%
25 # P4 +24-46% +100-150%
26 # Westmere +18-34% +87-160%
27 # Sandy Bridge +14-35% +120-185%
28 # Ivy Bridge +11-35% +125-180%
29 # Haswell +10-37% +160-200%
30 # Broadwell +24-58% +210-270%
31 # Atom +20-50% +180-240%
32 # VIA Nano +50-160% +480-480%
34 # (*) "without -DECP_NISTZ256_ASM" refers to build with
35 # "enable-ec_nistp_64_gcc_128";
37 # Ranges denote minimum and maximum improvement coefficients depending
38 # on benchmark. In "this/original" column lower coefficient is for
39 # ECDSA sign, while in "with/without" - for ECDH key agreement, and
40 # higher - for ECDSA sign, relatively fastest server-side operation.
41 # Keep in mind that +100% means 2x improvement.
43 # $output is the last argument if it looks like a file (it has an extension)
44 # $flavour is the first argument if it doesn't look like a file
45 $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m
|\
.\w
+$| ?
pop : undef;
46 $flavour = $#ARGV >= 0 && $ARGV[0] !~ m
|\
.| ?
shift : undef;
48 $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
50 $0 =~ m/(.*[\/\\])[^\
/\\]+$/; $dir=$1;
51 ( $xlate="${dir}x86_64-xlate.pl" and -f
$xlate ) or
52 ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f
$xlate) or
53 die "can't locate x86_64-xlate.pl";
55 open OUT
,"| \"$^X\" \"$xlate\" $flavour \"$output\""
56 or die "can't call $xlate: $!";
59 if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
60 =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
61 $avx = ($1>=2.19) + ($1>=2.22);
65 if (!$addx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM
} =~ /nasm/) &&
66 `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
67 $avx = ($1>=2.09) + ($1>=2.10);
71 if (!$addx && $win64 && ($flavour =~ /masm/ || $ENV{ASM
} =~ /ml64/) &&
72 `ml64 2>&1` =~ /Version ([0-9]+)\./) {
73 $avx = ($1>=10) + ($1>=11);
77 if (!$addx && `$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|.*based on LLVM) ([0-9]+)\.([0-9]+)/) {
78 my $ver = $2 + $3/100.0; # 3.1->3.01, 3.10->3.10
79 $avx = ($ver>=3.0) + ($ver>=3.01);
85 .extern OPENSSL_ia32cap_P
90 .quad
0xffffffffffffffff, 0x00000000ffffffff, 0x0000000000000000, 0xffffffff00000001
92 # 2^512 mod P precomputed for NIST P256 polynomial
94 .quad
0x0000000000000003, 0xfffffffbffffffff, 0xfffffffffffffffe, 0x00000004fffffffd
101 .long
3,3,3,3,3,3,3,3
103 .quad
0x0000000000000001, 0xffffffff00000000, 0xffffffffffffffff, 0x00000000fffffffe
105 # Constants for computations modulo ord(p256)
107 .quad
0xf3b9cac2fc632551, 0xbce6faada7179e84, 0xffffffffffffffff, 0xffffffff00000000
109 .quad
0xccd1c8aaee00bc4f
113 ################################################################################
114 # void ecp_nistz256_mul_by_2(uint64_t res[4], uint64_t a[4]);
116 my ($a0,$a1,$a2,$a3)=map("%r$_",(8..11));
117 my ($t0,$t1,$t2,$t3,$t4)=("%rax","%rdx","%rcx","%r12","%r13");
118 my ($r_ptr,$a_ptr,$b_ptr)=("%rdi","%rsi","%rdx");
122 .globl ecp_nistz256_mul_by_2
123 .type ecp_nistz256_mul_by_2
,\
@function,2
125 ecp_nistz256_mul_by_2
:
136 add
$a0, $a0 # a0:a3+a0:a3
140 lea
.Lpoly
(%rip), $a_ptr
169 .cfi_adjust_cfa_offset
-16
173 .size ecp_nistz256_mul_by_2
,.-ecp_nistz256_mul_by_2
175 ################################################################################
176 # void ecp_nistz256_div_by_2(uint64_t res[4], uint64_t a[4]);
177 .globl ecp_nistz256_div_by_2
178 .type ecp_nistz256_div_by_2
,\
@function,2
180 ecp_nistz256_div_by_2
:
193 lea
.Lpoly
(%rip), $a_ptr
204 xor $a_ptr, $a_ptr # borrow $a_ptr
213 mov
$a1, $t0 # a0:a3>>1
239 .cfi_adjust_cfa_offset
-16
243 .size ecp_nistz256_div_by_2
,.-ecp_nistz256_div_by_2
245 ################################################################################
246 # void ecp_nistz256_mul_by_3(uint64_t res[4], uint64_t a[4]);
247 .globl ecp_nistz256_mul_by_3
248 .type ecp_nistz256_mul_by_3
,\
@function,2
250 ecp_nistz256_mul_by_3
:
261 add
$a0, $a0 # a0:a3+a0:a3
273 sbb
.Lpoly
+8*1(%rip), $a1
276 sbb
.Lpoly
+8*3(%rip), $a3
285 add
8*0($a_ptr), $a0 # a0:a3+=a_ptr[0:3]
295 sbb
.Lpoly
+8*1(%rip), $a1
298 sbb
.Lpoly
+8*3(%rip), $a3
315 .cfi_adjust_cfa_offset
-16
319 .size ecp_nistz256_mul_by_3
,.-ecp_nistz256_mul_by_3
321 ################################################################################
322 # void ecp_nistz256_add(uint64_t res[4], uint64_t a[4], uint64_t b[4]);
323 .globl ecp_nistz256_add
324 .type ecp_nistz256_add
,\
@function,3
339 lea
.Lpoly
(%rip), $a_ptr
371 .cfi_adjust_cfa_offset
-16
375 .size ecp_nistz256_add
,.-ecp_nistz256_add
377 ################################################################################
378 # void ecp_nistz256_sub(uint64_t res[4], uint64_t a[4], uint64_t b[4]);
379 .globl ecp_nistz256_sub
380 .type ecp_nistz256_sub
,\
@function,3
395 lea
.Lpoly
(%rip), $a_ptr
427 .cfi_adjust_cfa_offset
-16
431 .size ecp_nistz256_sub
,.-ecp_nistz256_sub
433 ################################################################################
434 # void ecp_nistz256_neg(uint64_t res[4], uint64_t a[4]);
435 .globl ecp_nistz256_neg
436 .type ecp_nistz256_neg
,\
@function,2
457 lea
.Lpoly
(%rip), $a_ptr
483 .cfi_adjust_cfa_offset
-16
487 .size ecp_nistz256_neg
,.-ecp_nistz256_neg
491 my ($r_ptr,$a_ptr,$b_org,$b_ptr)=("%rdi","%rsi","%rdx","%rbx");
492 my ($acc0,$acc1,$acc2,$acc3,$acc4,$acc5,$acc6,$acc7)=map("%r$_",(8..15));
493 my ($t0,$t1,$t2,$t3,$t4)=("%rcx","%rbp","%rbx","%rdx","%rax");
494 my ($poly1,$poly3)=($acc6,$acc7);
497 ################################################################################
498 # void ecp_nistz256_ord_mul_mont(
503 .globl ecp_nistz256_ord_mul_mont
504 .type ecp_nistz256_ord_mul_mont
,\
@function,3
506 ecp_nistz256_ord_mul_mont
:
509 $code.=<<___
if ($addx);
511 and OPENSSL_ia32cap_P
+8(%rip), %ecx
513 je
.Lecp_nistz256_ord_mul_montx
530 mov
8*0($b_org), %rax
532 lea
.Lord
(%rip), %r14
533 mov
.LordK
(%rip), %r15
535 ################################# * b[0]
563 ################################# First reduction step
566 add
%rax, $acc5 # guaranteed to be zero
572 sbb \
$0, $acc0 # can't borrow
581 adc \
$0, $acc0 # can't overflow
586 mov
8*1($b_ptr), %rax
587 sbb
%rdx, $t1 # can't borrow
593 ################################# * b[1]
629 ################################# Second reduction step
632 add
%rax, $t0 # guaranteed to be zero
637 sbb \
$0, $acc1 # can't borrow
646 adc \
$0, $acc1 # can't overflow
651 mov
8*2($b_ptr), %rax
652 sbb
%rdx, $t1 # can't borrow
658 ################################## * b[2]
694 ################################# Third reduction step
697 add
%rax, $t0 # guaranteed to be zero
702 sbb \
$0, $acc2 # can't borrow
711 adc \
$0, $acc2 # can't overflow
716 mov
8*3($b_ptr), %rax
717 sbb
%rdx, $t1 # can't borrow
723 ################################# * b[3]
759 ################################# Last reduction step
762 add
%rax, $t0 # guaranteed to be zero
767 sbb \
$0, $acc3 # can't borrow
776 adc \
$0, $acc3 # can't overflow
781 sbb
%rdx, $t1 # can't borrow
787 ################################# Subtract ord
803 mov
$acc4, 8*0($r_ptr)
804 mov
$acc5, 8*1($r_ptr)
805 mov
$acc0, 8*2($r_ptr)
806 mov
$acc1, 8*3($r_ptr)
821 .cfi_adjust_cfa_offset
-48
825 .size ecp_nistz256_ord_mul_mont
,.-ecp_nistz256_ord_mul_mont
827 ################################################################################
828 # void ecp_nistz256_ord_sqr_mont(
833 .globl ecp_nistz256_ord_sqr_mont
834 .type ecp_nistz256_ord_sqr_mont
,\
@function,3
836 ecp_nistz256_ord_sqr_mont
:
839 $code.=<<___
if ($addx);
841 and OPENSSL_ia32cap_P
+8(%rip), %ecx
843 je
.Lecp_nistz256_ord_sqr_montx
860 mov
8*0($a_ptr), $acc0
861 mov
8*1($a_ptr), %rax
862 mov
8*2($a_ptr), $acc6
863 mov
8*3($a_ptr), $acc7
864 lea
.Lord
(%rip), $a_ptr # pointer to modulus
870 ################################# a[1:] * a[0]
871 mov
%rax, $t1 # put aside a[1]
872 mul
$acc0 # a[1] * a[0]
874 movq
$t1, %xmm1 # offload a[1]
878 mul
$acc0 # a[2] * a[0]
881 movq
$acc6, %xmm2 # offload a[2]
885 mul
$acc0 # a[3] * a[0]
888 movq
$acc7, %xmm3 # offload a[3]
892 ################################# a[3] * a[2]
893 mul
$acc6 # a[3] * a[2]
898 ################################# a[2:] * a[1]
899 mul
$t1 # a[2] * a[1]
905 mul
$t1 # a[3] * a[1]
911 adc \
$0, $acc6 # can't overflow
913 ################################# *2
924 ################################# Missing products
925 mul
%rax # a[0] * a[0]
930 mul
%rax # a[1] * a[1]
937 mul
%rax # a[2] * a[2]
945 imulq
8*4($a_ptr), $acc0 # *= .LordK
947 mul
%rax # a[3] * a[3]
950 mov
8*0($a_ptr), %rax # modulus[0]
951 adc
%rdx, $acc7 # can't overflow
953 ################################# First reduction step
956 add
%rax, $t0 # guaranteed to be zero
957 mov
8*1($a_ptr), %rax # modulus[1]
961 sbb \
$0, $t1 # can't borrow
970 adc \
$0, $t1 # can't overflow
973 imulq
8*4($a_ptr), $acc1 # *= .LordK
978 mov
8*0($a_ptr), %rax
979 sbb
%rdx, $acc0 # can't borrow
982 adc \
$0, $acc0 # can't overflow
984 ################################# Second reduction step
987 add
%rax, $t0 # guaranteed to be zero
988 mov
8*1($a_ptr), %rax
992 sbb \
$0, $t1 # can't borrow
1001 adc \
$0, $t1 # can't overflow
1004 imulq
8*4($a_ptr), $acc2 # *= .LordK
1009 mov
8*0($a_ptr), %rax
1010 sbb
%rdx, $acc1 # can't borrow
1013 adc \
$0, $acc1 # can't overflow
1015 ################################# Third reduction step
1018 add
%rax, $t0 # guaranteed to be zero
1019 mov
8*1($a_ptr), %rax
1023 sbb \
$0, $t1 # can't borrow
1032 adc \
$0, $t1 # can't overflow
1035 imulq
8*4($a_ptr), $acc3 # *= .LordK
1040 mov
8*0($a_ptr), %rax
1041 sbb
%rdx, $acc2 # can't borrow
1044 adc \
$0, $acc2 # can't overflow
1046 ################################# Last reduction step
1049 add
%rax, $t0 # guaranteed to be zero
1050 mov
8*1($a_ptr), %rax
1054 sbb \
$0, $t1 # can't borrow
1063 adc \
$0, $t1 # can't overflow
1068 sbb
%rdx, $acc3 # can't borrow
1071 adc \
$0, $acc3 # can't overflow
1073 ################################# Add bits [511:256] of the sqr result
1083 ################################# Compare to modulus
1084 sub 8*0($a_ptr), $acc0
1086 sbb
8*1($a_ptr), $acc1
1087 sbb
8*2($a_ptr), $acc2
1089 sbb
8*3($a_ptr), $acc3
1100 mov
$acc0, 8*0($r_ptr)
1101 mov
%rax, 8*1($r_ptr)
1103 mov
$acc6, 8*2($r_ptr)
1105 mov
$acc7, 8*3($r_ptr)
1121 .cfi_adjust_cfa_offset
-48
1125 .size ecp_nistz256_ord_sqr_mont
,.-ecp_nistz256_ord_sqr_mont
1128 $code.=<<___
if ($addx);
1129 ################################################################################
1130 .type ecp_nistz256_ord_mul_montx
,\
@function,3
1132 ecp_nistz256_ord_mul_montx
:
1134 .Lecp_nistz256_ord_mul_montx
:
1150 mov
8*0($b_org), %rdx
1151 mov
8*0($a_ptr), $acc1
1152 mov
8*1($a_ptr), $acc2
1153 mov
8*2($a_ptr), $acc3
1154 mov
8*3($a_ptr), $acc4
1155 lea
-128($a_ptr), $a_ptr # control u-op density
1156 lea
.Lord
-128(%rip), %r14
1157 mov
.LordK
(%rip), %r15
1159 ################################# Multiply by b[0]
1160 mulx
$acc1, $acc0, $acc1
1161 mulx
$acc2, $t0, $acc2
1162 mulx
$acc3, $t1, $acc3
1164 mulx
$acc4, $t0, $acc4
1166 mulx
%r15, %rdx, %rax
1171 ################################# reduction
1172 xor $acc5, $acc5 # $acc5=0, cf=0, of=0
1173 mulx
8*0+128(%r14), $t0, $t1
1174 adcx
$t0, $acc0 # guaranteed to be zero
1177 mulx
8*1+128(%r14), $t0, $t1
1181 mulx
8*2+128(%r14), $t0, $t1
1185 mulx
8*3+128(%r14), $t0, $t1
1186 mov
8*1($b_ptr), %rdx
1191 adc \
$0, $acc5 # cf=0, of=0
1193 ################################# Multiply by b[1]
1194 mulx
8*0+128($a_ptr), $t0, $t1
1198 mulx
8*1+128($a_ptr), $t0, $t1
1202 mulx
8*2+128($a_ptr), $t0, $t1
1206 mulx
8*3+128($a_ptr), $t0, $t1
1208 mulx
%r15, %rdx, %rax
1214 adc \
$0, $acc0 # cf=0, of=0
1216 ################################# reduction
1217 mulx
8*0+128(%r14), $t0, $t1
1218 adcx
$t0, $acc1 # guaranteed to be zero
1221 mulx
8*1+128(%r14), $t0, $t1
1225 mulx
8*2+128(%r14), $t0, $t1
1229 mulx
8*3+128(%r14), $t0, $t1
1230 mov
8*2($b_ptr), %rdx
1235 adc \
$0, $acc0 # cf=0, of=0
1237 ################################# Multiply by b[2]
1238 mulx
8*0+128($a_ptr), $t0, $t1
1242 mulx
8*1+128($a_ptr), $t0, $t1
1246 mulx
8*2+128($a_ptr), $t0, $t1
1250 mulx
8*3+128($a_ptr), $t0, $t1
1252 mulx
%r15, %rdx, %rax
1258 adc \
$0, $acc1 # cf=0, of=0
1260 ################################# reduction
1261 mulx
8*0+128(%r14), $t0, $t1
1262 adcx
$t0, $acc2 # guaranteed to be zero
1265 mulx
8*1+128(%r14), $t0, $t1
1269 mulx
8*2+128(%r14), $t0, $t1
1273 mulx
8*3+128(%r14), $t0, $t1
1274 mov
8*3($b_ptr), %rdx
1279 adc \
$0, $acc1 # cf=0, of=0
1281 ################################# Multiply by b[3]
1282 mulx
8*0+128($a_ptr), $t0, $t1
1286 mulx
8*1+128($a_ptr), $t0, $t1
1290 mulx
8*2+128($a_ptr), $t0, $t1
1294 mulx
8*3+128($a_ptr), $t0, $t1
1296 mulx
%r15, %rdx, %rax
1302 adc \
$0, $acc2 # cf=0, of=0
1304 ################################# reduction
1305 mulx
8*0+128(%r14), $t0, $t1
1306 adcx
$t0, $acc3 # guaranteed to be zero
1309 mulx
8*1+128(%r14), $t0, $t1
1313 mulx
8*2+128(%r14), $t0, $t1
1317 mulx
8*3+128(%r14), $t0, $t1
1327 #################################
1328 # Branch-less conditional subtraction of P
1330 sub 8*0(%r14), $acc4
1331 sbb
8*1(%r14), $acc5
1332 sbb
8*2(%r14), $acc0
1334 sbb
8*3(%r14), $acc1
1342 mov
$acc4, 8*0($r_ptr)
1343 mov
$acc5, 8*1($r_ptr)
1344 mov
$acc0, 8*2($r_ptr)
1345 mov
$acc1, 8*3($r_ptr)
1360 .cfi_adjust_cfa_offset
-48
1361 .Lord_mulx_epilogue
:
1364 .size ecp_nistz256_ord_mul_montx
,.-ecp_nistz256_ord_mul_montx
1366 .type ecp_nistz256_ord_sqr_montx
,\
@function,3
1368 ecp_nistz256_ord_sqr_montx
:
1370 .Lecp_nistz256_ord_sqr_montx
:
1386 mov
8*0($a_ptr), %rdx
1387 mov
8*1($a_ptr), $acc6
1388 mov
8*2($a_ptr), $acc7
1389 mov
8*3($a_ptr), $acc0
1390 lea
.Lord
(%rip), $a_ptr
1395 mulx
$acc6, $acc1, $acc2 # a[0]*a[1]
1396 mulx
$acc7, $t0, $acc3 # a[0]*a[2]
1397 mov
%rdx, %rax # offload a[0]
1398 movq
$acc6, %xmm1 # offload a[1]
1399 mulx
$acc0, $t1, $acc4 # a[0]*a[3]
1402 movq
$acc7, %xmm2 # offload a[2]
1405 xor $acc5, $acc5 # $acc5=0,cf=0,of=0
1406 #################################
1407 mulx
$acc7, $t0, $t1 # a[1]*a[2]
1411 mulx
$acc0, $t0, $t1 # a[1]*a[3]
1416 #################################
1417 mulx
$acc0, $t0, $acc6 # a[2]*a[3]
1419 movq
$acc0, %xmm3 # offload a[3]
1420 xor $acc7, $acc7 # $acc7=0,cf=0,of=0
1421 adcx
$acc1, $acc1 # acc1:6<<1
1424 adox
$acc7, $acc6 # of=0
1426 ################################# a[i]*a[i]
1427 mulx
%rdx, $acc0, $t1
1448 ################################# reduction
1450 mulx
8*4($a_ptr), %rdx, $t0
1452 xor %rax, %rax # cf=0, of=0
1453 mulx
8*0($a_ptr), $t0, $t1
1454 adcx
$t0, $acc0 # guaranteed to be zero
1456 mulx
8*1($a_ptr), $t0, $t1
1459 mulx
8*2($a_ptr), $t0, $t1
1462 mulx
8*3($a_ptr), $t0, $t1
1464 adox
$t1, $acc0 # of=0
1465 adcx
%rax, $acc0 # cf=0
1467 #################################
1469 mulx
8*4($a_ptr), %rdx, $t0
1471 mulx
8*0($a_ptr), $t0, $t1
1472 adox
$t0, $acc1 # guaranteed to be zero
1474 mulx
8*1($a_ptr), $t0, $t1
1477 mulx
8*2($a_ptr), $t0, $t1
1480 mulx
8*3($a_ptr), $t0, $t1
1482 adcx
$t1, $acc1 # cf=0
1483 adox
%rax, $acc1 # of=0
1485 #################################
1487 mulx
8*4($a_ptr), %rdx, $t0
1489 mulx
8*0($a_ptr), $t0, $t1
1490 adcx
$t0, $acc2 # guaranteed to be zero
1492 mulx
8*1($a_ptr), $t0, $t1
1495 mulx
8*2($a_ptr), $t0, $t1
1498 mulx
8*3($a_ptr), $t0, $t1
1500 adox
$t1, $acc2 # of=0
1501 adcx
%rax, $acc2 # cf=0
1503 #################################
1505 mulx
8*4($a_ptr), %rdx, $t0
1507 mulx
8*0($a_ptr), $t0, $t1
1508 adox
$t0, $acc3 # guaranteed to be zero
1510 mulx
8*1($a_ptr), $t0, $t1
1513 mulx
8*2($a_ptr), $t0, $t1
1516 mulx
8*3($a_ptr), $t0, $t1
1521 ################################# accumulate upper half
1522 add
$acc0, $acc4 # add $acc4, $acc0
1530 ################################# compare to modulus
1531 sub 8*0($a_ptr), $acc4
1533 sbb
8*1($a_ptr), $acc1
1534 sbb
8*2($a_ptr), $acc2
1536 sbb
8*3($a_ptr), $acc3
1547 mov
%rdx, 8*0($r_ptr)
1548 mov
$acc6, 8*1($r_ptr)
1550 mov
$acc7, 8*2($r_ptr)
1552 mov
$acc0, 8*3($r_ptr)
1568 .cfi_adjust_cfa_offset
-48
1569 .Lord_sqrx_epilogue
:
1572 .size ecp_nistz256_ord_sqr_montx
,.-ecp_nistz256_ord_sqr_montx
1576 ################################################################################
1577 # void ecp_nistz256_to_mont(
1580 .globl ecp_nistz256_to_mont
1581 .type ecp_nistz256_to_mont
,\
@function,2
1583 ecp_nistz256_to_mont
:
1586 $code.=<<___
if ($addx);
1588 and OPENSSL_ia32cap_P
+8(%rip), %ecx
1591 lea
.LRR
(%rip), $b_org
1594 .size ecp_nistz256_to_mont
,.-ecp_nistz256_to_mont
1596 ################################################################################
1597 # void ecp_nistz256_mul_mont(
1602 .globl ecp_nistz256_mul_mont
1603 .type ecp_nistz256_mul_mont
,\
@function,3
1605 ecp_nistz256_mul_mont
:
1608 $code.=<<___
if ($addx);
1610 and OPENSSL_ia32cap_P
+8(%rip), %ecx
1628 $code.=<<___
if ($addx);
1634 mov
8*0($b_org), %rax
1635 mov
8*0($a_ptr), $acc1
1636 mov
8*1($a_ptr), $acc2
1637 mov
8*2($a_ptr), $acc3
1638 mov
8*3($a_ptr), $acc4
1640 call __ecp_nistz256_mul_montq
1642 $code.=<<___
if ($addx);
1648 mov
8*0($b_org), %rdx
1649 mov
8*0($a_ptr), $acc1
1650 mov
8*1($a_ptr), $acc2
1651 mov
8*2($a_ptr), $acc3
1652 mov
8*3($a_ptr), $acc4
1653 lea
-128($a_ptr), $a_ptr # control u-op density
1655 call __ecp_nistz256_mul_montx
1672 .cfi_adjust_cfa_offset
-48
1676 .size ecp_nistz256_mul_mont
,.-ecp_nistz256_mul_mont
1678 .type __ecp_nistz256_mul_montq
,\
@abi-omnipotent
1680 __ecp_nistz256_mul_montq
:
1682 ########################################################################
1683 # Multiply a by b[0]
1686 mov
.Lpoly
+8*1(%rip),$poly1
1692 mov
.Lpoly
+8*3(%rip),$poly3
1711 ########################################################################
1712 # First reduction step
1713 # Basically now we want to multiply acc[0] by p256,
1714 # and add the result to the acc.
1715 # Due to the special form of p256 we do some optimizations
1717 # acc[0] x p256[0..1] = acc[0] x 2^96 - acc[0]
1718 # then we add acc[0] and get acc[0] x 2^96
1724 add
$acc0, $acc1 # +=acc[0]<<96
1727 mov
8*1($b_ptr), %rax
1732 ########################################################################
1765 ########################################################################
1766 # Second reduction step
1774 mov
8*2($b_ptr), %rax
1779 ########################################################################
1812 ########################################################################
1813 # Third reduction step
1821 mov
8*3($b_ptr), %rax
1826 ########################################################################
1859 ########################################################################
1860 # Final reduction step
1873 ########################################################################
1874 # Branch-less conditional subtraction of P
1875 sub \
$-1, $acc4 # .Lpoly[0]
1877 sbb
$poly1, $acc5 # .Lpoly[1]
1878 sbb \
$0, $acc0 # .Lpoly[2]
1880 sbb
$poly3, $acc1 # .Lpoly[3]
1885 mov
$acc4, 8*0($r_ptr)
1887 mov
$acc5, 8*1($r_ptr)
1889 mov
$acc0, 8*2($r_ptr)
1890 mov
$acc1, 8*3($r_ptr)
1894 .size __ecp_nistz256_mul_montq
,.-__ecp_nistz256_mul_montq
1896 ################################################################################
1897 # void ecp_nistz256_sqr_mont(
1901 # we optimize the square according to S.Gueron and V.Krasnov,
1902 # "Speeding up Big-Number Squaring"
1903 .globl ecp_nistz256_sqr_mont
1904 .type ecp_nistz256_sqr_mont
,\
@function,2
1906 ecp_nistz256_sqr_mont
:
1909 $code.=<<___
if ($addx);
1911 and OPENSSL_ia32cap_P
+8(%rip), %ecx
1928 $code.=<<___
if ($addx);
1933 mov
8*0($a_ptr), %rax
1934 mov
8*1($a_ptr), $acc6
1935 mov
8*2($a_ptr), $acc7
1936 mov
8*3($a_ptr), $acc0
1938 call __ecp_nistz256_sqr_montq
1940 $code.=<<___
if ($addx);
1945 mov
8*0($a_ptr), %rdx
1946 mov
8*1($a_ptr), $acc6
1947 mov
8*2($a_ptr), $acc7
1948 mov
8*3($a_ptr), $acc0
1949 lea
-128($a_ptr), $a_ptr # control u-op density
1951 call __ecp_nistz256_sqr_montx
1968 .cfi_adjust_cfa_offset
-48
1972 .size ecp_nistz256_sqr_mont
,.-ecp_nistz256_sqr_mont
1974 .type __ecp_nistz256_sqr_montq
,\
@abi-omnipotent
1976 __ecp_nistz256_sqr_montq
:
1979 mulq
$acc6 # a[1]*a[0]
1984 mulq
$acc5 # a[0]*a[2]
1990 mulq
$acc5 # a[0]*a[3]
1996 #################################
1997 mulq
$acc6 # a[1]*a[2]
2003 mulq
$acc6 # a[1]*a[3]
2011 #################################
2012 mulq
$acc7 # a[2]*a[3]
2015 mov
8*0($a_ptr), %rax
2019 add
$acc1, $acc1 # acc1:6<<1
2029 mov
8*1($a_ptr), %rax
2035 mov
8*2($a_ptr), %rax
2042 mov
8*3($a_ptr), %rax
2052 mov
.Lpoly
+8*1(%rip), $a_ptr
2053 mov
.Lpoly
+8*3(%rip), $t1
2055 ##########################################
2062 add
$acc0, $acc1 # +=acc[0]<<96
2068 ##########################################
2081 ##########################################
2094 ###########################################
2107 ############################################
2108 # Add the rest of the acc
2117 sub \
$-1, $acc4 # .Lpoly[0]
2119 sbb
$a_ptr, $acc5 # .Lpoly[1]
2120 sbb \
$0, $acc6 # .Lpoly[2]
2122 sbb
$t1, $acc7 # .Lpoly[3]
2127 mov
$acc4, 8*0($r_ptr)
2129 mov
$acc5, 8*1($r_ptr)
2131 mov
$acc6, 8*2($r_ptr)
2132 mov
$acc7, 8*3($r_ptr)
2136 .size __ecp_nistz256_sqr_montq
,.-__ecp_nistz256_sqr_montq
2141 .type __ecp_nistz256_mul_montx
,\
@abi-omnipotent
2143 __ecp_nistz256_mul_montx
:
2145 ########################################################################
2147 mulx
$acc1, $acc0, $acc1
2148 mulx
$acc2, $t0, $acc2
2150 xor $acc5, $acc5 # cf=0
2151 mulx
$acc3, $t1, $acc3
2152 mov
.Lpoly
+8*3(%rip), $poly3
2154 mulx
$acc4, $t0, $acc4
2157 shlx
$poly1,$acc0,$t1
2159 shrx
$poly1,$acc0,$t0
2162 ########################################################################
2163 # First reduction step
2167 mulx
$poly3, $t0, $t1
2168 mov
8*1($b_ptr), %rdx
2172 xor $acc0, $acc0 # $acc0=0,cf=0,of=0
2174 ########################################################################
2176 mulx
8*0+128($a_ptr), $t0, $t1
2180 mulx
8*1+128($a_ptr), $t0, $t1
2184 mulx
8*2+128($a_ptr), $t0, $t1
2188 mulx
8*3+128($a_ptr), $t0, $t1
2191 shlx
$poly1, $acc1, $t0
2193 shrx
$poly1, $acc1, $t1
2199 ########################################################################
2200 # Second reduction step
2204 mulx
$poly3, $t0, $t1
2205 mov
8*2($b_ptr), %rdx
2209 xor $acc1 ,$acc1 # $acc1=0,cf=0,of=0
2211 ########################################################################
2213 mulx
8*0+128($a_ptr), $t0, $t1
2217 mulx
8*1+128($a_ptr), $t0, $t1
2221 mulx
8*2+128($a_ptr), $t0, $t1
2225 mulx
8*3+128($a_ptr), $t0, $t1
2228 shlx
$poly1, $acc2, $t0
2230 shrx
$poly1, $acc2, $t1
2236 ########################################################################
2237 # Third reduction step
2241 mulx
$poly3, $t0, $t1
2242 mov
8*3($b_ptr), %rdx
2246 xor $acc2, $acc2 # $acc2=0,cf=0,of=0
2248 ########################################################################
2250 mulx
8*0+128($a_ptr), $t0, $t1
2254 mulx
8*1+128($a_ptr), $t0, $t1
2258 mulx
8*2+128($a_ptr), $t0, $t1
2262 mulx
8*3+128($a_ptr), $t0, $t1
2265 shlx
$poly1, $acc3, $t0
2267 shrx
$poly1, $acc3, $t1
2273 ########################################################################
2274 # Fourth reduction step
2278 mulx
$poly3, $t0, $t1
2280 mov
.Lpoly
+8*1(%rip), $poly1
2286 ########################################################################
2287 # Branch-less conditional subtraction of P
2290 sbb \
$-1, $acc4 # .Lpoly[0]
2291 sbb
$poly1, $acc5 # .Lpoly[1]
2292 sbb \
$0, $acc0 # .Lpoly[2]
2294 sbb
$poly3, $acc1 # .Lpoly[3]
2299 mov
$acc4, 8*0($r_ptr)
2301 mov
$acc5, 8*1($r_ptr)
2303 mov
$acc0, 8*2($r_ptr)
2304 mov
$acc1, 8*3($r_ptr)
2308 .size __ecp_nistz256_mul_montx
,.-__ecp_nistz256_mul_montx
2310 .type __ecp_nistz256_sqr_montx
,\
@abi-omnipotent
2312 __ecp_nistz256_sqr_montx
:
2314 mulx
$acc6, $acc1, $acc2 # a[0]*a[1]
2315 mulx
$acc7, $t0, $acc3 # a[0]*a[2]
2318 mulx
$acc0, $t1, $acc4 # a[0]*a[3]
2322 xor $acc5, $acc5 # $acc5=0,cf=0,of=0
2324 #################################
2325 mulx
$acc7, $t0, $t1 # a[1]*a[2]
2329 mulx
$acc0, $t0, $t1 # a[1]*a[3]
2335 #################################
2336 mulx
$acc0, $t0, $acc6 # a[2]*a[3]
2337 mov
8*0+128($a_ptr), %rdx
2338 xor $acc7, $acc7 # $acc7=0,cf=0,of=0
2339 adcx
$acc1, $acc1 # acc1:6<<1
2342 adox
$acc7, $acc6 # of=0
2344 mulx
%rdx, $acc0, $t1
2345 mov
8*1+128($a_ptr), %rdx
2350 mov
8*2+128($a_ptr), %rdx
2356 mov
8*3+128($a_ptr), %rdx
2364 mov
.Lpoly
+8*3(%rip), %rdx
2366 shlx
$a_ptr, $acc0, $t0
2368 shrx
$a_ptr, $acc0, $t4
2375 mulx
$acc0, $t0, $acc0
2377 shlx
$a_ptr, $acc1, $t0
2379 shrx
$a_ptr, $acc1, $t4
2385 mulx
$acc1, $t0, $acc1
2387 shlx
$a_ptr, $acc2, $t0
2389 shrx
$a_ptr, $acc2, $t4
2395 mulx
$acc2, $t0, $acc2
2397 shlx
$a_ptr, $acc3, $t0
2399 shrx
$a_ptr, $acc3, $t4
2405 mulx
$acc3, $t0, $acc3
2410 add
$acc0, $acc4 # accumulate upper half
2411 mov
.Lpoly
+8*1(%rip), $a_ptr
2419 sub \
$-1, $acc4 # .Lpoly[0]
2421 sbb
$a_ptr, $acc5 # .Lpoly[1]
2422 sbb \
$0, $acc6 # .Lpoly[2]
2424 sbb
$t1, $acc7 # .Lpoly[3]
2429 mov
$acc4, 8*0($r_ptr)
2431 mov
$acc5, 8*1($r_ptr)
2433 mov
$acc6, 8*2($r_ptr)
2434 mov
$acc7, 8*3($r_ptr)
2438 .size __ecp_nistz256_sqr_montx
,.-__ecp_nistz256_sqr_montx
2443 my ($r_ptr,$in_ptr)=("%rdi","%rsi");
2444 my ($acc0,$acc1,$acc2,$acc3)=map("%r$_",(8..11));
2445 my ($t0,$t1,$t2)=("%rcx","%r12","%r13");
2448 ################################################################################
2449 # void ecp_nistz256_from_mont(
2452 # This one performs Montgomery multiplication by 1, so we only need the reduction
2454 .globl ecp_nistz256_from_mont
2455 .type ecp_nistz256_from_mont
,\
@function,2
2457 ecp_nistz256_from_mont
:
2465 mov
8*0($in_ptr), %rax
2466 mov
.Lpoly
+8*3(%rip), $t2
2467 mov
8*1($in_ptr), $acc1
2468 mov
8*2($in_ptr), $acc2
2469 mov
8*3($in_ptr), $acc3
2471 mov
.Lpoly
+8*1(%rip), $t1
2473 #########################################
2485 #########################################
2498 ##########################################
2511 ###########################################
2525 ###########################################
2526 # Branch-less conditional subtraction
2536 cmovnz
$in_ptr, $acc1
2537 mov
$acc0, 8*0($r_ptr)
2539 mov
$acc1, 8*1($r_ptr)
2541 mov
$acc2, 8*2($r_ptr)
2542 mov
$acc3, 8*3($r_ptr)
2549 .cfi_adjust_cfa_offset
-16
2553 .size ecp_nistz256_from_mont
,.-ecp_nistz256_from_mont
2557 my ($val,$in_t,$index)=$win64?
("%rcx","%rdx","%r8d"):("%rdi","%rsi","%edx");
2558 my ($ONE,$INDEX,$Ra,$Rb,$Rc,$Rd,$Re,$Rf)=map("%xmm$_",(0..7));
2559 my ($M0,$T0a,$T0b,$T0c,$T0d,$T0e,$T0f,$TMP0)=map("%xmm$_",(8..15));
2560 my ($M1,$T2a,$T2b,$TMP2,$M2,$T2a,$T2b,$TMP2)=map("%xmm$_",(8..15));
2563 ################################################################################
2564 # void ecp_nistz256_scatter_w5(uint64_t *val, uint64_t *in_t, int index);
2565 .globl ecp_nistz256_scatter_w5
2566 .type ecp_nistz256_scatter_w5
,\
@abi-omnipotent
2568 ecp_nistz256_scatter_w5
:
2570 lea
-3($index,$index,2), $index
2571 movdqa
0x00($in_t), %xmm0
2573 movdqa
0x10($in_t), %xmm1
2574 movdqa
0x20($in_t), %xmm2
2575 movdqa
0x30($in_t), %xmm3
2576 movdqa
0x40($in_t), %xmm4
2577 movdqa
0x50($in_t), %xmm5
2578 movdqa
%xmm0, 0x00($val,$index)
2579 movdqa
%xmm1, 0x10($val,$index)
2580 movdqa
%xmm2, 0x20($val,$index)
2581 movdqa
%xmm3, 0x30($val,$index)
2582 movdqa
%xmm4, 0x40($val,$index)
2583 movdqa
%xmm5, 0x50($val,$index)
2587 .size ecp_nistz256_scatter_w5
,.-ecp_nistz256_scatter_w5
2589 ################################################################################
2590 # void ecp_nistz256_gather_w5(uint64_t *val, uint64_t *in_t, int index);
2591 .globl ecp_nistz256_gather_w5
2592 .type ecp_nistz256_gather_w5
,\
@abi-omnipotent
2594 ecp_nistz256_gather_w5
:
2597 $code.=<<___
if ($avx>1);
2598 mov OPENSSL_ia32cap_P
+8(%rip), %eax
2600 jnz
.Lavx2_gather_w5
2602 $code.=<<___
if ($win64);
2603 lea
-0x88(%rsp), %rax
2604 .LSEH_begin_ecp_nistz256_gather_w5
:
2605 .byte
0x48,0x8d,0x60,0xe0 #lea -0x20(%rax), %rsp
2606 .byte
0x0f,0x29,0x70,0xe0 #movaps %xmm6, -0x20(%rax)
2607 .byte
0x0f,0x29,0x78,0xf0 #movaps %xmm7, -0x10(%rax)
2608 .byte
0x44,0x0f,0x29,0x00 #movaps %xmm8, 0(%rax)
2609 .byte
0x44,0x0f,0x29,0x48,0x10 #movaps %xmm9, 0x10(%rax)
2610 .byte
0x44,0x0f,0x29,0x50,0x20 #movaps %xmm10, 0x20(%rax)
2611 .byte
0x44,0x0f,0x29,0x58,0x30 #movaps %xmm11, 0x30(%rax)
2612 .byte
0x44,0x0f,0x29,0x60,0x40 #movaps %xmm12, 0x40(%rax)
2613 .byte
0x44,0x0f,0x29,0x68,0x50 #movaps %xmm13, 0x50(%rax)
2614 .byte
0x44,0x0f,0x29,0x70,0x60 #movaps %xmm14, 0x60(%rax)
2615 .byte
0x44,0x0f,0x29,0x78,0x70 #movaps %xmm15, 0x70(%rax)
2618 movdqa
.LOne
(%rip), $ONE
2629 pshufd \
$0, $INDEX, $INDEX
2632 .Lselect_loop_sse_w5
:
2636 pcmpeqd
$INDEX, $TMP0
2638 movdqa
16*0($in_t), $T0a
2639 movdqa
16*1($in_t), $T0b
2640 movdqa
16*2($in_t), $T0c
2641 movdqa
16*3($in_t), $T0d
2642 movdqa
16*4($in_t), $T0e
2643 movdqa
16*5($in_t), $T0f
2644 lea
16*6($in_t), $in_t
2660 jnz
.Lselect_loop_sse_w5
2662 movdqu
$Ra, 16*0($val)
2663 movdqu
$Rb, 16*1($val)
2664 movdqu
$Rc, 16*2($val)
2665 movdqu
$Rd, 16*3($val)
2666 movdqu
$Re, 16*4($val)
2667 movdqu
$Rf, 16*5($val)
2669 $code.=<<___
if ($win64);
2670 movaps
(%rsp), %xmm6
2671 movaps
0x10(%rsp), %xmm7
2672 movaps
0x20(%rsp), %xmm8
2673 movaps
0x30(%rsp), %xmm9
2674 movaps
0x40(%rsp), %xmm10
2675 movaps
0x50(%rsp), %xmm11
2676 movaps
0x60(%rsp), %xmm12
2677 movaps
0x70(%rsp), %xmm13
2678 movaps
0x80(%rsp), %xmm14
2679 movaps
0x90(%rsp), %xmm15
2680 lea
0xa8(%rsp), %rsp
2685 .LSEH_end_ecp_nistz256_gather_w5
:
2686 .size ecp_nistz256_gather_w5
,.-ecp_nistz256_gather_w5
2688 ################################################################################
2689 # void ecp_nistz256_scatter_w7(uint64_t *val, uint64_t *in_t, int index);
2690 .globl ecp_nistz256_scatter_w7
2691 .type ecp_nistz256_scatter_w7
,\
@abi-omnipotent
2693 ecp_nistz256_scatter_w7
:
2695 movdqu
0x00($in_t), %xmm0
2697 movdqu
0x10($in_t), %xmm1
2698 movdqu
0x20($in_t), %xmm2
2699 movdqu
0x30($in_t), %xmm3
2700 movdqa
%xmm0, 0x00($val,$index)
2701 movdqa
%xmm1, 0x10($val,$index)
2702 movdqa
%xmm2, 0x20($val,$index)
2703 movdqa
%xmm3, 0x30($val,$index)
2707 .size ecp_nistz256_scatter_w7
,.-ecp_nistz256_scatter_w7
2709 ################################################################################
2710 # void ecp_nistz256_gather_w7(uint64_t *val, uint64_t *in_t, int index);
2711 .globl ecp_nistz256_gather_w7
2712 .type ecp_nistz256_gather_w7
,\
@abi-omnipotent
2714 ecp_nistz256_gather_w7
:
2717 $code.=<<___
if ($avx>1);
2718 mov OPENSSL_ia32cap_P
+8(%rip), %eax
2720 jnz
.Lavx2_gather_w7
2722 $code.=<<___
if ($win64);
2723 lea
-0x88(%rsp), %rax
2724 .LSEH_begin_ecp_nistz256_gather_w7
:
2725 .byte
0x48,0x8d,0x60,0xe0 #lea -0x20(%rax), %rsp
2726 .byte
0x0f,0x29,0x70,0xe0 #movaps %xmm6, -0x20(%rax)
2727 .byte
0x0f,0x29,0x78,0xf0 #movaps %xmm7, -0x10(%rax)
2728 .byte
0x44,0x0f,0x29,0x00 #movaps %xmm8, 0(%rax)
2729 .byte
0x44,0x0f,0x29,0x48,0x10 #movaps %xmm9, 0x10(%rax)
2730 .byte
0x44,0x0f,0x29,0x50,0x20 #movaps %xmm10, 0x20(%rax)
2731 .byte
0x44,0x0f,0x29,0x58,0x30 #movaps %xmm11, 0x30(%rax)
2732 .byte
0x44,0x0f,0x29,0x60,0x40 #movaps %xmm12, 0x40(%rax)
2733 .byte
0x44,0x0f,0x29,0x68,0x50 #movaps %xmm13, 0x50(%rax)
2734 .byte
0x44,0x0f,0x29,0x70,0x60 #movaps %xmm14, 0x60(%rax)
2735 .byte
0x44,0x0f,0x29,0x78,0x70 #movaps %xmm15, 0x70(%rax)
2738 movdqa
.LOne
(%rip), $M0
2747 pshufd \
$0, $INDEX, $INDEX
2750 .Lselect_loop_sse_w7
:
2753 movdqa
16*0($in_t), $T0a
2754 movdqa
16*1($in_t), $T0b
2755 pcmpeqd
$INDEX, $TMP0
2756 movdqa
16*2($in_t), $T0c
2757 movdqa
16*3($in_t), $T0d
2758 lea
16*4($in_t), $in_t
2767 prefetcht0
255($in_t)
2771 jnz
.Lselect_loop_sse_w7
2773 movdqu
$Ra, 16*0($val)
2774 movdqu
$Rb, 16*1($val)
2775 movdqu
$Rc, 16*2($val)
2776 movdqu
$Rd, 16*3($val)
2778 $code.=<<___
if ($win64);
2779 movaps
(%rsp), %xmm6
2780 movaps
0x10(%rsp), %xmm7
2781 movaps
0x20(%rsp), %xmm8
2782 movaps
0x30(%rsp), %xmm9
2783 movaps
0x40(%rsp), %xmm10
2784 movaps
0x50(%rsp), %xmm11
2785 movaps
0x60(%rsp), %xmm12
2786 movaps
0x70(%rsp), %xmm13
2787 movaps
0x80(%rsp), %xmm14
2788 movaps
0x90(%rsp), %xmm15
2789 lea
0xa8(%rsp), %rsp
2794 .LSEH_end_ecp_nistz256_gather_w7
:
2795 .size ecp_nistz256_gather_w7
,.-ecp_nistz256_gather_w7
2799 my ($val,$in_t,$index)=$win64?
("%rcx","%rdx","%r8d"):("%rdi","%rsi","%edx");
2800 my ($TWO,$INDEX,$Ra,$Rb,$Rc)=map("%ymm$_",(0..4));
2801 my ($M0,$T0a,$T0b,$T0c,$TMP0)=map("%ymm$_",(5..9));
2802 my ($M1,$T1a,$T1b,$T1c,$TMP1)=map("%ymm$_",(10..14));
2805 ################################################################################
2806 # void ecp_nistz256_avx2_gather_w5(uint64_t *val, uint64_t *in_t, int index);
2807 .type ecp_nistz256_avx2_gather_w5
,\
@abi-omnipotent
2809 ecp_nistz256_avx2_gather_w5
:
2814 $code.=<<___
if ($win64);
2815 lea
-0x88(%rsp), %rax
2817 .LSEH_begin_ecp_nistz256_avx2_gather_w5
:
2818 .byte
0x48,0x8d,0x60,0xe0 # lea -0x20(%rax), %rsp
2819 .byte
0xc5,0xf8,0x29,0x70,0xe0 # vmovaps %xmm6, -0x20(%rax)
2820 .byte
0xc5,0xf8,0x29,0x78,0xf0 # vmovaps %xmm7, -0x10(%rax)
2821 .byte
0xc5,0x78,0x29,0x40,0x00 # vmovaps %xmm8, 8(%rax)
2822 .byte
0xc5,0x78,0x29,0x48,0x10 # vmovaps %xmm9, 0x10(%rax)
2823 .byte
0xc5,0x78,0x29,0x50,0x20 # vmovaps %xmm10, 0x20(%rax)
2824 .byte
0xc5,0x78,0x29,0x58,0x30 # vmovaps %xmm11, 0x30(%rax)
2825 .byte
0xc5,0x78,0x29,0x60,0x40 # vmovaps %xmm12, 0x40(%rax)
2826 .byte
0xc5,0x78,0x29,0x68,0x50 # vmovaps %xmm13, 0x50(%rax)
2827 .byte
0xc5,0x78,0x29,0x70,0x60 # vmovaps %xmm14, 0x60(%rax)
2828 .byte
0xc5,0x78,0x29,0x78,0x70 # vmovaps %xmm15, 0x70(%rax)
2831 vmovdqa
.LTwo
(%rip), $TWO
2837 vmovdqa
.LOne
(%rip), $M0
2838 vmovdqa
.LTwo
(%rip), $M1
2841 vpermd
$INDEX, $Ra, $INDEX
2844 .Lselect_loop_avx2_w5
:
2846 vmovdqa
32*0($in_t), $T0a
2847 vmovdqa
32*1($in_t), $T0b
2848 vmovdqa
32*2($in_t), $T0c
2850 vmovdqa
32*3($in_t), $T1a
2851 vmovdqa
32*4($in_t), $T1b
2852 vmovdqa
32*5($in_t), $T1c
2854 vpcmpeqd
$INDEX, $M0, $TMP0
2855 vpcmpeqd
$INDEX, $M1, $TMP1
2857 vpaddd
$TWO, $M0, $M0
2858 vpaddd
$TWO, $M1, $M1
2859 lea
32*6($in_t), $in_t
2861 vpand
$TMP0, $T0a, $T0a
2862 vpand
$TMP0, $T0b, $T0b
2863 vpand
$TMP0, $T0c, $T0c
2864 vpand
$TMP1, $T1a, $T1a
2865 vpand
$TMP1, $T1b, $T1b
2866 vpand
$TMP1, $T1c, $T1c
2868 vpxor
$T0a, $Ra, $Ra
2869 vpxor
$T0b, $Rb, $Rb
2870 vpxor
$T0c, $Rc, $Rc
2871 vpxor
$T1a, $Ra, $Ra
2872 vpxor
$T1b, $Rb, $Rb
2873 vpxor
$T1c, $Rc, $Rc
2876 jnz
.Lselect_loop_avx2_w5
2878 vmovdqu
$Ra, 32*0($val)
2879 vmovdqu
$Rb, 32*1($val)
2880 vmovdqu
$Rc, 32*2($val)
2883 $code.=<<___
if ($win64);
2884 movaps
(%rsp), %xmm6
2885 movaps
0x10(%rsp), %xmm7
2886 movaps
0x20(%rsp), %xmm8
2887 movaps
0x30(%rsp), %xmm9
2888 movaps
0x40(%rsp), %xmm10
2889 movaps
0x50(%rsp), %xmm11
2890 movaps
0x60(%rsp), %xmm12
2891 movaps
0x70(%rsp), %xmm13
2892 movaps
0x80(%rsp), %xmm14
2893 movaps
0x90(%rsp), %xmm15
2899 .LSEH_end_ecp_nistz256_avx2_gather_w5
:
2900 .size ecp_nistz256_avx2_gather_w5
,.-ecp_nistz256_avx2_gather_w5
2904 my ($val,$in_t,$index)=$win64?
("%rcx","%rdx","%r8d"):("%rdi","%rsi","%edx");
2905 my ($THREE,$INDEX,$Ra,$Rb)=map("%ymm$_",(0..3));
2906 my ($M0,$T0a,$T0b,$TMP0)=map("%ymm$_",(4..7));
2907 my ($M1,$T1a,$T1b,$TMP1)=map("%ymm$_",(8..11));
2908 my ($M2,$T2a,$T2b,$TMP2)=map("%ymm$_",(12..15));
2912 ################################################################################
2913 # void ecp_nistz256_avx2_gather_w7(uint64_t *val, uint64_t *in_t, int index);
2914 .globl ecp_nistz256_avx2_gather_w7
2915 .type ecp_nistz256_avx2_gather_w7
,\
@abi-omnipotent
2917 ecp_nistz256_avx2_gather_w7
:
2922 $code.=<<___
if ($win64);
2924 lea
-0x88(%rsp), %rax
2925 .LSEH_begin_ecp_nistz256_avx2_gather_w7
:
2926 .byte
0x48,0x8d,0x60,0xe0 # lea -0x20(%rax), %rsp
2927 .byte
0xc5,0xf8,0x29,0x70,0xe0 # vmovaps %xmm6, -0x20(%rax)
2928 .byte
0xc5,0xf8,0x29,0x78,0xf0 # vmovaps %xmm7, -0x10(%rax)
2929 .byte
0xc5,0x78,0x29,0x40,0x00 # vmovaps %xmm8, 8(%rax)
2930 .byte
0xc5,0x78,0x29,0x48,0x10 # vmovaps %xmm9, 0x10(%rax)
2931 .byte
0xc5,0x78,0x29,0x50,0x20 # vmovaps %xmm10, 0x20(%rax)
2932 .byte
0xc5,0x78,0x29,0x58,0x30 # vmovaps %xmm11, 0x30(%rax)
2933 .byte
0xc5,0x78,0x29,0x60,0x40 # vmovaps %xmm12, 0x40(%rax)
2934 .byte
0xc5,0x78,0x29,0x68,0x50 # vmovaps %xmm13, 0x50(%rax)
2935 .byte
0xc5,0x78,0x29,0x70,0x60 # vmovaps %xmm14, 0x60(%rax)
2936 .byte
0xc5,0x78,0x29,0x78,0x70 # vmovaps %xmm15, 0x70(%rax)
2939 vmovdqa
.LThree
(%rip), $THREE
2944 vmovdqa
.LOne
(%rip), $M0
2945 vmovdqa
.LTwo
(%rip), $M1
2946 vmovdqa
.LThree
(%rip), $M2
2949 vpermd
$INDEX, $Ra, $INDEX
2950 # Skip index = 0, because it is implicitly the point at infinity
2953 .Lselect_loop_avx2_w7
:
2955 vmovdqa
32*0($in_t), $T0a
2956 vmovdqa
32*1($in_t), $T0b
2958 vmovdqa
32*2($in_t), $T1a
2959 vmovdqa
32*3($in_t), $T1b
2961 vmovdqa
32*4($in_t), $T2a
2962 vmovdqa
32*5($in_t), $T2b
2964 vpcmpeqd
$INDEX, $M0, $TMP0
2965 vpcmpeqd
$INDEX, $M1, $TMP1
2966 vpcmpeqd
$INDEX, $M2, $TMP2
2968 vpaddd
$THREE, $M0, $M0
2969 vpaddd
$THREE, $M1, $M1
2970 vpaddd
$THREE, $M2, $M2
2971 lea
32*6($in_t), $in_t
2973 vpand
$TMP0, $T0a, $T0a
2974 vpand
$TMP0, $T0b, $T0b
2975 vpand
$TMP1, $T1a, $T1a
2976 vpand
$TMP1, $T1b, $T1b
2977 vpand
$TMP2, $T2a, $T2a
2978 vpand
$TMP2, $T2b, $T2b
2980 vpxor
$T0a, $Ra, $Ra
2981 vpxor
$T0b, $Rb, $Rb
2982 vpxor
$T1a, $Ra, $Ra
2983 vpxor
$T1b, $Rb, $Rb
2984 vpxor
$T2a, $Ra, $Ra
2985 vpxor
$T2b, $Rb, $Rb
2988 jnz
.Lselect_loop_avx2_w7
2991 vmovdqa
32*0($in_t), $T0a
2992 vmovdqa
32*1($in_t), $T0b
2994 vpcmpeqd
$INDEX, $M0, $TMP0
2996 vpand
$TMP0, $T0a, $T0a
2997 vpand
$TMP0, $T0b, $T0b
2999 vpxor
$T0a, $Ra, $Ra
3000 vpxor
$T0b, $Rb, $Rb
3002 vmovdqu
$Ra, 32*0($val)
3003 vmovdqu
$Rb, 32*1($val)
3006 $code.=<<___
if ($win64);
3007 movaps
(%rsp), %xmm6
3008 movaps
0x10(%rsp), %xmm7
3009 movaps
0x20(%rsp), %xmm8
3010 movaps
0x30(%rsp), %xmm9
3011 movaps
0x40(%rsp), %xmm10
3012 movaps
0x50(%rsp), %xmm11
3013 movaps
0x60(%rsp), %xmm12
3014 movaps
0x70(%rsp), %xmm13
3015 movaps
0x80(%rsp), %xmm14
3016 movaps
0x90(%rsp), %xmm15
3022 .LSEH_end_ecp_nistz256_avx2_gather_w7
:
3023 .size ecp_nistz256_avx2_gather_w7
,.-ecp_nistz256_avx2_gather_w7
3027 .globl ecp_nistz256_avx2_gather_w7
3028 .type ecp_nistz256_avx2_gather_w7
,\
@function,3
3030 ecp_nistz256_avx2_gather_w7
:
3032 .byte
0x0f,0x0b # ud2
3035 .size ecp_nistz256_avx2_gather_w7
,.-ecp_nistz256_avx2_gather_w7
3039 ########################################################################
3040 # This block implements higher level point_double, point_add and
3041 # point_add_affine. The key to performance in this case is to allow
3042 # out-of-order execution logic to overlap computations from next step
3043 # with tail processing from current step. By using tailored calling
3044 # sequence we minimize inter-step overhead to give processor better
3045 # shot at overlapping operations...
3047 # You will notice that input data is copied to stack. Trouble is that
3048 # there are no registers to spare for holding original pointers and
3049 # reloading them, pointers, would create undesired dependencies on
3050 # effective addresses calculation paths. In other words it's too done
3051 # to favour out-of-order execution logic.
3052 # <appro@openssl.org>
3054 my ($r_ptr,$a_ptr,$b_org,$b_ptr)=("%rdi","%rsi","%rdx","%rbx");
3055 my ($acc0,$acc1,$acc2,$acc3,$acc4,$acc5,$acc6,$acc7)=map("%r$_",(8..15));
3056 my ($t0,$t1,$t2,$t3,$t4)=("%rax","%rbp","%rcx",$acc4,$acc4);
3057 my ($poly1,$poly3)=($acc6,$acc7);
3059 sub load_for_mul
() {
3060 my ($a,$b,$src0) = @_;
3061 my $bias = $src0 eq "%rax" ?
0 : -128;
3067 lea $bias+$a, $a_ptr
3072 sub load_for_sqr
() {
3074 my $bias = $src0 eq "%rax" ?
0 : -128;
3078 lea $bias+$a, $a_ptr
3084 ########################################################################
3085 # operate in 4-5-0-1 "name space" that matches multiplication output
3087 my ($a0,$a1,$a2,$a3,$t3,$t4)=($acc4,$acc5,$acc0,$acc1,$acc2,$acc3);
3090 .type __ecp_nistz256_add_toq
,\
@abi-omnipotent
3092 __ecp_nistz256_add_toq
:
3095 add
8*0($b_ptr), $a0
3096 adc
8*1($b_ptr), $a1
3098 adc
8*2($b_ptr), $a2
3099 adc
8*3($b_ptr), $a3
3113 mov
$a0, 8*0($r_ptr)
3115 mov
$a1, 8*1($r_ptr)
3117 mov
$a2, 8*2($r_ptr)
3118 mov
$a3, 8*3($r_ptr)
3122 .size __ecp_nistz256_add_toq
,.-__ecp_nistz256_add_toq
3124 .type __ecp_nistz256_sub_fromq
,\
@abi-omnipotent
3126 __ecp_nistz256_sub_fromq
:
3128 sub 8*0($b_ptr), $a0
3129 sbb
8*1($b_ptr), $a1
3131 sbb
8*2($b_ptr), $a2
3132 sbb
8*3($b_ptr), $a3
3146 mov
$a0, 8*0($r_ptr)
3148 mov
$a1, 8*1($r_ptr)
3150 mov
$a2, 8*2($r_ptr)
3151 mov
$a3, 8*3($r_ptr)
3155 .size __ecp_nistz256_sub_fromq
,.-__ecp_nistz256_sub_fromq
3157 .type __ecp_nistz256_subq
,\
@abi-omnipotent
3159 __ecp_nistz256_subq
:
3184 .size __ecp_nistz256_subq
,.-__ecp_nistz256_subq
3186 .type __ecp_nistz256_mul_by_2q
,\
@abi-omnipotent
3188 __ecp_nistz256_mul_by_2q
:
3191 add
$a0, $a0 # a0:a3+a0:a3
3209 mov
$a0, 8*0($r_ptr)
3211 mov
$a1, 8*1($r_ptr)
3213 mov
$a2, 8*2($r_ptr)
3214 mov
$a3, 8*3($r_ptr)
3218 .size __ecp_nistz256_mul_by_2q
,.-__ecp_nistz256_mul_by_2q
3223 my ($src0,$sfx,$bias);
3224 my ($S,$M,$Zsqr,$in_x,$tmp0)=map(32*$_,(0..4));
3232 .globl ecp_nistz256_point_double
3233 .type ecp_nistz256_point_double
,\
@function,2
3235 ecp_nistz256_point_double
:
3238 $code.=<<___
if ($addx);
3240 and OPENSSL_ia32cap_P
+8(%rip), %ecx
3250 .type ecp_nistz256_point_doublex
,\
@function,2
3252 ecp_nistz256_point_doublex
:
3271 .cfi_adjust_cfa_offset
32*5+8
3272 .Lpoint_double
${x
}_body
:
3274 .Lpoint_double_shortcut
$x:
3275 movdqu
0x00($a_ptr), %xmm0 # copy *(P256_POINT *)$a_ptr.x
3276 mov
$a_ptr, $b_ptr # backup copy
3277 movdqu
0x10($a_ptr), %xmm1
3278 mov
0x20+8*0($a_ptr), $acc4 # load in_y in "5-4-0-1" order
3279 mov
0x20+8*1($a_ptr), $acc5
3280 mov
0x20+8*2($a_ptr), $acc0
3281 mov
0x20+8*3($a_ptr), $acc1
3282 mov
.Lpoly
+8*1(%rip), $poly1
3283 mov
.Lpoly
+8*3(%rip), $poly3
3284 movdqa
%xmm0, $in_x(%rsp)
3285 movdqa
%xmm1, $in_x+0x10(%rsp)
3286 lea
0x20($r_ptr), $acc2
3287 lea
0x40($r_ptr), $acc3
3292 lea
$S(%rsp), $r_ptr
3293 call __ecp_nistz256_mul_by_2
$x # p256_mul_by_2(S, in_y);
3295 mov
0x40+8*0($a_ptr), $src0
3296 mov
0x40+8*1($a_ptr), $acc6
3297 mov
0x40+8*2($a_ptr), $acc7
3298 mov
0x40+8*3($a_ptr), $acc0
3299 lea
0x40-$bias($a_ptr), $a_ptr
3300 lea
$Zsqr(%rsp), $r_ptr
3301 call __ecp_nistz256_sqr_mont
$x # p256_sqr_mont(Zsqr, in_z);
3303 `&load_for_sqr("$S(%rsp)", "$src0")`
3304 lea
$S(%rsp), $r_ptr
3305 call __ecp_nistz256_sqr_mont
$x # p256_sqr_mont(S, S);
3307 mov
0x20($b_ptr), $src0 # $b_ptr is still valid
3308 mov
0x40+8*0($b_ptr), $acc1
3309 mov
0x40+8*1($b_ptr), $acc2
3310 mov
0x40+8*2($b_ptr), $acc3
3311 mov
0x40+8*3($b_ptr), $acc4
3312 lea
0x40-$bias($b_ptr), $a_ptr
3313 lea
0x20($b_ptr), $b_ptr
3315 call __ecp_nistz256_mul_mont
$x # p256_mul_mont(res_z, in_z, in_y);
3316 call __ecp_nistz256_mul_by_2
$x # p256_mul_by_2(res_z, res_z);
3318 mov
$in_x+8*0(%rsp), $acc4 # "5-4-0-1" order
3319 mov
$in_x+8*1(%rsp), $acc5
3320 lea
$Zsqr(%rsp), $b_ptr
3321 mov
$in_x+8*2(%rsp), $acc0
3322 mov
$in_x+8*3(%rsp), $acc1
3323 lea
$M(%rsp), $r_ptr
3324 call __ecp_nistz256_add_to
$x # p256_add(M, in_x, Zsqr);
3326 mov
$in_x+8*0(%rsp), $acc4 # "5-4-0-1" order
3327 mov
$in_x+8*1(%rsp), $acc5
3328 lea
$Zsqr(%rsp), $b_ptr
3329 mov
$in_x+8*2(%rsp), $acc0
3330 mov
$in_x+8*3(%rsp), $acc1
3331 lea
$Zsqr(%rsp), $r_ptr
3332 call __ecp_nistz256_sub_from
$x # p256_sub(Zsqr, in_x, Zsqr);
3334 `&load_for_sqr("$S(%rsp)", "$src0")`
3336 call __ecp_nistz256_sqr_mont
$x # p256_sqr_mont(res_y, S);
3339 ######## ecp_nistz256_div_by_2(res_y, res_y); ##########################
3340 # operate in 4-5-6-7 "name space" that matches squaring output
3342 my ($poly1,$poly3)=($a_ptr,$t1);
3343 my ($a0,$a1,$a2,$a3,$t3,$t4,$t1)=($acc4,$acc5,$acc6,$acc7,$acc0,$acc1,$acc2);
3356 xor $a_ptr, $a_ptr # borrow $a_ptr
3365 mov
$a1, $t0 # a0:a3>>1
3376 mov
$a0, 8*0($r_ptr)
3378 mov
$a1, 8*1($r_ptr)
3382 mov
$a2, 8*2($r_ptr)
3383 mov
$a3, 8*3($r_ptr)
3387 `&load_for_mul("$M(%rsp)", "$Zsqr(%rsp)", "$src0")`
3388 lea
$M(%rsp), $r_ptr
3389 call __ecp_nistz256_mul_mont
$x # p256_mul_mont(M, M, Zsqr);
3391 lea
$tmp0(%rsp), $r_ptr
3392 call __ecp_nistz256_mul_by_2
$x
3394 lea
$M(%rsp), $b_ptr
3395 lea
$M(%rsp), $r_ptr
3396 call __ecp_nistz256_add_to
$x # p256_mul_by_3(M, M);
3398 `&load_for_mul("$S(%rsp)", "$in_x(%rsp)", "$src0")`
3399 lea
$S(%rsp), $r_ptr
3400 call __ecp_nistz256_mul_mont
$x # p256_mul_mont(S, S, in_x);
3402 lea
$tmp0(%rsp), $r_ptr
3403 call __ecp_nistz256_mul_by_2
$x # p256_mul_by_2(tmp0, S);
3405 `&load_for_sqr("$M(%rsp)", "$src0")`
3407 call __ecp_nistz256_sqr_mont
$x # p256_sqr_mont(res_x, M);
3409 lea
$tmp0(%rsp), $b_ptr
3410 mov
$acc6, $acc0 # harmonize sqr output and sub input
3414 call __ecp_nistz256_sub_from
$x # p256_sub(res_x, res_x, tmp0);
3416 mov
$S+8*0(%rsp), $t0
3417 mov
$S+8*1(%rsp), $t1
3418 mov
$S+8*2(%rsp), $t2
3419 mov
$S+8*3(%rsp), $acc2 # "4-5-0-1" order
3420 lea
$S(%rsp), $r_ptr
3421 call __ecp_nistz256_sub
$x # p256_sub(S, S, res_x);
3424 lea
$M(%rsp), $b_ptr
3425 mov
$acc4, $acc6 # harmonize sub output and mul input
3427 mov
$acc4, $S+8*0(%rsp) # have to save:-(
3429 mov
$acc5, $S+8*1(%rsp)
3431 mov
$acc0, $S+8*2(%rsp)
3432 lea
$S-$bias(%rsp), $a_ptr
3434 mov
$acc1, $S+8*3(%rsp)
3436 lea
$S(%rsp), $r_ptr
3437 call __ecp_nistz256_mul_mont
$x # p256_mul_mont(S, S, M);
3441 call __ecp_nistz256_sub_from
$x # p256_sub(res_y, S, res_y);
3443 lea
32*5+56(%rsp), %rsi
3458 .cfi_def_cfa_register
%rsp
3459 .Lpoint_double
${x
}_epilogue
:
3462 .size ecp_nistz256_point_double
$sfx,.-ecp_nistz256_point_double
$sfx
3469 my ($src0,$sfx,$bias);
3470 my ($H,$Hsqr,$R,$Rsqr,$Hcub,
3472 $res_x,$res_y,$res_z,
3473 $in1_x,$in1_y,$in1_z,
3474 $in2_x,$in2_y,$in2_z)=map(32*$_,(0..17));
3475 my ($Z1sqr, $Z2sqr) = ($Hsqr, $Rsqr);
3483 .globl ecp_nistz256_point_add
3484 .type ecp_nistz256_point_add
,\
@function,3
3486 ecp_nistz256_point_add
:
3489 $code.=<<___
if ($addx);
3491 and OPENSSL_ia32cap_P
+8(%rip), %ecx
3501 .type ecp_nistz256_point_addx
,\
@function,3
3503 ecp_nistz256_point_addx
:
3522 .cfi_adjust_cfa_offset
32*18+8
3523 .Lpoint_add
${x
}_body
:
3525 movdqu
0x00($a_ptr), %xmm0 # copy *(P256_POINT *)$a_ptr
3526 movdqu
0x10($a_ptr), %xmm1
3527 movdqu
0x20($a_ptr), %xmm2
3528 movdqu
0x30($a_ptr), %xmm3
3529 movdqu
0x40($a_ptr), %xmm4
3530 movdqu
0x50($a_ptr), %xmm5
3531 mov
$a_ptr, $b_ptr # reassign
3532 mov
$b_org, $a_ptr # reassign
3533 movdqa
%xmm0, $in1_x(%rsp)
3534 movdqa
%xmm1, $in1_x+0x10(%rsp)
3535 movdqa
%xmm2, $in1_y(%rsp)
3536 movdqa
%xmm3, $in1_y+0x10(%rsp)
3537 movdqa
%xmm4, $in1_z(%rsp)
3538 movdqa
%xmm5, $in1_z+0x10(%rsp)
3541 movdqu
0x00($a_ptr), %xmm0 # copy *(P256_POINT *)$b_ptr
3542 pshufd \
$0xb1, %xmm5, %xmm3
3543 movdqu
0x10($a_ptr), %xmm1
3544 movdqu
0x20($a_ptr), %xmm2
3546 movdqu
0x30($a_ptr), %xmm3
3547 mov
0x40+8*0($a_ptr), $src0 # load original in2_z
3548 mov
0x40+8*1($a_ptr), $acc6
3549 mov
0x40+8*2($a_ptr), $acc7
3550 mov
0x40+8*3($a_ptr), $acc0
3551 movdqa
%xmm0, $in2_x(%rsp)
3552 pshufd \
$0x1e, %xmm5, %xmm4
3553 movdqa
%xmm1, $in2_x+0x10(%rsp)
3554 movdqu
0x40($a_ptr),%xmm0 # in2_z again
3555 movdqu
0x50($a_ptr),%xmm1
3556 movdqa
%xmm2, $in2_y(%rsp)
3557 movdqa
%xmm3, $in2_y+0x10(%rsp)
3561 movq
$r_ptr, %xmm0 # save $r_ptr
3563 lea
0x40-$bias($a_ptr), $a_ptr # $a_ptr is still valid
3564 mov
$src0, $in2_z+8*0(%rsp) # make in2_z copy
3565 mov
$acc6, $in2_z+8*1(%rsp)
3566 mov
$acc7, $in2_z+8*2(%rsp)
3567 mov
$acc0, $in2_z+8*3(%rsp)
3568 lea
$Z2sqr(%rsp), $r_ptr # Z2^2
3569 call __ecp_nistz256_sqr_mont
$x # p256_sqr_mont(Z2sqr, in2_z);
3571 pcmpeqd
%xmm4, %xmm5
3572 pshufd \
$0xb1, %xmm1, %xmm4
3574 pshufd \
$0, %xmm5, %xmm5 # in1infty
3575 pshufd \
$0x1e, %xmm4, %xmm3
3578 pcmpeqd
%xmm3, %xmm4
3579 pshufd \
$0, %xmm4, %xmm4 # in2infty
3580 mov
0x40+8*0($b_ptr), $src0 # load original in1_z
3581 mov
0x40+8*1($b_ptr), $acc6
3582 mov
0x40+8*2($b_ptr), $acc7
3583 mov
0x40+8*3($b_ptr), $acc0
3586 lea
0x40-$bias($b_ptr), $a_ptr
3587 lea
$Z1sqr(%rsp), $r_ptr # Z1^2
3588 call __ecp_nistz256_sqr_mont
$x # p256_sqr_mont(Z1sqr, in1_z);
3590 `&load_for_mul("$Z2sqr(%rsp)", "$in2_z(%rsp)", "$src0")`
3591 lea
$S1(%rsp), $r_ptr # S1 = Z2^3
3592 call __ecp_nistz256_mul_mont
$x # p256_mul_mont(S1, Z2sqr, in2_z);
3594 `&load_for_mul("$Z1sqr(%rsp)", "$in1_z(%rsp)", "$src0")`
3595 lea
$S2(%rsp), $r_ptr # S2 = Z1^3
3596 call __ecp_nistz256_mul_mont
$x # p256_mul_mont(S2, Z1sqr, in1_z);
3598 `&load_for_mul("$S1(%rsp)", "$in1_y(%rsp)", "$src0")`
3599 lea
$S1(%rsp), $r_ptr # S1 = Y1*Z2^3
3600 call __ecp_nistz256_mul_mont
$x # p256_mul_mont(S1, S1, in1_y);
3602 `&load_for_mul("$S2(%rsp)", "$in2_y(%rsp)", "$src0")`
3603 lea
$S2(%rsp), $r_ptr # S2 = Y2*Z1^3
3604 call __ecp_nistz256_mul_mont
$x # p256_mul_mont(S2, S2, in2_y);
3606 lea
$S1(%rsp), $b_ptr
3607 lea
$R(%rsp), $r_ptr # R = S2 - S1
3608 call __ecp_nistz256_sub_from
$x # p256_sub(R, S2, S1);
3610 or $acc5, $acc4 # see if result is zero
3614 por
%xmm5, %xmm2 # in1infty || in2infty
3617 `&load_for_mul("$Z2sqr(%rsp)", "$in1_x(%rsp)", "$src0")`
3618 lea
$U1(%rsp), $r_ptr # U1 = X1*Z2^2
3619 call __ecp_nistz256_mul_mont
$x # p256_mul_mont(U1, in1_x, Z2sqr);
3621 `&load_for_mul("$Z1sqr(%rsp)", "$in2_x(%rsp)", "$src0")`
3622 lea
$U2(%rsp), $r_ptr # U2 = X2*Z1^2
3623 call __ecp_nistz256_mul_mont
$x # p256_mul_mont(U2, in2_x, Z1sqr);
3625 lea
$U1(%rsp), $b_ptr
3626 lea
$H(%rsp), $r_ptr # H = U2 - U1
3627 call __ecp_nistz256_sub_from
$x # p256_sub(H, U2, U1);
3629 or $acc5, $acc4 # see if result is zero
3631 or $acc1, $acc4 # !is_equal(U1, U2)
3633 movq
%xmm2, $acc0 # in1infty | in2infty
3634 movq
%xmm3, $acc1 # !is_equal(S1, S2)
3639 # if (!is_equal(U1, U2) | in1infty | in2infty | !is_equal(S1, S2))
3640 .byte
0x3e # predict taken
3644 movq
%xmm1, $a_ptr # restore $a_ptr
3645 movq
%xmm0, $r_ptr # restore $r_ptr
3646 add \
$`32*(18-5)`, %rsp # difference in frame sizes
3647 .cfi_adjust_cfa_offset
`-32*(18-5)`
3648 jmp
.Lpoint_double_shortcut
$x
3649 .cfi_adjust_cfa_offset
`32*(18-5)`
3653 `&load_for_sqr("$R(%rsp)", "$src0")`
3654 lea
$Rsqr(%rsp), $r_ptr # R^2
3655 call __ecp_nistz256_sqr_mont
$x # p256_sqr_mont(Rsqr, R);
3657 `&load_for_mul("$H(%rsp)", "$in1_z(%rsp)", "$src0")`
3658 lea
$res_z(%rsp), $r_ptr # Z3 = H*Z1*Z2
3659 call __ecp_nistz256_mul_mont
$x # p256_mul_mont(res_z, H, in1_z);
3661 `&load_for_sqr("$H(%rsp)", "$src0")`
3662 lea
$Hsqr(%rsp), $r_ptr # H^2
3663 call __ecp_nistz256_sqr_mont
$x # p256_sqr_mont(Hsqr, H);
3665 `&load_for_mul("$res_z(%rsp)", "$in2_z(%rsp)", "$src0")`
3666 lea
$res_z(%rsp), $r_ptr # Z3 = H*Z1*Z2
3667 call __ecp_nistz256_mul_mont
$x # p256_mul_mont(res_z, res_z, in2_z);
3669 `&load_for_mul("$Hsqr(%rsp)", "$H(%rsp)", "$src0")`
3670 lea
$Hcub(%rsp), $r_ptr # H^3
3671 call __ecp_nistz256_mul_mont
$x # p256_mul_mont(Hcub, Hsqr, H);
3673 `&load_for_mul("$Hsqr(%rsp)", "$U1(%rsp)", "$src0")`
3674 lea
$U2(%rsp), $r_ptr # U1*H^2
3675 call __ecp_nistz256_mul_mont
$x # p256_mul_mont(U2, U1, Hsqr);
3678 #######################################################################
3679 # operate in 4-5-0-1 "name space" that matches multiplication output
3681 my ($acc0,$acc1,$acc2,$acc3,$t3,$t4)=($acc4,$acc5,$acc0,$acc1,$acc2,$acc3);
3682 my ($poly1, $poly3)=($acc6,$acc7);
3685 #lea $U2(%rsp), $a_ptr
3686 #lea $Hsqr(%rsp), $r_ptr # 2*U1*H^2
3687 #call __ecp_nistz256_mul_by_2 # ecp_nistz256_mul_by_2(Hsqr, U2);
3690 add
$acc0, $acc0 # a0:a3+a0:a3
3691 lea
$Rsqr(%rsp), $a_ptr
3708 mov
8*0($a_ptr), $t0
3710 mov
8*1($a_ptr), $t1
3712 mov
8*2($a_ptr), $t2
3714 mov
8*3($a_ptr), $t3
3716 call __ecp_nistz256_sub
$x # p256_sub(res_x, Rsqr, Hsqr);
3718 lea
$Hcub(%rsp), $b_ptr
3719 lea
$res_x(%rsp), $r_ptr
3720 call __ecp_nistz256_sub_from
$x # p256_sub(res_x, res_x, Hcub);
3722 mov
$U2+8*0(%rsp), $t0
3723 mov
$U2+8*1(%rsp), $t1
3724 mov
$U2+8*2(%rsp), $t2
3725 mov
$U2+8*3(%rsp), $t3
3726 lea
$res_y(%rsp), $r_ptr
3728 call __ecp_nistz256_sub
$x # p256_sub(res_y, U2, res_x);
3730 mov
$acc0, 8*0($r_ptr) # save the result, as
3731 mov
$acc1, 8*1($r_ptr) # __ecp_nistz256_sub doesn't
3732 mov
$acc2, 8*2($r_ptr)
3733 mov
$acc3, 8*3($r_ptr)
3737 `&load_for_mul("$S1(%rsp)", "$Hcub(%rsp)", "$src0")`
3738 lea
$S2(%rsp), $r_ptr
3739 call __ecp_nistz256_mul_mont
$x # p256_mul_mont(S2, S1, Hcub);
3741 `&load_for_mul("$R(%rsp)", "$res_y(%rsp)", "$src0")`
3742 lea
$res_y(%rsp), $r_ptr
3743 call __ecp_nistz256_mul_mont
$x # p256_mul_mont(res_y, R, res_y);
3745 lea
$S2(%rsp), $b_ptr
3746 lea
$res_y(%rsp), $r_ptr
3747 call __ecp_nistz256_sub_from
$x # p256_sub(res_y, res_y, S2);
3749 movq
%xmm0, $r_ptr # restore $r_ptr
3751 movdqa
%xmm5, %xmm0 # copy_conditional(res_z, in2_z, in1infty);
3753 pandn
$res_z(%rsp), %xmm0
3755 pandn
$res_z+0x10(%rsp), %xmm1
3757 pand
$in2_z(%rsp), %xmm2
3758 pand
$in2_z+0x10(%rsp), %xmm3
3762 movdqa
%xmm4, %xmm0 # copy_conditional(res_z, in1_z, in2infty);
3768 pand
$in1_z(%rsp), %xmm2
3769 pand
$in1_z+0x10(%rsp), %xmm3
3772 movdqu
%xmm2, 0x40($r_ptr)
3773 movdqu
%xmm3, 0x50($r_ptr)
3775 movdqa
%xmm5, %xmm0 # copy_conditional(res_x, in2_x, in1infty);
3777 pandn
$res_x(%rsp), %xmm0
3779 pandn
$res_x+0x10(%rsp), %xmm1
3781 pand
$in2_x(%rsp), %xmm2
3782 pand
$in2_x+0x10(%rsp), %xmm3
3786 movdqa
%xmm4, %xmm0 # copy_conditional(res_x, in1_x, in2infty);
3792 pand
$in1_x(%rsp), %xmm2
3793 pand
$in1_x+0x10(%rsp), %xmm3
3796 movdqu
%xmm2, 0x00($r_ptr)
3797 movdqu
%xmm3, 0x10($r_ptr)
3799 movdqa
%xmm5, %xmm0 # copy_conditional(res_y, in2_y, in1infty);
3801 pandn
$res_y(%rsp), %xmm0
3803 pandn
$res_y+0x10(%rsp), %xmm1
3805 pand
$in2_y(%rsp), %xmm2
3806 pand
$in2_y+0x10(%rsp), %xmm3
3810 movdqa
%xmm4, %xmm0 # copy_conditional(res_y, in1_y, in2infty);
3816 pand
$in1_y(%rsp), %xmm2
3817 pand
$in1_y+0x10(%rsp), %xmm3
3820 movdqu
%xmm2, 0x20($r_ptr)
3821 movdqu
%xmm3, 0x30($r_ptr)
3824 lea
32*18+56(%rsp), %rsi
3839 .cfi_def_cfa_register
%rsp
3840 .Lpoint_add
${x
}_epilogue
:
3843 .size ecp_nistz256_point_add
$sfx,.-ecp_nistz256_point_add
$sfx
3848 sub gen_add_affine
() {
3850 my ($src0,$sfx,$bias);
3851 my ($U2,$S2,$H,$R,$Hsqr,$Hcub,$Rsqr,
3852 $res_x,$res_y,$res_z,
3853 $in1_x,$in1_y,$in1_z,
3854 $in2_x,$in2_y)=map(32*$_,(0..14));
3863 .globl ecp_nistz256_point_add_affine
3864 .type ecp_nistz256_point_add_affine
,\
@function,3
3866 ecp_nistz256_point_add_affine
:
3869 $code.=<<___
if ($addx);
3871 and OPENSSL_ia32cap_P
+8(%rip), %ecx
3873 je
.Lpoint_add_affinex
3881 .type ecp_nistz256_point_add_affinex
,\
@function,3
3883 ecp_nistz256_point_add_affinex
:
3885 .Lpoint_add_affinex
:
3902 .cfi_adjust_cfa_offset
32*15+8
3903 .Ladd_affine
${x
}_body
:
3905 movdqu
0x00($a_ptr), %xmm0 # copy *(P256_POINT *)$a_ptr
3906 mov
$b_org, $b_ptr # reassign
3907 movdqu
0x10($a_ptr), %xmm1
3908 movdqu
0x20($a_ptr), %xmm2
3909 movdqu
0x30($a_ptr), %xmm3
3910 movdqu
0x40($a_ptr), %xmm4
3911 movdqu
0x50($a_ptr), %xmm5
3912 mov
0x40+8*0($a_ptr), $src0 # load original in1_z
3913 mov
0x40+8*1($a_ptr), $acc6
3914 mov
0x40+8*2($a_ptr), $acc7
3915 mov
0x40+8*3($a_ptr), $acc0
3916 movdqa
%xmm0, $in1_x(%rsp)
3917 movdqa
%xmm1, $in1_x+0x10(%rsp)
3918 movdqa
%xmm2, $in1_y(%rsp)
3919 movdqa
%xmm3, $in1_y+0x10(%rsp)
3920 movdqa
%xmm4, $in1_z(%rsp)
3921 movdqa
%xmm5, $in1_z+0x10(%rsp)
3924 movdqu
0x00($b_ptr), %xmm0 # copy *(P256_POINT_AFFINE *)$b_ptr
3925 pshufd \
$0xb1, %xmm5, %xmm3
3926 movdqu
0x10($b_ptr), %xmm1
3927 movdqu
0x20($b_ptr), %xmm2
3929 movdqu
0x30($b_ptr), %xmm3
3930 movdqa
%xmm0, $in2_x(%rsp)
3931 pshufd \
$0x1e, %xmm5, %xmm4
3932 movdqa
%xmm1, $in2_x+0x10(%rsp)
3934 movq
$r_ptr, %xmm0 # save $r_ptr
3935 movdqa
%xmm2, $in2_y(%rsp)
3936 movdqa
%xmm3, $in2_y+0x10(%rsp)
3942 lea
0x40-$bias($a_ptr), $a_ptr # $a_ptr is still valid
3943 lea
$Z1sqr(%rsp), $r_ptr # Z1^2
3944 call __ecp_nistz256_sqr_mont
$x # p256_sqr_mont(Z1sqr, in1_z);
3946 pcmpeqd
%xmm4, %xmm5
3947 pshufd \
$0xb1, %xmm3, %xmm4
3948 mov
0x00($b_ptr), $src0 # $b_ptr is still valid
3949 #lea 0x00($b_ptr), $b_ptr
3950 mov
$acc4, $acc1 # harmonize sqr output and mul input
3952 pshufd \
$0, %xmm5, %xmm5 # in1infty
3953 pshufd \
$0x1e, %xmm4, %xmm3
3958 pcmpeqd
%xmm3, %xmm4
3959 pshufd \
$0, %xmm4, %xmm4 # in2infty
3961 lea
$Z1sqr-$bias(%rsp), $a_ptr
3963 lea
$U2(%rsp), $r_ptr # U2 = X2*Z1^2
3964 call __ecp_nistz256_mul_mont
$x # p256_mul_mont(U2, Z1sqr, in2_x);
3966 lea
$in1_x(%rsp), $b_ptr
3967 lea
$H(%rsp), $r_ptr # H = U2 - U1
3968 call __ecp_nistz256_sub_from
$x # p256_sub(H, U2, in1_x);
3970 `&load_for_mul("$Z1sqr(%rsp)", "$in1_z(%rsp)", "$src0")`
3971 lea
$S2(%rsp), $r_ptr # S2 = Z1^3
3972 call __ecp_nistz256_mul_mont
$x # p256_mul_mont(S2, Z1sqr, in1_z);
3974 `&load_for_mul("$H(%rsp)", "$in1_z(%rsp)", "$src0")`
3975 lea
$res_z(%rsp), $r_ptr # Z3 = H*Z1*Z2
3976 call __ecp_nistz256_mul_mont
$x # p256_mul_mont(res_z, H, in1_z);
3978 `&load_for_mul("$S2(%rsp)", "$in2_y(%rsp)", "$src0")`
3979 lea
$S2(%rsp), $r_ptr # S2 = Y2*Z1^3
3980 call __ecp_nistz256_mul_mont
$x # p256_mul_mont(S2, S2, in2_y);
3982 lea
$in1_y(%rsp), $b_ptr
3983 lea
$R(%rsp), $r_ptr # R = S2 - S1
3984 call __ecp_nistz256_sub_from
$x # p256_sub(R, S2, in1_y);
3986 `&load_for_sqr("$H(%rsp)", "$src0")`
3987 lea
$Hsqr(%rsp), $r_ptr # H^2
3988 call __ecp_nistz256_sqr_mont
$x # p256_sqr_mont(Hsqr, H);
3990 `&load_for_sqr("$R(%rsp)", "$src0")`
3991 lea
$Rsqr(%rsp), $r_ptr # R^2
3992 call __ecp_nistz256_sqr_mont
$x # p256_sqr_mont(Rsqr, R);
3994 `&load_for_mul("$H(%rsp)", "$Hsqr(%rsp)", "$src0")`
3995 lea
$Hcub(%rsp), $r_ptr # H^3
3996 call __ecp_nistz256_mul_mont
$x # p256_mul_mont(Hcub, Hsqr, H);
3998 `&load_for_mul("$Hsqr(%rsp)", "$in1_x(%rsp)", "$src0")`
3999 lea
$U2(%rsp), $r_ptr # U1*H^2
4000 call __ecp_nistz256_mul_mont
$x # p256_mul_mont(U2, in1_x, Hsqr);
4003 #######################################################################
4004 # operate in 4-5-0-1 "name space" that matches multiplication output
4006 my ($acc0,$acc1,$acc2,$acc3,$t3,$t4)=($acc4,$acc5,$acc0,$acc1,$acc2,$acc3);
4007 my ($poly1, $poly3)=($acc6,$acc7);
4010 #lea $U2(%rsp), $a_ptr
4011 #lea $Hsqr(%rsp), $r_ptr # 2*U1*H^2
4012 #call __ecp_nistz256_mul_by_2 # ecp_nistz256_mul_by_2(Hsqr, U2);
4015 add
$acc0, $acc0 # a0:a3+a0:a3
4016 lea
$Rsqr(%rsp), $a_ptr
4033 mov
8*0($a_ptr), $t0
4035 mov
8*1($a_ptr), $t1
4037 mov
8*2($a_ptr), $t2
4039 mov
8*3($a_ptr), $t3
4041 call __ecp_nistz256_sub
$x # p256_sub(res_x, Rsqr, Hsqr);
4043 lea
$Hcub(%rsp), $b_ptr
4044 lea
$res_x(%rsp), $r_ptr
4045 call __ecp_nistz256_sub_from
$x # p256_sub(res_x, res_x, Hcub);
4047 mov
$U2+8*0(%rsp), $t0
4048 mov
$U2+8*1(%rsp), $t1
4049 mov
$U2+8*2(%rsp), $t2
4050 mov
$U2+8*3(%rsp), $t3
4051 lea
$H(%rsp), $r_ptr
4053 call __ecp_nistz256_sub
$x # p256_sub(H, U2, res_x);
4055 mov
$acc0, 8*0($r_ptr) # save the result, as
4056 mov
$acc1, 8*1($r_ptr) # __ecp_nistz256_sub doesn't
4057 mov
$acc2, 8*2($r_ptr)
4058 mov
$acc3, 8*3($r_ptr)
4062 `&load_for_mul("$Hcub(%rsp)", "$in1_y(%rsp)", "$src0")`
4063 lea
$S2(%rsp), $r_ptr
4064 call __ecp_nistz256_mul_mont
$x # p256_mul_mont(S2, Hcub, in1_y);
4066 `&load_for_mul("$H(%rsp)", "$R(%rsp)", "$src0")`
4067 lea
$H(%rsp), $r_ptr
4068 call __ecp_nistz256_mul_mont
$x # p256_mul_mont(H, H, R);
4070 lea
$S2(%rsp), $b_ptr
4071 lea
$res_y(%rsp), $r_ptr
4072 call __ecp_nistz256_sub_from
$x # p256_sub(res_y, H, S2);
4074 movq
%xmm0, $r_ptr # restore $r_ptr
4076 movdqa
%xmm5, %xmm0 # copy_conditional(res_z, ONE, in1infty);
4078 pandn
$res_z(%rsp), %xmm0
4080 pandn
$res_z+0x10(%rsp), %xmm1
4082 pand
.LONE_mont
(%rip), %xmm2
4083 pand
.LONE_mont
+0x10(%rip), %xmm3
4087 movdqa
%xmm4, %xmm0 # copy_conditional(res_z, in1_z, in2infty);
4093 pand
$in1_z(%rsp), %xmm2
4094 pand
$in1_z+0x10(%rsp), %xmm3
4097 movdqu
%xmm2, 0x40($r_ptr)
4098 movdqu
%xmm3, 0x50($r_ptr)
4100 movdqa
%xmm5, %xmm0 # copy_conditional(res_x, in2_x, in1infty);
4102 pandn
$res_x(%rsp), %xmm0
4104 pandn
$res_x+0x10(%rsp), %xmm1
4106 pand
$in2_x(%rsp), %xmm2
4107 pand
$in2_x+0x10(%rsp), %xmm3
4111 movdqa
%xmm4, %xmm0 # copy_conditional(res_x, in1_x, in2infty);
4117 pand
$in1_x(%rsp), %xmm2
4118 pand
$in1_x+0x10(%rsp), %xmm3
4121 movdqu
%xmm2, 0x00($r_ptr)
4122 movdqu
%xmm3, 0x10($r_ptr)
4124 movdqa
%xmm5, %xmm0 # copy_conditional(res_y, in2_y, in1infty);
4126 pandn
$res_y(%rsp), %xmm0
4128 pandn
$res_y+0x10(%rsp), %xmm1
4130 pand
$in2_y(%rsp), %xmm2
4131 pand
$in2_y+0x10(%rsp), %xmm3
4135 movdqa
%xmm4, %xmm0 # copy_conditional(res_y, in1_y, in2infty);
4141 pand
$in1_y(%rsp), %xmm2
4142 pand
$in1_y+0x10(%rsp), %xmm3
4145 movdqu
%xmm2, 0x20($r_ptr)
4146 movdqu
%xmm3, 0x30($r_ptr)
4148 lea
32*15+56(%rsp), %rsi
4163 .cfi_def_cfa_register
%rsp
4164 .Ladd_affine
${x
}_epilogue
:
4167 .size ecp_nistz256_point_add_affine
$sfx,.-ecp_nistz256_point_add_affine
$sfx
4170 &gen_add_affine
("q");
4172 ########################################################################
4176 ########################################################################
4177 # operate in 4-5-0-1 "name space" that matches multiplication output
4179 my ($a0,$a1,$a2,$a3,$t3,$t4)=($acc4,$acc5,$acc0,$acc1,$acc2,$acc3);
4182 .type __ecp_nistz256_add_tox
,\
@abi-omnipotent
4184 __ecp_nistz256_add_tox
:
4187 adc
8*0($b_ptr), $a0
4188 adc
8*1($b_ptr), $a1
4190 adc
8*2($b_ptr), $a2
4191 adc
8*3($b_ptr), $a3
4206 mov
$a0, 8*0($r_ptr)
4208 mov
$a1, 8*1($r_ptr)
4210 mov
$a2, 8*2($r_ptr)
4211 mov
$a3, 8*3($r_ptr)
4215 .size __ecp_nistz256_add_tox
,.-__ecp_nistz256_add_tox
4217 .type __ecp_nistz256_sub_fromx
,\
@abi-omnipotent
4219 __ecp_nistz256_sub_fromx
:
4222 sbb
8*0($b_ptr), $a0
4223 sbb
8*1($b_ptr), $a1
4225 sbb
8*2($b_ptr), $a2
4226 sbb
8*3($b_ptr), $a3
4241 mov
$a0, 8*0($r_ptr)
4243 mov
$a1, 8*1($r_ptr)
4245 mov
$a2, 8*2($r_ptr)
4246 mov
$a3, 8*3($r_ptr)
4250 .size __ecp_nistz256_sub_fromx
,.-__ecp_nistz256_sub_fromx
4252 .type __ecp_nistz256_subx
,\
@abi-omnipotent
4254 __ecp_nistz256_subx
:
4281 .size __ecp_nistz256_subx
,.-__ecp_nistz256_subx
4283 .type __ecp_nistz256_mul_by_2x
,\
@abi-omnipotent
4285 __ecp_nistz256_mul_by_2x
:
4288 adc
$a0, $a0 # a0:a3+a0:a3
4307 mov
$a0, 8*0($r_ptr)
4309 mov
$a1, 8*1($r_ptr)
4311 mov
$a2, 8*2($r_ptr)
4312 mov
$a3, 8*3($r_ptr)
4316 .size __ecp_nistz256_mul_by_2x
,.-__ecp_nistz256_mul_by_2x
4321 &gen_add_affine
("x");
4325 # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
4326 # CONTEXT *context,DISPATCHER_CONTEXT *disp)
4334 .extern __imp_RtlVirtualUnwind
4336 .type short_handler
,\
@abi-omnipotent
4350 mov
120($context),%rax # pull context->Rax
4351 mov
248($context),%rbx # pull context->Rip
4353 mov
8($disp),%rsi # disp->ImageBase
4354 mov
56($disp),%r11 # disp->HandlerData
4356 mov
0(%r11),%r10d # HandlerData[0]
4357 lea
(%rsi,%r10),%r10 # end of prologue label
4358 cmp %r10,%rbx # context->Rip<end of prologue label
4359 jb
.Lcommon_seh_tail
4361 mov
152($context),%rax # pull context->Rsp
4363 mov
4(%r11),%r10d # HandlerData[1]
4364 lea
(%rsi,%r10),%r10 # epilogue label
4365 cmp %r10,%rbx # context->Rip>=epilogue label
4366 jae
.Lcommon_seh_tail
4372 mov
%r12,216($context) # restore context->R12
4373 mov
%r13,224($context) # restore context->R13
4375 jmp
.Lcommon_seh_tail
4376 .size short_handler
,.-short_handler
4378 .type full_handler
,\
@abi-omnipotent
4392 mov
120($context),%rax # pull context->Rax
4393 mov
248($context),%rbx # pull context->Rip
4395 mov
8($disp),%rsi # disp->ImageBase
4396 mov
56($disp),%r11 # disp->HandlerData
4398 mov
0(%r11),%r10d # HandlerData[0]
4399 lea
(%rsi,%r10),%r10 # end of prologue label
4400 cmp %r10,%rbx # context->Rip<end of prologue label
4401 jb
.Lcommon_seh_tail
4403 mov
152($context),%rax # pull context->Rsp
4405 mov
4(%r11),%r10d # HandlerData[1]
4406 lea
(%rsi,%r10),%r10 # epilogue label
4407 cmp %r10,%rbx # context->Rip>=epilogue label
4408 jae
.Lcommon_seh_tail
4410 mov
8(%r11),%r10d # HandlerData[2]
4411 lea
(%rax,%r10),%rax
4419 mov
%rbx,144($context) # restore context->Rbx
4420 mov
%rbp,160($context) # restore context->Rbp
4421 mov
%r12,216($context) # restore context->R12
4422 mov
%r13,224($context) # restore context->R13
4423 mov
%r14,232($context) # restore context->R14
4424 mov
%r15,240($context) # restore context->R15
4429 mov
%rax,152($context) # restore context->Rsp
4430 mov
%rsi,168($context) # restore context->Rsi
4431 mov
%rdi,176($context) # restore context->Rdi
4433 mov
40($disp),%rdi # disp->ContextRecord
4434 mov
$context,%rsi # context
4435 mov \
$154,%ecx # sizeof(CONTEXT)
4436 .long
0xa548f3fc # cld; rep movsq
4439 xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER
4440 mov
8(%rsi),%rdx # arg2, disp->ImageBase
4441 mov
0(%rsi),%r8 # arg3, disp->ControlPc
4442 mov
16(%rsi),%r9 # arg4, disp->FunctionEntry
4443 mov
40(%rsi),%r10 # disp->ContextRecord
4444 lea
56(%rsi),%r11 # &disp->HandlerData
4445 lea
24(%rsi),%r12 # &disp->EstablisherFrame
4446 mov
%r10,32(%rsp) # arg5
4447 mov
%r11,40(%rsp) # arg6
4448 mov
%r12,48(%rsp) # arg7
4449 mov
%rcx,56(%rsp) # arg8, (NULL)
4450 call
*__imp_RtlVirtualUnwind
(%rip)
4452 mov \
$1,%eax # ExceptionContinueSearch
4464 .size full_handler
,.-full_handler
4468 .rva
.LSEH_begin_ecp_nistz256_mul_by_2
4469 .rva
.LSEH_end_ecp_nistz256_mul_by_2
4470 .rva
.LSEH_info_ecp_nistz256_mul_by_2
4472 .rva
.LSEH_begin_ecp_nistz256_div_by_2
4473 .rva
.LSEH_end_ecp_nistz256_div_by_2
4474 .rva
.LSEH_info_ecp_nistz256_div_by_2
4476 .rva
.LSEH_begin_ecp_nistz256_mul_by_3
4477 .rva
.LSEH_end_ecp_nistz256_mul_by_3
4478 .rva
.LSEH_info_ecp_nistz256_mul_by_3
4480 .rva
.LSEH_begin_ecp_nistz256_add
4481 .rva
.LSEH_end_ecp_nistz256_add
4482 .rva
.LSEH_info_ecp_nistz256_add
4484 .rva
.LSEH_begin_ecp_nistz256_sub
4485 .rva
.LSEH_end_ecp_nistz256_sub
4486 .rva
.LSEH_info_ecp_nistz256_sub
4488 .rva
.LSEH_begin_ecp_nistz256_neg
4489 .rva
.LSEH_end_ecp_nistz256_neg
4490 .rva
.LSEH_info_ecp_nistz256_neg
4492 .rva
.LSEH_begin_ecp_nistz256_ord_mul_mont
4493 .rva
.LSEH_end_ecp_nistz256_ord_mul_mont
4494 .rva
.LSEH_info_ecp_nistz256_ord_mul_mont
4496 .rva
.LSEH_begin_ecp_nistz256_ord_sqr_mont
4497 .rva
.LSEH_end_ecp_nistz256_ord_sqr_mont
4498 .rva
.LSEH_info_ecp_nistz256_ord_sqr_mont
4500 $code.=<<___
if ($addx);
4501 .rva
.LSEH_begin_ecp_nistz256_ord_mul_montx
4502 .rva
.LSEH_end_ecp_nistz256_ord_mul_montx
4503 .rva
.LSEH_info_ecp_nistz256_ord_mul_montx
4505 .rva
.LSEH_begin_ecp_nistz256_ord_sqr_montx
4506 .rva
.LSEH_end_ecp_nistz256_ord_sqr_montx
4507 .rva
.LSEH_info_ecp_nistz256_ord_sqr_montx
4510 .rva
.LSEH_begin_ecp_nistz256_to_mont
4511 .rva
.LSEH_end_ecp_nistz256_to_mont
4512 .rva
.LSEH_info_ecp_nistz256_to_mont
4514 .rva
.LSEH_begin_ecp_nistz256_mul_mont
4515 .rva
.LSEH_end_ecp_nistz256_mul_mont
4516 .rva
.LSEH_info_ecp_nistz256_mul_mont
4518 .rva
.LSEH_begin_ecp_nistz256_sqr_mont
4519 .rva
.LSEH_end_ecp_nistz256_sqr_mont
4520 .rva
.LSEH_info_ecp_nistz256_sqr_mont
4522 .rva
.LSEH_begin_ecp_nistz256_from_mont
4523 .rva
.LSEH_end_ecp_nistz256_from_mont
4524 .rva
.LSEH_info_ecp_nistz256_from_mont
4526 .rva
.LSEH_begin_ecp_nistz256_gather_w5
4527 .rva
.LSEH_end_ecp_nistz256_gather_w5
4528 .rva
.LSEH_info_ecp_nistz256_gather_wX
4530 .rva
.LSEH_begin_ecp_nistz256_gather_w7
4531 .rva
.LSEH_end_ecp_nistz256_gather_w7
4532 .rva
.LSEH_info_ecp_nistz256_gather_wX
4534 $code.=<<___
if ($avx>1);
4535 .rva
.LSEH_begin_ecp_nistz256_avx2_gather_w5
4536 .rva
.LSEH_end_ecp_nistz256_avx2_gather_w5
4537 .rva
.LSEH_info_ecp_nistz256_avx2_gather_wX
4539 .rva
.LSEH_begin_ecp_nistz256_avx2_gather_w7
4540 .rva
.LSEH_end_ecp_nistz256_avx2_gather_w7
4541 .rva
.LSEH_info_ecp_nistz256_avx2_gather_wX
4544 .rva
.LSEH_begin_ecp_nistz256_point_double
4545 .rva
.LSEH_end_ecp_nistz256_point_double
4546 .rva
.LSEH_info_ecp_nistz256_point_double
4548 .rva
.LSEH_begin_ecp_nistz256_point_add
4549 .rva
.LSEH_end_ecp_nistz256_point_add
4550 .rva
.LSEH_info_ecp_nistz256_point_add
4552 .rva
.LSEH_begin_ecp_nistz256_point_add_affine
4553 .rva
.LSEH_end_ecp_nistz256_point_add_affine
4554 .rva
.LSEH_info_ecp_nistz256_point_add_affine
4556 $code.=<<___
if ($addx);
4557 .rva
.LSEH_begin_ecp_nistz256_point_doublex
4558 .rva
.LSEH_end_ecp_nistz256_point_doublex
4559 .rva
.LSEH_info_ecp_nistz256_point_doublex
4561 .rva
.LSEH_begin_ecp_nistz256_point_addx
4562 .rva
.LSEH_end_ecp_nistz256_point_addx
4563 .rva
.LSEH_info_ecp_nistz256_point_addx
4565 .rva
.LSEH_begin_ecp_nistz256_point_add_affinex
4566 .rva
.LSEH_end_ecp_nistz256_point_add_affinex
4567 .rva
.LSEH_info_ecp_nistz256_point_add_affinex
4573 .LSEH_info_ecp_nistz256_mul_by_2
:
4576 .rva
.Lmul_by_2_body
,.Lmul_by_2_epilogue
# HandlerData[]
4577 .LSEH_info_ecp_nistz256_div_by_2
:
4580 .rva
.Ldiv_by_2_body
,.Ldiv_by_2_epilogue
# HandlerData[]
4581 .LSEH_info_ecp_nistz256_mul_by_3
:
4584 .rva
.Lmul_by_3_body
,.Lmul_by_3_epilogue
# HandlerData[]
4585 .LSEH_info_ecp_nistz256_add
:
4588 .rva
.Ladd_body
,.Ladd_epilogue
# HandlerData[]
4589 .LSEH_info_ecp_nistz256_sub
:
4592 .rva
.Lsub_body
,.Lsub_epilogue
# HandlerData[]
4593 .LSEH_info_ecp_nistz256_neg
:
4596 .rva
.Lneg_body
,.Lneg_epilogue
# HandlerData[]
4597 .LSEH_info_ecp_nistz256_ord_mul_mont
:
4600 .rva
.Lord_mul_body
,.Lord_mul_epilogue
# HandlerData[]
4602 .LSEH_info_ecp_nistz256_ord_sqr_mont
:
4605 .rva
.Lord_sqr_body
,.Lord_sqr_epilogue
# HandlerData[]
4608 $code.=<<___
if ($addx);
4609 .LSEH_info_ecp_nistz256_ord_mul_montx
:
4612 .rva
.Lord_mulx_body
,.Lord_mulx_epilogue
# HandlerData[]
4614 .LSEH_info_ecp_nistz256_ord_sqr_montx
:
4617 .rva
.Lord_sqrx_body
,.Lord_sqrx_epilogue
# HandlerData[]
4621 .LSEH_info_ecp_nistz256_to_mont
:
4624 .rva
.Lmul_body
,.Lmul_epilogue
# HandlerData[]
4626 .LSEH_info_ecp_nistz256_mul_mont
:
4629 .rva
.Lmul_body
,.Lmul_epilogue
# HandlerData[]
4631 .LSEH_info_ecp_nistz256_sqr_mont
:
4634 .rva
.Lsqr_body
,.Lsqr_epilogue
# HandlerData[]
4636 .LSEH_info_ecp_nistz256_from_mont
:
4639 .rva
.Lfrom_body
,.Lfrom_epilogue
# HandlerData[]
4640 .LSEH_info_ecp_nistz256_gather_wX
:
4641 .byte
0x01,0x33,0x16,0x00
4642 .byte
0x33,0xf8,0x09,0x00 #movaps 0x90(rsp),xmm15
4643 .byte
0x2e,0xe8,0x08,0x00 #movaps 0x80(rsp),xmm14
4644 .byte
0x29,0xd8,0x07,0x00 #movaps 0x70(rsp),xmm13
4645 .byte
0x24,0xc8,0x06,0x00 #movaps 0x60(rsp),xmm12
4646 .byte
0x1f,0xb8,0x05,0x00 #movaps 0x50(rsp),xmm11
4647 .byte
0x1a,0xa8,0x04,0x00 #movaps 0x40(rsp),xmm10
4648 .byte
0x15,0x98,0x03,0x00 #movaps 0x30(rsp),xmm9
4649 .byte
0x10,0x88,0x02,0x00 #movaps 0x20(rsp),xmm8
4650 .byte
0x0c,0x78,0x01,0x00 #movaps 0x10(rsp),xmm7
4651 .byte
0x08,0x68,0x00,0x00 #movaps 0x00(rsp),xmm6
4652 .byte
0x04,0x01,0x15,0x00 #sub rsp,0xa8
4655 $code.=<<___
if ($avx>1);
4656 .LSEH_info_ecp_nistz256_avx2_gather_wX
:
4657 .byte
0x01,0x36,0x17,0x0b
4658 .byte
0x36,0xf8,0x09,0x00 # vmovaps 0x90(rsp),xmm15
4659 .byte
0x31,0xe8,0x08,0x00 # vmovaps 0x80(rsp),xmm14
4660 .byte
0x2c,0xd8,0x07,0x00 # vmovaps 0x70(rsp),xmm13
4661 .byte
0x27,0xc8,0x06,0x00 # vmovaps 0x60(rsp),xmm12
4662 .byte
0x22,0xb8,0x05,0x00 # vmovaps 0x50(rsp),xmm11
4663 .byte
0x1d,0xa8,0x04,0x00 # vmovaps 0x40(rsp),xmm10
4664 .byte
0x18,0x98,0x03,0x00 # vmovaps 0x30(rsp),xmm9
4665 .byte
0x13,0x88,0x02,0x00 # vmovaps 0x20(rsp),xmm8
4666 .byte
0x0e,0x78,0x01,0x00 # vmovaps 0x10(rsp),xmm7
4667 .byte
0x09,0x68,0x00,0x00 # vmovaps 0x00(rsp),xmm6
4668 .byte
0x04,0x01,0x15,0x00 # sub rsp,0xa8
4669 .byte
0x00,0xb3,0x00,0x00 # set_frame r11
4673 .LSEH_info_ecp_nistz256_point_double
:
4676 .rva
.Lpoint_doubleq_body
,.Lpoint_doubleq_epilogue
# HandlerData[]
4678 .LSEH_info_ecp_nistz256_point_add
:
4681 .rva
.Lpoint_addq_body
,.Lpoint_addq_epilogue
# HandlerData[]
4683 .LSEH_info_ecp_nistz256_point_add_affine
:
4686 .rva
.Ladd_affineq_body
,.Ladd_affineq_epilogue
# HandlerData[]
4689 $code.=<<___
if ($addx);
4691 .LSEH_info_ecp_nistz256_point_doublex
:
4694 .rva
.Lpoint_doublex_body
,.Lpoint_doublex_epilogue
# HandlerData[]
4696 .LSEH_info_ecp_nistz256_point_addx
:
4699 .rva
.Lpoint_addx_body
,.Lpoint_addx_epilogue
# HandlerData[]
4701 .LSEH_info_ecp_nistz256_point_add_affinex
:
4704 .rva
.Ladd_affinex_body
,.Ladd_affinex_epilogue
# HandlerData[]
4709 ########################################################################
4710 # Convert ecp_nistz256_table.c to layout expected by ecp_nistz_gather_w7
4712 open TABLE
,"<ecp_nistz256_table.c" or
4713 open TABLE
,"<${dir}../ecp_nistz256_table.c" or
4714 die "failed to open ecp_nistz256_table.c:",$!;
4719 s/TOBN\(\s*(0x[0-9a-f]+),\s*(0x[0-9a-f]+)\s*\)/push @arr,hex($2),hex($1)/geo;
4723 die "insane number of elements" if ($#arr != 64*16*37-1);
4727 .globl ecp_nistz256_precomputed
4728 .type ecp_nistz256_precomputed
,\
@object
4730 ecp_nistz256_precomputed
:
4732 while (@line=splice(@arr,0,16)) {
4733 print ".long\t",join(',',map { sprintf "0x%08x",$_} @line),"\n";
4736 .size ecp_nistz256_precomputed
,.-ecp_nistz256_precomputed
4739 $code =~ s/\`([^\`]*)\`/eval $1/gem;
4741 close STDOUT
or die "error closing STDOUT: $!";