projects / thirdparty / openssl.git / blame
| Commit | Line | Data |
|---|---|---|
| 3c2bdd7d | 1 | # Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. |
| c781eb1c AM |
2 | # Copyright (c) 2020, Intel Corporation. All Rights Reserved. |
| 3 | # | |
| 4 | # Licensed under the Apache License 2.0 (the "License"). You may not use | |
| 5 | # this file except in compliance with the License. You can obtain a copy | |
| 6 | # in the file LICENSE in the source distribution or at | |
| 7 | # https://www.openssl.org/source/license.html | |
| 8 | # | |
| 9 | # | |
| f87b4c4e AM |
10 | # Originally written by Sergey Kirillov and Andrey Matyukov. |
| 11 | # Special thanks to Ilya Albrekht for his valuable hints. | |
| c781eb1c AM |
12 | # Intel Corporation |
| 13 | # | |
| 14 | # December 2020 | |
| 15 | # | |
| 16 | # Initial release. | |
| 17 | # | |
| 18 | # Implementation utilizes 256-bit (ymm) registers to avoid frequency scaling issues. | |
| 19 | # | |
| 20 | # IceLake-Client @ 1.3GHz | |
| 21 | # |---------+----------------------+--------------+-------------| | |
| 22 | # | | OpenSSL 3.0.0-alpha9 | this | Unit | | |
| 23 | # |---------+----------------------+--------------+-------------| | |
| 24 | # | rsa2048 | 2 127 659 | 1 015 625 | cycles/sign | | |
| 25 | # | | 611 | 1280 / +109% | sign/s | | |
| 26 | # |---------+----------------------+--------------+-------------| | |
| 27 | # | |
| 28 | ||
| 29 | # $output is the last argument if it looks like a file (it has an extension) | |
| 30 | # $flavour is the first argument if it doesn't look like a file | |
| 31 | $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; | |
| 32 | $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; | |
| 33 | ||
| 34 | $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); | |
| 35 | $avx512ifma=0; | |
| 36 | ||
| 37 | $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; | |
| 38 | ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or | |
| 39 | ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or | |
| 40 | die "can't locate x86_64-xlate.pl"; | |
| 41 | ||
| 42 | if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` | |
| 43 | =~ /GNU assembler version ([2-9]\.[0-9]+)/) { | |
| 44 | $avx512ifma = ($1>=2.26); | |
| 45 | } | |
| 46 | ||
| 47 | if (!$avx512 && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && | |
| 48 | `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)(?:\.([0-9]+))?/) { | |
| 49 | $avx512ifma = ($1==2.11 && $2>=8) + ($1>=2.12); | |
| 50 | } | |
| 51 | ||
| 52 | if (!$avx512 && `$ENV{CC} -v 2>&1` =~ /((?:clang|LLVM) version|.*based on LLVM) ([0-9]+\.[0-9]+)/) { | |
| 788a72e9 | 53 | $avx512ifma = ($2>=7.0); |
| c781eb1c AM |
54 | } |
| 55 | ||
| 56 | open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"" | |
| 57 | or die "can't call $xlate: $!"; | |
| 58 | *STDOUT=*OUT; | |
| 59 | ||
| 60 | if ($avx512ifma>0) {{{ | |
| 61 | @_6_args_universal_ABI = ("%rdi","%rsi","%rdx","%rcx","%r8","%r9"); | |
| 62 | ||
| 63 | $code.=<<___; | |
| 64 | .extern OPENSSL_ia32cap_P | |
| e475d9a4 P |
65 | .globl ossl_rsaz_avx512ifma_eligible |
| 66 | .type ossl_rsaz_avx512ifma_eligible,\@abi-omnipotent | |
| c781eb1c | 67 | .align 32 |
| e475d9a4 | 68 | ossl_rsaz_avx512ifma_eligible: |
| c781eb1c AM |
69 | mov OPENSSL_ia32cap_P+8(%rip), %ecx |
| 70 | xor %eax,%eax | |
| 71 | and \$`1<<31|1<<21|1<<17|1<<16`, %ecx # avx512vl + avx512ifma + avx512dq + avx512f | |
| 72 | cmp \$`1<<31|1<<21|1<<17|1<<16`, %ecx | |
| 73 | cmove %ecx,%eax | |
| 74 | ret | |
| e475d9a4 | 75 | .size ossl_rsaz_avx512ifma_eligible, .-ossl_rsaz_avx512ifma_eligible |
| c781eb1c AM |
76 | ___ |
| 77 | ||
| 78 | ############################################################################### | |
| 79 | # Almost Montgomery Multiplication (AMM) for 20-digit number in radix 2^52. | |
| 80 | # | |
| f87b4c4e | 81 | # AMM is defined as presented in the paper [1]. |
| c781eb1c AM |
82 | # |
| 83 | # The input and output are presented in 2^52 radix domain, i.e. | |
| 84 | # |res|, |a|, |b|, |m| are arrays of 20 64-bit qwords with 12 high bits zeroed. | |
| 85 | # |k0| is a Montgomery coefficient, which is here k0 = -1/m mod 2^64 | |
| c781eb1c | 86 | # |
| f87b4c4e AM |
87 | # NB: the AMM implementation does not perform "conditional" subtraction step |
| 88 | # specified in the original algorithm as according to the Lemma 1 from the paper | |
| 89 | # [2], the result will be always < 2*m and can be used as a direct input to | |
| 90 | # the next AMM iteration. This post-condition is true, provided the correct | |
| e304aa87 | 91 | # parameter |s| (notion of the Lemma 1 from [2]) is chosen, i.e. s >= n + 2 * k, |
| f87b4c4e | 92 | # which matches our case: 1040 > 1024 + 2 * 1. |
| c781eb1c | 93 | # |
| f87b4c4e AM |
94 | # [1] Gueron, S. Efficient software implementations of modular exponentiation. |
| 95 | # DOI: 10.1007/s13389-012-0031-5 | |
| 96 | # [2] Gueron, S. Enhanced Montgomery Multiplication. | |
| 97 | # DOI: 10.1007/3-540-36400-5_5 | |
| 98 | # | |
| 99 | # void ossl_rsaz_amm52x20_x1_ifma256(BN_ULONG *res, | |
| 100 | # const BN_ULONG *a, | |
| 101 | # const BN_ULONG *b, | |
| 102 | # const BN_ULONG *m, | |
| 103 | # BN_ULONG k0); | |
| c781eb1c AM |
104 | ############################################################################### |
| 105 | { | |
| 106 | # input parameters ("%rdi","%rsi","%rdx","%rcx","%r8") | |
| 107 | my ($res,$a,$b,$m,$k0) = @_6_args_universal_ABI; | |
| 108 | ||
| 109 | my $mask52 = "%rax"; | |
| 110 | my $acc0_0 = "%r9"; | |
| 111 | my $acc0_0_low = "%r9d"; | |
| 112 | my $acc0_1 = "%r15"; | |
| 113 | my $acc0_1_low = "%r15d"; | |
| 114 | my $b_ptr = "%r11"; | |
| 115 | ||
| 116 | my $iter = "%ebx"; | |
| 117 | ||
| 118 | my $zero = "%ymm0"; | |
| f87b4c4e AM |
119 | my $Bi = "%ymm1"; |
| 120 | my $Yi = "%ymm2"; | |
| 121 | my ($R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0) = ("%ymm3",map("%ymm$_",(16..19))); | |
| 122 | my ($R0_1,$R0_1h,$R1_1,$R1_1h,$R2_1) = ("%ymm4",map("%ymm$_",(20..23))); | |
| c781eb1c AM |
123 | |
| 124 | # Registers mapping for normalization. | |
| f87b4c4e | 125 | my ($T0,$T0h,$T1,$T1h,$T2) = ("$zero", "$Bi", "$Yi", map("%ymm$_", (25..26))); |
| c781eb1c AM |
126 | |
| 127 | sub amm52x20_x1() { | |
| 128 | # _data_offset - offset in the |a| or |m| arrays pointing to the beginning | |
| 129 | # of data for corresponding AMM operation; | |
| 130 | # _b_offset - offset in the |b| array pointing to the next qword digit; | |
| 131 | my ($_data_offset,$_b_offset,$_acc,$_R0,$_R0h,$_R1,$_R1h,$_R2,$_k0) = @_; | |
| d136db21 MC |
132 | my $_R0_xmm = $_R0; |
| 133 | $_R0_xmm =~ s/%y/%x/; | |
| c781eb1c AM |
134 | $code.=<<___; |
| 135 | movq $_b_offset($b_ptr), %r13 # b[i] | |
| 136 | ||
| 137 | vpbroadcastq %r13, $Bi # broadcast b[i] | |
| 138 | movq $_data_offset($a), %rdx | |
| 139 | mulx %r13, %r13, %r12 # a[0]*b[i] = (t0,t2) | |
| 140 | addq %r13, $_acc # acc += t0 | |
| 141 | movq %r12, %r10 | |
| 142 | adcq \$0, %r10 # t2 += CF | |
| 143 | ||
| 144 | movq $_k0, %r13 | |
| 145 | imulq $_acc, %r13 # acc * k0 | |
| 146 | andq $mask52, %r13 # yi = (acc * k0) & mask52 | |
| 147 | ||
| 148 | vpbroadcastq %r13, $Yi # broadcast y[i] | |
| 149 | movq $_data_offset($m), %rdx | |
| 150 | mulx %r13, %r13, %r12 # yi * m[0] = (t0,t1) | |
| 151 | addq %r13, $_acc # acc += t0 | |
| 152 | adcq %r12, %r10 # t2 += (t1 + CF) | |
| 153 | ||
| 154 | shrq \$52, $_acc | |
| 155 | salq \$12, %r10 | |
| 156 | or %r10, $_acc # acc = ((acc >> 52) | (t2 << 12)) | |
| 157 | ||
| 158 | vpmadd52luq `$_data_offset+64*0`($a), $Bi, $_R0 | |
| 159 | vpmadd52luq `$_data_offset+64*0+32`($a), $Bi, $_R0h | |
| 160 | vpmadd52luq `$_data_offset+64*1`($a), $Bi, $_R1 | |
| 161 | vpmadd52luq `$_data_offset+64*1+32`($a), $Bi, $_R1h | |
| 162 | vpmadd52luq `$_data_offset+64*2`($a), $Bi, $_R2 | |
| 163 | ||
| 164 | vpmadd52luq `$_data_offset+64*0`($m), $Yi, $_R0 | |
| 165 | vpmadd52luq `$_data_offset+64*0+32`($m), $Yi, $_R0h | |
| 166 | vpmadd52luq `$_data_offset+64*1`($m), $Yi, $_R1 | |
| 167 | vpmadd52luq `$_data_offset+64*1+32`($m), $Yi, $_R1h | |
| 168 | vpmadd52luq `$_data_offset+64*2`($m), $Yi, $_R2 | |
| 169 | ||
| 170 | # Shift accumulators right by 1 qword, zero extending the highest one | |
| 171 | valignq \$1, $_R0, $_R0h, $_R0 | |
| 172 | valignq \$1, $_R0h, $_R1, $_R0h | |
| 173 | valignq \$1, $_R1, $_R1h, $_R1 | |
| 174 | valignq \$1, $_R1h, $_R2, $_R1h | |
| 175 | valignq \$1, $_R2, $zero, $_R2 | |
| 176 | ||
| 177 | vmovq $_R0_xmm, %r13 | |
| 178 | addq %r13, $_acc # acc += R0[0] | |
| 179 | ||
| 180 | vpmadd52huq `$_data_offset+64*0`($a), $Bi, $_R0 | |
| 181 | vpmadd52huq `$_data_offset+64*0+32`($a), $Bi, $_R0h | |
| 182 | vpmadd52huq `$_data_offset+64*1`($a), $Bi, $_R1 | |
| 183 | vpmadd52huq `$_data_offset+64*1+32`($a), $Bi, $_R1h | |
| 184 | vpmadd52huq `$_data_offset+64*2`($a), $Bi, $_R2 | |
| 185 | ||
| 186 | vpmadd52huq `$_data_offset+64*0`($m), $Yi, $_R0 | |
| 187 | vpmadd52huq `$_data_offset+64*0+32`($m), $Yi, $_R0h | |
| 188 | vpmadd52huq `$_data_offset+64*1`($m), $Yi, $_R1 | |
| 189 | vpmadd52huq `$_data_offset+64*1+32`($m), $Yi, $_R1h | |
| 190 | vpmadd52huq `$_data_offset+64*2`($m), $Yi, $_R2 | |
| 191 | ___ | |
| 192 | } | |
| 193 | ||
| f87b4c4e AM |
194 | # Normalization routine: handles carry bits and gets bignum qwords to normalized |
| 195 | # 2^52 representation. | |
| c781eb1c AM |
196 | # |
| 197 | # Uses %r8-14,%e[bcd]x | |
| 198 | sub amm52x20_x1_norm { | |
| 199 | my ($_acc,$_R0,$_R0h,$_R1,$_R1h,$_R2) = @_; | |
| 200 | $code.=<<___; | |
| 201 | # Put accumulator to low qword in R0 | |
| f87b4c4e AM |
202 | vpbroadcastq $_acc, $T0 |
| 203 | vpblendd \$3, $T0, $_R0, $_R0 | |
| c781eb1c AM |
204 | |
| 205 | # Extract "carries" (12 high bits) from each QW of R0..R2 | |
| 206 | # Save them to LSB of QWs in T0..T2 | |
| 207 | vpsrlq \$52, $_R0, $T0 | |
| 208 | vpsrlq \$52, $_R0h, $T0h | |
| 209 | vpsrlq \$52, $_R1, $T1 | |
| 210 | vpsrlq \$52, $_R1h, $T1h | |
| 211 | vpsrlq \$52, $_R2, $T2 | |
| 212 | ||
| 213 | # "Shift left" T0..T2 by 1 QW | |
| 214 | valignq \$3, $T1h, $T2, $T2 | |
| 215 | valignq \$3, $T1, $T1h, $T1h | |
| 216 | valignq \$3, $T0h, $T1, $T1 | |
| 217 | valignq \$3, $T0, $T0h, $T0h | |
| f87b4c4e | 218 | valignq \$3, .Lzeros(%rip), $T0, $T0 |
| c781eb1c AM |
219 | |
| 220 | # Drop "carries" from R0..R2 QWs | |
| f87b4c4e AM |
221 | vpandq .Lmask52x4(%rip), $_R0, $_R0 |
| 222 | vpandq .Lmask52x4(%rip), $_R0h, $_R0h | |
| 223 | vpandq .Lmask52x4(%rip), $_R1, $_R1 | |
| 224 | vpandq .Lmask52x4(%rip), $_R1h, $_R1h | |
| 225 | vpandq .Lmask52x4(%rip), $_R2, $_R2 | |
| c781eb1c AM |
226 | |
| 227 | # Sum R0..R2 with corresponding adjusted carries | |
| 228 | vpaddq $T0, $_R0, $_R0 | |
| 229 | vpaddq $T0h, $_R0h, $_R0h | |
| 230 | vpaddq $T1, $_R1, $_R1 | |
| 231 | vpaddq $T1h, $_R1h, $_R1h | |
| 232 | vpaddq $T2, $_R2, $_R2 | |
| 233 | ||
| 234 | # Now handle carry bits from this addition | |
| 235 | # Get mask of QWs which 52-bit parts overflow... | |
| f87b4c4e AM |
236 | vpcmpuq \$6, .Lmask52x4(%rip), $_R0, %k1 # OP=nle (i.e. gt) |
| 237 | vpcmpuq \$6, .Lmask52x4(%rip), $_R0h, %k2 | |
| 238 | vpcmpuq \$6, .Lmask52x4(%rip), $_R1, %k3 | |
| 239 | vpcmpuq \$6, .Lmask52x4(%rip), $_R1h, %k4 | |
| 240 | vpcmpuq \$6, .Lmask52x4(%rip), $_R2, %k5 | |
| c781eb1c AM |
241 | kmovb %k1, %r14d # k1 |
| 242 | kmovb %k2, %r13d # k1h | |
| 243 | kmovb %k3, %r12d # k2 | |
| 244 | kmovb %k4, %r11d # k2h | |
| 245 | kmovb %k5, %r10d # k3 | |
| 246 | ||
| 247 | # ...or saturated | |
| f87b4c4e AM |
248 | vpcmpuq \$0, .Lmask52x4(%rip), $_R0, %k1 # OP=eq |
| 249 | vpcmpuq \$0, .Lmask52x4(%rip), $_R0h, %k2 | |
| 250 | vpcmpuq \$0, .Lmask52x4(%rip), $_R1, %k3 | |
| 251 | vpcmpuq \$0, .Lmask52x4(%rip), $_R1h, %k4 | |
| 252 | vpcmpuq \$0, .Lmask52x4(%rip), $_R2, %k5 | |
| c781eb1c AM |
253 | kmovb %k1, %r9d # k4 |
| 254 | kmovb %k2, %r8d # k4h | |
| 255 | kmovb %k3, %ebx # k5 | |
| 256 | kmovb %k4, %ecx # k5h | |
| 257 | kmovb %k5, %edx # k6 | |
| 258 | ||
| 259 | # Get mask of QWs where carries shall be propagated to. | |
| 260 | # Merge 4-bit masks to 8-bit values to use add with carry. | |
| 261 | shl \$4, %r13b | |
| 262 | or %r13b, %r14b | |
| 263 | shl \$4, %r11b | |
| 264 | or %r11b, %r12b | |
| 265 | ||
| 266 | add %r14b, %r14b | |
| 267 | adc %r12b, %r12b | |
| 268 | adc %r10b, %r10b | |
| 269 | ||
| 270 | shl \$4, %r8b | |
| 271 | or %r8b,%r9b | |
| 272 | shl \$4, %cl | |
| 273 | or %cl, %bl | |
| 274 | ||
| 275 | add %r9b, %r14b | |
| 276 | adc %bl, %r12b | |
| 277 | adc %dl, %r10b | |
| 278 | ||
| 279 | xor %r9b, %r14b | |
| 280 | xor %bl, %r12b | |
| 281 | xor %dl, %r10b | |
| 282 | ||
| 283 | kmovb %r14d, %k1 | |
| 284 | shr \$4, %r14b | |
| 285 | kmovb %r14d, %k2 | |
| 286 | kmovb %r12d, %k3 | |
| 287 | shr \$4, %r12b | |
| 288 | kmovb %r12d, %k4 | |
| 289 | kmovb %r10d, %k5 | |
| 290 | ||
| 291 | # Add carries according to the obtained mask | |
| f87b4c4e AM |
292 | vpsubq .Lmask52x4(%rip), $_R0, ${_R0}{%k1} |
| 293 | vpsubq .Lmask52x4(%rip), $_R0h, ${_R0h}{%k2} | |
| 294 | vpsubq .Lmask52x4(%rip), $_R1, ${_R1}{%k3} | |
| 295 | vpsubq .Lmask52x4(%rip), $_R1h, ${_R1h}{%k4} | |
| 296 | vpsubq .Lmask52x4(%rip), $_R2, ${_R2}{%k5} | |
| 297 | ||
| 298 | vpandq .Lmask52x4(%rip), $_R0, $_R0 | |
| 299 | vpandq .Lmask52x4(%rip), $_R0h, $_R0h | |
| 300 | vpandq .Lmask52x4(%rip), $_R1, $_R1 | |
| 301 | vpandq .Lmask52x4(%rip), $_R1h, $_R1h | |
| 302 | vpandq .Lmask52x4(%rip), $_R2, $_R2 | |
| c781eb1c AM |
303 | ___ |
| 304 | } | |
| 305 | ||
| 306 | $code.=<<___; | |
| 307 | .text | |
| 308 | ||
| f87b4c4e AM |
309 | .globl ossl_rsaz_amm52x20_x1_ifma256 |
| 310 | .type ossl_rsaz_amm52x20_x1_ifma256,\@function,5 | |
| c781eb1c | 311 | .align 32 |
| f87b4c4e | 312 | ossl_rsaz_amm52x20_x1_ifma256: |
| c781eb1c AM |
313 | .cfi_startproc |
| 314 | endbranch | |
| 315 | push %rbx | |
| 316 | .cfi_push %rbx | |
| 317 | push %rbp | |
| 318 | .cfi_push %rbp | |
| 319 | push %r12 | |
| 320 | .cfi_push %r12 | |
| 321 | push %r13 | |
| 322 | .cfi_push %r13 | |
| 323 | push %r14 | |
| 324 | .cfi_push %r14 | |
| 325 | push %r15 | |
| 326 | .cfi_push %r15 | |
| f87b4c4e | 327 | .Lossl_rsaz_amm52x20_x1_ifma256_body: |
| c781eb1c AM |
328 | |
| 329 | # Zeroing accumulators | |
| 330 | vpxord $zero, $zero, $zero | |
| 331 | vmovdqa64 $zero, $R0_0 | |
| 332 | vmovdqa64 $zero, $R0_0h | |
| 333 | vmovdqa64 $zero, $R1_0 | |
| 334 | vmovdqa64 $zero, $R1_0h | |
| 335 | vmovdqa64 $zero, $R2_0 | |
| 336 | ||
| 337 | xorl $acc0_0_low, $acc0_0_low | |
| 338 | ||
| 339 | movq $b, $b_ptr # backup address of b | |
| 340 | movq \$0xfffffffffffff, $mask52 # 52-bit mask | |
| 341 | ||
| 342 | # Loop over 20 digits unrolled by 4 | |
| 343 | mov \$5, $iter | |
| 344 | ||
| 345 | .align 32 | |
| 346 | .Lloop5: | |
| 347 | ___ | |
| 348 | foreach my $idx (0..3) { | |
| 349 | &amm52x20_x1(0,8*$idx,$acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$k0); | |
| 350 | } | |
| 351 | $code.=<<___; | |
| 352 | lea `4*8`($b_ptr), $b_ptr | |
| 353 | dec $iter | |
| 354 | jne .Lloop5 | |
| c781eb1c AM |
355 | ___ |
| 356 | &amm52x20_x1_norm($acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0); | |
| 357 | $code.=<<___; | |
| 358 | ||
| f87b4c4e AM |
359 | vmovdqu64 $R0_0, `0*32`($res) |
| 360 | vmovdqu64 $R0_0h, `1*32`($res) | |
| 361 | vmovdqu64 $R1_0, `2*32`($res) | |
| 362 | vmovdqu64 $R1_0h, `3*32`($res) | |
| 363 | vmovdqu64 $R2_0, `4*32`($res) | |
| c781eb1c AM |
364 | |
| 365 | vzeroupper | |
| 366 | mov 0(%rsp),%r15 | |
| 367 | .cfi_restore %r15 | |
| 368 | mov 8(%rsp),%r14 | |
| 369 | .cfi_restore %r14 | |
| 370 | mov 16(%rsp),%r13 | |
| 371 | .cfi_restore %r13 | |
| 372 | mov 24(%rsp),%r12 | |
| 373 | .cfi_restore %r12 | |
| 374 | mov 32(%rsp),%rbp | |
| 375 | .cfi_restore %rbp | |
| 376 | mov 40(%rsp),%rbx | |
| 377 | .cfi_restore %rbx | |
| 378 | lea 48(%rsp),%rsp | |
| 379 | .cfi_adjust_cfa_offset -48 | |
| f87b4c4e | 380 | .Lossl_rsaz_amm52x20_x1_ifma256_epilogue: |
| c781eb1c AM |
381 | ret |
| 382 | .cfi_endproc | |
| f87b4c4e | 383 | .size ossl_rsaz_amm52x20_x1_ifma256, .-ossl_rsaz_amm52x20_x1_ifma256 |
| c781eb1c AM |
384 | ___ |
| 385 | ||
| 386 | $code.=<<___; | |
| 387 | .data | |
| 388 | .align 32 | |
| 389 | .Lmask52x4: | |
| 390 | .quad 0xfffffffffffff | |
| 391 | .quad 0xfffffffffffff | |
| 392 | .quad 0xfffffffffffff | |
| 393 | .quad 0xfffffffffffff | |
| 394 | ___ | |
| 395 | ||
| 396 | ############################################################################### | |
| 397 | # Dual Almost Montgomery Multiplication for 20-digit number in radix 2^52 | |
| 398 | # | |
| f87b4c4e | 399 | # See description of ossl_rsaz_amm52x20_x1_ifma256() above for details about Almost |
| c781eb1c AM |
400 | # Montgomery Multiplication algorithm and function input parameters description. |
| 401 | # | |
| 402 | # This function does two AMMs for two independent inputs, hence dual. | |
| 403 | # | |
| f87b4c4e AM |
404 | # void ossl_rsaz_amm52x20_x2_ifma256(BN_ULONG out[2][20], |
| 405 | # const BN_ULONG a[2][20], | |
| 406 | # const BN_ULONG b[2][20], | |
| 407 | # const BN_ULONG m[2][20], | |
| 408 | # const BN_ULONG k0[2]); | |
| c781eb1c AM |
409 | ############################################################################### |
| 410 | ||
| 411 | $code.=<<___; | |
| 412 | .text | |
| 413 | ||
| f87b4c4e AM |
414 | .globl ossl_rsaz_amm52x20_x2_ifma256 |
| 415 | .type ossl_rsaz_amm52x20_x2_ifma256,\@function,5 | |
| c781eb1c | 416 | .align 32 |
| f87b4c4e | 417 | ossl_rsaz_amm52x20_x2_ifma256: |
| c781eb1c AM |
418 | .cfi_startproc |
| 419 | endbranch | |
| 420 | push %rbx | |
| 421 | .cfi_push %rbx | |
| 422 | push %rbp | |
| 423 | .cfi_push %rbp | |
| 424 | push %r12 | |
| 425 | .cfi_push %r12 | |
| 426 | push %r13 | |
| 427 | .cfi_push %r13 | |
| 428 | push %r14 | |
| 429 | .cfi_push %r14 | |
| 430 | push %r15 | |
| 431 | .cfi_push %r15 | |
| f87b4c4e | 432 | .Lossl_rsaz_amm52x20_x2_ifma256_body: |
| c781eb1c AM |
433 | |
| 434 | # Zeroing accumulators | |
| 435 | vpxord $zero, $zero, $zero | |
| 436 | vmovdqa64 $zero, $R0_0 | |
| 437 | vmovdqa64 $zero, $R0_0h | |
| 438 | vmovdqa64 $zero, $R1_0 | |
| 439 | vmovdqa64 $zero, $R1_0h | |
| 440 | vmovdqa64 $zero, $R2_0 | |
| 441 | vmovdqa64 $zero, $R0_1 | |
| 442 | vmovdqa64 $zero, $R0_1h | |
| 443 | vmovdqa64 $zero, $R1_1 | |
| 444 | vmovdqa64 $zero, $R1_1h | |
| 445 | vmovdqa64 $zero, $R2_1 | |
| 446 | ||
| 447 | xorl $acc0_0_low, $acc0_0_low | |
| 448 | xorl $acc0_1_low, $acc0_1_low | |
| 449 | ||
| 450 | movq $b, $b_ptr # backup address of b | |
| 451 | movq \$0xfffffffffffff, $mask52 # 52-bit mask | |
| 452 | ||
| 453 | mov \$20, $iter | |
| 454 | ||
| 455 | .align 32 | |
| 456 | .Lloop20: | |
| 457 | ___ | |
| 458 | &amm52x20_x1( 0, 0,$acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,"($k0)"); | |
| 459 | # 20*8 = offset of the next dimension in two-dimension array | |
| 460 | &amm52x20_x1(20*8,20*8,$acc0_1,$R0_1,$R0_1h,$R1_1,$R1_1h,$R2_1,"8($k0)"); | |
| 461 | $code.=<<___; | |
| 462 | lea 8($b_ptr), $b_ptr | |
| 463 | dec $iter | |
| 464 | jne .Lloop20 | |
| c781eb1c AM |
465 | ___ |
| 466 | &amm52x20_x1_norm($acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0); | |
| 467 | &amm52x20_x1_norm($acc0_1,$R0_1,$R0_1h,$R1_1,$R1_1h,$R2_1); | |
| 468 | $code.=<<___; | |
| 469 | ||
| f87b4c4e AM |
470 | vmovdqu64 $R0_0, `0*32`($res) |
| 471 | vmovdqu64 $R0_0h, `1*32`($res) | |
| 472 | vmovdqu64 $R1_0, `2*32`($res) | |
| 473 | vmovdqu64 $R1_0h, `3*32`($res) | |
| 474 | vmovdqu64 $R2_0, `4*32`($res) | |
| c781eb1c | 475 | |
| f87b4c4e AM |
476 | vmovdqu64 $R0_1, `5*32`($res) |
| 477 | vmovdqu64 $R0_1h, `6*32`($res) | |
| 478 | vmovdqu64 $R1_1, `7*32`($res) | |
| 479 | vmovdqu64 $R1_1h, `8*32`($res) | |
| 480 | vmovdqu64 $R2_1, `9*32`($res) | |
| c781eb1c AM |
481 | |
| 482 | vzeroupper | |
| 483 | mov 0(%rsp),%r15 | |
| 484 | .cfi_restore %r15 | |
| 485 | mov 8(%rsp),%r14 | |
| 486 | .cfi_restore %r14 | |
| 487 | mov 16(%rsp),%r13 | |
| 488 | .cfi_restore %r13 | |
| 489 | mov 24(%rsp),%r12 | |
| 490 | .cfi_restore %r12 | |
| 491 | mov 32(%rsp),%rbp | |
| 492 | .cfi_restore %rbp | |
| 493 | mov 40(%rsp),%rbx | |
| 494 | .cfi_restore %rbx | |
| 495 | lea 48(%rsp),%rsp | |
| 496 | .cfi_adjust_cfa_offset -48 | |
| f87b4c4e | 497 | .Lossl_rsaz_amm52x20_x2_ifma256_epilogue: |
| c781eb1c AM |
498 | ret |
| 499 | .cfi_endproc | |
| f87b4c4e | 500 | .size ossl_rsaz_amm52x20_x2_ifma256, .-ossl_rsaz_amm52x20_x2_ifma256 |
| c781eb1c AM |
501 | ___ |
| 502 | } | |
| 503 | ||
| 504 | ############################################################################### | |
| 505 | # Constant time extraction from the precomputed table of powers base^i, where | |
| 506 | # i = 0..2^EXP_WIN_SIZE-1 | |
| 507 | # | |
| f87b4c4e AM |
508 | # The input |red_table| contains precomputations for two independent base values. |
| 509 | # |red_table_idx1| and |red_table_idx2| are corresponding power indexes. | |
| c781eb1c | 510 | # |
| f87b4c4e | 511 | # Extracted value (output) is 2 20 digit numbers in 2^52 radix. |
| c781eb1c | 512 | # |
| 190c029e P |
513 | # void ossl_extract_multiplier_2x20_win5(BN_ULONG *red_Y, |
| 514 | # const BN_ULONG red_table[1 << EXP_WIN_SIZE][2][20], | |
| f87b4c4e | 515 | # int red_table_idx1, int red_table_idx2); |
| c781eb1c AM |
516 | # |
| 517 | # EXP_WIN_SIZE = 5 | |
| 518 | ############################################################################### | |
| 519 | { | |
| 520 | # input parameters | |
| f87b4c4e AM |
521 | my ($out,$red_tbl,$red_tbl_idx1,$red_tbl_idx2)=$win64 ? ("%rcx","%rdx","%r8", "%r9") : # Win64 order |
| 522 | ("%rdi","%rsi","%rdx","%rcx"); # Unix order | |
| c781eb1c | 523 | |
| f87b4c4e AM |
524 | my ($t0,$t1,$t2,$t3,$t4,$t5) = map("%ymm$_", (0..5)); |
| 525 | my ($t6,$t7,$t8,$t9) = map("%ymm$_", (16..19)); | |
| 526 | my ($tmp,$cur_idx,$idx1,$idx2,$ones) = map("%ymm$_", (20..24)); | |
| 527 | ||
| 528 | my @t = ($t0,$t1,$t2,$t3,$t4,$t5,$t6,$t7,$t8,$t9); | |
| 529 | my $t0xmm = $t0; | |
| 530 | $t0xmm =~ s/%y/%x/; | |
| c781eb1c AM |
531 | |
| 532 | $code.=<<___; | |
| 533 | .text | |
| 534 | ||
| 535 | .align 32 | |
| 190c029e | 536 | .globl ossl_extract_multiplier_2x20_win5 |
| f87b4c4e | 537 | .type ossl_extract_multiplier_2x20_win5,\@abi-omnipotent |
| 190c029e | 538 | ossl_extract_multiplier_2x20_win5: |
| c781eb1c AM |
539 | .cfi_startproc |
| 540 | endbranch | |
| c781eb1c | 541 | vmovdqa64 .Lones(%rip), $ones # broadcast ones |
| f87b4c4e AM |
542 | vpbroadcastq $red_tbl_idx1, $idx1 |
| 543 | vpbroadcastq $red_tbl_idx2, $idx2 | |
| c781eb1c AM |
544 | leaq `(1<<5)*2*20*8`($red_tbl), %rax # holds end of the tbl |
| 545 | ||
| f87b4c4e AM |
546 | # zeroing t0..n, cur_idx |
| 547 | vpxor $t0xmm, $t0xmm, $t0xmm | |
| 548 | vmovdqa64 $t0, $cur_idx | |
| 549 | ___ | |
| 550 | foreach (1..9) { | |
| 551 | $code.="vmovdqa64 $t0, $t[$_] \n"; | |
| 552 | } | |
| 553 | $code.=<<___; | |
| c781eb1c AM |
554 | |
| 555 | .align 32 | |
| 556 | .Lloop: | |
| f87b4c4e AM |
557 | vpcmpq \$0, $cur_idx, $idx1, %k1 # mask of (idx1 == cur_idx) |
| 558 | vpcmpq \$0, $cur_idx, $idx2, %k2 # mask of (idx2 == cur_idx) | |
| 559 | ___ | |
| 560 | foreach (0..9) { | |
| 561 | my $mask = $_<5?"%k1":"%k2"; | |
| 562 | $code.=<<___; | |
| 563 | vmovdqu64 `${_}*32`($red_tbl), $tmp # load data from red_tbl | |
| 564 | vpblendmq $tmp, $t[$_], ${t[$_]}{$mask} # extract data when mask is not zero | |
| 565 | ___ | |
| 566 | } | |
| 567 | $code.=<<___; | |
| 568 | vpaddq $ones, $cur_idx, $cur_idx # increment cur_idx | |
| 569 | addq \$`2*20*8`, $red_tbl | |
| c781eb1c AM |
570 | cmpq $red_tbl, %rax |
| 571 | jne .Lloop | |
| f87b4c4e AM |
572 | ___ |
| 573 | # store t0..n | |
| 574 | foreach (0..9) { | |
| 575 | $code.="vmovdqu64 $t[$_], `${_}*32`($out) \n"; | |
| 576 | } | |
| 577 | $code.=<<___; | |
| c781eb1c AM |
578 | ret |
| 579 | .cfi_endproc | |
| 190c029e | 580 | .size ossl_extract_multiplier_2x20_win5, .-ossl_extract_multiplier_2x20_win5 |
| c781eb1c AM |
581 | ___ |
| 582 | $code.=<<___; | |
| 583 | .data | |
| 584 | .align 32 | |
| 585 | .Lones: | |
| 586 | .quad 1,1,1,1 | |
| f87b4c4e AM |
587 | .Lzeros: |
| 588 | .quad 0,0,0,0 | |
| c781eb1c AM |
589 | ___ |
| 590 | } | |
| 591 | ||
| 592 | if ($win64) { | |
| 593 | $rec="%rcx"; | |
| 594 | $frame="%rdx"; | |
| 595 | $context="%r8"; | |
| 596 | $disp="%r9"; | |
| 597 | ||
| f87b4c4e | 598 | $code.=<<___; |
| c781eb1c AM |
599 | .extern __imp_RtlVirtualUnwind |
| 600 | .type rsaz_def_handler,\@abi-omnipotent | |
| 601 | .align 16 | |
| 602 | rsaz_def_handler: | |
| 603 | push %rsi | |
| 604 | push %rdi | |
| 605 | push %rbx | |
| 606 | push %rbp | |
| 607 | push %r12 | |
| 608 | push %r13 | |
| 609 | push %r14 | |
| 610 | push %r15 | |
| 611 | pushfq | |
| 612 | sub \$64,%rsp | |
| 613 | ||
| 614 | mov 120($context),%rax # pull context->Rax | |
| 615 | mov 248($context),%rbx # pull context->Rip | |
| 616 | ||
| 617 | mov 8($disp),%rsi # disp->ImageBase | |
| 618 | mov 56($disp),%r11 # disp->HandlerData | |
| 619 | ||
| 620 | mov 0(%r11),%r10d # HandlerData[0] | |
| 621 | lea (%rsi,%r10),%r10 # prologue label | |
| 622 | cmp %r10,%rbx # context->Rip<.Lprologue | |
| 623 | jb .Lcommon_seh_tail | |
| 624 | ||
| 625 | mov 152($context),%rax # pull context->Rsp | |
| 626 | ||
| 627 | mov 4(%r11),%r10d # HandlerData[1] | |
| 628 | lea (%rsi,%r10),%r10 # epilogue label | |
| 629 | cmp %r10,%rbx # context->Rip>=.Lepilogue | |
| 630 | jae .Lcommon_seh_tail | |
| 631 | ||
| 632 | lea 48(%rax),%rax | |
| 633 | ||
| 634 | mov -8(%rax),%rbx | |
| 635 | mov -16(%rax),%rbp | |
| 636 | mov -24(%rax),%r12 | |
| 637 | mov -32(%rax),%r13 | |
| 638 | mov -40(%rax),%r14 | |
| 639 | mov -48(%rax),%r15 | |
| 640 | mov %rbx,144($context) # restore context->Rbx | |
| 641 | mov %rbp,160($context) # restore context->Rbp | |
| 642 | mov %r12,216($context) # restore context->R12 | |
| 643 | mov %r13,224($context) # restore context->R13 | |
| 644 | mov %r14,232($context) # restore context->R14 | |
| 645 | mov %r15,240($context) # restore context->R14 | |
| 646 | ||
| 647 | .Lcommon_seh_tail: | |
| 648 | mov 8(%rax),%rdi | |
| 649 | mov 16(%rax),%rsi | |
| 650 | mov %rax,152($context) # restore context->Rsp | |
| 651 | mov %rsi,168($context) # restore context->Rsi | |
| 652 | mov %rdi,176($context) # restore context->Rdi | |
| 653 | ||
| 654 | mov 40($disp),%rdi # disp->ContextRecord | |
| 655 | mov $context,%rsi # context | |
| 656 | mov \$154,%ecx # sizeof(CONTEXT) | |
| 657 | .long 0xa548f3fc # cld; rep movsq | |
| 658 | ||
| 659 | mov $disp,%rsi | |
| 660 | xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER | |
| 661 | mov 8(%rsi),%rdx # arg2, disp->ImageBase | |
| 662 | mov 0(%rsi),%r8 # arg3, disp->ControlPc | |
| 663 | mov 16(%rsi),%r9 # arg4, disp->FunctionEntry | |
| 664 | mov 40(%rsi),%r10 # disp->ContextRecord | |
| 665 | lea 56(%rsi),%r11 # &disp->HandlerData | |
| 666 | lea 24(%rsi),%r12 # &disp->EstablisherFrame | |
| 667 | mov %r10,32(%rsp) # arg5 | |
| 668 | mov %r11,40(%rsp) # arg6 | |
| 669 | mov %r12,48(%rsp) # arg7 | |
| 670 | mov %rcx,56(%rsp) # arg8, (NULL) | |
| 671 | call *__imp_RtlVirtualUnwind(%rip) | |
| 672 | ||
| 673 | mov \$1,%eax # ExceptionContinueSearch | |
| 674 | add \$64,%rsp | |
| 675 | popfq | |
| 676 | pop %r15 | |
| 677 | pop %r14 | |
| 678 | pop %r13 | |
| 679 | pop %r12 | |
| 680 | pop %rbp | |
| 681 | pop %rbx | |
| 682 | pop %rdi | |
| 683 | pop %rsi | |
| 684 | ret | |
| 685 | .size rsaz_def_handler,.-rsaz_def_handler | |
| 686 | ||
| 687 | .section .pdata | |
| 688 | .align 4 | |
| f87b4c4e AM |
689 | .rva .LSEH_begin_ossl_rsaz_amm52x20_x1_ifma256 |
| 690 | .rva .LSEH_end_ossl_rsaz_amm52x20_x1_ifma256 | |
| 691 | .rva .LSEH_info_ossl_rsaz_amm52x20_x1_ifma256 | |
| c781eb1c | 692 | |
| f87b4c4e AM |
693 | .rva .LSEH_begin_ossl_rsaz_amm52x20_x2_ifma256 |
| 694 | .rva .LSEH_end_ossl_rsaz_amm52x20_x2_ifma256 | |
| 695 | .rva .LSEH_info_ossl_rsaz_amm52x20_x2_ifma256 | |
| b238e78f | 696 | |
| c781eb1c AM |
697 | .section .xdata |
| 698 | .align 8 | |
| f87b4c4e | 699 | .LSEH_info_ossl_rsaz_amm52x20_x1_ifma256: |
| c781eb1c AM |
700 | .byte 9,0,0,0 |
| 701 | .rva rsaz_def_handler | |
| f87b4c4e AM |
702 | .rva .Lossl_rsaz_amm52x20_x1_ifma256_body,.Lossl_rsaz_amm52x20_x1_ifma256_epilogue |
| 703 | .LSEH_info_ossl_rsaz_amm52x20_x2_ifma256: | |
| b238e78f AM |
704 | .byte 9,0,0,0 |
| 705 | .rva rsaz_def_handler | |
| f87b4c4e | 706 | .rva .Lossl_rsaz_amm52x20_x2_ifma256_body,.Lossl_rsaz_amm52x20_x2_ifma256_epilogue |
| c781eb1c AM |
707 | ___ |
| 708 | } | |
| 709 | }}} else {{{ # fallback for old assembler | |
| 710 | $code.=<<___; | |
| 711 | .text | |
| 712 | ||
| e475d9a4 P |
713 | .globl ossl_rsaz_avx512ifma_eligible |
| 714 | .type ossl_rsaz_avx512ifma_eligible,\@abi-omnipotent | |
| 715 | ossl_rsaz_avx512ifma_eligible: | |
| c781eb1c AM |
716 | xor %eax,%eax |
| 717 | ret | |
| e475d9a4 | 718 | .size ossl_rsaz_avx512ifma_eligible, .-ossl_rsaz_avx512ifma_eligible |
| c781eb1c | 719 | |
| f87b4c4e AM |
720 | .globl ossl_rsaz_amm52x20_x1_ifma256 |
| 721 | .globl ossl_rsaz_amm52x20_x2_ifma256 | |
| 190c029e | 722 | .globl ossl_extract_multiplier_2x20_win5 |
| f87b4c4e AM |
723 | .type ossl_rsaz_amm52x20_x1_ifma256,\@abi-omnipotent |
| 724 | ossl_rsaz_amm52x20_x1_ifma256: | |
| 725 | ossl_rsaz_amm52x20_x2_ifma256: | |
| 190c029e | 726 | ossl_extract_multiplier_2x20_win5: |
| c781eb1c AM |
727 | .byte 0x0f,0x0b # ud2 |
| 728 | ret | |
| f87b4c4e | 729 | .size ossl_rsaz_amm52x20_x1_ifma256, .-ossl_rsaz_amm52x20_x1_ifma256 |
| c781eb1c AM |
730 | ___ |
| 731 | }}} | |
| 732 | ||
| 733 | $code =~ s/\`([^\`]*)\`/eval $1/gem; | |
| 734 | print $code; | |
| 735 | close STDOUT or die "error closing STDOUT: $!"; |