crypto/bn/asm/rsaz-2k-avx512.pl

   1 # Copyright 2020-2022 The OpenSSL Project Authors. All Rights Reserved.
   2 # Copyright (c) 2020, Intel Corporation. All Rights Reserved.
   3 #
   4 # Licensed under the Apache License 2.0 (the "License").  You may not use
   5 # this file except in compliance with the License.  You can obtain a copy
   6 # in the file LICENSE in the source distribution or at
   7 # https://www.openssl.org/source/license.html
   8 #
   9 #
  10 # Originally written by Sergey Kirillov and Andrey Matyukov.
  11 # Special thanks to Ilya Albrekht for his valuable hints.
  12 # Intel Corporation
  13 #
  14 # December 2020
  15 #
  16 # Initial release.
  17 #
  18 # Implementation utilizes 256-bit (ymm) registers to avoid frequency scaling issues.
  19 #
  20 # IceLake-Client @ 1.3GHz
  21 # |---------+----------------------+--------------+-------------|
  22 # |         | OpenSSL 3.0.0-alpha9 | this         | Unit        |
  23 # |---------+----------------------+--------------+-------------|
  24 # | rsa2048 | 2 127 659            | 1 015 625    | cycles/sign |
  25 # |         | 611                  | 1280 / +109% | sign/s      |
  26 # |---------+----------------------+--------------+-------------|
  27 #
  28
  29 # $output is the last argument if it looks like a file (it has an extension)
  30 # $flavour is the first argument if it doesn't look like a file
  31 $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
  32 $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
  33
  34 $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
  35 $avx512ifma=0;
  36
  37 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  38 ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
  39 ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
  40 die "can't locate x86_64-xlate.pl";
  41
  42 if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
  43         =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
  44     $avx512ifma = ($1>=2.26);
  45 }
  46
  47 if (!$avx512 && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
  48        `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)(?:\.([0-9]+))?/) {
  49     $avx512ifma = ($1==2.11 && $2>=8) + ($1>=2.12);
  50 }
  51
  52 if (!$avx512 && `$ENV{CC} -v 2>&1` =~ /((?:clang|LLVM) version|.*based on LLVM) ([0-9]+\.[0-9]+)/) {
  53     $avx512ifma = ($2>=7.0);
  54 }
  55
  56 open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\""
  57     or die "can't call $xlate: $!";
  58 *STDOUT=*OUT;
  59
  60 if ($avx512ifma>0) {{{
  61 @_6_args_universal_ABI = ("%rdi","%rsi","%rdx","%rcx","%r8","%r9");
  62
  63 $code.=<<___;
  64 .extern OPENSSL_ia32cap_P
  65 .globl  ossl_rsaz_avx512ifma_eligible
  66 .type   ossl_rsaz_avx512ifma_eligible,\@abi-omnipotent
  67 .align  32
  68 ossl_rsaz_avx512ifma_eligible:
  69     mov OPENSSL_ia32cap_P+8(%rip), %ecx
  70     xor %eax,%eax
  71     and \$`1<<31|1<<21|1<<17|1<<16`, %ecx     # avx512vl + avx512ifma + avx512dq + avx512f
  72     cmp \$`1<<31|1<<21|1<<17|1<<16`, %ecx
  73     cmove %ecx,%eax
  74     ret
  75 .size   ossl_rsaz_avx512ifma_eligible, .-ossl_rsaz_avx512ifma_eligible
  76 ___
  77
  78 ###############################################################################
  79 # Almost Montgomery Multiplication (AMM) for 20-digit number in radix 2^52.
  80 #
  81 # AMM is defined as presented in the paper [1].
  82 #
  83 # The input and output are presented in 2^52 radix domain, i.e.
  84 #   |res|, |a|, |b|, |m| are arrays of 20 64-bit qwords with 12 high bits zeroed.
  85 #   |k0| is a Montgomery coefficient, which is here k0 = -1/m mod 2^64
  86 #
  87 # NB: the AMM implementation does not perform "conditional" subtraction step
  88 # specified in the original algorithm as according to the Lemma 1 from the paper
  89 # [2], the result will be always < 2*m and can be used as a direct input to
  90 # the next AMM iteration.  This post-condition is true, provided the correct
  91 # parameter |s| (notion of the Lemma 1 from [2]) is chosen, i.e.  s >= n + 2 * k,
  92 # which matches our case: 1040 > 1024 + 2 * 1.
  93 #
  94 # [1] Gueron, S. Efficient software implementations of modular exponentiation.
  95 #     DOI: 10.1007/s13389-012-0031-5
  96 # [2] Gueron, S. Enhanced Montgomery Multiplication.
  97 #     DOI: 10.1007/3-540-36400-5_5
  98 #
  99 # void ossl_rsaz_amm52x20_x1_ifma256(BN_ULONG *res,
 100 #                                    const BN_ULONG *a,
 101 #                                    const BN_ULONG *b,
 102 #                                    const BN_ULONG *m,
 103 #                                    BN_ULONG k0);
 104 ###############################################################################
 105 {
 106 # input parameters ("%rdi","%rsi","%rdx","%rcx","%r8")
 107 my ($res,$a,$b,$m,$k0) = @_6_args_universal_ABI;
 108
 109 my $mask52     = "%rax";
 110 my $acc0_0     = "%r9";
 111 my $acc0_0_low = "%r9d";
 112 my $acc0_1     = "%r15";
 113 my $acc0_1_low = "%r15d";
 114 my $b_ptr      = "%r11";
 115
 116 my $iter = "%ebx";
 117
 118 my $zero = "%ymm0";
 119 my $Bi   = "%ymm1";
 120 my $Yi   = "%ymm2";
 121 my ($R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0) = ("%ymm3",map("%ymm$_",(16..19)));
 122 my ($R0_1,$R0_1h,$R1_1,$R1_1h,$R2_1) = ("%ymm4",map("%ymm$_",(20..23)));
 123
 124 # Registers mapping for normalization.
 125 my ($T0,$T0h,$T1,$T1h,$T2) = ("$zero", "$Bi", "$Yi", map("%ymm$_", (25..26)));
 126
 127 sub amm52x20_x1() {
 128 # _data_offset - offset in the |a| or |m| arrays pointing to the beginning
 129 #                of data for corresponding AMM operation;
 130 # _b_offset    - offset in the |b| array pointing to the next qword digit;
 131 my ($_data_offset,$_b_offset,$_acc,$_R0,$_R0h,$_R1,$_R1h,$_R2,$_k0) = @_;
 132 my $_R0_xmm = $_R0;
 133 $_R0_xmm =~ s/%y/%x/;
 134 $code.=<<___;
 135     movq    $_b_offset($b_ptr), %r13             # b[i]
 136
 137     vpbroadcastq    %r13, $Bi                    # broadcast b[i]
 138     movq    $_data_offset($a), %rdx
 139     mulx    %r13, %r13, %r12                     # a[0]*b[i] = (t0,t2)
 140     addq    %r13, $_acc                          # acc += t0
 141     movq    %r12, %r10
 142     adcq    \$0, %r10                            # t2 += CF
 143
 144     movq    $_k0, %r13
 145     imulq   $_acc, %r13                          # acc * k0
 146     andq    $mask52, %r13                        # yi = (acc * k0) & mask52
 147
 148     vpbroadcastq    %r13, $Yi                    # broadcast y[i]
 149     movq    $_data_offset($m), %rdx
 150     mulx    %r13, %r13, %r12                     # yi * m[0] = (t0,t1)
 151     addq    %r13, $_acc                          # acc += t0
 152     adcq    %r12, %r10                           # t2 += (t1 + CF)
 153
 154     shrq    \$52, $_acc
 155     salq    \$12, %r10
 156     or      %r10, $_acc                          # acc = ((acc >> 52) | (t2 << 12))
 157
 158     vpmadd52luq `$_data_offset+64*0`($a), $Bi, $_R0
 159     vpmadd52luq `$_data_offset+64*0+32`($a), $Bi, $_R0h
 160     vpmadd52luq `$_data_offset+64*1`($a), $Bi, $_R1
 161     vpmadd52luq `$_data_offset+64*1+32`($a), $Bi, $_R1h
 162     vpmadd52luq `$_data_offset+64*2`($a), $Bi, $_R2
 163
 164     vpmadd52luq `$_data_offset+64*0`($m), $Yi, $_R0
 165     vpmadd52luq `$_data_offset+64*0+32`($m), $Yi, $_R0h
 166     vpmadd52luq `$_data_offset+64*1`($m), $Yi, $_R1
 167     vpmadd52luq `$_data_offset+64*1+32`($m), $Yi, $_R1h
 168     vpmadd52luq `$_data_offset+64*2`($m), $Yi, $_R2
 169
 170     # Shift accumulators right by 1 qword, zero extending the highest one
 171     valignq     \$1, $_R0, $_R0h, $_R0
 172     valignq     \$1, $_R0h, $_R1, $_R0h
 173     valignq     \$1, $_R1, $_R1h, $_R1
 174     valignq     \$1, $_R1h, $_R2, $_R1h
 175     valignq     \$1, $_R2, $zero, $_R2
 176
 177     vmovq   $_R0_xmm, %r13
 178     addq    %r13, $_acc    # acc += R0[0]
 179
 180     vpmadd52huq `$_data_offset+64*0`($a), $Bi, $_R0
 181     vpmadd52huq `$_data_offset+64*0+32`($a), $Bi, $_R0h
 182     vpmadd52huq `$_data_offset+64*1`($a), $Bi, $_R1
 183     vpmadd52huq `$_data_offset+64*1+32`($a), $Bi, $_R1h
 184     vpmadd52huq `$_data_offset+64*2`($a), $Bi, $_R2
 185
 186     vpmadd52huq `$_data_offset+64*0`($m), $Yi, $_R0
 187     vpmadd52huq `$_data_offset+64*0+32`($m), $Yi, $_R0h
 188     vpmadd52huq `$_data_offset+64*1`($m), $Yi, $_R1
 189     vpmadd52huq `$_data_offset+64*1+32`($m), $Yi, $_R1h
 190     vpmadd52huq `$_data_offset+64*2`($m), $Yi, $_R2
 191 ___
 192 }
 193
 194 # Normalization routine: handles carry bits and gets bignum qwords to normalized
 195 # 2^52 representation.
 196 #
 197 # Uses %r8-14,%e[bcd]x
 198 sub amm52x20_x1_norm {
 199 my ($_acc,$_R0,$_R0h,$_R1,$_R1h,$_R2) = @_;
 200 $code.=<<___;
 201     # Put accumulator to low qword in R0
 202     vpbroadcastq    $_acc, $T0
 203     vpblendd \$3, $T0, $_R0, $_R0
 204
 205     # Extract "carries" (12 high bits) from each QW of R0..R2
 206     # Save them to LSB of QWs in T0..T2
 207     vpsrlq    \$52, $_R0,   $T0
 208     vpsrlq    \$52, $_R0h,  $T0h
 209     vpsrlq    \$52, $_R1,   $T1
 210     vpsrlq    \$52, $_R1h,  $T1h
 211     vpsrlq    \$52, $_R2,   $T2
 212
 213     # "Shift left" T0..T2 by 1 QW
 214     valignq \$3, $T1h,  $T2,  $T2
 215     valignq \$3, $T1,   $T1h, $T1h
 216     valignq \$3, $T0h,  $T1,  $T1
 217     valignq \$3, $T0,   $T0h, $T0h
 218     valignq \$3, .Lzeros(%rip), $T0,  $T0
 219
 220     # Drop "carries" from R0..R2 QWs
 221     vpandq    .Lmask52x4(%rip), $_R0,  $_R0
 222     vpandq    .Lmask52x4(%rip), $_R0h, $_R0h
 223     vpandq    .Lmask52x4(%rip), $_R1,  $_R1
 224     vpandq    .Lmask52x4(%rip), $_R1h, $_R1h
 225     vpandq    .Lmask52x4(%rip), $_R2,  $_R2
 226
 227     # Sum R0..R2 with corresponding adjusted carries
 228     vpaddq  $T0,  $_R0,  $_R0
 229     vpaddq  $T0h, $_R0h, $_R0h
 230     vpaddq  $T1,  $_R1,  $_R1
 231     vpaddq  $T1h, $_R1h, $_R1h
 232     vpaddq  $T2,  $_R2,  $_R2
 233
 234     # Now handle carry bits from this addition
 235     # Get mask of QWs which 52-bit parts overflow...
 236     vpcmpuq   \$6, .Lmask52x4(%rip), $_R0,  %k1 # OP=nle (i.e. gt)
 237     vpcmpuq   \$6, .Lmask52x4(%rip), $_R0h, %k2
 238     vpcmpuq   \$6, .Lmask52x4(%rip), $_R1,  %k3
 239     vpcmpuq   \$6, .Lmask52x4(%rip), $_R1h, %k4
 240     vpcmpuq   \$6, .Lmask52x4(%rip), $_R2,  %k5
 241     kmovb   %k1, %r14d                   # k1
 242     kmovb   %k2, %r13d                   # k1h
 243     kmovb   %k3, %r12d                   # k2
 244     kmovb   %k4, %r11d                   # k2h
 245     kmovb   %k5, %r10d                   # k3
 246
 247     # ...or saturated
 248     vpcmpuq   \$0, .Lmask52x4(%rip), $_R0,  %k1 # OP=eq
 249     vpcmpuq   \$0, .Lmask52x4(%rip), $_R0h, %k2
 250     vpcmpuq   \$0, .Lmask52x4(%rip), $_R1,  %k3
 251     vpcmpuq   \$0, .Lmask52x4(%rip), $_R1h, %k4
 252     vpcmpuq   \$0, .Lmask52x4(%rip), $_R2,  %k5
 253     kmovb   %k1, %r9d                    # k4
 254     kmovb   %k2, %r8d                    # k4h
 255     kmovb   %k3, %ebx                    # k5
 256     kmovb   %k4, %ecx                    # k5h
 257     kmovb   %k5, %edx                    # k6
 258
 259     # Get mask of QWs where carries shall be propagated to.
 260     # Merge 4-bit masks to 8-bit values to use add with carry.
 261     shl   \$4, %r13b
 262     or    %r13b, %r14b
 263     shl   \$4, %r11b
 264     or    %r11b, %r12b
 265
 266     add   %r14b, %r14b
 267     adc   %r12b, %r12b
 268     adc   %r10b, %r10b
 269
 270     shl   \$4, %r8b
 271     or    %r8b,%r9b
 272     shl   \$4, %cl
 273     or    %cl, %bl
 274
 275     add   %r9b, %r14b
 276     adc   %bl, %r12b
 277     adc   %dl, %r10b
 278
 279     xor   %r9b, %r14b
 280     xor   %bl, %r12b
 281     xor   %dl, %r10b
 282
 283     kmovb   %r14d, %k1
 284     shr     \$4, %r14b
 285     kmovb   %r14d, %k2
 286     kmovb   %r12d, %k3
 287     shr     \$4, %r12b
 288     kmovb   %r12d, %k4
 289     kmovb   %r10d, %k5
 290
 291     # Add carries according to the obtained mask
 292     vpsubq  .Lmask52x4(%rip), $_R0,  ${_R0}{%k1}
 293     vpsubq  .Lmask52x4(%rip), $_R0h, ${_R0h}{%k2}
 294     vpsubq  .Lmask52x4(%rip), $_R1,  ${_R1}{%k3}
 295     vpsubq  .Lmask52x4(%rip), $_R1h, ${_R1h}{%k4}
 296     vpsubq  .Lmask52x4(%rip), $_R2,  ${_R2}{%k5}
 297
 298     vpandq   .Lmask52x4(%rip), $_R0,  $_R0
 299     vpandq   .Lmask52x4(%rip), $_R0h, $_R0h
 300     vpandq   .Lmask52x4(%rip), $_R1,  $_R1
 301     vpandq   .Lmask52x4(%rip), $_R1h, $_R1h
 302     vpandq   .Lmask52x4(%rip), $_R2,  $_R2
 303 ___
 304 }
 305
 306 $code.=<<___;
 307 .text
 308
 309 .globl  ossl_rsaz_amm52x20_x1_ifma256
 310 .type   ossl_rsaz_amm52x20_x1_ifma256,\@function,5
 311 .align 32
 312 ossl_rsaz_amm52x20_x1_ifma256:
 313 .cfi_startproc
 314     endbranch
 315     push    %rbx
 316 .cfi_push   %rbx
 317     push    %rbp
 318 .cfi_push   %rbp
 319     push    %r12
 320 .cfi_push   %r12
 321     push    %r13
 322 .cfi_push   %r13
 323     push    %r14
 324 .cfi_push   %r14
 325     push    %r15
 326 .cfi_push   %r15
 327 .Lossl_rsaz_amm52x20_x1_ifma256_body:
 328
 329     # Zeroing accumulators
 330     vpxord   $zero, $zero, $zero
 331     vmovdqa64   $zero, $R0_0
 332     vmovdqa64   $zero, $R0_0h
 333     vmovdqa64   $zero, $R1_0
 334     vmovdqa64   $zero, $R1_0h
 335     vmovdqa64   $zero, $R2_0
 336
 337     xorl    $acc0_0_low, $acc0_0_low
 338
 339     movq    $b, $b_ptr                       # backup address of b
 340     movq    \$0xfffffffffffff, $mask52       # 52-bit mask
 341
 342     # Loop over 20 digits unrolled by 4
 343     mov     \$5, $iter
 344
 345 .align 32
 346 .Lloop5:
 347 ___
 348     foreach my $idx (0..3) {
 349         &amm52x20_x1(0,8*$idx,$acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$k0);
 350     }
 351 $code.=<<___;
 352     lea    `4*8`($b_ptr), $b_ptr
 353     dec    $iter
 354     jne    .Lloop5
 355 ___
 356     &amm52x20_x1_norm($acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0);
 357 $code.=<<___;
 358
 359     vmovdqu64   $R0_0,  `0*32`($res)
 360     vmovdqu64   $R0_0h, `1*32`($res)
 361     vmovdqu64   $R1_0,  `2*32`($res)
 362     vmovdqu64   $R1_0h, `3*32`($res)
 363     vmovdqu64   $R2_0,  `4*32`($res)
 364
 365     vzeroupper
 366     mov  0(%rsp),%r15
 367 .cfi_restore    %r15
 368     mov  8(%rsp),%r14
 369 .cfi_restore    %r14
 370     mov  16(%rsp),%r13
 371 .cfi_restore    %r13
 372     mov  24(%rsp),%r12
 373 .cfi_restore    %r12
 374     mov  32(%rsp),%rbp
 375 .cfi_restore    %rbp
 376     mov  40(%rsp),%rbx
 377 .cfi_restore    %rbx
 378     lea  48(%rsp),%rsp
 379 .cfi_adjust_cfa_offset  -48
 380 .Lossl_rsaz_amm52x20_x1_ifma256_epilogue:
 381     ret
 382 .cfi_endproc
 383 .size   ossl_rsaz_amm52x20_x1_ifma256, .-ossl_rsaz_amm52x20_x1_ifma256
 384 ___
 385
 386 $code.=<<___;
 387 .data
 388 .align 32
 389 .Lmask52x4:
 390     .quad   0xfffffffffffff
 391     .quad   0xfffffffffffff
 392     .quad   0xfffffffffffff
 393     .quad   0xfffffffffffff
 394 ___
 395
 396 ###############################################################################
 397 # Dual Almost Montgomery Multiplication for 20-digit number in radix 2^52
 398 #
 399 # See description of ossl_rsaz_amm52x20_x1_ifma256() above for details about Almost
 400 # Montgomery Multiplication algorithm and function input parameters description.
 401 #
 402 # This function does two AMMs for two independent inputs, hence dual.
 403 #
 404 # void ossl_rsaz_amm52x20_x2_ifma256(BN_ULONG out[2][20],
 405 #                                    const BN_ULONG a[2][20],
 406 #                                    const BN_ULONG b[2][20],
 407 #                                    const BN_ULONG m[2][20],
 408 #                                    const BN_ULONG k0[2]);
 409 ###############################################################################
 410
 411 $code.=<<___;
 412 .text
 413
 414 .globl  ossl_rsaz_amm52x20_x2_ifma256
 415 .type   ossl_rsaz_amm52x20_x2_ifma256,\@function,5
 416 .align 32
 417 ossl_rsaz_amm52x20_x2_ifma256:
 418 .cfi_startproc
 419     endbranch
 420     push    %rbx
 421 .cfi_push   %rbx
 422     push    %rbp
 423 .cfi_push   %rbp
 424     push    %r12
 425 .cfi_push   %r12
 426     push    %r13
 427 .cfi_push   %r13
 428     push    %r14
 429 .cfi_push   %r14
 430     push    %r15
 431 .cfi_push   %r15
 432 .Lossl_rsaz_amm52x20_x2_ifma256_body:
 433
 434     # Zeroing accumulators
 435     vpxord   $zero, $zero, $zero
 436     vmovdqa64   $zero, $R0_0
 437     vmovdqa64   $zero, $R0_0h
 438     vmovdqa64   $zero, $R1_0
 439     vmovdqa64   $zero, $R1_0h
 440     vmovdqa64   $zero, $R2_0
 441     vmovdqa64   $zero, $R0_1
 442     vmovdqa64   $zero, $R0_1h
 443     vmovdqa64   $zero, $R1_1
 444     vmovdqa64   $zero, $R1_1h
 445     vmovdqa64   $zero, $R2_1
 446
 447     xorl    $acc0_0_low, $acc0_0_low
 448     xorl    $acc0_1_low, $acc0_1_low
 449
 450     movq    $b, $b_ptr                       # backup address of b
 451     movq    \$0xfffffffffffff, $mask52       # 52-bit mask
 452
 453     mov    \$20, $iter
 454
 455 .align 32
 456 .Lloop20:
 457 ___
 458     &amm52x20_x1(   0,   0,$acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,"($k0)");
 459     # 20*8 = offset of the next dimension in two-dimension array
 460     &amm52x20_x1(20*8,20*8,$acc0_1,$R0_1,$R0_1h,$R1_1,$R1_1h,$R2_1,"8($k0)");
 461 $code.=<<___;
 462     lea    8($b_ptr), $b_ptr
 463     dec    $iter
 464     jne    .Lloop20
 465 ___
 466     &amm52x20_x1_norm($acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0);
 467     &amm52x20_x1_norm($acc0_1,$R0_1,$R0_1h,$R1_1,$R1_1h,$R2_1);
 468 $code.=<<___;
 469
 470     vmovdqu64   $R0_0,  `0*32`($res)
 471     vmovdqu64   $R0_0h, `1*32`($res)
 472     vmovdqu64   $R1_0,  `2*32`($res)
 473     vmovdqu64   $R1_0h, `3*32`($res)
 474     vmovdqu64   $R2_0,  `4*32`($res)
 475
 476     vmovdqu64   $R0_1,  `5*32`($res)
 477     vmovdqu64   $R0_1h, `6*32`($res)
 478     vmovdqu64   $R1_1,  `7*32`($res)
 479     vmovdqu64   $R1_1h, `8*32`($res)
 480     vmovdqu64   $R2_1,  `9*32`($res)
 481
 482     vzeroupper
 483     mov  0(%rsp),%r15
 484 .cfi_restore    %r15
 485     mov  8(%rsp),%r14
 486 .cfi_restore    %r14
 487     mov  16(%rsp),%r13
 488 .cfi_restore    %r13
 489     mov  24(%rsp),%r12
 490 .cfi_restore    %r12
 491     mov  32(%rsp),%rbp
 492 .cfi_restore    %rbp
 493     mov  40(%rsp),%rbx
 494 .cfi_restore    %rbx
 495     lea  48(%rsp),%rsp
 496 .cfi_adjust_cfa_offset  -48
 497 .Lossl_rsaz_amm52x20_x2_ifma256_epilogue:
 498     ret
 499 .cfi_endproc
 500 .size   ossl_rsaz_amm52x20_x2_ifma256, .-ossl_rsaz_amm52x20_x2_ifma256
 501 ___
 502 }
 503
 504 ###############################################################################
 505 # Constant time extraction from the precomputed table of powers base^i, where
 506 #    i = 0..2^EXP_WIN_SIZE-1
 507 #
 508 # The input |red_table| contains precomputations for two independent base values.
 509 # |red_table_idx1| and |red_table_idx2| are corresponding power indexes.
 510 #
 511 # Extracted value (output) is 2 20 digit numbers in 2^52 radix.
 512 #
 513 # void ossl_extract_multiplier_2x20_win5(BN_ULONG *red_Y,
 514 #                                        const BN_ULONG red_table[1 << EXP_WIN_SIZE][2][20],
 515 #                                        int red_table_idx1, int red_table_idx2);
 516 #
 517 # EXP_WIN_SIZE = 5
 518 ###############################################################################
 519 {
 520 # input parameters
 521 my ($out,$red_tbl,$red_tbl_idx1,$red_tbl_idx2)=$win64 ? ("%rcx","%rdx","%r8", "%r9") :  # Win64 order
 522                                                         ("%rdi","%rsi","%rdx","%rcx");  # Unix order
 523
 524 my ($t0,$t1,$t2,$t3,$t4,$t5) = map("%ymm$_", (0..5));
 525 my ($t6,$t7,$t8,$t9) = map("%ymm$_", (16..19));
 526 my ($tmp,$cur_idx,$idx1,$idx2,$ones) = map("%ymm$_", (20..24));
 527
 528 my @t = ($t0,$t1,$t2,$t3,$t4,$t5,$t6,$t7,$t8,$t9);
 529 my $t0xmm = $t0;
 530 $t0xmm =~ s/%y/%x/;
 531
 532 $code.=<<___;
 533 .text
 534
 535 .align 32
 536 .globl  ossl_extract_multiplier_2x20_win5
 537 .type   ossl_extract_multiplier_2x20_win5,\@abi-omnipotent
 538 ossl_extract_multiplier_2x20_win5:
 539 .cfi_startproc
 540     endbranch
 541     vmovdqa64   .Lones(%rip), $ones         # broadcast ones
 542     vpbroadcastq    $red_tbl_idx1, $idx1
 543     vpbroadcastq    $red_tbl_idx2, $idx2
 544     leaq   `(1<<5)*2*20*8`($red_tbl), %rax  # holds end of the tbl
 545
 546     # zeroing t0..n, cur_idx
 547     vpxor   $t0xmm, $t0xmm, $t0xmm
 548     vmovdqa64   $t0, $cur_idx
 549 ___
 550 foreach (1..9) {
 551     $code.="vmovdqa64   $t0, $t[$_] \n";
 552 }
 553 $code.=<<___;
 554
 555 .align 32
 556 .Lloop:
 557     vpcmpq  \$0, $cur_idx, $idx1, %k1      # mask of (idx1 == cur_idx)
 558     vpcmpq  \$0, $cur_idx, $idx2, %k2      # mask of (idx2 == cur_idx)
 559 ___
 560 foreach (0..9) {
 561     my $mask = $_<5?"%k1":"%k2";
 562 $code.=<<___;
 563     vmovdqu64  `${_}*32`($red_tbl), $tmp     # load data from red_tbl
 564     vpblendmq  $tmp, $t[$_], ${t[$_]}{$mask} # extract data when mask is not zero
 565 ___
 566 }
 567 $code.=<<___;
 568     vpaddq  $ones, $cur_idx, $cur_idx      # increment cur_idx
 569     addq    \$`2*20*8`, $red_tbl
 570     cmpq    $red_tbl, %rax
 571     jne .Lloop
 572 ___
 573 # store t0..n
 574 foreach (0..9) {
 575     $code.="vmovdqu64   $t[$_], `${_}*32`($out) \n";
 576 }
 577 $code.=<<___;
 578     ret
 579 .cfi_endproc
 580 .size   ossl_extract_multiplier_2x20_win5, .-ossl_extract_multiplier_2x20_win5
 581 ___
 582 $code.=<<___;
 583 .data
 584 .align 32
 585 .Lones:
 586     .quad   1,1,1,1
 587 .Lzeros:
 588     .quad   0,0,0,0
 589 ___
 590 }
 591
 592 if ($win64) {
 593 $rec="%rcx";
 594 $frame="%rdx";
 595 $context="%r8";
 596 $disp="%r9";
 597
 598 $code.=<<___;
 599 .extern     __imp_RtlVirtualUnwind
 600 .type   rsaz_def_handler,\@abi-omnipotent
 601 .align  16
 602 rsaz_def_handler:
 603     push    %rsi
 604     push    %rdi
 605     push    %rbx
 606     push    %rbp
 607     push    %r12
 608     push    %r13
 609     push    %r14
 610     push    %r15
 611     pushfq
 612     sub     \$64,%rsp
 613
 614     mov     120($context),%rax # pull context->Rax
 615     mov     248($context),%rbx # pull context->Rip
 616
 617     mov     8($disp),%rsi      # disp->ImageBase
 618     mov     56($disp),%r11     # disp->HandlerData
 619
 620     mov     0(%r11),%r10d      # HandlerData[0]
 621     lea     (%rsi,%r10),%r10   # prologue label
 622     cmp     %r10,%rbx          # context->Rip<.Lprologue
 623     jb  .Lcommon_seh_tail
 624
 625     mov     152($context),%rax # pull context->Rsp
 626
 627     mov     4(%r11),%r10d      # HandlerData[1]
 628     lea     (%rsi,%r10),%r10   # epilogue label
 629     cmp     %r10,%rbx          # context->Rip>=.Lepilogue
 630     jae     .Lcommon_seh_tail
 631
 632     lea     48(%rax),%rax
 633
 634     mov     -8(%rax),%rbx
 635     mov     -16(%rax),%rbp
 636     mov     -24(%rax),%r12
 637     mov     -32(%rax),%r13
 638     mov     -40(%rax),%r14
 639     mov     -48(%rax),%r15
 640     mov     %rbx,144($context) # restore context->Rbx
 641     mov     %rbp,160($context) # restore context->Rbp
 642     mov     %r12,216($context) # restore context->R12
 643     mov     %r13,224($context) # restore context->R13
 644     mov     %r14,232($context) # restore context->R14
 645     mov     %r15,240($context) # restore context->R14
 646
 647 .Lcommon_seh_tail:
 648     mov     8(%rax),%rdi
 649     mov     16(%rax),%rsi
 650     mov     %rax,152($context) # restore context->Rsp
 651     mov     %rsi,168($context) # restore context->Rsi
 652     mov     %rdi,176($context) # restore context->Rdi
 653
 654     mov     40($disp),%rdi     # disp->ContextRecord
 655     mov     $context,%rsi      # context
 656     mov     \$154,%ecx         # sizeof(CONTEXT)
 657     .long   0xa548f3fc         # cld; rep movsq
 658
 659     mov     $disp,%rsi
 660     xor     %rcx,%rcx          # arg1, UNW_FLAG_NHANDLER
 661     mov     8(%rsi),%rdx       # arg2, disp->ImageBase
 662     mov     0(%rsi),%r8        # arg3, disp->ControlPc
 663     mov     16(%rsi),%r9       # arg4, disp->FunctionEntry
 664     mov     40(%rsi),%r10      # disp->ContextRecord
 665     lea     56(%rsi),%r11      # &disp->HandlerData
 666     lea     24(%rsi),%r12      # &disp->EstablisherFrame
 667     mov     %r10,32(%rsp)      # arg5
 668     mov     %r11,40(%rsp)      # arg6
 669     mov     %r12,48(%rsp)      # arg7
 670     mov     %rcx,56(%rsp)      # arg8, (NULL)
 671     call    *__imp_RtlVirtualUnwind(%rip)
 672
 673     mov     \$1,%eax           # ExceptionContinueSearch
 674     add     \$64,%rsp
 675     popfq
 676     pop     %r15
 677     pop     %r14
 678     pop     %r13
 679     pop     %r12
 680     pop     %rbp
 681     pop     %rbx
 682     pop     %rdi
 683     pop     %rsi
 684     ret
 685 .size   rsaz_def_handler,.-rsaz_def_handler
 686
 687 .section    .pdata
 688 .align  4
 689     .rva    .LSEH_begin_ossl_rsaz_amm52x20_x1_ifma256
 690     .rva    .LSEH_end_ossl_rsaz_amm52x20_x1_ifma256
 691     .rva    .LSEH_info_ossl_rsaz_amm52x20_x1_ifma256
 692
 693     .rva    .LSEH_begin_ossl_rsaz_amm52x20_x2_ifma256
 694     .rva    .LSEH_end_ossl_rsaz_amm52x20_x2_ifma256
 695     .rva    .LSEH_info_ossl_rsaz_amm52x20_x2_ifma256
 696
 697 .section    .xdata
 698 .align  8
 699 .LSEH_info_ossl_rsaz_amm52x20_x1_ifma256:
 700     .byte   9,0,0,0
 701     .rva    rsaz_def_handler
 702     .rva    .Lossl_rsaz_amm52x20_x1_ifma256_body,.Lossl_rsaz_amm52x20_x1_ifma256_epilogue
 703 .LSEH_info_ossl_rsaz_amm52x20_x2_ifma256:
 704     .byte   9,0,0,0
 705     .rva    rsaz_def_handler
 706     .rva    .Lossl_rsaz_amm52x20_x2_ifma256_body,.Lossl_rsaz_amm52x20_x2_ifma256_epilogue
 707 ___
 708 }
 709 }}} else {{{                # fallback for old assembler
 710 $code.=<<___;
 711 .text
 712
 713 .globl  ossl_rsaz_avx512ifma_eligible
 714 .type   ossl_rsaz_avx512ifma_eligible,\@abi-omnipotent
 715 ossl_rsaz_avx512ifma_eligible:
 716     xor     %eax,%eax
 717     ret
 718 .size   ossl_rsaz_avx512ifma_eligible, .-ossl_rsaz_avx512ifma_eligible
 719
 720 .globl  ossl_rsaz_amm52x20_x1_ifma256
 721 .globl  ossl_rsaz_amm52x20_x2_ifma256
 722 .globl  ossl_extract_multiplier_2x20_win5
 723 .type   ossl_rsaz_amm52x20_x1_ifma256,\@abi-omnipotent
 724 ossl_rsaz_amm52x20_x1_ifma256:
 725 ossl_rsaz_amm52x20_x2_ifma256:
 726 ossl_extract_multiplier_2x20_win5:
 727     .byte   0x0f,0x0b    # ud2
 728     ret
 729 .size   ossl_rsaz_amm52x20_x1_ifma256, .-ossl_rsaz_amm52x20_x1_ifma256
 730 ___
 731 }}}
 732
 733 $code =~ s/\`([^\`]*)\`/eval $1/gem;
 734 print $code;
 735 close STDOUT or die "error closing STDOUT: $!";