crypto/bn/asm/rsaz-avx512.pl

   1 # Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved.
   2 # Copyright (c) 2020, Intel Corporation. All Rights Reserved.
   3 #
   4 # Licensed under the Apache License 2.0 (the "License").  You may not use
   5 # this file except in compliance with the License.  You can obtain a copy
   6 # in the file LICENSE in the source distribution or at
   7 # https://www.openssl.org/source/license.html
   8 #
   9 #
  10 # Originally written by Ilya Albrekht, Sergey Kirillov and Andrey Matyukov
  11 # Intel Corporation
  12 #
  13 # December 2020
  14 #
  15 # Initial release.
  16 #
  17 # Implementation utilizes 256-bit (ymm) registers to avoid frequency scaling issues.
  18 #
  19 # IceLake-Client @ 1.3GHz
  20 # |---------+----------------------+--------------+-------------|
  21 # |         | OpenSSL 3.0.0-alpha9 | this         | Unit        |
  22 # |---------+----------------------+--------------+-------------|
  23 # | rsa2048 | 2 127 659            | 1 015 625    | cycles/sign |
  24 # |         | 611                  | 1280 / +109% | sign/s      |
  25 # |---------+----------------------+--------------+-------------|
  26 #
  27
  28 # $output is the last argument if it looks like a file (it has an extension)
  29 # $flavour is the first argument if it doesn't look like a file
  30 $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
  31 $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
  32
  33 $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
  34 $avx512ifma=0;
  35
  36 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  37 ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
  38 ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
  39 die "can't locate x86_64-xlate.pl";
  40
  41 if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
  42         =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
  43     $avx512ifma = ($1>=2.26);
  44 }
  45
  46 if (!$avx512 && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
  47        `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)(?:\.([0-9]+))?/) {
  48     $avx512ifma = ($1==2.11 && $2>=8) + ($1>=2.12);
  49 }
  50
  51 if (!$avx512 && `$ENV{CC} -v 2>&1` =~ /((?:clang|LLVM) version|.*based on LLVM) ([0-9]+\.[0-9]+)/) {
  52     $avx512ifma = ($2>=7.0);
  53 }
  54
  55 open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\""
  56     or die "can't call $xlate: $!";
  57 *STDOUT=*OUT;
  58
  59 if ($avx512ifma>0) {{{
  60 @_6_args_universal_ABI = ("%rdi","%rsi","%rdx","%rcx","%r8","%r9");
  61
  62 $code.=<<___;
  63 .extern OPENSSL_ia32cap_P
  64 .globl  rsaz_avx512ifma_eligible
  65 .type   rsaz_avx512ifma_eligible,\@abi-omnipotent
  66 .align  32
  67 rsaz_avx512ifma_eligible:
  68     mov OPENSSL_ia32cap_P+8(%rip), %ecx
  69     xor %eax,%eax
  70     and \$`1<<31|1<<21|1<<17|1<<16`, %ecx     # avx512vl + avx512ifma + avx512dq + avx512f
  71     cmp \$`1<<31|1<<21|1<<17|1<<16`, %ecx
  72     cmove %ecx,%eax
  73     ret
  74 .size   rsaz_avx512ifma_eligible, .-rsaz_avx512ifma_eligible
  75 ___
  76
  77 ###############################################################################
  78 # Almost Montgomery Multiplication (AMM) for 20-digit number in radix 2^52.
  79 #
  80 # AMM is defined as presented in the paper
  81 # "Efficient Software Implementations of Modular Exponentiation" by Shay Gueron.
  82 #
  83 # The input and output are presented in 2^52 radix domain, i.e.
  84 #   |res|, |a|, |b|, |m| are arrays of 20 64-bit qwords with 12 high bits zeroed.
  85 #   |k0| is a Montgomery coefficient, which is here k0 = -1/m mod 2^64
  86 #        (note, the implementation counts only 52 bits from it).
  87 #
  88 # NB: the AMM implementation does not perform "conditional" subtraction step as
  89 # specified in the original algorithm as according to the paper "Enhanced Montgomery
  90 # Multiplication" by Shay Gueron (see Lemma 1), the result will be always < 2*2^1024
  91 # and can be used as a direct input to the next AMM iteration.
  92 # This post-condition is true, provided the correct parameter |s| is choosen, i.e.
  93 # s >= n + 2 * k, which matches our case: 1040 > 1024 + 2 * 1.
  94 #
  95 # void RSAZ_amm52x20_x1_256(BN_ULONG *res,
  96 #                           const BN_ULONG *a,
  97 #                           const BN_ULONG *b,
  98 #                           const BN_ULONG *m,
  99 #                           BN_ULONG k0);
 100 ###############################################################################
 101 {
 102 # input parameters ("%rdi","%rsi","%rdx","%rcx","%r8")
 103 my ($res,$a,$b,$m,$k0) = @_6_args_universal_ABI;
 104
 105 my $mask52     = "%rax";
 106 my $acc0_0     = "%r9";
 107 my $acc0_0_low = "%r9d";
 108 my $acc0_1     = "%r15";
 109 my $acc0_1_low = "%r15d";
 110 my $b_ptr      = "%r11";
 111
 112 my $iter = "%ebx";
 113
 114 my $zero = "%ymm0";
 115 my ($R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0) = ("%ymm1", map("%ymm$_",(16..19)));
 116 my ($R0_1,$R0_1h,$R1_1,$R1_1h,$R2_1) = ("%ymm2", map("%ymm$_",(20..23)));
 117 my $Bi = "%ymm3";
 118 my $Yi = "%ymm4";
 119
 120 # Registers mapping for normalization.
 121 # We can reuse Bi, Yi registers here.
 122 my $TMP = $Bi;
 123 my $mask52x4 = $Yi;
 124 my ($T0,$T0h,$T1,$T1h,$T2) = map("%ymm$_", (24..28));
 125
 126 sub amm52x20_x1() {
 127 # _data_offset - offset in the |a| or |m| arrays pointing to the beginning
 128 #                of data for corresponding AMM operation;
 129 # _b_offset    - offset in the |b| array pointing to the next qword digit;
 130 my ($_data_offset,$_b_offset,$_acc,$_R0,$_R0h,$_R1,$_R1h,$_R2,$_k0) = @_;
 131 my $_R0_xmm = $_R0 =~ s/%y/%x/r;
 132 $code.=<<___;
 133     movq    $_b_offset($b_ptr), %r13             # b[i]
 134
 135     vpbroadcastq    %r13, $Bi                    # broadcast b[i]
 136     movq    $_data_offset($a), %rdx
 137     mulx    %r13, %r13, %r12                     # a[0]*b[i] = (t0,t2)
 138     addq    %r13, $_acc                          # acc += t0
 139     movq    %r12, %r10
 140     adcq    \$0, %r10                            # t2 += CF
 141
 142     movq    $_k0, %r13
 143     imulq   $_acc, %r13                          # acc * k0
 144     andq    $mask52, %r13                        # yi = (acc * k0) & mask52
 145
 146     vpbroadcastq    %r13, $Yi                    # broadcast y[i]
 147     movq    $_data_offset($m), %rdx
 148     mulx    %r13, %r13, %r12                     # yi * m[0] = (t0,t1)
 149     addq    %r13, $_acc                          # acc += t0
 150     adcq    %r12, %r10                           # t2 += (t1 + CF)
 151
 152     shrq    \$52, $_acc
 153     salq    \$12, %r10
 154     or      %r10, $_acc                          # acc = ((acc >> 52) | (t2 << 12))
 155
 156     vpmadd52luq `$_data_offset+64*0`($a), $Bi, $_R0
 157     vpmadd52luq `$_data_offset+64*0+32`($a), $Bi, $_R0h
 158     vpmadd52luq `$_data_offset+64*1`($a), $Bi, $_R1
 159     vpmadd52luq `$_data_offset+64*1+32`($a), $Bi, $_R1h
 160     vpmadd52luq `$_data_offset+64*2`($a), $Bi, $_R2
 161
 162     vpmadd52luq `$_data_offset+64*0`($m), $Yi, $_R0
 163     vpmadd52luq `$_data_offset+64*0+32`($m), $Yi, $_R0h
 164     vpmadd52luq `$_data_offset+64*1`($m), $Yi, $_R1
 165     vpmadd52luq `$_data_offset+64*1+32`($m), $Yi, $_R1h
 166     vpmadd52luq `$_data_offset+64*2`($m), $Yi, $_R2
 167
 168     # Shift accumulators right by 1 qword, zero extending the highest one
 169     valignq     \$1, $_R0, $_R0h, $_R0
 170     valignq     \$1, $_R0h, $_R1, $_R0h
 171     valignq     \$1, $_R1, $_R1h, $_R1
 172     valignq     \$1, $_R1h, $_R2, $_R1h
 173     valignq     \$1, $_R2, $zero, $_R2
 174
 175     vmovq   $_R0_xmm, %r13
 176     addq    %r13, $_acc    # acc += R0[0]
 177
 178     vpmadd52huq `$_data_offset+64*0`($a), $Bi, $_R0
 179     vpmadd52huq `$_data_offset+64*0+32`($a), $Bi, $_R0h
 180     vpmadd52huq `$_data_offset+64*1`($a), $Bi, $_R1
 181     vpmadd52huq `$_data_offset+64*1+32`($a), $Bi, $_R1h
 182     vpmadd52huq `$_data_offset+64*2`($a), $Bi, $_R2
 183
 184     vpmadd52huq `$_data_offset+64*0`($m), $Yi, $_R0
 185     vpmadd52huq `$_data_offset+64*0+32`($m), $Yi, $_R0h
 186     vpmadd52huq `$_data_offset+64*1`($m), $Yi, $_R1
 187     vpmadd52huq `$_data_offset+64*1+32`($m), $Yi, $_R1h
 188     vpmadd52huq `$_data_offset+64*2`($m), $Yi, $_R2
 189 ___
 190 }
 191
 192 # Normalization routine: handles carry bits in R0..R2 QWs and
 193 # gets R0..R2 back to normalized 2^52 representation.
 194 #
 195 # Uses %r8-14,%e[bcd]x
 196 sub amm52x20_x1_norm {
 197 my ($_acc,$_R0,$_R0h,$_R1,$_R1h,$_R2) = @_;
 198 $code.=<<___;
 199     # Put accumulator to low qword in R0
 200     vpbroadcastq    $_acc, $TMP
 201     vpblendd \$3, $TMP, $_R0, $_R0
 202
 203     # Extract "carries" (12 high bits) from each QW of R0..R2
 204     # Save them to LSB of QWs in T0..T2
 205     vpsrlq    \$52, $_R0,   $T0
 206     vpsrlq    \$52, $_R0h,  $T0h
 207     vpsrlq    \$52, $_R1,   $T1
 208     vpsrlq    \$52, $_R1h,  $T1h
 209     vpsrlq    \$52, $_R2,   $T2
 210
 211     # "Shift left" T0..T2 by 1 QW
 212     valignq \$3, $T1h,  $T2,  $T2
 213     valignq \$3, $T1,   $T1h, $T1h
 214     valignq \$3, $T0h,  $T1,  $T1
 215     valignq \$3, $T0,   $T0h, $T0h
 216     valignq \$3, $zero, $T0,  $T0
 217
 218     # Drop "carries" from R0..R2 QWs
 219     vpandq    $mask52x4, $_R0,  $_R0
 220     vpandq    $mask52x4, $_R0h, $_R0h
 221     vpandq    $mask52x4, $_R1,  $_R1
 222     vpandq    $mask52x4, $_R1h, $_R1h
 223     vpandq    $mask52x4, $_R2,  $_R2
 224
 225     # Sum R0..R2 with corresponding adjusted carries
 226     vpaddq  $T0,  $_R0,  $_R0
 227     vpaddq  $T0h, $_R0h, $_R0h
 228     vpaddq  $T1,  $_R1,  $_R1
 229     vpaddq  $T1h, $_R1h, $_R1h
 230     vpaddq  $T2,  $_R2,  $_R2
 231
 232     # Now handle carry bits from this addition
 233     # Get mask of QWs which 52-bit parts overflow...
 234     vpcmpuq   \$1, $_R0,  $mask52x4, %k1 # OP=lt
 235     vpcmpuq   \$1, $_R0h, $mask52x4, %k2
 236     vpcmpuq   \$1, $_R1,  $mask52x4, %k3
 237     vpcmpuq   \$1, $_R1h, $mask52x4, %k4
 238     vpcmpuq   \$1, $_R2,  $mask52x4, %k5
 239     kmovb   %k1, %r14d                   # k1
 240     kmovb   %k2, %r13d                   # k1h
 241     kmovb   %k3, %r12d                   # k2
 242     kmovb   %k4, %r11d                   # k2h
 243     kmovb   %k5, %r10d                   # k3
 244
 245     # ...or saturated
 246     vpcmpuq   \$0, $_R0,  $mask52x4, %k1 # OP=eq
 247     vpcmpuq   \$0, $_R0h, $mask52x4, %k2
 248     vpcmpuq   \$0, $_R1,  $mask52x4, %k3
 249     vpcmpuq   \$0, $_R1h, $mask52x4, %k4
 250     vpcmpuq   \$0, $_R2,  $mask52x4, %k5
 251     kmovb   %k1, %r9d                    # k4
 252     kmovb   %k2, %r8d                    # k4h
 253     kmovb   %k3, %ebx                    # k5
 254     kmovb   %k4, %ecx                    # k5h
 255     kmovb   %k5, %edx                    # k6
 256
 257     # Get mask of QWs where carries shall be propagated to.
 258     # Merge 4-bit masks to 8-bit values to use add with carry.
 259     shl   \$4, %r13b
 260     or    %r13b, %r14b
 261     shl   \$4, %r11b
 262     or    %r11b, %r12b
 263
 264     add   %r14b, %r14b
 265     adc   %r12b, %r12b
 266     adc   %r10b, %r10b
 267
 268     shl   \$4, %r8b
 269     or    %r8b,%r9b
 270     shl   \$4, %cl
 271     or    %cl, %bl
 272
 273     add   %r9b, %r14b
 274     adc   %bl, %r12b
 275     adc   %dl, %r10b
 276
 277     xor   %r9b, %r14b
 278     xor   %bl, %r12b
 279     xor   %dl, %r10b
 280
 281     kmovb   %r14d, %k1
 282     shr     \$4, %r14b
 283     kmovb   %r14d, %k2
 284     kmovb   %r12d, %k3
 285     shr     \$4, %r12b
 286     kmovb   %r12d, %k4
 287     kmovb   %r10d, %k5
 288
 289     # Add carries according to the obtained mask
 290     vpsubq  $mask52x4, $_R0,  ${_R0}{%k1}
 291     vpsubq  $mask52x4, $_R0h, ${_R0h}{%k2}
 292     vpsubq  $mask52x4, $_R1,  ${_R1}{%k3}
 293     vpsubq  $mask52x4, $_R1h, ${_R1h}{%k4}
 294     vpsubq  $mask52x4, $_R2,  ${_R2}{%k5}
 295
 296     vpandq   $mask52x4, $_R0,  $_R0
 297     vpandq   $mask52x4, $_R0h, $_R0h
 298     vpandq   $mask52x4, $_R1,  $_R1
 299     vpandq   $mask52x4, $_R1h, $_R1h
 300     vpandq   $mask52x4, $_R2,  $_R2
 301 ___
 302 }
 303
 304 $code.=<<___;
 305 .text
 306
 307 .globl  RSAZ_amm52x20_x1_256
 308 .type   RSAZ_amm52x20_x1_256,\@function,5
 309 .align 32
 310 RSAZ_amm52x20_x1_256:
 311 .cfi_startproc
 312     endbranch
 313     push    %rbx
 314 .cfi_push   %rbx
 315     push    %rbp
 316 .cfi_push   %rbp
 317     push    %r12
 318 .cfi_push   %r12
 319     push    %r13
 320 .cfi_push   %r13
 321     push    %r14
 322 .cfi_push   %r14
 323     push    %r15
 324 .cfi_push   %r15
 325 .Lrsaz_amm52x20_x1_256_body:
 326
 327     # Zeroing accumulators
 328     vpxord   $zero, $zero, $zero
 329     vmovdqa64   $zero, $R0_0
 330     vmovdqa64   $zero, $R0_0h
 331     vmovdqa64   $zero, $R1_0
 332     vmovdqa64   $zero, $R1_0h
 333     vmovdqa64   $zero, $R2_0
 334
 335     xorl    $acc0_0_low, $acc0_0_low
 336
 337     movq    $b, $b_ptr                       # backup address of b
 338     movq    \$0xfffffffffffff, $mask52       # 52-bit mask
 339
 340     # Loop over 20 digits unrolled by 4
 341     mov     \$5, $iter
 342
 343 .align 32
 344 .Lloop5:
 345 ___
 346     foreach my $idx (0..3) {
 347         &amm52x20_x1(0,8*$idx,$acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$k0);
 348     }
 349 $code.=<<___;
 350     lea    `4*8`($b_ptr), $b_ptr
 351     dec    $iter
 352     jne    .Lloop5
 353
 354     vmovdqa64   .Lmask52x4(%rip), $mask52x4
 355 ___
 356     &amm52x20_x1_norm($acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0);
 357 $code.=<<___;
 358
 359     vmovdqu64   $R0_0, ($res)
 360     vmovdqu64   $R0_0h, 32($res)
 361     vmovdqu64   $R1_0, 64($res)
 362     vmovdqu64   $R1_0h, 96($res)
 363     vmovdqu64   $R2_0, 128($res)
 364
 365     vzeroupper
 366     mov  0(%rsp),%r15
 367 .cfi_restore    %r15
 368     mov  8(%rsp),%r14
 369 .cfi_restore    %r14
 370     mov  16(%rsp),%r13
 371 .cfi_restore    %r13
 372     mov  24(%rsp),%r12
 373 .cfi_restore    %r12
 374     mov  32(%rsp),%rbp
 375 .cfi_restore    %rbp
 376     mov  40(%rsp),%rbx
 377 .cfi_restore    %rbx
 378     lea  48(%rsp),%rsp
 379 .cfi_adjust_cfa_offset  -48
 380 .Lrsaz_amm52x20_x1_256_epilogue:
 381     ret
 382 .cfi_endproc
 383 .size   RSAZ_amm52x20_x1_256, .-RSAZ_amm52x20_x1_256
 384 ___
 385
 386 $code.=<<___;
 387 .data
 388 .align 32
 389 .Lmask52x4:
 390     .quad   0xfffffffffffff
 391     .quad   0xfffffffffffff
 392     .quad   0xfffffffffffff
 393     .quad   0xfffffffffffff
 394 ___
 395
 396 ###############################################################################
 397 # Dual Almost Montgomery Multiplication for 20-digit number in radix 2^52
 398 #
 399 # See description of RSAZ_amm52x20_x1_256() above for details about Almost
 400 # Montgomery Multiplication algorithm and function input parameters description.
 401 #
 402 # This function does two AMMs for two independent inputs, hence dual.
 403 #
 404 # void RSAZ_amm52x20_x2_256(BN_ULONG out[2][20],
 405 #                           const BN_ULONG a[2][20],
 406 #                           const BN_ULONG b[2][20],
 407 #                           const BN_ULONG m[2][20],
 408 #                           const BN_ULONG k0[2]);
 409 ###############################################################################
 410
 411 $code.=<<___;
 412 .text
 413
 414 .globl  RSAZ_amm52x20_x2_256
 415 .type   RSAZ_amm52x20_x2_256,\@function,5
 416 .align 32
 417 RSAZ_amm52x20_x2_256:
 418 .cfi_startproc
 419     endbranch
 420     push    %rbx
 421 .cfi_push   %rbx
 422     push    %rbp
 423 .cfi_push   %rbp
 424     push    %r12
 425 .cfi_push   %r12
 426     push    %r13
 427 .cfi_push   %r13
 428     push    %r14
 429 .cfi_push   %r14
 430     push    %r15
 431 .cfi_push   %r15
 432 .Lrsaz_amm52x20_x2_256_body:
 433
 434     # Zeroing accumulators
 435     vpxord   $zero, $zero, $zero
 436     vmovdqa64   $zero, $R0_0
 437     vmovdqa64   $zero, $R0_0h
 438     vmovdqa64   $zero, $R1_0
 439     vmovdqa64   $zero, $R1_0h
 440     vmovdqa64   $zero, $R2_0
 441     vmovdqa64   $zero, $R0_1
 442     vmovdqa64   $zero, $R0_1h
 443     vmovdqa64   $zero, $R1_1
 444     vmovdqa64   $zero, $R1_1h
 445     vmovdqa64   $zero, $R2_1
 446
 447     xorl    $acc0_0_low, $acc0_0_low
 448     xorl    $acc0_1_low, $acc0_1_low
 449
 450     movq    $b, $b_ptr                       # backup address of b
 451     movq    \$0xfffffffffffff, $mask52       # 52-bit mask
 452
 453     mov    \$20, $iter
 454
 455 .align 32
 456 .Lloop20:
 457 ___
 458     &amm52x20_x1(   0,   0,$acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,"($k0)");
 459     # 20*8 = offset of the next dimension in two-dimension array
 460     &amm52x20_x1(20*8,20*8,$acc0_1,$R0_1,$R0_1h,$R1_1,$R1_1h,$R2_1,"8($k0)");
 461 $code.=<<___;
 462     lea    8($b_ptr), $b_ptr
 463     dec    $iter
 464     jne    .Lloop20
 465
 466     vmovdqa64   .Lmask52x4(%rip), $mask52x4
 467 ___
 468     &amm52x20_x1_norm($acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0);
 469     &amm52x20_x1_norm($acc0_1,$R0_1,$R0_1h,$R1_1,$R1_1h,$R2_1);
 470 $code.=<<___;
 471
 472     vmovdqu64   $R0_0, ($res)
 473     vmovdqu64   $R0_0h, 32($res)
 474     vmovdqu64   $R1_0, 64($res)
 475     vmovdqu64   $R1_0h, 96($res)
 476     vmovdqu64   $R2_0, 128($res)
 477
 478     vmovdqu64   $R0_1, 160($res)
 479     vmovdqu64   $R0_1h, 192($res)
 480     vmovdqu64   $R1_1, 224($res)
 481     vmovdqu64   $R1_1h, 256($res)
 482     vmovdqu64   $R2_1, 288($res)
 483
 484     vzeroupper
 485     mov  0(%rsp),%r15
 486 .cfi_restore    %r15
 487     mov  8(%rsp),%r14
 488 .cfi_restore    %r14
 489     mov  16(%rsp),%r13
 490 .cfi_restore    %r13
 491     mov  24(%rsp),%r12
 492 .cfi_restore    %r12
 493     mov  32(%rsp),%rbp
 494 .cfi_restore    %rbp
 495     mov  40(%rsp),%rbx
 496 .cfi_restore    %rbx
 497     lea  48(%rsp),%rsp
 498 .cfi_adjust_cfa_offset  -48
 499 .Lrsaz_amm52x20_x2_256_epilogue:
 500     ret
 501 .cfi_endproc
 502 .size   RSAZ_amm52x20_x2_256, .-RSAZ_amm52x20_x2_256
 503 ___
 504 }
 505
 506 ###############################################################################
 507 # Constant time extraction from the precomputed table of powers base^i, where
 508 #    i = 0..2^EXP_WIN_SIZE-1
 509 #
 510 # The input |red_table| contains precomputations for two independent base values,
 511 # so the |tbl_idx| indicates for which base shall we extract the value.
 512 # |red_table_idx| is a power index.
 513 #
 514 # Extracted value (output) is 20 digit number in 2^52 radix.
 515 #
 516 # void extract_multiplier_2x20_win5(BN_ULONG *red_Y,
 517 #                                   const BN_ULONG red_table[1 << EXP_WIN_SIZE][2][20],
 518 #                                   int red_table_idx,
 519 #                                   int tbl_idx);           # 0 or 1
 520 #
 521 # EXP_WIN_SIZE = 5
 522 ###############################################################################
 523 {
 524 # input parameters
 525 my ($out,$red_tbl,$red_tbl_idx,$tbl_idx) = @_6_args_universal_ABI;
 526
 527 my ($t0,$t1,$t2,$t3,$t4) = map("%ymm$_", (0..4));
 528 my $t4xmm = $t4 =~ s/%y/%x/r;
 529 my ($tmp0,$tmp1,$tmp2,$tmp3,$tmp4) = map("%ymm$_", (16..20));
 530 my ($cur_idx,$idx,$ones) = map("%ymm$_", (21..23));
 531
 532 $code.=<<___;
 533 .text
 534
 535 .align 32
 536 .globl  extract_multiplier_2x20_win5
 537 .type   extract_multiplier_2x20_win5,\@function,4
 538 extract_multiplier_2x20_win5:
 539 .cfi_startproc
 540     endbranch
 541     leaq    ($tbl_idx,$tbl_idx,4), %rax
 542     salq    \$5, %rax
 543     addq    %rax, $red_tbl
 544
 545     vmovdqa64   .Lones(%rip), $ones         # broadcast ones
 546     vpbroadcastq    $red_tbl_idx, $idx
 547     leaq   `(1<<5)*2*20*8`($red_tbl), %rax  # holds end of the tbl
 548
 549     vpxor   $t4xmm, $t4xmm, $t4xmm
 550     vmovdqa64   $t4, $t3                    # zeroing t0..4, cur_idx
 551     vmovdqa64   $t4, $t2
 552     vmovdqa64   $t4, $t1
 553     vmovdqa64   $t4, $t0
 554     vmovdqa64   $t4, $cur_idx
 555
 556 .align 32
 557 .Lloop:
 558     vpcmpq  \$0, $cur_idx, $idx, %k1        # mask of (idx == cur_idx)
 559     addq    \$320, $red_tbl                 # 320 = 2 * 20 digits * 8 bytes
 560     vpaddq  $ones, $cur_idx, $cur_idx       # increment cur_idx
 561     vmovdqu64  -320($red_tbl), $tmp0        # load data from red_tbl
 562     vmovdqu64  -288($red_tbl), $tmp1
 563     vmovdqu64  -256($red_tbl), $tmp2
 564     vmovdqu64  -224($red_tbl), $tmp3
 565     vmovdqu64  -192($red_tbl), $tmp4
 566     vpblendmq  $tmp0, $t0, ${t0}{%k1}       # extract data when mask is not zero
 567     vpblendmq  $tmp1, $t1, ${t1}{%k1}
 568     vpblendmq  $tmp2, $t2, ${t2}{%k1}
 569     vpblendmq  $tmp3, $t3, ${t3}{%k1}
 570     vpblendmq  $tmp4, $t4, ${t4}{%k1}
 571     cmpq    $red_tbl, %rax
 572     jne .Lloop
 573
 574     vmovdqu64   $t0, ($out)                 # store t0..4
 575     vmovdqu64   $t1, 32($out)
 576     vmovdqu64   $t2, 64($out)
 577     vmovdqu64   $t3, 96($out)
 578     vmovdqu64   $t4, 128($out)
 579
 580     ret
 581 .cfi_endproc
 582 .size   extract_multiplier_2x20_win5, .-extract_multiplier_2x20_win5
 583 ___
 584 $code.=<<___;
 585 .data
 586 .align 32
 587 .Lones:
 588     .quad   1,1,1,1
 589 ___
 590 }
 591
 592 if ($win64) {
 593 $rec="%rcx";
 594 $frame="%rdx";
 595 $context="%r8";
 596 $disp="%r9";
 597
 598 $code.=<<___
 599 .extern     __imp_RtlVirtualUnwind
 600 .type   rsaz_def_handler,\@abi-omnipotent
 601 .align  16
 602 rsaz_def_handler:
 603     push    %rsi
 604     push    %rdi
 605     push    %rbx
 606     push    %rbp
 607     push    %r12
 608     push    %r13
 609     push    %r14
 610     push    %r15
 611     pushfq
 612     sub     \$64,%rsp
 613
 614     mov     120($context),%rax # pull context->Rax
 615     mov     248($context),%rbx # pull context->Rip
 616
 617     mov     8($disp),%rsi      # disp->ImageBase
 618     mov     56($disp),%r11     # disp->HandlerData
 619
 620     mov     0(%r11),%r10d      # HandlerData[0]
 621     lea     (%rsi,%r10),%r10   # prologue label
 622     cmp     %r10,%rbx          # context->Rip<.Lprologue
 623     jb  .Lcommon_seh_tail
 624
 625     mov     152($context),%rax # pull context->Rsp
 626
 627     mov     4(%r11),%r10d      # HandlerData[1]
 628     lea     (%rsi,%r10),%r10   # epilogue label
 629     cmp     %r10,%rbx          # context->Rip>=.Lepilogue
 630     jae     .Lcommon_seh_tail
 631
 632     lea     48(%rax),%rax
 633
 634     mov     -8(%rax),%rbx
 635     mov     -16(%rax),%rbp
 636     mov     -24(%rax),%r12
 637     mov     -32(%rax),%r13
 638     mov     -40(%rax),%r14
 639     mov     -48(%rax),%r15
 640     mov     %rbx,144($context) # restore context->Rbx
 641     mov     %rbp,160($context) # restore context->Rbp
 642     mov     %r12,216($context) # restore context->R12
 643     mov     %r13,224($context) # restore context->R13
 644     mov     %r14,232($context) # restore context->R14
 645     mov     %r15,240($context) # restore context->R14
 646
 647 .Lcommon_seh_tail:
 648     mov     8(%rax),%rdi
 649     mov     16(%rax),%rsi
 650     mov     %rax,152($context) # restore context->Rsp
 651     mov     %rsi,168($context) # restore context->Rsi
 652     mov     %rdi,176($context) # restore context->Rdi
 653
 654     mov     40($disp),%rdi     # disp->ContextRecord
 655     mov     $context,%rsi      # context
 656     mov     \$154,%ecx         # sizeof(CONTEXT)
 657     .long   0xa548f3fc         # cld; rep movsq
 658
 659     mov     $disp,%rsi
 660     xor     %rcx,%rcx          # arg1, UNW_FLAG_NHANDLER
 661     mov     8(%rsi),%rdx       # arg2, disp->ImageBase
 662     mov     0(%rsi),%r8        # arg3, disp->ControlPc
 663     mov     16(%rsi),%r9       # arg4, disp->FunctionEntry
 664     mov     40(%rsi),%r10      # disp->ContextRecord
 665     lea     56(%rsi),%r11      # &disp->HandlerData
 666     lea     24(%rsi),%r12      # &disp->EstablisherFrame
 667     mov     %r10,32(%rsp)      # arg5
 668     mov     %r11,40(%rsp)      # arg6
 669     mov     %r12,48(%rsp)      # arg7
 670     mov     %rcx,56(%rsp)      # arg8, (NULL)
 671     call    *__imp_RtlVirtualUnwind(%rip)
 672
 673     mov     \$1,%eax           # ExceptionContinueSearch
 674     add     \$64,%rsp
 675     popfq
 676     pop     %r15
 677     pop     %r14
 678     pop     %r13
 679     pop     %r12
 680     pop     %rbp
 681     pop     %rbx
 682     pop     %rdi
 683     pop     %rsi
 684     ret
 685 .size   rsaz_def_handler,.-rsaz_def_handler
 686
 687 .section    .pdata
 688 .align  4
 689     .rva    .LSEH_begin_RSAZ_amm52x20_x1_256
 690     .rva    .LSEH_end_RSAZ_amm52x20_x1_256
 691     .rva    .LSEH_info_RSAZ_amm52x20_x1_256
 692
 693     .rva    .LSEH_begin_RSAZ_amm52x20_x2_256
 694     .rva    .LSEH_end_RSAZ_amm52x20_x2_256
 695     .rva    .LSEH_info_RSAZ_amm52x20_x2_256
 696
 697     .rva    .LSEH_begin_extract_multiplier_2x20_win5
 698     .rva    .LSEH_end_extract_multiplier_2x20_win5
 699     .rva    .LSEH_info_extract_multiplier_2x20_win5
 700
 701 .section    .xdata
 702 .align  8
 703 .LSEH_info_RSAZ_amm52x20_x1_256:
 704     .byte   9,0,0,0
 705     .rva    rsaz_def_handler
 706     .rva    .Lrsaz_amm52x20_x1_256_body,.Lrsaz_amm52x20_x1_256_epilogue
 707 .LSEH_info_RSAZ_amm52x20_x2_256:
 708     .byte   9,0,0,0
 709     .rva    rsaz_def_handler
 710     .rva    .Lrsaz_amm52x20_x2_256_body,.Lrsaz_amm52x20_x2_256_epilogue
 711 .LSEH_info_extract_multiplier_2x20_win5:
 712     .byte   9,0,0,0
 713     .rva    rsaz_def_handler
 714     .rva    .LSEH_begin_extract_multiplier_2x20_win5,.LSEH_begin_extract_multiplier_2x20_win5
 715 ___
 716 }
 717 }}} else {{{                # fallback for old assembler
 718 $code.=<<___;
 719 .text
 720
 721 .globl  rsaz_avx512ifma_eligible
 722 .type   rsaz_avx512ifma_eligible,\@abi-omnipotent
 723 rsaz_avx512ifma_eligible:
 724     xor     %eax,%eax
 725     ret
 726 .size   rsaz_avx512ifma_eligible, .-rsaz_avx512ifma_eligible
 727
 728 .globl  RSAZ_amm52x20_x1_256
 729 .globl  RSAZ_amm52x20_x2_256
 730 .globl  extract_multiplier_2x20_win5
 731 .type   RSAZ_amm52x20_x1_256,\@abi-omnipotent
 732 RSAZ_amm52x20_x1_256:
 733 RSAZ_amm52x20_x2_256:
 734 extract_multiplier_2x20_win5:
 735     .byte   0x0f,0x0b    # ud2
 736     ret
 737 .size   RSAZ_amm52x20_x1_256, .-RSAZ_amm52x20_x1_256
 738 ___
 739 }}}
 740
 741 $code =~ s/\`([^\`]*)\`/eval $1/gem;
 742 print $code;
 743 close STDOUT or die "error closing STDOUT: $!";