crypto/ec/asm/ecp_nistz256-ppc64.pl

   1 #! /usr/bin/env perl
   2 # Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved.
   3 #
   4 # Licensed under the Apache License 2.0 (the "License").  You may not use
   5 # this file except in compliance with the License.  You can obtain a copy
   6 # in the file LICENSE in the source distribution or at
   7 # https://www.openssl.org/source/license.html
   8
   9 #
  10 # ====================================================================
  11 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
  12 # project. The module is, however, dual licensed under OpenSSL and
  13 # CRYPTOGAMS licenses depending on where you obtain it. For further
  14 # details see http://www.openssl.org/~appro/cryptogams/.
  15 # ====================================================================
  16 #
  17 # ECP_NISTZ256 module for PPC64.
  18 #
  19 # August 2016.
  20 #
  21 # Original ECP_NISTZ256 submission targeting x86_64 is detailed in
  22 # http://eprint.iacr.org/2013/816.
  23 #
  24 #                       with/without -DECP_NISTZ256_ASM
  25 # POWER7                +260-530%
  26 # POWER8                +220-340%
  27
  28 # $output is the last argument if it looks like a file (it has an extension)
  29 # $flavour is the first argument if it doesn't look like a file
  30 $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
  31 $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
  32
  33 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  34 ( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
  35 ( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
  36 die "can't locate ppc-xlate.pl";
  37
  38 open OUT,"| \"$^X\" $xlate $flavour \"$output\""
  39     or die "can't call $xlate: $!";
  40 *STDOUT=*OUT;
  41
  42 my $sp="r1";
  43
  44 {
  45 my ($rp,$ap,$bp,$bi,$acc0,$acc1,$acc2,$acc3,$poly1,$poly3,
  46     $acc4,$acc5,$a0,$a1,$a2,$a3,$t0,$t1,$t2,$t3) =
  47     map("r$_",(3..12,22..31));
  48
  49 my ($acc6,$acc7)=($bp,$bi);     # used in __ecp_nistz256_sqr_mont
  50
  51 $code.=<<___;
  52 .machine        "any"
  53 .text
  54 ___
  55 ########################################################################
  56 # Convert ecp_nistz256_table.c to layout expected by ecp_nistz_gather_w7
  57 #
  58 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  59 open TABLE,"<ecp_nistz256_table.c"              or
  60 open TABLE,"<${dir}../ecp_nistz256_table.c"     or
  61 die "failed to open ecp_nistz256_table.c:",$!;
  62
  63 use integer;
  64
  65 foreach(<TABLE>) {
  66         s/TOBN\(\s*(0x[0-9a-f]+),\s*(0x[0-9a-f]+)\s*\)/push @arr,hex($2),hex($1)/geo;
  67 }
  68 close TABLE;
  69
  70 # See ecp_nistz256_table.c for explanation for why it's 64*16*37.
  71 # 64*16*37-1 is because $#arr returns last valid index or @arr, not
  72 # amount of elements.
  73 die "insane number of elements" if ($#arr != 64*16*37-1);
  74
  75 $code.=<<___;
  76 .type   ecp_nistz256_precomputed,\@object
  77 .globl  ecp_nistz256_precomputed
  78 .align  12
  79 ecp_nistz256_precomputed:
  80 ___
  81 ########################################################################
  82 # this conversion smashes P256_POINT_AFFINE by individual bytes with
  83 # 64 byte interval, similar to
  84 #       1111222233334444
  85 #       1234123412341234
  86 for(1..37) {
  87         @tbl = splice(@arr,0,64*16);
  88         for($i=0;$i<64;$i++) {
  89                 undef @line;
  90                 for($j=0;$j<64;$j++) {
  91                         push @line,(@tbl[$j*16+$i/4]>>(($i%4)*8))&0xff;
  92                 }
  93                 $code.=".byte\t";
  94                 $code.=join(',',map { sprintf "0x%02x",$_} @line);
  95                 $code.="\n";
  96         }
  97 }
  98
  99 $code.=<<___;
 100 .size   ecp_nistz256_precomputed,.-ecp_nistz256_precomputed
 101 .asciz  "ECP_NISTZ256 for PPC64, CRYPTOGAMS by <appro\@openssl.org>"
 102
 103 # void  ecp_nistz256_mul_mont(BN_ULONG x0[4],const BN_ULONG x1[4],
 104 #                                            const BN_ULONG x2[4]);
 105 .globl  ecp_nistz256_mul_mont
 106 .align  5
 107 ecp_nistz256_mul_mont:
 108         stdu    $sp,-128($sp)
 109         mflr    r0
 110         std     r22,48($sp)
 111         std     r23,56($sp)
 112         std     r24,64($sp)
 113         std     r25,72($sp)
 114         std     r26,80($sp)
 115         std     r27,88($sp)
 116         std     r28,96($sp)
 117         std     r29,104($sp)
 118         std     r30,112($sp)
 119         std     r31,120($sp)
 120
 121         ld      $a0,0($ap)
 122         ld      $bi,0($bp)
 123         ld      $a1,8($ap)
 124         ld      $a2,16($ap)
 125         ld      $a3,24($ap)
 126
 127         li      $poly1,-1
 128         srdi    $poly1,$poly1,32        # 0x00000000ffffffff
 129         li      $poly3,1
 130         orc     $poly3,$poly3,$poly1    # 0xffffffff00000001
 131
 132         bl      __ecp_nistz256_mul_mont
 133
 134         mtlr    r0
 135         ld      r22,48($sp)
 136         ld      r23,56($sp)
 137         ld      r24,64($sp)
 138         ld      r25,72($sp)
 139         ld      r26,80($sp)
 140         ld      r27,88($sp)
 141         ld      r28,96($sp)
 142         ld      r29,104($sp)
 143         ld      r30,112($sp)
 144         ld      r31,120($sp)
 145         addi    $sp,$sp,128
 146         blr
 147         .long   0
 148         .byte   0,12,4,0,0x80,10,3,0
 149         .long   0
 150 .size   ecp_nistz256_mul_mont,.-ecp_nistz256_mul_mont
 151
 152 # void  ecp_nistz256_sqr_mont(BN_ULONG x0[4],const BN_ULONG x1[4]);
 153 .globl  ecp_nistz256_sqr_mont
 154 .align  4
 155 ecp_nistz256_sqr_mont:
 156         stdu    $sp,-128($sp)
 157         mflr    r0
 158         std     r22,48($sp)
 159         std     r23,56($sp)
 160         std     r24,64($sp)
 161         std     r25,72($sp)
 162         std     r26,80($sp)
 163         std     r27,88($sp)
 164         std     r28,96($sp)
 165         std     r29,104($sp)
 166         std     r30,112($sp)
 167         std     r31,120($sp)
 168
 169         ld      $a0,0($ap)
 170         ld      $a1,8($ap)
 171         ld      $a2,16($ap)
 172         ld      $a3,24($ap)
 173
 174         li      $poly1,-1
 175         srdi    $poly1,$poly1,32        # 0x00000000ffffffff
 176         li      $poly3,1
 177         orc     $poly3,$poly3,$poly1    # 0xffffffff00000001
 178
 179         bl      __ecp_nistz256_sqr_mont
 180
 181         mtlr    r0
 182         ld      r22,48($sp)
 183         ld      r23,56($sp)
 184         ld      r24,64($sp)
 185         ld      r25,72($sp)
 186         ld      r26,80($sp)
 187         ld      r27,88($sp)
 188         ld      r28,96($sp)
 189         ld      r29,104($sp)
 190         ld      r30,112($sp)
 191         ld      r31,120($sp)
 192         addi    $sp,$sp,128
 193         blr
 194         .long   0
 195         .byte   0,12,4,0,0x80,10,2,0
 196         .long   0
 197 .size   ecp_nistz256_sqr_mont,.-ecp_nistz256_sqr_mont
 198
 199 # void  ecp_nistz256_add(BN_ULONG x0[4],const BN_ULONG x1[4],
 200 #                                       const BN_ULONG x2[4]);
 201 .globl  ecp_nistz256_add
 202 .align  4
 203 ecp_nistz256_add:
 204         stdu    $sp,-128($sp)
 205         mflr    r0
 206         std     r28,96($sp)
 207         std     r29,104($sp)
 208         std     r30,112($sp)
 209         std     r31,120($sp)
 210
 211         ld      $acc0,0($ap)
 212         ld      $t0,  0($bp)
 213         ld      $acc1,8($ap)
 214         ld      $t1,  8($bp)
 215         ld      $acc2,16($ap)
 216         ld      $t2,  16($bp)
 217         ld      $acc3,24($ap)
 218         ld      $t3,  24($bp)
 219
 220         li      $poly1,-1
 221         srdi    $poly1,$poly1,32        # 0x00000000ffffffff
 222         li      $poly3,1
 223         orc     $poly3,$poly3,$poly1    # 0xffffffff00000001
 224
 225         bl      __ecp_nistz256_add
 226
 227         mtlr    r0
 228         ld      r28,96($sp)
 229         ld      r29,104($sp)
 230         ld      r30,112($sp)
 231         ld      r31,120($sp)
 232         addi    $sp,$sp,128
 233         blr
 234         .long   0
 235         .byte   0,12,4,0,0x80,4,3,0
 236         .long   0
 237 .size   ecp_nistz256_add,.-ecp_nistz256_add
 238
 239 # void  ecp_nistz256_div_by_2(BN_ULONG x0[4],const BN_ULONG x1[4]);
 240 .globl  ecp_nistz256_div_by_2
 241 .align  4
 242 ecp_nistz256_div_by_2:
 243         stdu    $sp,-128($sp)
 244         mflr    r0
 245         std     r28,96($sp)
 246         std     r29,104($sp)
 247         std     r30,112($sp)
 248         std     r31,120($sp)
 249
 250         ld      $acc0,0($ap)
 251         ld      $acc1,8($ap)
 252         ld      $acc2,16($ap)
 253         ld      $acc3,24($ap)
 254
 255         li      $poly1,-1
 256         srdi    $poly1,$poly1,32        # 0x00000000ffffffff
 257         li      $poly3,1
 258         orc     $poly3,$poly3,$poly1    # 0xffffffff00000001
 259
 260         bl      __ecp_nistz256_div_by_2
 261
 262         mtlr    r0
 263         ld      r28,96($sp)
 264         ld      r29,104($sp)
 265         ld      r30,112($sp)
 266         ld      r31,120($sp)
 267         addi    $sp,$sp,128
 268         blr
 269         .long   0
 270         .byte   0,12,4,0,0x80,4,2,0
 271         .long   0
 272 .size   ecp_nistz256_div_by_2,.-ecp_nistz256_div_by_2
 273
 274 # void  ecp_nistz256_mul_by_2(BN_ULONG x0[4],const BN_ULONG x1[4]);
 275 .globl  ecp_nistz256_mul_by_2
 276 .align  4
 277 ecp_nistz256_mul_by_2:
 278         stdu    $sp,-128($sp)
 279         mflr    r0
 280         std     r28,96($sp)
 281         std     r29,104($sp)
 282         std     r30,112($sp)
 283         std     r31,120($sp)
 284
 285         ld      $acc0,0($ap)
 286         ld      $acc1,8($ap)
 287         ld      $acc2,16($ap)
 288         ld      $acc3,24($ap)
 289
 290         mr      $t0,$acc0
 291         mr      $t1,$acc1
 292         mr      $t2,$acc2
 293         mr      $t3,$acc3
 294
 295         li      $poly1,-1
 296         srdi    $poly1,$poly1,32        # 0x00000000ffffffff
 297         li      $poly3,1
 298         orc     $poly3,$poly3,$poly1    # 0xffffffff00000001
 299
 300         bl      __ecp_nistz256_add      # ret = a+a     // 2*a
 301
 302         mtlr    r0
 303         ld      r28,96($sp)
 304         ld      r29,104($sp)
 305         ld      r30,112($sp)
 306         ld      r31,120($sp)
 307         addi    $sp,$sp,128
 308         blr
 309         .long   0
 310         .byte   0,12,4,0,0x80,4,3,0
 311         .long   0
 312 .size   ecp_nistz256_mul_by_2,.-ecp_nistz256_mul_by_2
 313
 314 # void  ecp_nistz256_mul_by_3(BN_ULONG x0[4],const BN_ULONG x1[4]);
 315 .globl  ecp_nistz256_mul_by_3
 316 .align  4
 317 ecp_nistz256_mul_by_3:
 318         stdu    $sp,-128($sp)
 319         mflr    r0
 320         std     r28,96($sp)
 321         std     r29,104($sp)
 322         std     r30,112($sp)
 323         std     r31,120($sp)
 324
 325         ld      $acc0,0($ap)
 326         ld      $acc1,8($ap)
 327         ld      $acc2,16($ap)
 328         ld      $acc3,24($ap)
 329
 330         mr      $t0,$acc0
 331         std     $acc0,64($sp)
 332         mr      $t1,$acc1
 333         std     $acc1,72($sp)
 334         mr      $t2,$acc2
 335         std     $acc2,80($sp)
 336         mr      $t3,$acc3
 337         std     $acc3,88($sp)
 338
 339         li      $poly1,-1
 340         srdi    $poly1,$poly1,32        # 0x00000000ffffffff
 341         li      $poly3,1
 342         orc     $poly3,$poly3,$poly1    # 0xffffffff00000001
 343
 344         bl      __ecp_nistz256_add      # ret = a+a     // 2*a
 345
 346         ld      $t0,64($sp)
 347         ld      $t1,72($sp)
 348         ld      $t2,80($sp)
 349         ld      $t3,88($sp)
 350
 351         bl      __ecp_nistz256_add      # ret += a      // 2*a+a=3*a
 352
 353         mtlr    r0
 354         ld      r28,96($sp)
 355         ld      r29,104($sp)
 356         ld      r30,112($sp)
 357         ld      r31,120($sp)
 358         addi    $sp,$sp,128
 359         blr
 360         .long   0
 361         .byte   0,12,4,0,0x80,4,2,0
 362         .long   0
 363 .size   ecp_nistz256_mul_by_3,.-ecp_nistz256_mul_by_3
 364
 365 # void  ecp_nistz256_sub(BN_ULONG x0[4],const BN_ULONG x1[4],
 366 #                                       const BN_ULONG x2[4]);
 367 .globl  ecp_nistz256_sub
 368 .align  4
 369 ecp_nistz256_sub:
 370         stdu    $sp,-128($sp)
 371         mflr    r0
 372         std     r28,96($sp)
 373         std     r29,104($sp)
 374         std     r30,112($sp)
 375         std     r31,120($sp)
 376
 377         ld      $acc0,0($ap)
 378         ld      $acc1,8($ap)
 379         ld      $acc2,16($ap)
 380         ld      $acc3,24($ap)
 381
 382         li      $poly1,-1
 383         srdi    $poly1,$poly1,32        # 0x00000000ffffffff
 384         li      $poly3,1
 385         orc     $poly3,$poly3,$poly1    # 0xffffffff00000001
 386
 387         bl      __ecp_nistz256_sub_from
 388
 389         mtlr    r0
 390         ld      r28,96($sp)
 391         ld      r29,104($sp)
 392         ld      r30,112($sp)
 393         ld      r31,120($sp)
 394         addi    $sp,$sp,128
 395         blr
 396         .long   0
 397         .byte   0,12,4,0,0x80,4,3,0
 398         .long   0
 399 .size   ecp_nistz256_sub,.-ecp_nistz256_sub
 400
 401 # void  ecp_nistz256_neg(BN_ULONG x0[4],const BN_ULONG x1[4]);
 402 .globl  ecp_nistz256_neg
 403 .align  4
 404 ecp_nistz256_neg:
 405         stdu    $sp,-128($sp)
 406         mflr    r0
 407         std     r28,96($sp)
 408         std     r29,104($sp)
 409         std     r30,112($sp)
 410         std     r31,120($sp)
 411
 412         mr      $bp,$ap
 413         li      $acc0,0
 414         li      $acc1,0
 415         li      $acc2,0
 416         li      $acc3,0
 417
 418         li      $poly1,-1
 419         srdi    $poly1,$poly1,32        # 0x00000000ffffffff
 420         li      $poly3,1
 421         orc     $poly3,$poly3,$poly1    # 0xffffffff00000001
 422
 423         bl      __ecp_nistz256_sub_from
 424
 425         mtlr    r0
 426         ld      r28,96($sp)
 427         ld      r29,104($sp)
 428         ld      r30,112($sp)
 429         ld      r31,120($sp)
 430         addi    $sp,$sp,128
 431         blr
 432         .long   0
 433         .byte   0,12,4,0,0x80,4,2,0
 434         .long   0
 435 .size   ecp_nistz256_neg,.-ecp_nistz256_neg
 436
 437 # note that __ecp_nistz256_mul_mont expects a[0-3] input pre-loaded
 438 # to $a0-$a3 and b[0] - to $bi
 439 .type   __ecp_nistz256_mul_mont,\@function
 440 .align  4
 441 __ecp_nistz256_mul_mont:
 442         mulld   $acc0,$a0,$bi           # a[0]*b[0]
 443         mulhdu  $t0,$a0,$bi
 444
 445         mulld   $acc1,$a1,$bi           # a[1]*b[0]
 446         mulhdu  $t1,$a1,$bi
 447
 448         mulld   $acc2,$a2,$bi           # a[2]*b[0]
 449         mulhdu  $t2,$a2,$bi
 450
 451         mulld   $acc3,$a3,$bi           # a[3]*b[0]
 452         mulhdu  $t3,$a3,$bi
 453         ld      $bi,8($bp)              # b[1]
 454
 455         addc    $acc1,$acc1,$t0         # accumulate high parts of multiplication
 456          sldi   $t0,$acc0,32
 457         adde    $acc2,$acc2,$t1
 458          srdi   $t1,$acc0,32
 459         adde    $acc3,$acc3,$t2
 460         addze   $acc4,$t3
 461         li      $acc5,0
 462 ___
 463 for($i=1;$i<4;$i++) {
 464         ################################################################
 465         # Reduction iteration is normally performed by accumulating
 466         # result of multiplication of modulus by "magic" digit [and
 467         # omitting least significant word, which is guaranteed to
 468         # be 0], but thanks to special form of modulus and "magic"
 469         # digit being equal to least significant word, it can be
 470         # performed with additions and subtractions alone. Indeed:
 471         #
 472         #            ffff0001.00000000.0000ffff.ffffffff
 473         # *                                     abcdefgh
 474         # + xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx.abcdefgh
 475         #
 476         # Now observing that ff..ff*x = (2^n-1)*x = 2^n*x-x, we
 477         # rewrite above as:
 478         #
 479         #   xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx.abcdefgh
 480         # + abcdefgh.abcdefgh.0000abcd.efgh0000.00000000
 481         # - 0000abcd.efgh0000.00000000.00000000.abcdefgh
 482         #
 483         # or marking redundant operations:
 484         #
 485         #   xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx.--------
 486         # + abcdefgh.abcdefgh.0000abcd.efgh0000.--------
 487         # - 0000abcd.efgh0000.--------.--------.--------
 488
 489 $code.=<<___;
 490         subfc   $t2,$t0,$acc0           # "*0xffff0001"
 491         subfe   $t3,$t1,$acc0
 492         addc    $acc0,$acc1,$t0         # +=acc[0]<<96 and omit acc[0]
 493         adde    $acc1,$acc2,$t1
 494         adde    $acc2,$acc3,$t2         # +=acc[0]*0xffff0001
 495         adde    $acc3,$acc4,$t3
 496         addze   $acc4,$acc5
 497
 498         mulld   $t0,$a0,$bi             # lo(a[0]*b[i])
 499         mulld   $t1,$a1,$bi             # lo(a[1]*b[i])
 500         mulld   $t2,$a2,$bi             # lo(a[2]*b[i])
 501         mulld   $t3,$a3,$bi             # lo(a[3]*b[i])
 502         addc    $acc0,$acc0,$t0         # accumulate low parts of multiplication
 503          mulhdu $t0,$a0,$bi             # hi(a[0]*b[i])
 504         adde    $acc1,$acc1,$t1
 505          mulhdu $t1,$a1,$bi             # hi(a[1]*b[i])
 506         adde    $acc2,$acc2,$t2
 507          mulhdu $t2,$a2,$bi             # hi(a[2]*b[i])
 508         adde    $acc3,$acc3,$t3
 509          mulhdu $t3,$a3,$bi             # hi(a[3]*b[i])
 510         addze   $acc4,$acc4
 511 ___
 512 $code.=<<___    if ($i<3);
 513         ld      $bi,8*($i+1)($bp)       # b[$i+1]
 514 ___
 515 $code.=<<___;
 516         addc    $acc1,$acc1,$t0         # accumulate high parts of multiplication
 517          sldi   $t0,$acc0,32
 518         adde    $acc2,$acc2,$t1
 519          srdi   $t1,$acc0,32
 520         adde    $acc3,$acc3,$t2
 521         adde    $acc4,$acc4,$t3
 522         li      $acc5,0
 523         addze   $acc5,$acc5
 524 ___
 525 }
 526 $code.=<<___;
 527         # last reduction
 528         subfc   $t2,$t0,$acc0           # "*0xffff0001"
 529         subfe   $t3,$t1,$acc0
 530         addc    $acc0,$acc1,$t0         # +=acc[0]<<96 and omit acc[0]
 531         adde    $acc1,$acc2,$t1
 532         adde    $acc2,$acc3,$t2         # +=acc[0]*0xffff0001
 533         adde    $acc3,$acc4,$t3
 534         addze   $acc4,$acc5
 535
 536         li      $t2,0
 537         addic   $acc0,$acc0,1           # ret -= modulus
 538         subfe   $acc1,$poly1,$acc1
 539         subfe   $acc2,$t2,$acc2
 540         subfe   $acc3,$poly3,$acc3
 541         subfe   $acc4,$t2,$acc4
 542
 543         addc    $acc0,$acc0,$acc4       # ret += modulus if borrow
 544         and     $t1,$poly1,$acc4
 545         and     $t3,$poly3,$acc4
 546         adde    $acc1,$acc1,$t1
 547         addze   $acc2,$acc2
 548         adde    $acc3,$acc3,$t3
 549
 550         std     $acc0,0($rp)
 551         std     $acc1,8($rp)
 552         std     $acc2,16($rp)
 553         std     $acc3,24($rp)
 554
 555         blr
 556         .long   0
 557         .byte   0,12,0x14,0,0,0,1,0
 558         .long   0
 559 .size   __ecp_nistz256_mul_mont,.-__ecp_nistz256_mul_mont
 560
 561 # note that __ecp_nistz256_sqr_mont expects a[0-3] input pre-loaded
 562 # to $a0-$a3
 563 .type   __ecp_nistz256_sqr_mont,\@function
 564 .align  4
 565 __ecp_nistz256_sqr_mont:
 566         ################################################################
 567         #  |  |  |  |  |  |a1*a0|  |
 568         #  |  |  |  |  |a2*a0|  |  |
 569         #  |  |a3*a2|a3*a0|  |  |  |
 570         #  |  |  |  |a2*a1|  |  |  |
 571         #  |  |  |a3*a1|  |  |  |  |
 572         # *|  |  |  |  |  |  |  | 2|
 573         # +|a3*a3|a2*a2|a1*a1|a0*a0|
 574         #  |--+--+--+--+--+--+--+--|
 575         #  |A7|A6|A5|A4|A3|A2|A1|A0|, where Ax is $accx, i.e. follow $accx
 576         #
 577         #  "can't overflow" below mark carrying into high part of
 578         #  multiplication result, which can't overflow, because it
 579         #  can never be all ones.
 580
 581         mulld   $acc1,$a1,$a0           # a[1]*a[0]
 582         mulhdu  $t1,$a1,$a0
 583         mulld   $acc2,$a2,$a0           # a[2]*a[0]
 584         mulhdu  $t2,$a2,$a0
 585         mulld   $acc3,$a3,$a0           # a[3]*a[0]
 586         mulhdu  $acc4,$a3,$a0
 587
 588         addc    $acc2,$acc2,$t1         # accumulate high parts of multiplication
 589          mulld  $t0,$a2,$a1             # a[2]*a[1]
 590          mulhdu $t1,$a2,$a1
 591         adde    $acc3,$acc3,$t2
 592          mulld  $t2,$a3,$a1             # a[3]*a[1]
 593          mulhdu $t3,$a3,$a1
 594         addze   $acc4,$acc4             # can't overflow
 595
 596         mulld   $acc5,$a3,$a2           # a[3]*a[2]
 597         mulhdu  $acc6,$a3,$a2
 598
 599         addc    $t1,$t1,$t2             # accumulate high parts of multiplication
 600         addze   $t2,$t3                 # can't overflow
 601
 602         addc    $acc3,$acc3,$t0         # accumulate low parts of multiplication
 603         adde    $acc4,$acc4,$t1
 604         adde    $acc5,$acc5,$t2
 605         addze   $acc6,$acc6             # can't overflow
 606
 607         addc    $acc1,$acc1,$acc1       # acc[1-6]*=2
 608         adde    $acc2,$acc2,$acc2
 609         adde    $acc3,$acc3,$acc3
 610         adde    $acc4,$acc4,$acc4
 611         adde    $acc5,$acc5,$acc5
 612         adde    $acc6,$acc6,$acc6
 613         li      $acc7,0
 614         addze   $acc7,$acc7
 615
 616         mulld   $acc0,$a0,$a0           # a[0]*a[0]
 617         mulhdu  $a0,$a0,$a0
 618         mulld   $t1,$a1,$a1             # a[1]*a[1]
 619         mulhdu  $a1,$a1,$a1
 620         mulld   $t2,$a2,$a2             # a[2]*a[2]
 621         mulhdu  $a2,$a2,$a2
 622         mulld   $t3,$a3,$a3             # a[3]*a[3]
 623         mulhdu  $a3,$a3,$a3
 624         addc    $acc1,$acc1,$a0         # +a[i]*a[i]
 625          sldi   $t0,$acc0,32
 626         adde    $acc2,$acc2,$t1
 627          srdi   $t1,$acc0,32
 628         adde    $acc3,$acc3,$a1
 629         adde    $acc4,$acc4,$t2
 630         adde    $acc5,$acc5,$a2
 631         adde    $acc6,$acc6,$t3
 632         adde    $acc7,$acc7,$a3
 633 ___
 634 for($i=0;$i<3;$i++) {                   # reductions, see commentary in
 635                                         # multiplication for details
 636 $code.=<<___;
 637         subfc   $t2,$t0,$acc0           # "*0xffff0001"
 638         subfe   $t3,$t1,$acc0
 639         addc    $acc0,$acc1,$t0         # +=acc[0]<<96 and omit acc[0]
 640          sldi   $t0,$acc0,32
 641         adde    $acc1,$acc2,$t1
 642          srdi   $t1,$acc0,32
 643         adde    $acc2,$acc3,$t2         # +=acc[0]*0xffff0001
 644         addze   $acc3,$t3               # can't overflow
 645 ___
 646 }
 647 $code.=<<___;
 648         subfc   $t2,$t0,$acc0           # "*0xffff0001"
 649         subfe   $t3,$t1,$acc0
 650         addc    $acc0,$acc1,$t0         # +=acc[0]<<96 and omit acc[0]
 651         adde    $acc1,$acc2,$t1
 652         adde    $acc2,$acc3,$t2         # +=acc[0]*0xffff0001
 653         addze   $acc3,$t3               # can't overflow
 654
 655         addc    $acc0,$acc0,$acc4       # accumulate upper half
 656         adde    $acc1,$acc1,$acc5
 657         adde    $acc2,$acc2,$acc6
 658         adde    $acc3,$acc3,$acc7
 659         li      $t2,0
 660         addze   $acc4,$t2
 661
 662         addic   $acc0,$acc0,1           # ret -= modulus
 663         subfe   $acc1,$poly1,$acc1
 664         subfe   $acc2,$t2,$acc2
 665         subfe   $acc3,$poly3,$acc3
 666         subfe   $acc4,$t2,$acc4
 667
 668         addc    $acc0,$acc0,$acc4       # ret += modulus if borrow
 669         and     $t1,$poly1,$acc4
 670         and     $t3,$poly3,$acc4
 671         adde    $acc1,$acc1,$t1
 672         addze   $acc2,$acc2
 673         adde    $acc3,$acc3,$t3
 674
 675         std     $acc0,0($rp)
 676         std     $acc1,8($rp)
 677         std     $acc2,16($rp)
 678         std     $acc3,24($rp)
 679
 680         blr
 681         .long   0
 682         .byte   0,12,0x14,0,0,0,1,0
 683         .long   0
 684 .size   __ecp_nistz256_sqr_mont,.-__ecp_nistz256_sqr_mont
 685
 686 # Note that __ecp_nistz256_add expects both input vectors pre-loaded to
 687 # $a0-$a3 and $t0-$t3. This is done because it's used in multiple
 688 # contexts, e.g. in multiplication by 2 and 3...
 689 .type   __ecp_nistz256_add,\@function
 690 .align  4
 691 __ecp_nistz256_add:
 692         addc    $acc0,$acc0,$t0         # ret = a+b
 693         adde    $acc1,$acc1,$t1
 694         adde    $acc2,$acc2,$t2
 695         li      $t2,0
 696         adde    $acc3,$acc3,$t3
 697         addze   $t0,$t2
 698
 699         # if a+b >= modulus, subtract modulus
 700         #
 701         # But since comparison implies subtraction, we subtract
 702         # modulus and then add it back if subtraction borrowed.
 703
 704         subic   $acc0,$acc0,-1
 705         subfe   $acc1,$poly1,$acc1
 706         subfe   $acc2,$t2,$acc2
 707         subfe   $acc3,$poly3,$acc3
 708         subfe   $t0,$t2,$t0
 709
 710         addc    $acc0,$acc0,$t0
 711         and     $t1,$poly1,$t0
 712         and     $t3,$poly3,$t0
 713         adde    $acc1,$acc1,$t1
 714         addze   $acc2,$acc2
 715         adde    $acc3,$acc3,$t3
 716
 717         std     $acc0,0($rp)
 718         std     $acc1,8($rp)
 719         std     $acc2,16($rp)
 720         std     $acc3,24($rp)
 721
 722         blr
 723         .long   0
 724         .byte   0,12,0x14,0,0,0,3,0
 725         .long   0
 726 .size   __ecp_nistz256_add,.-__ecp_nistz256_add
 727
 728 .type   __ecp_nistz256_sub_from,\@function
 729 .align  4
 730 __ecp_nistz256_sub_from:
 731         ld      $t0,0($bp)
 732         ld      $t1,8($bp)
 733         ld      $t2,16($bp)
 734         ld      $t3,24($bp)
 735         subfc   $acc0,$t0,$acc0         # ret = a-b
 736         subfe   $acc1,$t1,$acc1
 737         subfe   $acc2,$t2,$acc2
 738         subfe   $acc3,$t3,$acc3
 739         subfe   $t0,$t0,$t0             # t0 = borrow ? -1 : 0
 740
 741         # if a-b borrowed, add modulus
 742
 743         addc    $acc0,$acc0,$t0         # ret -= modulus & t0
 744         and     $t1,$poly1,$t0
 745         and     $t3,$poly3,$t0
 746         adde    $acc1,$acc1,$t1
 747         addze   $acc2,$acc2
 748         adde    $acc3,$acc3,$t3
 749
 750         std     $acc0,0($rp)
 751         std     $acc1,8($rp)
 752         std     $acc2,16($rp)
 753         std     $acc3,24($rp)
 754
 755         blr
 756         .long   0
 757         .byte   0,12,0x14,0,0,0,3,0
 758         .long   0
 759 .size   __ecp_nistz256_sub_from,.-__ecp_nistz256_sub_from
 760
 761 .type   __ecp_nistz256_sub_morf,\@function
 762 .align  4
 763 __ecp_nistz256_sub_morf:
 764         ld      $t0,0($bp)
 765         ld      $t1,8($bp)
 766         ld      $t2,16($bp)
 767         ld      $t3,24($bp)
 768         subfc   $acc0,$acc0,$t0         # ret = b-a
 769         subfe   $acc1,$acc1,$t1
 770         subfe   $acc2,$acc2,$t2
 771         subfe   $acc3,$acc3,$t3
 772         subfe   $t0,$t0,$t0             # t0 = borrow ? -1 : 0
 773
 774         # if b-a borrowed, add modulus
 775
 776         addc    $acc0,$acc0,$t0         # ret -= modulus & t0
 777         and     $t1,$poly1,$t0
 778         and     $t3,$poly3,$t0
 779         adde    $acc1,$acc1,$t1
 780         addze   $acc2,$acc2
 781         adde    $acc3,$acc3,$t3
 782
 783         std     $acc0,0($rp)
 784         std     $acc1,8($rp)
 785         std     $acc2,16($rp)
 786         std     $acc3,24($rp)
 787
 788         blr
 789         .long   0
 790         .byte   0,12,0x14,0,0,0,3,0
 791         .long   0
 792 .size   __ecp_nistz256_sub_morf,.-__ecp_nistz256_sub_morf
 793
 794 .type   __ecp_nistz256_div_by_2,\@function
 795 .align  4
 796 __ecp_nistz256_div_by_2:
 797         andi.   $t0,$acc0,1
 798         addic   $acc0,$acc0,-1          # a += modulus
 799          neg    $t0,$t0
 800         adde    $acc1,$acc1,$poly1
 801          not    $t0,$t0
 802         addze   $acc2,$acc2
 803          li     $t2,0
 804         adde    $acc3,$acc3,$poly3
 805          and    $t1,$poly1,$t0
 806         addze   $ap,$t2                 # ap = carry
 807          and    $t3,$poly3,$t0
 808
 809         subfc   $acc0,$t0,$acc0         # a -= modulus if a was even
 810         subfe   $acc1,$t1,$acc1
 811         subfe   $acc2,$t2,$acc2
 812         subfe   $acc3,$t3,$acc3
 813         subfe   $ap,  $t2,$ap
 814
 815         srdi    $acc0,$acc0,1
 816         sldi    $t0,$acc1,63
 817         srdi    $acc1,$acc1,1
 818         sldi    $t1,$acc2,63
 819         srdi    $acc2,$acc2,1
 820         sldi    $t2,$acc3,63
 821         srdi    $acc3,$acc3,1
 822         sldi    $t3,$ap,63
 823         or      $acc0,$acc0,$t0
 824         or      $acc1,$acc1,$t1
 825         or      $acc2,$acc2,$t2
 826         or      $acc3,$acc3,$t3
 827
 828         std     $acc0,0($rp)
 829         std     $acc1,8($rp)
 830         std     $acc2,16($rp)
 831         std     $acc3,24($rp)
 832
 833         blr
 834         .long   0
 835         .byte   0,12,0x14,0,0,0,1,0
 836         .long   0
 837 .size   __ecp_nistz256_div_by_2,.-__ecp_nistz256_div_by_2
 838 ___
 839 ########################################################################
 840 # following subroutines are "literal" implementation of those found in
 841 # ecp_nistz256.c
 842 #
 843 ########################################################################
 844 # void ecp_nistz256_point_double(P256_POINT *out,const P256_POINT *inp);
 845 #
 846 if (1) {
 847 my $FRAME=64+32*4+12*8;
 848 my ($S,$M,$Zsqr,$tmp0)=map(64+32*$_,(0..3));
 849 # above map() describes stack layout with 4 temporary
 850 # 256-bit vectors on top.
 851 my ($rp_real,$ap_real) = map("r$_",(20,21));
 852
 853 $code.=<<___;
 854 .globl  ecp_nistz256_point_double
 855 .align  5
 856 ecp_nistz256_point_double:
 857         stdu    $sp,-$FRAME($sp)
 858         mflr    r0
 859         std     r20,$FRAME-8*12($sp)
 860         std     r21,$FRAME-8*11($sp)
 861         std     r22,$FRAME-8*10($sp)
 862         std     r23,$FRAME-8*9($sp)
 863         std     r24,$FRAME-8*8($sp)
 864         std     r25,$FRAME-8*7($sp)
 865         std     r26,$FRAME-8*6($sp)
 866         std     r27,$FRAME-8*5($sp)
 867         std     r28,$FRAME-8*4($sp)
 868         std     r29,$FRAME-8*3($sp)
 869         std     r30,$FRAME-8*2($sp)
 870         std     r31,$FRAME-8*1($sp)
 871
 872         li      $poly1,-1
 873         srdi    $poly1,$poly1,32        # 0x00000000ffffffff
 874         li      $poly3,1
 875         orc     $poly3,$poly3,$poly1    # 0xffffffff00000001
 876 .Ldouble_shortcut:
 877         ld      $acc0,32($ap)
 878         ld      $acc1,40($ap)
 879         ld      $acc2,48($ap)
 880         ld      $acc3,56($ap)
 881         mr      $t0,$acc0
 882         mr      $t1,$acc1
 883         mr      $t2,$acc2
 884         mr      $t3,$acc3
 885          ld     $a0,64($ap)             # forward load for p256_sqr_mont
 886          ld     $a1,72($ap)
 887          ld     $a2,80($ap)
 888          ld     $a3,88($ap)
 889          mr     $rp_real,$rp
 890          mr     $ap_real,$ap
 891         addi    $rp,$sp,$S
 892         bl      __ecp_nistz256_add      # p256_mul_by_2(S, in_y);
 893
 894         addi    $rp,$sp,$Zsqr
 895         bl      __ecp_nistz256_sqr_mont # p256_sqr_mont(Zsqr, in_z);
 896
 897         ld      $t0,0($ap_real)
 898         ld      $t1,8($ap_real)
 899         ld      $t2,16($ap_real)
 900         ld      $t3,24($ap_real)
 901         mr      $a0,$acc0               # put Zsqr aside for p256_sub
 902         mr      $a1,$acc1
 903         mr      $a2,$acc2
 904         mr      $a3,$acc3
 905         addi    $rp,$sp,$M
 906         bl      __ecp_nistz256_add      # p256_add(M, Zsqr, in_x);
 907
 908         addi    $bp,$ap_real,0
 909         mr      $acc0,$a0               # restore Zsqr
 910         mr      $acc1,$a1
 911         mr      $acc2,$a2
 912         mr      $acc3,$a3
 913          ld     $a0,$S+0($sp)           # forward load for p256_sqr_mont
 914          ld     $a1,$S+8($sp)
 915          ld     $a2,$S+16($sp)
 916          ld     $a3,$S+24($sp)
 917         addi    $rp,$sp,$Zsqr
 918         bl      __ecp_nistz256_sub_morf # p256_sub(Zsqr, in_x, Zsqr);
 919
 920         addi    $rp,$sp,$S
 921         bl      __ecp_nistz256_sqr_mont # p256_sqr_mont(S, S);
 922
 923         ld      $bi,32($ap_real)
 924         ld      $a0,64($ap_real)
 925         ld      $a1,72($ap_real)
 926         ld      $a2,80($ap_real)
 927         ld      $a3,88($ap_real)
 928         addi    $bp,$ap_real,32
 929         addi    $rp,$sp,$tmp0
 930         bl      __ecp_nistz256_mul_mont # p256_mul_mont(tmp0, in_z, in_y);
 931
 932         mr      $t0,$acc0
 933         mr      $t1,$acc1
 934         mr      $t2,$acc2
 935         mr      $t3,$acc3
 936          ld     $a0,$S+0($sp)           # forward load for p256_sqr_mont
 937          ld     $a1,$S+8($sp)
 938          ld     $a2,$S+16($sp)
 939          ld     $a3,$S+24($sp)
 940         addi    $rp,$rp_real,64
 941         bl      __ecp_nistz256_add      # p256_mul_by_2(res_z, tmp0);
 942
 943         addi    $rp,$sp,$tmp0
 944         bl      __ecp_nistz256_sqr_mont # p256_sqr_mont(tmp0, S);
 945
 946          ld     $bi,$Zsqr($sp)          # forward load for p256_mul_mont
 947          ld     $a0,$M+0($sp)
 948          ld     $a1,$M+8($sp)
 949          ld     $a2,$M+16($sp)
 950          ld     $a3,$M+24($sp)
 951         addi    $rp,$rp_real,32
 952         bl      __ecp_nistz256_div_by_2 # p256_div_by_2(res_y, tmp0);
 953
 954         addi    $bp,$sp,$Zsqr
 955         addi    $rp,$sp,$M
 956         bl      __ecp_nistz256_mul_mont # p256_mul_mont(M, M, Zsqr);
 957
 958         mr      $t0,$acc0               # duplicate M
 959         mr      $t1,$acc1
 960         mr      $t2,$acc2
 961         mr      $t3,$acc3
 962         mr      $a0,$acc0               # put M aside
 963         mr      $a1,$acc1
 964         mr      $a2,$acc2
 965         mr      $a3,$acc3
 966         addi    $rp,$sp,$M
 967         bl      __ecp_nistz256_add
 968         mr      $t0,$a0                 # restore M
 969         mr      $t1,$a1
 970         mr      $t2,$a2
 971         mr      $t3,$a3
 972          ld     $bi,0($ap_real)         # forward load for p256_mul_mont
 973          ld     $a0,$S+0($sp)
 974          ld     $a1,$S+8($sp)
 975          ld     $a2,$S+16($sp)
 976          ld     $a3,$S+24($sp)
 977         bl      __ecp_nistz256_add      # p256_mul_by_3(M, M);
 978
 979         addi    $bp,$ap_real,0
 980         addi    $rp,$sp,$S
 981         bl      __ecp_nistz256_mul_mont # p256_mul_mont(S, S, in_x);
 982
 983         mr      $t0,$acc0
 984         mr      $t1,$acc1
 985         mr      $t2,$acc2
 986         mr      $t3,$acc3
 987          ld     $a0,$M+0($sp)           # forward load for p256_sqr_mont
 988          ld     $a1,$M+8($sp)
 989          ld     $a2,$M+16($sp)
 990          ld     $a3,$M+24($sp)
 991         addi    $rp,$sp,$tmp0
 992         bl      __ecp_nistz256_add      # p256_mul_by_2(tmp0, S);
 993
 994         addi    $rp,$rp_real,0
 995         bl      __ecp_nistz256_sqr_mont # p256_sqr_mont(res_x, M);
 996
 997         addi    $bp,$sp,$tmp0
 998         bl      __ecp_nistz256_sub_from # p256_sub(res_x, res_x, tmp0);
 999
1000         addi    $bp,$sp,$S
1001         addi    $rp,$sp,$S
1002         bl      __ecp_nistz256_sub_morf # p256_sub(S, S, res_x);
1003
1004         ld      $bi,$M($sp)
1005         mr      $a0,$acc0               # copy S
1006         mr      $a1,$acc1
1007         mr      $a2,$acc2
1008         mr      $a3,$acc3
1009         addi    $bp,$sp,$M
1010         bl      __ecp_nistz256_mul_mont # p256_mul_mont(S, S, M);
1011
1012         addi    $bp,$rp_real,32
1013         addi    $rp,$rp_real,32
1014         bl      __ecp_nistz256_sub_from # p256_sub(res_y, S, res_y);
1015
1016         mtlr    r0
1017         ld      r20,$FRAME-8*12($sp)
1018         ld      r21,$FRAME-8*11($sp)
1019         ld      r22,$FRAME-8*10($sp)
1020         ld      r23,$FRAME-8*9($sp)
1021         ld      r24,$FRAME-8*8($sp)
1022         ld      r25,$FRAME-8*7($sp)
1023         ld      r26,$FRAME-8*6($sp)
1024         ld      r27,$FRAME-8*5($sp)
1025         ld      r28,$FRAME-8*4($sp)
1026         ld      r29,$FRAME-8*3($sp)
1027         ld      r30,$FRAME-8*2($sp)
1028         ld      r31,$FRAME-8*1($sp)
1029         addi    $sp,$sp,$FRAME
1030         blr
1031         .long   0
1032         .byte   0,12,4,0,0x80,12,2,0
1033         .long   0
1034 .size   ecp_nistz256_point_double,.-ecp_nistz256_point_double
1035 ___
1036 }
1037
1038 ########################################################################
1039 # void ecp_nistz256_point_add(P256_POINT *out,const P256_POINT *in1,
1040 #                             const P256_POINT *in2);
1041 if (1) {
1042 my $FRAME = 64 + 32*12 + 16*8;
1043 my ($res_x,$res_y,$res_z,
1044     $H,$Hsqr,$R,$Rsqr,$Hcub,
1045     $U1,$U2,$S1,$S2)=map(64+32*$_,(0..11));
1046 my ($Z1sqr, $Z2sqr) = ($Hsqr, $Rsqr);
1047 # above map() describes stack layout with 12 temporary
1048 # 256-bit vectors on top.
1049 my ($rp_real,$ap_real,$bp_real,$in1infty,$in2infty,$temp)=map("r$_",(16..21));
1050
1051 $code.=<<___;
1052 .globl  ecp_nistz256_point_add
1053 .align  5
1054 ecp_nistz256_point_add:
1055         stdu    $sp,-$FRAME($sp)
1056         mflr    r0
1057         std     r16,$FRAME-8*16($sp)
1058         std     r17,$FRAME-8*15($sp)
1059         std     r18,$FRAME-8*14($sp)
1060         std     r19,$FRAME-8*13($sp)
1061         std     r20,$FRAME-8*12($sp)
1062         std     r21,$FRAME-8*11($sp)
1063         std     r22,$FRAME-8*10($sp)
1064         std     r23,$FRAME-8*9($sp)
1065         std     r24,$FRAME-8*8($sp)
1066         std     r25,$FRAME-8*7($sp)
1067         std     r26,$FRAME-8*6($sp)
1068         std     r27,$FRAME-8*5($sp)
1069         std     r28,$FRAME-8*4($sp)
1070         std     r29,$FRAME-8*3($sp)
1071         std     r30,$FRAME-8*2($sp)
1072         std     r31,$FRAME-8*1($sp)
1073
1074         li      $poly1,-1
1075         srdi    $poly1,$poly1,32        # 0x00000000ffffffff
1076         li      $poly3,1
1077         orc     $poly3,$poly3,$poly1    # 0xffffffff00000001
1078
1079         ld      $a0,64($bp)             # in2_z
1080         ld      $a1,72($bp)
1081         ld      $a2,80($bp)
1082         ld      $a3,88($bp)
1083          mr     $rp_real,$rp
1084          mr     $ap_real,$ap
1085          mr     $bp_real,$bp
1086         or      $t0,$a0,$a1
1087         or      $t2,$a2,$a3
1088         or      $in2infty,$t0,$t2
1089         neg     $t0,$in2infty
1090         or      $in2infty,$in2infty,$t0
1091         sradi   $in2infty,$in2infty,63  # !in2infty
1092         addi    $rp,$sp,$Z2sqr
1093         bl      __ecp_nistz256_sqr_mont # p256_sqr_mont(Z2sqr, in2_z);
1094
1095         ld      $a0,64($ap_real)        # in1_z
1096         ld      $a1,72($ap_real)
1097         ld      $a2,80($ap_real)
1098         ld      $a3,88($ap_real)
1099         or      $t0,$a0,$a1
1100         or      $t2,$a2,$a3
1101         or      $in1infty,$t0,$t2
1102         neg     $t0,$in1infty
1103         or      $in1infty,$in1infty,$t0
1104         sradi   $in1infty,$in1infty,63  # !in1infty
1105         addi    $rp,$sp,$Z1sqr
1106         bl      __ecp_nistz256_sqr_mont # p256_sqr_mont(Z1sqr, in1_z);
1107
1108         ld      $bi,64($bp_real)
1109         ld      $a0,$Z2sqr+0($sp)
1110         ld      $a1,$Z2sqr+8($sp)
1111         ld      $a2,$Z2sqr+16($sp)
1112         ld      $a3,$Z2sqr+24($sp)
1113         addi    $bp,$bp_real,64
1114         addi    $rp,$sp,$S1
1115         bl      __ecp_nistz256_mul_mont # p256_mul_mont(S1, Z2sqr, in2_z);
1116
1117         ld      $bi,64($ap_real)
1118         ld      $a0,$Z1sqr+0($sp)
1119         ld      $a1,$Z1sqr+8($sp)
1120         ld      $a2,$Z1sqr+16($sp)
1121         ld      $a3,$Z1sqr+24($sp)
1122         addi    $bp,$ap_real,64
1123         addi    $rp,$sp,$S2
1124         bl      __ecp_nistz256_mul_mont # p256_mul_mont(S2, Z1sqr, in1_z);
1125
1126         ld      $bi,32($ap_real)
1127         ld      $a0,$S1+0($sp)
1128         ld      $a1,$S1+8($sp)
1129         ld      $a2,$S1+16($sp)
1130         ld      $a3,$S1+24($sp)
1131         addi    $bp,$ap_real,32
1132         addi    $rp,$sp,$S1
1133         bl      __ecp_nistz256_mul_mont # p256_mul_mont(S1, S1, in1_y);
1134
1135         ld      $bi,32($bp_real)
1136         ld      $a0,$S2+0($sp)
1137         ld      $a1,$S2+8($sp)
1138         ld      $a2,$S2+16($sp)
1139         ld      $a3,$S2+24($sp)
1140         addi    $bp,$bp_real,32
1141         addi    $rp,$sp,$S2
1142         bl      __ecp_nistz256_mul_mont # p256_mul_mont(S2, S2, in2_y);
1143
1144         addi    $bp,$sp,$S1
1145          ld     $bi,$Z2sqr($sp)         # forward load for p256_mul_mont
1146          ld     $a0,0($ap_real)
1147          ld     $a1,8($ap_real)
1148          ld     $a2,16($ap_real)
1149          ld     $a3,24($ap_real)
1150         addi    $rp,$sp,$R
1151         bl      __ecp_nistz256_sub_from # p256_sub(R, S2, S1);
1152
1153         or      $acc0,$acc0,$acc1       # see if result is zero
1154         or      $acc2,$acc2,$acc3
1155         or      $temp,$acc0,$acc2
1156
1157         addi    $bp,$sp,$Z2sqr
1158         addi    $rp,$sp,$U1
1159         bl      __ecp_nistz256_mul_mont # p256_mul_mont(U1, in1_x, Z2sqr);
1160
1161         ld      $bi,$Z1sqr($sp)
1162         ld      $a0,0($bp_real)
1163         ld      $a1,8($bp_real)
1164         ld      $a2,16($bp_real)
1165         ld      $a3,24($bp_real)
1166         addi    $bp,$sp,$Z1sqr
1167         addi    $rp,$sp,$U2
1168         bl      __ecp_nistz256_mul_mont # p256_mul_mont(U2, in2_x, Z1sqr);
1169
1170         addi    $bp,$sp,$U1
1171          ld     $a0,$R+0($sp)           # forward load for p256_sqr_mont
1172          ld     $a1,$R+8($sp)
1173          ld     $a2,$R+16($sp)
1174          ld     $a3,$R+24($sp)
1175         addi    $rp,$sp,$H
1176         bl      __ecp_nistz256_sub_from # p256_sub(H, U2, U1);
1177
1178         or      $acc0,$acc0,$acc1       # see if result is zero
1179         or      $acc2,$acc2,$acc3
1180         or.     $acc0,$acc0,$acc2
1181         bne     .Ladd_proceed           # is_equal(U1,U2)?
1182
1183         and.    $t0,$in1infty,$in2infty
1184         beq     .Ladd_proceed           # (in1infty || in2infty)?
1185
1186         cmpldi  $temp,0
1187         beq     .Ladd_double            # is_equal(S1,S2)?
1188
1189         xor     $a0,$a0,$a0
1190         std     $a0,0($rp_real)
1191         std     $a0,8($rp_real)
1192         std     $a0,16($rp_real)
1193         std     $a0,24($rp_real)
1194         std     $a0,32($rp_real)
1195         std     $a0,40($rp_real)
1196         std     $a0,48($rp_real)
1197         std     $a0,56($rp_real)
1198         std     $a0,64($rp_real)
1199         std     $a0,72($rp_real)
1200         std     $a0,80($rp_real)
1201         std     $a0,88($rp_real)
1202         b       .Ladd_done
1203
1204 .align  4
1205 .Ladd_double:
1206         ld      $bp,0($sp)              # back-link
1207         mr      $ap,$ap_real
1208         mr      $rp,$rp_real
1209         ld      r16,$FRAME-8*16($sp)
1210         ld      r17,$FRAME-8*15($sp)
1211         ld      r18,$FRAME-8*14($sp)
1212         ld      r19,$FRAME-8*13($sp)
1213         stdu    $bp,$FRAME-288($sp)     # difference in stack frame sizes
1214         b       .Ldouble_shortcut
1215
1216 .align  4
1217 .Ladd_proceed:
1218         addi    $rp,$sp,$Rsqr
1219         bl      __ecp_nistz256_sqr_mont # p256_sqr_mont(Rsqr, R);
1220
1221         ld      $bi,64($ap_real)
1222         ld      $a0,$H+0($sp)
1223         ld      $a1,$H+8($sp)
1224         ld      $a2,$H+16($sp)
1225         ld      $a3,$H+24($sp)
1226         addi    $bp,$ap_real,64
1227         addi    $rp,$sp,$res_z
1228         bl      __ecp_nistz256_mul_mont # p256_mul_mont(res_z, H, in1_z);
1229
1230         ld      $a0,$H+0($sp)
1231         ld      $a1,$H+8($sp)
1232         ld      $a2,$H+16($sp)
1233         ld      $a3,$H+24($sp)
1234         addi    $rp,$sp,$Hsqr
1235         bl      __ecp_nistz256_sqr_mont # p256_sqr_mont(Hsqr, H);
1236
1237         ld      $bi,64($bp_real)
1238         ld      $a0,$res_z+0($sp)
1239         ld      $a1,$res_z+8($sp)
1240         ld      $a2,$res_z+16($sp)
1241         ld      $a3,$res_z+24($sp)
1242         addi    $bp,$bp_real,64
1243         addi    $rp,$sp,$res_z
1244         bl      __ecp_nistz256_mul_mont # p256_mul_mont(res_z, res_z, in2_z);
1245
1246         ld      $bi,$H($sp)
1247         ld      $a0,$Hsqr+0($sp)
1248         ld      $a1,$Hsqr+8($sp)
1249         ld      $a2,$Hsqr+16($sp)
1250         ld      $a3,$Hsqr+24($sp)
1251         addi    $bp,$sp,$H
1252         addi    $rp,$sp,$Hcub
1253         bl      __ecp_nistz256_mul_mont # p256_mul_mont(Hcub, Hsqr, H);
1254
1255         ld      $bi,$Hsqr($sp)
1256         ld      $a0,$U1+0($sp)
1257         ld      $a1,$U1+8($sp)
1258         ld      $a2,$U1+16($sp)
1259         ld      $a3,$U1+24($sp)
1260         addi    $bp,$sp,$Hsqr
1261         addi    $rp,$sp,$U2
1262         bl      __ecp_nistz256_mul_mont # p256_mul_mont(U2, U1, Hsqr);
1263
1264         mr      $t0,$acc0
1265         mr      $t1,$acc1
1266         mr      $t2,$acc2
1267         mr      $t3,$acc3
1268         addi    $rp,$sp,$Hsqr
1269         bl      __ecp_nistz256_add      # p256_mul_by_2(Hsqr, U2);
1270
1271         addi    $bp,$sp,$Rsqr
1272         addi    $rp,$sp,$res_x
1273         bl      __ecp_nistz256_sub_morf # p256_sub(res_x, Rsqr, Hsqr);
1274
1275         addi    $bp,$sp,$Hcub
1276         bl      __ecp_nistz256_sub_from # p256_sub(res_x, res_x, Hcub);
1277
1278         addi    $bp,$sp,$U2
1279          ld     $bi,$Hcub($sp)          # forward load for p256_mul_mont
1280          ld     $a0,$S1+0($sp)
1281          ld     $a1,$S1+8($sp)
1282          ld     $a2,$S1+16($sp)
1283          ld     $a3,$S1+24($sp)
1284         addi    $rp,$sp,$res_y
1285         bl      __ecp_nistz256_sub_morf # p256_sub(res_y, U2, res_x);
1286
1287         addi    $bp,$sp,$Hcub
1288         addi    $rp,$sp,$S2
1289         bl      __ecp_nistz256_mul_mont # p256_mul_mont(S2, S1, Hcub);
1290
1291         ld      $bi,$R($sp)
1292         ld      $a0,$res_y+0($sp)
1293         ld      $a1,$res_y+8($sp)
1294         ld      $a2,$res_y+16($sp)
1295         ld      $a3,$res_y+24($sp)
1296         addi    $bp,$sp,$R
1297         addi    $rp,$sp,$res_y
1298         bl      __ecp_nistz256_mul_mont # p256_mul_mont(res_y, res_y, R);
1299
1300         addi    $bp,$sp,$S2
1301         bl      __ecp_nistz256_sub_from # p256_sub(res_y, res_y, S2);
1302
1303         ld      $t0,0($bp_real)         # in2
1304         ld      $t1,8($bp_real)
1305         ld      $t2,16($bp_real)
1306         ld      $t3,24($bp_real)
1307         ld      $a0,$res_x+0($sp)       # res
1308         ld      $a1,$res_x+8($sp)
1309         ld      $a2,$res_x+16($sp)
1310         ld      $a3,$res_x+24($sp)
1311 ___
1312 for($i=0;$i<64;$i+=32) {                # conditional moves
1313 $code.=<<___;
1314         ld      $acc0,$i+0($ap_real)    # in1
1315         ld      $acc1,$i+8($ap_real)
1316         ld      $acc2,$i+16($ap_real)
1317         ld      $acc3,$i+24($ap_real)
1318         andc    $t0,$t0,$in1infty
1319         andc    $t1,$t1,$in1infty
1320         andc    $t2,$t2,$in1infty
1321         andc    $t3,$t3,$in1infty
1322         and     $a0,$a0,$in1infty
1323         and     $a1,$a1,$in1infty
1324         and     $a2,$a2,$in1infty
1325         and     $a3,$a3,$in1infty
1326         or      $t0,$t0,$a0
1327         or      $t1,$t1,$a1
1328         or      $t2,$t2,$a2
1329         or      $t3,$t3,$a3
1330         andc    $acc0,$acc0,$in2infty
1331         andc    $acc1,$acc1,$in2infty
1332         andc    $acc2,$acc2,$in2infty
1333         andc    $acc3,$acc3,$in2infty
1334         and     $t0,$t0,$in2infty
1335         and     $t1,$t1,$in2infty
1336         and     $t2,$t2,$in2infty
1337         and     $t3,$t3,$in2infty
1338         or      $acc0,$acc0,$t0
1339         or      $acc1,$acc1,$t1
1340         or      $acc2,$acc2,$t2
1341         or      $acc3,$acc3,$t3
1342
1343         ld      $t0,$i+32($bp_real)     # in2
1344         ld      $t1,$i+40($bp_real)
1345         ld      $t2,$i+48($bp_real)
1346         ld      $t3,$i+56($bp_real)
1347         ld      $a0,$res_x+$i+32($sp)
1348         ld      $a1,$res_x+$i+40($sp)
1349         ld      $a2,$res_x+$i+48($sp)
1350         ld      $a3,$res_x+$i+56($sp)
1351         std     $acc0,$i+0($rp_real)
1352         std     $acc1,$i+8($rp_real)
1353         std     $acc2,$i+16($rp_real)
1354         std     $acc3,$i+24($rp_real)
1355 ___
1356 }
1357 $code.=<<___;
1358         ld      $acc0,$i+0($ap_real)    # in1
1359         ld      $acc1,$i+8($ap_real)
1360         ld      $acc2,$i+16($ap_real)
1361         ld      $acc3,$i+24($ap_real)
1362         andc    $t0,$t0,$in1infty
1363         andc    $t1,$t1,$in1infty
1364         andc    $t2,$t2,$in1infty
1365         andc    $t3,$t3,$in1infty
1366         and     $a0,$a0,$in1infty
1367         and     $a1,$a1,$in1infty
1368         and     $a2,$a2,$in1infty
1369         and     $a3,$a3,$in1infty
1370         or      $t0,$t0,$a0
1371         or      $t1,$t1,$a1
1372         or      $t2,$t2,$a2
1373         or      $t3,$t3,$a3
1374         andc    $acc0,$acc0,$in2infty
1375         andc    $acc1,$acc1,$in2infty
1376         andc    $acc2,$acc2,$in2infty
1377         andc    $acc3,$acc3,$in2infty
1378         and     $t0,$t0,$in2infty
1379         and     $t1,$t1,$in2infty
1380         and     $t2,$t2,$in2infty
1381         and     $t3,$t3,$in2infty
1382         or      $acc0,$acc0,$t0
1383         or      $acc1,$acc1,$t1
1384         or      $acc2,$acc2,$t2
1385         or      $acc3,$acc3,$t3
1386         std     $acc0,$i+0($rp_real)
1387         std     $acc1,$i+8($rp_real)
1388         std     $acc2,$i+16($rp_real)
1389         std     $acc3,$i+24($rp_real)
1390
1391 .Ladd_done:
1392         mtlr    r0
1393         ld      r16,$FRAME-8*16($sp)
1394         ld      r17,$FRAME-8*15($sp)
1395         ld      r18,$FRAME-8*14($sp)
1396         ld      r19,$FRAME-8*13($sp)
1397         ld      r20,$FRAME-8*12($sp)
1398         ld      r21,$FRAME-8*11($sp)
1399         ld      r22,$FRAME-8*10($sp)
1400         ld      r23,$FRAME-8*9($sp)
1401         ld      r24,$FRAME-8*8($sp)
1402         ld      r25,$FRAME-8*7($sp)
1403         ld      r26,$FRAME-8*6($sp)
1404         ld      r27,$FRAME-8*5($sp)
1405         ld      r28,$FRAME-8*4($sp)
1406         ld      r29,$FRAME-8*3($sp)
1407         ld      r30,$FRAME-8*2($sp)
1408         ld      r31,$FRAME-8*1($sp)
1409         addi    $sp,$sp,$FRAME
1410         blr
1411         .long   0
1412         .byte   0,12,4,0,0x80,16,3,0
1413         .long   0
1414 .size   ecp_nistz256_point_add,.-ecp_nistz256_point_add
1415 ___
1416 }
1417
1418 ########################################################################
1419 # void ecp_nistz256_point_add_affine(P256_POINT *out,const P256_POINT *in1,
1420 #                                    const P256_POINT_AFFINE *in2);
1421 if (1) {
1422 my $FRAME = 64 + 32*10 + 16*8;
1423 my ($res_x,$res_y,$res_z,
1424     $U2,$S2,$H,$R,$Hsqr,$Hcub,$Rsqr)=map(64+32*$_,(0..9));
1425 my $Z1sqr = $S2;
1426 # above map() describes stack layout with 10 temporary
1427 # 256-bit vectors on top.
1428 my ($rp_real,$ap_real,$bp_real,$in1infty,$in2infty,$temp)=map("r$_",(16..21));
1429
1430 $code.=<<___;
1431 .globl  ecp_nistz256_point_add_affine
1432 .align  5
1433 ecp_nistz256_point_add_affine:
1434         stdu    $sp,-$FRAME($sp)
1435         mflr    r0
1436         std     r16,$FRAME-8*16($sp)
1437         std     r17,$FRAME-8*15($sp)
1438         std     r18,$FRAME-8*14($sp)
1439         std     r19,$FRAME-8*13($sp)
1440         std     r20,$FRAME-8*12($sp)
1441         std     r21,$FRAME-8*11($sp)
1442         std     r22,$FRAME-8*10($sp)
1443         std     r23,$FRAME-8*9($sp)
1444         std     r24,$FRAME-8*8($sp)
1445         std     r25,$FRAME-8*7($sp)
1446         std     r26,$FRAME-8*6($sp)
1447         std     r27,$FRAME-8*5($sp)
1448         std     r28,$FRAME-8*4($sp)
1449         std     r29,$FRAME-8*3($sp)
1450         std     r30,$FRAME-8*2($sp)
1451         std     r31,$FRAME-8*1($sp)
1452
1453         li      $poly1,-1
1454         srdi    $poly1,$poly1,32        # 0x00000000ffffffff
1455         li      $poly3,1
1456         orc     $poly3,$poly3,$poly1    # 0xffffffff00000001
1457
1458         mr      $rp_real,$rp
1459         mr      $ap_real,$ap
1460         mr      $bp_real,$bp
1461
1462         ld      $a0,64($ap)             # in1_z
1463         ld      $a1,72($ap)
1464         ld      $a2,80($ap)
1465         ld      $a3,88($ap)
1466         or      $t0,$a0,$a1
1467         or      $t2,$a2,$a3
1468         or      $in1infty,$t0,$t2
1469         neg     $t0,$in1infty
1470         or      $in1infty,$in1infty,$t0
1471         sradi   $in1infty,$in1infty,63  # !in1infty
1472
1473         ld      $acc0,0($bp)            # in2_x
1474         ld      $acc1,8($bp)
1475         ld      $acc2,16($bp)
1476         ld      $acc3,24($bp)
1477         ld      $t0,32($bp)             # in2_y
1478         ld      $t1,40($bp)
1479         ld      $t2,48($bp)
1480         ld      $t3,56($bp)
1481         or      $acc0,$acc0,$acc1
1482         or      $acc2,$acc2,$acc3
1483         or      $acc0,$acc0,$acc2
1484         or      $t0,$t0,$t1
1485         or      $t2,$t2,$t3
1486         or      $t0,$t0,$t2
1487         or      $in2infty,$acc0,$t0
1488         neg     $t0,$in2infty
1489         or      $in2infty,$in2infty,$t0
1490         sradi   $in2infty,$in2infty,63  # !in2infty
1491
1492         addi    $rp,$sp,$Z1sqr
1493         bl      __ecp_nistz256_sqr_mont # p256_sqr_mont(Z1sqr, in1_z);
1494
1495         mr      $a0,$acc0
1496         mr      $a1,$acc1
1497         mr      $a2,$acc2
1498         mr      $a3,$acc3
1499         ld      $bi,0($bp_real)
1500         addi    $bp,$bp_real,0
1501         addi    $rp,$sp,$U2
1502         bl      __ecp_nistz256_mul_mont # p256_mul_mont(U2, Z1sqr, in2_x);
1503
1504         addi    $bp,$ap_real,0
1505          ld     $bi,64($ap_real)        # forward load for p256_mul_mont
1506          ld     $a0,$Z1sqr+0($sp)
1507          ld     $a1,$Z1sqr+8($sp)
1508          ld     $a2,$Z1sqr+16($sp)
1509          ld     $a3,$Z1sqr+24($sp)
1510         addi    $rp,$sp,$H
1511         bl      __ecp_nistz256_sub_from # p256_sub(H, U2, in1_x);
1512
1513         addi    $bp,$ap_real,64
1514         addi    $rp,$sp,$S2
1515         bl      __ecp_nistz256_mul_mont # p256_mul_mont(S2, Z1sqr, in1_z);
1516
1517         ld      $bi,64($ap_real)
1518         ld      $a0,$H+0($sp)
1519         ld      $a1,$H+8($sp)
1520         ld      $a2,$H+16($sp)
1521         ld      $a3,$H+24($sp)
1522         addi    $bp,$ap_real,64
1523         addi    $rp,$sp,$res_z
1524         bl      __ecp_nistz256_mul_mont # p256_mul_mont(res_z, H, in1_z);
1525
1526         ld      $bi,32($bp_real)
1527         ld      $a0,$S2+0($sp)
1528         ld      $a1,$S2+8($sp)
1529         ld      $a2,$S2+16($sp)
1530         ld      $a3,$S2+24($sp)
1531         addi    $bp,$bp_real,32
1532         addi    $rp,$sp,$S2
1533         bl      __ecp_nistz256_mul_mont # p256_mul_mont(S2, S2, in2_y);
1534
1535         addi    $bp,$ap_real,32
1536          ld     $a0,$H+0($sp)           # forward load for p256_sqr_mont
1537          ld     $a1,$H+8($sp)
1538          ld     $a2,$H+16($sp)
1539          ld     $a3,$H+24($sp)
1540         addi    $rp,$sp,$R
1541         bl      __ecp_nistz256_sub_from # p256_sub(R, S2, in1_y);
1542
1543         addi    $rp,$sp,$Hsqr
1544         bl      __ecp_nistz256_sqr_mont # p256_sqr_mont(Hsqr, H);
1545
1546         ld      $a0,$R+0($sp)
1547         ld      $a1,$R+8($sp)
1548         ld      $a2,$R+16($sp)
1549         ld      $a3,$R+24($sp)
1550         addi    $rp,$sp,$Rsqr
1551         bl      __ecp_nistz256_sqr_mont # p256_sqr_mont(Rsqr, R);
1552
1553         ld      $bi,$H($sp)
1554         ld      $a0,$Hsqr+0($sp)
1555         ld      $a1,$Hsqr+8($sp)
1556         ld      $a2,$Hsqr+16($sp)
1557         ld      $a3,$Hsqr+24($sp)
1558         addi    $bp,$sp,$H
1559         addi    $rp,$sp,$Hcub
1560         bl      __ecp_nistz256_mul_mont # p256_mul_mont(Hcub, Hsqr, H);
1561
1562         ld      $bi,0($ap_real)
1563         ld      $a0,$Hsqr+0($sp)
1564         ld      $a1,$Hsqr+8($sp)
1565         ld      $a2,$Hsqr+16($sp)
1566         ld      $a3,$Hsqr+24($sp)
1567         addi    $bp,$ap_real,0
1568         addi    $rp,$sp,$U2
1569         bl      __ecp_nistz256_mul_mont # p256_mul_mont(U2, in1_x, Hsqr);
1570
1571         mr      $t0,$acc0
1572         mr      $t1,$acc1
1573         mr      $t2,$acc2
1574         mr      $t3,$acc3
1575         addi    $rp,$sp,$Hsqr
1576         bl      __ecp_nistz256_add      # p256_mul_by_2(Hsqr, U2);
1577
1578         addi    $bp,$sp,$Rsqr
1579         addi    $rp,$sp,$res_x
1580         bl      __ecp_nistz256_sub_morf # p256_sub(res_x, Rsqr, Hsqr);
1581
1582         addi    $bp,$sp,$Hcub
1583         bl      __ecp_nistz256_sub_from #  p256_sub(res_x, res_x, Hcub);
1584
1585         addi    $bp,$sp,$U2
1586          ld     $bi,32($ap_real)        # forward load for p256_mul_mont
1587          ld     $a0,$Hcub+0($sp)
1588          ld     $a1,$Hcub+8($sp)
1589          ld     $a2,$Hcub+16($sp)
1590          ld     $a3,$Hcub+24($sp)
1591         addi    $rp,$sp,$res_y
1592         bl      __ecp_nistz256_sub_morf # p256_sub(res_y, U2, res_x);
1593
1594         addi    $bp,$ap_real,32
1595         addi    $rp,$sp,$S2
1596         bl      __ecp_nistz256_mul_mont # p256_mul_mont(S2, in1_y, Hcub);
1597
1598         ld      $bi,$R($sp)
1599         ld      $a0,$res_y+0($sp)
1600         ld      $a1,$res_y+8($sp)
1601         ld      $a2,$res_y+16($sp)
1602         ld      $a3,$res_y+24($sp)
1603         addi    $bp,$sp,$R
1604         addi    $rp,$sp,$res_y
1605         bl      __ecp_nistz256_mul_mont # p256_mul_mont(res_y, res_y, R);
1606
1607         addi    $bp,$sp,$S2
1608         bl      __ecp_nistz256_sub_from # p256_sub(res_y, res_y, S2);
1609
1610         ld      $t0,0($bp_real)         # in2
1611         ld      $t1,8($bp_real)
1612         ld      $t2,16($bp_real)
1613         ld      $t3,24($bp_real)
1614         ld      $a0,$res_x+0($sp)       # res
1615         ld      $a1,$res_x+8($sp)
1616         ld      $a2,$res_x+16($sp)
1617         ld      $a3,$res_x+24($sp)
1618 ___
1619 for($i=0;$i<64;$i+=32) {                # conditional moves
1620 $code.=<<___;
1621         ld      $acc0,$i+0($ap_real)    # in1
1622         ld      $acc1,$i+8($ap_real)
1623         ld      $acc2,$i+16($ap_real)
1624         ld      $acc3,$i+24($ap_real)
1625         andc    $t0,$t0,$in1infty
1626         andc    $t1,$t1,$in1infty
1627         andc    $t2,$t2,$in1infty
1628         andc    $t3,$t3,$in1infty
1629         and     $a0,$a0,$in1infty
1630         and     $a1,$a1,$in1infty
1631         and     $a2,$a2,$in1infty
1632         and     $a3,$a3,$in1infty
1633         or      $t0,$t0,$a0
1634         or      $t1,$t1,$a1
1635         or      $t2,$t2,$a2
1636         or      $t3,$t3,$a3
1637         andc    $acc0,$acc0,$in2infty
1638         andc    $acc1,$acc1,$in2infty
1639         andc    $acc2,$acc2,$in2infty
1640         andc    $acc3,$acc3,$in2infty
1641         and     $t0,$t0,$in2infty
1642         and     $t1,$t1,$in2infty
1643         and     $t2,$t2,$in2infty
1644         and     $t3,$t3,$in2infty
1645         or      $acc0,$acc0,$t0
1646         or      $acc1,$acc1,$t1
1647         or      $acc2,$acc2,$t2
1648         or      $acc3,$acc3,$t3
1649 ___
1650 $code.=<<___    if ($i==0);
1651         ld      $t0,32($bp_real)        # in2
1652         ld      $t1,40($bp_real)
1653         ld      $t2,48($bp_real)
1654         ld      $t3,56($bp_real)
1655 ___
1656 $code.=<<___    if ($i==32);
1657         li      $t0,1                   # Lone_mont
1658         not     $t1,$poly1
1659         li      $t2,-1
1660         not     $t3,$poly3
1661 ___
1662 $code.=<<___;
1663         ld      $a0,$res_x+$i+32($sp)
1664         ld      $a1,$res_x+$i+40($sp)
1665         ld      $a2,$res_x+$i+48($sp)
1666         ld      $a3,$res_x+$i+56($sp)
1667         std     $acc0,$i+0($rp_real)
1668         std     $acc1,$i+8($rp_real)
1669         std     $acc2,$i+16($rp_real)
1670         std     $acc3,$i+24($rp_real)
1671 ___
1672 }
1673 $code.=<<___;
1674         ld      $acc0,$i+0($ap_real)    # in1
1675         ld      $acc1,$i+8($ap_real)
1676         ld      $acc2,$i+16($ap_real)
1677         ld      $acc3,$i+24($ap_real)
1678         andc    $t0,$t0,$in1infty
1679         andc    $t1,$t1,$in1infty
1680         andc    $t2,$t2,$in1infty
1681         andc    $t3,$t3,$in1infty
1682         and     $a0,$a0,$in1infty
1683         and     $a1,$a1,$in1infty
1684         and     $a2,$a2,$in1infty
1685         and     $a3,$a3,$in1infty
1686         or      $t0,$t0,$a0
1687         or      $t1,$t1,$a1
1688         or      $t2,$t2,$a2
1689         or      $t3,$t3,$a3
1690         andc    $acc0,$acc0,$in2infty
1691         andc    $acc1,$acc1,$in2infty
1692         andc    $acc2,$acc2,$in2infty
1693         andc    $acc3,$acc3,$in2infty
1694         and     $t0,$t0,$in2infty
1695         and     $t1,$t1,$in2infty
1696         and     $t2,$t2,$in2infty
1697         and     $t3,$t3,$in2infty
1698         or      $acc0,$acc0,$t0
1699         or      $acc1,$acc1,$t1
1700         or      $acc2,$acc2,$t2
1701         or      $acc3,$acc3,$t3
1702         std     $acc0,$i+0($rp_real)
1703         std     $acc1,$i+8($rp_real)
1704         std     $acc2,$i+16($rp_real)
1705         std     $acc3,$i+24($rp_real)
1706
1707         mtlr    r0
1708         ld      r16,$FRAME-8*16($sp)
1709         ld      r17,$FRAME-8*15($sp)
1710         ld      r18,$FRAME-8*14($sp)
1711         ld      r19,$FRAME-8*13($sp)
1712         ld      r20,$FRAME-8*12($sp)
1713         ld      r21,$FRAME-8*11($sp)
1714         ld      r22,$FRAME-8*10($sp)
1715         ld      r23,$FRAME-8*9($sp)
1716         ld      r24,$FRAME-8*8($sp)
1717         ld      r25,$FRAME-8*7($sp)
1718         ld      r26,$FRAME-8*6($sp)
1719         ld      r27,$FRAME-8*5($sp)
1720         ld      r28,$FRAME-8*4($sp)
1721         ld      r29,$FRAME-8*3($sp)
1722         ld      r30,$FRAME-8*2($sp)
1723         ld      r31,$FRAME-8*1($sp)
1724         addi    $sp,$sp,$FRAME
1725         blr
1726         .long   0
1727         .byte   0,12,4,0,0x80,16,3,0
1728         .long   0
1729 .size   ecp_nistz256_point_add_affine,.-ecp_nistz256_point_add_affine
1730 ___
1731 }
1732 if (1) {
1733 my ($ordk,$ord0,$ord1,$t4) = map("r$_",(18..21));
1734 my ($ord2,$ord3,$zr) = ($poly1,$poly3,"r0");
1735
1736 $code.=<<___;
1737 ########################################################################
1738 # void ecp_nistz256_ord_mul_mont(uint64_t res[4], uint64_t a[4],
1739 #                                uint64_t b[4]);
1740 .globl  ecp_nistz256_ord_mul_mont
1741 .align  5
1742 ecp_nistz256_ord_mul_mont:
1743         stdu    $sp,-160($sp)
1744         std     r18,48($sp)
1745         std     r19,56($sp)
1746         std     r20,64($sp)
1747         std     r21,72($sp)
1748         std     r22,80($sp)
1749         std     r23,88($sp)
1750         std     r24,96($sp)
1751         std     r25,104($sp)
1752         std     r26,112($sp)
1753         std     r27,120($sp)
1754         std     r28,128($sp)
1755         std     r29,136($sp)
1756         std     r30,144($sp)
1757         std     r31,152($sp)
1758
1759         ld      $a0,0($ap)
1760         ld      $bi,0($bp)
1761         ld      $a1,8($ap)
1762         ld      $a2,16($ap)
1763         ld      $a3,24($ap)
1764
1765         lis     $ordk,0xccd1
1766         lis     $ord0,0xf3b9
1767         lis     $ord1,0xbce6
1768         ori     $ordk,$ordk,0xc8aa
1769         ori     $ord0,$ord0,0xcac2
1770         ori     $ord1,$ord1,0xfaad
1771         sldi    $ordk,$ordk,32
1772         sldi    $ord0,$ord0,32
1773         sldi    $ord1,$ord1,32
1774         oris    $ordk,$ordk,0xee00
1775         oris    $ord0,$ord0,0xfc63
1776         oris    $ord1,$ord1,0xa717
1777         ori     $ordk,$ordk,0xbc4f      # 0xccd1c8aaee00bc4f
1778         ori     $ord0,$ord0,0x2551      # 0xf3b9cac2fc632551
1779         ori     $ord1,$ord1,0x9e84      # 0xbce6faada7179e84
1780         li      $ord2,-1                # 0xffffffffffffffff
1781         sldi    $ord3,$ord2,32          # 0xffffffff00000000
1782         li      $zr,0
1783
1784         mulld   $acc0,$a0,$bi           # a[0]*b[0]
1785         mulhdu  $t0,$a0,$bi
1786
1787         mulld   $acc1,$a1,$bi           # a[1]*b[0]
1788         mulhdu  $t1,$a1,$bi
1789
1790         mulld   $acc2,$a2,$bi           # a[2]*b[0]
1791         mulhdu  $t2,$a2,$bi
1792
1793         mulld   $acc3,$a3,$bi           # a[3]*b[0]
1794         mulhdu  $acc4,$a3,$bi
1795
1796         mulld   $t4,$acc0,$ordk
1797
1798         addc    $acc1,$acc1,$t0         # accumulate high parts of multiplication
1799         adde    $acc2,$acc2,$t1
1800         adde    $acc3,$acc3,$t2
1801         addze   $acc4,$acc4
1802         li      $acc5,0
1803 ___
1804 for ($i=1;$i<4;$i++) {
1805         ################################################################
1806         #            ffff0000.ffffffff.yyyyyyyy.zzzzzzzz
1807         # *                                     abcdefgh
1808         # + xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx
1809         #
1810         # Now observing that ff..ff*x = (2^n-1)*x = 2^n*x-x, we
1811         # rewrite above as:
1812         #
1813         #   xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx
1814         # - 0000abcd.efgh0000.abcdefgh.00000000.00000000
1815         # + abcdefgh.abcdefgh.yzayzbyz.cyzdyzey.zfyzgyzh
1816 $code.=<<___;
1817         ld      $bi,8*$i($bp)           # b[i]
1818
1819         sldi    $t0,$t4,32
1820         subfc   $acc2,$t4,$acc2
1821         srdi    $t1,$t4,32
1822         subfe   $acc3,$t0,$acc3
1823         subfe   $acc4,$t1,$acc4
1824         subfe   $acc5,$zr,$acc5
1825
1826         addic   $t0,$acc0,-1            # discarded
1827         mulhdu  $t1,$ord0,$t4
1828         mulld   $t2,$ord1,$t4
1829         mulhdu  $t3,$ord1,$t4
1830
1831         adde    $t2,$t2,$t1
1832          mulld  $t0,$a0,$bi
1833         addze   $t3,$t3
1834          mulld  $t1,$a1,$bi
1835
1836         addc    $acc0,$acc1,$t2
1837          mulld  $t2,$a2,$bi
1838         adde    $acc1,$acc2,$t3
1839          mulld  $t3,$a3,$bi
1840         adde    $acc2,$acc3,$t4
1841         adde    $acc3,$acc4,$t4
1842         addze   $acc4,$acc5
1843
1844         addc    $acc0,$acc0,$t0         # accumulate low parts
1845         mulhdu  $t0,$a0,$bi
1846         adde    $acc1,$acc1,$t1
1847         mulhdu  $t1,$a1,$bi
1848         adde    $acc2,$acc2,$t2
1849         mulhdu  $t2,$a2,$bi
1850         adde    $acc3,$acc3,$t3
1851         mulhdu  $t3,$a3,$bi
1852         addze   $acc4,$acc4
1853         mulld   $t4,$acc0,$ordk
1854         addc    $acc1,$acc1,$t0         # accumulate high parts
1855         adde    $acc2,$acc2,$t1
1856         adde    $acc3,$acc3,$t2
1857         adde    $acc4,$acc4,$t3
1858         addze   $acc5,$zr
1859 ___
1860 }
1861 $code.=<<___;
1862         sldi    $t0,$t4,32              # last reduction
1863         subfc   $acc2,$t4,$acc2
1864         srdi    $t1,$t4,32
1865         subfe   $acc3,$t0,$acc3
1866         subfe   $acc4,$t1,$acc4
1867         subfe   $acc5,$zr,$acc5
1868
1869         addic   $t0,$acc0,-1            # discarded
1870         mulhdu  $t1,$ord0,$t4
1871         mulld   $t2,$ord1,$t4
1872         mulhdu  $t3,$ord1,$t4
1873
1874         adde    $t2,$t2,$t1
1875         addze   $t3,$t3
1876
1877         addc    $acc0,$acc1,$t2
1878         adde    $acc1,$acc2,$t3
1879         adde    $acc2,$acc3,$t4
1880         adde    $acc3,$acc4,$t4
1881         addze   $acc4,$acc5
1882
1883         subfc   $acc0,$ord0,$acc0       # ret -= modulus
1884         subfe   $acc1,$ord1,$acc1
1885         subfe   $acc2,$ord2,$acc2
1886         subfe   $acc3,$ord3,$acc3
1887         subfe   $acc4,$zr,$acc4
1888
1889         and     $t0,$ord0,$acc4
1890         and     $t1,$ord1,$acc4
1891         addc    $acc0,$acc0,$t0         # ret += modulus if borrow
1892         and     $t3,$ord3,$acc4
1893         adde    $acc1,$acc1,$t1
1894         adde    $acc2,$acc2,$acc4
1895         adde    $acc3,$acc3,$t3
1896
1897         std     $acc0,0($rp)
1898         std     $acc1,8($rp)
1899         std     $acc2,16($rp)
1900         std     $acc3,24($rp)
1901
1902         ld      r18,48($sp)
1903         ld      r19,56($sp)
1904         ld      r20,64($sp)
1905         ld      r21,72($sp)
1906         ld      r22,80($sp)
1907         ld      r23,88($sp)
1908         ld      r24,96($sp)
1909         ld      r25,104($sp)
1910         ld      r26,112($sp)
1911         ld      r27,120($sp)
1912         ld      r28,128($sp)
1913         ld      r29,136($sp)
1914         ld      r30,144($sp)
1915         ld      r31,152($sp)
1916         addi    $sp,$sp,160
1917         blr
1918         .long   0
1919         .byte   0,12,4,0,0x80,14,3,0
1920         .long   0
1921 .size   ecp_nistz256_ord_mul_mont,.-ecp_nistz256_ord_mul_mont
1922
1923 ################################################################################
1924 # void ecp_nistz256_ord_sqr_mont(uint64_t res[4], uint64_t a[4],
1925 #                                uint64_t rep);
1926 .globl  ecp_nistz256_ord_sqr_mont
1927 .align  5
1928 ecp_nistz256_ord_sqr_mont:
1929         stdu    $sp,-160($sp)
1930         std     r18,48($sp)
1931         std     r19,56($sp)
1932         std     r20,64($sp)
1933         std     r21,72($sp)
1934         std     r22,80($sp)
1935         std     r23,88($sp)
1936         std     r24,96($sp)
1937         std     r25,104($sp)
1938         std     r26,112($sp)
1939         std     r27,120($sp)
1940         std     r28,128($sp)
1941         std     r29,136($sp)
1942         std     r30,144($sp)
1943         std     r31,152($sp)
1944
1945         mtctr   $bp
1946
1947         ld      $a0,0($ap)
1948         ld      $a1,8($ap)
1949         ld      $a2,16($ap)
1950         ld      $a3,24($ap)
1951
1952         lis     $ordk,0xccd1
1953         lis     $ord0,0xf3b9
1954         lis     $ord1,0xbce6
1955         ori     $ordk,$ordk,0xc8aa
1956         ori     $ord0,$ord0,0xcac2
1957         ori     $ord1,$ord1,0xfaad
1958         sldi    $ordk,$ordk,32
1959         sldi    $ord0,$ord0,32
1960         sldi    $ord1,$ord1,32
1961         oris    $ordk,$ordk,0xee00
1962         oris    $ord0,$ord0,0xfc63
1963         oris    $ord1,$ord1,0xa717
1964         ori     $ordk,$ordk,0xbc4f      # 0xccd1c8aaee00bc4f
1965         ori     $ord0,$ord0,0x2551      # 0xf3b9cac2fc632551
1966         ori     $ord1,$ord1,0x9e84      # 0xbce6faada7179e84
1967         li      $ord2,-1                # 0xffffffffffffffff
1968         sldi    $ord3,$ord2,32          # 0xffffffff00000000
1969         li      $zr,0
1970         b       .Loop_ord_sqr
1971
1972 .align  5
1973 .Loop_ord_sqr:
1974         ################################################################
1975         #  |  |  |  |  |  |a1*a0|  |
1976         #  |  |  |  |  |a2*a0|  |  |
1977         #  |  |a3*a2|a3*a0|  |  |  |
1978         #  |  |  |  |a2*a1|  |  |  |
1979         #  |  |  |a3*a1|  |  |  |  |
1980         # *|  |  |  |  |  |  |  | 2|
1981         # +|a3*a3|a2*a2|a1*a1|a0*a0|
1982         #  |--+--+--+--+--+--+--+--|
1983         #  |A7|A6|A5|A4|A3|A2|A1|A0|, where Ax is $accx, i.e. follow $accx
1984         #
1985         #  "can't overflow" below mark carrying into high part of
1986         #  multiplication result, which can't overflow, because it
1987         #  can never be all ones.
1988
1989         mulld   $acc1,$a1,$a0           # a[1]*a[0]
1990         mulhdu  $t1,$a1,$a0
1991         mulld   $acc2,$a2,$a0           # a[2]*a[0]
1992         mulhdu  $t2,$a2,$a0
1993         mulld   $acc3,$a3,$a0           # a[3]*a[0]
1994         mulhdu  $acc4,$a3,$a0
1995
1996         addc    $acc2,$acc2,$t1         # accumulate high parts of multiplication
1997          mulld  $t0,$a2,$a1             # a[2]*a[1]
1998          mulhdu $t1,$a2,$a1
1999         adde    $acc3,$acc3,$t2
2000          mulld  $t2,$a3,$a1             # a[3]*a[1]
2001          mulhdu $t3,$a3,$a1
2002         addze   $acc4,$acc4             # can't overflow
2003
2004         mulld   $acc5,$a3,$a2           # a[3]*a[2]
2005         mulhdu  $acc6,$a3,$a2
2006
2007         addc    $t1,$t1,$t2             # accumulate high parts of multiplication
2008          mulld  $acc0,$a0,$a0           # a[0]*a[0]
2009         addze   $t2,$t3                 # can't overflow
2010
2011         addc    $acc3,$acc3,$t0         # accumulate low parts of multiplication
2012          mulhdu $a0,$a0,$a0
2013         adde    $acc4,$acc4,$t1
2014          mulld  $t1,$a1,$a1             # a[1]*a[1]
2015         adde    $acc5,$acc5,$t2
2016          mulhdu $a1,$a1,$a1
2017         addze   $acc6,$acc6             # can't overflow
2018
2019         addc    $acc1,$acc1,$acc1       # acc[1-6]*=2
2020          mulld  $t2,$a2,$a2             # a[2]*a[2]
2021         adde    $acc2,$acc2,$acc2
2022          mulhdu $a2,$a2,$a2
2023         adde    $acc3,$acc3,$acc3
2024          mulld  $t3,$a3,$a3             # a[3]*a[3]
2025         adde    $acc4,$acc4,$acc4
2026          mulhdu $a3,$a3,$a3
2027         adde    $acc5,$acc5,$acc5
2028         adde    $acc6,$acc6,$acc6
2029         addze   $acc7,$zr
2030
2031         addc    $acc1,$acc1,$a0         # +a[i]*a[i]
2032          mulld  $t4,$acc0,$ordk
2033         adde    $acc2,$acc2,$t1
2034         adde    $acc3,$acc3,$a1
2035         adde    $acc4,$acc4,$t2
2036         adde    $acc5,$acc5,$a2
2037         adde    $acc6,$acc6,$t3
2038         adde    $acc7,$acc7,$a3
2039 ___
2040 for($i=0; $i<4; $i++) {                 # reductions
2041 $code.=<<___;
2042         addic   $t0,$acc0,-1            # discarded
2043         mulhdu  $t1,$ord0,$t4
2044         mulld   $t2,$ord1,$t4
2045         mulhdu  $t3,$ord1,$t4
2046
2047         adde    $t2,$t2,$t1
2048         addze   $t3,$t3
2049
2050         addc    $acc0,$acc1,$t2
2051         adde    $acc1,$acc2,$t3
2052         adde    $acc2,$acc3,$t4
2053         adde    $acc3,$zr,$t4           # can't overflow
2054 ___
2055 $code.=<<___    if ($i<3);
2056         mulld   $t3,$acc0,$ordk
2057 ___
2058 $code.=<<___;
2059         sldi    $t0,$t4,32
2060         subfc   $acc1,$t4,$acc1
2061         srdi    $t1,$t4,32
2062         subfe   $acc2,$t0,$acc2
2063         subfe   $acc3,$t1,$acc3         # can't borrow
2064 ___
2065         ($t3,$t4) = ($t4,$t3);
2066 }
2067 $code.=<<___;
2068         addc    $acc0,$acc0,$acc4       # accumulate upper half
2069         adde    $acc1,$acc1,$acc5
2070         adde    $acc2,$acc2,$acc6
2071         adde    $acc3,$acc3,$acc7
2072         addze   $acc4,$zr
2073
2074         subfc   $acc0,$ord0,$acc0       # ret -= modulus
2075         subfe   $acc1,$ord1,$acc1
2076         subfe   $acc2,$ord2,$acc2
2077         subfe   $acc3,$ord3,$acc3
2078         subfe   $acc4,$zr,$acc4
2079
2080         and     $t0,$ord0,$acc4
2081         and     $t1,$ord1,$acc4
2082         addc    $a0,$acc0,$t0           # ret += modulus if borrow
2083         and     $t3,$ord3,$acc4
2084         adde    $a1,$acc1,$t1
2085         adde    $a2,$acc2,$acc4
2086         adde    $a3,$acc3,$t3
2087
2088         bdnz    .Loop_ord_sqr
2089
2090         std     $a0,0($rp)
2091         std     $a1,8($rp)
2092         std     $a2,16($rp)
2093         std     $a3,24($rp)
2094
2095         ld      r18,48($sp)
2096         ld      r19,56($sp)
2097         ld      r20,64($sp)
2098         ld      r21,72($sp)
2099         ld      r22,80($sp)
2100         ld      r23,88($sp)
2101         ld      r24,96($sp)
2102         ld      r25,104($sp)
2103         ld      r26,112($sp)
2104         ld      r27,120($sp)
2105         ld      r28,128($sp)
2106         ld      r29,136($sp)
2107         ld      r30,144($sp)
2108         ld      r31,152($sp)
2109         addi    $sp,$sp,160
2110         blr
2111         .long   0
2112         .byte   0,12,4,0,0x80,14,3,0
2113         .long   0
2114 .size   ecp_nistz256_ord_sqr_mont,.-ecp_nistz256_ord_sqr_mont
2115 ___
2116 }       }
2117
2118 ########################################################################
2119 # scatter-gather subroutines
2120 {
2121 my ($out,$inp,$index,$mask)=map("r$_",(3..7));
2122 $code.=<<___;
2123 ########################################################################
2124 # void  ecp_nistz256_scatter_w5(void *out, const P256_POINT *inp,
2125 #                               int index);
2126 .globl  ecp_nistz256_scatter_w5
2127 .align  4
2128 ecp_nistz256_scatter_w5:
2129         slwi    $index,$index,2
2130         add     $out,$out,$index
2131
2132         ld      r8, 0($inp)             # X
2133         ld      r9, 8($inp)
2134         ld      r10,16($inp)
2135         ld      r11,24($inp)
2136
2137         stw     r8, 64*0-4($out)
2138         srdi    r8, r8, 32
2139         stw     r9, 64*1-4($out)
2140         srdi    r9, r9, 32
2141         stw     r10,64*2-4($out)
2142         srdi    r10,r10,32
2143         stw     r11,64*3-4($out)
2144         srdi    r11,r11,32
2145         stw     r8, 64*4-4($out)
2146         stw     r9, 64*5-4($out)
2147         stw     r10,64*6-4($out)
2148         stw     r11,64*7-4($out)
2149         addi    $out,$out,64*8
2150
2151         ld      r8, 32($inp)            # Y
2152         ld      r9, 40($inp)
2153         ld      r10,48($inp)
2154         ld      r11,56($inp)
2155
2156         stw     r8, 64*0-4($out)
2157         srdi    r8, r8, 32
2158         stw     r9, 64*1-4($out)
2159         srdi    r9, r9, 32
2160         stw     r10,64*2-4($out)
2161         srdi    r10,r10,32
2162         stw     r11,64*3-4($out)
2163         srdi    r11,r11,32
2164         stw     r8, 64*4-4($out)
2165         stw     r9, 64*5-4($out)
2166         stw     r10,64*6-4($out)
2167         stw     r11,64*7-4($out)
2168         addi    $out,$out,64*8
2169
2170         ld      r8, 64($inp)            # Z
2171         ld      r9, 72($inp)
2172         ld      r10,80($inp)
2173         ld      r11,88($inp)
2174
2175         stw     r8, 64*0-4($out)
2176         srdi    r8, r8, 32
2177         stw     r9, 64*1-4($out)
2178         srdi    r9, r9, 32
2179         stw     r10,64*2-4($out)
2180         srdi    r10,r10,32
2181         stw     r11,64*3-4($out)
2182         srdi    r11,r11,32
2183         stw     r8, 64*4-4($out)
2184         stw     r9, 64*5-4($out)
2185         stw     r10,64*6-4($out)
2186         stw     r11,64*7-4($out)
2187
2188         blr
2189         .long   0
2190         .byte   0,12,0x14,0,0,0,3,0
2191         .long   0
2192 .size   ecp_nistz256_scatter_w5,.-ecp_nistz256_scatter_w5
2193
2194 ########################################################################
2195 # void  ecp_nistz256_gather_w5(P256_POINT *out, const void *inp,
2196 #                               int index);
2197 .globl  ecp_nistz256_gather_w5
2198 .align  4
2199 ecp_nistz256_gather_w5:
2200         neg     r0,$index
2201         sradi   r0,r0,63
2202
2203         add     $index,$index,r0
2204         slwi    $index,$index,2
2205         add     $inp,$inp,$index
2206
2207         lwz     r5, 64*0($inp)
2208         lwz     r6, 64*1($inp)
2209         lwz     r7, 64*2($inp)
2210         lwz     r8, 64*3($inp)
2211         lwz     r9, 64*4($inp)
2212         lwz     r10,64*5($inp)
2213         lwz     r11,64*6($inp)
2214         lwz     r12,64*7($inp)
2215         addi    $inp,$inp,64*8
2216         sldi    r9, r9, 32
2217         sldi    r10,r10,32
2218         sldi    r11,r11,32
2219         sldi    r12,r12,32
2220         or      r5,r5,r9
2221         or      r6,r6,r10
2222         or      r7,r7,r11
2223         or      r8,r8,r12
2224         and     r5,r5,r0
2225         and     r6,r6,r0
2226         and     r7,r7,r0
2227         and     r8,r8,r0
2228         std     r5,0($out)              # X
2229         std     r6,8($out)
2230         std     r7,16($out)
2231         std     r8,24($out)
2232
2233         lwz     r5, 64*0($inp)
2234         lwz     r6, 64*1($inp)
2235         lwz     r7, 64*2($inp)
2236         lwz     r8, 64*3($inp)
2237         lwz     r9, 64*4($inp)
2238         lwz     r10,64*5($inp)
2239         lwz     r11,64*6($inp)
2240         lwz     r12,64*7($inp)
2241         addi    $inp,$inp,64*8
2242         sldi    r9, r9, 32
2243         sldi    r10,r10,32
2244         sldi    r11,r11,32
2245         sldi    r12,r12,32
2246         or      r5,r5,r9
2247         or      r6,r6,r10
2248         or      r7,r7,r11
2249         or      r8,r8,r12
2250         and     r5,r5,r0
2251         and     r6,r6,r0
2252         and     r7,r7,r0
2253         and     r8,r8,r0
2254         std     r5,32($out)             # Y
2255         std     r6,40($out)
2256         std     r7,48($out)
2257         std     r8,56($out)
2258
2259         lwz     r5, 64*0($inp)
2260         lwz     r6, 64*1($inp)
2261         lwz     r7, 64*2($inp)
2262         lwz     r8, 64*3($inp)
2263         lwz     r9, 64*4($inp)
2264         lwz     r10,64*5($inp)
2265         lwz     r11,64*6($inp)
2266         lwz     r12,64*7($inp)
2267         sldi    r9, r9, 32
2268         sldi    r10,r10,32
2269         sldi    r11,r11,32
2270         sldi    r12,r12,32
2271         or      r5,r5,r9
2272         or      r6,r6,r10
2273         or      r7,r7,r11
2274         or      r8,r8,r12
2275         and     r5,r5,r0
2276         and     r6,r6,r0
2277         and     r7,r7,r0
2278         and     r8,r8,r0
2279         std     r5,64($out)             # Z
2280         std     r6,72($out)
2281         std     r7,80($out)
2282         std     r8,88($out)
2283
2284         blr
2285         .long   0
2286         .byte   0,12,0x14,0,0,0,3,0
2287         .long   0
2288 .size   ecp_nistz256_gather_w5,.-ecp_nistz256_gather_w5
2289
2290 ########################################################################
2291 # void  ecp_nistz256_scatter_w7(void *out, const P256_POINT_AFFINE *inp,
2292 #                               int index);
2293 .globl  ecp_nistz256_scatter_w7
2294 .align  4
2295 ecp_nistz256_scatter_w7:
2296         li      r0,8
2297         mtctr   r0
2298         add     $out,$out,$index
2299         subi    $inp,$inp,8
2300
2301 .Loop_scatter_w7:
2302         ldu     r0,8($inp)
2303         stb     r0,64*0($out)
2304         srdi    r0,r0,8
2305         stb     r0,64*1($out)
2306         srdi    r0,r0,8
2307         stb     r0,64*2($out)
2308         srdi    r0,r0,8
2309         stb     r0,64*3($out)
2310         srdi    r0,r0,8
2311         stb     r0,64*4($out)
2312         srdi    r0,r0,8
2313         stb     r0,64*5($out)
2314         srdi    r0,r0,8
2315         stb     r0,64*6($out)
2316         srdi    r0,r0,8
2317         stb     r0,64*7($out)
2318         addi    $out,$out,64*8
2319         bdnz    .Loop_scatter_w7
2320
2321         blr
2322         .long   0
2323         .byte   0,12,0x14,0,0,0,3,0
2324         .long   0
2325 .size   ecp_nistz256_scatter_w7,.-ecp_nistz256_scatter_w7
2326
2327 ########################################################################
2328 # void  ecp_nistz256_gather_w7(P256_POINT_AFFINE *out, const void *inp,
2329 #                               int index);
2330 .globl  ecp_nistz256_gather_w7
2331 .align  4
2332 ecp_nistz256_gather_w7:
2333         li      r0,8
2334         mtctr   r0
2335         neg     r0,$index
2336         sradi   r0,r0,63
2337
2338         add     $index,$index,r0
2339         add     $inp,$inp,$index
2340         subi    $out,$out,8
2341
2342 .Loop_gather_w7:
2343         lbz     r5, 64*0($inp)
2344         lbz     r6, 64*1($inp)
2345         lbz     r7, 64*2($inp)
2346         lbz     r8, 64*3($inp)
2347         lbz     r9, 64*4($inp)
2348         lbz     r10,64*5($inp)
2349         lbz     r11,64*6($inp)
2350         lbz     r12,64*7($inp)
2351         addi    $inp,$inp,64*8
2352
2353         sldi    r6, r6, 8
2354         sldi    r7, r7, 16
2355         sldi    r8, r8, 24
2356         sldi    r9, r9, 32
2357         sldi    r10,r10,40
2358         sldi    r11,r11,48
2359         sldi    r12,r12,56
2360
2361         or      r5,r5,r6
2362         or      r7,r7,r8
2363         or      r9,r9,r10
2364         or      r11,r11,r12
2365         or      r5,r5,r7
2366         or      r9,r9,r11
2367         or      r5,r5,r9
2368         and     r5,r5,r0
2369         stdu    r5,8($out)
2370         bdnz    .Loop_gather_w7
2371
2372         blr
2373         .long   0
2374         .byte   0,12,0x14,0,0,0,3,0
2375         .long   0
2376 .size   ecp_nistz256_gather_w7,.-ecp_nistz256_gather_w7
2377 ___
2378 }
2379
2380 foreach (split("\n",$code)) {
2381         s/\`([^\`]*)\`/eval $1/ge;
2382
2383         print $_,"\n";
2384 }
2385 close STDOUT or die "error closing STDOUT: $!"; # enforce flush