crypto/poly1305/asm/poly1305-ppc.pl

   1 #! /usr/bin/env perl
   2 # Copyright 2016-2018 The OpenSSL Project Authors. All Rights Reserved.
   3 #
   4 # Licensed under the OpenSSL license (the "License").  You may not use
   5 # this file except in compliance with the License.  You can obtain a copy
   6 # in the file LICENSE in the source distribution or at
   7 # https://www.openssl.org/source/license.html
   8
   9 #
  10 # ====================================================================
  11 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
  12 # project. The module is, however, dual licensed under OpenSSL and
  13 # CRYPTOGAMS licenses depending on where you obtain it. For further
  14 # details see http://www.openssl.org/~appro/cryptogams/.
  15 # ====================================================================
  16 #
  17 # This module implements Poly1305 hash for PowerPC.
  18 #
  19 # June 2015
  20 #
  21 # Numbers are cycles per processed byte with poly1305_blocks alone,
  22 # and improvement coefficients relative to gcc-generated code.
  23 #
  24 #                       -m32            -m64
  25 #
  26 # Freescale e300        14.8/+80%       -
  27 # PPC74x0               7.60/+60%       -
  28 # PPC970                7.00/+114%      3.51/+205%
  29 # POWER7                3.75/+260%      1.93/+100%
  30 # POWER8                -               2.03/+200%
  31 # POWER9                -               2.00/+150%
  32 #
  33 # Do we need floating-point implementation for PPC? Results presented
  34 # in poly1305_ieee754.c are tricky to compare to, because they are for
  35 # compiler-generated code. On the other hand it's known that floating-
  36 # point performance can be dominated by FPU latency, which means that
  37 # there is limit even for ideally optimized (and even vectorized) code.
  38 # And this limit is estimated to be higher than above -m64 results. Or
  39 # in other words floating-point implementation can be meaningful to
  40 # consider only in 32-bit application context. We probably have to
  41 # recognize that 32-bit builds are getting less popular on high-end
  42 # systems and therefore tend to target embedded ones, which might not
  43 # even have FPU...
  44 #
  45 # On side note, Power ISA 2.07 enables vector base 2^26 implementation,
  46 # and POWER8 might have capacity to break 1.0 cycle per byte barrier...
  47
  48 $flavour = shift;
  49
  50 if ($flavour =~ /64/) {
  51         $SIZE_T =8;
  52         $LRSAVE =2*$SIZE_T;
  53         $UCMP   ="cmpld";
  54         $STU    ="stdu";
  55         $POP    ="ld";
  56         $PUSH   ="std";
  57 } elsif ($flavour =~ /32/) {
  58         $SIZE_T =4;
  59         $LRSAVE =$SIZE_T;
  60         $UCMP   ="cmplw";
  61         $STU    ="stwu";
  62         $POP    ="lwz";
  63         $PUSH   ="stw";
  64 } else { die "nonsense $flavour"; }
  65
  66 # Define endianness based on flavour
  67 # i.e.: linux64le
  68 $LITTLE_ENDIAN = ($flavour=~/le$/) ? $SIZE_T : 0;
  69
  70 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  71 ( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
  72 ( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
  73 die "can't locate ppc-xlate.pl";
  74
  75 open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!";
  76
  77 $FRAME=24*$SIZE_T;
  78
  79 $sp="r1";
  80 my ($ctx,$inp,$len,$padbit) = map("r$_",(3..6));
  81 my ($mac,$nonce)=($inp,$len);
  82 my $mask = "r0";
  83
  84 $code=<<___;
  85 .machine        "any"
  86 .text
  87 ___
  88                                                         if ($flavour =~ /64/) {
  89 ###############################################################################
  90 # base 2^64 implementation
  91
  92 my ($h0,$h1,$h2,$d0,$d1,$d2, $r0,$r1,$s1, $t0,$t1) = map("r$_",(7..12,27..31));
  93
  94 $code.=<<___;
  95 .globl  .poly1305_init_int
  96 .align  4
  97 .poly1305_init_int:
  98         xor     r0,r0,r0
  99         std     r0,0($ctx)              # zero hash value
 100         std     r0,8($ctx)
 101         std     r0,16($ctx)
 102
 103         $UCMP   $inp,r0
 104         beq-    Lno_key
 105 ___
 106 $code.=<<___    if ($LITTLE_ENDIAN);
 107         ld      $d0,0($inp)             # load key material
 108         ld      $d1,8($inp)
 109 ___
 110 $code.=<<___    if (!$LITTLE_ENDIAN);
 111         li      $h0,4
 112         lwbrx   $d0,0,$inp              # load key material
 113         li      $d1,8
 114         lwbrx   $h0,$h0,$inp
 115         li      $h1,12
 116         lwbrx   $d1,$d1,$inp
 117         lwbrx   $h1,$h1,$inp
 118         insrdi  $d0,$h0,32,0
 119         insrdi  $d1,$h1,32,0
 120 ___
 121 $code.=<<___;
 122         lis     $h1,0xfff               # 0x0fff0000
 123         ori     $h1,$h1,0xfffc          # 0x0ffffffc
 124         insrdi  $h1,$h1,32,0            # 0x0ffffffc0ffffffc
 125         ori     $h0,$h1,3               # 0x0ffffffc0fffffff
 126
 127         and     $d0,$d0,$h0
 128         and     $d1,$d1,$h1
 129
 130         std     $d0,32($ctx)            # store key
 131         std     $d1,40($ctx)
 132
 133 Lno_key:
 134         xor     r3,r3,r3
 135         blr
 136         .long   0
 137         .byte   0,12,0x14,0,0,0,2,0
 138 .size   .poly1305_init_int,.-.poly1305_init_int
 139
 140 .globl  .poly1305_blocks
 141 .align  4
 142 .poly1305_blocks:
 143         srdi.   $len,$len,4
 144         beq-    Labort
 145
 146         $STU    $sp,-$FRAME($sp)
 147         mflr    r0
 148         $PUSH   r27,`$FRAME-$SIZE_T*5`($sp)
 149         $PUSH   r28,`$FRAME-$SIZE_T*4`($sp)
 150         $PUSH   r29,`$FRAME-$SIZE_T*3`($sp)
 151         $PUSH   r30,`$FRAME-$SIZE_T*2`($sp)
 152         $PUSH   r31,`$FRAME-$SIZE_T*1`($sp)
 153         $PUSH   r0,`$FRAME+$LRSAVE`($sp)
 154
 155         ld      $r0,32($ctx)            # load key
 156         ld      $r1,40($ctx)
 157
 158         ld      $h0,0($ctx)             # load hash value
 159         ld      $h1,8($ctx)
 160         ld      $h2,16($ctx)
 161
 162         srdi    $s1,$r1,2
 163         mtctr   $len
 164         add     $s1,$s1,$r1             # s1 = r1 + r1>>2
 165         li      $mask,3
 166         b       Loop
 167
 168 .align  4
 169 Loop:
 170 ___
 171 $code.=<<___    if ($LITTLE_ENDIAN);
 172         ld      $t0,0($inp)             # load input
 173         ld      $t1,8($inp)
 174 ___
 175 $code.=<<___    if (!$LITTLE_ENDIAN);
 176         li      $d0,4
 177         lwbrx   $t0,0,$inp              # load input
 178         li      $t1,8
 179         lwbrx   $d0,$d0,$inp
 180         li      $d1,12
 181         lwbrx   $t1,$t1,$inp
 182         lwbrx   $d1,$d1,$inp
 183         insrdi  $t0,$d0,32,0
 184         insrdi  $t1,$d1,32,0
 185 ___
 186 $code.=<<___;
 187         addi    $inp,$inp,16
 188
 189         addc    $h0,$h0,$t0             # accumulate input
 190         adde    $h1,$h1,$t1
 191
 192         mulld   $d0,$h0,$r0             # h0*r0
 193         mulhdu  $d1,$h0,$r0
 194         adde    $h2,$h2,$padbit
 195
 196         mulld   $t0,$h1,$s1             # h1*5*r1
 197         mulhdu  $t1,$h1,$s1
 198         addc    $d0,$d0,$t0
 199         adde    $d1,$d1,$t1
 200
 201         mulld   $t0,$h0,$r1             # h0*r1
 202         mulhdu  $d2,$h0,$r1
 203         addc    $d1,$d1,$t0
 204         addze   $d2,$d2
 205
 206         mulld   $t0,$h1,$r0             # h1*r0
 207         mulhdu  $t1,$h1,$r0
 208         addc    $d1,$d1,$t0
 209         adde    $d2,$d2,$t1
 210
 211         mulld   $t0,$h2,$s1             # h2*5*r1
 212         mulld   $t1,$h2,$r0             # h2*r0
 213         addc    $d1,$d1,$t0
 214         adde    $d2,$d2,$t1
 215
 216         andc    $t0,$d2,$mask           # final reduction step
 217         and     $h2,$d2,$mask
 218         srdi    $t1,$t0,2
 219         add     $t0,$t0,$t1
 220         addc    $h0,$d0,$t0
 221         addze   $h1,$d1
 222         addze   $h2,$h2
 223
 224         bdnz    Loop
 225
 226         std     $h0,0($ctx)             # store hash value
 227         std     $h1,8($ctx)
 228         std     $h2,16($ctx)
 229
 230         $POP    r27,`$FRAME-$SIZE_T*5`($sp)
 231         $POP    r28,`$FRAME-$SIZE_T*4`($sp)
 232         $POP    r29,`$FRAME-$SIZE_T*3`($sp)
 233         $POP    r30,`$FRAME-$SIZE_T*2`($sp)
 234         $POP    r31,`$FRAME-$SIZE_T*1`($sp)
 235         addi    $sp,$sp,$FRAME
 236 Labort:
 237         blr
 238         .long   0
 239         .byte   0,12,4,1,0x80,5,4,0
 240 .size   .poly1305_blocks,.-.poly1305_blocks
 241
 242 .globl  .poly1305_emit
 243 .align  4
 244 .poly1305_emit:
 245         ld      $h0,0($ctx)             # load hash
 246         ld      $h1,8($ctx)
 247         ld      $h2,16($ctx)
 248         ld      $padbit,0($nonce)       # load nonce
 249         ld      $nonce,8($nonce)
 250
 251         addic   $d0,$h0,5               # compare to modulus
 252         addze   $d1,$h1
 253         addze   $d2,$h2
 254
 255         srdi    $mask,$d2,2             # did it carry/borrow?
 256         neg     $mask,$mask
 257
 258         andc    $h0,$h0,$mask
 259         and     $d0,$d0,$mask
 260         andc    $h1,$h1,$mask
 261         and     $d1,$d1,$mask
 262         or      $h0,$h0,$d0
 263         or      $h1,$h1,$d1
 264 ___
 265 $code.=<<___    if (!$LITTLE_ENDIAN);
 266         rotldi  $padbit,$padbit,32      # flip nonce words
 267         rotldi  $nonce,$nonce,32
 268 ___
 269 $code.=<<___;
 270         addc    $h0,$h0,$padbit         # accumulate nonce
 271         adde    $h1,$h1,$nonce
 272 ___
 273 $code.=<<___    if ($LITTLE_ENDIAN);
 274         std     $h0,0($mac)             # write result
 275         std     $h1,8($mac)
 276 ___
 277 $code.=<<___    if (!$LITTLE_ENDIAN);
 278         extrdi  r0,$h0,32,0
 279         li      $d0,4
 280         stwbrx  $h0,0,$mac              # write result
 281         extrdi  $h0,$h1,32,0
 282         li      $d1,8
 283         stwbrx  r0,$d0,$mac
 284         li      $d2,12
 285         stwbrx  $h1,$d1,$mac
 286         stwbrx  $h0,$d2,$mac
 287 ___
 288 $code.=<<___;
 289         blr
 290         .long   0
 291         .byte   0,12,0x14,0,0,0,3,0
 292 .size   .poly1305_emit,.-.poly1305_emit
 293 ___
 294                                                         } else {
 295 ###############################################################################
 296 # base 2^32 implementation
 297
 298 my ($h0,$h1,$h2,$h3,$h4, $r0,$r1,$r2,$r3, $s1,$s2,$s3,
 299     $t0,$t1,$t2,$t3, $D0,$D1,$D2,$D3, $d0,$d1,$d2,$d3
 300    ) = map("r$_",(7..12,14..31));
 301
 302 $code.=<<___;
 303 .globl  .poly1305_init_int
 304 .align  4
 305 .poly1305_init_int:
 306         xor     r0,r0,r0
 307         stw     r0,0($ctx)              # zero hash value
 308         stw     r0,4($ctx)
 309         stw     r0,8($ctx)
 310         stw     r0,12($ctx)
 311         stw     r0,16($ctx)
 312
 313         $UCMP   $inp,r0
 314         beq-    Lno_key
 315 ___
 316 $code.=<<___    if ($LITTLE_ENDIAN);
 317         lw      $h0,0($inp)             # load key material
 318         lw      $h1,4($inp)
 319         lw      $h2,8($inp)
 320         lw      $h3,12($inp)
 321 ___
 322 $code.=<<___    if (!$LITTLE_ENDIAN);
 323         li      $h1,4
 324         lwbrx   $h0,0,$inp              # load key material
 325         li      $h2,8
 326         lwbrx   $h1,$h1,$inp
 327         li      $h3,12
 328         lwbrx   $h2,$h2,$inp
 329         lwbrx   $h3,$h3,$inp
 330 ___
 331 $code.=<<___;
 332         lis     $mask,0xf000            # 0xf0000000
 333         li      $r0,-4
 334         andc    $r0,$r0,$mask           # 0x0ffffffc
 335
 336         andc    $h0,$h0,$mask
 337         and     $h1,$h1,$r0
 338         and     $h2,$h2,$r0
 339         and     $h3,$h3,$r0
 340
 341         stw     $h0,32($ctx)            # store key
 342         stw     $h1,36($ctx)
 343         stw     $h2,40($ctx)
 344         stw     $h3,44($ctx)
 345
 346 Lno_key:
 347         xor     r3,r3,r3
 348         blr
 349         .long   0
 350         .byte   0,12,0x14,0,0,0,2,0
 351 .size   .poly1305_init_int,.-.poly1305_init_int
 352
 353 .globl  .poly1305_blocks
 354 .align  4
 355 .poly1305_blocks:
 356         srwi.   $len,$len,4
 357         beq-    Labort
 358
 359         $STU    $sp,-$FRAME($sp)
 360         mflr    r0
 361         $PUSH   r14,`$FRAME-$SIZE_T*18`($sp)
 362         $PUSH   r15,`$FRAME-$SIZE_T*17`($sp)
 363         $PUSH   r16,`$FRAME-$SIZE_T*16`($sp)
 364         $PUSH   r17,`$FRAME-$SIZE_T*15`($sp)
 365         $PUSH   r18,`$FRAME-$SIZE_T*14`($sp)
 366         $PUSH   r19,`$FRAME-$SIZE_T*13`($sp)
 367         $PUSH   r20,`$FRAME-$SIZE_T*12`($sp)
 368         $PUSH   r21,`$FRAME-$SIZE_T*11`($sp)
 369         $PUSH   r22,`$FRAME-$SIZE_T*10`($sp)
 370         $PUSH   r23,`$FRAME-$SIZE_T*9`($sp)
 371         $PUSH   r24,`$FRAME-$SIZE_T*8`($sp)
 372         $PUSH   r25,`$FRAME-$SIZE_T*7`($sp)
 373         $PUSH   r26,`$FRAME-$SIZE_T*6`($sp)
 374         $PUSH   r27,`$FRAME-$SIZE_T*5`($sp)
 375         $PUSH   r28,`$FRAME-$SIZE_T*4`($sp)
 376         $PUSH   r29,`$FRAME-$SIZE_T*3`($sp)
 377         $PUSH   r30,`$FRAME-$SIZE_T*2`($sp)
 378         $PUSH   r31,`$FRAME-$SIZE_T*1`($sp)
 379         $PUSH   r0,`$FRAME+$LRSAVE`($sp)
 380
 381         lwz     $r0,32($ctx)            # load key
 382         lwz     $r1,36($ctx)
 383         lwz     $r2,40($ctx)
 384         lwz     $r3,44($ctx)
 385
 386         lwz     $h0,0($ctx)             # load hash value
 387         lwz     $h1,4($ctx)
 388         lwz     $h2,8($ctx)
 389         lwz     $h3,12($ctx)
 390         lwz     $h4,16($ctx)
 391
 392         srwi    $s1,$r1,2
 393         srwi    $s2,$r2,2
 394         srwi    $s3,$r3,2
 395         add     $s1,$s1,$r1             # si = ri + ri>>2
 396         add     $s2,$s2,$r2
 397         add     $s3,$s3,$r3
 398         mtctr   $len
 399         li      $mask,3
 400         b       Loop
 401
 402 .align  4
 403 Loop:
 404 ___
 405 $code.=<<___    if ($LITTLE_ENDIAN);
 406         lwz     $d0,0($inp)             # load input
 407         lwz     $d1,4($inp)
 408         lwz     $d2,8($inp)
 409         lwz     $d3,12($inp)
 410 ___
 411 $code.=<<___    if (!$LITTLE_ENDIAN);
 412         li      $d1,4
 413         lwbrx   $d0,0,$inp              # load input
 414         li      $d2,8
 415         lwbrx   $d1,$d1,$inp
 416         li      $d3,12
 417         lwbrx   $d2,$d2,$inp
 418         lwbrx   $d3,$d3,$inp
 419 ___
 420 $code.=<<___;
 421         addi    $inp,$inp,16
 422
 423         addc    $h0,$h0,$d0             # accumulate input
 424         adde    $h1,$h1,$d1
 425         adde    $h2,$h2,$d2
 426
 427         mullw   $d0,$h0,$r0             # h0*r0
 428         mulhwu  $D0,$h0,$r0
 429
 430         mullw   $d1,$h0,$r1             # h0*r1
 431         mulhwu  $D1,$h0,$r1
 432
 433         mullw   $d2,$h0,$r2             # h0*r2
 434         mulhwu  $D2,$h0,$r2
 435
 436          adde   $h3,$h3,$d3
 437          adde   $h4,$h4,$padbit
 438
 439         mullw   $d3,$h0,$r3             # h0*r3
 440         mulhwu  $D3,$h0,$r3
 441
 442         mullw   $t0,$h1,$s3             # h1*s3
 443         mulhwu  $t1,$h1,$s3
 444
 445         mullw   $t2,$h1,$r0             # h1*r0
 446         mulhwu  $t3,$h1,$r0
 447          addc   $d0,$d0,$t0
 448          adde   $D0,$D0,$t1
 449
 450         mullw   $t0,$h1,$r1             # h1*r1
 451         mulhwu  $t1,$h1,$r1
 452          addc   $d1,$d1,$t2
 453          adde   $D1,$D1,$t3
 454
 455         mullw   $t2,$h1,$r2             # h1*r2
 456         mulhwu  $t3,$h1,$r2
 457          addc   $d2,$d2,$t0
 458          adde   $D2,$D2,$t1
 459
 460         mullw   $t0,$h2,$s2             # h2*s2
 461         mulhwu  $t1,$h2,$s2
 462          addc   $d3,$d3,$t2
 463          adde   $D3,$D3,$t3
 464
 465         mullw   $t2,$h2,$s3             # h2*s3
 466         mulhwu  $t3,$h2,$s3
 467          addc   $d0,$d0,$t0
 468          adde   $D0,$D0,$t1
 469
 470         mullw   $t0,$h2,$r0             # h2*r0
 471         mulhwu  $t1,$h2,$r0
 472          addc   $d1,$d1,$t2
 473          adde   $D1,$D1,$t3
 474
 475         mullw   $t2,$h2,$r1             # h2*r1
 476         mulhwu  $t3,$h2,$r1
 477          addc   $d2,$d2,$t0
 478          adde   $D2,$D2,$t1
 479
 480         mullw   $t0,$h3,$s1             # h3*s1
 481         mulhwu  $t1,$h3,$s1
 482          addc   $d3,$d3,$t2
 483          adde   $D3,$D3,$t3
 484
 485         mullw   $t2,$h3,$s2             # h3*s2
 486         mulhwu  $t3,$h3,$s2
 487          addc   $d0,$d0,$t0
 488          adde   $D0,$D0,$t1
 489
 490         mullw   $t0,$h3,$s3             # h3*s3
 491         mulhwu  $t1,$h3,$s3
 492          addc   $d1,$d1,$t2
 493          adde   $D1,$D1,$t3
 494
 495         mullw   $t2,$h3,$r0             # h3*r0
 496         mulhwu  $t3,$h3,$r0
 497          addc   $d2,$d2,$t0
 498          adde   $D2,$D2,$t1
 499
 500         mullw   $t0,$h4,$s1             # h4*s1
 501          addc   $d3,$d3,$t2
 502          adde   $D3,$D3,$t3
 503         addc    $d1,$d1,$t0
 504
 505         mullw   $t1,$h4,$s2             # h4*s2
 506          addze  $D1,$D1
 507         addc    $d2,$d2,$t1
 508         addze   $D2,$D2
 509
 510         mullw   $t2,$h4,$s3             # h4*s3
 511         addc    $d3,$d3,$t2
 512         addze   $D3,$D3
 513
 514         mullw   $h4,$h4,$r0             # h4*r0
 515
 516         addc    $h1,$d1,$D0
 517         adde    $h2,$d2,$D1
 518         adde    $h3,$d3,$D2
 519         adde    $h4,$h4,$D3
 520
 521         andc    $D0,$h4,$mask           # final reduction step
 522         and     $h4,$h4,$mask
 523         srwi    $D1,$D0,2
 524         add     $D0,$D0,$D1
 525         addc    $h0,$d0,$D0
 526         addze   $h1,$h1
 527         addze   $h2,$h2
 528         addze   $h3,$h3
 529         addze   $h4,$h4
 530
 531         bdnz    Loop
 532
 533         stw     $h0,0($ctx)             # store hash value
 534         stw     $h1,4($ctx)
 535         stw     $h2,8($ctx)
 536         stw     $h3,12($ctx)
 537         stw     $h4,16($ctx)
 538
 539         $POP    r14,`$FRAME-$SIZE_T*18`($sp)
 540         $POP    r15,`$FRAME-$SIZE_T*17`($sp)
 541         $POP    r16,`$FRAME-$SIZE_T*16`($sp)
 542         $POP    r17,`$FRAME-$SIZE_T*15`($sp)
 543         $POP    r18,`$FRAME-$SIZE_T*14`($sp)
 544         $POP    r19,`$FRAME-$SIZE_T*13`($sp)
 545         $POP    r20,`$FRAME-$SIZE_T*12`($sp)
 546         $POP    r21,`$FRAME-$SIZE_T*11`($sp)
 547         $POP    r22,`$FRAME-$SIZE_T*10`($sp)
 548         $POP    r23,`$FRAME-$SIZE_T*9`($sp)
 549         $POP    r24,`$FRAME-$SIZE_T*8`($sp)
 550         $POP    r25,`$FRAME-$SIZE_T*7`($sp)
 551         $POP    r26,`$FRAME-$SIZE_T*6`($sp)
 552         $POP    r27,`$FRAME-$SIZE_T*5`($sp)
 553         $POP    r28,`$FRAME-$SIZE_T*4`($sp)
 554         $POP    r29,`$FRAME-$SIZE_T*3`($sp)
 555         $POP    r30,`$FRAME-$SIZE_T*2`($sp)
 556         $POP    r31,`$FRAME-$SIZE_T*1`($sp)
 557         addi    $sp,$sp,$FRAME
 558 Labort:
 559         blr
 560         .long   0
 561         .byte   0,12,4,1,0x80,18,4,0
 562 .size   .poly1305_blocks,.-.poly1305_blocks
 563
 564 .globl  .poly1305_emit
 565 .align  4
 566 .poly1305_emit:
 567         $STU    $sp,-$FRAME($sp)
 568         mflr    r0
 569         $PUSH   r28,`$FRAME-$SIZE_T*4`($sp)
 570         $PUSH   r29,`$FRAME-$SIZE_T*3`($sp)
 571         $PUSH   r30,`$FRAME-$SIZE_T*2`($sp)
 572         $PUSH   r31,`$FRAME-$SIZE_T*1`($sp)
 573         $PUSH   r0,`$FRAME+$LRSAVE`($sp)
 574
 575         lwz     $h0,0($ctx)             # load hash
 576         lwz     $h1,4($ctx)
 577         lwz     $h2,8($ctx)
 578         lwz     $h3,12($ctx)
 579         lwz     $h4,16($ctx)
 580
 581         addic   $d0,$h0,5               # compare to modulus
 582         addze   $d1,$h1
 583         addze   $d2,$h2
 584         addze   $d3,$h3
 585         addze   $mask,$h4
 586
 587         srwi    $mask,$mask,2           # did it carry/borrow?
 588         neg     $mask,$mask
 589
 590         andc    $h0,$h0,$mask
 591         and     $d0,$d0,$mask
 592         andc    $h1,$h1,$mask
 593         and     $d1,$d1,$mask
 594         or      $h0,$h0,$d0
 595         lwz     $d0,0($nonce)           # load nonce
 596         andc    $h2,$h2,$mask
 597         and     $d2,$d2,$mask
 598         or      $h1,$h1,$d1
 599         lwz     $d1,4($nonce)
 600         andc    $h3,$h3,$mask
 601         and     $d3,$d3,$mask
 602         or      $h2,$h2,$d2
 603         lwz     $d2,8($nonce)
 604         or      $h3,$h3,$d3
 605         lwz     $d3,12($nonce)
 606
 607         addc    $h0,$h0,$d0             # accumulate nonce
 608         adde    $h1,$h1,$d1
 609         adde    $h2,$h2,$d2
 610         adde    $h3,$h3,$d3
 611 ___
 612 $code.=<<___    if ($LITTLE_ENDIAN);
 613         stw     $h0,0($mac)             # write result
 614         stw     $h1,4($mac)
 615         stw     $h2,8($mac)
 616         stw     $h3,12($mac)
 617 ___
 618 $code.=<<___    if (!$LITTLE_ENDIAN);
 619         li      $d1,4
 620         stwbrx  $h0,0,$mac              # write result
 621         li      $d2,8
 622         stwbrx  $h1,$d1,$mac
 623         li      $d3,12
 624         stwbrx  $h2,$d2,$mac
 625         stwbrx  $h3,$d3,$mac
 626 ___
 627 $code.=<<___;
 628         $POP    r28,`$FRAME-$SIZE_T*4`($sp)
 629         $POP    r29,`$FRAME-$SIZE_T*3`($sp)
 630         $POP    r30,`$FRAME-$SIZE_T*2`($sp)
 631         $POP    r31,`$FRAME-$SIZE_T*1`($sp)
 632         addi    $sp,$sp,$FRAME
 633         blr
 634         .long   0
 635         .byte   0,12,4,1,0x80,4,3,0
 636 .size   .poly1305_emit,.-.poly1305_emit
 637 ___
 638                                                         }
 639 $code.=<<___;
 640 .asciz  "Poly1305 for PPC, CRYPTOGAMS by <appro\@openssl.org>"
 641 ___
 642
 643 $code =~ s/\`([^\`]*)\`/eval $1/gem;
 644 print $code;
 645 close STDOUT;