crypto/sha/asm/sha512-armv4.pl

   1 #! /usr/bin/env perl
   2 # Copyright 2007-2023 The OpenSSL Project Authors. All Rights Reserved.
   3 #
   4 # Licensed under the Apache License 2.0 (the "License").  You may not use
   5 # this file except in compliance with the License.  You can obtain a copy
   6 # in the file LICENSE in the source distribution or at
   7 # https://www.openssl.org/source/license.html
   8
   9
  10 # ====================================================================
  11 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
  12 # project. The module is, however, dual licensed under OpenSSL and
  13 # CRYPTOGAMS licenses depending on where you obtain it. For further
  14 # details see http://www.openssl.org/~appro/cryptogams/.
  15 #
  16 # Permission to use under GPL terms is granted.
  17 # ====================================================================
  18
  19 # SHA512 block procedure for ARMv4. September 2007.
  20
  21 # This code is ~4.5 (four and a half) times faster than code generated
  22 # by gcc 3.4 and it spends ~72 clock cycles per byte [on single-issue
  23 # Xscale PXA250 core].
  24 #
  25 # July 2010.
  26 #
  27 # Rescheduling for dual-issue pipeline resulted in 6% improvement on
  28 # Cortex A8 core and ~40 cycles per processed byte.
  29
  30 # February 2011.
  31 #
  32 # Profiler-assisted and platform-specific optimization resulted in 7%
  33 # improvement on Coxtex A8 core and ~38 cycles per byte.
  34
  35 # March 2011.
  36 #
  37 # Add NEON implementation. On Cortex A8 it was measured to process
  38 # one byte in 23.3 cycles or ~60% faster than integer-only code.
  39
  40 # August 2012.
  41 #
  42 # Improve NEON performance by 12% on Snapdragon S4. In absolute
  43 # terms it's 22.6 cycles per byte, which is disappointing result.
  44 # Technical writers asserted that 3-way S4 pipeline can sustain
  45 # multiple NEON instructions per cycle, but dual NEON issue could
  46 # not be observed, see http://www.openssl.org/~appro/Snapdragon-S4.html
  47 # for further details. On side note Cortex-A15 processes one byte in
  48 # 16 cycles.
  49
  50 # Byte order [in]dependence. =========================================
  51 #
  52 # Originally caller was expected to maintain specific *dword* order in
  53 # h[0-7], namely with most significant dword at *lower* address, which
  54 # was reflected in below two parameters as 0 and 4. Now caller is
  55 # expected to maintain native byte order for whole 64-bit values.
  56 $hi="HI";
  57 $lo="LO";
  58 # ====================================================================
  59
  60 # $output is the last argument if it looks like a file (it has an extension)
  61 # $flavour is the first argument if it doesn't look like a file
  62 $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
  63 $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
  64
  65 if ($flavour && $flavour ne "void") {
  66     $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  67     ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
  68     ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
  69     die "can't locate arm-xlate.pl";
  70
  71     open STDOUT,"| \"$^X\" $xlate $flavour \"$output\""
  72         or die "can't call $xlate: $!";
  73 } else {
  74     $output and open STDOUT,">$output";
  75 }
  76
  77 $ctx="r0";      # parameter block
  78 $inp="r1";
  79 $len="r2";
  80
  81 $Tlo="r3";
  82 $Thi="r4";
  83 $Alo="r5";
  84 $Ahi="r6";
  85 $Elo="r7";
  86 $Ehi="r8";
  87 $t0="r9";
  88 $t1="r10";
  89 $t2="r11";
  90 $t3="r12";
  91 ############    r13 is stack pointer
  92 $Ktbl="r14";
  93 ############    r15 is program counter
  94
  95 $Aoff=8*0;
  96 $Boff=8*1;
  97 $Coff=8*2;
  98 $Doff=8*3;
  99 $Eoff=8*4;
 100 $Foff=8*5;
 101 $Goff=8*6;
 102 $Hoff=8*7;
 103 $Xoff=8*8;
 104
 105 sub BODY_00_15() {
 106 my $magic = shift;
 107 $code.=<<___;
 108         @ Sigma1(x)     (ROTR((x),14) ^ ROTR((x),18)  ^ ROTR((x),41))
 109         @ LO            lo>>14^hi<<18 ^ lo>>18^hi<<14 ^ hi>>9^lo<<23
 110         @ HI            hi>>14^lo<<18 ^ hi>>18^lo<<14 ^ lo>>9^hi<<23
 111         mov     $t0,$Elo,lsr#14
 112         str     $Tlo,[sp,#$Xoff+0]
 113         mov     $t1,$Ehi,lsr#14
 114         str     $Thi,[sp,#$Xoff+4]
 115         eor     $t0,$t0,$Ehi,lsl#18
 116         ldr     $t2,[sp,#$Hoff+0]       @ h.lo
 117         eor     $t1,$t1,$Elo,lsl#18
 118         ldr     $t3,[sp,#$Hoff+4]       @ h.hi
 119         eor     $t0,$t0,$Elo,lsr#18
 120         eor     $t1,$t1,$Ehi,lsr#18
 121         eor     $t0,$t0,$Ehi,lsl#14
 122         eor     $t1,$t1,$Elo,lsl#14
 123         eor     $t0,$t0,$Ehi,lsr#9
 124         eor     $t1,$t1,$Elo,lsr#9
 125         eor     $t0,$t0,$Elo,lsl#23
 126         eor     $t1,$t1,$Ehi,lsl#23     @ Sigma1(e)
 127         adds    $Tlo,$Tlo,$t0
 128         ldr     $t0,[sp,#$Foff+0]       @ f.lo
 129         adc     $Thi,$Thi,$t1           @ T += Sigma1(e)
 130         ldr     $t1,[sp,#$Foff+4]       @ f.hi
 131         adds    $Tlo,$Tlo,$t2
 132         ldr     $t2,[sp,#$Goff+0]       @ g.lo
 133         adc     $Thi,$Thi,$t3           @ T += h
 134         ldr     $t3,[sp,#$Goff+4]       @ g.hi
 135
 136         eor     $t0,$t0,$t2
 137         str     $Elo,[sp,#$Eoff+0]
 138         eor     $t1,$t1,$t3
 139         str     $Ehi,[sp,#$Eoff+4]
 140         and     $t0,$t0,$Elo
 141         str     $Alo,[sp,#$Aoff+0]
 142         and     $t1,$t1,$Ehi
 143         str     $Ahi,[sp,#$Aoff+4]
 144         eor     $t0,$t0,$t2
 145         ldr     $t2,[$Ktbl,#$lo]        @ K[i].lo
 146         eor     $t1,$t1,$t3             @ Ch(e,f,g)
 147         ldr     $t3,[$Ktbl,#$hi]        @ K[i].hi
 148
 149         adds    $Tlo,$Tlo,$t0
 150         ldr     $Elo,[sp,#$Doff+0]      @ d.lo
 151         adc     $Thi,$Thi,$t1           @ T += Ch(e,f,g)
 152         ldr     $Ehi,[sp,#$Doff+4]      @ d.hi
 153         adds    $Tlo,$Tlo,$t2
 154         and     $t0,$t2,#0xff
 155         adc     $Thi,$Thi,$t3           @ T += K[i]
 156         adds    $Elo,$Elo,$Tlo
 157         ldr     $t2,[sp,#$Boff+0]       @ b.lo
 158         adc     $Ehi,$Ehi,$Thi          @ d += T
 159         teq     $t0,#$magic
 160
 161         ldr     $t3,[sp,#$Coff+0]       @ c.lo
 162 #ifdef  __thumb2__
 163         it      eq                      @ Thumb2 thing, sanity check in ARM
 164 #endif
 165         orreq   $Ktbl,$Ktbl,#1
 166         @ Sigma0(x)     (ROTR((x),28) ^ ROTR((x),34) ^ ROTR((x),39))
 167         @ LO            lo>>28^hi<<4  ^ hi>>2^lo<<30 ^ hi>>7^lo<<25
 168         @ HI            hi>>28^lo<<4  ^ lo>>2^hi<<30 ^ lo>>7^hi<<25
 169         mov     $t0,$Alo,lsr#28
 170         mov     $t1,$Ahi,lsr#28
 171         eor     $t0,$t0,$Ahi,lsl#4
 172         eor     $t1,$t1,$Alo,lsl#4
 173         eor     $t0,$t0,$Ahi,lsr#2
 174         eor     $t1,$t1,$Alo,lsr#2
 175         eor     $t0,$t0,$Alo,lsl#30
 176         eor     $t1,$t1,$Ahi,lsl#30
 177         eor     $t0,$t0,$Ahi,lsr#7
 178         eor     $t1,$t1,$Alo,lsr#7
 179         eor     $t0,$t0,$Alo,lsl#25
 180         eor     $t1,$t1,$Ahi,lsl#25     @ Sigma0(a)
 181         adds    $Tlo,$Tlo,$t0
 182         and     $t0,$Alo,$t2
 183         adc     $Thi,$Thi,$t1           @ T += Sigma0(a)
 184
 185         ldr     $t1,[sp,#$Boff+4]       @ b.hi
 186         orr     $Alo,$Alo,$t2
 187         ldr     $t2,[sp,#$Coff+4]       @ c.hi
 188         and     $Alo,$Alo,$t3
 189         and     $t3,$Ahi,$t1
 190         orr     $Ahi,$Ahi,$t1
 191         orr     $Alo,$Alo,$t0           @ Maj(a,b,c).lo
 192         and     $Ahi,$Ahi,$t2
 193         adds    $Alo,$Alo,$Tlo
 194         orr     $Ahi,$Ahi,$t3           @ Maj(a,b,c).hi
 195         sub     sp,sp,#8
 196         adc     $Ahi,$Ahi,$Thi          @ h += T
 197         tst     $Ktbl,#1
 198         add     $Ktbl,$Ktbl,#8
 199 ___
 200 }
 201
 202 my $_word = ($flavour =~ /win/ ? "DCDU" : ".word");
 203
 204 $code=<<___;
 205 #ifndef __KERNEL__
 206 # include "arm_arch.h"
 207 # define VFP_ABI_PUSH   vstmdb  sp!,{d8-d15}
 208 # define VFP_ABI_POP    vldmia  sp!,{d8-d15}
 209 #else
 210 # define __ARM_ARCH__ __LINUX_ARM_ARCH__
 211 # define __ARM_MAX_ARCH__ 7
 212 # define VFP_ABI_PUSH
 213 # define VFP_ABI_POP
 214 #endif
 215
 216 #ifdef __ARMEL__
 217 # define LO 0
 218 # define HI 4
 219 # define WORD64(hi0,lo0,hi1,lo1)        $_word  lo0,hi0, lo1,hi1
 220 #else
 221 # define HI 0
 222 # define LO 4
 223 # define WORD64(hi0,lo0,hi1,lo1)        $_word  hi0,lo0, hi1,lo1
 224 #endif
 225
 226 #if defined(__thumb2__)
 227 .syntax unified
 228 .thumb
 229 # define adrl adr
 230 #else
 231 .code   32
 232 #endif
 233
 234 .text
 235
 236 .type   K512,%object
 237 .align  5
 238 K512:
 239 WORD64(0x428a2f98,0xd728ae22, 0x71374491,0x23ef65cd)
 240 WORD64(0xb5c0fbcf,0xec4d3b2f, 0xe9b5dba5,0x8189dbbc)
 241 WORD64(0x3956c25b,0xf348b538, 0x59f111f1,0xb605d019)
 242 WORD64(0x923f82a4,0xaf194f9b, 0xab1c5ed5,0xda6d8118)
 243 WORD64(0xd807aa98,0xa3030242, 0x12835b01,0x45706fbe)
 244 WORD64(0x243185be,0x4ee4b28c, 0x550c7dc3,0xd5ffb4e2)
 245 WORD64(0x72be5d74,0xf27b896f, 0x80deb1fe,0x3b1696b1)
 246 WORD64(0x9bdc06a7,0x25c71235, 0xc19bf174,0xcf692694)
 247 WORD64(0xe49b69c1,0x9ef14ad2, 0xefbe4786,0x384f25e3)
 248 WORD64(0x0fc19dc6,0x8b8cd5b5, 0x240ca1cc,0x77ac9c65)
 249 WORD64(0x2de92c6f,0x592b0275, 0x4a7484aa,0x6ea6e483)
 250 WORD64(0x5cb0a9dc,0xbd41fbd4, 0x76f988da,0x831153b5)
 251 WORD64(0x983e5152,0xee66dfab, 0xa831c66d,0x2db43210)
 252 WORD64(0xb00327c8,0x98fb213f, 0xbf597fc7,0xbeef0ee4)
 253 WORD64(0xc6e00bf3,0x3da88fc2, 0xd5a79147,0x930aa725)
 254 WORD64(0x06ca6351,0xe003826f, 0x14292967,0x0a0e6e70)
 255 WORD64(0x27b70a85,0x46d22ffc, 0x2e1b2138,0x5c26c926)
 256 WORD64(0x4d2c6dfc,0x5ac42aed, 0x53380d13,0x9d95b3df)
 257 WORD64(0x650a7354,0x8baf63de, 0x766a0abb,0x3c77b2a8)
 258 WORD64(0x81c2c92e,0x47edaee6, 0x92722c85,0x1482353b)
 259 WORD64(0xa2bfe8a1,0x4cf10364, 0xa81a664b,0xbc423001)
 260 WORD64(0xc24b8b70,0xd0f89791, 0xc76c51a3,0x0654be30)
 261 WORD64(0xd192e819,0xd6ef5218, 0xd6990624,0x5565a910)
 262 WORD64(0xf40e3585,0x5771202a, 0x106aa070,0x32bbd1b8)
 263 WORD64(0x19a4c116,0xb8d2d0c8, 0x1e376c08,0x5141ab53)
 264 WORD64(0x2748774c,0xdf8eeb99, 0x34b0bcb5,0xe19b48a8)
 265 WORD64(0x391c0cb3,0xc5c95a63, 0x4ed8aa4a,0xe3418acb)
 266 WORD64(0x5b9cca4f,0x7763e373, 0x682e6ff3,0xd6b2b8a3)
 267 WORD64(0x748f82ee,0x5defb2fc, 0x78a5636f,0x43172f60)
 268 WORD64(0x84c87814,0xa1f0ab72, 0x8cc70208,0x1a6439ec)
 269 WORD64(0x90befffa,0x23631e28, 0xa4506ceb,0xde82bde9)
 270 WORD64(0xbef9a3f7,0xb2c67915, 0xc67178f2,0xe372532b)
 271 WORD64(0xca273ece,0xea26619c, 0xd186b8c7,0x21c0c207)
 272 WORD64(0xeada7dd6,0xcde0eb1e, 0xf57d4f7f,0xee6ed178)
 273 WORD64(0x06f067aa,0x72176fba, 0x0a637dc5,0xa2c898a6)
 274 WORD64(0x113f9804,0xbef90dae, 0x1b710b35,0x131c471b)
 275 WORD64(0x28db77f5,0x23047d84, 0x32caab7b,0x40c72493)
 276 WORD64(0x3c9ebe0a,0x15c9bebc, 0x431d67c4,0x9c100d4c)
 277 WORD64(0x4cc5d4be,0xcb3e42b6, 0x597f299c,0xfc657e2a)
 278 WORD64(0x5fcb6fab,0x3ad6faec, 0x6c44198c,0x4a475817)
 279 .size   K512,.-K512
 280 #if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
 281 .LOPENSSL_armcap:
 282 # ifdef _WIN32
 283 .word   OPENSSL_armcap_P
 284 # else
 285 .word   OPENSSL_armcap_P-.Lsha512_block_data_order
 286 # endif
 287 .skip   32-4
 288 #else
 289 .skip   32
 290 #endif
 291
 292 .global sha512_block_data_order
 293 .type   sha512_block_data_order,%function
 294 sha512_block_data_order:
 295 .Lsha512_block_data_order:
 296 #if __ARM_ARCH__<7 && !defined(__thumb2__)
 297         sub     r3,pc,#8                @ sha512_block_data_order
 298 #else
 299         adr     r3,.Lsha512_block_data_order
 300 #endif
 301 #if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
 302         ldr     r12,.LOPENSSL_armcap
 303 # if !defined(_WIN32)
 304         ldr     r12,[r3,r12]            @ OPENSSL_armcap_P
 305 # endif
 306 # if defined(__APPLE__) || defined(_WIN32)
 307         ldr     r12,[r12]
 308 # endif
 309         tst     r12,#ARMV7_NEON
 310         bne     .LNEON
 311 #endif
 312         add     $len,$inp,$len,lsl#7    @ len to point at the end of inp
 313         stmdb   sp!,{r4-r12,lr}
 314         sub     $Ktbl,r3,#672           @ K512
 315         sub     sp,sp,#9*8
 316
 317         ldr     $Elo,[$ctx,#$Eoff+$lo]
 318         ldr     $Ehi,[$ctx,#$Eoff+$hi]
 319         ldr     $t0, [$ctx,#$Goff+$lo]
 320         ldr     $t1, [$ctx,#$Goff+$hi]
 321         ldr     $t2, [$ctx,#$Hoff+$lo]
 322         ldr     $t3, [$ctx,#$Hoff+$hi]
 323 .Loop:
 324         str     $t0, [sp,#$Goff+0]
 325         str     $t1, [sp,#$Goff+4]
 326         str     $t2, [sp,#$Hoff+0]
 327         str     $t3, [sp,#$Hoff+4]
 328         ldr     $Alo,[$ctx,#$Aoff+$lo]
 329         ldr     $Ahi,[$ctx,#$Aoff+$hi]
 330         ldr     $Tlo,[$ctx,#$Boff+$lo]
 331         ldr     $Thi,[$ctx,#$Boff+$hi]
 332         ldr     $t0, [$ctx,#$Coff+$lo]
 333         ldr     $t1, [$ctx,#$Coff+$hi]
 334         ldr     $t2, [$ctx,#$Doff+$lo]
 335         ldr     $t3, [$ctx,#$Doff+$hi]
 336         str     $Tlo,[sp,#$Boff+0]
 337         str     $Thi,[sp,#$Boff+4]
 338         str     $t0, [sp,#$Coff+0]
 339         str     $t1, [sp,#$Coff+4]
 340         str     $t2, [sp,#$Doff+0]
 341         str     $t3, [sp,#$Doff+4]
 342         ldr     $Tlo,[$ctx,#$Foff+$lo]
 343         ldr     $Thi,[$ctx,#$Foff+$hi]
 344         str     $Tlo,[sp,#$Foff+0]
 345         str     $Thi,[sp,#$Foff+4]
 346
 347 .L00_15:
 348 #if __ARM_ARCH__<7
 349         ldrb    $Tlo,[$inp,#7]
 350         ldrb    $t0, [$inp,#6]
 351         ldrb    $t1, [$inp,#5]
 352         ldrb    $t2, [$inp,#4]
 353         ldrb    $Thi,[$inp,#3]
 354         ldrb    $t3, [$inp,#2]
 355         orr     $Tlo,$Tlo,$t0,lsl#8
 356         ldrb    $t0, [$inp,#1]
 357         orr     $Tlo,$Tlo,$t1,lsl#16
 358         ldrb    $t1, [$inp],#8
 359         orr     $Tlo,$Tlo,$t2,lsl#24
 360         orr     $Thi,$Thi,$t3,lsl#8
 361         orr     $Thi,$Thi,$t0,lsl#16
 362         orr     $Thi,$Thi,$t1,lsl#24
 363 #else
 364         ldr     $Tlo,[$inp,#4]
 365         ldr     $Thi,[$inp],#8
 366 #ifdef __ARMEL__
 367         rev     $Tlo,$Tlo
 368         rev     $Thi,$Thi
 369 #endif
 370 #endif
 371 ___
 372         &BODY_00_15(0x94);
 373 $code.=<<___;
 374         tst     $Ktbl,#1
 375         beq     .L00_15
 376         ldr     $t0,[sp,#`$Xoff+8*(16-1)`+0]
 377         ldr     $t1,[sp,#`$Xoff+8*(16-1)`+4]
 378         bic     $Ktbl,$Ktbl,#1
 379 .L16_79:
 380         @ sigma0(x)     (ROTR((x),1)  ^ ROTR((x),8)  ^ ((x)>>7))
 381         @ LO            lo>>1^hi<<31  ^ lo>>8^hi<<24 ^ lo>>7^hi<<25
 382         @ HI            hi>>1^lo<<31  ^ hi>>8^lo<<24 ^ hi>>7
 383         mov     $Tlo,$t0,lsr#1
 384         ldr     $t2,[sp,#`$Xoff+8*(16-14)`+0]
 385         mov     $Thi,$t1,lsr#1
 386         ldr     $t3,[sp,#`$Xoff+8*(16-14)`+4]
 387         eor     $Tlo,$Tlo,$t1,lsl#31
 388         eor     $Thi,$Thi,$t0,lsl#31
 389         eor     $Tlo,$Tlo,$t0,lsr#8
 390         eor     $Thi,$Thi,$t1,lsr#8
 391         eor     $Tlo,$Tlo,$t1,lsl#24
 392         eor     $Thi,$Thi,$t0,lsl#24
 393         eor     $Tlo,$Tlo,$t0,lsr#7
 394         eor     $Thi,$Thi,$t1,lsr#7
 395         eor     $Tlo,$Tlo,$t1,lsl#25
 396
 397         @ sigma1(x)     (ROTR((x),19) ^ ROTR((x),61) ^ ((x)>>6))
 398         @ LO            lo>>19^hi<<13 ^ hi>>29^lo<<3 ^ lo>>6^hi<<26
 399         @ HI            hi>>19^lo<<13 ^ lo>>29^hi<<3 ^ hi>>6
 400         mov     $t0,$t2,lsr#19
 401         mov     $t1,$t3,lsr#19
 402         eor     $t0,$t0,$t3,lsl#13
 403         eor     $t1,$t1,$t2,lsl#13
 404         eor     $t0,$t0,$t3,lsr#29
 405         eor     $t1,$t1,$t2,lsr#29
 406         eor     $t0,$t0,$t2,lsl#3
 407         eor     $t1,$t1,$t3,lsl#3
 408         eor     $t0,$t0,$t2,lsr#6
 409         eor     $t1,$t1,$t3,lsr#6
 410         ldr     $t2,[sp,#`$Xoff+8*(16-9)`+0]
 411         eor     $t0,$t0,$t3,lsl#26
 412
 413         ldr     $t3,[sp,#`$Xoff+8*(16-9)`+4]
 414         adds    $Tlo,$Tlo,$t0
 415         ldr     $t0,[sp,#`$Xoff+8*16`+0]
 416         adc     $Thi,$Thi,$t1
 417
 418         ldr     $t1,[sp,#`$Xoff+8*16`+4]
 419         adds    $Tlo,$Tlo,$t2
 420         adc     $Thi,$Thi,$t3
 421         adds    $Tlo,$Tlo,$t0
 422         adc     $Thi,$Thi,$t1
 423 ___
 424         &BODY_00_15(0x17);
 425 $code.=<<___;
 426 #ifdef  __thumb2__
 427         ittt    eq                      @ Thumb2 thing, sanity check in ARM
 428 #endif
 429         ldreq   $t0,[sp,#`$Xoff+8*(16-1)`+0]
 430         ldreq   $t1,[sp,#`$Xoff+8*(16-1)`+4]
 431         beq     .L16_79
 432         bic     $Ktbl,$Ktbl,#1
 433
 434         ldr     $Tlo,[sp,#$Boff+0]
 435         ldr     $Thi,[sp,#$Boff+4]
 436         ldr     $t0, [$ctx,#$Aoff+$lo]
 437         ldr     $t1, [$ctx,#$Aoff+$hi]
 438         ldr     $t2, [$ctx,#$Boff+$lo]
 439         ldr     $t3, [$ctx,#$Boff+$hi]
 440         adds    $t0,$Alo,$t0
 441         str     $t0, [$ctx,#$Aoff+$lo]
 442         adc     $t1,$Ahi,$t1
 443         str     $t1, [$ctx,#$Aoff+$hi]
 444         adds    $t2,$Tlo,$t2
 445         str     $t2, [$ctx,#$Boff+$lo]
 446         adc     $t3,$Thi,$t3
 447         str     $t3, [$ctx,#$Boff+$hi]
 448
 449         ldr     $Alo,[sp,#$Coff+0]
 450         ldr     $Ahi,[sp,#$Coff+4]
 451         ldr     $Tlo,[sp,#$Doff+0]
 452         ldr     $Thi,[sp,#$Doff+4]
 453         ldr     $t0, [$ctx,#$Coff+$lo]
 454         ldr     $t1, [$ctx,#$Coff+$hi]
 455         ldr     $t2, [$ctx,#$Doff+$lo]
 456         ldr     $t3, [$ctx,#$Doff+$hi]
 457         adds    $t0,$Alo,$t0
 458         str     $t0, [$ctx,#$Coff+$lo]
 459         adc     $t1,$Ahi,$t1
 460         str     $t1, [$ctx,#$Coff+$hi]
 461         adds    $t2,$Tlo,$t2
 462         str     $t2, [$ctx,#$Doff+$lo]
 463         adc     $t3,$Thi,$t3
 464         str     $t3, [$ctx,#$Doff+$hi]
 465
 466         ldr     $Tlo,[sp,#$Foff+0]
 467         ldr     $Thi,[sp,#$Foff+4]
 468         ldr     $t0, [$ctx,#$Eoff+$lo]
 469         ldr     $t1, [$ctx,#$Eoff+$hi]
 470         ldr     $t2, [$ctx,#$Foff+$lo]
 471         ldr     $t3, [$ctx,#$Foff+$hi]
 472         adds    $Elo,$Elo,$t0
 473         str     $Elo,[$ctx,#$Eoff+$lo]
 474         adc     $Ehi,$Ehi,$t1
 475         str     $Ehi,[$ctx,#$Eoff+$hi]
 476         adds    $t2,$Tlo,$t2
 477         str     $t2, [$ctx,#$Foff+$lo]
 478         adc     $t3,$Thi,$t3
 479         str     $t3, [$ctx,#$Foff+$hi]
 480
 481         ldr     $Alo,[sp,#$Goff+0]
 482         ldr     $Ahi,[sp,#$Goff+4]
 483         ldr     $Tlo,[sp,#$Hoff+0]
 484         ldr     $Thi,[sp,#$Hoff+4]
 485         ldr     $t0, [$ctx,#$Goff+$lo]
 486         ldr     $t1, [$ctx,#$Goff+$hi]
 487         ldr     $t2, [$ctx,#$Hoff+$lo]
 488         ldr     $t3, [$ctx,#$Hoff+$hi]
 489         adds    $t0,$Alo,$t0
 490         str     $t0, [$ctx,#$Goff+$lo]
 491         adc     $t1,$Ahi,$t1
 492         str     $t1, [$ctx,#$Goff+$hi]
 493         adds    $t2,$Tlo,$t2
 494         str     $t2, [$ctx,#$Hoff+$lo]
 495         adc     $t3,$Thi,$t3
 496         str     $t3, [$ctx,#$Hoff+$hi]
 497
 498         add     sp,sp,#640
 499         sub     $Ktbl,$Ktbl,#640
 500
 501         teq     $inp,$len
 502         bne     .Loop
 503
 504         add     sp,sp,#8*9              @ destroy frame
 505 #if __ARM_ARCH__>=5
 506         ldmia   sp!,{r4-r12,pc}
 507 #else
 508         ldmia   sp!,{r4-r12,lr}
 509         tst     lr,#1
 510         moveq   pc,lr                   @ be binary compatible with V4, yet
 511         bx      lr                      @ interoperable with Thumb ISA:-)
 512 #endif
 513 .size   sha512_block_data_order,.-sha512_block_data_order
 514 ___
 515
 516 {
 517 my @Sigma0=(28,34,39);
 518 my @Sigma1=(14,18,41);
 519 my @sigma0=(1, 8, 7);
 520 my @sigma1=(19,61,6);
 521
 522 my $Ktbl="r3";
 523 my $cnt="r12";  # volatile register known as ip, intra-procedure-call scratch
 524
 525 my @X=map("d$_",(0..15));
 526 my @V=($A,$B,$C,$D,$E,$F,$G,$H)=map("d$_",(16..23));
 527
 528 sub NEON_00_15() {
 529 my $i=shift;
 530 my ($a,$b,$c,$d,$e,$f,$g,$h)=@_;
 531 my ($t0,$t1,$t2,$T1,$K,$Ch,$Maj)=map("d$_",(24..31));   # temps
 532
 533 $code.=<<___ if ($i<16 || $i&1);
 534         vshr.u64        $t0,$e,#@Sigma1[0]      @ $i
 535 #if $i<16
 536         vld1.64         {@X[$i%16]},[$inp]!     @ handles unaligned
 537 #endif
 538         vshr.u64        $t1,$e,#@Sigma1[1]
 539 #if $i>0
 540          vadd.i64       $a,$Maj                 @ h+=Maj from the past
 541 #endif
 542         vshr.u64        $t2,$e,#@Sigma1[2]
 543 ___
 544 $code.=<<___;
 545         vld1.64         {$K},[$Ktbl,:64]!       @ K[i++]
 546         vsli.64         $t0,$e,#`64-@Sigma1[0]`
 547         vsli.64         $t1,$e,#`64-@Sigma1[1]`
 548         vmov            $Ch,$e
 549         vsli.64         $t2,$e,#`64-@Sigma1[2]`
 550 #if $i<16 && defined(__ARMEL__)
 551         vrev64.8        @X[$i],@X[$i]
 552 #endif
 553         veor            $t1,$t0
 554         vbsl            $Ch,$f,$g               @ Ch(e,f,g)
 555         vshr.u64        $t0,$a,#@Sigma0[0]
 556         veor            $t2,$t1                 @ Sigma1(e)
 557         vadd.i64        $T1,$Ch,$h
 558         vshr.u64        $t1,$a,#@Sigma0[1]
 559         vsli.64         $t0,$a,#`64-@Sigma0[0]`
 560         vadd.i64        $T1,$t2
 561         vshr.u64        $t2,$a,#@Sigma0[2]
 562         vadd.i64        $K,@X[$i%16]
 563         vsli.64         $t1,$a,#`64-@Sigma0[1]`
 564         veor            $Maj,$a,$b
 565         vsli.64         $t2,$a,#`64-@Sigma0[2]`
 566         veor            $h,$t0,$t1
 567         vadd.i64        $T1,$K
 568         vbsl            $Maj,$c,$b              @ Maj(a,b,c)
 569         veor            $h,$t2                  @ Sigma0(a)
 570         vadd.i64        $d,$T1
 571         vadd.i64        $Maj,$T1
 572         @ vadd.i64      $h,$Maj
 573 ___
 574 }
 575
 576 sub NEON_16_79() {
 577 my $i=shift;
 578
 579 if ($i&1)       { &NEON_00_15($i,@_); return; }
 580
 581 # 2x-vectorized, therefore runs every 2nd round
 582 my @X=map("q$_",(0..7));                        # view @X as 128-bit vector
 583 my ($t0,$t1,$s0,$s1) = map("q$_",(12..15));     # temps
 584 my ($d0,$d1,$d2) = map("d$_",(24..26));         # temps from NEON_00_15
 585 my $e=@_[4];                                    # $e from NEON_00_15
 586 $i /= 2;
 587 $code.=<<___;
 588         vshr.u64        $t0,@X[($i+7)%8],#@sigma1[0]
 589         vshr.u64        $t1,@X[($i+7)%8],#@sigma1[1]
 590          vadd.i64       @_[0],d30                       @ h+=Maj from the past
 591         vshr.u64        $s1,@X[($i+7)%8],#@sigma1[2]
 592         vsli.64         $t0,@X[($i+7)%8],#`64-@sigma1[0]`
 593         vext.8          $s0,@X[$i%8],@X[($i+1)%8],#8    @ X[i+1]
 594         vsli.64         $t1,@X[($i+7)%8],#`64-@sigma1[1]`
 595         veor            $s1,$t0
 596         vshr.u64        $t0,$s0,#@sigma0[0]
 597         veor            $s1,$t1                         @ sigma1(X[i+14])
 598         vshr.u64        $t1,$s0,#@sigma0[1]
 599         vadd.i64        @X[$i%8],$s1
 600         vshr.u64        $s1,$s0,#@sigma0[2]
 601         vsli.64         $t0,$s0,#`64-@sigma0[0]`
 602         vsli.64         $t1,$s0,#`64-@sigma0[1]`
 603         vext.8          $s0,@X[($i+4)%8],@X[($i+5)%8],#8        @ X[i+9]
 604         veor            $s1,$t0
 605         vshr.u64        $d0,$e,#@Sigma1[0]              @ from NEON_00_15
 606         vadd.i64        @X[$i%8],$s0
 607         vshr.u64        $d1,$e,#@Sigma1[1]              @ from NEON_00_15
 608         veor            $s1,$t1                         @ sigma0(X[i+1])
 609         vshr.u64        $d2,$e,#@Sigma1[2]              @ from NEON_00_15
 610         vadd.i64        @X[$i%8],$s1
 611 ___
 612         &NEON_00_15(2*$i,@_);
 613 }
 614
 615 $code.=<<___;
 616 #if __ARM_MAX_ARCH__>=7
 617 .arch   armv7-a
 618 .fpu    neon
 619
 620 .global sha512_block_data_order_neon
 621 .type   sha512_block_data_order_neon,%function
 622 .align  4
 623 sha512_block_data_order_neon:
 624 .LNEON:
 625         dmb                             @ errata #451034 on early Cortex A8
 626         add     $len,$inp,$len,lsl#7    @ len to point at the end of inp
 627         adr     $Ktbl,K512
 628         VFP_ABI_PUSH
 629         vldmia  $ctx,{$A-$H}            @ load context
 630 .Loop_neon:
 631 ___
 632 for($i=0;$i<16;$i++)    { &NEON_00_15($i,@V); unshift(@V,pop(@V)); }
 633 $code.=<<___;
 634         mov             $cnt,#4
 635 .L16_79_neon:
 636         subs            $cnt,#1
 637 ___
 638 for(;$i<32;$i++)        { &NEON_16_79($i,@V); unshift(@V,pop(@V)); }
 639 $code.=<<___;
 640         bne             .L16_79_neon
 641
 642          vadd.i64       $A,d30          @ h+=Maj from the past
 643         vldmia          $ctx,{d24-d31}  @ load context to temp
 644         vadd.i64        q8,q12          @ vectorized accumulate
 645         vadd.i64        q9,q13
 646         vadd.i64        q10,q14
 647         vadd.i64        q11,q15
 648         vstmia          $ctx,{$A-$H}    @ save context
 649         teq             $inp,$len
 650         sub             $Ktbl,#640      @ rewind K512
 651         bne             .Loop_neon
 652
 653         VFP_ABI_POP
 654         ret                             @ bx lr
 655 .size   sha512_block_data_order_neon,.-sha512_block_data_order_neon
 656 #endif
 657 ___
 658 }
 659 $code.=<<___;
 660 .asciz  "SHA512 block transform for ARMv4/NEON, CRYPTOGAMS by <appro\@openssl.org>"
 661 .align  2
 662 #if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
 663 .extern OPENSSL_armcap_P
 664 #endif
 665 ___
 666
 667 $code =~ s/\`([^\`]*)\`/eval $1/gem;
 668 $code =~ s/\bbx\s+lr\b/.word\t0xe12fff1e/gm;    # make it possible to compile with -march=armv4
 669 $code =~ s/\bret\b/bx   lr/gm;
 670
 671 open SELF,$0;
 672 while(<SELF>) {
 673         next if (/^#!/);
 674         last if (!s/^#/@/ and !/^$/);
 675         print;
 676 }
 677 close SELF;
 678
 679 print $code;
 680 close STDOUT or die "error closing STDOUT: $!"; # enforce flush