2 # Copyright 2016-2018 The OpenSSL Project Authors. All Rights Reserved.
4 # Licensed under the Apache License 2.0 (the "License"). You may not use
5 # this file except in compliance with the License. You can obtain a copy
6 # in the file LICENSE in the source distribution or at
7 # https://www.openssl.org/source/license.html
10 # ====================================================================
11 # Written by Andy Polyakov, @dot-asm, initially for use in the OpenSSL
12 # project. The module is dual licensed under OpenSSL and CRYPTOGAMS
13 # licenses depending on where you obtain it. For further details see
14 # https://github.com/dot-asm/cryptogams/.
15 # ====================================================================
17 # This module implements Poly1305 hash for PowerPC.
21 # Numbers are cycles per processed byte with poly1305_blocks alone,
22 # and improvement coefficients relative to gcc-generated code.
26 # Freescale e300 14.8/+80% -
28 # PPC970 7.00/+114% 3.51/+205%
29 # POWER7 3.75/+260% 1.93/+100%
33 # Do we need floating-point implementation for PPC? Results presented
34 # in poly1305_ieee754.c are tricky to compare to, because they are for
35 # compiler-generated code. On the other hand it's known that floating-
36 # point performance can be dominated by FPU latency, which means that
37 # there is limit even for ideally optimized (and even vectorized) code.
38 # And this limit is estimated to be higher than above -m64 results. Or
39 # in other words floating-point implementation can be meaningful to
40 # consider only in 32-bit application context. We probably have to
41 # recognize that 32-bit builds are getting less popular on high-end
42 # systems and therefore tend to target embedded ones, which might not
45 # On side note, Power ISA 2.07 enables vector base 2^26 implementation,
46 # and POWER8 might have capacity to break 1.0 cycle per byte barrier...
50 # ... Unfortunately not:-( Estimate was a projection of ARM result,
51 # but ARM has vector multiply-n-add instruction, while PowerISA does
52 # not, not one usable in the context. Improvement is ~40% over -m64
53 # result above and is ~1.43 on little-endian systems.
55 # $output is the last argument if it looks like a file (it has an extension)
56 # $flavour is the first argument if it doesn't look like a file
57 $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m
|\
.\w
+$| ?
pop : undef;
58 $flavour = $#ARGV >= 0 && $ARGV[0] !~ m
|\
.| ?
shift : undef;
60 if ($flavour =~ /64/) {
67 } elsif ($flavour =~ /32/) {
74 } else { die "nonsense $flavour"; }
76 # Define endianness based on flavour
78 $LITTLE_ENDIAN = ($flavour=~/le$/) ?
$SIZE_T : 0;
80 $0 =~ m/(.*[\/\\])[^\
/\\]+$/; $dir=$1;
81 ( $xlate="${dir}ppc-xlate.pl" and -f
$xlate ) or
82 ( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f
$xlate) or
83 die "can't locate ppc-xlate.pl";
85 open STDOUT
,"| $^X $xlate $flavour \"$output\""
86 or die "can't call $xlate: $!";
91 my ($ctx,$inp,$len,$padbit) = map("r$_",(3..6));
92 my ($mac,$nonce)=($inp,$len);
99 if ($flavour =~ /64/) {
100 ###############################################################################
101 # base 2^64 implementation
103 my ($h0,$h1,$h2,$d0,$d1,$d2, $r0,$r1,$s1, $t0,$t1) = map("r$_",(7..12,27..31));
106 .globl
.poly1305_init_int
110 std r0
,0($ctx) # zero hash value
113 stw r0
,24($ctx) # clear is_base2_26
118 $code.=<<___
if ($LITTLE_ENDIAN);
119 ld
$d0,0($inp) # load key material
122 $code.=<<___
if (!$LITTLE_ENDIAN);
124 lwbrx
$d0,0,$inp # load key material
134 lis
$h1,0xfff # 0x0fff0000
135 ori
$h1,$h1,0xfffc # 0x0ffffffc
136 insrdi
$h1,$h1,32,0 # 0x0ffffffc0ffffffc
137 ori
$h0,$h1,3 # 0x0ffffffc0fffffff
142 std
$d0,32($ctx) # store key
149 .byte
0,12,0x14,0,0,0,2,0
150 .size
.poly1305_init_int
,.-.poly1305_init_int
152 .globl
.poly1305_blocks
159 $STU $sp,-$FRAME($sp)
161 $PUSH r27
,`$FRAME-$SIZE_T*5`($sp)
162 $PUSH r28
,`$FRAME-$SIZE_T*4`($sp)
163 $PUSH r29
,`$FRAME-$SIZE_T*3`($sp)
164 $PUSH r30
,`$FRAME-$SIZE_T*2`($sp)
165 $PUSH r31
,`$FRAME-$SIZE_T*1`($sp)
166 $PUSH r0
,`$FRAME+$LRSAVE`($sp)
168 ld
$r0,32($ctx) # load key
171 ld
$h0,0($ctx) # load hash value
177 add
$s1,$s1,$r1 # s1 = r1 + r1>>2
184 $code.=<<___
if ($LITTLE_ENDIAN);
185 ld
$t0,0($inp) # load input
188 $code.=<<___
if (!$LITTLE_ENDIAN);
190 lwbrx
$t0,0,$inp # load input
202 addc
$h0,$h0,$t0 # accumulate input
205 mulld
$d0,$h0,$r0 # h0*r0
209 mulld
$t0,$h1,$s1 # h1*5*r1
214 mulld
$t0,$h0,$r1 # h0*r1
219 mulld
$t0,$h1,$r0 # h1*r0
224 mulld
$t0,$h2,$s1 # h2*5*r1
225 mulld
$t1,$h2,$r0 # h2*r0
229 andc
$t0,$d2,$mask # final reduction step
239 std
$h0,0($ctx) # store hash value
243 $POP r27
,`$FRAME-$SIZE_T*5`($sp)
244 $POP r28
,`$FRAME-$SIZE_T*4`($sp)
245 $POP r29
,`$FRAME-$SIZE_T*3`($sp)
246 $POP r30
,`$FRAME-$SIZE_T*2`($sp)
247 $POP r31
,`$FRAME-$SIZE_T*1`($sp)
252 .byte
0,12,4,1,0x80,5,4,0
253 .size
.poly1305_blocks
,.-.poly1305_blocks
256 my ($h0,$h1,$h2,$h3,$h4,$t0) = map("r$_",(7..12));
259 .globl
.poly1305_emit
262 lwz
$h0,0($ctx) # load hash value base 2^26
267 lwz r0
,24($ctx) # is_base2_26
269 sldi
$h1,$h1,26 # base 2^26 -> base 2^64
281 ld
$h3,0($ctx) # load hash value base 2^64
286 xor $h0,$h0,$h3 # choose between radixes
296 addic
$h3,$h0,5 # compare to modulus
300 srdi
$t0,$t0,2 # see if it carried/borrowed
318 addc
$h0,$h0,$h3 # accumulate nonce
324 stbu
$h0,1($ctx) # write [little-endian] result
364 .byte
0,12,0x14,0,0,0,3,0
365 .size
.poly1305_emit
,.-.poly1305_emit
368 ###############################################################################
369 # base 2^32 implementation
371 my ($h0,$h1,$h2,$h3,$h4, $r0,$r1,$r2,$r3, $s1,$s2,$s3,
372 $t0,$t1,$t2,$t3, $D0,$D1,$D2,$D3, $d0,$d1,$d2,$d3
373 ) = map("r$_",(7..12,14..31));
376 .globl
.poly1305_init_int
380 stw r0
,0($ctx) # zero hash value
385 stw r0
,24($ctx) # clear is_base2_26
390 $code.=<<___
if ($LITTLE_ENDIAN);
391 lw
$h0,0($inp) # load key material
396 $code.=<<___
if (!$LITTLE_ENDIAN);
398 lwbrx
$h0,0,$inp # load key material
406 lis
$mask,0xf000 # 0xf0000000
408 andc
$r0,$r0,$mask # 0x0ffffffc
415 stw
$h0,32($ctx) # store key
424 .byte
0,12,0x14,0,0,0,2,0
425 .size
.poly1305_init_int
,.-.poly1305_init_int
427 .globl
.poly1305_blocks
434 $STU $sp,-$FRAME($sp)
436 $PUSH r14
,`$FRAME-$SIZE_T*18`($sp)
437 $PUSH r15
,`$FRAME-$SIZE_T*17`($sp)
438 $PUSH r16
,`$FRAME-$SIZE_T*16`($sp)
439 $PUSH r17
,`$FRAME-$SIZE_T*15`($sp)
440 $PUSH r18
,`$FRAME-$SIZE_T*14`($sp)
441 $PUSH r19
,`$FRAME-$SIZE_T*13`($sp)
442 $PUSH r20
,`$FRAME-$SIZE_T*12`($sp)
443 $PUSH r21
,`$FRAME-$SIZE_T*11`($sp)
444 $PUSH r22
,`$FRAME-$SIZE_T*10`($sp)
445 $PUSH r23
,`$FRAME-$SIZE_T*9`($sp)
446 $PUSH r24
,`$FRAME-$SIZE_T*8`($sp)
447 $PUSH r25
,`$FRAME-$SIZE_T*7`($sp)
448 $PUSH r26
,`$FRAME-$SIZE_T*6`($sp)
449 $PUSH r27
,`$FRAME-$SIZE_T*5`($sp)
450 $PUSH r28
,`$FRAME-$SIZE_T*4`($sp)
451 $PUSH r29
,`$FRAME-$SIZE_T*3`($sp)
452 $PUSH r30
,`$FRAME-$SIZE_T*2`($sp)
453 $PUSH r31
,`$FRAME-$SIZE_T*1`($sp)
454 $PUSH r0
,`$FRAME+$LRSAVE`($sp)
456 lwz
$r0,32($ctx) # load key
461 lwz
$h0,0($ctx) # load hash value
470 add
$s1,$s1,$r1 # si = ri + ri>>2
480 $code.=<<___
if ($LITTLE_ENDIAN);
481 lwz
$d0,0($inp) # load input
486 $code.=<<___
if (!$LITTLE_ENDIAN);
488 lwbrx
$d0,0,$inp # load input
498 addc
$h0,$h0,$d0 # accumulate input
502 mullw
$d0,$h0,$r0 # h0*r0
505 mullw
$d1,$h0,$r1 # h0*r1
508 mullw
$d2,$h0,$r2 # h0*r2
514 mullw
$d3,$h0,$r3 # h0*r3
517 mullw
$t0,$h1,$s3 # h1*s3
520 mullw
$t2,$h1,$r0 # h1*r0
525 mullw
$t0,$h1,$r1 # h1*r1
530 mullw
$t2,$h1,$r2 # h1*r2
535 mullw
$t0,$h2,$s2 # h2*s2
540 mullw
$t2,$h2,$s3 # h2*s3
545 mullw
$t0,$h2,$r0 # h2*r0
550 mullw
$t2,$h2,$r1 # h2*r1
555 mullw
$t0,$h3,$s1 # h3*s1
560 mullw
$t2,$h3,$s2 # h3*s2
565 mullw
$t0,$h3,$s3 # h3*s3
570 mullw
$t2,$h3,$r0 # h3*r0
575 mullw
$t0,$h4,$s1 # h4*s1
580 mullw
$t1,$h4,$s2 # h4*s2
585 mullw
$t2,$h4,$s3 # h4*s3
589 mullw
$h4,$h4,$r0 # h4*r0
596 andc
$D0,$h4,$mask # final reduction step
608 stw
$h0,0($ctx) # store hash value
614 $POP r14
,`$FRAME-$SIZE_T*18`($sp)
615 $POP r15
,`$FRAME-$SIZE_T*17`($sp)
616 $POP r16
,`$FRAME-$SIZE_T*16`($sp)
617 $POP r17
,`$FRAME-$SIZE_T*15`($sp)
618 $POP r18
,`$FRAME-$SIZE_T*14`($sp)
619 $POP r19
,`$FRAME-$SIZE_T*13`($sp)
620 $POP r20
,`$FRAME-$SIZE_T*12`($sp)
621 $POP r21
,`$FRAME-$SIZE_T*11`($sp)
622 $POP r22
,`$FRAME-$SIZE_T*10`($sp)
623 $POP r23
,`$FRAME-$SIZE_T*9`($sp)
624 $POP r24
,`$FRAME-$SIZE_T*8`($sp)
625 $POP r25
,`$FRAME-$SIZE_T*7`($sp)
626 $POP r26
,`$FRAME-$SIZE_T*6`($sp)
627 $POP r27
,`$FRAME-$SIZE_T*5`($sp)
628 $POP r28
,`$FRAME-$SIZE_T*4`($sp)
629 $POP r29
,`$FRAME-$SIZE_T*3`($sp)
630 $POP r30
,`$FRAME-$SIZE_T*2`($sp)
631 $POP r31
,`$FRAME-$SIZE_T*1`($sp)
636 .byte
0,12,4,1,0x80,18,4,0
637 .size
.poly1305_blocks
,.-.poly1305_blocks
640 my ($h0,$h1,$h2,$h3,$h4,$t0,$t1) = map("r$_",(6..12));
643 .globl
.poly1305_emit
646 lwz r0
,24($ctx) # is_base2_26
647 lwz
$h0,0($ctx) # load hash value
655 slwi
$t0,$h1,26 # base 2^26 -> base 2^32
670 addic r0
,$h0,5 # compare to modulus
676 srwi r0
,r0
,2 # see if it carried/borrowed
689 addc
$h0,$h0,r0
# accumulate nonce
697 stbu
$h0,1($ctx) # write [little-endian] result
735 .byte
0,12,0x14,0,0,0,3,0
736 .size
.poly1305_emit
,.-.poly1305_emit
740 ########################################################################
741 # PowerISA 2.07/VSX section #
742 ########################################################################
744 my $LOCALS= 6*$SIZE_T;
745 my $VSXFRAME = $LOCALS + 6*$SIZE_T;
746 $VSXFRAME += 128; # local variables
747 $VSXFRAME += 13*16; # v20-v31 offload
749 my $BIG_ENDIAN = ($flavour !~ /le/) ?
4 : 0;
751 ########################################################################
752 # Layout of opaque area is following:
754 # unsigned __int32 h[5]; # current hash value base 2^26
755 # unsigned __int32 pad;
756 # unsigned __int32 is_base2_26, pad;
757 # unsigned __int64 r[2]; # key value base 2^64
758 # struct { unsigned __int32 r^2, r^4, r^1, r^3; } r[9];
760 # where r^n are base 2^26 digits of powers of multiplier key. There are
761 # 5 digits, but last four are interleaved with multiples of 5, totalling
762 # in 9 elements: r0, r1, 5*r1, r2, 5*r2, r3, 5*r3, r4, 5*r4. Order of
763 # powers is as they appear in register, not memory.
765 my ($H0, $H1, $H2, $H3, $H4) = map("v$_",(0..4));
766 my ($I0, $I1, $I2, $I3, $I4) = map("v$_",(5..9));
767 my ($R0, $R1, $S1, $R2, $S2) = map("v$_",(10..14));
768 my ($R3, $S3, $R4, $S4) = ($R1, $S1, $R2, $S2);
769 my ($ACC0, $ACC1, $ACC2, $ACC3, $ACC4) = map("v$_",(15..19));
770 my ($T0, $T1, $T2, $T3, $T4) = map("v$_",(20..24));
771 my ($_26,$_4,$_40,$_14,$mask26,$padbits,$I2perm) = map("v$_",(25..31));
772 my ($x00,$x60,$x70,$x10,$x20,$x30,$x40,$x50) = (0, map("r$_",(7,8,27..31)));
773 my ($ctx_,$_ctx,$const) = map("r$_",(10..12));
775 if ($flavour =~ /64/) {
776 ###############################################################################
777 # setup phase of poly1305_blocks_vsx is different on 32- and 64-bit platforms,
778 # but the base 2^26 computational part is same...
780 my ($h0,$h1,$h2,$d0,$d1,$d2, $r0,$r1,$s1, $t0,$t1) = map("r$_",(6..11,27..31));
784 .globl
.poly1305_blocks_vsx
786 .poly1305_blocks_vsx
:
787 lwz r7
,24($ctx) # is_base2_26
789 bge __poly1305_blocks_vsx
791 neg r0
,r7
# is_base2_26 as mask
792 lwz r7
,0($ctx) # load hash base 2^26
798 sldi r8
,r8
,26 # base 2^26 -> base 2^64
810 ld r8
,0($ctx) # load hash base 2^64
814 xor r7
,r7
,r8
# select between radixes
825 std r7
,0($ctx) # store hash base 2^64
828 stw r0
,24($ctx) # clear is_base2_26
832 .byte
0,12,0x14,0,0,0,4,0
833 .size
.poly1305_blocks_vsx
,.-.poly1305_blocks_vsx
837 mulld
$d0,$h0,$r0 # h0*r0
840 mulld
$t0,$h1,$s1 # h1*5*r1
845 mulld
$t0,$h0,$r1 # h0*r1
850 mulld
$t0,$h1,$r0 # h1*r0
855 mulld
$t0,$h2,$s1 # h2*5*r1
856 mulld
$t1,$h2,$r0 # h2*r0
860 andc
$t0,$d2,$mask # final reduction step
870 .byte
0,12,0x14,0,0,0,0,0
871 .size __poly1305_mul
,.-__poly1305_mul
882 add
$d0,$d0,$d1 # * 5
888 add
$d0,$d0,$d2 # * 5
895 add
$d0,$d0,$d1 # * 5
901 add
$d0,$d0,$d2 # * 5
906 .byte
0,12,0x14,0,0,0,0,0
907 .size __poly1305_splat
,.-__poly1305_splat
910 __poly1305_blocks_vsx
:
911 $STU $sp,-$VSXFRAME($sp)
913 li r10
,`15+$LOCALS+128`
914 li r11
,`31+$LOCALS+128`
938 stw r12
,`$VSXFRAME-$SIZE_T*5-4`($sp)# save vrsave
940 mtspr
256,r12
# preserve all AltiVec registers
941 $PUSH r27
,`$VSXFRAME-$SIZE_T*5`($sp)
942 $PUSH r28
,`$VSXFRAME-$SIZE_T*4`($sp)
943 $PUSH r29
,`$VSXFRAME-$SIZE_T*3`($sp)
944 $PUSH r30
,`$VSXFRAME-$SIZE_T*2`($sp)
945 $PUSH r31
,`$VSXFRAME-$SIZE_T*1`($sp)
946 $PUSH r0
,`$VSXFRAME+$LRSAVE`($sp)
955 lvx_u
$mask26,$x00,$const
956 lvx_u
$_26,$x10,$const
957 lvx_u
$_40,$x20,$const
958 lvx_u
$I2perm,$x30,$const
959 lvx_u
$padbits,$x40,$const
961 cmplwi r7
,0 # is_base2_26?
964 ld
$r0,32($ctx) # load key base 2^64
968 add
$s1,$s1,$r1 # s1 = r1 + r1>>2
970 mr
$h0,$r0 # "calculate" r^1
973 addi
$t1,$ctx,`48+(12^$BIG_ENDIAN)`
976 bl __poly1305_mul
# calculate r^2
977 addi
$t1,$ctx,`48+(4^$BIG_ENDIAN)`
980 bl __poly1305_mul
# calculate r^3
981 addi
$t1,$ctx,`48+(8^$BIG_ENDIAN)`
984 bl __poly1305_mul
# calculate r^4
985 addi
$t1,$ctx,`48+(0^$BIG_ENDIAN)`
988 ld
$h0,0($ctx) # load hash
992 extrdi
$d0,$h0,26,38 # base 2^64 -> base 2^26
1006 ###############################################################################
1007 # 32-bit initialization
1009 my ($h0,$h1,$h2,$h3,$h4,$t0,$t1) = map("r$_",(7..11,0,12));
1010 my ($R3,$S3,$R4,$S4)=($I1,$I2,$I3,$I4);
1013 .globl
.poly1305_blocks_vsx
1015 .poly1305_blocks_vsx
:
1016 lwz r7
,24($ctx) # is_base2_26
1018 bge __poly1305_blocks_vsx
1020 beq Lpoly1305_blocks
1022 lwz
$h0,0($ctx) # load hash
1028 slwi
$t0,$h1,26 # base 2^26 -> base 2^32
1043 stw
$h0,0($ctx) # store hash base 2^32
1048 stw
$t0,24($ctx) # clear is_base2_26
1052 .byte
0,12,0x14,0,0,0,4,0
1053 .size
.poly1305_blocks_vsx
,.-.poly1305_blocks_vsx
1057 vmulouw
$ACC0,$H0,$R0
1058 vmulouw
$ACC1,$H1,$R0
1059 vmulouw
$ACC2,$H2,$R0
1060 vmulouw
$ACC3,$H3,$R0
1061 vmulouw
$ACC4,$H4,$R0
1064 vaddudm
$ACC0,$ACC0,$T0
1066 vaddudm
$ACC1,$ACC1,$T0
1068 vaddudm
$ACC2,$ACC2,$T0
1070 vaddudm
$ACC3,$ACC3,$T0
1072 vaddudm
$ACC4,$ACC4,$T0
1075 vaddudm
$ACC0,$ACC0,$T0
1077 vaddudm
$ACC1,$ACC1,$T0
1079 vaddudm
$ACC2,$ACC2,$T0
1081 vaddudm
$ACC3,$ACC3,$T0
1083 vaddudm
$ACC4,$ACC4,$T0
1086 vaddudm
$ACC0,$ACC0,$T0
1088 vaddudm
$ACC1,$ACC1,$T0
1090 vaddudm
$ACC2,$ACC2,$T0
1092 vaddudm
$ACC3,$ACC3,$T0
1094 vaddudm
$ACC4,$ACC4,$T0
1097 vaddudm
$ACC0,$ACC0,$T0
1099 vaddudm
$ACC1,$ACC1,$T0
1101 vaddudm
$ACC2,$ACC2,$T0
1103 vaddudm
$ACC3,$ACC3,$T0
1105 vaddudm
$ACC4,$ACC4,$T0
1107 ################################################################
1113 vand
$H3,$ACC3,$mask26
1114 vand
$H0,$ACC0,$mask26
1115 vaddudm
$H4,$H4,$ACC4 # h3 -> h4
1116 vaddudm
$H1,$H1,$ACC1 # h0 -> h1
1120 vand
$H4,$H4,$mask26
1121 vand
$H1,$H1,$mask26
1122 vaddudm
$H0,$H0,$ACC4
1123 vaddudm
$H2,$ACC2,$ACC1 # h1 -> h2
1125 vsld
$ACC4,$ACC4,$T0 # <<2
1127 vand
$H2,$H2,$mask26
1128 vaddudm
$H0,$H0,$ACC4 # h4 -> h0
1129 vaddudm
$H3,$H3,$ACC2 # h2 -> h3
1133 vand
$H0,$H0,$mask26
1134 vand
$H3,$H3,$mask26
1135 vaddudm
$H1,$H1,$ACC0 # h0 -> h1
1136 vaddudm
$H4,$H4,$ACC3 # h3 -> h4
1140 .byte
0,12,0x14,0,0,0,0,0
1141 .size __poly1305_mul
,.-__poly1305_mul
1144 __poly1305_blocks_vsx
:
1145 $STU $sp,-$VSXFRAME($sp)
1147 li r10
,`15+$LOCALS+128`
1148 li r11
,`31+$LOCALS+128`
1172 stw r12
,`$VSXFRAME-$SIZE_T*5-4`($sp)# save vrsave
1174 mtspr
256,r12
# preserve all AltiVec registers
1175 $PUSH r27
,`$VSXFRAME-$SIZE_T*5`($sp)
1176 $PUSH r28
,`$VSXFRAME-$SIZE_T*4`($sp)
1177 $PUSH r29
,`$VSXFRAME-$SIZE_T*3`($sp)
1178 $PUSH r30
,`$VSXFRAME-$SIZE_T*2`($sp)
1179 $PUSH r31
,`$VSXFRAME-$SIZE_T*1`($sp)
1180 $PUSH r0
,`$VSXFRAME+$LRSAVE`($sp)
1189 lvx_u
$mask26,$x00,$const
1190 lvx_u
$_26,$x10,$const
1191 lvx_u
$_40,$x20,$const
1192 lvx_u
$I2perm,$x30,$const
1193 lvx_u
$padbits,$x40,$const
1195 cmplwi r7
,0 # is_base2_26?
1198 lwz
$h1,32($ctx) # load key base 2^32
1203 extrwi
$h0,$h1,26,6 # base 2^32 -> base 2^26
1236 bl __poly1305_mul
# r^1:- * r^1:-
1238 vpermdi
$R0,$H0,$R0,0b00
1239 vpermdi
$R1,$H1,$R1,0b00
1240 vpermdi
$R2,$H2,$R2,0b00
1241 vpermdi
$R3,$H3,$R3,0b00
1242 vpermdi
$R4,$H4,$R4,0b00
1243 vpermdi
$H0,$H0,$H0,0b00
1244 vpermdi
$H1,$H1,$H1,0b00
1245 vpermdi
$H2,$H2,$H2,0b00
1246 vpermdi
$H3,$H3,$H3,0b00
1247 vpermdi
$H4,$H4,$H4,0b00
1248 vsld
$S1,$R1,$T0 # <<2
1257 bl __poly1305_mul
# r^2:r^2 * r^2:r^1
1260 lwz
$h1,0($ctx) # load hash
1266 vmrgow
$R0,$R0,$H0 # r^2:r^4:r^1:r^3
1271 vslw
$S1,$R1,$T0 # <<2
1280 stvx_u
$R0,$x30,$ctx
1281 stvx_u
$R1,$x40,$ctx
1282 stvx_u
$S1,$x50,$ctx
1290 extrwi
$h0,$h1,26,6 # base 2^32 -> base 2^26
1308 stw r0
,24($ctx) # set is_base2_26
1317 lvwzx_u
$H0,$x00,$ctx
1318 lvwzx_u
$H1,$x10,$ctx
1319 lvwzx_u
$H2,$x20,$ctx
1320 lvwzx_u
$H3,$x30,$ctx
1321 lvwzx_u
$H4,$x40,$ctx
1331 addi
$ctx_,$ctx,64 # &ctx->r[1]
1332 addi
$_ctx,$sp,`$LOCALS+15` # &ctx->r[1], r^2:r^4 shadow
1334 vxor
$T0,$T0,$T0 # ensure second half is zero
1335 vpermdi
$H0,$H0,$T0,0b00
1336 vpermdi
$H1,$H1,$T0,0b00
1337 vpermdi
$H2,$H2,$T0,0b00
1338 vpermdi
$H3,$H3,$T0,0b00
1339 vpermdi
$H4,$H4,$T0,0b00
1341 be?lvx_u
$_4,$x50,$const # byte swap mask
1342 lvx_u
$T1,$x00,$inp # load first input block
1346 be?vperm
$T1,$T1,$T1,$_4
1347 be?vperm
$T2,$T2,$T2,$_4
1348 be?vperm
$T3,$T3,$T3,$_4
1349 be?vperm
$T4,$T4,$T4,$_4
1351 vpermdi
$I0,$T1,$T2,0b00
# smash input to base 2^26
1353 vperm
$I2,$T1,$T2,$I2perm # 0x...0e0f0001...1e1f1011
1355 vpermdi
$I3,$T1,$T2,0b11
1361 vand
$I0,$I0,$mask26
1362 vand
$I1,$I1,$mask26
1363 vand
$I2,$I2,$mask26
1364 vand
$I3,$I3,$mask26
1366 vpermdi
$T1,$T3,$T4,0b00
1367 vperm
$T2,$T3,$T4,$I2perm # 0x...0e0f0001...1e1f1011
1368 vpermdi
$T3,$T3,$T4,0b11
1374 vand
$T1,$T1,$mask26
1375 vand
$T0,$T0,$mask26
1376 vand
$T2,$T2,$mask26
1377 vand
$T3,$T3,$mask26
1379 # inp[2]:inp[0]:inp[3]:inp[1]
1385 vor
$I4,$I4,$padbits
1387 lvx_splt
$R0,$x30,$ctx # taking lvx_vsplt out of loop
1388 lvx_splt
$R1,$x00,$ctx_ # gives ~8% improvement
1389 lvx_splt
$S1,$x10,$ctx_
1390 lvx_splt
$R2,$x20,$ctx_
1391 lvx_splt
$S2,$x30,$ctx_
1392 lvx_splt
$T1,$x40,$ctx_
1393 lvx_splt
$T2,$x50,$ctx_
1394 lvx_splt
$T3,$x60,$ctx_
1395 lvx_splt
$T4,$x70,$ctx_
1406 addi
$const,$const,0x50
1414 ################################################################
1415 ## ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2
1416 ## ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r
1417 ## \___________________/
1419 ## Note that we start with inp[2:3]*r^2. This is because it
1420 ## doesn't depend on reduction in previous iteration.
1421 ################################################################
1422 ## d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
1423 ## d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
1424 ## d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
1425 ## d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
1426 ## d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
1428 vmuleuw
$ACC0,$I0,$R0
1429 vmuleuw
$ACC1,$I0,$R1
1430 vmuleuw
$ACC2,$I0,$R2
1431 vmuleuw
$ACC3,$I1,$R2
1434 vaddudm
$ACC1,$ACC1,$T0
1436 vaddudm
$ACC2,$ACC2,$T0
1437 vmuleuw
$ACC4,$I2,$R2
1439 vaddudm
$ACC0,$ACC0,$T0
1441 vaddudm
$ACC3,$ACC3,$T0
1444 vaddudm
$ACC4,$ACC4,$T0
1454 vaddudm
$ACC0,$ACC0,$T0
1456 vaddudm
$ACC1,$ACC1,$T0
1458 vaddudm
$ACC2,$ACC2,$T0
1460 vaddudm
$ACC3,$ACC3,$T0
1463 vaddudm
$ACC4,$ACC4,$T0
1467 vaddudm
$ACC0,$ACC0,$T0
1469 vaddudm
$ACC1,$ACC1,$T0
1471 vaddudm
$ACC2,$ACC2,$T0
1473 vaddudm
$ACC3,$ACC3,$T0
1475 vaddudm
$ACC4,$ACC4,$T0
1477 be?lvx_u
$_4,$x00,$const # byte swap mask
1478 lvx_u
$T1,$x00,$inp # load next input block
1482 be?vperm
$T1,$T1,$T1,$_4
1483 be?vperm
$T2,$T2,$T2,$_4
1484 be?vperm
$T3,$T3,$T3,$_4
1485 be?vperm
$T4,$T4,$T4,$_4
1488 vaddudm
$ACC0,$ACC0,$T0
1490 vaddudm
$ACC1,$ACC1,$T0
1492 vaddudm
$ACC2,$ACC2,$T0
1494 vaddudm
$ACC3,$ACC3,$T0
1496 vaddudm
$ACC4,$ACC4,$T0
1498 vpermdi
$I0,$T1,$T2,0b00
# smash input to base 2^26
1500 vperm
$I2,$T1,$T2,$I2perm # 0x...0e0f0001...1e1f1011
1501 vpermdi
$I3,$T1,$T2,0b11
1503 # (hash + inp[0:1]) * r^4
1505 vaddudm
$ACC0,$ACC0,$T0
1507 vaddudm
$ACC1,$ACC1,$T0
1509 vaddudm
$ACC2,$ACC2,$T0
1511 vaddudm
$ACC3,$ACC3,$T0
1513 vaddudm
$ACC4,$ACC4,$T0
1515 vpermdi
$T1,$T3,$T4,0b00
1516 vperm
$T2,$T3,$T4,$I2perm # 0x...0e0f0001...1e1f1011
1517 vpermdi
$T3,$T3,$T4,0b11
1520 vaddudm
$ACC0,$ACC0,$T0
1522 vaddudm
$ACC1,$ACC1,$T0
1524 vaddudm
$ACC2,$ACC2,$T0
1526 vaddudm
$ACC3,$ACC3,$T0
1529 vaddudm
$ACC4,$ACC4,$T0
1538 vaddudm
$ACC0,$ACC0,$T0
1540 vaddudm
$ACC1,$ACC1,$T0
1542 vaddudm
$ACC2,$ACC2,$T0
1544 vaddudm
$ACC3,$ACC3,$T0
1547 vaddudm
$ACC4,$ACC4,$T0
1550 vand
$I0,$I0,$mask26
1551 vand
$I1,$I1,$mask26
1552 vand
$I2,$I2,$mask26
1553 vand
$I3,$I3,$mask26
1556 vaddudm
$ACC0,$ACC0,$T0
1558 vaddudm
$ACC1,$ACC1,$T0
1560 vaddudm
$ACC2,$ACC2,$T0
1562 vaddudm
$ACC3,$ACC3,$T0
1564 vaddudm
$ACC4,$ACC4,$T0
1572 vaddudm
$ACC0,$ACC0,$T0
1574 vaddudm
$ACC1,$ACC1,$T0
1576 vaddudm
$ACC2,$ACC2,$T0
1578 vaddudm
$ACC3,$ACC3,$T0
1580 vaddudm
$ACC4,$ACC4,$T0
1582 vand
$T1,$T1,$mask26
1583 vand
$_4,$_4,$mask26
1584 vand
$T2,$T2,$mask26
1585 vand
$T3,$T3,$mask26
1587 ################################################################
1588 # lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
1594 vand
$H3,$ACC3,$mask26
1595 vand
$H0,$ACC0,$mask26
1596 vaddudm
$H4,$H4,$ACC4 # h3 -> h4
1597 vaddudm
$H1,$H1,$ACC1 # h0 -> h1
1604 vor
$I4,$I4,$padbits
1608 vand
$H4,$H4,$mask26
1609 vand
$H1,$H1,$mask26
1610 vaddudm
$H0,$H0,$ACC4
1611 vaddudm
$H2,$ACC2,$ACC1 # h1 -> h2
1613 vsld
$ACC4,$ACC4,$T0 # <<2
1615 vand
$H2,$H2,$mask26
1616 vaddudm
$H0,$H0,$ACC4 # h4 -> h0
1617 vaddudm
$H3,$H3,$ACC2 # h2 -> h3
1621 vand
$H0,$H0,$mask26
1622 vand
$H3,$H3,$mask26
1623 vaddudm
$H1,$H1,$ACC0 # h0 -> h1
1624 vaddudm
$H4,$H4,$ACC3 # h3 -> h4
1630 andi
. $len,$len,0x30
1633 lvx_u
$R0,$x30,$ctx # load all powers
1634 lvx_u
$R1,$x00,$ctx_
1635 lvx_u
$S1,$x10,$ctx_
1636 lvx_u
$R2,$x20,$ctx_
1637 lvx_u
$S2,$x30,$ctx_
1640 vmuleuw
$ACC0,$I0,$R0
1641 vmuleuw
$ACC1,$I1,$R0
1642 vmuleuw
$ACC2,$I2,$R0
1643 vmuleuw
$ACC3,$I3,$R0
1644 vmuleuw
$ACC4,$I4,$R0
1647 vaddudm
$ACC0,$ACC0,$T0
1649 vaddudm
$ACC1,$ACC1,$T0
1651 vaddudm
$ACC2,$ACC2,$T0
1653 vaddudm
$ACC3,$ACC3,$T0
1654 lvx_u
$S3,$x50,$ctx_
1656 vaddudm
$ACC4,$ACC4,$T0
1657 lvx_u
$R3,$x40,$ctx_
1666 vaddudm
$ACC0,$ACC0,$T0
1668 vaddudm
$ACC1,$ACC1,$T0
1670 vaddudm
$ACC2,$ACC2,$T0
1672 vaddudm
$ACC3,$ACC3,$T0
1673 lvx_u
$S4,$x70,$ctx_
1675 vaddudm
$ACC4,$ACC4,$T0
1676 lvx_u
$R4,$x60,$ctx_
1679 vaddudm
$ACC0,$ACC0,$T0
1681 vaddudm
$ACC1,$ACC1,$T0
1683 vaddudm
$ACC2,$ACC2,$T0
1685 vaddudm
$ACC3,$ACC3,$T0
1687 vaddudm
$ACC4,$ACC4,$T0
1690 vaddudm
$ACC0,$ACC0,$T0
1692 vaddudm
$ACC1,$ACC1,$T0
1694 vaddudm
$ACC2,$ACC2,$T0
1696 vaddudm
$ACC3,$ACC3,$T0
1698 vaddudm
$ACC4,$ACC4,$T0
1700 # (hash + inp[0:1]) * r^4
1702 vaddudm
$ACC0,$ACC0,$T0
1704 vaddudm
$ACC1,$ACC1,$T0
1706 vaddudm
$ACC2,$ACC2,$T0
1708 vaddudm
$ACC3,$ACC3,$T0
1710 vaddudm
$ACC4,$ACC4,$T0
1713 vaddudm
$ACC0,$ACC0,$T0
1715 vaddudm
$ACC1,$ACC1,$T0
1717 vaddudm
$ACC2,$ACC2,$T0
1719 vaddudm
$ACC3,$ACC3,$T0
1720 lvx_u
$S1,$x10,$ctx_
1722 vaddudm
$ACC4,$ACC4,$T0
1723 lvx_u
$R1,$x00,$ctx_
1726 vaddudm
$ACC0,$ACC0,$T0
1728 vaddudm
$ACC1,$ACC1,$T0
1730 vaddudm
$ACC2,$ACC2,$T0
1732 vaddudm
$ACC3,$ACC3,$T0
1733 lvx_u
$S2,$x30,$ctx_
1735 vaddudm
$ACC4,$ACC4,$T0
1736 lvx_u
$R2,$x20,$ctx_
1739 vaddudm
$ACC0,$ACC0,$T0
1741 vaddudm
$ACC1,$ACC1,$T0
1743 vaddudm
$ACC2,$ACC2,$T0
1745 vaddudm
$ACC3,$ACC3,$T0
1747 vaddudm
$ACC4,$ACC4,$T0
1750 vaddudm
$ACC0,$ACC0,$T0
1752 vaddudm
$ACC1,$ACC1,$T0
1754 vaddudm
$ACC2,$ACC2,$T0
1756 vaddudm
$ACC3,$ACC3,$T0
1758 vaddudm
$ACC4,$ACC4,$T0
1760 ################################################################
1761 # horizontal addition
1763 vpermdi
$H0,$ACC0,$ACC0,0b10
1764 vpermdi
$H1,$ACC1,$ACC1,0b10
1765 vpermdi
$H2,$ACC2,$ACC2,0b10
1766 vpermdi
$H3,$ACC3,$ACC3,0b10
1767 vpermdi
$H4,$ACC4,$ACC4,0b10
1768 vaddudm
$ACC0,$ACC0,$H0
1769 vaddudm
$ACC1,$ACC1,$H1
1770 vaddudm
$ACC2,$ACC2,$H2
1771 vaddudm
$ACC3,$ACC3,$H3
1772 vaddudm
$ACC4,$ACC4,$H4
1774 ################################################################
1780 vand
$H3,$ACC3,$mask26
1781 vand
$H0,$ACC0,$mask26
1782 vaddudm
$H4,$H4,$ACC4 # h3 -> h4
1783 vaddudm
$H1,$H1,$ACC1 # h0 -> h1
1787 vand
$H4,$H4,$mask26
1788 vand
$H1,$H1,$mask26
1789 vaddudm
$H0,$H0,$ACC4
1790 vaddudm
$H2,$ACC2,$ACC1 # h1 -> h2
1792 vsld
$ACC4,$ACC4,$T0 # <<2
1794 vand
$H2,$H2,$mask26
1795 vaddudm
$H0,$H0,$ACC4 # h4 -> h0
1796 vaddudm
$H3,$H3,$ACC2 # h2 -> h3
1800 vand
$H0,$H0,$mask26
1801 vand
$H3,$H3,$mask26
1802 vaddudm
$H1,$H1,$ACC0 # h0 -> h1
1803 vaddudm
$H4,$H4,$ACC3 # h3 -> h4
1809 be?lvx_u
$_4,$x00,$const # byte swap mask
1810 lvx_u
$T1,$x00,$inp # load last partial input block
1814 be?vperm
$T1,$T1,$T1,$_4
1815 be?vperm
$T2,$T2,$T2,$_4
1816 be?vperm
$T3,$T3,$T3,$_4
1817 be?vperm
$T4,$T4,$T4,$_4
1819 vpermdi
$I0,$T1,$T2,0b00
# smash input to base 2^26
1821 vperm
$I2,$T1,$T2,$I2perm # 0x...0e0f0001...1e1f1011
1822 vpermdi
$I3,$T1,$T2,0b11
1828 vand
$I0,$I0,$mask26
1829 vand
$I1,$I1,$mask26
1830 vand
$I2,$I2,$mask26
1831 vand
$I3,$I3,$mask26
1833 vpermdi
$T0,$T3,$T4,0b00
1834 vperm
$T1,$T3,$T4,$I2perm # 0x...0e0f0001...1e1f1011
1835 vpermdi
$T2,$T3,$T4,0b11
1844 vand
$T0,$T0,$mask26
1845 vand
$T3,$T3,$mask26
1846 vand
$T1,$T1,$mask26
1847 vand
$T2,$T2,$mask26
1849 # inp[2]:inp[0]:inp[3]:inp[1]
1855 vor
$I4,$I4,$padbits
1857 vperm
$H0,$H0,$H0,$ACC0 # move hash to right lane
1858 vand
$I0,$I0, $ACC1 # mask redundant input lane[s]
1859 vperm
$H1,$H1,$H1,$ACC0
1861 vperm
$H2,$H2,$H2,$ACC0
1863 vperm
$H3,$H3,$H3,$ACC0
1865 vperm
$H4,$H4,$H4,$ACC0
1868 vaddudm
$I0,$I0,$H0 # accumulate hash
1869 vxor
$H0,$H0,$H0 # wipe hash value
1884 $POP r0
,`$VSXFRAME+$LRSAVE`($sp)
1889 stvwx_u
$H0,$x00,$ctx # store hash
1890 stvwx_u
$H1,$x10,$ctx
1891 stvwx_u
$H2,$x20,$ctx
1892 stvwx_u
$H3,$x30,$ctx
1893 stvwx_u
$H4,$x40,$ctx
1895 lwz r12
,`$VSXFRAME-$SIZE_T*5-4`($sp)# pull vrsave
1897 li r10
,`15+$LOCALS+128`
1898 li r11
,`31+$LOCALS+128`
1899 mtspr
256,r12
# restore vrsave
1922 $POP r27
,`$VSXFRAME-$SIZE_T*5`($sp)
1923 $POP r28
,`$VSXFRAME-$SIZE_T*4`($sp)
1924 $POP r29
,`$VSXFRAME-$SIZE_T*3`($sp)
1925 $POP r30
,`$VSXFRAME-$SIZE_T*2`($sp)
1926 $POP r31
,`$VSXFRAME-$SIZE_T*1`($sp)
1927 addi
$sp,$sp,$VSXFRAME
1930 .byte
0,12,0x04,1,0x80,5,4,0
1932 .size __poly1305_blocks_vsx
,.-__poly1305_blocks_vsx
1938 mflr
$const # vvvvvv "distance" between . and 1st data entry
1939 addi
$const,$const,`64-8`
1943 .byte
0,12,0x14,0,0,0,0,0
1946 .quad
0x0000000003ffffff,0x0000000003ffffff # mask26
1947 .quad
0x000000000000001a,0x000000000000001a # _26
1948 .quad
0x0000000000000028,0x0000000000000028 # _40
1949 .quad
0x000000000e0f0001,0x000000001e1f1011 # I2perm
1950 .quad
0x0100000001000000,0x0100000001000000 # padbits
1951 .quad
0x0706050403020100,0x0f0e0d0c0b0a0908 # byte swap for big-endian
1953 .quad
0x0000000000000000,0x0000000004050607 # magic tail masks
1954 .quad
0x0405060700000000,0x0000000000000000
1955 .quad
0x0000000000000000,0x0405060700000000
1957 .quad
0xffffffff00000000,0xffffffffffffffff
1958 .quad
0xffffffff00000000,0xffffffff00000000
1959 .quad
0x0000000000000000,0xffffffff00000000
1963 .asciz
"Poly1305 for PPC, CRYPTOGAMS by \@dot-asm"
1966 foreach (split("\n",$code)) {
1967 s/\`([^\`]*)\`/eval($1)/ge;
1969 # instructions prefixed with '?' are endian-specific and need
1970 # to be adjusted accordingly...
1971 if ($flavour !~ /le$/) { # big-endian
1974 } else { # little-endian
1981 close STDOUT
or die "error closing STDOUT: $!";