2 # Copyright 2011-2016 The OpenSSL Project Authors. All Rights Reserved.
4 # Licensed under the Apache License 2.0 (the "License"). You may not use
5 # this file except in compliance with the License. You can obtain a copy
6 # in the file LICENSE in the source distribution or at
7 # https://www.openssl.org/source/license.html
10 # ====================================================================
11 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
12 # project. The module is, however, dual licensed under OpenSSL and
13 # CRYPTOGAMS licenses depending on where you obtain it. For further
14 # details see http://www.openssl.org/~appro/cryptogams/.
15 # ====================================================================
19 # The module implements bn_GF2m_mul_2x2 polynomial multiplication
20 # used in bn_gf2m.c. It's kind of low-hanging mechanical port from
21 # C for the time being... Except that it has two code paths: pure
22 # integer code suitable for any ARMv4 and later CPU and NEON code
23 # suitable for ARMv7. Pure integer 1x1 multiplication subroutine runs
24 # in ~45 cycles on dual-issue core such as Cortex A8, which is ~50%
25 # faster than compiler-generated code. For ECDH and ECDSA verify (but
26 # not for ECDSA sign) it means 25%-45% improvement depending on key
27 # length, more for longer keys. Even though NEON 1x1 multiplication
28 # runs in even less cycles, ~30, improvement is measurable only on
29 # longer keys. One has to optimize code elsewhere to get NEON glow...
33 # Double bn_GF2m_mul_2x2 performance by using algorithm from paper
34 # referred below, which improves ECDH and ECDSA verify benchmarks
37 # Câmara, D.; Gouvêa, C. P. L.; López, J. & Dahab, R.: Fast Software
38 # Polynomial Multiplication on ARM Processors using the NEON Engine.
40 # http://conradoplg.cryptoland.net/files/2010/12/mocrysen13.pdf
42 # $output is the last argument if it looks like a file (it has an extension)
43 # $flavour is the first argument if it doesn't look like a file
44 $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m
|\
.\w
+$| ?
pop : undef;
45 $flavour = $#ARGV >= 0 && $ARGV[0] !~ m
|\
.| ?
shift : undef;
47 if ($flavour && $flavour ne "void") {
48 $0 =~ m/(.*[\/\\])[^\
/\\]+$/; $dir=$1;
49 ( $xlate="${dir}arm-xlate.pl" and -f
$xlate ) or
50 ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f
$xlate) or
51 die "can't locate arm-xlate.pl";
53 open STDOUT
,"| \"$^X\" $xlate $flavour \"$output\""
54 or die "can't call $xlate: $1";
56 $output and open STDOUT
,">$output";
62 #if defined(__thumb2__)
72 # private interface to mul_1x1_ialu
77 ($a0,$a1,$a2,$a12,$a4,$a14)=
78 ($hi,$lo,$t0,$t1, $i0,$i1 )=map("r$_",(4..9),12);
83 .type mul_1x1_ialu
,%function
87 bic
$a1,$a,#3<<30 @ a1=a&0x3fffffff
88 str
$a0,[sp
,#0] @ tab[0]=0
89 add
$a2,$a1,$a1 @ a2
=a1
<<1
90 str
$a1,[sp
,#4] @ tab[1]=a1
91 eor
$a12,$a1,$a2 @ a1
^a2
92 str
$a2,[sp
,#8] @ tab[2]=a2
93 mov
$a4,$a1,lsl
#2 @ a4=a1<<2
94 str
$a12,[sp
,#12] @ tab[3]=a1^a2
95 eor
$a14,$a1,$a4 @ a1
^a4
96 str
$a4,[sp
,#16] @ tab[4]=a4
97 eor
$a0,$a2,$a4 @ a2
^a4
98 str
$a14,[sp
,#20] @ tab[5]=a1^a4
99 eor
$a12,$a12,$a4 @ a1
^a2
^a4
100 str
$a0,[sp
,#24] @ tab[6]=a2^a4
101 and $i0,$mask,$b,lsl
#2
102 str
$a12,[sp
,#28] @ tab[7]=a1^a2^a4
104 and $i1,$mask,$b,lsr
#1
105 ldr
$lo,[sp
,$i0] @ tab
[b
& 0x7]
106 and $i0,$mask,$b,lsr
#4
107 ldr
$t1,[sp
,$i1] @ tab
[b
>> 3 & 0x7]
108 and $i1,$mask,$b,lsr
#7
109 ldr
$t0,[sp
,$i0] @ tab
[b
>> 6 & 0x7]
110 eor
$lo,$lo,$t1,lsl
#3 @ stall
112 ldr
$t1,[sp
,$i1] @ tab
[b
>> 9 & 0x7]
114 and $i0,$mask,$b,lsr
#10
115 eor
$lo,$lo,$t0,lsl
#6
116 eor
$hi,$hi,$t0,lsr
#26
117 ldr
$t0,[sp
,$i0] @ tab
[b
>> 12 & 0x7]
119 and $i1,$mask,$b,lsr
#13
120 eor
$lo,$lo,$t1,lsl
#9
121 eor
$hi,$hi,$t1,lsr
#23
122 ldr
$t1,[sp
,$i1] @ tab
[b
>> 15 & 0x7]
124 and $i0,$mask,$b,lsr
#16
125 eor
$lo,$lo,$t0,lsl
#12
126 eor
$hi,$hi,$t0,lsr
#20
127 ldr
$t0,[sp
,$i0] @ tab
[b
>> 18 & 0x7]
129 and $i1,$mask,$b,lsr
#19
130 eor
$lo,$lo,$t1,lsl
#15
131 eor
$hi,$hi,$t1,lsr
#17
132 ldr
$t1,[sp
,$i1] @ tab
[b
>> 21 & 0x7]
134 and $i0,$mask,$b,lsr
#22
135 eor
$lo,$lo,$t0,lsl
#18
136 eor
$hi,$hi,$t0,lsr
#14
137 ldr
$t0,[sp
,$i0] @ tab
[b
>> 24 & 0x7]
139 and $i1,$mask,$b,lsr
#25
140 eor
$lo,$lo,$t1,lsl
#21
141 eor
$hi,$hi,$t1,lsr
#11
142 ldr
$t1,[sp
,$i1] @ tab
[b
>> 27 & 0x7]
145 and $i0,$mask,$b,lsr
#28
146 eor
$lo,$lo,$t0,lsl
#24
147 eor
$hi,$hi,$t0,lsr
#8
148 ldr
$t0,[sp
,$i0] @ tab
[b
>> 30 ]
153 eorne
$lo,$lo,$b,lsl
#30
154 eorne
$hi,$hi,$b,lsr
#2
156 eor
$lo,$lo,$t1,lsl
#27
157 eor
$hi,$hi,$t1,lsr
#5
161 eorne
$lo,$lo,$b,lsl
#31
162 eorne
$hi,$hi,$b,lsr
#1
163 eor
$lo,$lo,$t0,lsl
#30
164 eor
$hi,$hi,$t0,lsr
#2
167 .size mul_1x1_ialu
,.-mul_1x1_ialu
170 # void bn_GF2m_mul_2x2(BN_ULONG *r,
171 # BN_ULONG a1,BN_ULONG a0,
172 # BN_ULONG b1,BN_ULONG b0); # r[3..0]=a1a0·b1b0
175 .global bn_GF2m_mul_2x2
176 .type bn_GF2m_mul_2x2
,%function
179 #if __ARM_MAX_ARCH__>=7
181 ldr r12
,.LOPENSSL_armcap
182 # if !defined(_WIN32)
183 adr r10
,.LOPENSSL_armcap
186 # if defined(__APPLE__) || defined(_WIN32)
195 stmdb sp
!,{r4
-r10
,lr
}
198 $ret="r10"; # reassigned 1st argument
200 mov
$ret,r0 @ reassign
1st argument
205 ldr r3
,[sp
,#32] @ load b0
207 mov sp
,r7 @ allocate tab
[8]
210 bl mul_1x1_ialu @ a1·b1
214 eor
$b,$b,r3 @ flip b0
and b1
215 eor
$a,$a,r2 @ flip a0
and a1
220 bl mul_1x1_ialu @ a0·b0
226 bl mul_1x1_ialu @
(a1
+a0
)·
(b1
+b0
)
228 @r=map("r$_",(6..9));
230 ldmia
$ret,{@r[0]-@r[3]}
232 ldr sp
,[sp
,#32] @ destroy tab[8]
243 ldmia sp
!,{r4
-r10
,pc
}
245 ldmia sp
!,{r4
-r10
,lr
}
247 moveq pc
,lr @ be binary compatible with V4
, yet
248 bx lr @ interoperable with Thumb ISA
:-)
253 my ($r,$t0,$t1,$t2,$t3)=map("q$_",(0..3,8..12));
254 my ($a,$b,$k48,$k32,$k16)=map("d$_",(26..31));
257 #if __ARM_MAX_ARCH__>=7
263 ldr r12
, [sp
] @
5th argument
266 vmov
.i64
$k48, #0x0000ffffffffffff
267 vmov
.i64
$k32, #0x00000000ffffffff
268 vmov
.i64
$k16, #0x000000000000ffff
270 vext
.8 $t0#lo, $a, $a, #1 @ A1
271 vmull
.p8
$t0, $t0#lo, $b @ F = A1*B
272 vext
.8 $r#lo, $b, $b, #1 @ B1
273 vmull
.p8
$r, $a, $r#lo @ E = A*B1
274 vext
.8 $t1#lo, $a, $a, #2 @ A2
275 vmull
.p8
$t1, $t1#lo, $b @ H = A2*B
276 vext
.8 $t3#lo, $b, $b, #2 @ B2
277 vmull
.p8
$t3, $a, $t3#lo @ G = A*B2
278 vext
.8 $t2#lo, $a, $a, #3 @ A3
279 veor
$t0, $t0, $r @ L
= E
+ F
280 vmull
.p8
$t2, $t2#lo, $b @ J = A3*B
281 vext
.8 $r#lo, $b, $b, #3 @ B3
282 veor
$t1, $t1, $t3 @ M
= G
+ H
283 vmull
.p8
$r, $a, $r#lo @ I = A*B3
284 veor
$t0#lo, $t0#lo, $t0#hi @ t0 = (L) (P0 + P1) << 8
285 vand
$t0#hi, $t0#hi, $k48
286 vext
.8 $t3#lo, $b, $b, #4 @ B4
287 veor
$t1#lo, $t1#lo, $t1#hi @ t1 = (M) (P2 + P3) << 16
288 vand
$t1#hi, $t1#hi, $k32
289 vmull
.p8
$t3, $a, $t3#lo @ K = A*B4
290 veor
$t2, $t2, $r @ N
= I
+ J
291 veor
$t0#lo, $t0#lo, $t0#hi
292 veor
$t1#lo, $t1#lo, $t1#hi
293 veor
$t2#lo, $t2#lo, $t2#hi @ t2 = (N) (P4 + P5) << 24
294 vand
$t2#hi, $t2#hi, $k16
295 vext
.8 $t0, $t0, $t0, #15
296 veor
$t3#lo, $t3#lo, $t3#hi @ t3 = (K) (P6 + P7) << 32
298 vext
.8 $t1, $t1, $t1, #14
299 veor
$t2#lo, $t2#lo, $t2#hi
300 vmull
.p8
$r, $a, $b @ D
= A
*B
301 vext
.8 $t3, $t3, $t3, #12
302 vext
.8 $t2, $t2, $t2, #13
314 .size bn_GF2m_mul_2x2
,.-bn_GF2m_mul_2x2
315 #if __ARM_MAX_ARCH__>=7
319 .word OPENSSL_armcap_P
321 .word OPENSSL_armcap_P
-.
324 .asciz
"GF(2^m) Multiplication for ARMv4/NEON, CRYPTOGAMS by <appro\@openssl.org>"
327 #if __ARM_MAX_ARCH__>=7
328 .comm OPENSSL_armcap_P
,4,4
332 foreach (split("\n",$code)) {
333 s/\`([^\`]*)\`/eval $1/geo;
335 s/\bq([0-9]+)#(lo|hi)/sprintf "d%d",2*$1+($2 eq "hi")/geo or
336 s/\bret\b/bx lr/go or
337 s/\bbx\s+lr\b/.word\t0xe12fff1e/go; # make it possible to compile with -march=armv4
341 close STDOUT
or die "error closing STDOUT: $!"; # enforce flush