]>
Commit | Line | Data |
---|---|---|
d2e9e320 | 1 | /* |
fecb3aae | 2 | * Copyright 2011-2022 The OpenSSL Project Authors. All Rights Reserved. |
d2e9e320 | 3 | * |
0e9725bc | 4 | * Licensed under the Apache License 2.0 (the "License"). You may not use |
d2e9e320 RS |
5 | * this file except in compliance with the License. You can obtain a copy |
6 | * in the file LICENSE in the source distribution or at | |
7 | * https://www.openssl.org/source/license.html | |
8 | */ | |
9 | ||
87873f43 AP |
10 | #include <stdio.h> |
11 | #include <stdlib.h> | |
12 | #include <string.h> | |
13 | #include <setjmp.h> | |
14 | #include <signal.h> | |
313e6ec1 | 15 | #include <openssl/crypto.h> |
f1a45f68 DC |
16 | #ifdef __APPLE__ |
17 | #include <sys/sysctl.h> | |
18 | #endif | |
d807db26 | 19 | #include "internal/cryptlib.h" |
efa1f224 | 20 | #include <unistd.h> |
87873f43 AP |
21 | |
22 | #include "arm_arch.h" | |
23 | ||
0f113f3e | 24 | unsigned int OPENSSL_armcap_P = 0; |
5ea64b45 | 25 | unsigned int OPENSSL_arm_midr = 0; |
10646160 | 26 | unsigned int OPENSSL_armv8_rsa_neonized = 0; |
87873f43 | 27 | |
c1669e1c | 28 | #if __ARM_MAX_ARCH__<7 |
0f113f3e MC |
29 | void OPENSSL_cpuid_setup(void) |
30 | { | |
31 | } | |
32 | ||
d807db26 | 33 | uint32_t OPENSSL_rdtsc(void) |
0f113f3e MC |
34 | { |
35 | return 0; | |
36 | } | |
c1669e1c | 37 | #else |
87873f43 AP |
38 | static sigset_t all_masked; |
39 | ||
40 | static sigjmp_buf ill_jmp; | |
0f113f3e MC |
41 | static void ill_handler(int sig) |
42 | { | |
43 | siglongjmp(ill_jmp, sig); | |
44 | } | |
87873f43 AP |
45 | |
46 | /* | |
47 | * Following subroutines could have been inlined, but it's not all | |
48 | * ARM compilers support inline assembler... | |
49 | */ | |
50 | void _armv7_neon_probe(void); | |
4afa9f03 AP |
51 | void _armv8_aes_probe(void); |
52 | void _armv8_sha1_probe(void); | |
53 | void _armv8_sha256_probe(void); | |
54 | void _armv8_pmull_probe(void); | |
77f3612e | 55 | # ifdef __aarch64__ |
71396cd0 | 56 | void _armv8_sm3_probe(void); |
15b7175f | 57 | void _armv8_sm4_probe(void); |
77f3612e | 58 | void _armv8_sha512_probe(void); |
5ea64b45 | 59 | unsigned int _armv8_cpuid_probe(void); |
b1b2146d DH |
60 | void _armv8_sve_probe(void); |
61 | void _armv8_sve2_probe(void); | |
efa1f224 OT |
62 | void _armv8_rng_probe(void); |
63 | ||
64 | size_t OPENSSL_rndr_asm(unsigned char *buf, size_t len); | |
65 | size_t OPENSSL_rndrrs_asm(unsigned char *buf, size_t len); | |
66 | ||
67 | size_t OPENSSL_rndr_bytes(unsigned char *buf, size_t len); | |
68 | size_t OPENSSL_rndrrs_bytes(unsigned char *buf, size_t len); | |
69 | ||
70 | static size_t OPENSSL_rndr_wrapper(size_t (*func)(unsigned char *, size_t), unsigned char *buf, size_t len) | |
71 | { | |
abc4345a | 72 | size_t buffer_size = 0; |
efa1f224 OT |
73 | int i; |
74 | ||
75 | for (i = 0; i < 8; i++) { | |
76 | buffer_size = func(buf, len); | |
77 | if (buffer_size == len) | |
78 | break; | |
79 | usleep(5000); /* 5000 microseconds (5 milliseconds) */ | |
80 | } | |
81 | return buffer_size; | |
82 | } | |
83 | ||
84 | size_t OPENSSL_rndr_bytes(unsigned char *buf, size_t len) | |
85 | { | |
86 | return OPENSSL_rndr_wrapper(OPENSSL_rndr_asm, buf, len); | |
87 | } | |
88 | ||
89 | size_t OPENSSL_rndrrs_bytes(unsigned char *buf, size_t len) | |
90 | { | |
91 | return OPENSSL_rndr_wrapper(OPENSSL_rndrrs_asm, buf, len); | |
92 | } | |
77f3612e | 93 | # endif |
d807db26 | 94 | uint32_t _armv7_tick(void); |
87873f43 | 95 | |
d807db26 | 96 | uint32_t OPENSSL_rdtsc(void) |
0f113f3e MC |
97 | { |
98 | if (OPENSSL_armcap_P & ARMV7_TICK) | |
99 | return _armv7_tick(); | |
100 | else | |
101 | return 0; | |
102 | } | |
87873f43 | 103 | |
9b05cbc3 AP |
104 | # if defined(__GNUC__) && __GNUC__>=2 |
105 | void OPENSSL_cpuid_setup(void) __attribute__ ((constructor)); | |
106 | # endif | |
5f40dd15 RL |
107 | |
108 | # if defined(__GLIBC__) && defined(__GLIBC_PREREQ) | |
109 | # if __GLIBC_PREREQ(2, 16) | |
110 | # include <sys/auxv.h> | |
111 | # define OSSL_IMPLEMENT_GETAUXVAL | |
112 | # endif | |
d5567d5f | 113 | # elif defined(__ANDROID_API__) |
114 | /* see https://developer.android.google.cn/ndk/guides/cpu-features */ | |
115 | # if __ANDROID_API__ >= 18 | |
116 | # include <sys/auxv.h> | |
117 | # define OSSL_IMPLEMENT_GETAUXVAL | |
118 | # endif | |
0f113f3e | 119 | # endif |
5eb24fbd DC |
120 | # if defined(__FreeBSD__) |
121 | # include <sys/param.h> | |
122 | # if __FreeBSD_version >= 1200000 | |
123 | # include <sys/auxv.h> | |
124 | # define OSSL_IMPLEMENT_GETAUXVAL | |
125 | ||
126 | static unsigned long getauxval(unsigned long key) | |
127 | { | |
128 | unsigned long val = 0ul; | |
129 | ||
130 | if (elf_aux_info((int)key, &val, sizeof(val)) != 0) | |
131 | return 0ul; | |
132 | ||
133 | return val; | |
134 | } | |
135 | # endif | |
136 | # endif | |
e8d93e34 | 137 | |
a5d250e5 LI |
138 | /* |
139 | * Android: according to https://developer.android.com/ndk/guides/cpu-features, | |
140 | * getauxval is supported starting with API level 18 | |
141 | */ | |
142 | # if defined(__ANDROID__) && defined(__ANDROID_API__) && __ANDROID_API__ >= 18 | |
143 | # include <sys/auxv.h> | |
144 | # define OSSL_IMPLEMENT_GETAUXVAL | |
145 | # endif | |
146 | ||
e8d93e34 | 147 | /* |
c9a41d7d | 148 | * ARM puts the feature bits for Crypto Extensions in AT_HWCAP2, whereas |
e8d93e34 AP |
149 | * AArch64 used AT_HWCAP. |
150 | */ | |
c1dabe26 AJ |
151 | # ifndef AT_HWCAP |
152 | # define AT_HWCAP 16 | |
153 | # endif | |
154 | # ifndef AT_HWCAP2 | |
155 | # define AT_HWCAP2 26 | |
156 | # endif | |
0f113f3e | 157 | # if defined(__arm__) || defined (__arm) |
c1dabe26 | 158 | # define HWCAP AT_HWCAP |
0f113f3e MC |
159 | # define HWCAP_NEON (1 << 12) |
160 | ||
c1dabe26 | 161 | # define HWCAP_CE AT_HWCAP2 |
0f113f3e MC |
162 | # define HWCAP_CE_AES (1 << 0) |
163 | # define HWCAP_CE_PMULL (1 << 1) | |
164 | # define HWCAP_CE_SHA1 (1 << 2) | |
165 | # define HWCAP_CE_SHA256 (1 << 3) | |
166 | # elif defined(__aarch64__) | |
c1dabe26 | 167 | # define HWCAP AT_HWCAP |
0f113f3e MC |
168 | # define HWCAP_NEON (1 << 1) |
169 | ||
170 | # define HWCAP_CE HWCAP | |
171 | # define HWCAP_CE_AES (1 << 3) | |
172 | # define HWCAP_CE_PMULL (1 << 4) | |
173 | # define HWCAP_CE_SHA1 (1 << 5) | |
174 | # define HWCAP_CE_SHA256 (1 << 6) | |
5ea64b45 | 175 | # define HWCAP_CPUID (1 << 11) |
954f45ba | 176 | # define HWCAP_SHA3 (1 << 17) |
71396cd0 | 177 | # define HWCAP_CE_SM3 (1 << 18) |
15b7175f | 178 | # define HWCAP_CE_SM4 (1 << 19) |
77f3612e | 179 | # define HWCAP_CE_SHA512 (1 << 21) |
b1b2146d | 180 | # define HWCAP_SVE (1 << 22) |
efa1f224 OT |
181 | /* AT_HWCAP2 */ |
182 | # define HWCAP2 26 | |
b1b2146d | 183 | # define HWCAP2_SVE2 (1 << 1) |
efa1f224 | 184 | # define HWCAP2_RNG (1 << 16) |
0f113f3e | 185 | # endif |
e8d93e34 | 186 | |
87873f43 | 187 | void OPENSSL_cpuid_setup(void) |
0f113f3e | 188 | { |
6ea3bca4 | 189 | const char *e; |
0f113f3e MC |
190 | struct sigaction ill_oact, ill_act; |
191 | sigset_t oset; | |
192 | static int trigger = 0; | |
193 | ||
194 | if (trigger) | |
195 | return; | |
196 | trigger = 1; | |
197 | ||
bb97dc50 TC |
198 | OPENSSL_armcap_P = 0; |
199 | ||
0f113f3e MC |
200 | if ((e = getenv("OPENSSL_armcap"))) { |
201 | OPENSSL_armcap_P = (unsigned int)strtoul(e, NULL, 0); | |
202 | return; | |
203 | } | |
204 | ||
f1a45f68 DC |
205 | # if defined(__APPLE__) |
206 | # if !defined(__aarch64__) | |
8653e78f AP |
207 | /* |
208 | * Capability probing by catching SIGILL appears to be problematic | |
209 | * on iOS. But since Apple universe is "monocultural", it's actually | |
210 | * possible to simply set pre-defined processor capability mask. | |
211 | */ | |
212 | if (1) { | |
213 | OPENSSL_armcap_P = ARMV7_NEON; | |
214 | return; | |
215 | } | |
216 | /* | |
217 | * One could do same even for __aarch64__ iOS builds. It's not done | |
218 | * exclusively for reasons of keeping code unified across platforms. | |
219 | * Unified code works because it never triggers SIGILL on Apple | |
220 | * devices... | |
221 | */ | |
f1a45f68 DC |
222 | # else |
223 | { | |
954f45ba X |
224 | unsigned int feature; |
225 | size_t len = sizeof(feature); | |
226 | char uarch[64]; | |
f1a45f68 | 227 | |
954f45ba | 228 | if (sysctlbyname("hw.optional.armv8_2_sha512", &feature, &len, NULL, 0) == 0 && feature == 1) |
f1a45f68 | 229 | OPENSSL_armcap_P |= ARMV8_SHA512; |
954f45ba X |
230 | feature = 0; |
231 | if (sysctlbyname("hw.optional.armv8_2_sha3", &feature, &len, NULL, 0) == 0 && feature == 1) { | |
232 | OPENSSL_armcap_P |= ARMV8_SHA3; | |
233 | len = sizeof(uarch); | |
234 | if ((sysctlbyname("machdep.cpu.brand_string", uarch, &len, NULL, 0) == 0) && | |
235 | (strncmp(uarch, "Apple M1", 8) == 0)) | |
236 | OPENSSL_armcap_P |= ARMV8_UNROLL8_EOR3; | |
237 | } | |
f1a45f68 DC |
238 | } |
239 | # endif | |
8653e78f AP |
240 | # endif |
241 | ||
5f40dd15 RL |
242 | # ifdef OSSL_IMPLEMENT_GETAUXVAL |
243 | if (getauxval(HWCAP) & HWCAP_NEON) { | |
244 | unsigned long hwcap = getauxval(HWCAP_CE); | |
245 | ||
246 | OPENSSL_armcap_P |= ARMV7_NEON; | |
247 | ||
248 | if (hwcap & HWCAP_CE_AES) | |
249 | OPENSSL_armcap_P |= ARMV8_AES; | |
250 | ||
251 | if (hwcap & HWCAP_CE_PMULL) | |
252 | OPENSSL_armcap_P |= ARMV8_PMULL; | |
253 | ||
254 | if (hwcap & HWCAP_CE_SHA1) | |
255 | OPENSSL_armcap_P |= ARMV8_SHA1; | |
256 | ||
257 | if (hwcap & HWCAP_CE_SHA256) | |
258 | OPENSSL_armcap_P |= ARMV8_SHA256; | |
259 | ||
260 | # ifdef __aarch64__ | |
15b7175f DH |
261 | if (hwcap & HWCAP_CE_SM4) |
262 | OPENSSL_armcap_P |= ARMV8_SM4; | |
263 | ||
5f40dd15 RL |
264 | if (hwcap & HWCAP_CE_SHA512) |
265 | OPENSSL_armcap_P |= ARMV8_SHA512; | |
5ea64b45 FF |
266 | |
267 | if (hwcap & HWCAP_CPUID) | |
268 | OPENSSL_armcap_P |= ARMV8_CPUID; | |
71396cd0 | 269 | |
270 | if (hwcap & HWCAP_CE_SM3) | |
271 | OPENSSL_armcap_P |= ARMV8_SM3; | |
954f45ba X |
272 | if (hwcap & HWCAP_SHA3) |
273 | OPENSSL_armcap_P |= ARMV8_SHA3; | |
5f40dd15 RL |
274 | # endif |
275 | } | |
efa1f224 | 276 | # ifdef __aarch64__ |
b1b2146d DH |
277 | if (getauxval(HWCAP) & HWCAP_SVE) |
278 | OPENSSL_armcap_P |= ARMV8_SVE; | |
279 | ||
280 | if (getauxval(HWCAP2) & HWCAP2_SVE2) | |
281 | OPENSSL_armcap_P |= ARMV8_SVE2; | |
282 | ||
efa1f224 OT |
283 | if (getauxval(HWCAP2) & HWCAP2_RNG) |
284 | OPENSSL_armcap_P |= ARMV8_RNG; | |
285 | # endif | |
5f40dd15 RL |
286 | # endif |
287 | ||
0f113f3e MC |
288 | sigfillset(&all_masked); |
289 | sigdelset(&all_masked, SIGILL); | |
290 | sigdelset(&all_masked, SIGTRAP); | |
291 | sigdelset(&all_masked, SIGFPE); | |
292 | sigdelset(&all_masked, SIGBUS); | |
293 | sigdelset(&all_masked, SIGSEGV); | |
294 | ||
0f113f3e MC |
295 | memset(&ill_act, 0, sizeof(ill_act)); |
296 | ill_act.sa_handler = ill_handler; | |
297 | ill_act.sa_mask = all_masked; | |
298 | ||
299 | sigprocmask(SIG_SETMASK, &ill_act.sa_mask, &oset); | |
300 | sigaction(SIGILL, &ill_act, &ill_oact); | |
301 | ||
5f40dd15 RL |
302 | /* If we used getauxval, we already have all the values */ |
303 | # ifndef OSSL_IMPLEMENT_GETAUXVAL | |
304 | if (sigsetjmp(ill_jmp, 1) == 0) { | |
0f113f3e MC |
305 | _armv7_neon_probe(); |
306 | OPENSSL_armcap_P |= ARMV7_NEON; | |
307 | if (sigsetjmp(ill_jmp, 1) == 0) { | |
308 | _armv8_pmull_probe(); | |
309 | OPENSSL_armcap_P |= ARMV8_PMULL | ARMV8_AES; | |
310 | } else if (sigsetjmp(ill_jmp, 1) == 0) { | |
311 | _armv8_aes_probe(); | |
312 | OPENSSL_armcap_P |= ARMV8_AES; | |
313 | } | |
314 | if (sigsetjmp(ill_jmp, 1) == 0) { | |
315 | _armv8_sha1_probe(); | |
316 | OPENSSL_armcap_P |= ARMV8_SHA1; | |
317 | } | |
318 | if (sigsetjmp(ill_jmp, 1) == 0) { | |
319 | _armv8_sha256_probe(); | |
320 | OPENSSL_armcap_P |= ARMV8_SHA256; | |
321 | } | |
5f40dd15 | 322 | # if defined(__aarch64__) && !defined(__APPLE__) |
77f3612e | 323 | if (sigsetjmp(ill_jmp, 1) == 0) { |
15b7175f DH |
324 | _armv8_sm4_probe(); |
325 | OPENSSL_armcap_P |= ARMV8_SM4; | |
326 | } | |
327 | ||
328 | if (sigsetjmp(ill_jmp, 1) == 0) { | |
77f3612e AP |
329 | _armv8_sha512_probe(); |
330 | OPENSSL_armcap_P |= ARMV8_SHA512; | |
331 | } | |
71396cd0 | 332 | |
333 | if (sigsetjmp(ill_jmp, 1) == 0) { | |
334 | _armv8_sm3_probe(); | |
335 | OPENSSL_armcap_P |= ARMV8_SM3; | |
954f45ba X |
336 | if (sigsetjmp(ill_jmp, 1) == 0) { |
337 | _armv8_eor3_probe(); | |
338 | OPENSSL_armcap_P |= ARMV8_SHA3; | |
71396cd0 | 339 | } |
5f40dd15 | 340 | # endif |
0f113f3e | 341 | } |
efa1f224 | 342 | # ifdef __aarch64__ |
b1b2146d DH |
343 | if (sigsetjmp(ill_jmp, 1) == 0) { |
344 | _armv8_sve_probe(); | |
345 | OPENSSL_armcap_P |= ARMV8_SVE; | |
346 | } | |
347 | ||
348 | if (sigsetjmp(ill_jmp, 1) == 0) { | |
349 | _armv8_sve2_probe(); | |
350 | OPENSSL_armcap_P |= ARMV8_SVE2; | |
351 | } | |
352 | ||
efa1f224 OT |
353 | if (sigsetjmp(ill_jmp, 1) == 0) { |
354 | _armv8_rng_probe(); | |
355 | OPENSSL_armcap_P |= ARMV8_RNG; | |
356 | } | |
357 | # endif | |
5f40dd15 RL |
358 | # endif |
359 | ||
360 | /* Things that getauxval didn't tell us */ | |
0f113f3e MC |
361 | if (sigsetjmp(ill_jmp, 1) == 0) { |
362 | _armv7_tick(); | |
363 | OPENSSL_armcap_P |= ARMV7_TICK; | |
364 | } | |
365 | ||
366 | sigaction(SIGILL, &ill_oact, NULL); | |
367 | sigprocmask(SIG_SETMASK, &oset, NULL); | |
5ea64b45 FF |
368 | |
369 | # ifdef __aarch64__ | |
370 | if (OPENSSL_armcap_P & ARMV8_CPUID) | |
371 | OPENSSL_arm_midr = _armv8_cpuid_probe(); | |
10646160 | 372 | |
373 | if ((MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A72) || | |
374 | MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_ARM, ARM_CPU_PART_N1)) && | |
375 | (OPENSSL_armcap_P & ARMV7_NEON)) { | |
376 | OPENSSL_armv8_rsa_neonized = 1; | |
377 | } | |
9224a407 X |
378 | if ((MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_ARM, ARM_CPU_PART_V1) || |
379 | MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_ARM, ARM_CPU_PART_N2)) && | |
954f45ba X |
380 | (OPENSSL_armcap_P & ARMV8_SHA3)) |
381 | OPENSSL_armcap_P |= ARMV8_UNROLL8_EOR3; | |
5ea64b45 | 382 | # endif |
0f113f3e | 383 | } |
c1669e1c | 384 | #endif |