]>
Commit | Line | Data |
---|---|---|
d2e9e320 | 1 | /* |
fecb3aae | 2 | * Copyright 2011-2022 The OpenSSL Project Authors. All Rights Reserved. |
d2e9e320 | 3 | * |
0e9725bc | 4 | * Licensed under the Apache License 2.0 (the "License"). You may not use |
d2e9e320 RS |
5 | * this file except in compliance with the License. You can obtain a copy |
6 | * in the file LICENSE in the source distribution or at | |
7 | * https://www.openssl.org/source/license.html | |
8 | */ | |
9 | ||
87873f43 AP |
10 | #include <stdio.h> |
11 | #include <stdlib.h> | |
12 | #include <string.h> | |
13 | #include <setjmp.h> | |
14 | #include <signal.h> | |
313e6ec1 | 15 | #include <openssl/crypto.h> |
f1a45f68 DC |
16 | #ifdef __APPLE__ |
17 | #include <sys/sysctl.h> | |
18 | #endif | |
d807db26 | 19 | #include "internal/cryptlib.h" |
b863e1e4 | 20 | #ifndef _WIN32 |
efa1f224 | 21 | #include <unistd.h> |
b863e1e4 EC |
22 | #else |
23 | #include <windows.h> | |
24 | #endif | |
87873f43 AP |
25 | #include "arm_arch.h" |
26 | ||
0f113f3e | 27 | unsigned int OPENSSL_armcap_P = 0; |
5ea64b45 | 28 | unsigned int OPENSSL_arm_midr = 0; |
10646160 | 29 | unsigned int OPENSSL_armv8_rsa_neonized = 0; |
87873f43 | 30 | |
b863e1e4 EC |
31 | #ifdef _WIN32 |
32 | void OPENSSL_cpuid_setup(void) | |
33 | { | |
34 | OPENSSL_armcap_P |= ARMV7_NEON; | |
35 | OPENSSL_armv8_rsa_neonized = 1; | |
36 | if (IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE)) { | |
37 | // These are all covered by one call in Windows | |
38 | OPENSSL_armcap_P |= ARMV8_AES; | |
39 | OPENSSL_armcap_P |= ARMV8_PMULL; | |
40 | OPENSSL_armcap_P |= ARMV8_SHA1; | |
41 | OPENSSL_armcap_P |= ARMV8_SHA256; | |
42 | } | |
43 | } | |
44 | ||
45 | uint32_t OPENSSL_rdtsc(void) | |
46 | { | |
47 | return 0; | |
48 | } | |
49 | #elif __ARM_MAX_ARCH__<7 | |
0f113f3e MC |
50 | void OPENSSL_cpuid_setup(void) |
51 | { | |
52 | } | |
53 | ||
d807db26 | 54 | uint32_t OPENSSL_rdtsc(void) |
0f113f3e MC |
55 | { |
56 | return 0; | |
57 | } | |
c1669e1c | 58 | #else |
87873f43 AP |
59 | static sigset_t all_masked; |
60 | ||
61 | static sigjmp_buf ill_jmp; | |
0f113f3e MC |
62 | static void ill_handler(int sig) |
63 | { | |
64 | siglongjmp(ill_jmp, sig); | |
65 | } | |
87873f43 AP |
66 | |
67 | /* | |
68 | * Following subroutines could have been inlined, but it's not all | |
69 | * ARM compilers support inline assembler... | |
70 | */ | |
71 | void _armv7_neon_probe(void); | |
4afa9f03 AP |
72 | void _armv8_aes_probe(void); |
73 | void _armv8_sha1_probe(void); | |
74 | void _armv8_sha256_probe(void); | |
75 | void _armv8_pmull_probe(void); | |
77f3612e | 76 | # ifdef __aarch64__ |
71396cd0 | 77 | void _armv8_sm3_probe(void); |
15b7175f | 78 | void _armv8_sm4_probe(void); |
77f3612e | 79 | void _armv8_sha512_probe(void); |
5ea64b45 | 80 | unsigned int _armv8_cpuid_probe(void); |
b1b2146d DH |
81 | void _armv8_sve_probe(void); |
82 | void _armv8_sve2_probe(void); | |
efa1f224 OT |
83 | void _armv8_rng_probe(void); |
84 | ||
85 | size_t OPENSSL_rndr_asm(unsigned char *buf, size_t len); | |
86 | size_t OPENSSL_rndrrs_asm(unsigned char *buf, size_t len); | |
87 | ||
88 | size_t OPENSSL_rndr_bytes(unsigned char *buf, size_t len); | |
89 | size_t OPENSSL_rndrrs_bytes(unsigned char *buf, size_t len); | |
90 | ||
91 | static size_t OPENSSL_rndr_wrapper(size_t (*func)(unsigned char *, size_t), unsigned char *buf, size_t len) | |
92 | { | |
abc4345a | 93 | size_t buffer_size = 0; |
efa1f224 OT |
94 | int i; |
95 | ||
96 | for (i = 0; i < 8; i++) { | |
97 | buffer_size = func(buf, len); | |
98 | if (buffer_size == len) | |
99 | break; | |
100 | usleep(5000); /* 5000 microseconds (5 milliseconds) */ | |
101 | } | |
102 | return buffer_size; | |
103 | } | |
104 | ||
105 | size_t OPENSSL_rndr_bytes(unsigned char *buf, size_t len) | |
106 | { | |
107 | return OPENSSL_rndr_wrapper(OPENSSL_rndr_asm, buf, len); | |
108 | } | |
109 | ||
110 | size_t OPENSSL_rndrrs_bytes(unsigned char *buf, size_t len) | |
111 | { | |
112 | return OPENSSL_rndr_wrapper(OPENSSL_rndrrs_asm, buf, len); | |
113 | } | |
77f3612e | 114 | # endif |
d807db26 | 115 | uint32_t _armv7_tick(void); |
87873f43 | 116 | |
d807db26 | 117 | uint32_t OPENSSL_rdtsc(void) |
0f113f3e MC |
118 | { |
119 | if (OPENSSL_armcap_P & ARMV7_TICK) | |
120 | return _armv7_tick(); | |
121 | else | |
122 | return 0; | |
123 | } | |
87873f43 | 124 | |
9b05cbc3 AP |
125 | # if defined(__GNUC__) && __GNUC__>=2 |
126 | void OPENSSL_cpuid_setup(void) __attribute__ ((constructor)); | |
127 | # endif | |
5f40dd15 RL |
128 | |
129 | # if defined(__GLIBC__) && defined(__GLIBC_PREREQ) | |
130 | # if __GLIBC_PREREQ(2, 16) | |
131 | # include <sys/auxv.h> | |
132 | # define OSSL_IMPLEMENT_GETAUXVAL | |
133 | # endif | |
d5567d5f | 134 | # elif defined(__ANDROID_API__) |
135 | /* see https://developer.android.google.cn/ndk/guides/cpu-features */ | |
136 | # if __ANDROID_API__ >= 18 | |
137 | # include <sys/auxv.h> | |
138 | # define OSSL_IMPLEMENT_GETAUXVAL | |
139 | # endif | |
0f113f3e | 140 | # endif |
5eb24fbd DC |
141 | # if defined(__FreeBSD__) |
142 | # include <sys/param.h> | |
143 | # if __FreeBSD_version >= 1200000 | |
144 | # include <sys/auxv.h> | |
145 | # define OSSL_IMPLEMENT_GETAUXVAL | |
146 | ||
147 | static unsigned long getauxval(unsigned long key) | |
148 | { | |
149 | unsigned long val = 0ul; | |
150 | ||
151 | if (elf_aux_info((int)key, &val, sizeof(val)) != 0) | |
152 | return 0ul; | |
153 | ||
154 | return val; | |
155 | } | |
156 | # endif | |
157 | # endif | |
e8d93e34 | 158 | |
a5d250e5 LI |
159 | /* |
160 | * Android: according to https://developer.android.com/ndk/guides/cpu-features, | |
161 | * getauxval is supported starting with API level 18 | |
162 | */ | |
163 | # if defined(__ANDROID__) && defined(__ANDROID_API__) && __ANDROID_API__ >= 18 | |
164 | # include <sys/auxv.h> | |
165 | # define OSSL_IMPLEMENT_GETAUXVAL | |
166 | # endif | |
167 | ||
e8d93e34 | 168 | /* |
c9a41d7d | 169 | * ARM puts the feature bits for Crypto Extensions in AT_HWCAP2, whereas |
e8d93e34 AP |
170 | * AArch64 used AT_HWCAP. |
171 | */ | |
c1dabe26 AJ |
172 | # ifndef AT_HWCAP |
173 | # define AT_HWCAP 16 | |
174 | # endif | |
175 | # ifndef AT_HWCAP2 | |
176 | # define AT_HWCAP2 26 | |
177 | # endif | |
0f113f3e | 178 | # if defined(__arm__) || defined (__arm) |
c1dabe26 | 179 | # define HWCAP AT_HWCAP |
0f113f3e MC |
180 | # define HWCAP_NEON (1 << 12) |
181 | ||
c1dabe26 | 182 | # define HWCAP_CE AT_HWCAP2 |
0f113f3e MC |
183 | # define HWCAP_CE_AES (1 << 0) |
184 | # define HWCAP_CE_PMULL (1 << 1) | |
185 | # define HWCAP_CE_SHA1 (1 << 2) | |
186 | # define HWCAP_CE_SHA256 (1 << 3) | |
187 | # elif defined(__aarch64__) | |
c1dabe26 | 188 | # define HWCAP AT_HWCAP |
0f113f3e MC |
189 | # define HWCAP_NEON (1 << 1) |
190 | ||
191 | # define HWCAP_CE HWCAP | |
192 | # define HWCAP_CE_AES (1 << 3) | |
193 | # define HWCAP_CE_PMULL (1 << 4) | |
194 | # define HWCAP_CE_SHA1 (1 << 5) | |
195 | # define HWCAP_CE_SHA256 (1 << 6) | |
5ea64b45 | 196 | # define HWCAP_CPUID (1 << 11) |
954f45ba | 197 | # define HWCAP_SHA3 (1 << 17) |
71396cd0 | 198 | # define HWCAP_CE_SM3 (1 << 18) |
15b7175f | 199 | # define HWCAP_CE_SM4 (1 << 19) |
77f3612e | 200 | # define HWCAP_CE_SHA512 (1 << 21) |
b1b2146d | 201 | # define HWCAP_SVE (1 << 22) |
efa1f224 OT |
202 | /* AT_HWCAP2 */ |
203 | # define HWCAP2 26 | |
b1b2146d | 204 | # define HWCAP2_SVE2 (1 << 1) |
efa1f224 | 205 | # define HWCAP2_RNG (1 << 16) |
0f113f3e | 206 | # endif |
e8d93e34 | 207 | |
87873f43 | 208 | void OPENSSL_cpuid_setup(void) |
0f113f3e | 209 | { |
6ea3bca4 | 210 | const char *e; |
0f113f3e MC |
211 | struct sigaction ill_oact, ill_act; |
212 | sigset_t oset; | |
213 | static int trigger = 0; | |
214 | ||
215 | if (trigger) | |
216 | return; | |
217 | trigger = 1; | |
218 | ||
bb97dc50 TC |
219 | OPENSSL_armcap_P = 0; |
220 | ||
0f113f3e MC |
221 | if ((e = getenv("OPENSSL_armcap"))) { |
222 | OPENSSL_armcap_P = (unsigned int)strtoul(e, NULL, 0); | |
223 | return; | |
224 | } | |
225 | ||
f1a45f68 DC |
226 | # if defined(__APPLE__) |
227 | # if !defined(__aarch64__) | |
8653e78f AP |
228 | /* |
229 | * Capability probing by catching SIGILL appears to be problematic | |
230 | * on iOS. But since Apple universe is "monocultural", it's actually | |
231 | * possible to simply set pre-defined processor capability mask. | |
232 | */ | |
233 | if (1) { | |
234 | OPENSSL_armcap_P = ARMV7_NEON; | |
235 | return; | |
236 | } | |
237 | /* | |
238 | * One could do same even for __aarch64__ iOS builds. It's not done | |
239 | * exclusively for reasons of keeping code unified across platforms. | |
240 | * Unified code works because it never triggers SIGILL on Apple | |
241 | * devices... | |
242 | */ | |
f1a45f68 DC |
243 | # else |
244 | { | |
954f45ba X |
245 | unsigned int feature; |
246 | size_t len = sizeof(feature); | |
247 | char uarch[64]; | |
f1a45f68 | 248 | |
954f45ba | 249 | if (sysctlbyname("hw.optional.armv8_2_sha512", &feature, &len, NULL, 0) == 0 && feature == 1) |
f1a45f68 | 250 | OPENSSL_armcap_P |= ARMV8_SHA512; |
954f45ba X |
251 | feature = 0; |
252 | if (sysctlbyname("hw.optional.armv8_2_sha3", &feature, &len, NULL, 0) == 0 && feature == 1) { | |
253 | OPENSSL_armcap_P |= ARMV8_SHA3; | |
254 | len = sizeof(uarch); | |
255 | if ((sysctlbyname("machdep.cpu.brand_string", uarch, &len, NULL, 0) == 0) && | |
256 | (strncmp(uarch, "Apple M1", 8) == 0)) | |
257 | OPENSSL_armcap_P |= ARMV8_UNROLL8_EOR3; | |
258 | } | |
f1a45f68 DC |
259 | } |
260 | # endif | |
8653e78f AP |
261 | # endif |
262 | ||
5f40dd15 RL |
263 | # ifdef OSSL_IMPLEMENT_GETAUXVAL |
264 | if (getauxval(HWCAP) & HWCAP_NEON) { | |
265 | unsigned long hwcap = getauxval(HWCAP_CE); | |
266 | ||
267 | OPENSSL_armcap_P |= ARMV7_NEON; | |
268 | ||
269 | if (hwcap & HWCAP_CE_AES) | |
270 | OPENSSL_armcap_P |= ARMV8_AES; | |
271 | ||
272 | if (hwcap & HWCAP_CE_PMULL) | |
273 | OPENSSL_armcap_P |= ARMV8_PMULL; | |
274 | ||
275 | if (hwcap & HWCAP_CE_SHA1) | |
276 | OPENSSL_armcap_P |= ARMV8_SHA1; | |
277 | ||
278 | if (hwcap & HWCAP_CE_SHA256) | |
279 | OPENSSL_armcap_P |= ARMV8_SHA256; | |
280 | ||
281 | # ifdef __aarch64__ | |
15b7175f DH |
282 | if (hwcap & HWCAP_CE_SM4) |
283 | OPENSSL_armcap_P |= ARMV8_SM4; | |
284 | ||
5f40dd15 RL |
285 | if (hwcap & HWCAP_CE_SHA512) |
286 | OPENSSL_armcap_P |= ARMV8_SHA512; | |
5ea64b45 FF |
287 | |
288 | if (hwcap & HWCAP_CPUID) | |
289 | OPENSSL_armcap_P |= ARMV8_CPUID; | |
71396cd0 | 290 | |
291 | if (hwcap & HWCAP_CE_SM3) | |
292 | OPENSSL_armcap_P |= ARMV8_SM3; | |
954f45ba X |
293 | if (hwcap & HWCAP_SHA3) |
294 | OPENSSL_armcap_P |= ARMV8_SHA3; | |
5f40dd15 RL |
295 | # endif |
296 | } | |
efa1f224 | 297 | # ifdef __aarch64__ |
b1b2146d DH |
298 | if (getauxval(HWCAP) & HWCAP_SVE) |
299 | OPENSSL_armcap_P |= ARMV8_SVE; | |
300 | ||
301 | if (getauxval(HWCAP2) & HWCAP2_SVE2) | |
302 | OPENSSL_armcap_P |= ARMV8_SVE2; | |
303 | ||
efa1f224 OT |
304 | if (getauxval(HWCAP2) & HWCAP2_RNG) |
305 | OPENSSL_armcap_P |= ARMV8_RNG; | |
306 | # endif | |
5f40dd15 RL |
307 | # endif |
308 | ||
0f113f3e MC |
309 | sigfillset(&all_masked); |
310 | sigdelset(&all_masked, SIGILL); | |
311 | sigdelset(&all_masked, SIGTRAP); | |
312 | sigdelset(&all_masked, SIGFPE); | |
313 | sigdelset(&all_masked, SIGBUS); | |
314 | sigdelset(&all_masked, SIGSEGV); | |
315 | ||
0f113f3e MC |
316 | memset(&ill_act, 0, sizeof(ill_act)); |
317 | ill_act.sa_handler = ill_handler; | |
318 | ill_act.sa_mask = all_masked; | |
319 | ||
320 | sigprocmask(SIG_SETMASK, &ill_act.sa_mask, &oset); | |
321 | sigaction(SIGILL, &ill_act, &ill_oact); | |
322 | ||
5f40dd15 RL |
323 | /* If we used getauxval, we already have all the values */ |
324 | # ifndef OSSL_IMPLEMENT_GETAUXVAL | |
325 | if (sigsetjmp(ill_jmp, 1) == 0) { | |
0f113f3e MC |
326 | _armv7_neon_probe(); |
327 | OPENSSL_armcap_P |= ARMV7_NEON; | |
328 | if (sigsetjmp(ill_jmp, 1) == 0) { | |
329 | _armv8_pmull_probe(); | |
330 | OPENSSL_armcap_P |= ARMV8_PMULL | ARMV8_AES; | |
331 | } else if (sigsetjmp(ill_jmp, 1) == 0) { | |
332 | _armv8_aes_probe(); | |
333 | OPENSSL_armcap_P |= ARMV8_AES; | |
334 | } | |
335 | if (sigsetjmp(ill_jmp, 1) == 0) { | |
336 | _armv8_sha1_probe(); | |
337 | OPENSSL_armcap_P |= ARMV8_SHA1; | |
338 | } | |
339 | if (sigsetjmp(ill_jmp, 1) == 0) { | |
340 | _armv8_sha256_probe(); | |
341 | OPENSSL_armcap_P |= ARMV8_SHA256; | |
342 | } | |
5f40dd15 | 343 | # if defined(__aarch64__) && !defined(__APPLE__) |
77f3612e | 344 | if (sigsetjmp(ill_jmp, 1) == 0) { |
15b7175f DH |
345 | _armv8_sm4_probe(); |
346 | OPENSSL_armcap_P |= ARMV8_SM4; | |
347 | } | |
348 | ||
349 | if (sigsetjmp(ill_jmp, 1) == 0) { | |
77f3612e AP |
350 | _armv8_sha512_probe(); |
351 | OPENSSL_armcap_P |= ARMV8_SHA512; | |
352 | } | |
71396cd0 | 353 | |
354 | if (sigsetjmp(ill_jmp, 1) == 0) { | |
355 | _armv8_sm3_probe(); | |
356 | OPENSSL_armcap_P |= ARMV8_SM3; | |
954f45ba X |
357 | if (sigsetjmp(ill_jmp, 1) == 0) { |
358 | _armv8_eor3_probe(); | |
359 | OPENSSL_armcap_P |= ARMV8_SHA3; | |
71396cd0 | 360 | } |
5f40dd15 | 361 | # endif |
0f113f3e | 362 | } |
efa1f224 | 363 | # ifdef __aarch64__ |
b1b2146d DH |
364 | if (sigsetjmp(ill_jmp, 1) == 0) { |
365 | _armv8_sve_probe(); | |
366 | OPENSSL_armcap_P |= ARMV8_SVE; | |
367 | } | |
368 | ||
369 | if (sigsetjmp(ill_jmp, 1) == 0) { | |
370 | _armv8_sve2_probe(); | |
371 | OPENSSL_armcap_P |= ARMV8_SVE2; | |
372 | } | |
373 | ||
efa1f224 OT |
374 | if (sigsetjmp(ill_jmp, 1) == 0) { |
375 | _armv8_rng_probe(); | |
376 | OPENSSL_armcap_P |= ARMV8_RNG; | |
377 | } | |
378 | # endif | |
5f40dd15 RL |
379 | # endif |
380 | ||
f2ec24c9 CG |
381 | /* |
382 | * Probing for ARMV7_TICK is known to produce unreliable results, | |
383 | * so we will only use the feature when the user explicitly enables | |
384 | * it with OPENSSL_armcap. | |
385 | */ | |
0f113f3e MC |
386 | |
387 | sigaction(SIGILL, &ill_oact, NULL); | |
388 | sigprocmask(SIG_SETMASK, &oset, NULL); | |
5ea64b45 FF |
389 | |
390 | # ifdef __aarch64__ | |
391 | if (OPENSSL_armcap_P & ARMV8_CPUID) | |
392 | OPENSSL_arm_midr = _armv8_cpuid_probe(); | |
10646160 | 393 | |
394 | if ((MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A72) || | |
395 | MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_ARM, ARM_CPU_PART_N1)) && | |
396 | (OPENSSL_armcap_P & ARMV7_NEON)) { | |
397 | OPENSSL_armv8_rsa_neonized = 1; | |
398 | } | |
9224a407 X |
399 | if ((MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_ARM, ARM_CPU_PART_V1) || |
400 | MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_ARM, ARM_CPU_PART_N2)) && | |
954f45ba X |
401 | (OPENSSL_armcap_P & ARMV8_SHA3)) |
402 | OPENSSL_armcap_P |= ARMV8_UNROLL8_EOR3; | |
5ea64b45 | 403 | # endif |
0f113f3e | 404 | } |
c1669e1c | 405 | #endif |