]>
Commit | Line | Data |
---|---|---|
1585f6ac MT |
1 | /* fat-arm64.c |
2 | ||
3 | Copyright (C) 2021 Mamone Tarsha | |
4 | ||
5 | This file is part of GNU Nettle. | |
6 | ||
7 | GNU Nettle is free software: you can redistribute it and/or | |
8 | modify it under the terms of either: | |
9 | ||
10 | * the GNU Lesser General Public License as published by the Free | |
11 | Software Foundation; either version 3 of the License, or (at your | |
12 | option) any later version. | |
13 | ||
14 | or | |
15 | ||
16 | * the GNU General Public License as published by the Free | |
17 | Software Foundation; either version 2 of the License, or (at your | |
18 | option) any later version. | |
19 | ||
20 | or both in parallel, as here. | |
21 | ||
22 | GNU Nettle is distributed in the hope that it will be useful, | |
23 | but WITHOUT ANY WARRANTY; without even the implied warranty of | |
24 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
25 | General Public License for more details. | |
26 | ||
27 | You should have received copies of the GNU General Public License and | |
28 | the GNU Lesser General Public License along with this program. If | |
29 | not, see http://www.gnu.org/licenses/. | |
30 | */ | |
31 | ||
32 | #define _GNU_SOURCE | |
33 | ||
34 | #if HAVE_CONFIG_H | |
35 | # include "config.h" | |
36 | #endif | |
37 | ||
38 | #include <assert.h> | |
39 | #include <stdio.h> | |
40 | #include <stdlib.h> | |
41 | #include <string.h> | |
42 | ||
43 | #if defined(__linux__) && defined(__GLIBC__) && defined(__GLIBC_PREREQ) | |
44 | # if __GLIBC_PREREQ(2, 16) | |
45 | # define USE_GETAUXVAL 1 | |
46 | # include <asm/hwcap.h> | |
47 | # include <sys/auxv.h> | |
48 | # endif | |
b874bdea CW |
49 | #elif defined(__OpenBSD__) |
50 | # include <sys/sysctl.h> | |
51 | # include <machine/cpu.h> | |
52 | # include <machine/armreg.h> | |
0dd1ce39 TK |
53 | #elif defined(__APPLE__) |
54 | # include <sys/sysctl.h> | |
1585f6ac MT |
55 | #endif |
56 | ||
57 | #include "nettle-types.h" | |
58 | ||
39d1e2a3 | 59 | #include "aes.h" |
9939f866 | 60 | #include "ghash-internal.h" |
1585f6ac MT |
61 | #include "fat-setup.h" |
62 | ||
63 | /* Defines from arch/arm64/include/uapi/asm/hwcap.h in Linux kernel */ | |
64 | #ifndef HWCAP_ASIMD | |
65 | #define HWCAP_ASIMD (1 << 1) | |
66 | #endif | |
39d1e2a3 MT |
67 | #ifndef HWCAP_AES |
68 | #define HWCAP_AES (1 << 3) | |
69 | #endif | |
1585f6ac MT |
70 | #ifndef HWCAP_PMULL |
71 | #define HWCAP_PMULL (1 << 4) | |
72 | #endif | |
e5a9dbf4 MT |
73 | #ifndef HWCAP_SHA1 |
74 | #define HWCAP_SHA1 (1 << 5) | |
75 | #endif | |
7b446327 MT |
76 | #ifndef HWCAP_SHA2 |
77 | #define HWCAP_SHA2 (1 << 6) | |
78 | #endif | |
1585f6ac MT |
79 | |
80 | struct arm64_features | |
81 | { | |
39d1e2a3 | 82 | int have_aes; |
1585f6ac | 83 | int have_pmull; |
e5a9dbf4 | 84 | int have_sha1; |
7b446327 | 85 | int have_sha2; |
1585f6ac MT |
86 | }; |
87 | ||
88 | #define MATCH(s, slen, literal, llen) \ | |
89 | ((slen) == (llen) && memcmp ((s), (literal), llen) == 0) | |
90 | ||
0dd1ce39 TK |
91 | #if defined(__APPLE__) |
92 | static int | |
93 | check_sysctlbyname(const char* name) | |
94 | { | |
95 | int val; | |
96 | size_t s = sizeof(val); | |
97 | return sysctlbyname(name, &val, &s, NULL, 0) ? 0 : val; | |
98 | } | |
99 | #endif | |
100 | ||
1585f6ac MT |
101 | static void |
102 | get_arm64_features (struct arm64_features *features) | |
103 | { | |
104 | const char *s; | |
39d1e2a3 | 105 | features->have_aes = 0; |
1585f6ac | 106 | features->have_pmull = 0; |
e5a9dbf4 | 107 | features->have_sha1 = 0; |
7b446327 | 108 | features->have_sha2 = 0; |
1585f6ac MT |
109 | |
110 | s = secure_getenv (ENV_OVERRIDE); | |
111 | if (s) | |
112 | for (;;) | |
113 | { | |
114 | const char *sep = strchr (s, ','); | |
115 | size_t length = sep ? (size_t) (sep - s) : strlen(s); | |
116 | ||
39d1e2a3 MT |
117 | if (MATCH (s, length, "aes", 3)) |
118 | features->have_aes = 1; | |
119 | else if (MATCH (s, length, "pmull", 5)) | |
1585f6ac | 120 | features->have_pmull = 1; |
7b446327 | 121 | else if (MATCH (s, length, "sha1", 4)) |
e5a9dbf4 | 122 | features->have_sha1 = 1; |
7b446327 MT |
123 | else if (MATCH (s, length, "sha2", 4)) |
124 | features->have_sha2 = 1; | |
1585f6ac MT |
125 | if (!sep) |
126 | break; | |
127 | s = sep + 1; | |
128 | } | |
129 | else | |
130 | { | |
131 | #if USE_GETAUXVAL | |
132 | unsigned long hwcap = getauxval(AT_HWCAP); | |
39d1e2a3 MT |
133 | features->have_aes |
134 | = ((hwcap & (HWCAP_ASIMD | HWCAP_AES)) == (HWCAP_ASIMD | HWCAP_AES)); | |
1585f6ac MT |
135 | features->have_pmull |
136 | = ((hwcap & (HWCAP_ASIMD | HWCAP_PMULL)) == (HWCAP_ASIMD | HWCAP_PMULL)); | |
e5a9dbf4 MT |
137 | features->have_sha1 |
138 | = ((hwcap & (HWCAP_ASIMD | HWCAP_SHA1)) == (HWCAP_ASIMD | HWCAP_SHA1)); | |
7b446327 MT |
139 | features->have_sha2 |
140 | = ((hwcap & (HWCAP_ASIMD | HWCAP_SHA2)) == (HWCAP_ASIMD | HWCAP_SHA2)); | |
b874bdea CW |
141 | #elif defined(__OpenBSD__) |
142 | const int isar0_mib[] = { CTL_MACHDEP, CPU_ID_AA64ISAR0 }; | |
143 | uint64_t isar0; | |
144 | size_t len = sizeof(isar0); | |
145 | ||
146 | if (sysctl(isar0_mib, 2, &isar0, &len, NULL, 0) < 0) | |
147 | return; | |
148 | features->have_aes | |
149 | = (ID_AA64ISAR0_AES(isar0) >= ID_AA64ISAR0_AES_BASE); | |
150 | features->have_pmull | |
151 | = (ID_AA64ISAR0_AES(isar0) >= ID_AA64ISAR0_AES_PMULL); | |
152 | features->have_sha1 | |
153 | = (ID_AA64ISAR0_SHA1(isar0) >= ID_AA64ISAR0_SHA1_BASE); | |
154 | features->have_sha2 | |
155 | = (ID_AA64ISAR0_SHA2(isar0) >= ID_AA64ISAR0_SHA2_BASE); | |
0dd1ce39 TK |
156 | #elif defined(__APPLE__) |
157 | /* See https://developer.apple.com/documentation/kernel/1387446-sysctlbyname/determining_instruction_set_characteristics */ | |
158 | features->have_aes = check_sysctlbyname("hw.optional.arm.FEAT_AES"); | |
159 | features->have_pmull = check_sysctlbyname("hw.optional.arm.FEAT_PMULL"); | |
160 | features->have_sha1 = check_sysctlbyname("hw.optional.arm.FEAT_SHA1"); | |
161 | features->have_sha2 = check_sysctlbyname("hw.optional.arm.FEAT_SHA256"); | |
1585f6ac MT |
162 | #endif |
163 | } | |
164 | } | |
165 | ||
39d1e2a3 MT |
166 | DECLARE_FAT_FUNC(nettle_aes128_encrypt, aes128_crypt_func) |
167 | DECLARE_FAT_FUNC_VAR(aes128_encrypt, aes128_crypt_func, c) | |
168 | DECLARE_FAT_FUNC_VAR(aes128_encrypt, aes128_crypt_func, arm64) | |
169 | DECLARE_FAT_FUNC(nettle_aes128_decrypt, aes128_crypt_func) | |
170 | DECLARE_FAT_FUNC_VAR(aes128_decrypt, aes128_crypt_func, c) | |
171 | DECLARE_FAT_FUNC_VAR(aes128_decrypt, aes128_crypt_func, arm64) | |
172 | ||
173 | DECLARE_FAT_FUNC(nettle_aes192_encrypt, aes192_crypt_func) | |
174 | DECLARE_FAT_FUNC_VAR(aes192_encrypt, aes192_crypt_func, c) | |
175 | DECLARE_FAT_FUNC_VAR(aes192_encrypt, aes192_crypt_func, arm64) | |
176 | DECLARE_FAT_FUNC(nettle_aes192_decrypt, aes192_crypt_func) | |
177 | DECLARE_FAT_FUNC_VAR(aes192_decrypt, aes192_crypt_func, c) | |
178 | DECLARE_FAT_FUNC_VAR(aes192_decrypt, aes192_crypt_func, arm64) | |
179 | ||
180 | DECLARE_FAT_FUNC(nettle_aes256_encrypt, aes256_crypt_func) | |
181 | DECLARE_FAT_FUNC_VAR(aes256_encrypt, aes256_crypt_func, c) | |
182 | DECLARE_FAT_FUNC_VAR(aes256_encrypt, aes256_crypt_func, arm64) | |
183 | DECLARE_FAT_FUNC(nettle_aes256_decrypt, aes256_crypt_func) | |
184 | DECLARE_FAT_FUNC_VAR(aes256_decrypt, aes256_crypt_func, c) | |
185 | DECLARE_FAT_FUNC_VAR(aes256_decrypt, aes256_crypt_func, arm64) | |
186 | ||
9939f866 NM |
187 | DECLARE_FAT_FUNC(_nettle_ghash_set_key, ghash_set_key_func) |
188 | DECLARE_FAT_FUNC_VAR(ghash_set_key, ghash_set_key_func, c) | |
189 | DECLARE_FAT_FUNC_VAR(ghash_set_key, ghash_set_key_func, arm64) | |
1585f6ac | 190 | |
9939f866 NM |
191 | DECLARE_FAT_FUNC(_nettle_ghash_update, ghash_update_func) |
192 | DECLARE_FAT_FUNC_VAR(ghash_update, ghash_update_func, c) | |
193 | DECLARE_FAT_FUNC_VAR(ghash_update, ghash_update_func, arm64) | |
1585f6ac | 194 | |
e5a9dbf4 MT |
195 | DECLARE_FAT_FUNC(nettle_sha1_compress, sha1_compress_func) |
196 | DECLARE_FAT_FUNC_VAR(sha1_compress, sha1_compress_func, c) | |
197 | DECLARE_FAT_FUNC_VAR(sha1_compress, sha1_compress_func, arm64) | |
198 | ||
dba9cacc NM |
199 | DECLARE_FAT_FUNC(_nettle_sha256_compress_n, sha256_compress_n_func) |
200 | DECLARE_FAT_FUNC_VAR(sha256_compress_n, sha256_compress_n_func, c) | |
201 | DECLARE_FAT_FUNC_VAR(sha256_compress_n, sha256_compress_n_func, arm64) | |
7b446327 | 202 | |
1585f6ac MT |
203 | static void CONSTRUCTOR |
204 | fat_init (void) | |
205 | { | |
206 | struct arm64_features features; | |
207 | int verbose; | |
208 | ||
209 | get_arm64_features (&features); | |
210 | ||
211 | verbose = getenv (ENV_VERBOSE) != NULL; | |
212 | if (verbose) | |
39d1e2a3 MT |
213 | fprintf (stderr, "libnettle: cpu features:%s%s%s%s\n", |
214 | features.have_aes ? " aes instructions" : "", | |
e5a9dbf4 | 215 | features.have_pmull ? " polynomial multiply long instructions (PMULL/PMULL2)" : "", |
7b446327 MT |
216 | features.have_sha1 ? " sha1 instructions" : "", |
217 | features.have_sha2 ? " sha2 instructions" : ""); | |
218 | ||
39d1e2a3 MT |
219 | if (features.have_aes) |
220 | { | |
221 | if (verbose) | |
222 | fprintf (stderr, "libnettle: enabling hardware accelerated AES encrypt/decrypt code.\n"); | |
223 | nettle_aes128_encrypt_vec = _nettle_aes128_encrypt_arm64; | |
224 | nettle_aes128_decrypt_vec = _nettle_aes128_decrypt_arm64; | |
225 | nettle_aes192_encrypt_vec = _nettle_aes192_encrypt_arm64; | |
226 | nettle_aes192_decrypt_vec = _nettle_aes192_decrypt_arm64; | |
227 | nettle_aes256_encrypt_vec = _nettle_aes256_encrypt_arm64; | |
228 | nettle_aes256_decrypt_vec = _nettle_aes256_decrypt_arm64; | |
229 | } | |
230 | else | |
231 | { | |
232 | nettle_aes128_encrypt_vec = _nettle_aes128_encrypt_c; | |
233 | nettle_aes128_decrypt_vec = _nettle_aes128_decrypt_c; | |
234 | nettle_aes192_encrypt_vec = _nettle_aes192_encrypt_c; | |
235 | nettle_aes192_decrypt_vec = _nettle_aes192_decrypt_c; | |
236 | nettle_aes256_encrypt_vec = _nettle_aes256_encrypt_c; | |
237 | nettle_aes256_decrypt_vec = _nettle_aes256_decrypt_c; | |
238 | } | |
239 | ||
1585f6ac MT |
240 | if (features.have_pmull) |
241 | { | |
242 | if (verbose) | |
243 | fprintf (stderr, "libnettle: enabling hardware-accelerated polynomial multiply code.\n"); | |
d966ea0d | 244 | |
9939f866 NM |
245 | /* Make sure _nettle_ghash_set_key_vec function is compatible |
246 | with _nettle_ghash_update_vec function e.g. _nettle_ghash_key_c() | |
1585f6ac | 247 | fills gcm_key table with values that are incompatible with |
9939f866 NM |
248 | _nettle_ghash_update_arm64() */ |
249 | _nettle_ghash_set_key_vec = _nettle_ghash_set_key_arm64; | |
250 | _nettle_ghash_update_vec = _nettle_ghash_update_arm64; | |
1585f6ac MT |
251 | } |
252 | else | |
253 | { | |
9939f866 NM |
254 | _nettle_ghash_set_key_vec = _nettle_ghash_set_key_c; |
255 | _nettle_ghash_update_vec = _nettle_ghash_update_c; | |
1585f6ac | 256 | } |
e5a9dbf4 MT |
257 | if (features.have_sha1) |
258 | { | |
259 | if (verbose) | |
260 | fprintf (stderr, "libnettle: enabling hardware-accelerated sha1 compress code.\n"); | |
261 | nettle_sha1_compress_vec = _nettle_sha1_compress_arm64; | |
262 | } | |
263 | else | |
264 | { | |
265 | nettle_sha1_compress_vec = _nettle_sha1_compress_c; | |
266 | } | |
7b446327 MT |
267 | if (features.have_sha2) |
268 | { | |
269 | if (verbose) | |
270 | fprintf (stderr, "libnettle: enabling hardware-accelerated sha256 compress code.\n"); | |
dba9cacc | 271 | _nettle_sha256_compress_n_vec = _nettle_sha256_compress_n_arm64; |
7b446327 MT |
272 | } |
273 | else | |
274 | { | |
dba9cacc | 275 | _nettle_sha256_compress_n_vec = _nettle_sha256_compress_n_c; |
7b446327 | 276 | } |
1585f6ac MT |
277 | } |
278 | ||
39d1e2a3 MT |
279 | DEFINE_FAT_FUNC(nettle_aes128_encrypt, void, |
280 | (const struct aes128_ctx *ctx, size_t length, | |
281 | uint8_t *dst,const uint8_t *src), | |
282 | (ctx, length, dst, src)) | |
283 | DEFINE_FAT_FUNC(nettle_aes128_decrypt, void, | |
284 | (const struct aes128_ctx *ctx, size_t length, | |
285 | uint8_t *dst,const uint8_t *src), | |
286 | (ctx, length, dst, src)) | |
287 | ||
288 | DEFINE_FAT_FUNC(nettle_aes192_encrypt, void, | |
289 | (const struct aes192_ctx *ctx, size_t length, | |
290 | uint8_t *dst,const uint8_t *src), | |
291 | (ctx, length, dst, src)) | |
292 | DEFINE_FAT_FUNC(nettle_aes192_decrypt, void, | |
293 | (const struct aes192_ctx *ctx, size_t length, | |
294 | uint8_t *dst,const uint8_t *src), | |
295 | (ctx, length, dst, src)) | |
296 | ||
297 | DEFINE_FAT_FUNC(nettle_aes256_encrypt, void, | |
298 | (const struct aes256_ctx *ctx, size_t length, | |
299 | uint8_t *dst,const uint8_t *src), | |
300 | (ctx, length, dst, src)) | |
301 | DEFINE_FAT_FUNC(nettle_aes256_decrypt, void, | |
302 | (const struct aes256_ctx *ctx, size_t length, | |
303 | uint8_t *dst,const uint8_t *src), | |
304 | (ctx, length, dst, src)) | |
305 | ||
9939f866 NM |
306 | DEFINE_FAT_FUNC(_nettle_ghash_set_key, void, |
307 | (struct gcm_key *ctx, const union nettle_block16 *key), | |
308 | (ctx, key)) | |
309 | DEFINE_FAT_FUNC(_nettle_ghash_update, const uint8_t *, | |
310 | (const struct gcm_key *ctx, union nettle_block16 *state, | |
311 | size_t blocks, const uint8_t *data), | |
312 | (ctx, state, blocks, data)) | |
e5a9dbf4 MT |
313 | |
314 | DEFINE_FAT_FUNC(nettle_sha1_compress, void, | |
315 | (uint32_t *state, const uint8_t *input), | |
316 | (state, input)) | |
7b446327 | 317 | |
dba9cacc NM |
318 | DEFINE_FAT_FUNC(_nettle_sha256_compress_n, const uint8_t *, |
319 | (uint32_t *state, const uint32_t *k, | |
320 | size_t blocks, const uint8_t *input), | |
321 | (state, k, blocks, input)) |