]>
Commit | Line | Data |
---|---|---|
2874c5fd | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
66be8951 MK |
2 | /* |
3 | * Cryptographic API. | |
4 | * | |
5 | * Glue code for the SHA1 Secure Hash Algorithm assembler implementation using | |
6 | * Supplemental SSE3 instructions. | |
7 | * | |
8 | * This file is based on sha1_generic.c | |
9 | * | |
10 | * Copyright (c) Alan Smithee. | |
11 | * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk> | |
12 | * Copyright (c) Jean-Francois Dive <jef@linuxbe.org> | |
13 | * Copyright (c) Mathias Krause <minipli@googlemail.com> | |
7c1da8d0 | 14 | * Copyright (c) Chandramouli Narayanan <mouli@linux.intel.com> |
66be8951 MK |
15 | */ |
16 | ||
17 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt | |
18 | ||
19 | #include <crypto/internal/hash.h> | |
f2abe0d7 | 20 | #include <crypto/internal/simd.h> |
66be8951 MK |
21 | #include <linux/init.h> |
22 | #include <linux/module.h> | |
23 | #include <linux/mm.h> | |
66be8951 MK |
24 | #include <linux/types.h> |
25 | #include <crypto/sha.h> | |
824b4376 | 26 | #include <crypto/sha1_base.h> |
f2abe0d7 | 27 | #include <asm/simd.h> |
66be8951 | 28 | |
85c66ecd | 29 | static int sha1_update(struct shash_desc *desc, const u8 *data, |
41419a28 | 30 | unsigned int len, sha1_block_fn *sha1_xform) |
66be8951 MK |
31 | { |
32 | struct sha1_state *sctx = shash_desc_ctx(desc); | |
66be8951 | 33 | |
f2abe0d7 | 34 | if (!crypto_simd_usable() || |
824b4376 AB |
35 | (sctx->count % SHA1_BLOCK_SIZE) + len < SHA1_BLOCK_SIZE) |
36 | return crypto_sha1_update(desc, data, len); | |
66be8951 | 37 | |
41419a28 KC |
38 | /* |
39 | * Make sure struct sha1_state begins directly with the SHA1 | |
40 | * 160-bit internal state, as this is what the asm functions expect. | |
41 | */ | |
824b4376 | 42 | BUILD_BUG_ON(offsetof(struct sha1_state, state) != 0); |
66be8951 | 43 | |
824b4376 | 44 | kernel_fpu_begin(); |
41419a28 | 45 | sha1_base_do_update(desc, data, len, sha1_xform); |
824b4376 | 46 | kernel_fpu_end(); |
66be8951 MK |
47 | |
48 | return 0; | |
49 | } | |
50 | ||
85c66ecd | 51 | static int sha1_finup(struct shash_desc *desc, const u8 *data, |
41419a28 | 52 | unsigned int len, u8 *out, sha1_block_fn *sha1_xform) |
66be8951 | 53 | { |
f2abe0d7 | 54 | if (!crypto_simd_usable()) |
824b4376 | 55 | return crypto_sha1_finup(desc, data, len, out); |
66be8951 | 56 | |
824b4376 AB |
57 | kernel_fpu_begin(); |
58 | if (len) | |
41419a28 KC |
59 | sha1_base_do_update(desc, data, len, sha1_xform); |
60 | sha1_base_do_finalize(desc, sha1_xform); | |
824b4376 | 61 | kernel_fpu_end(); |
66be8951 | 62 | |
824b4376 | 63 | return sha1_base_finish(desc, out); |
66be8951 MK |
64 | } |
65 | ||
41419a28 KC |
66 | asmlinkage void sha1_transform_ssse3(struct sha1_state *state, |
67 | const u8 *data, int blocks); | |
85c66ecd | 68 | |
69 | static int sha1_ssse3_update(struct shash_desc *desc, const u8 *data, | |
70 | unsigned int len) | |
66be8951 | 71 | { |
41419a28 | 72 | return sha1_update(desc, data, len, sha1_transform_ssse3); |
66be8951 MK |
73 | } |
74 | ||
85c66ecd | 75 | static int sha1_ssse3_finup(struct shash_desc *desc, const u8 *data, |
76 | unsigned int len, u8 *out) | |
7c1da8d0 | 77 | { |
41419a28 | 78 | return sha1_finup(desc, data, len, out, sha1_transform_ssse3); |
85c66ecd | 79 | } |
80 | ||
81 | /* Add padding and return the message digest. */ | |
82 | static int sha1_ssse3_final(struct shash_desc *desc, u8 *out) | |
83 | { | |
84 | return sha1_ssse3_finup(desc, NULL, 0, out); | |
7c1da8d0 | 85 | } |
7c1da8d0 | 86 | |
85c66ecd | 87 | static struct shash_alg sha1_ssse3_alg = { |
66be8951 | 88 | .digestsize = SHA1_DIGEST_SIZE, |
824b4376 | 89 | .init = sha1_base_init, |
66be8951 MK |
90 | .update = sha1_ssse3_update, |
91 | .final = sha1_ssse3_final, | |
824b4376 | 92 | .finup = sha1_ssse3_finup, |
66be8951 | 93 | .descsize = sizeof(struct sha1_state), |
66be8951 MK |
94 | .base = { |
95 | .cra_name = "sha1", | |
85c66ecd | 96 | .cra_driver_name = "sha1-ssse3", |
66be8951 | 97 | .cra_priority = 150, |
66be8951 MK |
98 | .cra_blocksize = SHA1_BLOCK_SIZE, |
99 | .cra_module = THIS_MODULE, | |
100 | } | |
101 | }; | |
102 | ||
85c66ecd | 103 | static int register_sha1_ssse3(void) |
104 | { | |
105 | if (boot_cpu_has(X86_FEATURE_SSSE3)) | |
106 | return crypto_register_shash(&sha1_ssse3_alg); | |
107 | return 0; | |
108 | } | |
109 | ||
110 | static void unregister_sha1_ssse3(void) | |
111 | { | |
112 | if (boot_cpu_has(X86_FEATURE_SSSE3)) | |
113 | crypto_unregister_shash(&sha1_ssse3_alg); | |
114 | } | |
115 | ||
41419a28 KC |
116 | asmlinkage void sha1_transform_avx(struct sha1_state *state, |
117 | const u8 *data, int blocks); | |
85c66ecd | 118 | |
119 | static int sha1_avx_update(struct shash_desc *desc, const u8 *data, | |
120 | unsigned int len) | |
121 | { | |
41419a28 | 122 | return sha1_update(desc, data, len, sha1_transform_avx); |
85c66ecd | 123 | } |
124 | ||
125 | static int sha1_avx_finup(struct shash_desc *desc, const u8 *data, | |
126 | unsigned int len, u8 *out) | |
127 | { | |
41419a28 | 128 | return sha1_finup(desc, data, len, out, sha1_transform_avx); |
85c66ecd | 129 | } |
130 | ||
131 | static int sha1_avx_final(struct shash_desc *desc, u8 *out) | |
132 | { | |
133 | return sha1_avx_finup(desc, NULL, 0, out); | |
134 | } | |
135 | ||
136 | static struct shash_alg sha1_avx_alg = { | |
137 | .digestsize = SHA1_DIGEST_SIZE, | |
138 | .init = sha1_base_init, | |
139 | .update = sha1_avx_update, | |
140 | .final = sha1_avx_final, | |
141 | .finup = sha1_avx_finup, | |
142 | .descsize = sizeof(struct sha1_state), | |
143 | .base = { | |
144 | .cra_name = "sha1", | |
145 | .cra_driver_name = "sha1-avx", | |
146 | .cra_priority = 160, | |
85c66ecd | 147 | .cra_blocksize = SHA1_BLOCK_SIZE, |
148 | .cra_module = THIS_MODULE, | |
149 | } | |
150 | }; | |
151 | ||
152 | static bool avx_usable(void) | |
66be8951 | 153 | { |
d91cab78 | 154 | if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) { |
da154e82 | 155 | if (boot_cpu_has(X86_FEATURE_AVX)) |
d1e50966 | 156 | pr_info("AVX detected but unusable.\n"); |
66be8951 MK |
157 | return false; |
158 | } | |
159 | ||
160 | return true; | |
161 | } | |
6ca5afb8 | 162 | |
85c66ecd | 163 | static int register_sha1_avx(void) |
164 | { | |
165 | if (avx_usable()) | |
166 | return crypto_register_shash(&sha1_avx_alg); | |
167 | return 0; | |
168 | } | |
169 | ||
170 | static void unregister_sha1_avx(void) | |
6ca5afb8 | 171 | { |
85c66ecd | 172 | if (avx_usable()) |
173 | crypto_unregister_shash(&sha1_avx_alg); | |
174 | } | |
175 | ||
85c66ecd | 176 | #define SHA1_AVX2_BLOCK_OPTSIZE 4 /* optimal 4*64 bytes of SHA1 blocks */ |
177 | ||
41419a28 KC |
178 | asmlinkage void sha1_transform_avx2(struct sha1_state *state, |
179 | const u8 *data, int blocks); | |
85c66ecd | 180 | |
181 | static bool avx2_usable(void) | |
6ca5afb8 | 182 | { |
8861249c | 183 | if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2) |
85c66ecd | 184 | && boot_cpu_has(X86_FEATURE_BMI1) |
185 | && boot_cpu_has(X86_FEATURE_BMI2)) | |
6ca5afb8 MK |
186 | return true; |
187 | ||
188 | return false; | |
189 | } | |
66be8951 | 190 | |
41419a28 KC |
191 | static void sha1_apply_transform_avx2(struct sha1_state *state, |
192 | const u8 *data, int blocks) | |
66be8951 | 193 | { |
85c66ecd | 194 | /* Select the optimal transform based on data block size */ |
41419a28 KC |
195 | if (blocks >= SHA1_AVX2_BLOCK_OPTSIZE) |
196 | sha1_transform_avx2(state, data, blocks); | |
85c66ecd | 197 | else |
41419a28 | 198 | sha1_transform_avx(state, data, blocks); |
85c66ecd | 199 | } |
6ca5afb8 | 200 | |
85c66ecd | 201 | static int sha1_avx2_update(struct shash_desc *desc, const u8 *data, |
202 | unsigned int len) | |
203 | { | |
41419a28 | 204 | return sha1_update(desc, data, len, sha1_apply_transform_avx2); |
85c66ecd | 205 | } |
66be8951 | 206 | |
85c66ecd | 207 | static int sha1_avx2_finup(struct shash_desc *desc, const u8 *data, |
208 | unsigned int len, u8 *out) | |
209 | { | |
41419a28 | 210 | return sha1_finup(desc, data, len, out, sha1_apply_transform_avx2); |
85c66ecd | 211 | } |
212 | ||
213 | static int sha1_avx2_final(struct shash_desc *desc, u8 *out) | |
214 | { | |
215 | return sha1_avx2_finup(desc, NULL, 0, out); | |
216 | } | |
217 | ||
218 | static struct shash_alg sha1_avx2_alg = { | |
219 | .digestsize = SHA1_DIGEST_SIZE, | |
220 | .init = sha1_base_init, | |
221 | .update = sha1_avx2_update, | |
222 | .final = sha1_avx2_final, | |
223 | .finup = sha1_avx2_finup, | |
224 | .descsize = sizeof(struct sha1_state), | |
225 | .base = { | |
226 | .cra_name = "sha1", | |
227 | .cra_driver_name = "sha1-avx2", | |
228 | .cra_priority = 170, | |
85c66ecd | 229 | .cra_blocksize = SHA1_BLOCK_SIZE, |
230 | .cra_module = THIS_MODULE, | |
7c1da8d0 | 231 | } |
85c66ecd | 232 | }; |
233 | ||
234 | static int register_sha1_avx2(void) | |
235 | { | |
236 | if (avx2_usable()) | |
237 | return crypto_register_shash(&sha1_avx2_alg); | |
238 | return 0; | |
239 | } | |
240 | ||
241 | static void unregister_sha1_avx2(void) | |
242 | { | |
243 | if (avx2_usable()) | |
244 | crypto_unregister_shash(&sha1_avx2_alg); | |
245 | } | |
246 | ||
95fca7df | 247 | #ifdef CONFIG_AS_SHA1_NI |
41419a28 KC |
248 | asmlinkage void sha1_ni_transform(struct sha1_state *digest, const u8 *data, |
249 | int rounds); | |
85c66ecd | 250 | |
251 | static int sha1_ni_update(struct shash_desc *desc, const u8 *data, | |
252 | unsigned int len) | |
253 | { | |
41419a28 | 254 | return sha1_update(desc, data, len, sha1_ni_transform); |
85c66ecd | 255 | } |
256 | ||
257 | static int sha1_ni_finup(struct shash_desc *desc, const u8 *data, | |
258 | unsigned int len, u8 *out) | |
259 | { | |
41419a28 | 260 | return sha1_finup(desc, data, len, out, sha1_ni_transform); |
85c66ecd | 261 | } |
262 | ||
263 | static int sha1_ni_final(struct shash_desc *desc, u8 *out) | |
264 | { | |
265 | return sha1_ni_finup(desc, NULL, 0, out); | |
266 | } | |
267 | ||
268 | static struct shash_alg sha1_ni_alg = { | |
269 | .digestsize = SHA1_DIGEST_SIZE, | |
270 | .init = sha1_base_init, | |
271 | .update = sha1_ni_update, | |
272 | .final = sha1_ni_final, | |
273 | .finup = sha1_ni_finup, | |
274 | .descsize = sizeof(struct sha1_state), | |
275 | .base = { | |
276 | .cra_name = "sha1", | |
277 | .cra_driver_name = "sha1-ni", | |
278 | .cra_priority = 250, | |
85c66ecd | 279 | .cra_blocksize = SHA1_BLOCK_SIZE, |
280 | .cra_module = THIS_MODULE, | |
95fca7df | 281 | } |
85c66ecd | 282 | }; |
283 | ||
284 | static int register_sha1_ni(void) | |
285 | { | |
286 | if (boot_cpu_has(X86_FEATURE_SHA_NI)) | |
287 | return crypto_register_shash(&sha1_ni_alg); | |
288 | return 0; | |
289 | } | |
290 | ||
291 | static void unregister_sha1_ni(void) | |
292 | { | |
293 | if (boot_cpu_has(X86_FEATURE_SHA_NI)) | |
294 | crypto_unregister_shash(&sha1_ni_alg); | |
295 | } | |
296 | ||
297 | #else | |
298 | static inline int register_sha1_ni(void) { return 0; } | |
299 | static inline void unregister_sha1_ni(void) { } | |
66be8951 MK |
300 | #endif |
301 | ||
302 | static int __init sha1_ssse3_mod_init(void) | |
303 | { | |
85c66ecd | 304 | if (register_sha1_ssse3()) |
305 | goto fail; | |
6ca5afb8 | 306 | |
85c66ecd | 307 | if (register_sha1_avx()) { |
308 | unregister_sha1_ssse3(); | |
309 | goto fail; | |
7c1da8d0 | 310 | } |
66be8951 | 311 | |
85c66ecd | 312 | if (register_sha1_avx2()) { |
313 | unregister_sha1_avx(); | |
314 | unregister_sha1_ssse3(); | |
315 | goto fail; | |
7c1da8d0 | 316 | } |
66be8951 | 317 | |
85c66ecd | 318 | if (register_sha1_ni()) { |
319 | unregister_sha1_avx2(); | |
320 | unregister_sha1_avx(); | |
321 | unregister_sha1_ssse3(); | |
322 | goto fail; | |
66be8951 | 323 | } |
66be8951 | 324 | |
85c66ecd | 325 | return 0; |
326 | fail: | |
66be8951 MK |
327 | return -ENODEV; |
328 | } | |
329 | ||
330 | static void __exit sha1_ssse3_mod_fini(void) | |
331 | { | |
85c66ecd | 332 | unregister_sha1_ni(); |
333 | unregister_sha1_avx2(); | |
334 | unregister_sha1_avx(); | |
335 | unregister_sha1_ssse3(); | |
66be8951 MK |
336 | } |
337 | ||
338 | module_init(sha1_ssse3_mod_init); | |
339 | module_exit(sha1_ssse3_mod_fini); | |
340 | ||
341 | MODULE_LICENSE("GPL"); | |
342 | MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, Supplemental SSE3 accelerated"); | |
343 | ||
5d26a105 | 344 | MODULE_ALIAS_CRYPTO("sha1"); |
1a445e8e SM |
345 | MODULE_ALIAS_CRYPTO("sha1-ssse3"); |
346 | MODULE_ALIAS_CRYPTO("sha1-avx"); | |
347 | MODULE_ALIAS_CRYPTO("sha1-avx2"); | |
348 | #ifdef CONFIG_AS_SHA1_NI | |
349 | MODULE_ALIAS_CRYPTO("sha1-ni"); | |
350 | #endif |