]>
Commit | Line | Data |
---|---|---|
68d2658a GKH |
1 | From 357d065a44cdd77ed5ff35155a989f2a763e96ef Mon Sep 17 00:00:00 2001 |
2 | From: Daniel Axtens <dja@axtens.net> | |
3 | Date: Fri, 17 May 2019 01:40:02 +1000 | |
4 | Subject: crypto: vmx - ghash: do nosimd fallback manually | |
5 | ||
6 | From: Daniel Axtens <dja@axtens.net> | |
7 | ||
8 | commit 357d065a44cdd77ed5ff35155a989f2a763e96ef upstream. | |
9 | ||
10 | VMX ghash was using a fallback that did not support interleaving simd | |
11 | and nosimd operations, leading to failures in the extended test suite. | |
12 | ||
13 | If I understood correctly, Eric's suggestion was to use the same | |
14 | data format that the generic code uses, allowing us to call into it | |
15 | with the same contexts. I wasn't able to get that to work - I think | |
16 | there's a very different key structure and data layout being used. | |
17 | ||
18 | So instead steal the arm64 approach and perform the fallback | |
19 | operations directly if required. | |
20 | ||
21 | Fixes: cc333cd68dfa ("crypto: vmx - Adding GHASH routines for VMX module") | |
22 | Cc: stable@vger.kernel.org # v4.1+ | |
23 | Reported-by: Eric Biggers <ebiggers@google.com> | |
24 | Signed-off-by: Daniel Axtens <dja@axtens.net> | |
25 | Acked-by: Ard Biesheuvel <ard.biesheuvel@linaro.org> | |
26 | Tested-by: Michael Ellerman <mpe@ellerman.id.au> | |
27 | Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au> | |
28 | Signed-off-by: Daniel Axtens <dja@axtens.net> | |
29 | Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org> | |
30 | ||
31 | --- | |
32 | drivers/crypto/vmx/ghash.c | 212 ++++++++++++++++++--------------------------- | |
33 | 1 file changed, 86 insertions(+), 126 deletions(-) | |
34 | ||
35 | --- a/drivers/crypto/vmx/ghash.c | |
36 | +++ b/drivers/crypto/vmx/ghash.c | |
37 | @@ -1,22 +1,14 @@ | |
38 | +// SPDX-License-Identifier: GPL-2.0 | |
39 | /** | |
40 | * GHASH routines supporting VMX instructions on the Power 8 | |
41 | * | |
42 | - * Copyright (C) 2015 International Business Machines Inc. | |
43 | - * | |
44 | - * This program is free software; you can redistribute it and/or modify | |
45 | - * it under the terms of the GNU General Public License as published by | |
46 | - * the Free Software Foundation; version 2 only. | |
47 | - * | |
48 | - * This program is distributed in the hope that it will be useful, | |
49 | - * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
50 | - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
51 | - * GNU General Public License for more details. | |
52 | - * | |
53 | - * You should have received a copy of the GNU General Public License | |
54 | - * along with this program; if not, write to the Free Software | |
55 | - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. | |
56 | + * Copyright (C) 2015, 2019 International Business Machines Inc. | |
57 | * | |
58 | * Author: Marcelo Henrique Cerri <mhcerri@br.ibm.com> | |
59 | + * | |
60 | + * Extended by Daniel Axtens <dja@axtens.net> to replace the fallback | |
61 | + * mechanism. The new approach is based on arm64 code, which is: | |
62 | + * Copyright (C) 2014 - 2018 Linaro Ltd. <ard.biesheuvel@linaro.org> | |
63 | */ | |
64 | ||
65 | #include <linux/types.h> | |
66 | @@ -39,71 +31,25 @@ void gcm_ghash_p8(u64 Xi[2], const u128 | |
67 | const u8 *in, size_t len); | |
68 | ||
69 | struct p8_ghash_ctx { | |
70 | + /* key used by vector asm */ | |
71 | u128 htable[16]; | |
72 | - struct crypto_shash *fallback; | |
73 | + /* key used by software fallback */ | |
74 | + be128 key; | |
75 | }; | |
76 | ||
77 | struct p8_ghash_desc_ctx { | |
78 | u64 shash[2]; | |
79 | u8 buffer[GHASH_DIGEST_SIZE]; | |
80 | int bytes; | |
81 | - struct shash_desc fallback_desc; | |
82 | }; | |
83 | ||
84 | -static int p8_ghash_init_tfm(struct crypto_tfm *tfm) | |
85 | -{ | |
86 | - const char *alg = "ghash-generic"; | |
87 | - struct crypto_shash *fallback; | |
88 | - struct crypto_shash *shash_tfm = __crypto_shash_cast(tfm); | |
89 | - struct p8_ghash_ctx *ctx = crypto_tfm_ctx(tfm); | |
90 | - | |
91 | - fallback = crypto_alloc_shash(alg, 0, CRYPTO_ALG_NEED_FALLBACK); | |
92 | - if (IS_ERR(fallback)) { | |
93 | - printk(KERN_ERR | |
94 | - "Failed to allocate transformation for '%s': %ld\n", | |
95 | - alg, PTR_ERR(fallback)); | |
96 | - return PTR_ERR(fallback); | |
97 | - } | |
98 | - | |
99 | - crypto_shash_set_flags(fallback, | |
100 | - crypto_shash_get_flags((struct crypto_shash | |
101 | - *) tfm)); | |
102 | - | |
103 | - /* Check if the descsize defined in the algorithm is still enough. */ | |
104 | - if (shash_tfm->descsize < sizeof(struct p8_ghash_desc_ctx) | |
105 | - + crypto_shash_descsize(fallback)) { | |
106 | - printk(KERN_ERR | |
107 | - "Desc size of the fallback implementation (%s) does not match the expected value: %lu vs %u\n", | |
108 | - alg, | |
109 | - shash_tfm->descsize - sizeof(struct p8_ghash_desc_ctx), | |
110 | - crypto_shash_descsize(fallback)); | |
111 | - return -EINVAL; | |
112 | - } | |
113 | - ctx->fallback = fallback; | |
114 | - | |
115 | - return 0; | |
116 | -} | |
117 | - | |
118 | -static void p8_ghash_exit_tfm(struct crypto_tfm *tfm) | |
119 | -{ | |
120 | - struct p8_ghash_ctx *ctx = crypto_tfm_ctx(tfm); | |
121 | - | |
122 | - if (ctx->fallback) { | |
123 | - crypto_free_shash(ctx->fallback); | |
124 | - ctx->fallback = NULL; | |
125 | - } | |
126 | -} | |
127 | - | |
128 | static int p8_ghash_init(struct shash_desc *desc) | |
129 | { | |
130 | - struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm)); | |
131 | struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc); | |
132 | ||
133 | dctx->bytes = 0; | |
134 | memset(dctx->shash, 0, GHASH_DIGEST_SIZE); | |
135 | - dctx->fallback_desc.tfm = ctx->fallback; | |
136 | - dctx->fallback_desc.flags = desc->flags; | |
137 | - return crypto_shash_init(&dctx->fallback_desc); | |
138 | + return 0; | |
139 | } | |
140 | ||
141 | static int p8_ghash_setkey(struct crypto_shash *tfm, const u8 *key, | |
142 | @@ -121,7 +67,51 @@ static int p8_ghash_setkey(struct crypto | |
143 | disable_kernel_vsx(); | |
144 | pagefault_enable(); | |
145 | preempt_enable(); | |
146 | - return crypto_shash_setkey(ctx->fallback, key, keylen); | |
147 | + | |
148 | + memcpy(&ctx->key, key, GHASH_BLOCK_SIZE); | |
149 | + | |
150 | + return 0; | |
151 | +} | |
152 | + | |
153 | +static inline void __ghash_block(struct p8_ghash_ctx *ctx, | |
154 | + struct p8_ghash_desc_ctx *dctx) | |
155 | +{ | |
156 | + if (!IN_INTERRUPT) { | |
157 | + preempt_disable(); | |
158 | + pagefault_disable(); | |
159 | + enable_kernel_vsx(); | |
160 | + gcm_ghash_p8(dctx->shash, ctx->htable, | |
161 | + dctx->buffer, GHASH_DIGEST_SIZE); | |
162 | + disable_kernel_vsx(); | |
163 | + pagefault_enable(); | |
164 | + preempt_enable(); | |
165 | + } else { | |
166 | + crypto_xor((u8 *)dctx->shash, dctx->buffer, GHASH_BLOCK_SIZE); | |
167 | + gf128mul_lle((be128 *)dctx->shash, &ctx->key); | |
168 | + } | |
169 | +} | |
170 | + | |
171 | +static inline void __ghash_blocks(struct p8_ghash_ctx *ctx, | |
172 | + struct p8_ghash_desc_ctx *dctx, | |
173 | + const u8 *src, unsigned int srclen) | |
174 | +{ | |
175 | + if (!IN_INTERRUPT) { | |
176 | + preempt_disable(); | |
177 | + pagefault_disable(); | |
178 | + enable_kernel_vsx(); | |
179 | + gcm_ghash_p8(dctx->shash, ctx->htable, | |
180 | + src, srclen); | |
181 | + disable_kernel_vsx(); | |
182 | + pagefault_enable(); | |
183 | + preempt_enable(); | |
184 | + } else { | |
185 | + while (srclen >= GHASH_BLOCK_SIZE) { | |
186 | + crypto_xor((u8 *)dctx->shash, src, GHASH_BLOCK_SIZE); | |
187 | + gf128mul_lle((be128 *)dctx->shash, &ctx->key); | |
188 | + srclen -= GHASH_BLOCK_SIZE; | |
189 | + src += GHASH_BLOCK_SIZE; | |
190 | + } | |
191 | + } | |
192 | } | |
193 | ||
194 | static int p8_ghash_update(struct shash_desc *desc, | |
195 | @@ -131,49 +121,33 @@ static int p8_ghash_update(struct shash_ | |
196 | struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm)); | |
197 | struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc); | |
198 | ||
199 | - if (IN_INTERRUPT) { | |
200 | - return crypto_shash_update(&dctx->fallback_desc, src, | |
201 | - srclen); | |
202 | - } else { | |
203 | - if (dctx->bytes) { | |
204 | - if (dctx->bytes + srclen < GHASH_DIGEST_SIZE) { | |
205 | - memcpy(dctx->buffer + dctx->bytes, src, | |
206 | - srclen); | |
207 | - dctx->bytes += srclen; | |
208 | - return 0; | |
209 | - } | |
210 | + if (dctx->bytes) { | |
211 | + if (dctx->bytes + srclen < GHASH_DIGEST_SIZE) { | |
212 | memcpy(dctx->buffer + dctx->bytes, src, | |
213 | - GHASH_DIGEST_SIZE - dctx->bytes); | |
214 | - preempt_disable(); | |
215 | - pagefault_disable(); | |
216 | - enable_kernel_vsx(); | |
217 | - gcm_ghash_p8(dctx->shash, ctx->htable, | |
218 | - dctx->buffer, GHASH_DIGEST_SIZE); | |
219 | - disable_kernel_vsx(); | |
220 | - pagefault_enable(); | |
221 | - preempt_enable(); | |
222 | - src += GHASH_DIGEST_SIZE - dctx->bytes; | |
223 | - srclen -= GHASH_DIGEST_SIZE - dctx->bytes; | |
224 | - dctx->bytes = 0; | |
225 | + srclen); | |
226 | + dctx->bytes += srclen; | |
227 | + return 0; | |
228 | } | |
229 | - len = srclen & ~(GHASH_DIGEST_SIZE - 1); | |
230 | - if (len) { | |
231 | - preempt_disable(); | |
232 | - pagefault_disable(); | |
233 | - enable_kernel_vsx(); | |
234 | - gcm_ghash_p8(dctx->shash, ctx->htable, src, len); | |
235 | - disable_kernel_vsx(); | |
236 | - pagefault_enable(); | |
237 | - preempt_enable(); | |
238 | - src += len; | |
239 | - srclen -= len; | |
240 | - } | |
241 | - if (srclen) { | |
242 | - memcpy(dctx->buffer, src, srclen); | |
243 | - dctx->bytes = srclen; | |
244 | - } | |
245 | - return 0; | |
246 | + memcpy(dctx->buffer + dctx->bytes, src, | |
247 | + GHASH_DIGEST_SIZE - dctx->bytes); | |
248 | + | |
249 | + __ghash_block(ctx, dctx); | |
250 | + | |
251 | + src += GHASH_DIGEST_SIZE - dctx->bytes; | |
252 | + srclen -= GHASH_DIGEST_SIZE - dctx->bytes; | |
253 | + dctx->bytes = 0; | |
254 | + } | |
255 | + len = srclen & ~(GHASH_DIGEST_SIZE - 1); | |
256 | + if (len) { | |
257 | + __ghash_blocks(ctx, dctx, src, len); | |
258 | + src += len; | |
259 | + srclen -= len; | |
260 | } | |
261 | + if (srclen) { | |
262 | + memcpy(dctx->buffer, src, srclen); | |
263 | + dctx->bytes = srclen; | |
264 | + } | |
265 | + return 0; | |
266 | } | |
267 | ||
268 | static int p8_ghash_final(struct shash_desc *desc, u8 *out) | |
269 | @@ -182,25 +156,14 @@ static int p8_ghash_final(struct shash_d | |
270 | struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm)); | |
271 | struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc); | |
272 | ||
273 | - if (IN_INTERRUPT) { | |
274 | - return crypto_shash_final(&dctx->fallback_desc, out); | |
275 | - } else { | |
276 | - if (dctx->bytes) { | |
277 | - for (i = dctx->bytes; i < GHASH_DIGEST_SIZE; i++) | |
278 | - dctx->buffer[i] = 0; | |
279 | - preempt_disable(); | |
280 | - pagefault_disable(); | |
281 | - enable_kernel_vsx(); | |
282 | - gcm_ghash_p8(dctx->shash, ctx->htable, | |
283 | - dctx->buffer, GHASH_DIGEST_SIZE); | |
284 | - disable_kernel_vsx(); | |
285 | - pagefault_enable(); | |
286 | - preempt_enable(); | |
287 | - dctx->bytes = 0; | |
288 | - } | |
289 | - memcpy(out, dctx->shash, GHASH_DIGEST_SIZE); | |
290 | - return 0; | |
291 | + if (dctx->bytes) { | |
292 | + for (i = dctx->bytes; i < GHASH_DIGEST_SIZE; i++) | |
293 | + dctx->buffer[i] = 0; | |
294 | + __ghash_block(ctx, dctx); | |
295 | + dctx->bytes = 0; | |
296 | } | |
297 | + memcpy(out, dctx->shash, GHASH_DIGEST_SIZE); | |
298 | + return 0; | |
299 | } | |
300 | ||
301 | struct shash_alg p8_ghash_alg = { | |
302 | @@ -215,11 +178,8 @@ struct shash_alg p8_ghash_alg = { | |
303 | .cra_name = "ghash", | |
304 | .cra_driver_name = "p8_ghash", | |
305 | .cra_priority = 1000, | |
306 | - .cra_flags = CRYPTO_ALG_NEED_FALLBACK, | |
307 | .cra_blocksize = GHASH_BLOCK_SIZE, | |
308 | .cra_ctxsize = sizeof(struct p8_ghash_ctx), | |
309 | .cra_module = THIS_MODULE, | |
310 | - .cra_init = p8_ghash_init_tfm, | |
311 | - .cra_exit = p8_ghash_exit_tfm, | |
312 | }, | |
313 | }; |