]> git.ipfire.org Git - thirdparty/linux.git/blob - drivers/crypto/ccp/ccp-ops.c
projects / thirdparty / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge tag 'hyperv-fixes-signed' of git://git.kernel.org/pub/scm/linux/kernel/git...
[thirdparty/linux.git] / drivers / crypto / ccp / ccp-ops.c
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * AMD Cryptographic Coprocessor (CCP) driver
4 *
5 * Copyright (C) 2013-2019 Advanced Micro Devices, Inc.
6 *
7 * Author: Tom Lendacky <thomas.lendacky@amd.com>
8 * Author: Gary R Hook <gary.hook@amd.com>
9 */
10
11 #include <linux/module.h>
12 #include <linux/kernel.h>
13 #include <linux/pci.h>
14 #include <linux/interrupt.h>
15 #include <crypto/scatterwalk.h>
16 #include <crypto/des.h>
17 #include <linux/ccp.h>
18
19 #include "ccp-dev.h"
20
21 /* SHA initial context values */
22 static const __be32 ccp_sha1_init[SHA1_DIGEST_SIZE / sizeof(__be32)] = {
23 cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1),
24 cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3),
25 cpu_to_be32(SHA1_H4),
26 };
27
28 static const __be32 ccp_sha224_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
29 cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1),
30 cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3),
31 cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5),
32 cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7),
33 };
34
35 static const __be32 ccp_sha256_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
36 cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1),
37 cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3),
38 cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5),
39 cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7),
40 };
41
42 static const __be64 ccp_sha384_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
43 cpu_to_be64(SHA384_H0), cpu_to_be64(SHA384_H1),
44 cpu_to_be64(SHA384_H2), cpu_to_be64(SHA384_H3),
45 cpu_to_be64(SHA384_H4), cpu_to_be64(SHA384_H5),
46 cpu_to_be64(SHA384_H6), cpu_to_be64(SHA384_H7),
47 };
48
49 static const __be64 ccp_sha512_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
50 cpu_to_be64(SHA512_H0), cpu_to_be64(SHA512_H1),
51 cpu_to_be64(SHA512_H2), cpu_to_be64(SHA512_H3),
52 cpu_to_be64(SHA512_H4), cpu_to_be64(SHA512_H5),
53 cpu_to_be64(SHA512_H6), cpu_to_be64(SHA512_H7),
54 };
55
56 #define CCP_NEW_JOBID(ccp) ((ccp->vdata->version == CCP_VERSION(3, 0)) ? \
57 ccp_gen_jobid(ccp) : 0)
58
59 static u32 ccp_gen_jobid(struct ccp_device *ccp)
60 {
61 return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK;
62 }
63
64 static void ccp_sg_free(struct ccp_sg_workarea *wa)
65 {
66 if (wa->dma_count)
67 dma_unmap_sg(wa->dma_dev, wa->dma_sg, wa->nents, wa->dma_dir);
68
69 wa->dma_count = 0;
70 }
71
72 static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev,
73 struct scatterlist *sg, u64 len,
74 enum dma_data_direction dma_dir)
75 {
76 memset(wa, 0, sizeof(*wa));
77
78 wa->sg = sg;
79 if (!sg)
80 return 0;
81
82 wa->nents = sg_nents_for_len(sg, len);
83 if (wa->nents < 0)
84 return wa->nents;
85
86 wa->bytes_left = len;
87 wa->sg_used = 0;
88
89 if (len == 0)
90 return 0;
91
92 if (dma_dir == DMA_NONE)
93 return 0;
94
95 wa->dma_sg = sg;
96 wa->dma_dev = dev;
97 wa->dma_dir = dma_dir;
98 wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir);
99 if (!wa->dma_count)
100 return -ENOMEM;
101
102 return 0;
103 }
104
105 static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len)
106 {
107 unsigned int nbytes = min_t(u64, len, wa->bytes_left);
108
109 if (!wa->sg)
110 return;
111
112 wa->sg_used += nbytes;
113 wa->bytes_left -= nbytes;
114 if (wa->sg_used == wa->sg->length) {
115 wa->sg = sg_next(wa->sg);
116 wa->sg_used = 0;
117 }
118 }
119
120 static void ccp_dm_free(struct ccp_dm_workarea *wa)
121 {
122 if (wa->length <= CCP_DMAPOOL_MAX_SIZE) {
123 if (wa->address)
124 dma_pool_free(wa->dma_pool, wa->address,
125 wa->dma.address);
126 } else {
127 if (wa->dma.address)
128 dma_unmap_single(wa->dev, wa->dma.address, wa->length,
129 wa->dma.dir);
130 kfree(wa->address);
131 }
132
133 wa->address = NULL;
134 wa->dma.address = 0;
135 }
136
137 static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa,
138 struct ccp_cmd_queue *cmd_q,
139 unsigned int len,
140 enum dma_data_direction dir)
141 {
142 memset(wa, 0, sizeof(*wa));
143
144 if (!len)
145 return 0;
146
147 wa->dev = cmd_q->ccp->dev;
148 wa->length = len;
149
150 if (len <= CCP_DMAPOOL_MAX_SIZE) {
151 wa->dma_pool = cmd_q->dma_pool;
152
153 wa->address = dma_pool_alloc(wa->dma_pool, GFP_KERNEL,
154 &wa->dma.address);
155 if (!wa->address)
156 return -ENOMEM;
157
158 wa->dma.length = CCP_DMAPOOL_MAX_SIZE;
159
160 memset(wa->address, 0, CCP_DMAPOOL_MAX_SIZE);
161 } else {
162 wa->address = kzalloc(len, GFP_KERNEL);
163 if (!wa->address)
164 return -ENOMEM;
165
166 wa->dma.address = dma_map_single(wa->dev, wa->address, len,
167 dir);
168 if (dma_mapping_error(wa->dev, wa->dma.address))
169 return -ENOMEM;
170
171 wa->dma.length = len;
172 }
173 wa->dma.dir = dir;
174
175 return 0;
176 }
177
178 static int ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
179 struct scatterlist *sg, unsigned int sg_offset,
180 unsigned int len)
181 {
182 WARN_ON(!wa->address);
183
184 if (len > (wa->length - wa_offset))
185 return -EINVAL;
186
187 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
188 0);
189 return 0;
190 }
191
192 static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
193 struct scatterlist *sg, unsigned int sg_offset,
194 unsigned int len)
195 {
196 WARN_ON(!wa->address);
197
198 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
199 1);
200 }
201
202 static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa,
203 unsigned int wa_offset,
204 struct scatterlist *sg,
205 unsigned int sg_offset,
206 unsigned int len)
207 {
208 u8 *p, *q;
209 int rc;
210
211 rc = ccp_set_dm_area(wa, wa_offset, sg, sg_offset, len);
212 if (rc)
213 return rc;
214
215 p = wa->address + wa_offset;
216 q = p + len - 1;
217 while (p < q) {
218 *p = *p ^ *q;
219 *q = *p ^ *q;
220 *p = *p ^ *q;
221 p++;
222 q--;
223 }
224 return 0;
225 }
226
227 static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa,
228 unsigned int wa_offset,
229 struct scatterlist *sg,
230 unsigned int sg_offset,
231 unsigned int len)
232 {
233 u8 *p, *q;
234
235 p = wa->address + wa_offset;
236 q = p + len - 1;
237 while (p < q) {
238 *p = *p ^ *q;
239 *q = *p ^ *q;
240 *p = *p ^ *q;
241 p++;
242 q--;
243 }
244
245 ccp_get_dm_area(wa, wa_offset, sg, sg_offset, len);
246 }
247
248 static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q)
249 {
250 ccp_dm_free(&data->dm_wa);
251 ccp_sg_free(&data->sg_wa);
252 }
253
254 static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q,
255 struct scatterlist *sg, u64 sg_len,
256 unsigned int dm_len,
257 enum dma_data_direction dir)
258 {
259 int ret;
260
261 memset(data, 0, sizeof(*data));
262
263 ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len,
264 dir);
265 if (ret)
266 goto e_err;
267
268 ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir);
269 if (ret)
270 goto e_err;
271
272 return 0;
273
274 e_err:
275 ccp_free_data(data, cmd_q);
276
277 return ret;
278 }
279
280 static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from)
281 {
282 struct ccp_sg_workarea *sg_wa = &data->sg_wa;
283 struct ccp_dm_workarea *dm_wa = &data->dm_wa;
284 unsigned int buf_count, nbytes;
285
286 /* Clear the buffer if setting it */
287 if (!from)
288 memset(dm_wa->address, 0, dm_wa->length);
289
290 if (!sg_wa->sg)
291 return 0;
292
293 /* Perform the copy operation
294 * nbytes will always be <= UINT_MAX because dm_wa->length is
295 * an unsigned int
296 */
297 nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length);
298 scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used,
299 nbytes, from);
300
301 /* Update the structures and generate the count */
302 buf_count = 0;
303 while (sg_wa->bytes_left && (buf_count < dm_wa->length)) {
304 nbytes = min(sg_wa->sg->length - sg_wa->sg_used,
305 dm_wa->length - buf_count);
306 nbytes = min_t(u64, sg_wa->bytes_left, nbytes);
307
308 buf_count += nbytes;
309 ccp_update_sg_workarea(sg_wa, nbytes);
310 }
311
312 return buf_count;
313 }
314
315 static unsigned int ccp_fill_queue_buf(struct ccp_data *data)
316 {
317 return ccp_queue_buf(data, 0);
318 }
319
320 static unsigned int ccp_empty_queue_buf(struct ccp_data *data)
321 {
322 return ccp_queue_buf(data, 1);
323 }
324
325 static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst,
326 struct ccp_op *op, unsigned int block_size,
327 bool blocksize_op)
328 {
329 unsigned int sg_src_len, sg_dst_len, op_len;
330
331 /* The CCP can only DMA from/to one address each per operation. This
332 * requires that we find the smallest DMA area between the source
333 * and destination. The resulting len values will always be <= UINT_MAX
334 * because the dma length is an unsigned int.
335 */
336 sg_src_len = sg_dma_len(src->sg_wa.sg) - src->sg_wa.sg_used;
337 sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len);
338
339 if (dst) {
340 sg_dst_len = sg_dma_len(dst->sg_wa.sg) - dst->sg_wa.sg_used;
341 sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len);
342 op_len = min(sg_src_len, sg_dst_len);
343 } else {
344 op_len = sg_src_len;
345 }
346
347 /* The data operation length will be at least block_size in length
348 * or the smaller of available sg room remaining for the source or
349 * the destination
350 */
351 op_len = max(op_len, block_size);
352
353 /* Unless we have to buffer data, there's no reason to wait */
354 op->soc = 0;
355
356 if (sg_src_len < block_size) {
357 /* Not enough data in the sg element, so it
358 * needs to be buffered into a blocksize chunk
359 */
360 int cp_len = ccp_fill_queue_buf(src);
361
362 op->soc = 1;
363 op->src.u.dma.address = src->dm_wa.dma.address;
364 op->src.u.dma.offset = 0;
365 op->src.u.dma.length = (blocksize_op) ? block_size : cp_len;
366 } else {
367 /* Enough data in the sg element, but we need to
368 * adjust for any previously copied data
369 */
370 op->src.u.dma.address = sg_dma_address(src->sg_wa.sg);
371 op->src.u.dma.offset = src->sg_wa.sg_used;
372 op->src.u.dma.length = op_len & ~(block_size - 1);
373
374 ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length);
375 }
376
377 if (dst) {
378 if (sg_dst_len < block_size) {
379 /* Not enough room in the sg element or we're on the
380 * last piece of data (when using padding), so the
381 * output needs to be buffered into a blocksize chunk
382 */
383 op->soc = 1;
384 op->dst.u.dma.address = dst->dm_wa.dma.address;
385 op->dst.u.dma.offset = 0;
386 op->dst.u.dma.length = op->src.u.dma.length;
387 } else {
388 /* Enough room in the sg element, but we need to
389 * adjust for any previously used area
390 */
391 op->dst.u.dma.address = sg_dma_address(dst->sg_wa.sg);
392 op->dst.u.dma.offset = dst->sg_wa.sg_used;
393 op->dst.u.dma.length = op->src.u.dma.length;
394 }
395 }
396 }
397
398 static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst,
399 struct ccp_op *op)
400 {
401 op->init = 0;
402
403 if (dst) {
404 if (op->dst.u.dma.address == dst->dm_wa.dma.address)
405 ccp_empty_queue_buf(dst);
406 else
407 ccp_update_sg_workarea(&dst->sg_wa,
408 op->dst.u.dma.length);
409 }
410 }
411
412 static int ccp_copy_to_from_sb(struct ccp_cmd_queue *cmd_q,
413 struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
414 u32 byte_swap, bool from)
415 {
416 struct ccp_op op;
417
418 memset(&op, 0, sizeof(op));
419
420 op.cmd_q = cmd_q;
421 op.jobid = jobid;
422 op.eom = 1;
423
424 if (from) {
425 op.soc = 1;
426 op.src.type = CCP_MEMTYPE_SB;
427 op.src.u.sb = sb;
428 op.dst.type = CCP_MEMTYPE_SYSTEM;
429 op.dst.u.dma.address = wa->dma.address;
430 op.dst.u.dma.length = wa->length;
431 } else {
432 op.src.type = CCP_MEMTYPE_SYSTEM;
433 op.src.u.dma.address = wa->dma.address;
434 op.src.u.dma.length = wa->length;
435 op.dst.type = CCP_MEMTYPE_SB;
436 op.dst.u.sb = sb;
437 }
438
439 op.u.passthru.byte_swap = byte_swap;
440
441 return cmd_q->ccp->vdata->perform->passthru(&op);
442 }
443
444 static int ccp_copy_to_sb(struct ccp_cmd_queue *cmd_q,
445 struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
446 u32 byte_swap)
447 {
448 return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, false);
449 }
450
451 static int ccp_copy_from_sb(struct ccp_cmd_queue *cmd_q,
452 struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
453 u32 byte_swap)
454 {
455 return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, true);
456 }
457
458 static int ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q,
459 struct ccp_cmd *cmd)
460 {
461 struct ccp_aes_engine *aes = &cmd->u.aes;
462 struct ccp_dm_workarea key, ctx;
463 struct ccp_data src;
464 struct ccp_op op;
465 unsigned int dm_offset;
466 int ret;
467
468 if (!((aes->key_len == AES_KEYSIZE_128) ||
469 (aes->key_len == AES_KEYSIZE_192) ||
470 (aes->key_len == AES_KEYSIZE_256)))
471 return -EINVAL;
472
473 if (aes->src_len & (AES_BLOCK_SIZE - 1))
474 return -EINVAL;
475
476 if (aes->iv_len != AES_BLOCK_SIZE)
477 return -EINVAL;
478
479 if (!aes->key || !aes->iv || !aes->src)
480 return -EINVAL;
481
482 if (aes->cmac_final) {
483 if (aes->cmac_key_len != AES_BLOCK_SIZE)
484 return -EINVAL;
485
486 if (!aes->cmac_key)
487 return -EINVAL;
488 }
489
490 BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
491 BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
492
493 ret = -EIO;
494 memset(&op, 0, sizeof(op));
495 op.cmd_q = cmd_q;
496 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
497 op.sb_key = cmd_q->sb_key;
498 op.sb_ctx = cmd_q->sb_ctx;
499 op.init = 1;
500 op.u.aes.type = aes->type;
501 op.u.aes.mode = aes->mode;
502 op.u.aes.action = aes->action;
503
504 /* All supported key sizes fit in a single (32-byte) SB entry
505 * and must be in little endian format. Use the 256-bit byte
506 * swap passthru option to convert from big endian to little
507 * endian.
508 */
509 ret = ccp_init_dm_workarea(&key, cmd_q,
510 CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
511 DMA_TO_DEVICE);
512 if (ret)
513 return ret;
514
515 dm_offset = CCP_SB_BYTES - aes->key_len;
516 ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
517 if (ret)
518 goto e_key;
519 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
520 CCP_PASSTHRU_BYTESWAP_256BIT);
521 if (ret) {
522 cmd->engine_error = cmd_q->cmd_error;
523 goto e_key;
524 }
525
526 /* The AES context fits in a single (32-byte) SB entry and
527 * must be in little endian format. Use the 256-bit byte swap
528 * passthru option to convert from big endian to little endian.
529 */
530 ret = ccp_init_dm_workarea(&ctx, cmd_q,
531 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
532 DMA_BIDIRECTIONAL);
533 if (ret)
534 goto e_key;
535
536 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
537 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
538 if (ret)
539 goto e_ctx;
540 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
541 CCP_PASSTHRU_BYTESWAP_256BIT);
542 if (ret) {
543 cmd->engine_error = cmd_q->cmd_error;
544 goto e_ctx;
545 }
546
547 /* Send data to the CCP AES engine */
548 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
549 AES_BLOCK_SIZE, DMA_TO_DEVICE);
550 if (ret)
551 goto e_ctx;
552
553 while (src.sg_wa.bytes_left) {
554 ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true);
555 if (aes->cmac_final && !src.sg_wa.bytes_left) {
556 op.eom = 1;
557
558 /* Push the K1/K2 key to the CCP now */
559 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid,
560 op.sb_ctx,
561 CCP_PASSTHRU_BYTESWAP_256BIT);
562 if (ret) {
563 cmd->engine_error = cmd_q->cmd_error;
564 goto e_src;
565 }
566
567 ret = ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0,
568 aes->cmac_key_len);
569 if (ret)
570 goto e_src;
571 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
572 CCP_PASSTHRU_BYTESWAP_256BIT);
573 if (ret) {
574 cmd->engine_error = cmd_q->cmd_error;
575 goto e_src;
576 }
577 }
578
579 ret = cmd_q->ccp->vdata->perform->aes(&op);
580 if (ret) {
581 cmd->engine_error = cmd_q->cmd_error;
582 goto e_src;
583 }
584
585 ccp_process_data(&src, NULL, &op);
586 }
587
588 /* Retrieve the AES context - convert from LE to BE using
589 * 32-byte (256-bit) byteswapping
590 */
591 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
592 CCP_PASSTHRU_BYTESWAP_256BIT);
593 if (ret) {
594 cmd->engine_error = cmd_q->cmd_error;
595 goto e_src;
596 }
597
598 /* ...but we only need AES_BLOCK_SIZE bytes */
599 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
600 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
601
602 e_src:
603 ccp_free_data(&src, cmd_q);
604
605 e_ctx:
606 ccp_dm_free(&ctx);
607
608 e_key:
609 ccp_dm_free(&key);
610
611 return ret;
612 }
613
614 static int ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q,
615 struct ccp_cmd *cmd)
616 {
617 struct ccp_aes_engine *aes = &cmd->u.aes;
618 struct ccp_dm_workarea key, ctx, final_wa, tag;
619 struct ccp_data src, dst;
620 struct ccp_data aad;
621 struct ccp_op op;
622
623 unsigned long long *final;
624 unsigned int dm_offset;
625 unsigned int authsize;
626 unsigned int jobid;
627 unsigned int ilen;
628 bool in_place = true; /* Default value */
629 int ret;
630
631 struct scatterlist *p_inp, sg_inp[2];
632 struct scatterlist *p_tag, sg_tag[2];
633 struct scatterlist *p_outp, sg_outp[2];
634 struct scatterlist *p_aad;
635
636 if (!aes->iv)
637 return -EINVAL;
638
639 if (!((aes->key_len == AES_KEYSIZE_128) ||
640 (aes->key_len == AES_KEYSIZE_192) ||
641 (aes->key_len == AES_KEYSIZE_256)))
642 return -EINVAL;
643
644 if (!aes->key) /* Gotta have a key SGL */
645 return -EINVAL;
646
647 /* Zero defaults to 16 bytes, the maximum size */
648 authsize = aes->authsize ? aes->authsize : AES_BLOCK_SIZE;
649 switch (authsize) {
650 case 16:
651 case 15:
652 case 14:
653 case 13:
654 case 12:
655 case 8:
656 case 4:
657 break;
658 default:
659 return -EINVAL;
660 }
661
662 /* First, decompose the source buffer into AAD & PT,
663 * and the destination buffer into AAD, CT & tag, or
664 * the input into CT & tag.
665 * It is expected that the input and output SGs will
666 * be valid, even if the AAD and input lengths are 0.
667 */
668 p_aad = aes->src;
669 p_inp = scatterwalk_ffwd(sg_inp, aes->src, aes->aad_len);
670 p_outp = scatterwalk_ffwd(sg_outp, aes->dst, aes->aad_len);
671 if (aes->action == CCP_AES_ACTION_ENCRYPT) {
672 ilen = aes->src_len;
673 p_tag = scatterwalk_ffwd(sg_tag, p_outp, ilen);
674 } else {
675 /* Input length for decryption includes tag */
676 ilen = aes->src_len - authsize;
677 p_tag = scatterwalk_ffwd(sg_tag, p_inp, ilen);
678 }
679
680 jobid = CCP_NEW_JOBID(cmd_q->ccp);
681
682 memset(&op, 0, sizeof(op));
683 op.cmd_q = cmd_q;
684 op.jobid = jobid;
685 op.sb_key = cmd_q->sb_key; /* Pre-allocated */
686 op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
687 op.init = 1;
688 op.u.aes.type = aes->type;
689
690 /* Copy the key to the LSB */
691 ret = ccp_init_dm_workarea(&key, cmd_q,
692 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
693 DMA_TO_DEVICE);
694 if (ret)
695 return ret;
696
697 dm_offset = CCP_SB_BYTES - aes->key_len;
698 ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
699 if (ret)
700 goto e_key;
701 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
702 CCP_PASSTHRU_BYTESWAP_256BIT);
703 if (ret) {
704 cmd->engine_error = cmd_q->cmd_error;
705 goto e_key;
706 }
707
708 /* Copy the context (IV) to the LSB.
709 * There is an assumption here that the IV is 96 bits in length, plus
710 * a nonce of 32 bits. If no IV is present, use a zeroed buffer.
711 */
712 ret = ccp_init_dm_workarea(&ctx, cmd_q,
713 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
714 DMA_BIDIRECTIONAL);
715 if (ret)
716 goto e_key;
717
718 dm_offset = CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES - aes->iv_len;
719 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
720 if (ret)
721 goto e_ctx;
722
723 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
724 CCP_PASSTHRU_BYTESWAP_256BIT);
725 if (ret) {
726 cmd->engine_error = cmd_q->cmd_error;
727 goto e_ctx;
728 }
729
730 op.init = 1;
731 if (aes->aad_len > 0) {
732 /* Step 1: Run a GHASH over the Additional Authenticated Data */
733 ret = ccp_init_data(&aad, cmd_q, p_aad, aes->aad_len,
734 AES_BLOCK_SIZE,
735 DMA_TO_DEVICE);
736 if (ret)
737 goto e_ctx;
738
739 op.u.aes.mode = CCP_AES_MODE_GHASH;
740 op.u.aes.action = CCP_AES_GHASHAAD;
741
742 while (aad.sg_wa.bytes_left) {
743 ccp_prepare_data(&aad, NULL, &op, AES_BLOCK_SIZE, true);
744
745 ret = cmd_q->ccp->vdata->perform->aes(&op);
746 if (ret) {
747 cmd->engine_error = cmd_q->cmd_error;
748 goto e_aad;
749 }
750
751 ccp_process_data(&aad, NULL, &op);
752 op.init = 0;
753 }
754 }
755
756 op.u.aes.mode = CCP_AES_MODE_GCTR;
757 op.u.aes.action = aes->action;
758
759 if (ilen > 0) {
760 /* Step 2: Run a GCTR over the plaintext */
761 in_place = (sg_virt(p_inp) == sg_virt(p_outp)) ? true : false;
762
763 ret = ccp_init_data(&src, cmd_q, p_inp, ilen,
764 AES_BLOCK_SIZE,
765 in_place ? DMA_BIDIRECTIONAL
766 : DMA_TO_DEVICE);
767 if (ret)
768 goto e_ctx;
769
770 if (in_place) {
771 dst = src;
772 } else {
773 ret = ccp_init_data(&dst, cmd_q, p_outp, ilen,
774 AES_BLOCK_SIZE, DMA_FROM_DEVICE);
775 if (ret)
776 goto e_src;
777 }
778
779 op.soc = 0;
780 op.eom = 0;
781 op.init = 1;
782 while (src.sg_wa.bytes_left) {
783 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
784 if (!src.sg_wa.bytes_left) {
785 unsigned int nbytes = ilen % AES_BLOCK_SIZE;
786
787 if (nbytes) {
788 op.eom = 1;
789 op.u.aes.size = (nbytes * 8) - 1;
790 }
791 }
792
793 ret = cmd_q->ccp->vdata->perform->aes(&op);
794 if (ret) {
795 cmd->engine_error = cmd_q->cmd_error;
796 goto e_dst;
797 }
798
799 ccp_process_data(&src, &dst, &op);
800 op.init = 0;
801 }
802 }
803
804 /* Step 3: Update the IV portion of the context with the original IV */
805 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
806 CCP_PASSTHRU_BYTESWAP_256BIT);
807 if (ret) {
808 cmd->engine_error = cmd_q->cmd_error;
809 goto e_dst;
810 }
811
812 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
813 if (ret)
814 goto e_dst;
815
816 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
817 CCP_PASSTHRU_BYTESWAP_256BIT);
818 if (ret) {
819 cmd->engine_error = cmd_q->cmd_error;
820 goto e_dst;
821 }
822
823 /* Step 4: Concatenate the lengths of the AAD and source, and
824 * hash that 16 byte buffer.
825 */
826 ret = ccp_init_dm_workarea(&final_wa, cmd_q, AES_BLOCK_SIZE,
827 DMA_BIDIRECTIONAL);
828 if (ret)
829 goto e_dst;
830 final = (unsigned long long *) final_wa.address;
831 final[0] = cpu_to_be64(aes->aad_len * 8);
832 final[1] = cpu_to_be64(ilen * 8);
833
834 memset(&op, 0, sizeof(op));
835 op.cmd_q = cmd_q;
836 op.jobid = jobid;
837 op.sb_key = cmd_q->sb_key; /* Pre-allocated */
838 op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
839 op.init = 1;
840 op.u.aes.type = aes->type;
841 op.u.aes.mode = CCP_AES_MODE_GHASH;
842 op.u.aes.action = CCP_AES_GHASHFINAL;
843 op.src.type = CCP_MEMTYPE_SYSTEM;
844 op.src.u.dma.address = final_wa.dma.address;
845 op.src.u.dma.length = AES_BLOCK_SIZE;
846 op.dst.type = CCP_MEMTYPE_SYSTEM;
847 op.dst.u.dma.address = final_wa.dma.address;
848 op.dst.u.dma.length = AES_BLOCK_SIZE;
849 op.eom = 1;
850 op.u.aes.size = 0;
851 ret = cmd_q->ccp->vdata->perform->aes(&op);
852 if (ret)
853 goto e_dst;
854
855 if (aes->action == CCP_AES_ACTION_ENCRYPT) {
856 /* Put the ciphered tag after the ciphertext. */
857 ccp_get_dm_area(&final_wa, 0, p_tag, 0, authsize);
858 } else {
859 /* Does this ciphered tag match the input? */
860 ret = ccp_init_dm_workarea(&tag, cmd_q, authsize,
861 DMA_BIDIRECTIONAL);
862 if (ret)
863 goto e_tag;
864 ret = ccp_set_dm_area(&tag, 0, p_tag, 0, authsize);
865 if (ret)
866 goto e_tag;
867
868 ret = crypto_memneq(tag.address, final_wa.address,
869 authsize) ? -EBADMSG : 0;
870 ccp_dm_free(&tag);
871 }
872
873 e_tag:
874 ccp_dm_free(&final_wa);
875
876 e_dst:
877 if (ilen > 0 && !in_place)
878 ccp_free_data(&dst, cmd_q);
879
880 e_src:
881 if (ilen > 0)
882 ccp_free_data(&src, cmd_q);
883
884 e_aad:
885 if (aes->aad_len)
886 ccp_free_data(&aad, cmd_q);
887
888 e_ctx:
889 ccp_dm_free(&ctx);
890
891 e_key:
892 ccp_dm_free(&key);
893
894 return ret;
895 }
896
897 static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
898 {
899 struct ccp_aes_engine *aes = &cmd->u.aes;
900 struct ccp_dm_workarea key, ctx;
901 struct ccp_data src, dst;
902 struct ccp_op op;
903 unsigned int dm_offset;
904 bool in_place = false;
905 int ret;
906
907 if (aes->mode == CCP_AES_MODE_CMAC)
908 return ccp_run_aes_cmac_cmd(cmd_q, cmd);
909
910 if (aes->mode == CCP_AES_MODE_GCM)
911 return ccp_run_aes_gcm_cmd(cmd_q, cmd);
912
913 if (!((aes->key_len == AES_KEYSIZE_128) ||
914 (aes->key_len == AES_KEYSIZE_192) ||
915 (aes->key_len == AES_KEYSIZE_256)))
916 return -EINVAL;
917
918 if (((aes->mode == CCP_AES_MODE_ECB) ||
919 (aes->mode == CCP_AES_MODE_CBC)) &&
920 (aes->src_len & (AES_BLOCK_SIZE - 1)))
921 return -EINVAL;
922
923 if (!aes->key || !aes->src || !aes->dst)
924 return -EINVAL;
925
926 if (aes->mode != CCP_AES_MODE_ECB) {
927 if (aes->iv_len != AES_BLOCK_SIZE)
928 return -EINVAL;
929
930 if (!aes->iv)
931 return -EINVAL;
932 }
933
934 BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
935 BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
936
937 ret = -EIO;
938 memset(&op, 0, sizeof(op));
939 op.cmd_q = cmd_q;
940 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
941 op.sb_key = cmd_q->sb_key;
942 op.sb_ctx = cmd_q->sb_ctx;
943 op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1;
944 op.u.aes.type = aes->type;
945 op.u.aes.mode = aes->mode;
946 op.u.aes.action = aes->action;
947
948 /* All supported key sizes fit in a single (32-byte) SB entry
949 * and must be in little endian format. Use the 256-bit byte
950 * swap passthru option to convert from big endian to little
951 * endian.
952 */
953 ret = ccp_init_dm_workarea(&key, cmd_q,
954 CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
955 DMA_TO_DEVICE);
956 if (ret)
957 return ret;
958
959 dm_offset = CCP_SB_BYTES - aes->key_len;
960 ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
961 if (ret)
962 goto e_key;
963 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
964 CCP_PASSTHRU_BYTESWAP_256BIT);
965 if (ret) {
966 cmd->engine_error = cmd_q->cmd_error;
967 goto e_key;
968 }
969
970 /* The AES context fits in a single (32-byte) SB entry and
971 * must be in little endian format. Use the 256-bit byte swap
972 * passthru option to convert from big endian to little endian.
973 */
974 ret = ccp_init_dm_workarea(&ctx, cmd_q,
975 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
976 DMA_BIDIRECTIONAL);
977 if (ret)
978 goto e_key;
979
980 if (aes->mode != CCP_AES_MODE_ECB) {
981 /* Load the AES context - convert to LE */
982 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
983 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
984 if (ret)
985 goto e_ctx;
986 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
987 CCP_PASSTHRU_BYTESWAP_256BIT);
988 if (ret) {
989 cmd->engine_error = cmd_q->cmd_error;
990 goto e_ctx;
991 }
992 }
993 switch (aes->mode) {
994 case CCP_AES_MODE_CFB: /* CFB128 only */
995 case CCP_AES_MODE_CTR:
996 op.u.aes.size = AES_BLOCK_SIZE * BITS_PER_BYTE - 1;
997 break;
998 default:
999 op.u.aes.size = 0;
1000 }
1001
1002 /* Prepare the input and output data workareas. For in-place
1003 * operations we need to set the dma direction to BIDIRECTIONAL
1004 * and copy the src workarea to the dst workarea.
1005 */
1006 if (sg_virt(aes->src) == sg_virt(aes->dst))
1007 in_place = true;
1008
1009 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
1010 AES_BLOCK_SIZE,
1011 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1012 if (ret)
1013 goto e_ctx;
1014
1015 if (in_place) {
1016 dst = src;
1017 } else {
1018 ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len,
1019 AES_BLOCK_SIZE, DMA_FROM_DEVICE);
1020 if (ret)
1021 goto e_src;
1022 }
1023
1024 /* Send data to the CCP AES engine */
1025 while (src.sg_wa.bytes_left) {
1026 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
1027 if (!src.sg_wa.bytes_left) {
1028 op.eom = 1;
1029
1030 /* Since we don't retrieve the AES context in ECB
1031 * mode we have to wait for the operation to complete
1032 * on the last piece of data
1033 */
1034 if (aes->mode == CCP_AES_MODE_ECB)
1035 op.soc = 1;
1036 }
1037
1038 ret = cmd_q->ccp->vdata->perform->aes(&op);
1039 if (ret) {
1040 cmd->engine_error = cmd_q->cmd_error;
1041 goto e_dst;
1042 }
1043
1044 ccp_process_data(&src, &dst, &op);
1045 }
1046
1047 if (aes->mode != CCP_AES_MODE_ECB) {
1048 /* Retrieve the AES context - convert from LE to BE using
1049 * 32-byte (256-bit) byteswapping
1050 */
1051 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1052 CCP_PASSTHRU_BYTESWAP_256BIT);
1053 if (ret) {
1054 cmd->engine_error = cmd_q->cmd_error;
1055 goto e_dst;
1056 }
1057
1058 /* ...but we only need AES_BLOCK_SIZE bytes */
1059 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
1060 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
1061 }
1062
1063 e_dst:
1064 if (!in_place)
1065 ccp_free_data(&dst, cmd_q);
1066
1067 e_src:
1068 ccp_free_data(&src, cmd_q);
1069
1070 e_ctx:
1071 ccp_dm_free(&ctx);
1072
1073 e_key:
1074 ccp_dm_free(&key);
1075
1076 return ret;
1077 }
1078
1079 static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q,
1080 struct ccp_cmd *cmd)
1081 {
1082 struct ccp_xts_aes_engine *xts = &cmd->u.xts;
1083 struct ccp_dm_workarea key, ctx;
1084 struct ccp_data src, dst;
1085 struct ccp_op op;
1086 unsigned int unit_size, dm_offset;
1087 bool in_place = false;
1088 unsigned int sb_count;
1089 enum ccp_aes_type aestype;
1090 int ret;
1091
1092 switch (xts->unit_size) {
1093 case CCP_XTS_AES_UNIT_SIZE_16:
1094 unit_size = 16;
1095 break;
1096 case CCP_XTS_AES_UNIT_SIZE_512:
1097 unit_size = 512;
1098 break;
1099 case CCP_XTS_AES_UNIT_SIZE_1024:
1100 unit_size = 1024;
1101 break;
1102 case CCP_XTS_AES_UNIT_SIZE_2048:
1103 unit_size = 2048;
1104 break;
1105 case CCP_XTS_AES_UNIT_SIZE_4096:
1106 unit_size = 4096;
1107 break;
1108
1109 default:
1110 return -EINVAL;
1111 }
1112
1113 if (xts->key_len == AES_KEYSIZE_128)
1114 aestype = CCP_AES_TYPE_128;
1115 else if (xts->key_len == AES_KEYSIZE_256)
1116 aestype = CCP_AES_TYPE_256;
1117 else
1118 return -EINVAL;
1119
1120 if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1)))
1121 return -EINVAL;
1122
1123 if (xts->iv_len != AES_BLOCK_SIZE)
1124 return -EINVAL;
1125
1126 if (!xts->key || !xts->iv || !xts->src || !xts->dst)
1127 return -EINVAL;
1128
1129 BUILD_BUG_ON(CCP_XTS_AES_KEY_SB_COUNT != 1);
1130 BUILD_BUG_ON(CCP_XTS_AES_CTX_SB_COUNT != 1);
1131
1132 ret = -EIO;
1133 memset(&op, 0, sizeof(op));
1134 op.cmd_q = cmd_q;
1135 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1136 op.sb_key = cmd_q->sb_key;
1137 op.sb_ctx = cmd_q->sb_ctx;
1138 op.init = 1;
1139 op.u.xts.type = aestype;
1140 op.u.xts.action = xts->action;
1141 op.u.xts.unit_size = xts->unit_size;
1142
1143 /* A version 3 device only supports 128-bit keys, which fits into a
1144 * single SB entry. A version 5 device uses a 512-bit vector, so two
1145 * SB entries.
1146 */
1147 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0))
1148 sb_count = CCP_XTS_AES_KEY_SB_COUNT;
1149 else
1150 sb_count = CCP5_XTS_AES_KEY_SB_COUNT;
1151 ret = ccp_init_dm_workarea(&key, cmd_q,
1152 sb_count * CCP_SB_BYTES,
1153 DMA_TO_DEVICE);
1154 if (ret)
1155 return ret;
1156
1157 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
1158 /* All supported key sizes must be in little endian format.
1159 * Use the 256-bit byte swap passthru option to convert from
1160 * big endian to little endian.
1161 */
1162 dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128;
1163 ret = ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len);
1164 if (ret)
1165 goto e_key;
1166 ret = ccp_set_dm_area(&key, 0, xts->key, xts->key_len, xts->key_len);
1167 if (ret)
1168 goto e_key;
1169 } else {
1170 /* Version 5 CCPs use a 512-bit space for the key: each portion
1171 * occupies 256 bits, or one entire slot, and is zero-padded.
1172 */
1173 unsigned int pad;
1174
1175 dm_offset = CCP_SB_BYTES;
1176 pad = dm_offset - xts->key_len;
1177 ret = ccp_set_dm_area(&key, pad, xts->key, 0, xts->key_len);
1178 if (ret)
1179 goto e_key;
1180 ret = ccp_set_dm_area(&key, dm_offset + pad, xts->key,
1181 xts->key_len, xts->key_len);
1182 if (ret)
1183 goto e_key;
1184 }
1185 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
1186 CCP_PASSTHRU_BYTESWAP_256BIT);
1187 if (ret) {
1188 cmd->engine_error = cmd_q->cmd_error;
1189 goto e_key;
1190 }
1191
1192 /* The AES context fits in a single (32-byte) SB entry and
1193 * for XTS is already in little endian format so no byte swapping
1194 * is needed.
1195 */
1196 ret = ccp_init_dm_workarea(&ctx, cmd_q,
1197 CCP_XTS_AES_CTX_SB_COUNT * CCP_SB_BYTES,
1198 DMA_BIDIRECTIONAL);
1199 if (ret)
1200 goto e_key;
1201
1202 ret = ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len);
1203 if (ret)
1204 goto e_ctx;
1205 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1206 CCP_PASSTHRU_BYTESWAP_NOOP);
1207 if (ret) {
1208 cmd->engine_error = cmd_q->cmd_error;
1209 goto e_ctx;
1210 }
1211
1212 /* Prepare the input and output data workareas. For in-place
1213 * operations we need to set the dma direction to BIDIRECTIONAL
1214 * and copy the src workarea to the dst workarea.
1215 */
1216 if (sg_virt(xts->src) == sg_virt(xts->dst))
1217 in_place = true;
1218
1219 ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len,
1220 unit_size,
1221 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1222 if (ret)
1223 goto e_ctx;
1224
1225 if (in_place) {
1226 dst = src;
1227 } else {
1228 ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len,
1229 unit_size, DMA_FROM_DEVICE);
1230 if (ret)
1231 goto e_src;
1232 }
1233
1234 /* Send data to the CCP AES engine */
1235 while (src.sg_wa.bytes_left) {
1236 ccp_prepare_data(&src, &dst, &op, unit_size, true);
1237 if (!src.sg_wa.bytes_left)
1238 op.eom = 1;
1239
1240 ret = cmd_q->ccp->vdata->perform->xts_aes(&op);
1241 if (ret) {
1242 cmd->engine_error = cmd_q->cmd_error;
1243 goto e_dst;
1244 }
1245
1246 ccp_process_data(&src, &dst, &op);
1247 }
1248
1249 /* Retrieve the AES context - convert from LE to BE using
1250 * 32-byte (256-bit) byteswapping
1251 */
1252 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1253 CCP_PASSTHRU_BYTESWAP_256BIT);
1254 if (ret) {
1255 cmd->engine_error = cmd_q->cmd_error;
1256 goto e_dst;
1257 }
1258
1259 /* ...but we only need AES_BLOCK_SIZE bytes */
1260 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
1261 ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len);
1262
1263 e_dst:
1264 if (!in_place)
1265 ccp_free_data(&dst, cmd_q);
1266
1267 e_src:
1268 ccp_free_data(&src, cmd_q);
1269
1270 e_ctx:
1271 ccp_dm_free(&ctx);
1272
1273 e_key:
1274 ccp_dm_free(&key);
1275
1276 return ret;
1277 }
1278
1279 static int ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1280 {
1281 struct ccp_des3_engine *des3 = &cmd->u.des3;
1282
1283 struct ccp_dm_workarea key, ctx;
1284 struct ccp_data src, dst;
1285 struct ccp_op op;
1286 unsigned int dm_offset;
1287 unsigned int len_singlekey;
1288 bool in_place = false;
1289 int ret;
1290
1291 /* Error checks */
1292 if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0))
1293 return -EINVAL;
1294
1295 if (!cmd_q->ccp->vdata->perform->des3)
1296 return -EINVAL;
1297
1298 if (des3->key_len != DES3_EDE_KEY_SIZE)
1299 return -EINVAL;
1300
1301 if (((des3->mode == CCP_DES3_MODE_ECB) ||
1302 (des3->mode == CCP_DES3_MODE_CBC)) &&
1303 (des3->src_len & (DES3_EDE_BLOCK_SIZE - 1)))
1304 return -EINVAL;
1305
1306 if (!des3->key || !des3->src || !des3->dst)
1307 return -EINVAL;
1308
1309 if (des3->mode != CCP_DES3_MODE_ECB) {
1310 if (des3->iv_len != DES3_EDE_BLOCK_SIZE)
1311 return -EINVAL;
1312
1313 if (!des3->iv)
1314 return -EINVAL;
1315 }
1316
1317 ret = -EIO;
1318 /* Zero out all the fields of the command desc */
1319 memset(&op, 0, sizeof(op));
1320
1321 /* Set up the Function field */
1322 op.cmd_q = cmd_q;
1323 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1324 op.sb_key = cmd_q->sb_key;
1325
1326 op.init = (des3->mode == CCP_DES3_MODE_ECB) ? 0 : 1;
1327 op.u.des3.type = des3->type;
1328 op.u.des3.mode = des3->mode;
1329 op.u.des3.action = des3->action;
1330
1331 /*
1332 * All supported key sizes fit in a single (32-byte) KSB entry and
1333 * (like AES) must be in little endian format. Use the 256-bit byte
1334 * swap passthru option to convert from big endian to little endian.
1335 */
1336 ret = ccp_init_dm_workarea(&key, cmd_q,
1337 CCP_DES3_KEY_SB_COUNT * CCP_SB_BYTES,
1338 DMA_TO_DEVICE);
1339 if (ret)
1340 return ret;
1341
1342 /*
1343 * The contents of the key triplet are in the reverse order of what
1344 * is required by the engine. Copy the 3 pieces individually to put
1345 * them where they belong.
1346 */
1347 dm_offset = CCP_SB_BYTES - des3->key_len; /* Basic offset */
1348
1349 len_singlekey = des3->key_len / 3;
1350 ret = ccp_set_dm_area(&key, dm_offset + 2 * len_singlekey,
1351 des3->key, 0, len_singlekey);
1352 if (ret)
1353 goto e_key;
1354 ret = ccp_set_dm_area(&key, dm_offset + len_singlekey,
1355 des3->key, len_singlekey, len_singlekey);
1356 if (ret)
1357 goto e_key;
1358 ret = ccp_set_dm_area(&key, dm_offset,
1359 des3->key, 2 * len_singlekey, len_singlekey);
1360 if (ret)
1361 goto e_key;
1362
1363 /* Copy the key to the SB */
1364 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
1365 CCP_PASSTHRU_BYTESWAP_256BIT);
1366 if (ret) {
1367 cmd->engine_error = cmd_q->cmd_error;
1368 goto e_key;
1369 }
1370
1371 /*
1372 * The DES3 context fits in a single (32-byte) KSB entry and
1373 * must be in little endian format. Use the 256-bit byte swap
1374 * passthru option to convert from big endian to little endian.
1375 */
1376 if (des3->mode != CCP_DES3_MODE_ECB) {
1377 op.sb_ctx = cmd_q->sb_ctx;
1378
1379 ret = ccp_init_dm_workarea(&ctx, cmd_q,
1380 CCP_DES3_CTX_SB_COUNT * CCP_SB_BYTES,
1381 DMA_BIDIRECTIONAL);
1382 if (ret)
1383 goto e_key;
1384
1385 /* Load the context into the LSB */
1386 dm_offset = CCP_SB_BYTES - des3->iv_len;
1387 ret = ccp_set_dm_area(&ctx, dm_offset, des3->iv, 0,
1388 des3->iv_len);
1389 if (ret)
1390 goto e_ctx;
1391
1392 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1393 CCP_PASSTHRU_BYTESWAP_256BIT);
1394 if (ret) {
1395 cmd->engine_error = cmd_q->cmd_error;
1396 goto e_ctx;
1397 }
1398 }
1399
1400 /*
1401 * Prepare the input and output data workareas. For in-place
1402 * operations we need to set the dma direction to BIDIRECTIONAL
1403 * and copy the src workarea to the dst workarea.
1404 */
1405 if (sg_virt(des3->src) == sg_virt(des3->dst))
1406 in_place = true;
1407
1408 ret = ccp_init_data(&src, cmd_q, des3->src, des3->src_len,
1409 DES3_EDE_BLOCK_SIZE,
1410 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1411 if (ret)
1412 goto e_ctx;
1413
1414 if (in_place)
1415 dst = src;
1416 else {
1417 ret = ccp_init_data(&dst, cmd_q, des3->dst, des3->src_len,
1418 DES3_EDE_BLOCK_SIZE, DMA_FROM_DEVICE);
1419 if (ret)
1420 goto e_src;
1421 }
1422
1423 /* Send data to the CCP DES3 engine */
1424 while (src.sg_wa.bytes_left) {
1425 ccp_prepare_data(&src, &dst, &op, DES3_EDE_BLOCK_SIZE, true);
1426 if (!src.sg_wa.bytes_left) {
1427 op.eom = 1;
1428
1429 /* Since we don't retrieve the context in ECB mode
1430 * we have to wait for the operation to complete
1431 * on the last piece of data
1432 */
1433 op.soc = 0;
1434 }
1435
1436 ret = cmd_q->ccp->vdata->perform->des3(&op);
1437 if (ret) {
1438 cmd->engine_error = cmd_q->cmd_error;
1439 goto e_dst;
1440 }
1441
1442 ccp_process_data(&src, &dst, &op);
1443 }
1444
1445 if (des3->mode != CCP_DES3_MODE_ECB) {
1446 /* Retrieve the context and make BE */
1447 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1448 CCP_PASSTHRU_BYTESWAP_256BIT);
1449 if (ret) {
1450 cmd->engine_error = cmd_q->cmd_error;
1451 goto e_dst;
1452 }
1453
1454 /* ...but we only need the last DES3_EDE_BLOCK_SIZE bytes */
1455 ccp_get_dm_area(&ctx, dm_offset, des3->iv, 0,
1456 DES3_EDE_BLOCK_SIZE);
1457 }
1458 e_dst:
1459 if (!in_place)
1460 ccp_free_data(&dst, cmd_q);
1461
1462 e_src:
1463 ccp_free_data(&src, cmd_q);
1464
1465 e_ctx:
1466 if (des3->mode != CCP_DES3_MODE_ECB)
1467 ccp_dm_free(&ctx);
1468
1469 e_key:
1470 ccp_dm_free(&key);
1471
1472 return ret;
1473 }
1474
1475 static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1476 {
1477 struct ccp_sha_engine *sha = &cmd->u.sha;
1478 struct ccp_dm_workarea ctx;
1479 struct ccp_data src;
1480 struct ccp_op op;
1481 unsigned int ioffset, ooffset;
1482 unsigned int digest_size;
1483 int sb_count;
1484 const void *init;
1485 u64 block_size;
1486 int ctx_size;
1487 int ret;
1488
1489 switch (sha->type) {
1490 case CCP_SHA_TYPE_1:
1491 if (sha->ctx_len < SHA1_DIGEST_SIZE)
1492 return -EINVAL;
1493 block_size = SHA1_BLOCK_SIZE;
1494 break;
1495 case CCP_SHA_TYPE_224:
1496 if (sha->ctx_len < SHA224_DIGEST_SIZE)
1497 return -EINVAL;
1498 block_size = SHA224_BLOCK_SIZE;
1499 break;
1500 case CCP_SHA_TYPE_256:
1501 if (sha->ctx_len < SHA256_DIGEST_SIZE)
1502 return -EINVAL;
1503 block_size = SHA256_BLOCK_SIZE;
1504 break;
1505 case CCP_SHA_TYPE_384:
1506 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
1507 || sha->ctx_len < SHA384_DIGEST_SIZE)
1508 return -EINVAL;
1509 block_size = SHA384_BLOCK_SIZE;
1510 break;
1511 case CCP_SHA_TYPE_512:
1512 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
1513 || sha->ctx_len < SHA512_DIGEST_SIZE)
1514 return -EINVAL;
1515 block_size = SHA512_BLOCK_SIZE;
1516 break;
1517 default:
1518 return -EINVAL;
1519 }
1520
1521 if (!sha->ctx)
1522 return -EINVAL;
1523
1524 if (!sha->final && (sha->src_len & (block_size - 1)))
1525 return -EINVAL;
1526
1527 /* The version 3 device can't handle zero-length input */
1528 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
1529
1530 if (!sha->src_len) {
1531 unsigned int digest_len;
1532 const u8 *sha_zero;
1533
1534 /* Not final, just return */
1535 if (!sha->final)
1536 return 0;
1537
1538 /* CCP can't do a zero length sha operation so the
1539 * caller must buffer the data.
1540 */
1541 if (sha->msg_bits)
1542 return -EINVAL;
1543
1544 /* The CCP cannot perform zero-length sha operations
1545 * so the caller is required to buffer data for the
1546 * final operation. However, a sha operation for a
1547 * message with a total length of zero is valid so
1548 * known values are required to supply the result.
1549 */
1550 switch (sha->type) {
1551 case CCP_SHA_TYPE_1:
1552 sha_zero = sha1_zero_message_hash;
1553 digest_len = SHA1_DIGEST_SIZE;
1554 break;
1555 case CCP_SHA_TYPE_224:
1556 sha_zero = sha224_zero_message_hash;
1557 digest_len = SHA224_DIGEST_SIZE;
1558 break;
1559 case CCP_SHA_TYPE_256:
1560 sha_zero = sha256_zero_message_hash;
1561 digest_len = SHA256_DIGEST_SIZE;
1562 break;
1563 default:
1564 return -EINVAL;
1565 }
1566
1567 scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0,
1568 digest_len, 1);
1569
1570 return 0;
1571 }
1572 }
1573
1574 /* Set variables used throughout */
1575 switch (sha->type) {
1576 case CCP_SHA_TYPE_1:
1577 digest_size = SHA1_DIGEST_SIZE;
1578 init = (void *) ccp_sha1_init;
1579 ctx_size = SHA1_DIGEST_SIZE;
1580 sb_count = 1;
1581 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1582 ooffset = ioffset = CCP_SB_BYTES - SHA1_DIGEST_SIZE;
1583 else
1584 ooffset = ioffset = 0;
1585 break;
1586 case CCP_SHA_TYPE_224:
1587 digest_size = SHA224_DIGEST_SIZE;
1588 init = (void *) ccp_sha224_init;
1589 ctx_size = SHA256_DIGEST_SIZE;
1590 sb_count = 1;
1591 ioffset = 0;
1592 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1593 ooffset = CCP_SB_BYTES - SHA224_DIGEST_SIZE;
1594 else
1595 ooffset = 0;
1596 break;
1597 case CCP_SHA_TYPE_256:
1598 digest_size = SHA256_DIGEST_SIZE;
1599 init = (void *) ccp_sha256_init;
1600 ctx_size = SHA256_DIGEST_SIZE;
1601 sb_count = 1;
1602 ooffset = ioffset = 0;
1603 break;
1604 case CCP_SHA_TYPE_384:
1605 digest_size = SHA384_DIGEST_SIZE;
1606 init = (void *) ccp_sha384_init;
1607 ctx_size = SHA512_DIGEST_SIZE;
1608 sb_count = 2;
1609 ioffset = 0;
1610 ooffset = 2 * CCP_SB_BYTES - SHA384_DIGEST_SIZE;
1611 break;
1612 case CCP_SHA_TYPE_512:
1613 digest_size = SHA512_DIGEST_SIZE;
1614 init = (void *) ccp_sha512_init;
1615 ctx_size = SHA512_DIGEST_SIZE;
1616 sb_count = 2;
1617 ooffset = ioffset = 0;
1618 break;
1619 default:
1620 ret = -EINVAL;
1621 goto e_data;
1622 }
1623
1624 /* For zero-length plaintext the src pointer is ignored;
1625 * otherwise both parts must be valid
1626 */
1627 if (sha->src_len && !sha->src)
1628 return -EINVAL;
1629
1630 memset(&op, 0, sizeof(op));
1631 op.cmd_q = cmd_q;
1632 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1633 op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
1634 op.u.sha.type = sha->type;
1635 op.u.sha.msg_bits = sha->msg_bits;
1636
1637 /* For SHA1/224/256 the context fits in a single (32-byte) SB entry;
1638 * SHA384/512 require 2 adjacent SB slots, with the right half in the
1639 * first slot, and the left half in the second. Each portion must then
1640 * be in little endian format: use the 256-bit byte swap option.
1641 */
1642 ret = ccp_init_dm_workarea(&ctx, cmd_q, sb_count * CCP_SB_BYTES,
1643 DMA_BIDIRECTIONAL);
1644 if (ret)
1645 return ret;
1646 if (sha->first) {
1647 switch (sha->type) {
1648 case CCP_SHA_TYPE_1:
1649 case CCP_SHA_TYPE_224:
1650 case CCP_SHA_TYPE_256:
1651 memcpy(ctx.address + ioffset, init, ctx_size);
1652 break;
1653 case CCP_SHA_TYPE_384:
1654 case CCP_SHA_TYPE_512:
1655 memcpy(ctx.address + ctx_size / 2, init,
1656 ctx_size / 2);
1657 memcpy(ctx.address, init + ctx_size / 2,
1658 ctx_size / 2);
1659 break;
1660 default:
1661 ret = -EINVAL;
1662 goto e_ctx;
1663 }
1664 } else {
1665 /* Restore the context */
1666 ret = ccp_set_dm_area(&ctx, 0, sha->ctx, 0,
1667 sb_count * CCP_SB_BYTES);
1668 if (ret)
1669 goto e_ctx;
1670 }
1671
1672 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1673 CCP_PASSTHRU_BYTESWAP_256BIT);
1674 if (ret) {
1675 cmd->engine_error = cmd_q->cmd_error;
1676 goto e_ctx;
1677 }
1678
1679 if (sha->src) {
1680 /* Send data to the CCP SHA engine; block_size is set above */
1681 ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len,
1682 block_size, DMA_TO_DEVICE);
1683 if (ret)
1684 goto e_ctx;
1685
1686 while (src.sg_wa.bytes_left) {
1687 ccp_prepare_data(&src, NULL, &op, block_size, false);
1688 if (sha->final && !src.sg_wa.bytes_left)
1689 op.eom = 1;
1690
1691 ret = cmd_q->ccp->vdata->perform->sha(&op);
1692 if (ret) {
1693 cmd->engine_error = cmd_q->cmd_error;
1694 goto e_data;
1695 }
1696
1697 ccp_process_data(&src, NULL, &op);
1698 }
1699 } else {
1700 op.eom = 1;
1701 ret = cmd_q->ccp->vdata->perform->sha(&op);
1702 if (ret) {
1703 cmd->engine_error = cmd_q->cmd_error;
1704 goto e_data;
1705 }
1706 }
1707
1708 /* Retrieve the SHA context - convert from LE to BE using
1709 * 32-byte (256-bit) byteswapping to BE
1710 */
1711 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1712 CCP_PASSTHRU_BYTESWAP_256BIT);
1713 if (ret) {
1714 cmd->engine_error = cmd_q->cmd_error;
1715 goto e_data;
1716 }
1717
1718 if (sha->final) {
1719 /* Finishing up, so get the digest */
1720 switch (sha->type) {
1721 case CCP_SHA_TYPE_1:
1722 case CCP_SHA_TYPE_224:
1723 case CCP_SHA_TYPE_256:
1724 ccp_get_dm_area(&ctx, ooffset,
1725 sha->ctx, 0,
1726 digest_size);
1727 break;
1728 case CCP_SHA_TYPE_384:
1729 case CCP_SHA_TYPE_512:
1730 ccp_get_dm_area(&ctx, 0,
1731 sha->ctx, LSB_ITEM_SIZE - ooffset,
1732 LSB_ITEM_SIZE);
1733 ccp_get_dm_area(&ctx, LSB_ITEM_SIZE + ooffset,
1734 sha->ctx, 0,
1735 LSB_ITEM_SIZE - ooffset);
1736 break;
1737 default:
1738 ret = -EINVAL;
1739 goto e_ctx;
1740 }
1741 } else {
1742 /* Stash the context */
1743 ccp_get_dm_area(&ctx, 0, sha->ctx, 0,
1744 sb_count * CCP_SB_BYTES);
1745 }
1746
1747 if (sha->final && sha->opad) {
1748 /* HMAC operation, recursively perform final SHA */
1749 struct ccp_cmd hmac_cmd;
1750 struct scatterlist sg;
1751 u8 *hmac_buf;
1752
1753 if (sha->opad_len != block_size) {
1754 ret = -EINVAL;
1755 goto e_data;
1756 }
1757
1758 hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL);
1759 if (!hmac_buf) {
1760 ret = -ENOMEM;
1761 goto e_data;
1762 }
1763 sg_init_one(&sg, hmac_buf, block_size + digest_size);
1764
1765 scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0);
1766 switch (sha->type) {
1767 case CCP_SHA_TYPE_1:
1768 case CCP_SHA_TYPE_224:
1769 case CCP_SHA_TYPE_256:
1770 memcpy(hmac_buf + block_size,
1771 ctx.address + ooffset,
1772 digest_size);
1773 break;
1774 case CCP_SHA_TYPE_384:
1775 case CCP_SHA_TYPE_512:
1776 memcpy(hmac_buf + block_size,
1777 ctx.address + LSB_ITEM_SIZE + ooffset,
1778 LSB_ITEM_SIZE);
1779 memcpy(hmac_buf + block_size +
1780 (LSB_ITEM_SIZE - ooffset),
1781 ctx.address,
1782 LSB_ITEM_SIZE);
1783 break;
1784 default:
1785 ret = -EINVAL;
1786 goto e_ctx;
1787 }
1788
1789 memset(&hmac_cmd, 0, sizeof(hmac_cmd));
1790 hmac_cmd.engine = CCP_ENGINE_SHA;
1791 hmac_cmd.u.sha.type = sha->type;
1792 hmac_cmd.u.sha.ctx = sha->ctx;
1793 hmac_cmd.u.sha.ctx_len = sha->ctx_len;
1794 hmac_cmd.u.sha.src = &sg;
1795 hmac_cmd.u.sha.src_len = block_size + digest_size;
1796 hmac_cmd.u.sha.opad = NULL;
1797 hmac_cmd.u.sha.opad_len = 0;
1798 hmac_cmd.u.sha.first = 1;
1799 hmac_cmd.u.sha.final = 1;
1800 hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3;
1801
1802 ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd);
1803 if (ret)
1804 cmd->engine_error = hmac_cmd.engine_error;
1805
1806 kfree(hmac_buf);
1807 }
1808
1809 e_data:
1810 if (sha->src)
1811 ccp_free_data(&src, cmd_q);
1812
1813 e_ctx:
1814 ccp_dm_free(&ctx);
1815
1816 return ret;
1817 }
1818
1819 static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1820 {
1821 struct ccp_rsa_engine *rsa = &cmd->u.rsa;
1822 struct ccp_dm_workarea exp, src, dst;
1823 struct ccp_op op;
1824 unsigned int sb_count, i_len, o_len;
1825 int ret;
1826
1827 /* Check against the maximum allowable size, in bits */
1828 if (rsa->key_size > cmd_q->ccp->vdata->rsamax)
1829 return -EINVAL;
1830
1831 if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst)
1832 return -EINVAL;
1833
1834 memset(&op, 0, sizeof(op));
1835 op.cmd_q = cmd_q;
1836 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1837
1838 /* The RSA modulus must precede the message being acted upon, so
1839 * it must be copied to a DMA area where the message and the
1840 * modulus can be concatenated. Therefore the input buffer
1841 * length required is twice the output buffer length (which
1842 * must be a multiple of 256-bits). Compute o_len, i_len in bytes.
1843 * Buffer sizes must be a multiple of 32 bytes; rounding up may be
1844 * required.
1845 */
1846 o_len = 32 * ((rsa->key_size + 255) / 256);
1847 i_len = o_len * 2;
1848
1849 sb_count = 0;
1850 if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
1851 /* sb_count is the number of storage block slots required
1852 * for the modulus.
1853 */
1854 sb_count = o_len / CCP_SB_BYTES;
1855 op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q,
1856 sb_count);
1857 if (!op.sb_key)
1858 return -EIO;
1859 } else {
1860 /* A version 5 device allows a modulus size that will not fit
1861 * in the LSB, so the command will transfer it from memory.
1862 * Set the sb key to the default, even though it's not used.
1863 */
1864 op.sb_key = cmd_q->sb_key;
1865 }
1866
1867 /* The RSA exponent must be in little endian format. Reverse its
1868 * byte order.
1869 */
1870 ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE);
1871 if (ret)
1872 goto e_sb;
1873
1874 ret = ccp_reverse_set_dm_area(&exp, 0, rsa->exp, 0, rsa->exp_len);
1875 if (ret)
1876 goto e_exp;
1877
1878 if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
1879 /* Copy the exponent to the local storage block, using
1880 * as many 32-byte blocks as were allocated above. It's
1881 * already little endian, so no further change is required.
1882 */
1883 ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key,
1884 CCP_PASSTHRU_BYTESWAP_NOOP);
1885 if (ret) {
1886 cmd->engine_error = cmd_q->cmd_error;
1887 goto e_exp;
1888 }
1889 } else {
1890 /* The exponent can be retrieved from memory via DMA. */
1891 op.exp.u.dma.address = exp.dma.address;
1892 op.exp.u.dma.offset = 0;
1893 }
1894
1895 /* Concatenate the modulus and the message. Both the modulus and
1896 * the operands must be in little endian format. Since the input
1897 * is in big endian format it must be converted.
1898 */
1899 ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE);
1900 if (ret)
1901 goto e_exp;
1902
1903 ret = ccp_reverse_set_dm_area(&src, 0, rsa->mod, 0, rsa->mod_len);
1904 if (ret)
1905 goto e_src;
1906 ret = ccp_reverse_set_dm_area(&src, o_len, rsa->src, 0, rsa->src_len);
1907 if (ret)
1908 goto e_src;
1909
1910 /* Prepare the output area for the operation */
1911 ret = ccp_init_dm_workarea(&dst, cmd_q, o_len, DMA_FROM_DEVICE);
1912 if (ret)
1913 goto e_src;
1914
1915 op.soc = 1;
1916 op.src.u.dma.address = src.dma.address;
1917 op.src.u.dma.offset = 0;
1918 op.src.u.dma.length = i_len;
1919 op.dst.u.dma.address = dst.dma.address;
1920 op.dst.u.dma.offset = 0;
1921 op.dst.u.dma.length = o_len;
1922
1923 op.u.rsa.mod_size = rsa->key_size;
1924 op.u.rsa.input_len = i_len;
1925
1926 ret = cmd_q->ccp->vdata->perform->rsa(&op);
1927 if (ret) {
1928 cmd->engine_error = cmd_q->cmd_error;
1929 goto e_dst;
1930 }
1931
1932 ccp_reverse_get_dm_area(&dst, 0, rsa->dst, 0, rsa->mod_len);
1933
1934 e_dst:
1935 ccp_dm_free(&dst);
1936
1937 e_src:
1938 ccp_dm_free(&src);
1939
1940 e_exp:
1941 ccp_dm_free(&exp);
1942
1943 e_sb:
1944 if (sb_count)
1945 cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count);
1946
1947 return ret;
1948 }
1949
1950 static int ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q,
1951 struct ccp_cmd *cmd)
1952 {
1953 struct ccp_passthru_engine *pt = &cmd->u.passthru;
1954 struct ccp_dm_workarea mask;
1955 struct ccp_data src, dst;
1956 struct ccp_op op;
1957 bool in_place = false;
1958 unsigned int i;
1959 int ret = 0;
1960
1961 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
1962 return -EINVAL;
1963
1964 if (!pt->src || !pt->dst)
1965 return -EINVAL;
1966
1967 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1968 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
1969 return -EINVAL;
1970 if (!pt->mask)
1971 return -EINVAL;
1972 }
1973
1974 BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
1975
1976 memset(&op, 0, sizeof(op));
1977 op.cmd_q = cmd_q;
1978 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1979
1980 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1981 /* Load the mask */
1982 op.sb_key = cmd_q->sb_key;
1983
1984 ret = ccp_init_dm_workarea(&mask, cmd_q,
1985 CCP_PASSTHRU_SB_COUNT *
1986 CCP_SB_BYTES,
1987 DMA_TO_DEVICE);
1988 if (ret)
1989 return ret;
1990
1991 ret = ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len);
1992 if (ret)
1993 goto e_mask;
1994 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
1995 CCP_PASSTHRU_BYTESWAP_NOOP);
1996 if (ret) {
1997 cmd->engine_error = cmd_q->cmd_error;
1998 goto e_mask;
1999 }
2000 }
2001
2002 /* Prepare the input and output data workareas. For in-place
2003 * operations we need to set the dma direction to BIDIRECTIONAL
2004 * and copy the src workarea to the dst workarea.
2005 */
2006 if (sg_virt(pt->src) == sg_virt(pt->dst))
2007 in_place = true;
2008
2009 ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len,
2010 CCP_PASSTHRU_MASKSIZE,
2011 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
2012 if (ret)
2013 goto e_mask;
2014
2015 if (in_place) {
2016 dst = src;
2017 } else {
2018 ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len,
2019 CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE);
2020 if (ret)
2021 goto e_src;
2022 }
2023
2024 /* Send data to the CCP Passthru engine
2025 * Because the CCP engine works on a single source and destination
2026 * dma address at a time, each entry in the source scatterlist
2027 * (after the dma_map_sg call) must be less than or equal to the
2028 * (remaining) length in the destination scatterlist entry and the
2029 * length must be a multiple of CCP_PASSTHRU_BLOCKSIZE
2030 */
2031 dst.sg_wa.sg_used = 0;
2032 for (i = 1; i <= src.sg_wa.dma_count; i++) {
2033 if (!dst.sg_wa.sg ||
2034 (dst.sg_wa.sg->length < src.sg_wa.sg->length)) {
2035 ret = -EINVAL;
2036 goto e_dst;
2037 }
2038
2039 if (i == src.sg_wa.dma_count) {
2040 op.eom = 1;
2041 op.soc = 1;
2042 }
2043
2044 op.src.type = CCP_MEMTYPE_SYSTEM;
2045 op.src.u.dma.address = sg_dma_address(src.sg_wa.sg);
2046 op.src.u.dma.offset = 0;
2047 op.src.u.dma.length = sg_dma_len(src.sg_wa.sg);
2048
2049 op.dst.type = CCP_MEMTYPE_SYSTEM;
2050 op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg);
2051 op.dst.u.dma.offset = dst.sg_wa.sg_used;
2052 op.dst.u.dma.length = op.src.u.dma.length;
2053
2054 ret = cmd_q->ccp->vdata->perform->passthru(&op);
2055 if (ret) {
2056 cmd->engine_error = cmd_q->cmd_error;
2057 goto e_dst;
2058 }
2059
2060 dst.sg_wa.sg_used += src.sg_wa.sg->length;
2061 if (dst.sg_wa.sg_used == dst.sg_wa.sg->length) {
2062 dst.sg_wa.sg = sg_next(dst.sg_wa.sg);
2063 dst.sg_wa.sg_used = 0;
2064 }
2065 src.sg_wa.sg = sg_next(src.sg_wa.sg);
2066 }
2067
2068 e_dst:
2069 if (!in_place)
2070 ccp_free_data(&dst, cmd_q);
2071
2072 e_src:
2073 ccp_free_data(&src, cmd_q);
2074
2075 e_mask:
2076 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
2077 ccp_dm_free(&mask);
2078
2079 return ret;
2080 }
2081
2082 static int ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q,
2083 struct ccp_cmd *cmd)
2084 {
2085 struct ccp_passthru_nomap_engine *pt = &cmd->u.passthru_nomap;
2086 struct ccp_dm_workarea mask;
2087 struct ccp_op op;
2088 int ret;
2089
2090 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
2091 return -EINVAL;
2092
2093 if (!pt->src_dma || !pt->dst_dma)
2094 return -EINVAL;
2095
2096 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
2097 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
2098 return -EINVAL;
2099 if (!pt->mask)
2100 return -EINVAL;
2101 }
2102
2103 BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
2104
2105 memset(&op, 0, sizeof(op));
2106 op.cmd_q = cmd_q;
2107 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2108
2109 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
2110 /* Load the mask */
2111 op.sb_key = cmd_q->sb_key;
2112
2113 mask.length = pt->mask_len;
2114 mask.dma.address = pt->mask;
2115 mask.dma.length = pt->mask_len;
2116
2117 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
2118 CCP_PASSTHRU_BYTESWAP_NOOP);
2119 if (ret) {
2120 cmd->engine_error = cmd_q->cmd_error;
2121 return ret;
2122 }
2123 }
2124
2125 /* Send data to the CCP Passthru engine */
2126 op.eom = 1;
2127 op.soc = 1;
2128
2129 op.src.type = CCP_MEMTYPE_SYSTEM;
2130 op.src.u.dma.address = pt->src_dma;
2131 op.src.u.dma.offset = 0;
2132 op.src.u.dma.length = pt->src_len;
2133
2134 op.dst.type = CCP_MEMTYPE_SYSTEM;
2135 op.dst.u.dma.address = pt->dst_dma;
2136 op.dst.u.dma.offset = 0;
2137 op.dst.u.dma.length = pt->src_len;
2138
2139 ret = cmd_q->ccp->vdata->perform->passthru(&op);
2140 if (ret)
2141 cmd->engine_error = cmd_q->cmd_error;
2142
2143 return ret;
2144 }
2145
2146 static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2147 {
2148 struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2149 struct ccp_dm_workarea src, dst;
2150 struct ccp_op op;
2151 int ret;
2152 u8 *save;
2153
2154 if (!ecc->u.mm.operand_1 ||
2155 (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES))
2156 return -EINVAL;
2157
2158 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT)
2159 if (!ecc->u.mm.operand_2 ||
2160 (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES))
2161 return -EINVAL;
2162
2163 if (!ecc->u.mm.result ||
2164 (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES))
2165 return -EINVAL;
2166
2167 memset(&op, 0, sizeof(op));
2168 op.cmd_q = cmd_q;
2169 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2170
2171 /* Concatenate the modulus and the operands. Both the modulus and
2172 * the operands must be in little endian format. Since the input
2173 * is in big endian format it must be converted and placed in a
2174 * fixed length buffer.
2175 */
2176 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
2177 DMA_TO_DEVICE);
2178 if (ret)
2179 return ret;
2180
2181 /* Save the workarea address since it is updated in order to perform
2182 * the concatenation
2183 */
2184 save = src.address;
2185
2186 /* Copy the ECC modulus */
2187 ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);
2188 if (ret)
2189 goto e_src;
2190 src.address += CCP_ECC_OPERAND_SIZE;
2191
2192 /* Copy the first operand */
2193 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_1, 0,
2194 ecc->u.mm.operand_1_len);
2195 if (ret)
2196 goto e_src;
2197 src.address += CCP_ECC_OPERAND_SIZE;
2198
2199 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) {
2200 /* Copy the second operand */
2201 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_2, 0,
2202 ecc->u.mm.operand_2_len);
2203 if (ret)
2204 goto e_src;
2205 src.address += CCP_ECC_OPERAND_SIZE;
2206 }
2207
2208 /* Restore the workarea address */
2209 src.address = save;
2210
2211 /* Prepare the output area for the operation */
2212 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
2213 DMA_FROM_DEVICE);
2214 if (ret)
2215 goto e_src;
2216
2217 op.soc = 1;
2218 op.src.u.dma.address = src.dma.address;
2219 op.src.u.dma.offset = 0;
2220 op.src.u.dma.length = src.length;
2221 op.dst.u.dma.address = dst.dma.address;
2222 op.dst.u.dma.offset = 0;
2223 op.dst.u.dma.length = dst.length;
2224
2225 op.u.ecc.function = cmd->u.ecc.function;
2226
2227 ret = cmd_q->ccp->vdata->perform->ecc(&op);
2228 if (ret) {
2229 cmd->engine_error = cmd_q->cmd_error;
2230 goto e_dst;
2231 }
2232
2233 ecc->ecc_result = le16_to_cpup(
2234 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
2235 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
2236 ret = -EIO;
2237 goto e_dst;
2238 }
2239
2240 /* Save the ECC result */
2241 ccp_reverse_get_dm_area(&dst, 0, ecc->u.mm.result, 0,
2242 CCP_ECC_MODULUS_BYTES);
2243
2244 e_dst:
2245 ccp_dm_free(&dst);
2246
2247 e_src:
2248 ccp_dm_free(&src);
2249
2250 return ret;
2251 }
2252
2253 static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2254 {
2255 struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2256 struct ccp_dm_workarea src, dst;
2257 struct ccp_op op;
2258 int ret;
2259 u8 *save;
2260
2261 if (!ecc->u.pm.point_1.x ||
2262 (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) ||
2263 !ecc->u.pm.point_1.y ||
2264 (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES))
2265 return -EINVAL;
2266
2267 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
2268 if (!ecc->u.pm.point_2.x ||
2269 (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) ||
2270 !ecc->u.pm.point_2.y ||
2271 (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES))
2272 return -EINVAL;
2273 } else {
2274 if (!ecc->u.pm.domain_a ||
2275 (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES))
2276 return -EINVAL;
2277
2278 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT)
2279 if (!ecc->u.pm.scalar ||
2280 (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES))
2281 return -EINVAL;
2282 }
2283
2284 if (!ecc->u.pm.result.x ||
2285 (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) ||
2286 !ecc->u.pm.result.y ||
2287 (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES))
2288 return -EINVAL;
2289
2290 memset(&op, 0, sizeof(op));
2291 op.cmd_q = cmd_q;
2292 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2293
2294 /* Concatenate the modulus and the operands. Both the modulus and
2295 * the operands must be in little endian format. Since the input
2296 * is in big endian format it must be converted and placed in a
2297 * fixed length buffer.
2298 */
2299 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
2300 DMA_TO_DEVICE);
2301 if (ret)
2302 return ret;
2303
2304 /* Save the workarea address since it is updated in order to perform
2305 * the concatenation
2306 */
2307 save = src.address;
2308
2309 /* Copy the ECC modulus */
2310 ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);
2311 if (ret)
2312 goto e_src;
2313 src.address += CCP_ECC_OPERAND_SIZE;
2314
2315 /* Copy the first point X and Y coordinate */
2316 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.x, 0,
2317 ecc->u.pm.point_1.x_len);
2318 if (ret)
2319 goto e_src;
2320 src.address += CCP_ECC_OPERAND_SIZE;
2321 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.y, 0,
2322 ecc->u.pm.point_1.y_len);
2323 if (ret)
2324 goto e_src;
2325 src.address += CCP_ECC_OPERAND_SIZE;
2326
2327 /* Set the first point Z coordinate to 1 */
2328 *src.address = 0x01;
2329 src.address += CCP_ECC_OPERAND_SIZE;
2330
2331 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
2332 /* Copy the second point X and Y coordinate */
2333 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.x, 0,
2334 ecc->u.pm.point_2.x_len);
2335 if (ret)
2336 goto e_src;
2337 src.address += CCP_ECC_OPERAND_SIZE;
2338 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.y, 0,
2339 ecc->u.pm.point_2.y_len);
2340 if (ret)
2341 goto e_src;
2342 src.address += CCP_ECC_OPERAND_SIZE;
2343
2344 /* Set the second point Z coordinate to 1 */
2345 *src.address = 0x01;
2346 src.address += CCP_ECC_OPERAND_SIZE;
2347 } else {
2348 /* Copy the Domain "a" parameter */
2349 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.domain_a, 0,
2350 ecc->u.pm.domain_a_len);
2351 if (ret)
2352 goto e_src;
2353 src.address += CCP_ECC_OPERAND_SIZE;
2354
2355 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) {
2356 /* Copy the scalar value */
2357 ret = ccp_reverse_set_dm_area(&src, 0,
2358 ecc->u.pm.scalar, 0,
2359 ecc->u.pm.scalar_len);
2360 if (ret)
2361 goto e_src;
2362 src.address += CCP_ECC_OPERAND_SIZE;
2363 }
2364 }
2365
2366 /* Restore the workarea address */
2367 src.address = save;
2368
2369 /* Prepare the output area for the operation */
2370 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
2371 DMA_FROM_DEVICE);
2372 if (ret)
2373 goto e_src;
2374
2375 op.soc = 1;
2376 op.src.u.dma.address = src.dma.address;
2377 op.src.u.dma.offset = 0;
2378 op.src.u.dma.length = src.length;
2379 op.dst.u.dma.address = dst.dma.address;
2380 op.dst.u.dma.offset = 0;
2381 op.dst.u.dma.length = dst.length;
2382
2383 op.u.ecc.function = cmd->u.ecc.function;
2384
2385 ret = cmd_q->ccp->vdata->perform->ecc(&op);
2386 if (ret) {
2387 cmd->engine_error = cmd_q->cmd_error;
2388 goto e_dst;
2389 }
2390
2391 ecc->ecc_result = le16_to_cpup(
2392 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
2393 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
2394 ret = -EIO;
2395 goto e_dst;
2396 }
2397
2398 /* Save the workarea address since it is updated as we walk through
2399 * to copy the point math result
2400 */
2401 save = dst.address;
2402
2403 /* Save the ECC result X and Y coordinates */
2404 ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.x, 0,
2405 CCP_ECC_MODULUS_BYTES);
2406 dst.address += CCP_ECC_OUTPUT_SIZE;
2407 ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.y, 0,
2408 CCP_ECC_MODULUS_BYTES);
2409 dst.address += CCP_ECC_OUTPUT_SIZE;
2410
2411 /* Restore the workarea address */
2412 dst.address = save;
2413
2414 e_dst:
2415 ccp_dm_free(&dst);
2416
2417 e_src:
2418 ccp_dm_free(&src);
2419
2420 return ret;
2421 }
2422
2423 static int ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2424 {
2425 struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2426
2427 ecc->ecc_result = 0;
2428
2429 if (!ecc->mod ||
2430 (ecc->mod_len > CCP_ECC_MODULUS_BYTES))
2431 return -EINVAL;
2432
2433 switch (ecc->function) {
2434 case CCP_ECC_FUNCTION_MMUL_384BIT:
2435 case CCP_ECC_FUNCTION_MADD_384BIT:
2436 case CCP_ECC_FUNCTION_MINV_384BIT:
2437 return ccp_run_ecc_mm_cmd(cmd_q, cmd);
2438
2439 case CCP_ECC_FUNCTION_PADD_384BIT:
2440 case CCP_ECC_FUNCTION_PMUL_384BIT:
2441 case CCP_ECC_FUNCTION_PDBL_384BIT:
2442 return ccp_run_ecc_pm_cmd(cmd_q, cmd);
2443
2444 default:
2445 return -EINVAL;
2446 }
2447 }
2448
2449 int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2450 {
2451 int ret;
2452
2453 cmd->engine_error = 0;
2454 cmd_q->cmd_error = 0;
2455 cmd_q->int_rcvd = 0;
2456 cmd_q->free_slots = cmd_q->ccp->vdata->perform->get_free_slots(cmd_q);
2457
2458 switch (cmd->engine) {
2459 case CCP_ENGINE_AES:
2460 ret = ccp_run_aes_cmd(cmd_q, cmd);
2461 break;
2462 case CCP_ENGINE_XTS_AES_128:
2463 ret = ccp_run_xts_aes_cmd(cmd_q, cmd);
2464 break;
2465 case CCP_ENGINE_DES3:
2466 ret = ccp_run_des3_cmd(cmd_q, cmd);
2467 break;
2468 case CCP_ENGINE_SHA:
2469 ret = ccp_run_sha_cmd(cmd_q, cmd);
2470 break;
2471 case CCP_ENGINE_RSA:
2472 ret = ccp_run_rsa_cmd(cmd_q, cmd);
2473 break;
2474 case CCP_ENGINE_PASSTHRU:
2475 if (cmd->flags & CCP_CMD_PASSTHRU_NO_DMA_MAP)
2476 ret = ccp_run_passthru_nomap_cmd(cmd_q, cmd);
2477 else
2478 ret = ccp_run_passthru_cmd(cmd_q, cmd);
2479 break;
2480 case CCP_ENGINE_ECC:
2481 ret = ccp_run_ecc_cmd(cmd_q, cmd);
2482 break;
2483 default:
2484 ret = -EINVAL;
2485 }
2486
2487 return ret;
2488 }
A mirror of Linus' kernel repository