1 // SPDX-License-Identifier: GPL-2.0+
3 * caam - Freescale FSL CAAM support for crypto API
5 * Copyright 2008-2011 Freescale Semiconductor, Inc.
6 * Copyright 2016-2019 NXP
8 * Based on talitos crypto API driver.
10 * relationship of job descriptors to shared descriptors (SteveC Dec 10 2008):
12 * --------------- ---------------
13 * | JobDesc #1 |-------------------->| ShareDesc |
14 * | *(packet 1) | | (PDB) |
15 * --------------- |------------->| (hashKey) |
17 * . | |-------->| (operation) |
18 * --------------- | | ---------------
19 * | JobDesc #2 |------| |
25 * | JobDesc #3 |------------
29 * The SharedDesc never changes for a connection unless rekeyed, but
30 * each packet will likely be in a different place. So all we need
31 * to know to process the packet is where the input is, where the
32 * output goes, and what context we want to process with. Context is
33 * in the SharedDesc, packet references in the JobDesc.
35 * So, a job desc looks like:
37 * ---------------------
39 * | ShareDesc Pointer |
46 * ---------------------
53 #include "desc_constr.h"
56 #include "sg_sw_sec4.h"
58 #include "caamalg_desc.h"
59 #include <crypto/engine.h>
64 #define CAAM_CRA_PRIORITY 3000
65 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */
66 #define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \
67 CTR_RFC3686_NONCE_SIZE + \
68 SHA512_DIGEST_SIZE * 2)
70 #define AEAD_DESC_JOB_IO_LEN (DESC_JOB_IO_LEN + CAAM_CMD_SZ * 2)
71 #define GCM_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \
73 #define AUTHENC_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \
76 #define CHACHAPOLY_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + CAAM_CMD_SZ * 6)
78 #define DESC_MAX_USED_BYTES (CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN_MIN)
79 #define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ)
81 struct caam_alg_entry
{
89 struct caam_aead_alg
{
91 struct caam_alg_entry caam
;
95 struct caam_skcipher_alg
{
96 struct skcipher_alg skcipher
;
97 struct caam_alg_entry caam
;
102 * per-session context
105 struct crypto_engine_ctx enginectx
;
106 u32 sh_desc_enc
[DESC_MAX_USED_LEN
];
107 u32 sh_desc_dec
[DESC_MAX_USED_LEN
];
108 u8 key
[CAAM_MAX_KEY_SIZE
];
109 dma_addr_t sh_desc_enc_dma
;
110 dma_addr_t sh_desc_dec_dma
;
112 enum dma_data_direction dir
;
113 struct device
*jrdev
;
114 struct alginfo adata
;
115 struct alginfo cdata
;
116 unsigned int authsize
;
119 struct caam_skcipher_req_ctx
{
120 struct skcipher_edesc
*edesc
;
123 struct caam_aead_req_ctx
{
124 struct aead_edesc
*edesc
;
127 static int aead_null_set_sh_desc(struct crypto_aead
*aead
)
129 struct caam_ctx
*ctx
= crypto_aead_ctx(aead
);
130 struct device
*jrdev
= ctx
->jrdev
;
131 struct caam_drv_private
*ctrlpriv
= dev_get_drvdata(jrdev
->parent
);
133 int rem_bytes
= CAAM_DESC_BYTES_MAX
- AEAD_DESC_JOB_IO_LEN
-
134 ctx
->adata
.keylen_pad
;
137 * Job Descriptor and Shared Descriptors
138 * must all fit into the 64-word Descriptor h/w Buffer
140 if (rem_bytes
>= DESC_AEAD_NULL_ENC_LEN
) {
141 ctx
->adata
.key_inline
= true;
142 ctx
->adata
.key_virt
= ctx
->key
;
144 ctx
->adata
.key_inline
= false;
145 ctx
->adata
.key_dma
= ctx
->key_dma
;
148 /* aead_encrypt shared descriptor */
149 desc
= ctx
->sh_desc_enc
;
150 cnstr_shdsc_aead_null_encap(desc
, &ctx
->adata
, ctx
->authsize
,
152 dma_sync_single_for_device(jrdev
, ctx
->sh_desc_enc_dma
,
153 desc_bytes(desc
), ctx
->dir
);
156 * Job Descriptor and Shared Descriptors
157 * must all fit into the 64-word Descriptor h/w Buffer
159 if (rem_bytes
>= DESC_AEAD_NULL_DEC_LEN
) {
160 ctx
->adata
.key_inline
= true;
161 ctx
->adata
.key_virt
= ctx
->key
;
163 ctx
->adata
.key_inline
= false;
164 ctx
->adata
.key_dma
= ctx
->key_dma
;
167 /* aead_decrypt shared descriptor */
168 desc
= ctx
->sh_desc_dec
;
169 cnstr_shdsc_aead_null_decap(desc
, &ctx
->adata
, ctx
->authsize
,
171 dma_sync_single_for_device(jrdev
, ctx
->sh_desc_dec_dma
,
172 desc_bytes(desc
), ctx
->dir
);
177 static int aead_set_sh_desc(struct crypto_aead
*aead
)
179 struct caam_aead_alg
*alg
= container_of(crypto_aead_alg(aead
),
180 struct caam_aead_alg
, aead
);
181 unsigned int ivsize
= crypto_aead_ivsize(aead
);
182 struct caam_ctx
*ctx
= crypto_aead_ctx(aead
);
183 struct device
*jrdev
= ctx
->jrdev
;
184 struct caam_drv_private
*ctrlpriv
= dev_get_drvdata(jrdev
->parent
);
186 u32
*desc
, *nonce
= NULL
;
188 unsigned int data_len
[2];
189 const bool ctr_mode
= ((ctx
->cdata
.algtype
& OP_ALG_AAI_MASK
) ==
190 OP_ALG_AAI_CTR_MOD128
);
191 const bool is_rfc3686
= alg
->caam
.rfc3686
;
196 /* NULL encryption / decryption */
197 if (!ctx
->cdata
.keylen
)
198 return aead_null_set_sh_desc(aead
);
201 * AES-CTR needs to load IV in CONTEXT1 reg
202 * at an offset of 128bits (16bytes)
203 * CONTEXT1[255:128] = IV
210 * CONTEXT1[255:128] = {NONCE, IV, COUNTER}
213 ctx1_iv_off
= 16 + CTR_RFC3686_NONCE_SIZE
;
214 nonce
= (u32
*)((void *)ctx
->key
+ ctx
->adata
.keylen_pad
+
215 ctx
->cdata
.keylen
- CTR_RFC3686_NONCE_SIZE
);
219 * In case |user key| > |derived key|, using DKP<imm,imm>
220 * would result in invalid opcodes (last bytes of user key) in
221 * the resulting descriptor. Use DKP<ptr,imm> instead => both
222 * virtual and dma key addresses are needed.
224 ctx
->adata
.key_virt
= ctx
->key
;
225 ctx
->adata
.key_dma
= ctx
->key_dma
;
227 ctx
->cdata
.key_virt
= ctx
->key
+ ctx
->adata
.keylen_pad
;
228 ctx
->cdata
.key_dma
= ctx
->key_dma
+ ctx
->adata
.keylen_pad
;
230 data_len
[0] = ctx
->adata
.keylen_pad
;
231 data_len
[1] = ctx
->cdata
.keylen
;
237 * Job Descriptor and Shared Descriptors
238 * must all fit into the 64-word Descriptor h/w Buffer
240 if (desc_inline_query(DESC_AEAD_ENC_LEN
+
241 (is_rfc3686
? DESC_AEAD_CTR_RFC3686_LEN
: 0),
242 AUTHENC_DESC_JOB_IO_LEN
, data_len
, &inl_mask
,
243 ARRAY_SIZE(data_len
)) < 0)
246 ctx
->adata
.key_inline
= !!(inl_mask
& 1);
247 ctx
->cdata
.key_inline
= !!(inl_mask
& 2);
249 /* aead_encrypt shared descriptor */
250 desc
= ctx
->sh_desc_enc
;
251 cnstr_shdsc_aead_encap(desc
, &ctx
->cdata
, &ctx
->adata
, ivsize
,
252 ctx
->authsize
, is_rfc3686
, nonce
, ctx1_iv_off
,
253 false, ctrlpriv
->era
);
254 dma_sync_single_for_device(jrdev
, ctx
->sh_desc_enc_dma
,
255 desc_bytes(desc
), ctx
->dir
);
259 * Job Descriptor and Shared Descriptors
260 * must all fit into the 64-word Descriptor h/w Buffer
262 if (desc_inline_query(DESC_AEAD_DEC_LEN
+
263 (is_rfc3686
? DESC_AEAD_CTR_RFC3686_LEN
: 0),
264 AUTHENC_DESC_JOB_IO_LEN
, data_len
, &inl_mask
,
265 ARRAY_SIZE(data_len
)) < 0)
268 ctx
->adata
.key_inline
= !!(inl_mask
& 1);
269 ctx
->cdata
.key_inline
= !!(inl_mask
& 2);
271 /* aead_decrypt shared descriptor */
272 desc
= ctx
->sh_desc_dec
;
273 cnstr_shdsc_aead_decap(desc
, &ctx
->cdata
, &ctx
->adata
, ivsize
,
274 ctx
->authsize
, alg
->caam
.geniv
, is_rfc3686
,
275 nonce
, ctx1_iv_off
, false, ctrlpriv
->era
);
276 dma_sync_single_for_device(jrdev
, ctx
->sh_desc_dec_dma
,
277 desc_bytes(desc
), ctx
->dir
);
279 if (!alg
->caam
.geniv
)
283 * Job Descriptor and Shared Descriptors
284 * must all fit into the 64-word Descriptor h/w Buffer
286 if (desc_inline_query(DESC_AEAD_GIVENC_LEN
+
287 (is_rfc3686
? DESC_AEAD_CTR_RFC3686_LEN
: 0),
288 AUTHENC_DESC_JOB_IO_LEN
, data_len
, &inl_mask
,
289 ARRAY_SIZE(data_len
)) < 0)
292 ctx
->adata
.key_inline
= !!(inl_mask
& 1);
293 ctx
->cdata
.key_inline
= !!(inl_mask
& 2);
295 /* aead_givencrypt shared descriptor */
296 desc
= ctx
->sh_desc_enc
;
297 cnstr_shdsc_aead_givencap(desc
, &ctx
->cdata
, &ctx
->adata
, ivsize
,
298 ctx
->authsize
, is_rfc3686
, nonce
,
299 ctx1_iv_off
, false, ctrlpriv
->era
);
300 dma_sync_single_for_device(jrdev
, ctx
->sh_desc_enc_dma
,
301 desc_bytes(desc
), ctx
->dir
);
307 static int aead_setauthsize(struct crypto_aead
*authenc
,
308 unsigned int authsize
)
310 struct caam_ctx
*ctx
= crypto_aead_ctx(authenc
);
312 ctx
->authsize
= authsize
;
313 aead_set_sh_desc(authenc
);
318 static int gcm_set_sh_desc(struct crypto_aead
*aead
)
320 struct caam_ctx
*ctx
= crypto_aead_ctx(aead
);
321 struct device
*jrdev
= ctx
->jrdev
;
322 unsigned int ivsize
= crypto_aead_ivsize(aead
);
324 int rem_bytes
= CAAM_DESC_BYTES_MAX
- GCM_DESC_JOB_IO_LEN
-
327 if (!ctx
->cdata
.keylen
|| !ctx
->authsize
)
331 * AES GCM encrypt shared descriptor
332 * Job Descriptor and Shared Descriptor
333 * must fit into the 64-word Descriptor h/w Buffer
335 if (rem_bytes
>= DESC_GCM_ENC_LEN
) {
336 ctx
->cdata
.key_inline
= true;
337 ctx
->cdata
.key_virt
= ctx
->key
;
339 ctx
->cdata
.key_inline
= false;
340 ctx
->cdata
.key_dma
= ctx
->key_dma
;
343 desc
= ctx
->sh_desc_enc
;
344 cnstr_shdsc_gcm_encap(desc
, &ctx
->cdata
, ivsize
, ctx
->authsize
, false);
345 dma_sync_single_for_device(jrdev
, ctx
->sh_desc_enc_dma
,
346 desc_bytes(desc
), ctx
->dir
);
349 * Job Descriptor and Shared Descriptors
350 * must all fit into the 64-word Descriptor h/w Buffer
352 if (rem_bytes
>= DESC_GCM_DEC_LEN
) {
353 ctx
->cdata
.key_inline
= true;
354 ctx
->cdata
.key_virt
= ctx
->key
;
356 ctx
->cdata
.key_inline
= false;
357 ctx
->cdata
.key_dma
= ctx
->key_dma
;
360 desc
= ctx
->sh_desc_dec
;
361 cnstr_shdsc_gcm_decap(desc
, &ctx
->cdata
, ivsize
, ctx
->authsize
, false);
362 dma_sync_single_for_device(jrdev
, ctx
->sh_desc_dec_dma
,
363 desc_bytes(desc
), ctx
->dir
);
368 static int gcm_setauthsize(struct crypto_aead
*authenc
, unsigned int authsize
)
370 struct caam_ctx
*ctx
= crypto_aead_ctx(authenc
);
373 err
= crypto_gcm_check_authsize(authsize
);
377 ctx
->authsize
= authsize
;
378 gcm_set_sh_desc(authenc
);
383 static int rfc4106_set_sh_desc(struct crypto_aead
*aead
)
385 struct caam_ctx
*ctx
= crypto_aead_ctx(aead
);
386 struct device
*jrdev
= ctx
->jrdev
;
387 unsigned int ivsize
= crypto_aead_ivsize(aead
);
389 int rem_bytes
= CAAM_DESC_BYTES_MAX
- GCM_DESC_JOB_IO_LEN
-
392 if (!ctx
->cdata
.keylen
|| !ctx
->authsize
)
396 * RFC4106 encrypt shared descriptor
397 * Job Descriptor and Shared Descriptor
398 * must fit into the 64-word Descriptor h/w Buffer
400 if (rem_bytes
>= DESC_RFC4106_ENC_LEN
) {
401 ctx
->cdata
.key_inline
= true;
402 ctx
->cdata
.key_virt
= ctx
->key
;
404 ctx
->cdata
.key_inline
= false;
405 ctx
->cdata
.key_dma
= ctx
->key_dma
;
408 desc
= ctx
->sh_desc_enc
;
409 cnstr_shdsc_rfc4106_encap(desc
, &ctx
->cdata
, ivsize
, ctx
->authsize
,
411 dma_sync_single_for_device(jrdev
, ctx
->sh_desc_enc_dma
,
412 desc_bytes(desc
), ctx
->dir
);
415 * Job Descriptor and Shared Descriptors
416 * must all fit into the 64-word Descriptor h/w Buffer
418 if (rem_bytes
>= DESC_RFC4106_DEC_LEN
) {
419 ctx
->cdata
.key_inline
= true;
420 ctx
->cdata
.key_virt
= ctx
->key
;
422 ctx
->cdata
.key_inline
= false;
423 ctx
->cdata
.key_dma
= ctx
->key_dma
;
426 desc
= ctx
->sh_desc_dec
;
427 cnstr_shdsc_rfc4106_decap(desc
, &ctx
->cdata
, ivsize
, ctx
->authsize
,
429 dma_sync_single_for_device(jrdev
, ctx
->sh_desc_dec_dma
,
430 desc_bytes(desc
), ctx
->dir
);
435 static int rfc4106_setauthsize(struct crypto_aead
*authenc
,
436 unsigned int authsize
)
438 struct caam_ctx
*ctx
= crypto_aead_ctx(authenc
);
441 err
= crypto_rfc4106_check_authsize(authsize
);
445 ctx
->authsize
= authsize
;
446 rfc4106_set_sh_desc(authenc
);
451 static int rfc4543_set_sh_desc(struct crypto_aead
*aead
)
453 struct caam_ctx
*ctx
= crypto_aead_ctx(aead
);
454 struct device
*jrdev
= ctx
->jrdev
;
455 unsigned int ivsize
= crypto_aead_ivsize(aead
);
457 int rem_bytes
= CAAM_DESC_BYTES_MAX
- GCM_DESC_JOB_IO_LEN
-
460 if (!ctx
->cdata
.keylen
|| !ctx
->authsize
)
464 * RFC4543 encrypt shared descriptor
465 * Job Descriptor and Shared Descriptor
466 * must fit into the 64-word Descriptor h/w Buffer
468 if (rem_bytes
>= DESC_RFC4543_ENC_LEN
) {
469 ctx
->cdata
.key_inline
= true;
470 ctx
->cdata
.key_virt
= ctx
->key
;
472 ctx
->cdata
.key_inline
= false;
473 ctx
->cdata
.key_dma
= ctx
->key_dma
;
476 desc
= ctx
->sh_desc_enc
;
477 cnstr_shdsc_rfc4543_encap(desc
, &ctx
->cdata
, ivsize
, ctx
->authsize
,
479 dma_sync_single_for_device(jrdev
, ctx
->sh_desc_enc_dma
,
480 desc_bytes(desc
), ctx
->dir
);
483 * Job Descriptor and Shared Descriptors
484 * must all fit into the 64-word Descriptor h/w Buffer
486 if (rem_bytes
>= DESC_RFC4543_DEC_LEN
) {
487 ctx
->cdata
.key_inline
= true;
488 ctx
->cdata
.key_virt
= ctx
->key
;
490 ctx
->cdata
.key_inline
= false;
491 ctx
->cdata
.key_dma
= ctx
->key_dma
;
494 desc
= ctx
->sh_desc_dec
;
495 cnstr_shdsc_rfc4543_decap(desc
, &ctx
->cdata
, ivsize
, ctx
->authsize
,
497 dma_sync_single_for_device(jrdev
, ctx
->sh_desc_dec_dma
,
498 desc_bytes(desc
), ctx
->dir
);
503 static int rfc4543_setauthsize(struct crypto_aead
*authenc
,
504 unsigned int authsize
)
506 struct caam_ctx
*ctx
= crypto_aead_ctx(authenc
);
511 ctx
->authsize
= authsize
;
512 rfc4543_set_sh_desc(authenc
);
517 static int chachapoly_set_sh_desc(struct crypto_aead
*aead
)
519 struct caam_ctx
*ctx
= crypto_aead_ctx(aead
);
520 struct device
*jrdev
= ctx
->jrdev
;
521 unsigned int ivsize
= crypto_aead_ivsize(aead
);
524 if (!ctx
->cdata
.keylen
|| !ctx
->authsize
)
527 desc
= ctx
->sh_desc_enc
;
528 cnstr_shdsc_chachapoly(desc
, &ctx
->cdata
, &ctx
->adata
, ivsize
,
529 ctx
->authsize
, true, false);
530 dma_sync_single_for_device(jrdev
, ctx
->sh_desc_enc_dma
,
531 desc_bytes(desc
), ctx
->dir
);
533 desc
= ctx
->sh_desc_dec
;
534 cnstr_shdsc_chachapoly(desc
, &ctx
->cdata
, &ctx
->adata
, ivsize
,
535 ctx
->authsize
, false, false);
536 dma_sync_single_for_device(jrdev
, ctx
->sh_desc_dec_dma
,
537 desc_bytes(desc
), ctx
->dir
);
542 static int chachapoly_setauthsize(struct crypto_aead
*aead
,
543 unsigned int authsize
)
545 struct caam_ctx
*ctx
= crypto_aead_ctx(aead
);
547 if (authsize
!= POLY1305_DIGEST_SIZE
)
550 ctx
->authsize
= authsize
;
551 return chachapoly_set_sh_desc(aead
);
554 static int chachapoly_setkey(struct crypto_aead
*aead
, const u8
*key
,
557 struct caam_ctx
*ctx
= crypto_aead_ctx(aead
);
558 unsigned int ivsize
= crypto_aead_ivsize(aead
);
559 unsigned int saltlen
= CHACHAPOLY_IV_SIZE
- ivsize
;
561 if (keylen
!= CHACHA_KEY_SIZE
+ saltlen
)
564 ctx
->cdata
.key_virt
= key
;
565 ctx
->cdata
.keylen
= keylen
- saltlen
;
567 return chachapoly_set_sh_desc(aead
);
570 static int aead_setkey(struct crypto_aead
*aead
,
571 const u8
*key
, unsigned int keylen
)
573 struct caam_ctx
*ctx
= crypto_aead_ctx(aead
);
574 struct device
*jrdev
= ctx
->jrdev
;
575 struct caam_drv_private
*ctrlpriv
= dev_get_drvdata(jrdev
->parent
);
576 struct crypto_authenc_keys keys
;
579 if (crypto_authenc_extractkeys(&keys
, key
, keylen
) != 0)
582 dev_dbg(jrdev
, "keylen %d enckeylen %d authkeylen %d\n",
583 keys
.authkeylen
+ keys
.enckeylen
, keys
.enckeylen
,
585 print_hex_dump_debug("key in @"__stringify(__LINE__
)": ",
586 DUMP_PREFIX_ADDRESS
, 16, 4, key
, keylen
, 1);
589 * If DKP is supported, use it in the shared descriptor to generate
592 if (ctrlpriv
->era
>= 6) {
593 ctx
->adata
.keylen
= keys
.authkeylen
;
594 ctx
->adata
.keylen_pad
= split_key_len(ctx
->adata
.algtype
&
597 if (ctx
->adata
.keylen_pad
+ keys
.enckeylen
> CAAM_MAX_KEY_SIZE
)
600 memcpy(ctx
->key
, keys
.authkey
, keys
.authkeylen
);
601 memcpy(ctx
->key
+ ctx
->adata
.keylen_pad
, keys
.enckey
,
603 dma_sync_single_for_device(jrdev
, ctx
->key_dma
,
604 ctx
->adata
.keylen_pad
+
605 keys
.enckeylen
, ctx
->dir
);
609 ret
= gen_split_key(ctx
->jrdev
, ctx
->key
, &ctx
->adata
, keys
.authkey
,
610 keys
.authkeylen
, CAAM_MAX_KEY_SIZE
-
616 /* postpend encryption key to auth split key */
617 memcpy(ctx
->key
+ ctx
->adata
.keylen_pad
, keys
.enckey
, keys
.enckeylen
);
618 dma_sync_single_for_device(jrdev
, ctx
->key_dma
, ctx
->adata
.keylen_pad
+
619 keys
.enckeylen
, ctx
->dir
);
621 print_hex_dump_debug("ctx.key@"__stringify(__LINE__
)": ",
622 DUMP_PREFIX_ADDRESS
, 16, 4, ctx
->key
,
623 ctx
->adata
.keylen_pad
+ keys
.enckeylen
, 1);
626 ctx
->cdata
.keylen
= keys
.enckeylen
;
627 memzero_explicit(&keys
, sizeof(keys
));
628 return aead_set_sh_desc(aead
);
630 memzero_explicit(&keys
, sizeof(keys
));
634 static int des3_aead_setkey(struct crypto_aead
*aead
, const u8
*key
,
637 struct crypto_authenc_keys keys
;
640 err
= crypto_authenc_extractkeys(&keys
, key
, keylen
);
644 err
= verify_aead_des3_key(aead
, keys
.enckey
, keys
.enckeylen
) ?:
645 aead_setkey(aead
, key
, keylen
);
647 memzero_explicit(&keys
, sizeof(keys
));
651 static int gcm_setkey(struct crypto_aead
*aead
,
652 const u8
*key
, unsigned int keylen
)
654 struct caam_ctx
*ctx
= crypto_aead_ctx(aead
);
655 struct device
*jrdev
= ctx
->jrdev
;
658 err
= aes_check_keylen(keylen
);
662 print_hex_dump_debug("key in @"__stringify(__LINE__
)": ",
663 DUMP_PREFIX_ADDRESS
, 16, 4, key
, keylen
, 1);
665 memcpy(ctx
->key
, key
, keylen
);
666 dma_sync_single_for_device(jrdev
, ctx
->key_dma
, keylen
, ctx
->dir
);
667 ctx
->cdata
.keylen
= keylen
;
669 return gcm_set_sh_desc(aead
);
672 static int rfc4106_setkey(struct crypto_aead
*aead
,
673 const u8
*key
, unsigned int keylen
)
675 struct caam_ctx
*ctx
= crypto_aead_ctx(aead
);
676 struct device
*jrdev
= ctx
->jrdev
;
679 err
= aes_check_keylen(keylen
- 4);
683 print_hex_dump_debug("key in @"__stringify(__LINE__
)": ",
684 DUMP_PREFIX_ADDRESS
, 16, 4, key
, keylen
, 1);
686 memcpy(ctx
->key
, key
, keylen
);
689 * The last four bytes of the key material are used as the salt value
690 * in the nonce. Update the AES key length.
692 ctx
->cdata
.keylen
= keylen
- 4;
693 dma_sync_single_for_device(jrdev
, ctx
->key_dma
, ctx
->cdata
.keylen
,
695 return rfc4106_set_sh_desc(aead
);
698 static int rfc4543_setkey(struct crypto_aead
*aead
,
699 const u8
*key
, unsigned int keylen
)
701 struct caam_ctx
*ctx
= crypto_aead_ctx(aead
);
702 struct device
*jrdev
= ctx
->jrdev
;
705 err
= aes_check_keylen(keylen
- 4);
709 print_hex_dump_debug("key in @"__stringify(__LINE__
)": ",
710 DUMP_PREFIX_ADDRESS
, 16, 4, key
, keylen
, 1);
712 memcpy(ctx
->key
, key
, keylen
);
715 * The last four bytes of the key material are used as the salt value
716 * in the nonce. Update the AES key length.
718 ctx
->cdata
.keylen
= keylen
- 4;
719 dma_sync_single_for_device(jrdev
, ctx
->key_dma
, ctx
->cdata
.keylen
,
721 return rfc4543_set_sh_desc(aead
);
724 static int skcipher_setkey(struct crypto_skcipher
*skcipher
, const u8
*key
,
725 unsigned int keylen
, const u32 ctx1_iv_off
)
727 struct caam_ctx
*ctx
= crypto_skcipher_ctx(skcipher
);
728 struct caam_skcipher_alg
*alg
=
729 container_of(crypto_skcipher_alg(skcipher
), typeof(*alg
),
731 struct device
*jrdev
= ctx
->jrdev
;
732 unsigned int ivsize
= crypto_skcipher_ivsize(skcipher
);
734 const bool is_rfc3686
= alg
->caam
.rfc3686
;
736 print_hex_dump_debug("key in @"__stringify(__LINE__
)": ",
737 DUMP_PREFIX_ADDRESS
, 16, 4, key
, keylen
, 1);
739 ctx
->cdata
.keylen
= keylen
;
740 ctx
->cdata
.key_virt
= key
;
741 ctx
->cdata
.key_inline
= true;
743 /* skcipher_encrypt shared descriptor */
744 desc
= ctx
->sh_desc_enc
;
745 cnstr_shdsc_skcipher_encap(desc
, &ctx
->cdata
, ivsize
, is_rfc3686
,
747 dma_sync_single_for_device(jrdev
, ctx
->sh_desc_enc_dma
,
748 desc_bytes(desc
), ctx
->dir
);
750 /* skcipher_decrypt shared descriptor */
751 desc
= ctx
->sh_desc_dec
;
752 cnstr_shdsc_skcipher_decap(desc
, &ctx
->cdata
, ivsize
, is_rfc3686
,
754 dma_sync_single_for_device(jrdev
, ctx
->sh_desc_dec_dma
,
755 desc_bytes(desc
), ctx
->dir
);
760 static int aes_skcipher_setkey(struct crypto_skcipher
*skcipher
,
761 const u8
*key
, unsigned int keylen
)
765 err
= aes_check_keylen(keylen
);
769 return skcipher_setkey(skcipher
, key
, keylen
, 0);
772 static int rfc3686_skcipher_setkey(struct crypto_skcipher
*skcipher
,
773 const u8
*key
, unsigned int keylen
)
780 * | CONTEXT1[255:128] = {NONCE, IV, COUNTER}
781 * | *key = {KEY, NONCE}
783 ctx1_iv_off
= 16 + CTR_RFC3686_NONCE_SIZE
;
784 keylen
-= CTR_RFC3686_NONCE_SIZE
;
786 err
= aes_check_keylen(keylen
);
790 return skcipher_setkey(skcipher
, key
, keylen
, ctx1_iv_off
);
793 static int ctr_skcipher_setkey(struct crypto_skcipher
*skcipher
,
794 const u8
*key
, unsigned int keylen
)
800 * AES-CTR needs to load IV in CONTEXT1 reg
801 * at an offset of 128bits (16bytes)
802 * CONTEXT1[255:128] = IV
806 err
= aes_check_keylen(keylen
);
810 return skcipher_setkey(skcipher
, key
, keylen
, ctx1_iv_off
);
813 static int arc4_skcipher_setkey(struct crypto_skcipher
*skcipher
,
814 const u8
*key
, unsigned int keylen
)
816 return skcipher_setkey(skcipher
, key
, keylen
, 0);
819 static int des_skcipher_setkey(struct crypto_skcipher
*skcipher
,
820 const u8
*key
, unsigned int keylen
)
822 return verify_skcipher_des_key(skcipher
, key
) ?:
823 skcipher_setkey(skcipher
, key
, keylen
, 0);
826 static int des3_skcipher_setkey(struct crypto_skcipher
*skcipher
,
827 const u8
*key
, unsigned int keylen
)
829 return verify_skcipher_des3_key(skcipher
, key
) ?:
830 skcipher_setkey(skcipher
, key
, keylen
, 0);
833 static int xts_skcipher_setkey(struct crypto_skcipher
*skcipher
, const u8
*key
,
836 struct caam_ctx
*ctx
= crypto_skcipher_ctx(skcipher
);
837 struct device
*jrdev
= ctx
->jrdev
;
840 if (keylen
!= 2 * AES_MIN_KEY_SIZE
&& keylen
!= 2 * AES_MAX_KEY_SIZE
) {
841 dev_err(jrdev
, "key size mismatch\n");
845 ctx
->cdata
.keylen
= keylen
;
846 ctx
->cdata
.key_virt
= key
;
847 ctx
->cdata
.key_inline
= true;
849 /* xts_skcipher_encrypt shared descriptor */
850 desc
= ctx
->sh_desc_enc
;
851 cnstr_shdsc_xts_skcipher_encap(desc
, &ctx
->cdata
);
852 dma_sync_single_for_device(jrdev
, ctx
->sh_desc_enc_dma
,
853 desc_bytes(desc
), ctx
->dir
);
855 /* xts_skcipher_decrypt shared descriptor */
856 desc
= ctx
->sh_desc_dec
;
857 cnstr_shdsc_xts_skcipher_decap(desc
, &ctx
->cdata
);
858 dma_sync_single_for_device(jrdev
, ctx
->sh_desc_dec_dma
,
859 desc_bytes(desc
), ctx
->dir
);
865 * aead_edesc - s/w-extended aead descriptor
866 * @src_nents: number of segments in input s/w scatterlist
867 * @dst_nents: number of segments in output s/w scatterlist
868 * @mapped_src_nents: number of segments in input h/w link table
869 * @mapped_dst_nents: number of segments in output h/w link table
870 * @sec4_sg_bytes: length of dma mapped sec4_sg space
871 * @bklog: stored to determine if the request needs backlog
872 * @sec4_sg_dma: bus physical mapped address of h/w link table
873 * @sec4_sg: pointer to h/w link table
874 * @hw_desc: the h/w job descriptor followed by any referenced link tables
879 int mapped_src_nents
;
880 int mapped_dst_nents
;
883 dma_addr_t sec4_sg_dma
;
884 struct sec4_sg_entry
*sec4_sg
;
889 * skcipher_edesc - s/w-extended skcipher descriptor
890 * @src_nents: number of segments in input s/w scatterlist
891 * @dst_nents: number of segments in output s/w scatterlist
892 * @mapped_src_nents: number of segments in input h/w link table
893 * @mapped_dst_nents: number of segments in output h/w link table
894 * @iv_dma: dma address of iv for checking continuity and link table
895 * @sec4_sg_bytes: length of dma mapped sec4_sg space
896 * @bklog: stored to determine if the request needs backlog
897 * @sec4_sg_dma: bus physical mapped address of h/w link table
898 * @sec4_sg: pointer to h/w link table
899 * @hw_desc: the h/w job descriptor followed by any referenced link tables
902 struct skcipher_edesc
{
905 int mapped_src_nents
;
906 int mapped_dst_nents
;
910 dma_addr_t sec4_sg_dma
;
911 struct sec4_sg_entry
*sec4_sg
;
915 static void caam_unmap(struct device
*dev
, struct scatterlist
*src
,
916 struct scatterlist
*dst
, int src_nents
,
918 dma_addr_t iv_dma
, int ivsize
, dma_addr_t sec4_sg_dma
,
923 dma_unmap_sg(dev
, src
, src_nents
, DMA_TO_DEVICE
);
925 dma_unmap_sg(dev
, dst
, dst_nents
, DMA_FROM_DEVICE
);
927 dma_unmap_sg(dev
, src
, src_nents
, DMA_BIDIRECTIONAL
);
931 dma_unmap_single(dev
, iv_dma
, ivsize
, DMA_BIDIRECTIONAL
);
933 dma_unmap_single(dev
, sec4_sg_dma
, sec4_sg_bytes
,
937 static void aead_unmap(struct device
*dev
,
938 struct aead_edesc
*edesc
,
939 struct aead_request
*req
)
941 caam_unmap(dev
, req
->src
, req
->dst
,
942 edesc
->src_nents
, edesc
->dst_nents
, 0, 0,
943 edesc
->sec4_sg_dma
, edesc
->sec4_sg_bytes
);
946 static void skcipher_unmap(struct device
*dev
, struct skcipher_edesc
*edesc
,
947 struct skcipher_request
*req
)
949 struct crypto_skcipher
*skcipher
= crypto_skcipher_reqtfm(req
);
950 int ivsize
= crypto_skcipher_ivsize(skcipher
);
952 caam_unmap(dev
, req
->src
, req
->dst
,
953 edesc
->src_nents
, edesc
->dst_nents
,
954 edesc
->iv_dma
, ivsize
,
955 edesc
->sec4_sg_dma
, edesc
->sec4_sg_bytes
);
958 static void aead_crypt_done(struct device
*jrdev
, u32
*desc
, u32 err
,
961 struct aead_request
*req
= context
;
962 struct caam_aead_req_ctx
*rctx
= aead_request_ctx(req
);
963 struct caam_drv_private_jr
*jrp
= dev_get_drvdata(jrdev
);
964 struct aead_edesc
*edesc
;
967 dev_dbg(jrdev
, "%s %d: err 0x%x\n", __func__
, __LINE__
, err
);
972 ecode
= caam_jr_strstatus(jrdev
, err
);
974 aead_unmap(jrdev
, edesc
, req
);
979 * If no backlog flag, the completion of the request is done
980 * by CAAM, not crypto engine.
983 aead_request_complete(req
, ecode
);
985 crypto_finalize_aead_request(jrp
->engine
, req
, ecode
);
988 static void skcipher_crypt_done(struct device
*jrdev
, u32
*desc
, u32 err
,
991 struct skcipher_request
*req
= context
;
992 struct skcipher_edesc
*edesc
;
993 struct caam_skcipher_req_ctx
*rctx
= skcipher_request_ctx(req
);
994 struct crypto_skcipher
*skcipher
= crypto_skcipher_reqtfm(req
);
995 struct caam_drv_private_jr
*jrp
= dev_get_drvdata(jrdev
);
996 int ivsize
= crypto_skcipher_ivsize(skcipher
);
999 dev_dbg(jrdev
, "%s %d: err 0x%x\n", __func__
, __LINE__
, err
);
1001 edesc
= rctx
->edesc
;
1003 ecode
= caam_jr_strstatus(jrdev
, err
);
1005 skcipher_unmap(jrdev
, edesc
, req
);
1008 * The crypto API expects us to set the IV (req->iv) to the last
1009 * ciphertext block (CBC mode) or last counter (CTR mode).
1010 * This is used e.g. by the CTS mode.
1012 if (ivsize
&& !ecode
) {
1013 memcpy(req
->iv
, (u8
*)edesc
->sec4_sg
+ edesc
->sec4_sg_bytes
,
1016 print_hex_dump_debug("dstiv @" __stringify(__LINE__
)": ",
1017 DUMP_PREFIX_ADDRESS
, 16, 4, req
->iv
,
1021 caam_dump_sg("dst @" __stringify(__LINE__
)": ",
1022 DUMP_PREFIX_ADDRESS
, 16, 4, req
->dst
,
1023 edesc
->dst_nents
> 1 ? 100 : req
->cryptlen
, 1);
1028 * If no backlog flag, the completion of the request is done
1029 * by CAAM, not crypto engine.
1032 skcipher_request_complete(req
, ecode
);
1034 crypto_finalize_skcipher_request(jrp
->engine
, req
, ecode
);
1038 * Fill in aead job descriptor
1040 static void init_aead_job(struct aead_request
*req
,
1041 struct aead_edesc
*edesc
,
1042 bool all_contig
, bool encrypt
)
1044 struct crypto_aead
*aead
= crypto_aead_reqtfm(req
);
1045 struct caam_ctx
*ctx
= crypto_aead_ctx(aead
);
1046 int authsize
= ctx
->authsize
;
1047 u32
*desc
= edesc
->hw_desc
;
1048 u32 out_options
, in_options
;
1049 dma_addr_t dst_dma
, src_dma
;
1050 int len
, sec4_sg_index
= 0;
1054 sh_desc
= encrypt
? ctx
->sh_desc_enc
: ctx
->sh_desc_dec
;
1055 ptr
= encrypt
? ctx
->sh_desc_enc_dma
: ctx
->sh_desc_dec_dma
;
1057 len
= desc_len(sh_desc
);
1058 init_job_desc_shared(desc
, ptr
, len
, HDR_SHARE_DEFER
| HDR_REVERSE
);
1061 src_dma
= edesc
->mapped_src_nents
? sg_dma_address(req
->src
) :
1065 src_dma
= edesc
->sec4_sg_dma
;
1066 sec4_sg_index
+= edesc
->mapped_src_nents
;
1067 in_options
= LDST_SGF
;
1070 append_seq_in_ptr(desc
, src_dma
, req
->assoclen
+ req
->cryptlen
,
1074 out_options
= in_options
;
1076 if (unlikely(req
->src
!= req
->dst
)) {
1077 if (!edesc
->mapped_dst_nents
) {
1080 } else if (edesc
->mapped_dst_nents
== 1) {
1081 dst_dma
= sg_dma_address(req
->dst
);
1084 dst_dma
= edesc
->sec4_sg_dma
+
1086 sizeof(struct sec4_sg_entry
);
1087 out_options
= LDST_SGF
;
1092 append_seq_out_ptr(desc
, dst_dma
,
1093 req
->assoclen
+ req
->cryptlen
+ authsize
,
1096 append_seq_out_ptr(desc
, dst_dma
,
1097 req
->assoclen
+ req
->cryptlen
- authsize
,
1101 static void init_gcm_job(struct aead_request
*req
,
1102 struct aead_edesc
*edesc
,
1103 bool all_contig
, bool encrypt
)
1105 struct crypto_aead
*aead
= crypto_aead_reqtfm(req
);
1106 struct caam_ctx
*ctx
= crypto_aead_ctx(aead
);
1107 unsigned int ivsize
= crypto_aead_ivsize(aead
);
1108 u32
*desc
= edesc
->hw_desc
;
1109 bool generic_gcm
= (ivsize
== GCM_AES_IV_SIZE
);
1112 init_aead_job(req
, edesc
, all_contig
, encrypt
);
1113 append_math_add_imm_u32(desc
, REG3
, ZERO
, IMM
, req
->assoclen
);
1115 /* BUG This should not be specific to generic GCM. */
1117 if (encrypt
&& generic_gcm
&& !(req
->assoclen
+ req
->cryptlen
))
1118 last
= FIFOLD_TYPE_LAST1
;
1121 append_cmd(desc
, CMD_FIFO_LOAD
| FIFOLD_CLASS_CLASS1
| IMMEDIATE
|
1122 FIFOLD_TYPE_IV
| FIFOLD_TYPE_FLUSH1
| GCM_AES_IV_SIZE
| last
);
1125 append_data(desc
, ctx
->key
+ ctx
->cdata
.keylen
, 4);
1127 append_data(desc
, req
->iv
, ivsize
);
1128 /* End of blank commands */
1131 static void init_chachapoly_job(struct aead_request
*req
,
1132 struct aead_edesc
*edesc
, bool all_contig
,
1135 struct crypto_aead
*aead
= crypto_aead_reqtfm(req
);
1136 unsigned int ivsize
= crypto_aead_ivsize(aead
);
1137 unsigned int assoclen
= req
->assoclen
;
1138 u32
*desc
= edesc
->hw_desc
;
1141 init_aead_job(req
, edesc
, all_contig
, encrypt
);
1143 if (ivsize
!= CHACHAPOLY_IV_SIZE
) {
1144 /* IPsec specific: CONTEXT1[223:128] = {NONCE, IV} */
1148 * The associated data comes already with the IV but we need
1149 * to skip it when we authenticate or encrypt...
1154 append_math_add_imm_u32(desc
, REG3
, ZERO
, IMM
, assoclen
);
1157 * For IPsec load the IV further in the same register.
1158 * For RFC7539 simply load the 12 bytes nonce in a single operation
1160 append_load_as_imm(desc
, req
->iv
, ivsize
, LDST_CLASS_1_CCB
|
1161 LDST_SRCDST_BYTE_CONTEXT
|
1162 ctx_iv_off
<< LDST_OFFSET_SHIFT
);
1165 static void init_authenc_job(struct aead_request
*req
,
1166 struct aead_edesc
*edesc
,
1167 bool all_contig
, bool encrypt
)
1169 struct crypto_aead
*aead
= crypto_aead_reqtfm(req
);
1170 struct caam_aead_alg
*alg
= container_of(crypto_aead_alg(aead
),
1171 struct caam_aead_alg
, aead
);
1172 unsigned int ivsize
= crypto_aead_ivsize(aead
);
1173 struct caam_ctx
*ctx
= crypto_aead_ctx(aead
);
1174 struct caam_drv_private
*ctrlpriv
= dev_get_drvdata(ctx
->jrdev
->parent
);
1175 const bool ctr_mode
= ((ctx
->cdata
.algtype
& OP_ALG_AAI_MASK
) ==
1176 OP_ALG_AAI_CTR_MOD128
);
1177 const bool is_rfc3686
= alg
->caam
.rfc3686
;
1178 u32
*desc
= edesc
->hw_desc
;
1182 * AES-CTR needs to load IV in CONTEXT1 reg
1183 * at an offset of 128bits (16bytes)
1184 * CONTEXT1[255:128] = IV
1191 * CONTEXT1[255:128] = {NONCE, IV, COUNTER}
1194 ivoffset
= 16 + CTR_RFC3686_NONCE_SIZE
;
1196 init_aead_job(req
, edesc
, all_contig
, encrypt
);
1199 * {REG3, DPOVRD} = assoclen, depending on whether MATH command supports
1200 * having DPOVRD as destination.
1202 if (ctrlpriv
->era
< 3)
1203 append_math_add_imm_u32(desc
, REG3
, ZERO
, IMM
, req
->assoclen
);
1205 append_math_add_imm_u32(desc
, DPOVRD
, ZERO
, IMM
, req
->assoclen
);
1207 if (ivsize
&& ((is_rfc3686
&& encrypt
) || !alg
->caam
.geniv
))
1208 append_load_as_imm(desc
, req
->iv
, ivsize
,
1210 LDST_SRCDST_BYTE_CONTEXT
|
1211 (ivoffset
<< LDST_OFFSET_SHIFT
));
1215 * Fill in skcipher job descriptor
1217 static void init_skcipher_job(struct skcipher_request
*req
,
1218 struct skcipher_edesc
*edesc
,
1221 struct crypto_skcipher
*skcipher
= crypto_skcipher_reqtfm(req
);
1222 struct caam_ctx
*ctx
= crypto_skcipher_ctx(skcipher
);
1223 struct device
*jrdev
= ctx
->jrdev
;
1224 int ivsize
= crypto_skcipher_ivsize(skcipher
);
1225 u32
*desc
= edesc
->hw_desc
;
1227 u32 in_options
= 0, out_options
= 0;
1228 dma_addr_t src_dma
, dst_dma
, ptr
;
1229 int len
, sec4_sg_index
= 0;
1231 print_hex_dump_debug("presciv@"__stringify(__LINE__
)": ",
1232 DUMP_PREFIX_ADDRESS
, 16, 4, req
->iv
, ivsize
, 1);
1233 dev_dbg(jrdev
, "asked=%d, cryptlen%d\n",
1234 (int)edesc
->src_nents
> 1 ? 100 : req
->cryptlen
, req
->cryptlen
);
1236 caam_dump_sg("src @" __stringify(__LINE__
)": ",
1237 DUMP_PREFIX_ADDRESS
, 16, 4, req
->src
,
1238 edesc
->src_nents
> 1 ? 100 : req
->cryptlen
, 1);
1240 sh_desc
= encrypt
? ctx
->sh_desc_enc
: ctx
->sh_desc_dec
;
1241 ptr
= encrypt
? ctx
->sh_desc_enc_dma
: ctx
->sh_desc_dec_dma
;
1243 len
= desc_len(sh_desc
);
1244 init_job_desc_shared(desc
, ptr
, len
, HDR_SHARE_DEFER
| HDR_REVERSE
);
1246 if (ivsize
|| edesc
->mapped_src_nents
> 1) {
1247 src_dma
= edesc
->sec4_sg_dma
;
1248 sec4_sg_index
= edesc
->mapped_src_nents
+ !!ivsize
;
1249 in_options
= LDST_SGF
;
1251 src_dma
= sg_dma_address(req
->src
);
1254 append_seq_in_ptr(desc
, src_dma
, req
->cryptlen
+ ivsize
, in_options
);
1256 if (likely(req
->src
== req
->dst
)) {
1257 dst_dma
= src_dma
+ !!ivsize
* sizeof(struct sec4_sg_entry
);
1258 out_options
= in_options
;
1259 } else if (!ivsize
&& edesc
->mapped_dst_nents
== 1) {
1260 dst_dma
= sg_dma_address(req
->dst
);
1262 dst_dma
= edesc
->sec4_sg_dma
+ sec4_sg_index
*
1263 sizeof(struct sec4_sg_entry
);
1264 out_options
= LDST_SGF
;
1267 append_seq_out_ptr(desc
, dst_dma
, req
->cryptlen
+ ivsize
, out_options
);
1271 * allocate and map the aead extended descriptor
1273 static struct aead_edesc
*aead_edesc_alloc(struct aead_request
*req
,
1274 int desc_bytes
, bool *all_contig_ptr
,
1277 struct crypto_aead
*aead
= crypto_aead_reqtfm(req
);
1278 struct caam_ctx
*ctx
= crypto_aead_ctx(aead
);
1279 struct device
*jrdev
= ctx
->jrdev
;
1280 struct caam_aead_req_ctx
*rctx
= aead_request_ctx(req
);
1281 gfp_t flags
= (req
->base
.flags
& CRYPTO_TFM_REQ_MAY_SLEEP
) ?
1282 GFP_KERNEL
: GFP_ATOMIC
;
1283 int src_nents
, mapped_src_nents
, dst_nents
= 0, mapped_dst_nents
= 0;
1284 int src_len
, dst_len
= 0;
1285 struct aead_edesc
*edesc
;
1286 int sec4_sg_index
, sec4_sg_len
, sec4_sg_bytes
;
1287 unsigned int authsize
= ctx
->authsize
;
1289 if (unlikely(req
->dst
!= req
->src
)) {
1290 src_len
= req
->assoclen
+ req
->cryptlen
;
1291 dst_len
= src_len
+ (encrypt
? authsize
: (-authsize
));
1293 src_nents
= sg_nents_for_len(req
->src
, src_len
);
1294 if (unlikely(src_nents
< 0)) {
1295 dev_err(jrdev
, "Insufficient bytes (%d) in src S/G\n",
1297 return ERR_PTR(src_nents
);
1300 dst_nents
= sg_nents_for_len(req
->dst
, dst_len
);
1301 if (unlikely(dst_nents
< 0)) {
1302 dev_err(jrdev
, "Insufficient bytes (%d) in dst S/G\n",
1304 return ERR_PTR(dst_nents
);
1307 src_len
= req
->assoclen
+ req
->cryptlen
+
1308 (encrypt
? authsize
: 0);
1310 src_nents
= sg_nents_for_len(req
->src
, src_len
);
1311 if (unlikely(src_nents
< 0)) {
1312 dev_err(jrdev
, "Insufficient bytes (%d) in src S/G\n",
1314 return ERR_PTR(src_nents
);
1318 if (likely(req
->src
== req
->dst
)) {
1319 mapped_src_nents
= dma_map_sg(jrdev
, req
->src
, src_nents
,
1321 if (unlikely(!mapped_src_nents
)) {
1322 dev_err(jrdev
, "unable to map source\n");
1323 return ERR_PTR(-ENOMEM
);
1326 /* Cover also the case of null (zero length) input data */
1328 mapped_src_nents
= dma_map_sg(jrdev
, req
->src
,
1329 src_nents
, DMA_TO_DEVICE
);
1330 if (unlikely(!mapped_src_nents
)) {
1331 dev_err(jrdev
, "unable to map source\n");
1332 return ERR_PTR(-ENOMEM
);
1335 mapped_src_nents
= 0;
1338 /* Cover also the case of null (zero length) output data */
1340 mapped_dst_nents
= dma_map_sg(jrdev
, req
->dst
,
1343 if (unlikely(!mapped_dst_nents
)) {
1344 dev_err(jrdev
, "unable to map destination\n");
1345 dma_unmap_sg(jrdev
, req
->src
, src_nents
,
1347 return ERR_PTR(-ENOMEM
);
1350 mapped_dst_nents
= 0;
1355 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
1356 * the end of the table by allocating more S/G entries.
1358 sec4_sg_len
= mapped_src_nents
> 1 ? mapped_src_nents
: 0;
1359 if (mapped_dst_nents
> 1)
1360 sec4_sg_len
+= pad_sg_nents(mapped_dst_nents
);
1362 sec4_sg_len
= pad_sg_nents(sec4_sg_len
);
1364 sec4_sg_bytes
= sec4_sg_len
* sizeof(struct sec4_sg_entry
);
1366 /* allocate space for base edesc and hw desc commands, link tables */
1367 edesc
= kzalloc(sizeof(*edesc
) + desc_bytes
+ sec4_sg_bytes
,
1370 caam_unmap(jrdev
, req
->src
, req
->dst
, src_nents
, dst_nents
, 0,
1372 return ERR_PTR(-ENOMEM
);
1375 edesc
->src_nents
= src_nents
;
1376 edesc
->dst_nents
= dst_nents
;
1377 edesc
->mapped_src_nents
= mapped_src_nents
;
1378 edesc
->mapped_dst_nents
= mapped_dst_nents
;
1379 edesc
->sec4_sg
= (void *)edesc
+ sizeof(struct aead_edesc
) +
1382 rctx
->edesc
= edesc
;
1384 *all_contig_ptr
= !(mapped_src_nents
> 1);
1387 if (mapped_src_nents
> 1) {
1388 sg_to_sec4_sg_last(req
->src
, src_len
,
1389 edesc
->sec4_sg
+ sec4_sg_index
, 0);
1390 sec4_sg_index
+= mapped_src_nents
;
1392 if (mapped_dst_nents
> 1) {
1393 sg_to_sec4_sg_last(req
->dst
, dst_len
,
1394 edesc
->sec4_sg
+ sec4_sg_index
, 0);
1400 edesc
->sec4_sg_dma
= dma_map_single(jrdev
, edesc
->sec4_sg
,
1401 sec4_sg_bytes
, DMA_TO_DEVICE
);
1402 if (dma_mapping_error(jrdev
, edesc
->sec4_sg_dma
)) {
1403 dev_err(jrdev
, "unable to map S/G table\n");
1404 aead_unmap(jrdev
, edesc
, req
);
1406 return ERR_PTR(-ENOMEM
);
1409 edesc
->sec4_sg_bytes
= sec4_sg_bytes
;
1414 static int aead_enqueue_req(struct device
*jrdev
, struct aead_request
*req
)
1416 struct caam_drv_private_jr
*jrpriv
= dev_get_drvdata(jrdev
);
1417 struct caam_aead_req_ctx
*rctx
= aead_request_ctx(req
);
1418 struct aead_edesc
*edesc
= rctx
->edesc
;
1419 u32
*desc
= edesc
->hw_desc
;
1423 * Only the backlog request are sent to crypto-engine since the others
1424 * can be handled by CAAM, if free, especially since JR has up to 1024
1425 * entries (more than the 10 entries from crypto-engine).
1427 if (req
->base
.flags
& CRYPTO_TFM_REQ_MAY_BACKLOG
)
1428 ret
= crypto_transfer_aead_request_to_engine(jrpriv
->engine
,
1431 ret
= caam_jr_enqueue(jrdev
, desc
, aead_crypt_done
, req
);
1433 if ((ret
!= -EINPROGRESS
) && (ret
!= -EBUSY
)) {
1434 aead_unmap(jrdev
, edesc
, req
);
1441 static inline int chachapoly_crypt(struct aead_request
*req
, bool encrypt
)
1443 struct aead_edesc
*edesc
;
1444 struct crypto_aead
*aead
= crypto_aead_reqtfm(req
);
1445 struct caam_ctx
*ctx
= crypto_aead_ctx(aead
);
1446 struct device
*jrdev
= ctx
->jrdev
;
1450 edesc
= aead_edesc_alloc(req
, CHACHAPOLY_DESC_JOB_IO_LEN
, &all_contig
,
1453 return PTR_ERR(edesc
);
1455 desc
= edesc
->hw_desc
;
1457 init_chachapoly_job(req
, edesc
, all_contig
, encrypt
);
1458 print_hex_dump_debug("chachapoly jobdesc@" __stringify(__LINE__
)": ",
1459 DUMP_PREFIX_ADDRESS
, 16, 4, desc
, desc_bytes(desc
),
1462 return aead_enqueue_req(jrdev
, req
);
1465 static int chachapoly_encrypt(struct aead_request
*req
)
1467 return chachapoly_crypt(req
, true);
1470 static int chachapoly_decrypt(struct aead_request
*req
)
1472 return chachapoly_crypt(req
, false);
1475 static inline int aead_crypt(struct aead_request
*req
, bool encrypt
)
1477 struct aead_edesc
*edesc
;
1478 struct crypto_aead
*aead
= crypto_aead_reqtfm(req
);
1479 struct caam_ctx
*ctx
= crypto_aead_ctx(aead
);
1480 struct device
*jrdev
= ctx
->jrdev
;
1483 /* allocate extended descriptor */
1484 edesc
= aead_edesc_alloc(req
, AUTHENC_DESC_JOB_IO_LEN
,
1485 &all_contig
, encrypt
);
1487 return PTR_ERR(edesc
);
1489 /* Create and submit job descriptor */
1490 init_authenc_job(req
, edesc
, all_contig
, encrypt
);
1492 print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__
)": ",
1493 DUMP_PREFIX_ADDRESS
, 16, 4, edesc
->hw_desc
,
1494 desc_bytes(edesc
->hw_desc
), 1);
1496 return aead_enqueue_req(jrdev
, req
);
1499 static int aead_encrypt(struct aead_request
*req
)
1501 return aead_crypt(req
, true);
1504 static int aead_decrypt(struct aead_request
*req
)
1506 return aead_crypt(req
, false);
1509 static int aead_do_one_req(struct crypto_engine
*engine
, void *areq
)
1511 struct aead_request
*req
= aead_request_cast(areq
);
1512 struct caam_ctx
*ctx
= crypto_aead_ctx(crypto_aead_reqtfm(req
));
1513 struct caam_aead_req_ctx
*rctx
= aead_request_ctx(req
);
1514 u32
*desc
= rctx
->edesc
->hw_desc
;
1517 rctx
->edesc
->bklog
= true;
1519 ret
= caam_jr_enqueue(ctx
->jrdev
, desc
, aead_crypt_done
, req
);
1521 if (ret
!= -EINPROGRESS
) {
1522 aead_unmap(ctx
->jrdev
, rctx
->edesc
, req
);
1531 static inline int gcm_crypt(struct aead_request
*req
, bool encrypt
)
1533 struct aead_edesc
*edesc
;
1534 struct crypto_aead
*aead
= crypto_aead_reqtfm(req
);
1535 struct caam_ctx
*ctx
= crypto_aead_ctx(aead
);
1536 struct device
*jrdev
= ctx
->jrdev
;
1539 /* allocate extended descriptor */
1540 edesc
= aead_edesc_alloc(req
, GCM_DESC_JOB_IO_LEN
, &all_contig
,
1543 return PTR_ERR(edesc
);
1545 /* Create and submit job descriptor */
1546 init_gcm_job(req
, edesc
, all_contig
, encrypt
);
1548 print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__
)": ",
1549 DUMP_PREFIX_ADDRESS
, 16, 4, edesc
->hw_desc
,
1550 desc_bytes(edesc
->hw_desc
), 1);
1552 return aead_enqueue_req(jrdev
, req
);
1555 static int gcm_encrypt(struct aead_request
*req
)
1557 return gcm_crypt(req
, true);
1560 static int gcm_decrypt(struct aead_request
*req
)
1562 return gcm_crypt(req
, false);
1565 static int ipsec_gcm_encrypt(struct aead_request
*req
)
1567 return crypto_ipsec_check_assoclen(req
->assoclen
) ? : gcm_encrypt(req
);
1570 static int ipsec_gcm_decrypt(struct aead_request
*req
)
1572 return crypto_ipsec_check_assoclen(req
->assoclen
) ? : gcm_decrypt(req
);
1576 * allocate and map the skcipher extended descriptor for skcipher
1578 static struct skcipher_edesc
*skcipher_edesc_alloc(struct skcipher_request
*req
,
1581 struct crypto_skcipher
*skcipher
= crypto_skcipher_reqtfm(req
);
1582 struct caam_ctx
*ctx
= crypto_skcipher_ctx(skcipher
);
1583 struct caam_skcipher_req_ctx
*rctx
= skcipher_request_ctx(req
);
1584 struct device
*jrdev
= ctx
->jrdev
;
1585 gfp_t flags
= (req
->base
.flags
& CRYPTO_TFM_REQ_MAY_SLEEP
) ?
1586 GFP_KERNEL
: GFP_ATOMIC
;
1587 int src_nents
, mapped_src_nents
, dst_nents
= 0, mapped_dst_nents
= 0;
1588 struct skcipher_edesc
*edesc
;
1589 dma_addr_t iv_dma
= 0;
1591 int ivsize
= crypto_skcipher_ivsize(skcipher
);
1592 int dst_sg_idx
, sec4_sg_ents
, sec4_sg_bytes
;
1594 src_nents
= sg_nents_for_len(req
->src
, req
->cryptlen
);
1595 if (unlikely(src_nents
< 0)) {
1596 dev_err(jrdev
, "Insufficient bytes (%d) in src S/G\n",
1598 return ERR_PTR(src_nents
);
1601 if (req
->dst
!= req
->src
) {
1602 dst_nents
= sg_nents_for_len(req
->dst
, req
->cryptlen
);
1603 if (unlikely(dst_nents
< 0)) {
1604 dev_err(jrdev
, "Insufficient bytes (%d) in dst S/G\n",
1606 return ERR_PTR(dst_nents
);
1610 if (likely(req
->src
== req
->dst
)) {
1611 mapped_src_nents
= dma_map_sg(jrdev
, req
->src
, src_nents
,
1613 if (unlikely(!mapped_src_nents
)) {
1614 dev_err(jrdev
, "unable to map source\n");
1615 return ERR_PTR(-ENOMEM
);
1618 mapped_src_nents
= dma_map_sg(jrdev
, req
->src
, src_nents
,
1620 if (unlikely(!mapped_src_nents
)) {
1621 dev_err(jrdev
, "unable to map source\n");
1622 return ERR_PTR(-ENOMEM
);
1624 mapped_dst_nents
= dma_map_sg(jrdev
, req
->dst
, dst_nents
,
1626 if (unlikely(!mapped_dst_nents
)) {
1627 dev_err(jrdev
, "unable to map destination\n");
1628 dma_unmap_sg(jrdev
, req
->src
, src_nents
, DMA_TO_DEVICE
);
1629 return ERR_PTR(-ENOMEM
);
1633 if (!ivsize
&& mapped_src_nents
== 1)
1634 sec4_sg_ents
= 0; // no need for an input hw s/g table
1636 sec4_sg_ents
= mapped_src_nents
+ !!ivsize
;
1637 dst_sg_idx
= sec4_sg_ents
;
1640 * Input, output HW S/G tables: [IV, src][dst, IV]
1641 * IV entries point to the same buffer
1642 * If src == dst, S/G entries are reused (S/G tables overlap)
1644 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
1645 * the end of the table by allocating more S/G entries. Logic:
1647 * pad output S/G, if needed
1648 * else if (input S/G) ...
1649 * pad input S/G, if needed
1651 if (ivsize
|| mapped_dst_nents
> 1) {
1652 if (req
->src
== req
->dst
)
1653 sec4_sg_ents
= !!ivsize
+ pad_sg_nents(sec4_sg_ents
);
1655 sec4_sg_ents
+= pad_sg_nents(mapped_dst_nents
+
1658 sec4_sg_ents
= pad_sg_nents(sec4_sg_ents
);
1661 sec4_sg_bytes
= sec4_sg_ents
* sizeof(struct sec4_sg_entry
);
1664 * allocate space for base edesc and hw desc commands, link tables, IV
1666 edesc
= kzalloc(sizeof(*edesc
) + desc_bytes
+ sec4_sg_bytes
+ ivsize
,
1669 dev_err(jrdev
, "could not allocate extended descriptor\n");
1670 caam_unmap(jrdev
, req
->src
, req
->dst
, src_nents
, dst_nents
, 0,
1672 return ERR_PTR(-ENOMEM
);
1675 edesc
->src_nents
= src_nents
;
1676 edesc
->dst_nents
= dst_nents
;
1677 edesc
->mapped_src_nents
= mapped_src_nents
;
1678 edesc
->mapped_dst_nents
= mapped_dst_nents
;
1679 edesc
->sec4_sg_bytes
= sec4_sg_bytes
;
1680 edesc
->sec4_sg
= (struct sec4_sg_entry
*)((u8
*)edesc
->hw_desc
+
1682 rctx
->edesc
= edesc
;
1684 /* Make sure IV is located in a DMAable area */
1686 iv
= (u8
*)edesc
->sec4_sg
+ sec4_sg_bytes
;
1687 memcpy(iv
, req
->iv
, ivsize
);
1689 iv_dma
= dma_map_single(jrdev
, iv
, ivsize
, DMA_BIDIRECTIONAL
);
1690 if (dma_mapping_error(jrdev
, iv_dma
)) {
1691 dev_err(jrdev
, "unable to map IV\n");
1692 caam_unmap(jrdev
, req
->src
, req
->dst
, src_nents
,
1693 dst_nents
, 0, 0, 0, 0);
1695 return ERR_PTR(-ENOMEM
);
1698 dma_to_sec4_sg_one(edesc
->sec4_sg
, iv_dma
, ivsize
, 0);
1701 sg_to_sec4_sg(req
->src
, req
->cryptlen
, edesc
->sec4_sg
+
1704 if (req
->src
!= req
->dst
&& (ivsize
|| mapped_dst_nents
> 1))
1705 sg_to_sec4_sg(req
->dst
, req
->cryptlen
, edesc
->sec4_sg
+
1709 dma_to_sec4_sg_one(edesc
->sec4_sg
+ dst_sg_idx
+
1710 mapped_dst_nents
, iv_dma
, ivsize
, 0);
1712 if (ivsize
|| mapped_dst_nents
> 1)
1713 sg_to_sec4_set_last(edesc
->sec4_sg
+ dst_sg_idx
+
1716 if (sec4_sg_bytes
) {
1717 edesc
->sec4_sg_dma
= dma_map_single(jrdev
, edesc
->sec4_sg
,
1720 if (dma_mapping_error(jrdev
, edesc
->sec4_sg_dma
)) {
1721 dev_err(jrdev
, "unable to map S/G table\n");
1722 caam_unmap(jrdev
, req
->src
, req
->dst
, src_nents
,
1723 dst_nents
, iv_dma
, ivsize
, 0, 0);
1725 return ERR_PTR(-ENOMEM
);
1729 edesc
->iv_dma
= iv_dma
;
1731 print_hex_dump_debug("skcipher sec4_sg@" __stringify(__LINE__
)": ",
1732 DUMP_PREFIX_ADDRESS
, 16, 4, edesc
->sec4_sg
,
1738 static int skcipher_do_one_req(struct crypto_engine
*engine
, void *areq
)
1740 struct skcipher_request
*req
= skcipher_request_cast(areq
);
1741 struct caam_ctx
*ctx
= crypto_skcipher_ctx(crypto_skcipher_reqtfm(req
));
1742 struct caam_skcipher_req_ctx
*rctx
= skcipher_request_ctx(req
);
1743 u32
*desc
= rctx
->edesc
->hw_desc
;
1746 rctx
->edesc
->bklog
= true;
1748 ret
= caam_jr_enqueue(ctx
->jrdev
, desc
, skcipher_crypt_done
, req
);
1750 if (ret
!= -EINPROGRESS
) {
1751 skcipher_unmap(ctx
->jrdev
, rctx
->edesc
, req
);
1760 static inline int skcipher_crypt(struct skcipher_request
*req
, bool encrypt
)
1762 struct skcipher_edesc
*edesc
;
1763 struct crypto_skcipher
*skcipher
= crypto_skcipher_reqtfm(req
);
1764 struct caam_ctx
*ctx
= crypto_skcipher_ctx(skcipher
);
1765 struct device
*jrdev
= ctx
->jrdev
;
1766 struct caam_drv_private_jr
*jrpriv
= dev_get_drvdata(jrdev
);
1773 /* allocate extended descriptor */
1774 edesc
= skcipher_edesc_alloc(req
, DESC_JOB_IO_LEN
* CAAM_CMD_SZ
);
1776 return PTR_ERR(edesc
);
1778 /* Create and submit job descriptor*/
1779 init_skcipher_job(req
, edesc
, encrypt
);
1781 print_hex_dump_debug("skcipher jobdesc@" __stringify(__LINE__
)": ",
1782 DUMP_PREFIX_ADDRESS
, 16, 4, edesc
->hw_desc
,
1783 desc_bytes(edesc
->hw_desc
), 1);
1785 desc
= edesc
->hw_desc
;
1787 * Only the backlog request are sent to crypto-engine since the others
1788 * can be handled by CAAM, if free, especially since JR has up to 1024
1789 * entries (more than the 10 entries from crypto-engine).
1791 if (req
->base
.flags
& CRYPTO_TFM_REQ_MAY_BACKLOG
)
1792 ret
= crypto_transfer_skcipher_request_to_engine(jrpriv
->engine
,
1795 ret
= caam_jr_enqueue(jrdev
, desc
, skcipher_crypt_done
, req
);
1797 if ((ret
!= -EINPROGRESS
) && (ret
!= -EBUSY
)) {
1798 skcipher_unmap(jrdev
, edesc
, req
);
1805 static int skcipher_encrypt(struct skcipher_request
*req
)
1807 return skcipher_crypt(req
, true);
1810 static int skcipher_decrypt(struct skcipher_request
*req
)
1812 return skcipher_crypt(req
, false);
1815 static struct caam_skcipher_alg driver_algs
[] = {
1819 .cra_name
= "cbc(aes)",
1820 .cra_driver_name
= "cbc-aes-caam",
1821 .cra_blocksize
= AES_BLOCK_SIZE
,
1823 .setkey
= aes_skcipher_setkey
,
1824 .encrypt
= skcipher_encrypt
,
1825 .decrypt
= skcipher_decrypt
,
1826 .min_keysize
= AES_MIN_KEY_SIZE
,
1827 .max_keysize
= AES_MAX_KEY_SIZE
,
1828 .ivsize
= AES_BLOCK_SIZE
,
1830 .caam
.class1_alg_type
= OP_ALG_ALGSEL_AES
| OP_ALG_AAI_CBC
,
1835 .cra_name
= "cbc(des3_ede)",
1836 .cra_driver_name
= "cbc-3des-caam",
1837 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
1839 .setkey
= des3_skcipher_setkey
,
1840 .encrypt
= skcipher_encrypt
,
1841 .decrypt
= skcipher_decrypt
,
1842 .min_keysize
= DES3_EDE_KEY_SIZE
,
1843 .max_keysize
= DES3_EDE_KEY_SIZE
,
1844 .ivsize
= DES3_EDE_BLOCK_SIZE
,
1846 .caam
.class1_alg_type
= OP_ALG_ALGSEL_3DES
| OP_ALG_AAI_CBC
,
1851 .cra_name
= "cbc(des)",
1852 .cra_driver_name
= "cbc-des-caam",
1853 .cra_blocksize
= DES_BLOCK_SIZE
,
1855 .setkey
= des_skcipher_setkey
,
1856 .encrypt
= skcipher_encrypt
,
1857 .decrypt
= skcipher_decrypt
,
1858 .min_keysize
= DES_KEY_SIZE
,
1859 .max_keysize
= DES_KEY_SIZE
,
1860 .ivsize
= DES_BLOCK_SIZE
,
1862 .caam
.class1_alg_type
= OP_ALG_ALGSEL_DES
| OP_ALG_AAI_CBC
,
1867 .cra_name
= "ctr(aes)",
1868 .cra_driver_name
= "ctr-aes-caam",
1871 .setkey
= ctr_skcipher_setkey
,
1872 .encrypt
= skcipher_encrypt
,
1873 .decrypt
= skcipher_decrypt
,
1874 .min_keysize
= AES_MIN_KEY_SIZE
,
1875 .max_keysize
= AES_MAX_KEY_SIZE
,
1876 .ivsize
= AES_BLOCK_SIZE
,
1877 .chunksize
= AES_BLOCK_SIZE
,
1879 .caam
.class1_alg_type
= OP_ALG_ALGSEL_AES
|
1880 OP_ALG_AAI_CTR_MOD128
,
1885 .cra_name
= "rfc3686(ctr(aes))",
1886 .cra_driver_name
= "rfc3686-ctr-aes-caam",
1889 .setkey
= rfc3686_skcipher_setkey
,
1890 .encrypt
= skcipher_encrypt
,
1891 .decrypt
= skcipher_decrypt
,
1892 .min_keysize
= AES_MIN_KEY_SIZE
+
1893 CTR_RFC3686_NONCE_SIZE
,
1894 .max_keysize
= AES_MAX_KEY_SIZE
+
1895 CTR_RFC3686_NONCE_SIZE
,
1896 .ivsize
= CTR_RFC3686_IV_SIZE
,
1897 .chunksize
= AES_BLOCK_SIZE
,
1900 .class1_alg_type
= OP_ALG_ALGSEL_AES
|
1901 OP_ALG_AAI_CTR_MOD128
,
1908 .cra_name
= "xts(aes)",
1909 .cra_driver_name
= "xts-aes-caam",
1910 .cra_blocksize
= AES_BLOCK_SIZE
,
1912 .setkey
= xts_skcipher_setkey
,
1913 .encrypt
= skcipher_encrypt
,
1914 .decrypt
= skcipher_decrypt
,
1915 .min_keysize
= 2 * AES_MIN_KEY_SIZE
,
1916 .max_keysize
= 2 * AES_MAX_KEY_SIZE
,
1917 .ivsize
= AES_BLOCK_SIZE
,
1919 .caam
.class1_alg_type
= OP_ALG_ALGSEL_AES
| OP_ALG_AAI_XTS
,
1924 .cra_name
= "ecb(des)",
1925 .cra_driver_name
= "ecb-des-caam",
1926 .cra_blocksize
= DES_BLOCK_SIZE
,
1928 .setkey
= des_skcipher_setkey
,
1929 .encrypt
= skcipher_encrypt
,
1930 .decrypt
= skcipher_decrypt
,
1931 .min_keysize
= DES_KEY_SIZE
,
1932 .max_keysize
= DES_KEY_SIZE
,
1934 .caam
.class1_alg_type
= OP_ALG_ALGSEL_DES
| OP_ALG_AAI_ECB
,
1939 .cra_name
= "ecb(aes)",
1940 .cra_driver_name
= "ecb-aes-caam",
1941 .cra_blocksize
= AES_BLOCK_SIZE
,
1943 .setkey
= aes_skcipher_setkey
,
1944 .encrypt
= skcipher_encrypt
,
1945 .decrypt
= skcipher_decrypt
,
1946 .min_keysize
= AES_MIN_KEY_SIZE
,
1947 .max_keysize
= AES_MAX_KEY_SIZE
,
1949 .caam
.class1_alg_type
= OP_ALG_ALGSEL_AES
| OP_ALG_AAI_ECB
,
1954 .cra_name
= "ecb(des3_ede)",
1955 .cra_driver_name
= "ecb-des3-caam",
1956 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
1958 .setkey
= des3_skcipher_setkey
,
1959 .encrypt
= skcipher_encrypt
,
1960 .decrypt
= skcipher_decrypt
,
1961 .min_keysize
= DES3_EDE_KEY_SIZE
,
1962 .max_keysize
= DES3_EDE_KEY_SIZE
,
1964 .caam
.class1_alg_type
= OP_ALG_ALGSEL_3DES
| OP_ALG_AAI_ECB
,
1969 .cra_name
= "ecb(arc4)",
1970 .cra_driver_name
= "ecb-arc4-caam",
1971 .cra_blocksize
= ARC4_BLOCK_SIZE
,
1973 .setkey
= arc4_skcipher_setkey
,
1974 .encrypt
= skcipher_encrypt
,
1975 .decrypt
= skcipher_decrypt
,
1976 .min_keysize
= ARC4_MIN_KEY_SIZE
,
1977 .max_keysize
= ARC4_MAX_KEY_SIZE
,
1979 .caam
.class1_alg_type
= OP_ALG_ALGSEL_ARC4
| OP_ALG_AAI_ECB
,
1983 static struct caam_aead_alg driver_aeads
[] = {
1987 .cra_name
= "rfc4106(gcm(aes))",
1988 .cra_driver_name
= "rfc4106-gcm-aes-caam",
1991 .setkey
= rfc4106_setkey
,
1992 .setauthsize
= rfc4106_setauthsize
,
1993 .encrypt
= ipsec_gcm_encrypt
,
1994 .decrypt
= ipsec_gcm_decrypt
,
1995 .ivsize
= GCM_RFC4106_IV_SIZE
,
1996 .maxauthsize
= AES_BLOCK_SIZE
,
1999 .class1_alg_type
= OP_ALG_ALGSEL_AES
| OP_ALG_AAI_GCM
,
2006 .cra_name
= "rfc4543(gcm(aes))",
2007 .cra_driver_name
= "rfc4543-gcm-aes-caam",
2010 .setkey
= rfc4543_setkey
,
2011 .setauthsize
= rfc4543_setauthsize
,
2012 .encrypt
= ipsec_gcm_encrypt
,
2013 .decrypt
= ipsec_gcm_decrypt
,
2014 .ivsize
= GCM_RFC4543_IV_SIZE
,
2015 .maxauthsize
= AES_BLOCK_SIZE
,
2018 .class1_alg_type
= OP_ALG_ALGSEL_AES
| OP_ALG_AAI_GCM
,
2022 /* Galois Counter Mode */
2026 .cra_name
= "gcm(aes)",
2027 .cra_driver_name
= "gcm-aes-caam",
2030 .setkey
= gcm_setkey
,
2031 .setauthsize
= gcm_setauthsize
,
2032 .encrypt
= gcm_encrypt
,
2033 .decrypt
= gcm_decrypt
,
2034 .ivsize
= GCM_AES_IV_SIZE
,
2035 .maxauthsize
= AES_BLOCK_SIZE
,
2038 .class1_alg_type
= OP_ALG_ALGSEL_AES
| OP_ALG_AAI_GCM
,
2042 /* single-pass ipsec_esp descriptor */
2046 .cra_name
= "authenc(hmac(md5),"
2047 "ecb(cipher_null))",
2048 .cra_driver_name
= "authenc-hmac-md5-"
2049 "ecb-cipher_null-caam",
2050 .cra_blocksize
= NULL_BLOCK_SIZE
,
2052 .setkey
= aead_setkey
,
2053 .setauthsize
= aead_setauthsize
,
2054 .encrypt
= aead_encrypt
,
2055 .decrypt
= aead_decrypt
,
2056 .ivsize
= NULL_IV_SIZE
,
2057 .maxauthsize
= MD5_DIGEST_SIZE
,
2060 .class2_alg_type
= OP_ALG_ALGSEL_MD5
|
2061 OP_ALG_AAI_HMAC_PRECOMP
,
2067 .cra_name
= "authenc(hmac(sha1),"
2068 "ecb(cipher_null))",
2069 .cra_driver_name
= "authenc-hmac-sha1-"
2070 "ecb-cipher_null-caam",
2071 .cra_blocksize
= NULL_BLOCK_SIZE
,
2073 .setkey
= aead_setkey
,
2074 .setauthsize
= aead_setauthsize
,
2075 .encrypt
= aead_encrypt
,
2076 .decrypt
= aead_decrypt
,
2077 .ivsize
= NULL_IV_SIZE
,
2078 .maxauthsize
= SHA1_DIGEST_SIZE
,
2081 .class2_alg_type
= OP_ALG_ALGSEL_SHA1
|
2082 OP_ALG_AAI_HMAC_PRECOMP
,
2088 .cra_name
= "authenc(hmac(sha224),"
2089 "ecb(cipher_null))",
2090 .cra_driver_name
= "authenc-hmac-sha224-"
2091 "ecb-cipher_null-caam",
2092 .cra_blocksize
= NULL_BLOCK_SIZE
,
2094 .setkey
= aead_setkey
,
2095 .setauthsize
= aead_setauthsize
,
2096 .encrypt
= aead_encrypt
,
2097 .decrypt
= aead_decrypt
,
2098 .ivsize
= NULL_IV_SIZE
,
2099 .maxauthsize
= SHA224_DIGEST_SIZE
,
2102 .class2_alg_type
= OP_ALG_ALGSEL_SHA224
|
2103 OP_ALG_AAI_HMAC_PRECOMP
,
2109 .cra_name
= "authenc(hmac(sha256),"
2110 "ecb(cipher_null))",
2111 .cra_driver_name
= "authenc-hmac-sha256-"
2112 "ecb-cipher_null-caam",
2113 .cra_blocksize
= NULL_BLOCK_SIZE
,
2115 .setkey
= aead_setkey
,
2116 .setauthsize
= aead_setauthsize
,
2117 .encrypt
= aead_encrypt
,
2118 .decrypt
= aead_decrypt
,
2119 .ivsize
= NULL_IV_SIZE
,
2120 .maxauthsize
= SHA256_DIGEST_SIZE
,
2123 .class2_alg_type
= OP_ALG_ALGSEL_SHA256
|
2124 OP_ALG_AAI_HMAC_PRECOMP
,
2130 .cra_name
= "authenc(hmac(sha384),"
2131 "ecb(cipher_null))",
2132 .cra_driver_name
= "authenc-hmac-sha384-"
2133 "ecb-cipher_null-caam",
2134 .cra_blocksize
= NULL_BLOCK_SIZE
,
2136 .setkey
= aead_setkey
,
2137 .setauthsize
= aead_setauthsize
,
2138 .encrypt
= aead_encrypt
,
2139 .decrypt
= aead_decrypt
,
2140 .ivsize
= NULL_IV_SIZE
,
2141 .maxauthsize
= SHA384_DIGEST_SIZE
,
2144 .class2_alg_type
= OP_ALG_ALGSEL_SHA384
|
2145 OP_ALG_AAI_HMAC_PRECOMP
,
2151 .cra_name
= "authenc(hmac(sha512),"
2152 "ecb(cipher_null))",
2153 .cra_driver_name
= "authenc-hmac-sha512-"
2154 "ecb-cipher_null-caam",
2155 .cra_blocksize
= NULL_BLOCK_SIZE
,
2157 .setkey
= aead_setkey
,
2158 .setauthsize
= aead_setauthsize
,
2159 .encrypt
= aead_encrypt
,
2160 .decrypt
= aead_decrypt
,
2161 .ivsize
= NULL_IV_SIZE
,
2162 .maxauthsize
= SHA512_DIGEST_SIZE
,
2165 .class2_alg_type
= OP_ALG_ALGSEL_SHA512
|
2166 OP_ALG_AAI_HMAC_PRECOMP
,
2172 .cra_name
= "authenc(hmac(md5),cbc(aes))",
2173 .cra_driver_name
= "authenc-hmac-md5-"
2175 .cra_blocksize
= AES_BLOCK_SIZE
,
2177 .setkey
= aead_setkey
,
2178 .setauthsize
= aead_setauthsize
,
2179 .encrypt
= aead_encrypt
,
2180 .decrypt
= aead_decrypt
,
2181 .ivsize
= AES_BLOCK_SIZE
,
2182 .maxauthsize
= MD5_DIGEST_SIZE
,
2185 .class1_alg_type
= OP_ALG_ALGSEL_AES
| OP_ALG_AAI_CBC
,
2186 .class2_alg_type
= OP_ALG_ALGSEL_MD5
|
2187 OP_ALG_AAI_HMAC_PRECOMP
,
2193 .cra_name
= "echainiv(authenc(hmac(md5),"
2195 .cra_driver_name
= "echainiv-authenc-hmac-md5-"
2197 .cra_blocksize
= AES_BLOCK_SIZE
,
2199 .setkey
= aead_setkey
,
2200 .setauthsize
= aead_setauthsize
,
2201 .encrypt
= aead_encrypt
,
2202 .decrypt
= aead_decrypt
,
2203 .ivsize
= AES_BLOCK_SIZE
,
2204 .maxauthsize
= MD5_DIGEST_SIZE
,
2207 .class1_alg_type
= OP_ALG_ALGSEL_AES
| OP_ALG_AAI_CBC
,
2208 .class2_alg_type
= OP_ALG_ALGSEL_MD5
|
2209 OP_ALG_AAI_HMAC_PRECOMP
,
2216 .cra_name
= "authenc(hmac(sha1),cbc(aes))",
2217 .cra_driver_name
= "authenc-hmac-sha1-"
2219 .cra_blocksize
= AES_BLOCK_SIZE
,
2221 .setkey
= aead_setkey
,
2222 .setauthsize
= aead_setauthsize
,
2223 .encrypt
= aead_encrypt
,
2224 .decrypt
= aead_decrypt
,
2225 .ivsize
= AES_BLOCK_SIZE
,
2226 .maxauthsize
= SHA1_DIGEST_SIZE
,
2229 .class1_alg_type
= OP_ALG_ALGSEL_AES
| OP_ALG_AAI_CBC
,
2230 .class2_alg_type
= OP_ALG_ALGSEL_SHA1
|
2231 OP_ALG_AAI_HMAC_PRECOMP
,
2237 .cra_name
= "echainiv(authenc(hmac(sha1),"
2239 .cra_driver_name
= "echainiv-authenc-"
2240 "hmac-sha1-cbc-aes-caam",
2241 .cra_blocksize
= AES_BLOCK_SIZE
,
2243 .setkey
= aead_setkey
,
2244 .setauthsize
= aead_setauthsize
,
2245 .encrypt
= aead_encrypt
,
2246 .decrypt
= aead_decrypt
,
2247 .ivsize
= AES_BLOCK_SIZE
,
2248 .maxauthsize
= SHA1_DIGEST_SIZE
,
2251 .class1_alg_type
= OP_ALG_ALGSEL_AES
| OP_ALG_AAI_CBC
,
2252 .class2_alg_type
= OP_ALG_ALGSEL_SHA1
|
2253 OP_ALG_AAI_HMAC_PRECOMP
,
2260 .cra_name
= "authenc(hmac(sha224),cbc(aes))",
2261 .cra_driver_name
= "authenc-hmac-sha224-"
2263 .cra_blocksize
= AES_BLOCK_SIZE
,
2265 .setkey
= aead_setkey
,
2266 .setauthsize
= aead_setauthsize
,
2267 .encrypt
= aead_encrypt
,
2268 .decrypt
= aead_decrypt
,
2269 .ivsize
= AES_BLOCK_SIZE
,
2270 .maxauthsize
= SHA224_DIGEST_SIZE
,
2273 .class1_alg_type
= OP_ALG_ALGSEL_AES
| OP_ALG_AAI_CBC
,
2274 .class2_alg_type
= OP_ALG_ALGSEL_SHA224
|
2275 OP_ALG_AAI_HMAC_PRECOMP
,
2281 .cra_name
= "echainiv(authenc(hmac(sha224),"
2283 .cra_driver_name
= "echainiv-authenc-"
2284 "hmac-sha224-cbc-aes-caam",
2285 .cra_blocksize
= AES_BLOCK_SIZE
,
2287 .setkey
= aead_setkey
,
2288 .setauthsize
= aead_setauthsize
,
2289 .encrypt
= aead_encrypt
,
2290 .decrypt
= aead_decrypt
,
2291 .ivsize
= AES_BLOCK_SIZE
,
2292 .maxauthsize
= SHA224_DIGEST_SIZE
,
2295 .class1_alg_type
= OP_ALG_ALGSEL_AES
| OP_ALG_AAI_CBC
,
2296 .class2_alg_type
= OP_ALG_ALGSEL_SHA224
|
2297 OP_ALG_AAI_HMAC_PRECOMP
,
2304 .cra_name
= "authenc(hmac(sha256),cbc(aes))",
2305 .cra_driver_name
= "authenc-hmac-sha256-"
2307 .cra_blocksize
= AES_BLOCK_SIZE
,
2309 .setkey
= aead_setkey
,
2310 .setauthsize
= aead_setauthsize
,
2311 .encrypt
= aead_encrypt
,
2312 .decrypt
= aead_decrypt
,
2313 .ivsize
= AES_BLOCK_SIZE
,
2314 .maxauthsize
= SHA256_DIGEST_SIZE
,
2317 .class1_alg_type
= OP_ALG_ALGSEL_AES
| OP_ALG_AAI_CBC
,
2318 .class2_alg_type
= OP_ALG_ALGSEL_SHA256
|
2319 OP_ALG_AAI_HMAC_PRECOMP
,
2325 .cra_name
= "echainiv(authenc(hmac(sha256),"
2327 .cra_driver_name
= "echainiv-authenc-"
2328 "hmac-sha256-cbc-aes-caam",
2329 .cra_blocksize
= AES_BLOCK_SIZE
,
2331 .setkey
= aead_setkey
,
2332 .setauthsize
= aead_setauthsize
,
2333 .encrypt
= aead_encrypt
,
2334 .decrypt
= aead_decrypt
,
2335 .ivsize
= AES_BLOCK_SIZE
,
2336 .maxauthsize
= SHA256_DIGEST_SIZE
,
2339 .class1_alg_type
= OP_ALG_ALGSEL_AES
| OP_ALG_AAI_CBC
,
2340 .class2_alg_type
= OP_ALG_ALGSEL_SHA256
|
2341 OP_ALG_AAI_HMAC_PRECOMP
,
2348 .cra_name
= "authenc(hmac(sha384),cbc(aes))",
2349 .cra_driver_name
= "authenc-hmac-sha384-"
2351 .cra_blocksize
= AES_BLOCK_SIZE
,
2353 .setkey
= aead_setkey
,
2354 .setauthsize
= aead_setauthsize
,
2355 .encrypt
= aead_encrypt
,
2356 .decrypt
= aead_decrypt
,
2357 .ivsize
= AES_BLOCK_SIZE
,
2358 .maxauthsize
= SHA384_DIGEST_SIZE
,
2361 .class1_alg_type
= OP_ALG_ALGSEL_AES
| OP_ALG_AAI_CBC
,
2362 .class2_alg_type
= OP_ALG_ALGSEL_SHA384
|
2363 OP_ALG_AAI_HMAC_PRECOMP
,
2369 .cra_name
= "echainiv(authenc(hmac(sha384),"
2371 .cra_driver_name
= "echainiv-authenc-"
2372 "hmac-sha384-cbc-aes-caam",
2373 .cra_blocksize
= AES_BLOCK_SIZE
,
2375 .setkey
= aead_setkey
,
2376 .setauthsize
= aead_setauthsize
,
2377 .encrypt
= aead_encrypt
,
2378 .decrypt
= aead_decrypt
,
2379 .ivsize
= AES_BLOCK_SIZE
,
2380 .maxauthsize
= SHA384_DIGEST_SIZE
,
2383 .class1_alg_type
= OP_ALG_ALGSEL_AES
| OP_ALG_AAI_CBC
,
2384 .class2_alg_type
= OP_ALG_ALGSEL_SHA384
|
2385 OP_ALG_AAI_HMAC_PRECOMP
,
2392 .cra_name
= "authenc(hmac(sha512),cbc(aes))",
2393 .cra_driver_name
= "authenc-hmac-sha512-"
2395 .cra_blocksize
= AES_BLOCK_SIZE
,
2397 .setkey
= aead_setkey
,
2398 .setauthsize
= aead_setauthsize
,
2399 .encrypt
= aead_encrypt
,
2400 .decrypt
= aead_decrypt
,
2401 .ivsize
= AES_BLOCK_SIZE
,
2402 .maxauthsize
= SHA512_DIGEST_SIZE
,
2405 .class1_alg_type
= OP_ALG_ALGSEL_AES
| OP_ALG_AAI_CBC
,
2406 .class2_alg_type
= OP_ALG_ALGSEL_SHA512
|
2407 OP_ALG_AAI_HMAC_PRECOMP
,
2413 .cra_name
= "echainiv(authenc(hmac(sha512),"
2415 .cra_driver_name
= "echainiv-authenc-"
2416 "hmac-sha512-cbc-aes-caam",
2417 .cra_blocksize
= AES_BLOCK_SIZE
,
2419 .setkey
= aead_setkey
,
2420 .setauthsize
= aead_setauthsize
,
2421 .encrypt
= aead_encrypt
,
2422 .decrypt
= aead_decrypt
,
2423 .ivsize
= AES_BLOCK_SIZE
,
2424 .maxauthsize
= SHA512_DIGEST_SIZE
,
2427 .class1_alg_type
= OP_ALG_ALGSEL_AES
| OP_ALG_AAI_CBC
,
2428 .class2_alg_type
= OP_ALG_ALGSEL_SHA512
|
2429 OP_ALG_AAI_HMAC_PRECOMP
,
2436 .cra_name
= "authenc(hmac(md5),cbc(des3_ede))",
2437 .cra_driver_name
= "authenc-hmac-md5-"
2438 "cbc-des3_ede-caam",
2439 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
2441 .setkey
= des3_aead_setkey
,
2442 .setauthsize
= aead_setauthsize
,
2443 .encrypt
= aead_encrypt
,
2444 .decrypt
= aead_decrypt
,
2445 .ivsize
= DES3_EDE_BLOCK_SIZE
,
2446 .maxauthsize
= MD5_DIGEST_SIZE
,
2449 .class1_alg_type
= OP_ALG_ALGSEL_3DES
| OP_ALG_AAI_CBC
,
2450 .class2_alg_type
= OP_ALG_ALGSEL_MD5
|
2451 OP_ALG_AAI_HMAC_PRECOMP
,
2457 .cra_name
= "echainiv(authenc(hmac(md5),"
2459 .cra_driver_name
= "echainiv-authenc-hmac-md5-"
2460 "cbc-des3_ede-caam",
2461 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
2463 .setkey
= des3_aead_setkey
,
2464 .setauthsize
= aead_setauthsize
,
2465 .encrypt
= aead_encrypt
,
2466 .decrypt
= aead_decrypt
,
2467 .ivsize
= DES3_EDE_BLOCK_SIZE
,
2468 .maxauthsize
= MD5_DIGEST_SIZE
,
2471 .class1_alg_type
= OP_ALG_ALGSEL_3DES
| OP_ALG_AAI_CBC
,
2472 .class2_alg_type
= OP_ALG_ALGSEL_MD5
|
2473 OP_ALG_AAI_HMAC_PRECOMP
,
2480 .cra_name
= "authenc(hmac(sha1),"
2482 .cra_driver_name
= "authenc-hmac-sha1-"
2483 "cbc-des3_ede-caam",
2484 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
2486 .setkey
= des3_aead_setkey
,
2487 .setauthsize
= aead_setauthsize
,
2488 .encrypt
= aead_encrypt
,
2489 .decrypt
= aead_decrypt
,
2490 .ivsize
= DES3_EDE_BLOCK_SIZE
,
2491 .maxauthsize
= SHA1_DIGEST_SIZE
,
2494 .class1_alg_type
= OP_ALG_ALGSEL_3DES
| OP_ALG_AAI_CBC
,
2495 .class2_alg_type
= OP_ALG_ALGSEL_SHA1
|
2496 OP_ALG_AAI_HMAC_PRECOMP
,
2502 .cra_name
= "echainiv(authenc(hmac(sha1),"
2504 .cra_driver_name
= "echainiv-authenc-"
2506 "cbc-des3_ede-caam",
2507 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
2509 .setkey
= des3_aead_setkey
,
2510 .setauthsize
= aead_setauthsize
,
2511 .encrypt
= aead_encrypt
,
2512 .decrypt
= aead_decrypt
,
2513 .ivsize
= DES3_EDE_BLOCK_SIZE
,
2514 .maxauthsize
= SHA1_DIGEST_SIZE
,
2517 .class1_alg_type
= OP_ALG_ALGSEL_3DES
| OP_ALG_AAI_CBC
,
2518 .class2_alg_type
= OP_ALG_ALGSEL_SHA1
|
2519 OP_ALG_AAI_HMAC_PRECOMP
,
2526 .cra_name
= "authenc(hmac(sha224),"
2528 .cra_driver_name
= "authenc-hmac-sha224-"
2529 "cbc-des3_ede-caam",
2530 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
2532 .setkey
= des3_aead_setkey
,
2533 .setauthsize
= aead_setauthsize
,
2534 .encrypt
= aead_encrypt
,
2535 .decrypt
= aead_decrypt
,
2536 .ivsize
= DES3_EDE_BLOCK_SIZE
,
2537 .maxauthsize
= SHA224_DIGEST_SIZE
,
2540 .class1_alg_type
= OP_ALG_ALGSEL_3DES
| OP_ALG_AAI_CBC
,
2541 .class2_alg_type
= OP_ALG_ALGSEL_SHA224
|
2542 OP_ALG_AAI_HMAC_PRECOMP
,
2548 .cra_name
= "echainiv(authenc(hmac(sha224),"
2550 .cra_driver_name
= "echainiv-authenc-"
2552 "cbc-des3_ede-caam",
2553 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
2555 .setkey
= des3_aead_setkey
,
2556 .setauthsize
= aead_setauthsize
,
2557 .encrypt
= aead_encrypt
,
2558 .decrypt
= aead_decrypt
,
2559 .ivsize
= DES3_EDE_BLOCK_SIZE
,
2560 .maxauthsize
= SHA224_DIGEST_SIZE
,
2563 .class1_alg_type
= OP_ALG_ALGSEL_3DES
| OP_ALG_AAI_CBC
,
2564 .class2_alg_type
= OP_ALG_ALGSEL_SHA224
|
2565 OP_ALG_AAI_HMAC_PRECOMP
,
2572 .cra_name
= "authenc(hmac(sha256),"
2574 .cra_driver_name
= "authenc-hmac-sha256-"
2575 "cbc-des3_ede-caam",
2576 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
2578 .setkey
= des3_aead_setkey
,
2579 .setauthsize
= aead_setauthsize
,
2580 .encrypt
= aead_encrypt
,
2581 .decrypt
= aead_decrypt
,
2582 .ivsize
= DES3_EDE_BLOCK_SIZE
,
2583 .maxauthsize
= SHA256_DIGEST_SIZE
,
2586 .class1_alg_type
= OP_ALG_ALGSEL_3DES
| OP_ALG_AAI_CBC
,
2587 .class2_alg_type
= OP_ALG_ALGSEL_SHA256
|
2588 OP_ALG_AAI_HMAC_PRECOMP
,
2594 .cra_name
= "echainiv(authenc(hmac(sha256),"
2596 .cra_driver_name
= "echainiv-authenc-"
2598 "cbc-des3_ede-caam",
2599 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
2601 .setkey
= des3_aead_setkey
,
2602 .setauthsize
= aead_setauthsize
,
2603 .encrypt
= aead_encrypt
,
2604 .decrypt
= aead_decrypt
,
2605 .ivsize
= DES3_EDE_BLOCK_SIZE
,
2606 .maxauthsize
= SHA256_DIGEST_SIZE
,
2609 .class1_alg_type
= OP_ALG_ALGSEL_3DES
| OP_ALG_AAI_CBC
,
2610 .class2_alg_type
= OP_ALG_ALGSEL_SHA256
|
2611 OP_ALG_AAI_HMAC_PRECOMP
,
2618 .cra_name
= "authenc(hmac(sha384),"
2620 .cra_driver_name
= "authenc-hmac-sha384-"
2621 "cbc-des3_ede-caam",
2622 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
2624 .setkey
= des3_aead_setkey
,
2625 .setauthsize
= aead_setauthsize
,
2626 .encrypt
= aead_encrypt
,
2627 .decrypt
= aead_decrypt
,
2628 .ivsize
= DES3_EDE_BLOCK_SIZE
,
2629 .maxauthsize
= SHA384_DIGEST_SIZE
,
2632 .class1_alg_type
= OP_ALG_ALGSEL_3DES
| OP_ALG_AAI_CBC
,
2633 .class2_alg_type
= OP_ALG_ALGSEL_SHA384
|
2634 OP_ALG_AAI_HMAC_PRECOMP
,
2640 .cra_name
= "echainiv(authenc(hmac(sha384),"
2642 .cra_driver_name
= "echainiv-authenc-"
2644 "cbc-des3_ede-caam",
2645 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
2647 .setkey
= des3_aead_setkey
,
2648 .setauthsize
= aead_setauthsize
,
2649 .encrypt
= aead_encrypt
,
2650 .decrypt
= aead_decrypt
,
2651 .ivsize
= DES3_EDE_BLOCK_SIZE
,
2652 .maxauthsize
= SHA384_DIGEST_SIZE
,
2655 .class1_alg_type
= OP_ALG_ALGSEL_3DES
| OP_ALG_AAI_CBC
,
2656 .class2_alg_type
= OP_ALG_ALGSEL_SHA384
|
2657 OP_ALG_AAI_HMAC_PRECOMP
,
2664 .cra_name
= "authenc(hmac(sha512),"
2666 .cra_driver_name
= "authenc-hmac-sha512-"
2667 "cbc-des3_ede-caam",
2668 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
2670 .setkey
= des3_aead_setkey
,
2671 .setauthsize
= aead_setauthsize
,
2672 .encrypt
= aead_encrypt
,
2673 .decrypt
= aead_decrypt
,
2674 .ivsize
= DES3_EDE_BLOCK_SIZE
,
2675 .maxauthsize
= SHA512_DIGEST_SIZE
,
2678 .class1_alg_type
= OP_ALG_ALGSEL_3DES
| OP_ALG_AAI_CBC
,
2679 .class2_alg_type
= OP_ALG_ALGSEL_SHA512
|
2680 OP_ALG_AAI_HMAC_PRECOMP
,
2686 .cra_name
= "echainiv(authenc(hmac(sha512),"
2688 .cra_driver_name
= "echainiv-authenc-"
2690 "cbc-des3_ede-caam",
2691 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
2693 .setkey
= des3_aead_setkey
,
2694 .setauthsize
= aead_setauthsize
,
2695 .encrypt
= aead_encrypt
,
2696 .decrypt
= aead_decrypt
,
2697 .ivsize
= DES3_EDE_BLOCK_SIZE
,
2698 .maxauthsize
= SHA512_DIGEST_SIZE
,
2701 .class1_alg_type
= OP_ALG_ALGSEL_3DES
| OP_ALG_AAI_CBC
,
2702 .class2_alg_type
= OP_ALG_ALGSEL_SHA512
|
2703 OP_ALG_AAI_HMAC_PRECOMP
,
2710 .cra_name
= "authenc(hmac(md5),cbc(des))",
2711 .cra_driver_name
= "authenc-hmac-md5-"
2713 .cra_blocksize
= DES_BLOCK_SIZE
,
2715 .setkey
= aead_setkey
,
2716 .setauthsize
= aead_setauthsize
,
2717 .encrypt
= aead_encrypt
,
2718 .decrypt
= aead_decrypt
,
2719 .ivsize
= DES_BLOCK_SIZE
,
2720 .maxauthsize
= MD5_DIGEST_SIZE
,
2723 .class1_alg_type
= OP_ALG_ALGSEL_DES
| OP_ALG_AAI_CBC
,
2724 .class2_alg_type
= OP_ALG_ALGSEL_MD5
|
2725 OP_ALG_AAI_HMAC_PRECOMP
,
2731 .cra_name
= "echainiv(authenc(hmac(md5),"
2733 .cra_driver_name
= "echainiv-authenc-hmac-md5-"
2735 .cra_blocksize
= DES_BLOCK_SIZE
,
2737 .setkey
= aead_setkey
,
2738 .setauthsize
= aead_setauthsize
,
2739 .encrypt
= aead_encrypt
,
2740 .decrypt
= aead_decrypt
,
2741 .ivsize
= DES_BLOCK_SIZE
,
2742 .maxauthsize
= MD5_DIGEST_SIZE
,
2745 .class1_alg_type
= OP_ALG_ALGSEL_DES
| OP_ALG_AAI_CBC
,
2746 .class2_alg_type
= OP_ALG_ALGSEL_MD5
|
2747 OP_ALG_AAI_HMAC_PRECOMP
,
2754 .cra_name
= "authenc(hmac(sha1),cbc(des))",
2755 .cra_driver_name
= "authenc-hmac-sha1-"
2757 .cra_blocksize
= DES_BLOCK_SIZE
,
2759 .setkey
= aead_setkey
,
2760 .setauthsize
= aead_setauthsize
,
2761 .encrypt
= aead_encrypt
,
2762 .decrypt
= aead_decrypt
,
2763 .ivsize
= DES_BLOCK_SIZE
,
2764 .maxauthsize
= SHA1_DIGEST_SIZE
,
2767 .class1_alg_type
= OP_ALG_ALGSEL_DES
| OP_ALG_AAI_CBC
,
2768 .class2_alg_type
= OP_ALG_ALGSEL_SHA1
|
2769 OP_ALG_AAI_HMAC_PRECOMP
,
2775 .cra_name
= "echainiv(authenc(hmac(sha1),"
2777 .cra_driver_name
= "echainiv-authenc-"
2778 "hmac-sha1-cbc-des-caam",
2779 .cra_blocksize
= DES_BLOCK_SIZE
,
2781 .setkey
= aead_setkey
,
2782 .setauthsize
= aead_setauthsize
,
2783 .encrypt
= aead_encrypt
,
2784 .decrypt
= aead_decrypt
,
2785 .ivsize
= DES_BLOCK_SIZE
,
2786 .maxauthsize
= SHA1_DIGEST_SIZE
,
2789 .class1_alg_type
= OP_ALG_ALGSEL_DES
| OP_ALG_AAI_CBC
,
2790 .class2_alg_type
= OP_ALG_ALGSEL_SHA1
|
2791 OP_ALG_AAI_HMAC_PRECOMP
,
2798 .cra_name
= "authenc(hmac(sha224),cbc(des))",
2799 .cra_driver_name
= "authenc-hmac-sha224-"
2801 .cra_blocksize
= DES_BLOCK_SIZE
,
2803 .setkey
= aead_setkey
,
2804 .setauthsize
= aead_setauthsize
,
2805 .encrypt
= aead_encrypt
,
2806 .decrypt
= aead_decrypt
,
2807 .ivsize
= DES_BLOCK_SIZE
,
2808 .maxauthsize
= SHA224_DIGEST_SIZE
,
2811 .class1_alg_type
= OP_ALG_ALGSEL_DES
| OP_ALG_AAI_CBC
,
2812 .class2_alg_type
= OP_ALG_ALGSEL_SHA224
|
2813 OP_ALG_AAI_HMAC_PRECOMP
,
2819 .cra_name
= "echainiv(authenc(hmac(sha224),"
2821 .cra_driver_name
= "echainiv-authenc-"
2822 "hmac-sha224-cbc-des-caam",
2823 .cra_blocksize
= DES_BLOCK_SIZE
,
2825 .setkey
= aead_setkey
,
2826 .setauthsize
= aead_setauthsize
,
2827 .encrypt
= aead_encrypt
,
2828 .decrypt
= aead_decrypt
,
2829 .ivsize
= DES_BLOCK_SIZE
,
2830 .maxauthsize
= SHA224_DIGEST_SIZE
,
2833 .class1_alg_type
= OP_ALG_ALGSEL_DES
| OP_ALG_AAI_CBC
,
2834 .class2_alg_type
= OP_ALG_ALGSEL_SHA224
|
2835 OP_ALG_AAI_HMAC_PRECOMP
,
2842 .cra_name
= "authenc(hmac(sha256),cbc(des))",
2843 .cra_driver_name
= "authenc-hmac-sha256-"
2845 .cra_blocksize
= DES_BLOCK_SIZE
,
2847 .setkey
= aead_setkey
,
2848 .setauthsize
= aead_setauthsize
,
2849 .encrypt
= aead_encrypt
,
2850 .decrypt
= aead_decrypt
,
2851 .ivsize
= DES_BLOCK_SIZE
,
2852 .maxauthsize
= SHA256_DIGEST_SIZE
,
2855 .class1_alg_type
= OP_ALG_ALGSEL_DES
| OP_ALG_AAI_CBC
,
2856 .class2_alg_type
= OP_ALG_ALGSEL_SHA256
|
2857 OP_ALG_AAI_HMAC_PRECOMP
,
2863 .cra_name
= "echainiv(authenc(hmac(sha256),"
2865 .cra_driver_name
= "echainiv-authenc-"
2866 "hmac-sha256-cbc-des-caam",
2867 .cra_blocksize
= DES_BLOCK_SIZE
,
2869 .setkey
= aead_setkey
,
2870 .setauthsize
= aead_setauthsize
,
2871 .encrypt
= aead_encrypt
,
2872 .decrypt
= aead_decrypt
,
2873 .ivsize
= DES_BLOCK_SIZE
,
2874 .maxauthsize
= SHA256_DIGEST_SIZE
,
2877 .class1_alg_type
= OP_ALG_ALGSEL_DES
| OP_ALG_AAI_CBC
,
2878 .class2_alg_type
= OP_ALG_ALGSEL_SHA256
|
2879 OP_ALG_AAI_HMAC_PRECOMP
,
2886 .cra_name
= "authenc(hmac(sha384),cbc(des))",
2887 .cra_driver_name
= "authenc-hmac-sha384-"
2889 .cra_blocksize
= DES_BLOCK_SIZE
,
2891 .setkey
= aead_setkey
,
2892 .setauthsize
= aead_setauthsize
,
2893 .encrypt
= aead_encrypt
,
2894 .decrypt
= aead_decrypt
,
2895 .ivsize
= DES_BLOCK_SIZE
,
2896 .maxauthsize
= SHA384_DIGEST_SIZE
,
2899 .class1_alg_type
= OP_ALG_ALGSEL_DES
| OP_ALG_AAI_CBC
,
2900 .class2_alg_type
= OP_ALG_ALGSEL_SHA384
|
2901 OP_ALG_AAI_HMAC_PRECOMP
,
2907 .cra_name
= "echainiv(authenc(hmac(sha384),"
2909 .cra_driver_name
= "echainiv-authenc-"
2910 "hmac-sha384-cbc-des-caam",
2911 .cra_blocksize
= DES_BLOCK_SIZE
,
2913 .setkey
= aead_setkey
,
2914 .setauthsize
= aead_setauthsize
,
2915 .encrypt
= aead_encrypt
,
2916 .decrypt
= aead_decrypt
,
2917 .ivsize
= DES_BLOCK_SIZE
,
2918 .maxauthsize
= SHA384_DIGEST_SIZE
,
2921 .class1_alg_type
= OP_ALG_ALGSEL_DES
| OP_ALG_AAI_CBC
,
2922 .class2_alg_type
= OP_ALG_ALGSEL_SHA384
|
2923 OP_ALG_AAI_HMAC_PRECOMP
,
2930 .cra_name
= "authenc(hmac(sha512),cbc(des))",
2931 .cra_driver_name
= "authenc-hmac-sha512-"
2933 .cra_blocksize
= DES_BLOCK_SIZE
,
2935 .setkey
= aead_setkey
,
2936 .setauthsize
= aead_setauthsize
,
2937 .encrypt
= aead_encrypt
,
2938 .decrypt
= aead_decrypt
,
2939 .ivsize
= DES_BLOCK_SIZE
,
2940 .maxauthsize
= SHA512_DIGEST_SIZE
,
2943 .class1_alg_type
= OP_ALG_ALGSEL_DES
| OP_ALG_AAI_CBC
,
2944 .class2_alg_type
= OP_ALG_ALGSEL_SHA512
|
2945 OP_ALG_AAI_HMAC_PRECOMP
,
2951 .cra_name
= "echainiv(authenc(hmac(sha512),"
2953 .cra_driver_name
= "echainiv-authenc-"
2954 "hmac-sha512-cbc-des-caam",
2955 .cra_blocksize
= DES_BLOCK_SIZE
,
2957 .setkey
= aead_setkey
,
2958 .setauthsize
= aead_setauthsize
,
2959 .encrypt
= aead_encrypt
,
2960 .decrypt
= aead_decrypt
,
2961 .ivsize
= DES_BLOCK_SIZE
,
2962 .maxauthsize
= SHA512_DIGEST_SIZE
,
2965 .class1_alg_type
= OP_ALG_ALGSEL_DES
| OP_ALG_AAI_CBC
,
2966 .class2_alg_type
= OP_ALG_ALGSEL_SHA512
|
2967 OP_ALG_AAI_HMAC_PRECOMP
,
2974 .cra_name
= "authenc(hmac(md5),"
2975 "rfc3686(ctr(aes)))",
2976 .cra_driver_name
= "authenc-hmac-md5-"
2977 "rfc3686-ctr-aes-caam",
2980 .setkey
= aead_setkey
,
2981 .setauthsize
= aead_setauthsize
,
2982 .encrypt
= aead_encrypt
,
2983 .decrypt
= aead_decrypt
,
2984 .ivsize
= CTR_RFC3686_IV_SIZE
,
2985 .maxauthsize
= MD5_DIGEST_SIZE
,
2988 .class1_alg_type
= OP_ALG_ALGSEL_AES
|
2989 OP_ALG_AAI_CTR_MOD128
,
2990 .class2_alg_type
= OP_ALG_ALGSEL_MD5
|
2991 OP_ALG_AAI_HMAC_PRECOMP
,
2998 .cra_name
= "seqiv(authenc("
2999 "hmac(md5),rfc3686(ctr(aes))))",
3000 .cra_driver_name
= "seqiv-authenc-hmac-md5-"
3001 "rfc3686-ctr-aes-caam",
3004 .setkey
= aead_setkey
,
3005 .setauthsize
= aead_setauthsize
,
3006 .encrypt
= aead_encrypt
,
3007 .decrypt
= aead_decrypt
,
3008 .ivsize
= CTR_RFC3686_IV_SIZE
,
3009 .maxauthsize
= MD5_DIGEST_SIZE
,
3012 .class1_alg_type
= OP_ALG_ALGSEL_AES
|
3013 OP_ALG_AAI_CTR_MOD128
,
3014 .class2_alg_type
= OP_ALG_ALGSEL_MD5
|
3015 OP_ALG_AAI_HMAC_PRECOMP
,
3023 .cra_name
= "authenc(hmac(sha1),"
3024 "rfc3686(ctr(aes)))",
3025 .cra_driver_name
= "authenc-hmac-sha1-"
3026 "rfc3686-ctr-aes-caam",
3029 .setkey
= aead_setkey
,
3030 .setauthsize
= aead_setauthsize
,
3031 .encrypt
= aead_encrypt
,
3032 .decrypt
= aead_decrypt
,
3033 .ivsize
= CTR_RFC3686_IV_SIZE
,
3034 .maxauthsize
= SHA1_DIGEST_SIZE
,
3037 .class1_alg_type
= OP_ALG_ALGSEL_AES
|
3038 OP_ALG_AAI_CTR_MOD128
,
3039 .class2_alg_type
= OP_ALG_ALGSEL_SHA1
|
3040 OP_ALG_AAI_HMAC_PRECOMP
,
3047 .cra_name
= "seqiv(authenc("
3048 "hmac(sha1),rfc3686(ctr(aes))))",
3049 .cra_driver_name
= "seqiv-authenc-hmac-sha1-"
3050 "rfc3686-ctr-aes-caam",
3053 .setkey
= aead_setkey
,
3054 .setauthsize
= aead_setauthsize
,
3055 .encrypt
= aead_encrypt
,
3056 .decrypt
= aead_decrypt
,
3057 .ivsize
= CTR_RFC3686_IV_SIZE
,
3058 .maxauthsize
= SHA1_DIGEST_SIZE
,
3061 .class1_alg_type
= OP_ALG_ALGSEL_AES
|
3062 OP_ALG_AAI_CTR_MOD128
,
3063 .class2_alg_type
= OP_ALG_ALGSEL_SHA1
|
3064 OP_ALG_AAI_HMAC_PRECOMP
,
3072 .cra_name
= "authenc(hmac(sha224),"
3073 "rfc3686(ctr(aes)))",
3074 .cra_driver_name
= "authenc-hmac-sha224-"
3075 "rfc3686-ctr-aes-caam",
3078 .setkey
= aead_setkey
,
3079 .setauthsize
= aead_setauthsize
,
3080 .encrypt
= aead_encrypt
,
3081 .decrypt
= aead_decrypt
,
3082 .ivsize
= CTR_RFC3686_IV_SIZE
,
3083 .maxauthsize
= SHA224_DIGEST_SIZE
,
3086 .class1_alg_type
= OP_ALG_ALGSEL_AES
|
3087 OP_ALG_AAI_CTR_MOD128
,
3088 .class2_alg_type
= OP_ALG_ALGSEL_SHA224
|
3089 OP_ALG_AAI_HMAC_PRECOMP
,
3096 .cra_name
= "seqiv(authenc("
3097 "hmac(sha224),rfc3686(ctr(aes))))",
3098 .cra_driver_name
= "seqiv-authenc-hmac-sha224-"
3099 "rfc3686-ctr-aes-caam",
3102 .setkey
= aead_setkey
,
3103 .setauthsize
= aead_setauthsize
,
3104 .encrypt
= aead_encrypt
,
3105 .decrypt
= aead_decrypt
,
3106 .ivsize
= CTR_RFC3686_IV_SIZE
,
3107 .maxauthsize
= SHA224_DIGEST_SIZE
,
3110 .class1_alg_type
= OP_ALG_ALGSEL_AES
|
3111 OP_ALG_AAI_CTR_MOD128
,
3112 .class2_alg_type
= OP_ALG_ALGSEL_SHA224
|
3113 OP_ALG_AAI_HMAC_PRECOMP
,
3121 .cra_name
= "authenc(hmac(sha256),"
3122 "rfc3686(ctr(aes)))",
3123 .cra_driver_name
= "authenc-hmac-sha256-"
3124 "rfc3686-ctr-aes-caam",
3127 .setkey
= aead_setkey
,
3128 .setauthsize
= aead_setauthsize
,
3129 .encrypt
= aead_encrypt
,
3130 .decrypt
= aead_decrypt
,
3131 .ivsize
= CTR_RFC3686_IV_SIZE
,
3132 .maxauthsize
= SHA256_DIGEST_SIZE
,
3135 .class1_alg_type
= OP_ALG_ALGSEL_AES
|
3136 OP_ALG_AAI_CTR_MOD128
,
3137 .class2_alg_type
= OP_ALG_ALGSEL_SHA256
|
3138 OP_ALG_AAI_HMAC_PRECOMP
,
3145 .cra_name
= "seqiv(authenc(hmac(sha256),"
3146 "rfc3686(ctr(aes))))",
3147 .cra_driver_name
= "seqiv-authenc-hmac-sha256-"
3148 "rfc3686-ctr-aes-caam",
3151 .setkey
= aead_setkey
,
3152 .setauthsize
= aead_setauthsize
,
3153 .encrypt
= aead_encrypt
,
3154 .decrypt
= aead_decrypt
,
3155 .ivsize
= CTR_RFC3686_IV_SIZE
,
3156 .maxauthsize
= SHA256_DIGEST_SIZE
,
3159 .class1_alg_type
= OP_ALG_ALGSEL_AES
|
3160 OP_ALG_AAI_CTR_MOD128
,
3161 .class2_alg_type
= OP_ALG_ALGSEL_SHA256
|
3162 OP_ALG_AAI_HMAC_PRECOMP
,
3170 .cra_name
= "authenc(hmac(sha384),"
3171 "rfc3686(ctr(aes)))",
3172 .cra_driver_name
= "authenc-hmac-sha384-"
3173 "rfc3686-ctr-aes-caam",
3176 .setkey
= aead_setkey
,
3177 .setauthsize
= aead_setauthsize
,
3178 .encrypt
= aead_encrypt
,
3179 .decrypt
= aead_decrypt
,
3180 .ivsize
= CTR_RFC3686_IV_SIZE
,
3181 .maxauthsize
= SHA384_DIGEST_SIZE
,
3184 .class1_alg_type
= OP_ALG_ALGSEL_AES
|
3185 OP_ALG_AAI_CTR_MOD128
,
3186 .class2_alg_type
= OP_ALG_ALGSEL_SHA384
|
3187 OP_ALG_AAI_HMAC_PRECOMP
,
3194 .cra_name
= "seqiv(authenc(hmac(sha384),"
3195 "rfc3686(ctr(aes))))",
3196 .cra_driver_name
= "seqiv-authenc-hmac-sha384-"
3197 "rfc3686-ctr-aes-caam",
3200 .setkey
= aead_setkey
,
3201 .setauthsize
= aead_setauthsize
,
3202 .encrypt
= aead_encrypt
,
3203 .decrypt
= aead_decrypt
,
3204 .ivsize
= CTR_RFC3686_IV_SIZE
,
3205 .maxauthsize
= SHA384_DIGEST_SIZE
,
3208 .class1_alg_type
= OP_ALG_ALGSEL_AES
|
3209 OP_ALG_AAI_CTR_MOD128
,
3210 .class2_alg_type
= OP_ALG_ALGSEL_SHA384
|
3211 OP_ALG_AAI_HMAC_PRECOMP
,
3219 .cra_name
= "authenc(hmac(sha512),"
3220 "rfc3686(ctr(aes)))",
3221 .cra_driver_name
= "authenc-hmac-sha512-"
3222 "rfc3686-ctr-aes-caam",
3225 .setkey
= aead_setkey
,
3226 .setauthsize
= aead_setauthsize
,
3227 .encrypt
= aead_encrypt
,
3228 .decrypt
= aead_decrypt
,
3229 .ivsize
= CTR_RFC3686_IV_SIZE
,
3230 .maxauthsize
= SHA512_DIGEST_SIZE
,
3233 .class1_alg_type
= OP_ALG_ALGSEL_AES
|
3234 OP_ALG_AAI_CTR_MOD128
,
3235 .class2_alg_type
= OP_ALG_ALGSEL_SHA512
|
3236 OP_ALG_AAI_HMAC_PRECOMP
,
3243 .cra_name
= "seqiv(authenc(hmac(sha512),"
3244 "rfc3686(ctr(aes))))",
3245 .cra_driver_name
= "seqiv-authenc-hmac-sha512-"
3246 "rfc3686-ctr-aes-caam",
3249 .setkey
= aead_setkey
,
3250 .setauthsize
= aead_setauthsize
,
3251 .encrypt
= aead_encrypt
,
3252 .decrypt
= aead_decrypt
,
3253 .ivsize
= CTR_RFC3686_IV_SIZE
,
3254 .maxauthsize
= SHA512_DIGEST_SIZE
,
3257 .class1_alg_type
= OP_ALG_ALGSEL_AES
|
3258 OP_ALG_AAI_CTR_MOD128
,
3259 .class2_alg_type
= OP_ALG_ALGSEL_SHA512
|
3260 OP_ALG_AAI_HMAC_PRECOMP
,
3268 .cra_name
= "rfc7539(chacha20,poly1305)",
3269 .cra_driver_name
= "rfc7539-chacha20-poly1305-"
3273 .setkey
= chachapoly_setkey
,
3274 .setauthsize
= chachapoly_setauthsize
,
3275 .encrypt
= chachapoly_encrypt
,
3276 .decrypt
= chachapoly_decrypt
,
3277 .ivsize
= CHACHAPOLY_IV_SIZE
,
3278 .maxauthsize
= POLY1305_DIGEST_SIZE
,
3281 .class1_alg_type
= OP_ALG_ALGSEL_CHACHA20
|
3283 .class2_alg_type
= OP_ALG_ALGSEL_POLY1305
|
3291 .cra_name
= "rfc7539esp(chacha20,poly1305)",
3292 .cra_driver_name
= "rfc7539esp-chacha20-"
3296 .setkey
= chachapoly_setkey
,
3297 .setauthsize
= chachapoly_setauthsize
,
3298 .encrypt
= chachapoly_encrypt
,
3299 .decrypt
= chachapoly_decrypt
,
3301 .maxauthsize
= POLY1305_DIGEST_SIZE
,
3304 .class1_alg_type
= OP_ALG_ALGSEL_CHACHA20
|
3306 .class2_alg_type
= OP_ALG_ALGSEL_POLY1305
|
3313 static int caam_init_common(struct caam_ctx
*ctx
, struct caam_alg_entry
*caam
,
3316 dma_addr_t dma_addr
;
3317 struct caam_drv_private
*priv
;
3318 const size_t sh_desc_enc_offset
= offsetof(struct caam_ctx
,
3321 ctx
->jrdev
= caam_jr_alloc();
3322 if (IS_ERR(ctx
->jrdev
)) {
3323 pr_err("Job Ring Device allocation for transform failed\n");
3324 return PTR_ERR(ctx
->jrdev
);
3327 priv
= dev_get_drvdata(ctx
->jrdev
->parent
);
3328 if (priv
->era
>= 6 && uses_dkp
)
3329 ctx
->dir
= DMA_BIDIRECTIONAL
;
3331 ctx
->dir
= DMA_TO_DEVICE
;
3333 dma_addr
= dma_map_single_attrs(ctx
->jrdev
, ctx
->sh_desc_enc
,
3334 offsetof(struct caam_ctx
,
3337 ctx
->dir
, DMA_ATTR_SKIP_CPU_SYNC
);
3338 if (dma_mapping_error(ctx
->jrdev
, dma_addr
)) {
3339 dev_err(ctx
->jrdev
, "unable to map key, shared descriptors\n");
3340 caam_jr_free(ctx
->jrdev
);
3344 ctx
->sh_desc_enc_dma
= dma_addr
;
3345 ctx
->sh_desc_dec_dma
= dma_addr
+ offsetof(struct caam_ctx
,
3348 ctx
->key_dma
= dma_addr
+ offsetof(struct caam_ctx
, key
) -
3351 /* copy descriptor header template value */
3352 ctx
->cdata
.algtype
= OP_TYPE_CLASS1_ALG
| caam
->class1_alg_type
;
3353 ctx
->adata
.algtype
= OP_TYPE_CLASS2_ALG
| caam
->class2_alg_type
;
3358 static int caam_cra_init(struct crypto_skcipher
*tfm
)
3360 struct skcipher_alg
*alg
= crypto_skcipher_alg(tfm
);
3361 struct caam_skcipher_alg
*caam_alg
=
3362 container_of(alg
, typeof(*caam_alg
), skcipher
);
3363 struct caam_ctx
*ctx
= crypto_skcipher_ctx(tfm
);
3365 crypto_skcipher_set_reqsize(tfm
, sizeof(struct caam_skcipher_req_ctx
));
3367 ctx
->enginectx
.op
.do_one_request
= skcipher_do_one_req
;
3369 return caam_init_common(crypto_skcipher_ctx(tfm
), &caam_alg
->caam
,
3373 static int caam_aead_init(struct crypto_aead
*tfm
)
3375 struct aead_alg
*alg
= crypto_aead_alg(tfm
);
3376 struct caam_aead_alg
*caam_alg
=
3377 container_of(alg
, struct caam_aead_alg
, aead
);
3378 struct caam_ctx
*ctx
= crypto_aead_ctx(tfm
);
3380 crypto_aead_set_reqsize(tfm
, sizeof(struct caam_aead_req_ctx
));
3382 ctx
->enginectx
.op
.do_one_request
= aead_do_one_req
;
3384 return caam_init_common(ctx
, &caam_alg
->caam
, !caam_alg
->caam
.nodkp
);
3387 static void caam_exit_common(struct caam_ctx
*ctx
)
3389 dma_unmap_single_attrs(ctx
->jrdev
, ctx
->sh_desc_enc_dma
,
3390 offsetof(struct caam_ctx
, sh_desc_enc_dma
) -
3391 offsetof(struct caam_ctx
, sh_desc_enc
),
3392 ctx
->dir
, DMA_ATTR_SKIP_CPU_SYNC
);
3393 caam_jr_free(ctx
->jrdev
);
3396 static void caam_cra_exit(struct crypto_skcipher
*tfm
)
3398 caam_exit_common(crypto_skcipher_ctx(tfm
));
3401 static void caam_aead_exit(struct crypto_aead
*tfm
)
3403 caam_exit_common(crypto_aead_ctx(tfm
));
3406 void caam_algapi_exit(void)
3410 for (i
= 0; i
< ARRAY_SIZE(driver_aeads
); i
++) {
3411 struct caam_aead_alg
*t_alg
= driver_aeads
+ i
;
3413 if (t_alg
->registered
)
3414 crypto_unregister_aead(&t_alg
->aead
);
3417 for (i
= 0; i
< ARRAY_SIZE(driver_algs
); i
++) {
3418 struct caam_skcipher_alg
*t_alg
= driver_algs
+ i
;
3420 if (t_alg
->registered
)
3421 crypto_unregister_skcipher(&t_alg
->skcipher
);
3425 static void caam_skcipher_alg_init(struct caam_skcipher_alg
*t_alg
)
3427 struct skcipher_alg
*alg
= &t_alg
->skcipher
;
3429 alg
->base
.cra_module
= THIS_MODULE
;
3430 alg
->base
.cra_priority
= CAAM_CRA_PRIORITY
;
3431 alg
->base
.cra_ctxsize
= sizeof(struct caam_ctx
);
3432 alg
->base
.cra_flags
= CRYPTO_ALG_ASYNC
| CRYPTO_ALG_KERN_DRIVER_ONLY
;
3434 alg
->init
= caam_cra_init
;
3435 alg
->exit
= caam_cra_exit
;
3438 static void caam_aead_alg_init(struct caam_aead_alg
*t_alg
)
3440 struct aead_alg
*alg
= &t_alg
->aead
;
3442 alg
->base
.cra_module
= THIS_MODULE
;
3443 alg
->base
.cra_priority
= CAAM_CRA_PRIORITY
;
3444 alg
->base
.cra_ctxsize
= sizeof(struct caam_ctx
);
3445 alg
->base
.cra_flags
= CRYPTO_ALG_ASYNC
| CRYPTO_ALG_KERN_DRIVER_ONLY
;
3447 alg
->init
= caam_aead_init
;
3448 alg
->exit
= caam_aead_exit
;
3451 int caam_algapi_init(struct device
*ctrldev
)
3453 struct caam_drv_private
*priv
= dev_get_drvdata(ctrldev
);
3455 u32 aes_vid
, aes_inst
, des_inst
, md_vid
, md_inst
, ccha_inst
, ptha_inst
;
3457 unsigned int md_limit
= SHA512_DIGEST_SIZE
;
3458 bool registered
= false, gcm_support
;
3461 * Register crypto algorithms the device supports.
3462 * First, detect presence and attributes of DES, AES, and MD blocks.
3464 if (priv
->era
< 10) {
3465 u32 cha_vid
, cha_inst
, aes_rn
;
3467 cha_vid
= rd_reg32(&priv
->ctrl
->perfmon
.cha_id_ls
);
3468 aes_vid
= cha_vid
& CHA_ID_LS_AES_MASK
;
3469 md_vid
= (cha_vid
& CHA_ID_LS_MD_MASK
) >> CHA_ID_LS_MD_SHIFT
;
3471 cha_inst
= rd_reg32(&priv
->ctrl
->perfmon
.cha_num_ls
);
3472 des_inst
= (cha_inst
& CHA_ID_LS_DES_MASK
) >>
3473 CHA_ID_LS_DES_SHIFT
;
3474 aes_inst
= cha_inst
& CHA_ID_LS_AES_MASK
;
3475 md_inst
= (cha_inst
& CHA_ID_LS_MD_MASK
) >> CHA_ID_LS_MD_SHIFT
;
3476 arc4_inst
= (cha_inst
& CHA_ID_LS_ARC4_MASK
) >>
3477 CHA_ID_LS_ARC4_SHIFT
;
3481 aes_rn
= rd_reg32(&priv
->ctrl
->perfmon
.cha_rev_ls
) &
3483 gcm_support
= !(aes_vid
== CHA_VER_VID_AES_LP
&& aes_rn
< 8);
3487 aesa
= rd_reg32(&priv
->ctrl
->vreg
.aesa
);
3488 mdha
= rd_reg32(&priv
->ctrl
->vreg
.mdha
);
3490 aes_vid
= (aesa
& CHA_VER_VID_MASK
) >> CHA_VER_VID_SHIFT
;
3491 md_vid
= (mdha
& CHA_VER_VID_MASK
) >> CHA_VER_VID_SHIFT
;
3493 des_inst
= rd_reg32(&priv
->ctrl
->vreg
.desa
) & CHA_VER_NUM_MASK
;
3494 aes_inst
= aesa
& CHA_VER_NUM_MASK
;
3495 md_inst
= mdha
& CHA_VER_NUM_MASK
;
3496 ccha_inst
= rd_reg32(&priv
->ctrl
->vreg
.ccha
) & CHA_VER_NUM_MASK
;
3497 ptha_inst
= rd_reg32(&priv
->ctrl
->vreg
.ptha
) & CHA_VER_NUM_MASK
;
3498 arc4_inst
= rd_reg32(&priv
->ctrl
->vreg
.afha
) & CHA_VER_NUM_MASK
;
3500 gcm_support
= aesa
& CHA_VER_MISC_AES_GCM
;
3503 /* If MD is present, limit digest size based on LP256 */
3504 if (md_inst
&& md_vid
== CHA_VER_VID_MD_LP256
)
3505 md_limit
= SHA256_DIGEST_SIZE
;
3507 for (i
= 0; i
< ARRAY_SIZE(driver_algs
); i
++) {
3508 struct caam_skcipher_alg
*t_alg
= driver_algs
+ i
;
3509 u32 alg_sel
= t_alg
->caam
.class1_alg_type
& OP_ALG_ALGSEL_MASK
;
3511 /* Skip DES algorithms if not supported by device */
3513 ((alg_sel
== OP_ALG_ALGSEL_3DES
) ||
3514 (alg_sel
== OP_ALG_ALGSEL_DES
)))
3517 /* Skip AES algorithms if not supported by device */
3518 if (!aes_inst
&& (alg_sel
== OP_ALG_ALGSEL_AES
))
3521 /* Skip ARC4 algorithms if not supported by device */
3522 if (!arc4_inst
&& alg_sel
== OP_ALG_ALGSEL_ARC4
)
3526 * Check support for AES modes not available
3529 if (aes_vid
== CHA_VER_VID_AES_LP
&&
3530 (t_alg
->caam
.class1_alg_type
& OP_ALG_AAI_MASK
) ==
3534 caam_skcipher_alg_init(t_alg
);
3536 err
= crypto_register_skcipher(&t_alg
->skcipher
);
3538 pr_warn("%s alg registration failed\n",
3539 t_alg
->skcipher
.base
.cra_driver_name
);
3543 t_alg
->registered
= true;
3547 for (i
= 0; i
< ARRAY_SIZE(driver_aeads
); i
++) {
3548 struct caam_aead_alg
*t_alg
= driver_aeads
+ i
;
3549 u32 c1_alg_sel
= t_alg
->caam
.class1_alg_type
&
3551 u32 c2_alg_sel
= t_alg
->caam
.class2_alg_type
&
3553 u32 alg_aai
= t_alg
->caam
.class1_alg_type
& OP_ALG_AAI_MASK
;
3555 /* Skip DES algorithms if not supported by device */
3557 ((c1_alg_sel
== OP_ALG_ALGSEL_3DES
) ||
3558 (c1_alg_sel
== OP_ALG_ALGSEL_DES
)))
3561 /* Skip AES algorithms if not supported by device */
3562 if (!aes_inst
&& (c1_alg_sel
== OP_ALG_ALGSEL_AES
))
3565 /* Skip CHACHA20 algorithms if not supported by device */
3566 if (c1_alg_sel
== OP_ALG_ALGSEL_CHACHA20
&& !ccha_inst
)
3569 /* Skip POLY1305 algorithms if not supported by device */
3570 if (c2_alg_sel
== OP_ALG_ALGSEL_POLY1305
&& !ptha_inst
)
3573 /* Skip GCM algorithms if not supported by device */
3574 if (c1_alg_sel
== OP_ALG_ALGSEL_AES
&&
3575 alg_aai
== OP_ALG_AAI_GCM
&& !gcm_support
)
3579 * Skip algorithms requiring message digests
3580 * if MD or MD size is not supported by device.
3582 if (is_mdha(c2_alg_sel
) &&
3583 (!md_inst
|| t_alg
->aead
.maxauthsize
> md_limit
))
3586 caam_aead_alg_init(t_alg
);
3588 err
= crypto_register_aead(&t_alg
->aead
);
3590 pr_warn("%s alg registration failed\n",
3591 t_alg
->aead
.base
.cra_driver_name
);
3595 t_alg
->registered
= true;
3600 pr_info("caam algorithms registered in /proc/crypto\n");