1 // SPDX-License-Identifier: GPL-2.0-only
3 * GCM: Galois/Counter Mode.
5 * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi>
8 #include <crypto/gf128mul.h>
9 #include <crypto/internal/aead.h>
10 #include <crypto/internal/skcipher.h>
11 #include <crypto/internal/hash.h>
12 #include <crypto/null.h>
13 #include <crypto/scatterwalk.h>
14 #include <crypto/gcm.h>
15 #include <crypto/hash.h>
16 #include <linux/err.h>
17 #include <linux/init.h>
18 #include <linux/kernel.h>
19 #include <linux/module.h>
20 #include <linux/slab.h>
22 struct gcm_instance_ctx
{
23 struct crypto_skcipher_spawn ctr
;
24 struct crypto_ahash_spawn ghash
;
27 struct crypto_gcm_ctx
{
28 struct crypto_skcipher
*ctr
;
29 struct crypto_ahash
*ghash
;
32 struct crypto_rfc4106_ctx
{
33 struct crypto_aead
*child
;
37 struct crypto_rfc4106_req_ctx
{
38 struct scatterlist src
[3];
39 struct scatterlist dst
[3];
40 struct aead_request subreq
;
43 struct crypto_rfc4543_instance_ctx
{
44 struct crypto_aead_spawn aead
;
47 struct crypto_rfc4543_ctx
{
48 struct crypto_aead
*child
;
49 struct crypto_sync_skcipher
*null
;
53 struct crypto_rfc4543_req_ctx
{
54 struct aead_request subreq
;
57 struct crypto_gcm_ghash_ctx
{
58 unsigned int cryptlen
;
59 struct scatterlist
*src
;
60 int (*complete
)(struct aead_request
*req
, u32 flags
);
63 struct crypto_gcm_req_priv_ctx
{
67 struct scatterlist src
[3];
68 struct scatterlist dst
[3];
69 struct scatterlist sg
;
70 struct crypto_gcm_ghash_ctx ghash_ctx
;
72 struct ahash_request ahreq
;
73 struct skcipher_request skreq
;
79 struct scatterlist sg
;
82 static int crypto_rfc4543_copy_src_to_dst(struct aead_request
*req
, bool enc
);
84 static inline struct crypto_gcm_req_priv_ctx
*crypto_gcm_reqctx(
85 struct aead_request
*req
)
87 unsigned long align
= crypto_aead_alignmask(crypto_aead_reqtfm(req
));
89 return (void *)PTR_ALIGN((u8
*)aead_request_ctx(req
), align
+ 1);
92 static int crypto_gcm_setkey(struct crypto_aead
*aead
, const u8
*key
,
95 struct crypto_gcm_ctx
*ctx
= crypto_aead_ctx(aead
);
96 struct crypto_ahash
*ghash
= ctx
->ghash
;
97 struct crypto_skcipher
*ctr
= ctx
->ctr
;
102 struct crypto_wait wait
;
104 struct scatterlist sg
[1];
105 struct skcipher_request req
;
109 crypto_skcipher_clear_flags(ctr
, CRYPTO_TFM_REQ_MASK
);
110 crypto_skcipher_set_flags(ctr
, crypto_aead_get_flags(aead
) &
111 CRYPTO_TFM_REQ_MASK
);
112 err
= crypto_skcipher_setkey(ctr
, key
, keylen
);
116 data
= kzalloc(sizeof(*data
) + crypto_skcipher_reqsize(ctr
),
121 crypto_init_wait(&data
->wait
);
122 sg_init_one(data
->sg
, &data
->hash
, sizeof(data
->hash
));
123 skcipher_request_set_tfm(&data
->req
, ctr
);
124 skcipher_request_set_callback(&data
->req
, CRYPTO_TFM_REQ_MAY_SLEEP
|
125 CRYPTO_TFM_REQ_MAY_BACKLOG
,
128 skcipher_request_set_crypt(&data
->req
, data
->sg
, data
->sg
,
129 sizeof(data
->hash
), data
->iv
);
131 err
= crypto_wait_req(crypto_skcipher_encrypt(&data
->req
),
137 crypto_ahash_clear_flags(ghash
, CRYPTO_TFM_REQ_MASK
);
138 crypto_ahash_set_flags(ghash
, crypto_aead_get_flags(aead
) &
139 CRYPTO_TFM_REQ_MASK
);
140 err
= crypto_ahash_setkey(ghash
, (u8
*)&data
->hash
, sizeof(be128
));
142 kfree_sensitive(data
);
146 static int crypto_gcm_setauthsize(struct crypto_aead
*tfm
,
147 unsigned int authsize
)
149 return crypto_gcm_check_authsize(authsize
);
152 static void crypto_gcm_init_common(struct aead_request
*req
)
154 struct crypto_gcm_req_priv_ctx
*pctx
= crypto_gcm_reqctx(req
);
155 __be32 counter
= cpu_to_be32(1);
156 struct scatterlist
*sg
;
158 memset(pctx
->auth_tag
, 0, sizeof(pctx
->auth_tag
));
159 memcpy(pctx
->iv
, req
->iv
, GCM_AES_IV_SIZE
);
160 memcpy(pctx
->iv
+ GCM_AES_IV_SIZE
, &counter
, 4);
162 sg_init_table(pctx
->src
, 3);
163 sg_set_buf(pctx
->src
, pctx
->auth_tag
, sizeof(pctx
->auth_tag
));
164 sg
= scatterwalk_ffwd(pctx
->src
+ 1, req
->src
, req
->assoclen
);
165 if (sg
!= pctx
->src
+ 1)
166 sg_chain(pctx
->src
, 2, sg
);
168 if (req
->src
!= req
->dst
) {
169 sg_init_table(pctx
->dst
, 3);
170 sg_set_buf(pctx
->dst
, pctx
->auth_tag
, sizeof(pctx
->auth_tag
));
171 sg
= scatterwalk_ffwd(pctx
->dst
+ 1, req
->dst
, req
->assoclen
);
172 if (sg
!= pctx
->dst
+ 1)
173 sg_chain(pctx
->dst
, 2, sg
);
177 static void crypto_gcm_init_crypt(struct aead_request
*req
,
178 unsigned int cryptlen
)
180 struct crypto_aead
*aead
= crypto_aead_reqtfm(req
);
181 struct crypto_gcm_ctx
*ctx
= crypto_aead_ctx(aead
);
182 struct crypto_gcm_req_priv_ctx
*pctx
= crypto_gcm_reqctx(req
);
183 struct skcipher_request
*skreq
= &pctx
->u
.skreq
;
184 struct scatterlist
*dst
;
186 dst
= req
->src
== req
->dst
? pctx
->src
: pctx
->dst
;
188 skcipher_request_set_tfm(skreq
, ctx
->ctr
);
189 skcipher_request_set_crypt(skreq
, pctx
->src
, dst
,
190 cryptlen
+ sizeof(pctx
->auth_tag
),
194 static inline unsigned int gcm_remain(unsigned int len
)
197 return len
? 16 - len
: 0;
200 static void gcm_hash_len_done(void *data
, int err
);
202 static int gcm_hash_update(struct aead_request
*req
,
203 crypto_completion_t
compl,
204 struct scatterlist
*src
,
205 unsigned int len
, u32 flags
)
207 struct crypto_gcm_req_priv_ctx
*pctx
= crypto_gcm_reqctx(req
);
208 struct ahash_request
*ahreq
= &pctx
->u
.ahreq
;
210 ahash_request_set_callback(ahreq
, flags
, compl, req
);
211 ahash_request_set_crypt(ahreq
, src
, NULL
, len
);
213 return crypto_ahash_update(ahreq
);
216 static int gcm_hash_remain(struct aead_request
*req
,
218 crypto_completion_t
compl, u32 flags
)
220 return gcm_hash_update(req
, compl, &gcm_zeroes
->sg
, remain
, flags
);
223 static int gcm_hash_len(struct aead_request
*req
, u32 flags
)
225 struct crypto_gcm_req_priv_ctx
*pctx
= crypto_gcm_reqctx(req
);
226 struct ahash_request
*ahreq
= &pctx
->u
.ahreq
;
227 struct crypto_gcm_ghash_ctx
*gctx
= &pctx
->ghash_ctx
;
230 lengths
.a
= cpu_to_be64(req
->assoclen
* 8);
231 lengths
.b
= cpu_to_be64(gctx
->cryptlen
* 8);
232 memcpy(pctx
->iauth_tag
, &lengths
, 16);
233 sg_init_one(&pctx
->sg
, pctx
->iauth_tag
, 16);
234 ahash_request_set_callback(ahreq
, flags
, gcm_hash_len_done
, req
);
235 ahash_request_set_crypt(ahreq
, &pctx
->sg
,
236 pctx
->iauth_tag
, sizeof(lengths
));
238 return crypto_ahash_finup(ahreq
);
241 static int gcm_hash_len_continue(struct aead_request
*req
, u32 flags
)
243 struct crypto_gcm_req_priv_ctx
*pctx
= crypto_gcm_reqctx(req
);
244 struct crypto_gcm_ghash_ctx
*gctx
= &pctx
->ghash_ctx
;
246 return gctx
->complete(req
, flags
);
249 static void gcm_hash_len_done(void *data
, int err
)
251 struct aead_request
*req
= data
;
256 err
= gcm_hash_len_continue(req
, 0);
257 if (err
== -EINPROGRESS
)
261 aead_request_complete(req
, err
);
264 static int gcm_hash_crypt_remain_continue(struct aead_request
*req
, u32 flags
)
266 return gcm_hash_len(req
, flags
) ?:
267 gcm_hash_len_continue(req
, flags
);
270 static void gcm_hash_crypt_remain_done(void *data
, int err
)
272 struct aead_request
*req
= data
;
277 err
= gcm_hash_crypt_remain_continue(req
, 0);
278 if (err
== -EINPROGRESS
)
282 aead_request_complete(req
, err
);
285 static int gcm_hash_crypt_continue(struct aead_request
*req
, u32 flags
)
287 struct crypto_gcm_req_priv_ctx
*pctx
= crypto_gcm_reqctx(req
);
288 struct crypto_gcm_ghash_ctx
*gctx
= &pctx
->ghash_ctx
;
291 remain
= gcm_remain(gctx
->cryptlen
);
293 return gcm_hash_remain(req
, remain
,
294 gcm_hash_crypt_remain_done
, flags
) ?:
295 gcm_hash_crypt_remain_continue(req
, flags
);
297 return gcm_hash_crypt_remain_continue(req
, flags
);
300 static void gcm_hash_crypt_done(void *data
, int err
)
302 struct aead_request
*req
= data
;
307 err
= gcm_hash_crypt_continue(req
, 0);
308 if (err
== -EINPROGRESS
)
312 aead_request_complete(req
, err
);
315 static int gcm_hash_assoc_remain_continue(struct aead_request
*req
, u32 flags
)
317 struct crypto_gcm_req_priv_ctx
*pctx
= crypto_gcm_reqctx(req
);
318 struct crypto_gcm_ghash_ctx
*gctx
= &pctx
->ghash_ctx
;
321 return gcm_hash_update(req
, gcm_hash_crypt_done
,
322 gctx
->src
, gctx
->cryptlen
, flags
) ?:
323 gcm_hash_crypt_continue(req
, flags
);
325 return gcm_hash_crypt_remain_continue(req
, flags
);
328 static void gcm_hash_assoc_remain_done(void *data
, int err
)
330 struct aead_request
*req
= data
;
335 err
= gcm_hash_assoc_remain_continue(req
, 0);
336 if (err
== -EINPROGRESS
)
340 aead_request_complete(req
, err
);
343 static int gcm_hash_assoc_continue(struct aead_request
*req
, u32 flags
)
347 remain
= gcm_remain(req
->assoclen
);
349 return gcm_hash_remain(req
, remain
,
350 gcm_hash_assoc_remain_done
, flags
) ?:
351 gcm_hash_assoc_remain_continue(req
, flags
);
353 return gcm_hash_assoc_remain_continue(req
, flags
);
356 static void gcm_hash_assoc_done(void *data
, int err
)
358 struct aead_request
*req
= data
;
363 err
= gcm_hash_assoc_continue(req
, 0);
364 if (err
== -EINPROGRESS
)
368 aead_request_complete(req
, err
);
371 static int gcm_hash_init_continue(struct aead_request
*req
, u32 flags
)
374 return gcm_hash_update(req
, gcm_hash_assoc_done
,
375 req
->src
, req
->assoclen
, flags
) ?:
376 gcm_hash_assoc_continue(req
, flags
);
378 return gcm_hash_assoc_remain_continue(req
, flags
);
381 static void gcm_hash_init_done(void *data
, int err
)
383 struct aead_request
*req
= data
;
388 err
= gcm_hash_init_continue(req
, 0);
389 if (err
== -EINPROGRESS
)
393 aead_request_complete(req
, err
);
396 static int gcm_hash(struct aead_request
*req
, u32 flags
)
398 struct crypto_gcm_req_priv_ctx
*pctx
= crypto_gcm_reqctx(req
);
399 struct ahash_request
*ahreq
= &pctx
->u
.ahreq
;
400 struct crypto_gcm_ctx
*ctx
= crypto_aead_ctx(crypto_aead_reqtfm(req
));
402 ahash_request_set_tfm(ahreq
, ctx
->ghash
);
404 ahash_request_set_callback(ahreq
, flags
, gcm_hash_init_done
, req
);
405 return crypto_ahash_init(ahreq
) ?:
406 gcm_hash_init_continue(req
, flags
);
409 static int gcm_enc_copy_hash(struct aead_request
*req
, u32 flags
)
411 struct crypto_gcm_req_priv_ctx
*pctx
= crypto_gcm_reqctx(req
);
412 struct crypto_aead
*aead
= crypto_aead_reqtfm(req
);
413 u8
*auth_tag
= pctx
->auth_tag
;
415 crypto_xor(auth_tag
, pctx
->iauth_tag
, 16);
416 scatterwalk_map_and_copy(auth_tag
, req
->dst
,
417 req
->assoclen
+ req
->cryptlen
,
418 crypto_aead_authsize(aead
), 1);
422 static int gcm_encrypt_continue(struct aead_request
*req
, u32 flags
)
424 struct crypto_gcm_req_priv_ctx
*pctx
= crypto_gcm_reqctx(req
);
425 struct crypto_gcm_ghash_ctx
*gctx
= &pctx
->ghash_ctx
;
427 gctx
->src
= sg_next(req
->src
== req
->dst
? pctx
->src
: pctx
->dst
);
428 gctx
->cryptlen
= req
->cryptlen
;
429 gctx
->complete
= gcm_enc_copy_hash
;
431 return gcm_hash(req
, flags
);
434 static void gcm_encrypt_done(void *data
, int err
)
436 struct aead_request
*req
= data
;
441 err
= gcm_encrypt_continue(req
, 0);
442 if (err
== -EINPROGRESS
)
446 aead_request_complete(req
, err
);
449 static int crypto_gcm_encrypt(struct aead_request
*req
)
451 struct crypto_gcm_req_priv_ctx
*pctx
= crypto_gcm_reqctx(req
);
452 struct skcipher_request
*skreq
= &pctx
->u
.skreq
;
453 u32 flags
= aead_request_flags(req
);
455 crypto_gcm_init_common(req
);
456 crypto_gcm_init_crypt(req
, req
->cryptlen
);
457 skcipher_request_set_callback(skreq
, flags
, gcm_encrypt_done
, req
);
459 return crypto_skcipher_encrypt(skreq
) ?:
460 gcm_encrypt_continue(req
, flags
);
463 static int crypto_gcm_verify(struct aead_request
*req
)
465 struct crypto_gcm_req_priv_ctx
*pctx
= crypto_gcm_reqctx(req
);
466 struct crypto_aead
*aead
= crypto_aead_reqtfm(req
);
467 u8
*auth_tag
= pctx
->auth_tag
;
468 u8
*iauth_tag
= pctx
->iauth_tag
;
469 unsigned int authsize
= crypto_aead_authsize(aead
);
470 unsigned int cryptlen
= req
->cryptlen
- authsize
;
472 crypto_xor(auth_tag
, iauth_tag
, 16);
473 scatterwalk_map_and_copy(iauth_tag
, req
->src
,
474 req
->assoclen
+ cryptlen
, authsize
, 0);
475 return crypto_memneq(iauth_tag
, auth_tag
, authsize
) ? -EBADMSG
: 0;
478 static void gcm_decrypt_done(void *data
, int err
)
480 struct aead_request
*req
= data
;
483 err
= crypto_gcm_verify(req
);
485 aead_request_complete(req
, err
);
488 static int gcm_dec_hash_continue(struct aead_request
*req
, u32 flags
)
490 struct crypto_gcm_req_priv_ctx
*pctx
= crypto_gcm_reqctx(req
);
491 struct skcipher_request
*skreq
= &pctx
->u
.skreq
;
492 struct crypto_gcm_ghash_ctx
*gctx
= &pctx
->ghash_ctx
;
494 crypto_gcm_init_crypt(req
, gctx
->cryptlen
);
495 skcipher_request_set_callback(skreq
, flags
, gcm_decrypt_done
, req
);
496 return crypto_skcipher_decrypt(skreq
) ?: crypto_gcm_verify(req
);
499 static int crypto_gcm_decrypt(struct aead_request
*req
)
501 struct crypto_aead
*aead
= crypto_aead_reqtfm(req
);
502 struct crypto_gcm_req_priv_ctx
*pctx
= crypto_gcm_reqctx(req
);
503 struct crypto_gcm_ghash_ctx
*gctx
= &pctx
->ghash_ctx
;
504 unsigned int authsize
= crypto_aead_authsize(aead
);
505 unsigned int cryptlen
= req
->cryptlen
;
506 u32 flags
= aead_request_flags(req
);
508 cryptlen
-= authsize
;
510 crypto_gcm_init_common(req
);
512 gctx
->src
= sg_next(pctx
->src
);
513 gctx
->cryptlen
= cryptlen
;
514 gctx
->complete
= gcm_dec_hash_continue
;
516 return gcm_hash(req
, flags
);
519 static int crypto_gcm_init_tfm(struct crypto_aead
*tfm
)
521 struct aead_instance
*inst
= aead_alg_instance(tfm
);
522 struct gcm_instance_ctx
*ictx
= aead_instance_ctx(inst
);
523 struct crypto_gcm_ctx
*ctx
= crypto_aead_ctx(tfm
);
524 struct crypto_skcipher
*ctr
;
525 struct crypto_ahash
*ghash
;
529 ghash
= crypto_spawn_ahash(&ictx
->ghash
);
531 return PTR_ERR(ghash
);
533 ctr
= crypto_spawn_skcipher(&ictx
->ctr
);
541 align
= crypto_aead_alignmask(tfm
);
542 align
&= ~(crypto_tfm_ctx_alignment() - 1);
543 crypto_aead_set_reqsize(tfm
,
544 align
+ offsetof(struct crypto_gcm_req_priv_ctx
, u
) +
545 max(sizeof(struct skcipher_request
) +
546 crypto_skcipher_reqsize(ctr
),
547 sizeof(struct ahash_request
) +
548 crypto_ahash_reqsize(ghash
)));
553 crypto_free_ahash(ghash
);
557 static void crypto_gcm_exit_tfm(struct crypto_aead
*tfm
)
559 struct crypto_gcm_ctx
*ctx
= crypto_aead_ctx(tfm
);
561 crypto_free_ahash(ctx
->ghash
);
562 crypto_free_skcipher(ctx
->ctr
);
565 static void crypto_gcm_free(struct aead_instance
*inst
)
567 struct gcm_instance_ctx
*ctx
= aead_instance_ctx(inst
);
569 crypto_drop_skcipher(&ctx
->ctr
);
570 crypto_drop_ahash(&ctx
->ghash
);
574 static int crypto_gcm_create_common(struct crypto_template
*tmpl
,
576 const char *ctr_name
,
577 const char *ghash_name
)
580 struct aead_instance
*inst
;
581 struct gcm_instance_ctx
*ctx
;
582 struct skcipher_alg
*ctr
;
583 struct hash_alg_common
*ghash
;
586 err
= crypto_check_attr_type(tb
, CRYPTO_ALG_TYPE_AEAD
, &mask
);
590 inst
= kzalloc(sizeof(*inst
) + sizeof(*ctx
), GFP_KERNEL
);
593 ctx
= aead_instance_ctx(inst
);
595 err
= crypto_grab_ahash(&ctx
->ghash
, aead_crypto_instance(inst
),
596 ghash_name
, 0, mask
);
599 ghash
= crypto_spawn_ahash_alg(&ctx
->ghash
);
602 if (strcmp(ghash
->base
.cra_name
, "ghash") != 0 ||
603 ghash
->digestsize
!= 16)
606 err
= crypto_grab_skcipher(&ctx
->ctr
, aead_crypto_instance(inst
),
610 ctr
= crypto_spawn_skcipher_alg(&ctx
->ctr
);
612 /* The skcipher algorithm must be CTR mode, using 16-byte blocks. */
614 if (strncmp(ctr
->base
.cra_name
, "ctr(", 4) != 0 ||
615 crypto_skcipher_alg_ivsize(ctr
) != 16 ||
616 ctr
->base
.cra_blocksize
!= 1)
620 if (snprintf(inst
->alg
.base
.cra_name
, CRYPTO_MAX_ALG_NAME
,
621 "gcm(%s", ctr
->base
.cra_name
+ 4) >= CRYPTO_MAX_ALG_NAME
)
624 if (snprintf(inst
->alg
.base
.cra_driver_name
, CRYPTO_MAX_ALG_NAME
,
625 "gcm_base(%s,%s)", ctr
->base
.cra_driver_name
,
626 ghash
->base
.cra_driver_name
) >=
630 inst
->alg
.base
.cra_priority
= (ghash
->base
.cra_priority
+
631 ctr
->base
.cra_priority
) / 2;
632 inst
->alg
.base
.cra_blocksize
= 1;
633 inst
->alg
.base
.cra_alignmask
= ghash
->base
.cra_alignmask
|
634 ctr
->base
.cra_alignmask
;
635 inst
->alg
.base
.cra_ctxsize
= sizeof(struct crypto_gcm_ctx
);
636 inst
->alg
.ivsize
= GCM_AES_IV_SIZE
;
637 inst
->alg
.chunksize
= crypto_skcipher_alg_chunksize(ctr
);
638 inst
->alg
.maxauthsize
= 16;
639 inst
->alg
.init
= crypto_gcm_init_tfm
;
640 inst
->alg
.exit
= crypto_gcm_exit_tfm
;
641 inst
->alg
.setkey
= crypto_gcm_setkey
;
642 inst
->alg
.setauthsize
= crypto_gcm_setauthsize
;
643 inst
->alg
.encrypt
= crypto_gcm_encrypt
;
644 inst
->alg
.decrypt
= crypto_gcm_decrypt
;
646 inst
->free
= crypto_gcm_free
;
648 err
= aead_register_instance(tmpl
, inst
);
651 crypto_gcm_free(inst
);
656 static int crypto_gcm_create(struct crypto_template
*tmpl
, struct rtattr
**tb
)
658 const char *cipher_name
;
659 char ctr_name
[CRYPTO_MAX_ALG_NAME
];
661 cipher_name
= crypto_attr_alg_name(tb
[1]);
662 if (IS_ERR(cipher_name
))
663 return PTR_ERR(cipher_name
);
665 if (snprintf(ctr_name
, CRYPTO_MAX_ALG_NAME
, "ctr(%s)", cipher_name
) >=
667 return -ENAMETOOLONG
;
669 return crypto_gcm_create_common(tmpl
, tb
, ctr_name
, "ghash");
672 static int crypto_gcm_base_create(struct crypto_template
*tmpl
,
675 const char *ctr_name
;
676 const char *ghash_name
;
678 ctr_name
= crypto_attr_alg_name(tb
[1]);
679 if (IS_ERR(ctr_name
))
680 return PTR_ERR(ctr_name
);
682 ghash_name
= crypto_attr_alg_name(tb
[2]);
683 if (IS_ERR(ghash_name
))
684 return PTR_ERR(ghash_name
);
686 return crypto_gcm_create_common(tmpl
, tb
, ctr_name
, ghash_name
);
689 static int crypto_rfc4106_setkey(struct crypto_aead
*parent
, const u8
*key
,
692 struct crypto_rfc4106_ctx
*ctx
= crypto_aead_ctx(parent
);
693 struct crypto_aead
*child
= ctx
->child
;
699 memcpy(ctx
->nonce
, key
+ keylen
, 4);
701 crypto_aead_clear_flags(child
, CRYPTO_TFM_REQ_MASK
);
702 crypto_aead_set_flags(child
, crypto_aead_get_flags(parent
) &
703 CRYPTO_TFM_REQ_MASK
);
704 return crypto_aead_setkey(child
, key
, keylen
);
707 static int crypto_rfc4106_setauthsize(struct crypto_aead
*parent
,
708 unsigned int authsize
)
710 struct crypto_rfc4106_ctx
*ctx
= crypto_aead_ctx(parent
);
713 err
= crypto_rfc4106_check_authsize(authsize
);
717 return crypto_aead_setauthsize(ctx
->child
, authsize
);
720 static struct aead_request
*crypto_rfc4106_crypt(struct aead_request
*req
)
722 struct crypto_rfc4106_req_ctx
*rctx
= aead_request_ctx(req
);
723 struct crypto_aead
*aead
= crypto_aead_reqtfm(req
);
724 struct crypto_rfc4106_ctx
*ctx
= crypto_aead_ctx(aead
);
725 struct aead_request
*subreq
= &rctx
->subreq
;
726 struct crypto_aead
*child
= ctx
->child
;
727 struct scatterlist
*sg
;
728 u8
*iv
= PTR_ALIGN((u8
*)(subreq
+ 1) + crypto_aead_reqsize(child
),
729 crypto_aead_alignmask(child
) + 1);
731 scatterwalk_map_and_copy(iv
+ GCM_AES_IV_SIZE
, req
->src
, 0, req
->assoclen
- 8, 0);
733 memcpy(iv
, ctx
->nonce
, 4);
734 memcpy(iv
+ 4, req
->iv
, 8);
736 sg_init_table(rctx
->src
, 3);
737 sg_set_buf(rctx
->src
, iv
+ GCM_AES_IV_SIZE
, req
->assoclen
- 8);
738 sg
= scatterwalk_ffwd(rctx
->src
+ 1, req
->src
, req
->assoclen
);
739 if (sg
!= rctx
->src
+ 1)
740 sg_chain(rctx
->src
, 2, sg
);
742 if (req
->src
!= req
->dst
) {
743 sg_init_table(rctx
->dst
, 3);
744 sg_set_buf(rctx
->dst
, iv
+ GCM_AES_IV_SIZE
, req
->assoclen
- 8);
745 sg
= scatterwalk_ffwd(rctx
->dst
+ 1, req
->dst
, req
->assoclen
);
746 if (sg
!= rctx
->dst
+ 1)
747 sg_chain(rctx
->dst
, 2, sg
);
750 aead_request_set_tfm(subreq
, child
);
751 aead_request_set_callback(subreq
, req
->base
.flags
, req
->base
.complete
,
753 aead_request_set_crypt(subreq
, rctx
->src
,
754 req
->src
== req
->dst
? rctx
->src
: rctx
->dst
,
756 aead_request_set_ad(subreq
, req
->assoclen
- 8);
761 static int crypto_rfc4106_encrypt(struct aead_request
*req
)
765 err
= crypto_ipsec_check_assoclen(req
->assoclen
);
769 req
= crypto_rfc4106_crypt(req
);
771 return crypto_aead_encrypt(req
);
774 static int crypto_rfc4106_decrypt(struct aead_request
*req
)
778 err
= crypto_ipsec_check_assoclen(req
->assoclen
);
782 req
= crypto_rfc4106_crypt(req
);
784 return crypto_aead_decrypt(req
);
787 static int crypto_rfc4106_init_tfm(struct crypto_aead
*tfm
)
789 struct aead_instance
*inst
= aead_alg_instance(tfm
);
790 struct crypto_aead_spawn
*spawn
= aead_instance_ctx(inst
);
791 struct crypto_rfc4106_ctx
*ctx
= crypto_aead_ctx(tfm
);
792 struct crypto_aead
*aead
;
795 aead
= crypto_spawn_aead(spawn
);
797 return PTR_ERR(aead
);
801 align
= crypto_aead_alignmask(aead
);
802 align
&= ~(crypto_tfm_ctx_alignment() - 1);
803 crypto_aead_set_reqsize(
805 sizeof(struct crypto_rfc4106_req_ctx
) +
806 ALIGN(crypto_aead_reqsize(aead
), crypto_tfm_ctx_alignment()) +
812 static void crypto_rfc4106_exit_tfm(struct crypto_aead
*tfm
)
814 struct crypto_rfc4106_ctx
*ctx
= crypto_aead_ctx(tfm
);
816 crypto_free_aead(ctx
->child
);
819 static void crypto_rfc4106_free(struct aead_instance
*inst
)
821 crypto_drop_aead(aead_instance_ctx(inst
));
825 static int crypto_rfc4106_create(struct crypto_template
*tmpl
,
829 struct aead_instance
*inst
;
830 struct crypto_aead_spawn
*spawn
;
831 struct aead_alg
*alg
;
834 err
= crypto_check_attr_type(tb
, CRYPTO_ALG_TYPE_AEAD
, &mask
);
838 inst
= kzalloc(sizeof(*inst
) + sizeof(*spawn
), GFP_KERNEL
);
842 spawn
= aead_instance_ctx(inst
);
843 err
= crypto_grab_aead(spawn
, aead_crypto_instance(inst
),
844 crypto_attr_alg_name(tb
[1]), 0, mask
);
848 alg
= crypto_spawn_aead_alg(spawn
);
852 /* Underlying IV size must be 12. */
853 if (crypto_aead_alg_ivsize(alg
) != GCM_AES_IV_SIZE
)
856 /* Not a stream cipher? */
857 if (alg
->base
.cra_blocksize
!= 1)
861 if (snprintf(inst
->alg
.base
.cra_name
, CRYPTO_MAX_ALG_NAME
,
862 "rfc4106(%s)", alg
->base
.cra_name
) >=
863 CRYPTO_MAX_ALG_NAME
||
864 snprintf(inst
->alg
.base
.cra_driver_name
, CRYPTO_MAX_ALG_NAME
,
865 "rfc4106(%s)", alg
->base
.cra_driver_name
) >=
869 inst
->alg
.base
.cra_priority
= alg
->base
.cra_priority
;
870 inst
->alg
.base
.cra_blocksize
= 1;
871 inst
->alg
.base
.cra_alignmask
= alg
->base
.cra_alignmask
;
873 inst
->alg
.base
.cra_ctxsize
= sizeof(struct crypto_rfc4106_ctx
);
875 inst
->alg
.ivsize
= GCM_RFC4106_IV_SIZE
;
876 inst
->alg
.chunksize
= crypto_aead_alg_chunksize(alg
);
877 inst
->alg
.maxauthsize
= crypto_aead_alg_maxauthsize(alg
);
879 inst
->alg
.init
= crypto_rfc4106_init_tfm
;
880 inst
->alg
.exit
= crypto_rfc4106_exit_tfm
;
882 inst
->alg
.setkey
= crypto_rfc4106_setkey
;
883 inst
->alg
.setauthsize
= crypto_rfc4106_setauthsize
;
884 inst
->alg
.encrypt
= crypto_rfc4106_encrypt
;
885 inst
->alg
.decrypt
= crypto_rfc4106_decrypt
;
887 inst
->free
= crypto_rfc4106_free
;
889 err
= aead_register_instance(tmpl
, inst
);
892 crypto_rfc4106_free(inst
);
897 static int crypto_rfc4543_setkey(struct crypto_aead
*parent
, const u8
*key
,
900 struct crypto_rfc4543_ctx
*ctx
= crypto_aead_ctx(parent
);
901 struct crypto_aead
*child
= ctx
->child
;
907 memcpy(ctx
->nonce
, key
+ keylen
, 4);
909 crypto_aead_clear_flags(child
, CRYPTO_TFM_REQ_MASK
);
910 crypto_aead_set_flags(child
, crypto_aead_get_flags(parent
) &
911 CRYPTO_TFM_REQ_MASK
);
912 return crypto_aead_setkey(child
, key
, keylen
);
915 static int crypto_rfc4543_setauthsize(struct crypto_aead
*parent
,
916 unsigned int authsize
)
918 struct crypto_rfc4543_ctx
*ctx
= crypto_aead_ctx(parent
);
923 return crypto_aead_setauthsize(ctx
->child
, authsize
);
926 static int crypto_rfc4543_crypt(struct aead_request
*req
, bool enc
)
928 struct crypto_aead
*aead
= crypto_aead_reqtfm(req
);
929 struct crypto_rfc4543_ctx
*ctx
= crypto_aead_ctx(aead
);
930 struct crypto_rfc4543_req_ctx
*rctx
= aead_request_ctx(req
);
931 struct aead_request
*subreq
= &rctx
->subreq
;
932 unsigned int authsize
= crypto_aead_authsize(aead
);
933 u8
*iv
= PTR_ALIGN((u8
*)(rctx
+ 1) + crypto_aead_reqsize(ctx
->child
),
934 crypto_aead_alignmask(ctx
->child
) + 1);
937 if (req
->src
!= req
->dst
) {
938 err
= crypto_rfc4543_copy_src_to_dst(req
, enc
);
943 memcpy(iv
, ctx
->nonce
, 4);
944 memcpy(iv
+ 4, req
->iv
, 8);
946 aead_request_set_tfm(subreq
, ctx
->child
);
947 aead_request_set_callback(subreq
, req
->base
.flags
,
948 req
->base
.complete
, req
->base
.data
);
949 aead_request_set_crypt(subreq
, req
->src
, req
->dst
,
950 enc
? 0 : authsize
, iv
);
951 aead_request_set_ad(subreq
, req
->assoclen
+ req
->cryptlen
-
954 return enc
? crypto_aead_encrypt(subreq
) : crypto_aead_decrypt(subreq
);
957 static int crypto_rfc4543_copy_src_to_dst(struct aead_request
*req
, bool enc
)
959 struct crypto_aead
*aead
= crypto_aead_reqtfm(req
);
960 struct crypto_rfc4543_ctx
*ctx
= crypto_aead_ctx(aead
);
961 unsigned int authsize
= crypto_aead_authsize(aead
);
962 unsigned int nbytes
= req
->assoclen
+ req
->cryptlen
-
963 (enc
? 0 : authsize
);
964 SYNC_SKCIPHER_REQUEST_ON_STACK(nreq
, ctx
->null
);
966 skcipher_request_set_sync_tfm(nreq
, ctx
->null
);
967 skcipher_request_set_callback(nreq
, req
->base
.flags
, NULL
, NULL
);
968 skcipher_request_set_crypt(nreq
, req
->src
, req
->dst
, nbytes
, NULL
);
970 return crypto_skcipher_encrypt(nreq
);
973 static int crypto_rfc4543_encrypt(struct aead_request
*req
)
975 return crypto_ipsec_check_assoclen(req
->assoclen
) ?:
976 crypto_rfc4543_crypt(req
, true);
979 static int crypto_rfc4543_decrypt(struct aead_request
*req
)
981 return crypto_ipsec_check_assoclen(req
->assoclen
) ?:
982 crypto_rfc4543_crypt(req
, false);
985 static int crypto_rfc4543_init_tfm(struct crypto_aead
*tfm
)
987 struct aead_instance
*inst
= aead_alg_instance(tfm
);
988 struct crypto_rfc4543_instance_ctx
*ictx
= aead_instance_ctx(inst
);
989 struct crypto_aead_spawn
*spawn
= &ictx
->aead
;
990 struct crypto_rfc4543_ctx
*ctx
= crypto_aead_ctx(tfm
);
991 struct crypto_aead
*aead
;
992 struct crypto_sync_skcipher
*null
;
996 aead
= crypto_spawn_aead(spawn
);
998 return PTR_ERR(aead
);
1000 null
= crypto_get_default_null_skcipher();
1001 err
= PTR_ERR(null
);
1008 align
= crypto_aead_alignmask(aead
);
1009 align
&= ~(crypto_tfm_ctx_alignment() - 1);
1010 crypto_aead_set_reqsize(
1012 sizeof(struct crypto_rfc4543_req_ctx
) +
1013 ALIGN(crypto_aead_reqsize(aead
), crypto_tfm_ctx_alignment()) +
1014 align
+ GCM_AES_IV_SIZE
);
1019 crypto_free_aead(aead
);
1023 static void crypto_rfc4543_exit_tfm(struct crypto_aead
*tfm
)
1025 struct crypto_rfc4543_ctx
*ctx
= crypto_aead_ctx(tfm
);
1027 crypto_free_aead(ctx
->child
);
1028 crypto_put_default_null_skcipher();
1031 static void crypto_rfc4543_free(struct aead_instance
*inst
)
1033 struct crypto_rfc4543_instance_ctx
*ctx
= aead_instance_ctx(inst
);
1035 crypto_drop_aead(&ctx
->aead
);
1040 static int crypto_rfc4543_create(struct crypto_template
*tmpl
,
1044 struct aead_instance
*inst
;
1045 struct aead_alg
*alg
;
1046 struct crypto_rfc4543_instance_ctx
*ctx
;
1049 err
= crypto_check_attr_type(tb
, CRYPTO_ALG_TYPE_AEAD
, &mask
);
1053 inst
= kzalloc(sizeof(*inst
) + sizeof(*ctx
), GFP_KERNEL
);
1057 ctx
= aead_instance_ctx(inst
);
1058 err
= crypto_grab_aead(&ctx
->aead
, aead_crypto_instance(inst
),
1059 crypto_attr_alg_name(tb
[1]), 0, mask
);
1063 alg
= crypto_spawn_aead_alg(&ctx
->aead
);
1067 /* Underlying IV size must be 12. */
1068 if (crypto_aead_alg_ivsize(alg
) != GCM_AES_IV_SIZE
)
1071 /* Not a stream cipher? */
1072 if (alg
->base
.cra_blocksize
!= 1)
1075 err
= -ENAMETOOLONG
;
1076 if (snprintf(inst
->alg
.base
.cra_name
, CRYPTO_MAX_ALG_NAME
,
1077 "rfc4543(%s)", alg
->base
.cra_name
) >=
1078 CRYPTO_MAX_ALG_NAME
||
1079 snprintf(inst
->alg
.base
.cra_driver_name
, CRYPTO_MAX_ALG_NAME
,
1080 "rfc4543(%s)", alg
->base
.cra_driver_name
) >=
1081 CRYPTO_MAX_ALG_NAME
)
1084 inst
->alg
.base
.cra_priority
= alg
->base
.cra_priority
;
1085 inst
->alg
.base
.cra_blocksize
= 1;
1086 inst
->alg
.base
.cra_alignmask
= alg
->base
.cra_alignmask
;
1088 inst
->alg
.base
.cra_ctxsize
= sizeof(struct crypto_rfc4543_ctx
);
1090 inst
->alg
.ivsize
= GCM_RFC4543_IV_SIZE
;
1091 inst
->alg
.chunksize
= crypto_aead_alg_chunksize(alg
);
1092 inst
->alg
.maxauthsize
= crypto_aead_alg_maxauthsize(alg
);
1094 inst
->alg
.init
= crypto_rfc4543_init_tfm
;
1095 inst
->alg
.exit
= crypto_rfc4543_exit_tfm
;
1097 inst
->alg
.setkey
= crypto_rfc4543_setkey
;
1098 inst
->alg
.setauthsize
= crypto_rfc4543_setauthsize
;
1099 inst
->alg
.encrypt
= crypto_rfc4543_encrypt
;
1100 inst
->alg
.decrypt
= crypto_rfc4543_decrypt
;
1102 inst
->free
= crypto_rfc4543_free
;
1104 err
= aead_register_instance(tmpl
, inst
);
1107 crypto_rfc4543_free(inst
);
1112 static struct crypto_template crypto_gcm_tmpls
[] = {
1115 .create
= crypto_gcm_base_create
,
1116 .module
= THIS_MODULE
,
1119 .create
= crypto_gcm_create
,
1120 .module
= THIS_MODULE
,
1123 .create
= crypto_rfc4106_create
,
1124 .module
= THIS_MODULE
,
1127 .create
= crypto_rfc4543_create
,
1128 .module
= THIS_MODULE
,
1132 static int __init
crypto_gcm_module_init(void)
1136 gcm_zeroes
= kzalloc(sizeof(*gcm_zeroes
), GFP_KERNEL
);
1140 sg_init_one(&gcm_zeroes
->sg
, gcm_zeroes
->buf
, sizeof(gcm_zeroes
->buf
));
1142 err
= crypto_register_templates(crypto_gcm_tmpls
,
1143 ARRAY_SIZE(crypto_gcm_tmpls
));
1150 static void __exit
crypto_gcm_module_exit(void)
1153 crypto_unregister_templates(crypto_gcm_tmpls
,
1154 ARRAY_SIZE(crypto_gcm_tmpls
));
1157 subsys_initcall(crypto_gcm_module_init
);
1158 module_exit(crypto_gcm_module_exit
);
1160 MODULE_LICENSE("GPL");
1161 MODULE_DESCRIPTION("Galois/Counter Mode");
1162 MODULE_AUTHOR("Mikko Herranen <mh1@iki.fi>");
1163 MODULE_ALIAS_CRYPTO("gcm_base");
1164 MODULE_ALIAS_CRYPTO("rfc4106");
1165 MODULE_ALIAS_CRYPTO("rfc4543");
1166 MODULE_ALIAS_CRYPTO("gcm");