1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright 2019 Google LLC
7 * Refer to Documentation/block/inline-encryption.rst for detailed explanation.
10 #define pr_fmt(fmt) "blk-crypto-fallback: " fmt
12 #include <crypto/skcipher.h>
13 #include <linux/blk-crypto.h>
14 #include <linux/blk-crypto-profile.h>
15 #include <linux/blkdev.h>
16 #include <linux/crypto.h>
17 #include <linux/mempool.h>
18 #include <linux/module.h>
19 #include <linux/random.h>
20 #include <linux/scatterlist.h>
22 #include "blk-cgroup.h"
23 #include "blk-crypto-internal.h"
25 static unsigned int num_prealloc_bounce_pg
= 32;
26 module_param(num_prealloc_bounce_pg
, uint
, 0);
27 MODULE_PARM_DESC(num_prealloc_bounce_pg
,
28 "Number of preallocated bounce pages for the blk-crypto crypto API fallback");
30 static unsigned int blk_crypto_num_keyslots
= 100;
31 module_param_named(num_keyslots
, blk_crypto_num_keyslots
, uint
, 0);
32 MODULE_PARM_DESC(num_keyslots
,
33 "Number of keyslots for the blk-crypto crypto API fallback");
35 static unsigned int num_prealloc_fallback_crypt_ctxs
= 128;
36 module_param(num_prealloc_fallback_crypt_ctxs
, uint
, 0);
37 MODULE_PARM_DESC(num_prealloc_crypt_fallback_ctxs
,
38 "Number of preallocated bio fallback crypto contexts for blk-crypto to use during crypto API fallback");
40 struct bio_fallback_crypt_ctx
{
41 struct bio_crypt_ctx crypt_ctx
;
43 * Copy of the bvec_iter when this bio was submitted.
44 * We only want to en/decrypt the part of the bio as described by the
45 * bvec_iter upon submission because bio might be split before being
48 struct bvec_iter crypt_iter
;
51 struct work_struct work
;
55 void *bi_private_orig
;
56 bio_end_io_t
*bi_end_io_orig
;
61 static struct kmem_cache
*bio_fallback_crypt_ctx_cache
;
62 static mempool_t
*bio_fallback_crypt_ctx_pool
;
65 * Allocating a crypto tfm during I/O can deadlock, so we have to preallocate
66 * all of a mode's tfms when that mode starts being used. Since each mode may
67 * need all the keyslots at some point, each mode needs its own tfm for each
68 * keyslot; thus, a keyslot may contain tfms for multiple modes. However, to
69 * match the behavior of real inline encryption hardware (which only supports a
70 * single encryption context per keyslot), we only allow one tfm per keyslot to
71 * be used at a time - the rest of the unused tfms have their keys cleared.
73 static DEFINE_MUTEX(tfms_init_lock
);
74 static bool tfms_inited
[BLK_ENCRYPTION_MODE_MAX
];
76 static struct blk_crypto_fallback_keyslot
{
77 enum blk_crypto_mode_num crypto_mode
;
78 struct crypto_skcipher
*tfms
[BLK_ENCRYPTION_MODE_MAX
];
79 } *blk_crypto_keyslots
;
81 static struct blk_crypto_profile
*blk_crypto_fallback_profile
;
82 static struct workqueue_struct
*blk_crypto_wq
;
83 static mempool_t
*blk_crypto_bounce_page_pool
;
84 static struct bio_set crypto_bio_split
;
87 * This is the key we set when evicting a keyslot. This *should* be the all 0's
88 * key, but AES-XTS rejects that key, so we use some random bytes instead.
90 static u8 blank_key
[BLK_CRYPTO_MAX_KEY_SIZE
];
92 static void blk_crypto_fallback_evict_keyslot(unsigned int slot
)
94 struct blk_crypto_fallback_keyslot
*slotp
= &blk_crypto_keyslots
[slot
];
95 enum blk_crypto_mode_num crypto_mode
= slotp
->crypto_mode
;
98 WARN_ON(slotp
->crypto_mode
== BLK_ENCRYPTION_MODE_INVALID
);
100 /* Clear the key in the skcipher */
101 err
= crypto_skcipher_setkey(slotp
->tfms
[crypto_mode
], blank_key
,
102 blk_crypto_modes
[crypto_mode
].keysize
);
104 slotp
->crypto_mode
= BLK_ENCRYPTION_MODE_INVALID
;
108 blk_crypto_fallback_keyslot_program(struct blk_crypto_profile
*profile
,
109 const struct blk_crypto_key
*key
,
112 struct blk_crypto_fallback_keyslot
*slotp
= &blk_crypto_keyslots
[slot
];
113 const enum blk_crypto_mode_num crypto_mode
=
114 key
->crypto_cfg
.crypto_mode
;
117 if (crypto_mode
!= slotp
->crypto_mode
&&
118 slotp
->crypto_mode
!= BLK_ENCRYPTION_MODE_INVALID
)
119 blk_crypto_fallback_evict_keyslot(slot
);
121 slotp
->crypto_mode
= crypto_mode
;
122 err
= crypto_skcipher_setkey(slotp
->tfms
[crypto_mode
], key
->raw
,
125 blk_crypto_fallback_evict_keyslot(slot
);
131 static int blk_crypto_fallback_keyslot_evict(struct blk_crypto_profile
*profile
,
132 const struct blk_crypto_key
*key
,
135 blk_crypto_fallback_evict_keyslot(slot
);
139 static const struct blk_crypto_ll_ops blk_crypto_fallback_ll_ops
= {
140 .keyslot_program
= blk_crypto_fallback_keyslot_program
,
141 .keyslot_evict
= blk_crypto_fallback_keyslot_evict
,
144 static void blk_crypto_fallback_encrypt_endio(struct bio
*enc_bio
)
146 struct bio
*src_bio
= enc_bio
->bi_private
;
149 for (i
= 0; i
< enc_bio
->bi_vcnt
; i
++)
150 mempool_free(enc_bio
->bi_io_vec
[i
].bv_page
,
151 blk_crypto_bounce_page_pool
);
153 src_bio
->bi_status
= enc_bio
->bi_status
;
160 static struct bio
*blk_crypto_fallback_clone_bio(struct bio
*bio_src
)
162 unsigned int nr_segs
= bio_segments(bio_src
);
163 struct bvec_iter iter
;
167 bio
= bio_kmalloc(nr_segs
, GFP_NOIO
);
170 bio_init(bio
, bio_src
->bi_bdev
, bio
->bi_inline_vecs
, nr_segs
,
172 if (bio_flagged(bio_src
, BIO_REMAPPED
))
173 bio_set_flag(bio
, BIO_REMAPPED
);
174 bio
->bi_ioprio
= bio_src
->bi_ioprio
;
175 bio
->bi_iter
.bi_sector
= bio_src
->bi_iter
.bi_sector
;
176 bio
->bi_iter
.bi_size
= bio_src
->bi_iter
.bi_size
;
178 bio_for_each_segment(bv
, bio_src
, iter
)
179 bio
->bi_io_vec
[bio
->bi_vcnt
++] = bv
;
181 bio_clone_blkg_association(bio
, bio_src
);
187 blk_crypto_fallback_alloc_cipher_req(struct blk_crypto_keyslot
*slot
,
188 struct skcipher_request
**ciph_req_ret
,
189 struct crypto_wait
*wait
)
191 struct skcipher_request
*ciph_req
;
192 const struct blk_crypto_fallback_keyslot
*slotp
;
193 int keyslot_idx
= blk_crypto_keyslot_index(slot
);
195 slotp
= &blk_crypto_keyslots
[keyslot_idx
];
196 ciph_req
= skcipher_request_alloc(slotp
->tfms
[slotp
->crypto_mode
],
201 skcipher_request_set_callback(ciph_req
,
202 CRYPTO_TFM_REQ_MAY_BACKLOG
|
203 CRYPTO_TFM_REQ_MAY_SLEEP
,
204 crypto_req_done
, wait
);
205 *ciph_req_ret
= ciph_req
;
210 static bool blk_crypto_fallback_split_bio_if_needed(struct bio
**bio_ptr
)
212 struct bio
*bio
= *bio_ptr
;
214 unsigned int num_sectors
= 0;
216 struct bvec_iter iter
;
218 bio_for_each_segment(bv
, bio
, iter
) {
219 num_sectors
+= bv
.bv_len
>> SECTOR_SHIFT
;
220 if (++i
== BIO_MAX_VECS
)
223 if (num_sectors
< bio_sectors(bio
)) {
224 struct bio
*split_bio
;
226 split_bio
= bio_split(bio
, num_sectors
, GFP_NOIO
,
229 bio
->bi_status
= BLK_STS_RESOURCE
;
232 bio_chain(split_bio
, bio
);
233 submit_bio_noacct(bio
);
234 *bio_ptr
= split_bio
;
240 union blk_crypto_iv
{
241 __le64 dun
[BLK_CRYPTO_DUN_ARRAY_SIZE
];
242 u8 bytes
[BLK_CRYPTO_MAX_IV_SIZE
];
245 static void blk_crypto_dun_to_iv(const u64 dun
[BLK_CRYPTO_DUN_ARRAY_SIZE
],
246 union blk_crypto_iv
*iv
)
250 for (i
= 0; i
< BLK_CRYPTO_DUN_ARRAY_SIZE
; i
++)
251 iv
->dun
[i
] = cpu_to_le64(dun
[i
]);
255 * The crypto API fallback's encryption routine.
256 * Allocate a bounce bio for encryption, encrypt the input bio using crypto API,
257 * and replace *bio_ptr with the bounce bio. May split input bio if it's too
258 * large. Returns true on success. Returns false and sets bio->bi_status on
261 static bool blk_crypto_fallback_encrypt_bio(struct bio
**bio_ptr
)
263 struct bio
*src_bio
, *enc_bio
;
264 struct bio_crypt_ctx
*bc
;
265 struct blk_crypto_keyslot
*slot
;
267 struct skcipher_request
*ciph_req
= NULL
;
268 DECLARE_CRYPTO_WAIT(wait
);
269 u64 curr_dun
[BLK_CRYPTO_DUN_ARRAY_SIZE
];
270 struct scatterlist src
, dst
;
271 union blk_crypto_iv iv
;
276 /* Split the bio if it's too big for single page bvec */
277 if (!blk_crypto_fallback_split_bio_if_needed(bio_ptr
))
281 bc
= src_bio
->bi_crypt_context
;
282 data_unit_size
= bc
->bc_key
->crypto_cfg
.data_unit_size
;
284 /* Allocate bounce bio for encryption */
285 enc_bio
= blk_crypto_fallback_clone_bio(src_bio
);
287 src_bio
->bi_status
= BLK_STS_RESOURCE
;
292 * Get a blk-crypto-fallback keyslot that contains a crypto_skcipher for
293 * this bio's algorithm and key.
295 blk_st
= blk_crypto_get_keyslot(blk_crypto_fallback_profile
,
297 if (blk_st
!= BLK_STS_OK
) {
298 src_bio
->bi_status
= blk_st
;
299 goto out_put_enc_bio
;
302 /* and then allocate an skcipher_request for it */
303 if (!blk_crypto_fallback_alloc_cipher_req(slot
, &ciph_req
, &wait
)) {
304 src_bio
->bi_status
= BLK_STS_RESOURCE
;
305 goto out_release_keyslot
;
308 memcpy(curr_dun
, bc
->bc_dun
, sizeof(curr_dun
));
309 sg_init_table(&src
, 1);
310 sg_init_table(&dst
, 1);
312 skcipher_request_set_crypt(ciph_req
, &src
, &dst
, data_unit_size
,
315 /* Encrypt each page in the bounce bio */
316 for (i
= 0; i
< enc_bio
->bi_vcnt
; i
++) {
317 struct bio_vec
*enc_bvec
= &enc_bio
->bi_io_vec
[i
];
318 struct page
*plaintext_page
= enc_bvec
->bv_page
;
319 struct page
*ciphertext_page
=
320 mempool_alloc(blk_crypto_bounce_page_pool
, GFP_NOIO
);
322 enc_bvec
->bv_page
= ciphertext_page
;
324 if (!ciphertext_page
) {
325 src_bio
->bi_status
= BLK_STS_RESOURCE
;
326 goto out_free_bounce_pages
;
329 sg_set_page(&src
, plaintext_page
, data_unit_size
,
330 enc_bvec
->bv_offset
);
331 sg_set_page(&dst
, ciphertext_page
, data_unit_size
,
332 enc_bvec
->bv_offset
);
334 /* Encrypt each data unit in this page */
335 for (j
= 0; j
< enc_bvec
->bv_len
; j
+= data_unit_size
) {
336 blk_crypto_dun_to_iv(curr_dun
, &iv
);
337 if (crypto_wait_req(crypto_skcipher_encrypt(ciph_req
),
340 src_bio
->bi_status
= BLK_STS_IOERR
;
341 goto out_free_bounce_pages
;
343 bio_crypt_dun_increment(curr_dun
, 1);
344 src
.offset
+= data_unit_size
;
345 dst
.offset
+= data_unit_size
;
349 enc_bio
->bi_private
= src_bio
;
350 enc_bio
->bi_end_io
= blk_crypto_fallback_encrypt_endio
;
355 goto out_free_ciph_req
;
357 out_free_bounce_pages
:
359 mempool_free(enc_bio
->bi_io_vec
[--i
].bv_page
,
360 blk_crypto_bounce_page_pool
);
362 skcipher_request_free(ciph_req
);
364 blk_crypto_put_keyslot(slot
);
373 * The crypto API fallback's main decryption routine.
374 * Decrypts input bio in place, and calls bio_endio on the bio.
376 static void blk_crypto_fallback_decrypt_bio(struct work_struct
*work
)
378 struct bio_fallback_crypt_ctx
*f_ctx
=
379 container_of(work
, struct bio_fallback_crypt_ctx
, work
);
380 struct bio
*bio
= f_ctx
->bio
;
381 struct bio_crypt_ctx
*bc
= &f_ctx
->crypt_ctx
;
382 struct blk_crypto_keyslot
*slot
;
383 struct skcipher_request
*ciph_req
= NULL
;
384 DECLARE_CRYPTO_WAIT(wait
);
385 u64 curr_dun
[BLK_CRYPTO_DUN_ARRAY_SIZE
];
386 union blk_crypto_iv iv
;
387 struct scatterlist sg
;
389 struct bvec_iter iter
;
390 const int data_unit_size
= bc
->bc_key
->crypto_cfg
.data_unit_size
;
395 * Get a blk-crypto-fallback keyslot that contains a crypto_skcipher for
396 * this bio's algorithm and key.
398 blk_st
= blk_crypto_get_keyslot(blk_crypto_fallback_profile
,
400 if (blk_st
!= BLK_STS_OK
) {
401 bio
->bi_status
= blk_st
;
405 /* and then allocate an skcipher_request for it */
406 if (!blk_crypto_fallback_alloc_cipher_req(slot
, &ciph_req
, &wait
)) {
407 bio
->bi_status
= BLK_STS_RESOURCE
;
411 memcpy(curr_dun
, bc
->bc_dun
, sizeof(curr_dun
));
412 sg_init_table(&sg
, 1);
413 skcipher_request_set_crypt(ciph_req
, &sg
, &sg
, data_unit_size
,
416 /* Decrypt each segment in the bio */
417 __bio_for_each_segment(bv
, bio
, iter
, f_ctx
->crypt_iter
) {
418 struct page
*page
= bv
.bv_page
;
420 sg_set_page(&sg
, page
, data_unit_size
, bv
.bv_offset
);
422 /* Decrypt each data unit in the segment */
423 for (i
= 0; i
< bv
.bv_len
; i
+= data_unit_size
) {
424 blk_crypto_dun_to_iv(curr_dun
, &iv
);
425 if (crypto_wait_req(crypto_skcipher_decrypt(ciph_req
),
427 bio
->bi_status
= BLK_STS_IOERR
;
430 bio_crypt_dun_increment(curr_dun
, 1);
431 sg
.offset
+= data_unit_size
;
436 skcipher_request_free(ciph_req
);
437 blk_crypto_put_keyslot(slot
);
439 mempool_free(f_ctx
, bio_fallback_crypt_ctx_pool
);
444 * blk_crypto_fallback_decrypt_endio - queue bio for fallback decryption
446 * @bio: the bio to queue
448 * Restore bi_private and bi_end_io, and queue the bio for decryption into a
449 * workqueue, since this function will be called from an atomic context.
451 static void blk_crypto_fallback_decrypt_endio(struct bio
*bio
)
453 struct bio_fallback_crypt_ctx
*f_ctx
= bio
->bi_private
;
455 bio
->bi_private
= f_ctx
->bi_private_orig
;
456 bio
->bi_end_io
= f_ctx
->bi_end_io_orig
;
458 /* If there was an IO error, don't queue for decrypt. */
459 if (bio
->bi_status
) {
460 mempool_free(f_ctx
, bio_fallback_crypt_ctx_pool
);
465 INIT_WORK(&f_ctx
->work
, blk_crypto_fallback_decrypt_bio
);
467 queue_work(blk_crypto_wq
, &f_ctx
->work
);
471 * blk_crypto_fallback_bio_prep - Prepare a bio to use fallback en/decryption
473 * @bio_ptr: pointer to the bio to prepare
475 * If bio is doing a WRITE operation, this splits the bio into two parts if it's
476 * too big (see blk_crypto_fallback_split_bio_if_needed()). It then allocates a
477 * bounce bio for the first part, encrypts it, and updates bio_ptr to point to
480 * For a READ operation, we mark the bio for decryption by using bi_private and
483 * In either case, this function will make the bio look like a regular bio (i.e.
484 * as if no encryption context was ever specified) for the purposes of the rest
485 * of the stack except for blk-integrity (blk-integrity and blk-crypto are not
486 * currently supported together).
488 * Return: true on success. Sets bio->bi_status and returns false on error.
490 bool blk_crypto_fallback_bio_prep(struct bio
**bio_ptr
)
492 struct bio
*bio
= *bio_ptr
;
493 struct bio_crypt_ctx
*bc
= bio
->bi_crypt_context
;
494 struct bio_fallback_crypt_ctx
*f_ctx
;
496 if (WARN_ON_ONCE(!tfms_inited
[bc
->bc_key
->crypto_cfg
.crypto_mode
])) {
497 /* User didn't call blk_crypto_start_using_key() first */
498 bio
->bi_status
= BLK_STS_IOERR
;
502 if (!__blk_crypto_cfg_supported(blk_crypto_fallback_profile
,
503 &bc
->bc_key
->crypto_cfg
)) {
504 bio
->bi_status
= BLK_STS_NOTSUPP
;
508 if (bio_data_dir(bio
) == WRITE
)
509 return blk_crypto_fallback_encrypt_bio(bio_ptr
);
512 * bio READ case: Set up a f_ctx in the bio's bi_private and set the
513 * bi_end_io appropriately to trigger decryption when the bio is ended.
515 f_ctx
= mempool_alloc(bio_fallback_crypt_ctx_pool
, GFP_NOIO
);
516 f_ctx
->crypt_ctx
= *bc
;
517 f_ctx
->crypt_iter
= bio
->bi_iter
;
518 f_ctx
->bi_private_orig
= bio
->bi_private
;
519 f_ctx
->bi_end_io_orig
= bio
->bi_end_io
;
520 bio
->bi_private
= (void *)f_ctx
;
521 bio
->bi_end_io
= blk_crypto_fallback_decrypt_endio
;
522 bio_crypt_free_ctx(bio
);
527 int blk_crypto_fallback_evict_key(const struct blk_crypto_key
*key
)
529 return __blk_crypto_evict_key(blk_crypto_fallback_profile
, key
);
532 static bool blk_crypto_fallback_inited
;
533 static int blk_crypto_fallback_init(void)
538 if (blk_crypto_fallback_inited
)
541 get_random_bytes(blank_key
, BLK_CRYPTO_MAX_KEY_SIZE
);
543 err
= bioset_init(&crypto_bio_split
, 64, 0, 0);
547 /* Dynamic allocation is needed because of lockdep_register_key(). */
548 blk_crypto_fallback_profile
=
549 kzalloc(sizeof(*blk_crypto_fallback_profile
), GFP_KERNEL
);
550 if (!blk_crypto_fallback_profile
) {
552 goto fail_free_bioset
;
555 err
= blk_crypto_profile_init(blk_crypto_fallback_profile
,
556 blk_crypto_num_keyslots
);
558 goto fail_free_profile
;
561 blk_crypto_fallback_profile
->ll_ops
= blk_crypto_fallback_ll_ops
;
562 blk_crypto_fallback_profile
->max_dun_bytes_supported
= BLK_CRYPTO_MAX_IV_SIZE
;
564 /* All blk-crypto modes have a crypto API fallback. */
565 for (i
= 0; i
< BLK_ENCRYPTION_MODE_MAX
; i
++)
566 blk_crypto_fallback_profile
->modes_supported
[i
] = 0xFFFFFFFF;
567 blk_crypto_fallback_profile
->modes_supported
[BLK_ENCRYPTION_MODE_INVALID
] = 0;
569 blk_crypto_wq
= alloc_workqueue("blk_crypto_wq",
570 WQ_UNBOUND
| WQ_HIGHPRI
|
571 WQ_MEM_RECLAIM
, num_online_cpus());
573 goto fail_destroy_profile
;
575 blk_crypto_keyslots
= kcalloc(blk_crypto_num_keyslots
,
576 sizeof(blk_crypto_keyslots
[0]),
578 if (!blk_crypto_keyslots
)
581 blk_crypto_bounce_page_pool
=
582 mempool_create_page_pool(num_prealloc_bounce_pg
, 0);
583 if (!blk_crypto_bounce_page_pool
)
584 goto fail_free_keyslots
;
586 bio_fallback_crypt_ctx_cache
= KMEM_CACHE(bio_fallback_crypt_ctx
, 0);
587 if (!bio_fallback_crypt_ctx_cache
)
588 goto fail_free_bounce_page_pool
;
590 bio_fallback_crypt_ctx_pool
=
591 mempool_create_slab_pool(num_prealloc_fallback_crypt_ctxs
,
592 bio_fallback_crypt_ctx_cache
);
593 if (!bio_fallback_crypt_ctx_pool
)
594 goto fail_free_crypt_ctx_cache
;
596 blk_crypto_fallback_inited
= true;
599 fail_free_crypt_ctx_cache
:
600 kmem_cache_destroy(bio_fallback_crypt_ctx_cache
);
601 fail_free_bounce_page_pool
:
602 mempool_destroy(blk_crypto_bounce_page_pool
);
604 kfree(blk_crypto_keyslots
);
606 destroy_workqueue(blk_crypto_wq
);
607 fail_destroy_profile
:
608 blk_crypto_profile_destroy(blk_crypto_fallback_profile
);
610 kfree(blk_crypto_fallback_profile
);
612 bioset_exit(&crypto_bio_split
);
618 * Prepare blk-crypto-fallback for the specified crypto mode.
619 * Returns -ENOPKG if the needed crypto API support is missing.
621 int blk_crypto_fallback_start_using_mode(enum blk_crypto_mode_num mode_num
)
623 const char *cipher_str
= blk_crypto_modes
[mode_num
].cipher_str
;
624 struct blk_crypto_fallback_keyslot
*slotp
;
630 * Ensure that updates to blk_crypto_keyslots[i].tfms[mode_num]
631 * for each i are visible before we try to access them.
633 if (likely(smp_load_acquire(&tfms_inited
[mode_num
])))
636 mutex_lock(&tfms_init_lock
);
637 if (tfms_inited
[mode_num
])
640 err
= blk_crypto_fallback_init();
644 for (i
= 0; i
< blk_crypto_num_keyslots
; i
++) {
645 slotp
= &blk_crypto_keyslots
[i
];
646 slotp
->tfms
[mode_num
] = crypto_alloc_skcipher(cipher_str
, 0, 0);
647 if (IS_ERR(slotp
->tfms
[mode_num
])) {
648 err
= PTR_ERR(slotp
->tfms
[mode_num
]);
649 if (err
== -ENOENT
) {
650 pr_warn_once("Missing crypto API support for \"%s\"\n",
654 slotp
->tfms
[mode_num
] = NULL
;
658 crypto_skcipher_set_flags(slotp
->tfms
[mode_num
],
659 CRYPTO_TFM_REQ_FORBID_WEAK_KEYS
);
663 * Ensure that updates to blk_crypto_keyslots[i].tfms[mode_num]
664 * for each i are visible before we set tfms_inited[mode_num].
666 smp_store_release(&tfms_inited
[mode_num
], true);
670 for (i
= 0; i
< blk_crypto_num_keyslots
; i
++) {
671 slotp
= &blk_crypto_keyslots
[i
];
672 crypto_free_skcipher(slotp
->tfms
[mode_num
]);
673 slotp
->tfms
[mode_num
] = NULL
;
676 mutex_unlock(&tfms_init_lock
);