dm-verity: use SHA-256 library for SHA-256

When the hash algorithm is SHA-256 and the verity version is not 0, use
the SHA-256 library instead of crypto_shash.

This is a prerequisite for making dm-verity interleave the computation
of SHA-256 hashes for increased performance.  That optimization is
available in the SHA-256 library but not in crypto_shash.

Even without interleaved hashing, switching to the library also slightly
improves performance by itself because it avoids the overhead of
crypto_shash, including indirect calls and other API overhead.
(Benchmark on x86_64, AMD Zen 5: hashing 4K blocks gets 2.1% faster.)

SHA-256 is by far the most common hash algorithm used with dm-verity.
It makes sense to optimize for the common case and fall back to the
generic crypto layer for uncommon cases, as suggested by Linus:
https://lore.kernel.org/r/CAHk-=wgp-fOSsZsYrbyzqCAfEvrt5jQs1jL-97Wc4seMNTUyng@mail.gmail.com

Signed-off-by: Eric Biggers <ebiggers@kernel.org>
Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>

diff --git a/drivers/md/Kconfig b/drivers/md/Kconfig
index dcd232a2ca244be65e3d36dfcf7d8a6e7bdceda5..239c1744a9268ed5adb136efc354fe9629a8106f 100644 (file)
--- a/drivers/md/Kconfig
+++ b/drivers/md/Kconfig
@@ -547,6 +547,7 @@ config DM_VERITY
        depends on BLK_DEV_DM
        select CRYPTO
        select CRYPTO_HASH
+       select CRYPTO_LIB_SHA256
        select DM_BUFIO
        help
          This device-mapper target creates a read-only device that

diff --git a/drivers/md/dm-verity-target.c b/drivers/md/dm-verity-target.c
index 20ddf560d22e301ecb89c293c1f07e44b54bf0cb..bba981080563171c06c9bab53681acae97c0acf1 100644 (file)
--- a/drivers/md/dm-verity-target.c
+++ b/drivers/md/dm-verity-target.c
@@ -117,11 +117,25 @@ static sector_t verity_position_at_level(struct dm_verity *v, sector_t block,
 int verity_hash(struct dm_verity *v, struct dm_verity_io *io,
                const u8 *data, size_t len, u8 *digest)
 {
-       struct shash_desc *desc = &io->hash_desc;
+       struct shash_desc *desc;
        int r;
 
+       if (likely(v->use_sha256_lib)) {
+               struct sha256_ctx *ctx = &io->hash_ctx.sha256;
+
+               /*
+                * Fast path using SHA-256 library.  This is enabled only for
+                * verity version 1, where the salt is at the beginning.
+                */
+               *ctx = *v->initial_hashstate.sha256;
+               sha256_update(ctx, data, len);
+               sha256_final(ctx, digest);
+               return 0;
+       }
+
+       desc = &io->hash_ctx.shash;
        desc->tfm = v->shash_tfm;
-       if (unlikely(v->initial_hashstate == NULL)) {
+       if (unlikely(v->initial_hashstate.shash == NULL)) {
                /* Version 0: salt at end */
                r = crypto_shash_init(desc) ?:
                    crypto_shash_update(desc, data, len) ?:
@@ -129,7 +143,7 @@ int verity_hash(struct dm_verity *v, struct dm_verity_io *io,
                    crypto_shash_final(desc, digest);
        } else {
                /* Version 1: salt at beginning */
-               r = crypto_shash_import(desc, v->initial_hashstate) ?:
+               r = crypto_shash_import(desc, v->initial_hashstate.shash) ?:
                    crypto_shash_finup(desc, data, len, digest);
        }
        if (unlikely(r))
@@ -1004,7 +1018,7 @@ static void verity_dtr(struct dm_target *ti)
 
        kvfree(v->validated_blocks);
        kfree(v->salt);
-       kfree(v->initial_hashstate);
+       kfree(v->initial_hashstate.shash);
        kfree(v->root_digest);
        kfree(v->zero_digest);
        verity_free_sig(v);
@@ -1069,8 +1083,7 @@ static int verity_alloc_zero_digest(struct dm_verity *v)
        if (!v->zero_digest)
                return r;
 
-       io = kmalloc(sizeof(*io) + crypto_shash_descsize(v->shash_tfm),
-                    GFP_KERNEL);
+       io = kmalloc(v->ti->per_io_data_size, GFP_KERNEL);
 
        if (!io)
                return r; /* verity_dtr will free zero_digest */
@@ -1256,6 +1269,20 @@ static int verity_setup_hash_alg(struct dm_verity *v, const char *alg_name)
                ti->error = "Digest size too big";
                return -EINVAL;
        }
+       if (likely(v->version && strcmp(alg_name, "sha256") == 0)) {
+               /*
+                * Fast path: use the library API for reduced overhead and
+                * interleaved hashing support.
+                */
+               v->use_sha256_lib = true;
+               ti->per_io_data_size =
+                       offsetofend(struct dm_verity_io, hash_ctx.sha256);
+       } else {
+               /* Fallback case: use the generic crypto API. */
+               ti->per_io_data_size =
+                       offsetofend(struct dm_verity_io, hash_ctx.shash) +
+                       crypto_shash_descsize(shash);
+       }
        return 0;
 }
 
@@ -1276,7 +1303,18 @@ static int verity_setup_salt_and_hashstate(struct dm_verity *v, const char *arg)
                        return -EINVAL;
                }
        }
-       if (v->version) { /* Version 1: salt at beginning */
+       if (likely(v->use_sha256_lib)) {
+               /* Implies version 1: salt at beginning */
+               v->initial_hashstate.sha256 =
+                       kmalloc(sizeof(struct sha256_ctx), GFP_KERNEL);
+               if (!v->initial_hashstate.sha256) {
+                       ti->error = "Cannot allocate initial hash state";
+                       return -ENOMEM;
+               }
+               sha256_init(v->initial_hashstate.sha256);
+               sha256_update(v->initial_hashstate.sha256,
+                             v->salt, v->salt_size);
+       } else if (v->version) { /* Version 1: salt at beginning */
                SHASH_DESC_ON_STACK(desc, v->shash_tfm);
                int r;
 
@@ -1284,16 +1322,16 @@ static int verity_setup_salt_and_hashstate(struct dm_verity *v, const char *arg)
                 * Compute the pre-salted hash state that can be passed to
                 * crypto_shash_import() for each block later.
                 */
-               v->initial_hashstate = kmalloc(
+               v->initial_hashstate.shash = kmalloc(
                        crypto_shash_statesize(v->shash_tfm), GFP_KERNEL);
-               if (!v->initial_hashstate) {
+               if (!v->initial_hashstate.shash) {
                        ti->error = "Cannot allocate initial hash state";
                        return -ENOMEM;
                }
                desc->tfm = v->shash_tfm;
                r = crypto_shash_init(desc) ?:
                    crypto_shash_update(desc, v->salt, v->salt_size) ?:
-                   crypto_shash_export(desc, v->initial_hashstate);
+                   crypto_shash_export(desc, v->initial_hashstate.shash);
                if (r) {
                        ti->error = "Cannot set up initial hash state";
                        return r;
@@ -1555,9 +1593,6 @@ static int verity_ctr(struct dm_target *ti, unsigned int argc, char **argv)
                goto bad;
        }
 
-       ti->per_io_data_size = sizeof(struct dm_verity_io) +
-                              crypto_shash_descsize(v->shash_tfm);
-
        r = verity_fec_ctr(v);
        if (r)
                goto bad;

diff --git a/drivers/md/dm-verity.h b/drivers/md/dm-verity.h
index 6d141abd965c77205a783c02201d29700a938564..cdcee68a4bc0a5a23fddae3658bc500a9ae00680 100644 (file)
--- a/drivers/md/dm-verity.h
+++ b/drivers/md/dm-verity.h
@@ -16,6 +16,7 @@
 #include <linux/device-mapper.h>
 #include <linux/interrupt.h>
 #include <crypto/hash.h>
+#include <crypto/sha2.h>
 
 #define DM_VERITY_MAX_LEVELS           63
 
@@ -42,7 +43,10 @@ struct dm_verity {
        struct crypto_shash *shash_tfm;
        u8 *root_digest;        /* digest of the root block */
        u8 *salt;               /* salt: its size is salt_size */
-       u8 *initial_hashstate;  /* salted initial state, if version >= 1 */
+       union {
+               struct sha256_ctx *sha256;      /* for use_sha256_lib=1 */
+               u8 *shash;                      /* for use_sha256_lib=0 */
+       } initial_hashstate; /* salted initial state, if version >= 1 */
        u8 *zero_digest;        /* digest for a zero block */
 #ifdef CONFIG_SECURITY
        u8 *root_digest_sig;    /* signature of the root digest */
@@ -59,6 +63,7 @@ struct dm_verity {
        unsigned char version;
        bool hash_failed:1;     /* set if hash of any block failed */
        bool use_bh_wq:1;       /* try to verify in BH wq before normal work-queue */
+       bool use_sha256_lib:1;  /* use SHA-256 library instead of generic crypto API */
        unsigned int digest_size;       /* digest size for the current hash algorithm */
        enum verity_mode mode;  /* mode for handling verification errors */
        enum verity_mode error_mode;/* mode for handling I/O errors */
@@ -98,11 +103,16 @@ struct dm_verity_io {
        u8 want_digest[HASH_MAX_DIGESTSIZE];
 
        /*
-        * Temporary space for hashing.  This is variable-length and must be at
-        * the end of the struct.  struct shash_desc is just the fixed part;
-        * it's followed by a context of size crypto_shash_descsize(shash_tfm).
+        * Temporary space for hashing.  Either sha256 or shash is used,
+        * depending on the value of use_sha256_lib.  If shash is used,
+        * then this field is variable-length, with total size
+        * sizeof(struct shash_desc) + crypto_shash_descsize(shash_tfm).
+        * For this reason, this field must be the end of the struct.
         */
-       struct shash_desc hash_desc;
+       union {
+               struct sha256_ctx sha256;
+               struct shash_desc shash;
+       } hash_ctx;
 };
 
 static inline u8 *verity_io_real_digest(struct dm_verity *v,