crypto: crc32c - Provide crc32c-arch driver for accelerated library code

crc32c-generic is currently backed by the architecture's CRC-32c library
code, which may offer a variety of implementations depending on the
capabilities of the platform. These are not covered by the crypto
subsystem's fuzz testing capabilities because crc32c-generic is the
reference driver that the fuzzing logic uses as a source of truth.

Fix this by providing a crc32c-arch implementation which is based on the
arch library code if available, and modify crc32c-generic so it is
always based on the generic C implementation. If the arch has no CRC-32c
library code, this change does nothing.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>

diff --git a/crypto/Makefile b/crypto/Makefile
index 3e49a820f148deccc8869c0b6492ed6149a2bf26..77abca715445d08a0c42fcc53ede77bd055c2de4 100644 (file)
--- a/crypto/Makefile
+++ b/crypto/Makefile
@@ -155,6 +155,7 @@ obj-$(CONFIG_CRYPTO_DEFLATE) += deflate.o
 obj-$(CONFIG_CRYPTO_MICHAEL_MIC) += michael_mic.o
 obj-$(CONFIG_CRYPTO_CRC32C) += crc32c_generic.o
 obj-$(CONFIG_CRYPTO_CRC32) += crc32_generic.o
+CFLAGS_crc32c_generic.o += -DARCH=$(ARCH)
 CFLAGS_crc32_generic.o += -DARCH=$(ARCH)
 obj-$(CONFIG_CRYPTO_CRCT10DIF) += crct10dif_common.o crct10dif_generic.o
 obj-$(CONFIG_CRYPTO_CRC64_ROCKSOFT) += crc64_rocksoft_generic.o

diff --git a/crypto/crc32c_generic.c b/crypto/crc32c_generic.c
index 7686147385412a42e74dac14101172c69d611b75..3f928d7f4ade969f55cf3b1ae74cbcf121e16f12 100644 (file)
--- a/crypto/crc32c_generic.c
+++ b/crypto/crc32c_generic.c
@@ -85,6 +85,15 @@ static int chksum_update(struct shash_desc *desc, const u8 *data,
 {
        struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
 
+       ctx->crc = __crc32c_le_base(ctx->crc, data, length);
+       return 0;
+}
+
+static int chksum_update_arch(struct shash_desc *desc, const u8 *data,
+                             unsigned int length)
+{
+       struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
+
        ctx->crc = __crc32c_le(ctx->crc, data, length);
        return 0;
 }
@@ -98,6 +107,13 @@ static int chksum_final(struct shash_desc *desc, u8 *out)
 }
 
 static int __chksum_finup(u32 *crcp, const u8 *data, unsigned int len, u8 *out)
+{
+       put_unaligned_le32(~__crc32c_le_base(*crcp, data, len), out);
+       return 0;
+}
+
+static int __chksum_finup_arch(u32 *crcp, const u8 *data, unsigned int len,
+                              u8 *out)
 {
        put_unaligned_le32(~__crc32c_le(*crcp, data, len), out);
        return 0;
@@ -111,6 +127,14 @@ static int chksum_finup(struct shash_desc *desc, const u8 *data,
        return __chksum_finup(&ctx->crc, data, len, out);
 }
 
+static int chksum_finup_arch(struct shash_desc *desc, const u8 *data,
+                            unsigned int len, u8 *out)
+{
+       struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
+
+       return __chksum_finup_arch(&ctx->crc, data, len, out);
+}
+
 static int chksum_digest(struct shash_desc *desc, const u8 *data,
                         unsigned int length, u8 *out)
 {
@@ -119,6 +143,14 @@ static int chksum_digest(struct shash_desc *desc, const u8 *data,
        return __chksum_finup(&mctx->key, data, length, out);
 }
 
+static int chksum_digest_arch(struct shash_desc *desc, const u8 *data,
+                             unsigned int length, u8 *out)
+{
+       struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
+
+       return __chksum_finup_arch(&mctx->key, data, length, out);
+}
+
 static int crc32c_cra_init(struct crypto_tfm *tfm)
 {
        struct chksum_ctx *mctx = crypto_tfm_ctx(tfm);
@@ -127,35 +159,53 @@ static int crc32c_cra_init(struct crypto_tfm *tfm)
        return 0;
 }
 
-static struct shash_alg alg = {
-       .digestsize             =       CHKSUM_DIGEST_SIZE,
-       .setkey                 =       chksum_setkey,
-       .init           =       chksum_init,
-       .update         =       chksum_update,
-       .final          =       chksum_final,
-       .finup          =       chksum_finup,
-       .digest         =       chksum_digest,
-       .descsize               =       sizeof(struct chksum_desc_ctx),
-       .base                   =       {
-               .cra_name               =       "crc32c",
-               .cra_driver_name        =       "crc32c-generic",
-               .cra_priority           =       100,
-               .cra_flags              =       CRYPTO_ALG_OPTIONAL_KEY,
-               .cra_blocksize          =       CHKSUM_BLOCK_SIZE,
-               .cra_ctxsize            =       sizeof(struct chksum_ctx),
-               .cra_module             =       THIS_MODULE,
-               .cra_init               =       crc32c_cra_init,
-       }
-};
+static struct shash_alg algs[] = {{
+       .digestsize             = CHKSUM_DIGEST_SIZE,
+       .setkey                 = chksum_setkey,
+       .init                   = chksum_init,
+       .update                 = chksum_update,
+       .final                  = chksum_final,
+       .finup                  = chksum_finup,
+       .digest                 = chksum_digest,
+       .descsize               = sizeof(struct chksum_desc_ctx),
+
+       .base.cra_name          = "crc32c",
+       .base.cra_driver_name   = "crc32c-generic",
+       .base.cra_priority      = 100,
+       .base.cra_flags         = CRYPTO_ALG_OPTIONAL_KEY,
+       .base.cra_blocksize     = CHKSUM_BLOCK_SIZE,
+       .base.cra_ctxsize       = sizeof(struct chksum_ctx),
+       .base.cra_module        = THIS_MODULE,
+       .base.cra_init          = crc32c_cra_init,
+}, {
+       .digestsize             = CHKSUM_DIGEST_SIZE,
+       .setkey                 = chksum_setkey,
+       .init                   = chksum_init,
+       .update                 = chksum_update_arch,
+       .final                  = chksum_final,
+       .finup                  = chksum_finup_arch,
+       .digest                 = chksum_digest_arch,
+       .descsize               = sizeof(struct chksum_desc_ctx),
+
+       .base.cra_name          = "crc32c",
+       .base.cra_driver_name   = "crc32c-" __stringify(ARCH),
+       .base.cra_priority      = 150,
+       .base.cra_flags         = CRYPTO_ALG_OPTIONAL_KEY,
+       .base.cra_blocksize     = CHKSUM_BLOCK_SIZE,
+       .base.cra_ctxsize       = sizeof(struct chksum_ctx),
+       .base.cra_module        = THIS_MODULE,
+       .base.cra_init          = crc32c_cra_init,
+}};
 
 static int __init crc32c_mod_init(void)
 {
-       return crypto_register_shash(&alg);
+       /* register the arch flavor only if it differs from the generic one */
+       return crypto_register_shashes(algs, 1 + (&__crc32c_le != &__crc32c_le_base));
 }
 
 static void __exit crc32c_mod_fini(void)
 {
-       crypto_unregister_shash(&alg);
+       crypto_unregister_shashes(algs, 1 + (&__crc32c_le != &__crc32c_le_base));
 }
 
 subsys_initcall(crc32c_mod_init);

diff --git a/lib/crc32.c b/lib/crc32.c
index a54ba87b70730bd1856e8a99ab55695777bdfad3..ff587fee3893dd9732ddec328cdd216a078e7ee0 100644 (file)
--- a/lib/crc32.c
+++ b/lib/crc32.c
@@ -208,6 +208,8 @@ u32 __pure crc32_le_base(u32, unsigned char const *, size_t) __alias(crc32_le);
 EXPORT_SYMBOL(crc32_le_base);
 
 u32 __pure __crc32c_le_base(u32, unsigned char const *, size_t) __alias(__crc32c_le);
+EXPORT_SYMBOL(__crc32c_le_base);
+
 u32 __pure crc32_be_base(u32, unsigned char const *, size_t) __alias(crc32_be);
 
 /*