]> git.ipfire.org Git - thirdparty/linux.git/commitdiff
s390/crc32: expose CRC32 functions through lib
authorEric Biggers <ebiggers@google.com>
Mon, 2 Dec 2024 01:08:34 +0000 (17:08 -0800)
committerEric Biggers <ebiggers@google.com>
Mon, 2 Dec 2024 01:23:01 +0000 (17:23 -0800)
Move the s390 CRC32 assembly code into the lib directory and wire it up
to the library interface.  This allows it to be used without going
through the crypto API.  It remains usable via the crypto API too via
the shash algorithms that use the library interface.  Thus all the
arch-specific "shash" code becomes unnecessary and is removed.

Note: to see the diff from arch/s390/crypto/crc32-vx.c to
arch/s390/lib/crc32-glue.c, view this commit with 'git show -M10'.

Reviewed-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20241202010844.144356-10-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@google.com>
arch/s390/Kconfig
arch/s390/configs/debug_defconfig
arch/s390/configs/defconfig
arch/s390/crypto/Kconfig
arch/s390/crypto/Makefile
arch/s390/crypto/crc32-vx.c [deleted file]
arch/s390/lib/Makefile
arch/s390/lib/crc32-glue.c [new file with mode: 0644]
arch/s390/lib/crc32-vx.h [moved from arch/s390/crypto/crc32-vx.h with 100% similarity]
arch/s390/lib/crc32be-vx.c [moved from arch/s390/crypto/crc32be-vx.c with 100% similarity]
arch/s390/lib/crc32le-vx.c [moved from arch/s390/crypto/crc32le-vx.c with 100% similarity]

index 0077969170e8b4ca4c99e87ec75f6ea94f3e8e00..d272230cc885259504dde3eef774881da16af02b 100644 (file)
@@ -72,6 +72,7 @@ config S390
        select ARCH_ENABLE_MEMORY_HOTPLUG if SPARSEMEM
        select ARCH_ENABLE_MEMORY_HOTREMOVE
        select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2
+       select ARCH_HAS_CRC32
        select ARCH_HAS_CURRENT_STACK_POINTER
        select ARCH_HAS_DEBUG_VIRTUAL
        select ARCH_HAS_DEBUG_VM_PGTABLE
index d8d227ab82de0e78bb9f1766d3d547f702e47881..f2a149b4fafd0e45f9d06ce569cc0ffe474ac920 100644 (file)
@@ -795,7 +795,6 @@ CONFIG_CRYPTO_USER_API_HASH=m
 CONFIG_CRYPTO_USER_API_SKCIPHER=m
 CONFIG_CRYPTO_USER_API_RNG=m
 CONFIG_CRYPTO_USER_API_AEAD=m
-CONFIG_CRYPTO_CRC32_S390=y
 CONFIG_CRYPTO_SHA512_S390=m
 CONFIG_CRYPTO_SHA1_S390=m
 CONFIG_CRYPTO_SHA256_S390=m
index 6c2f2bb4fbf8f676e847818cd81bbfaa34af826a..d521aabc31d7d0d9c0b9c5388564a66099f375bf 100644 (file)
@@ -782,7 +782,6 @@ CONFIG_CRYPTO_USER_API_HASH=m
 CONFIG_CRYPTO_USER_API_SKCIPHER=m
 CONFIG_CRYPTO_USER_API_RNG=m
 CONFIG_CRYPTO_USER_API_AEAD=m
-CONFIG_CRYPTO_CRC32_S390=y
 CONFIG_CRYPTO_SHA512_S390=m
 CONFIG_CRYPTO_SHA1_S390=m
 CONFIG_CRYPTO_SHA256_S390=m
index d3eb3a23369321868f76cd422883dc18855577d0..b760232537f1c6942c6a8d6b70ce04aa013fd8be 100644 (file)
@@ -2,18 +2,6 @@
 
 menu "Accelerated Cryptographic Algorithms for CPU (s390)"
 
-config CRYPTO_CRC32_S390
-       tristate "CRC32c and CRC32"
-       depends on S390
-       select CRYPTO_HASH
-       select CRC32
-       help
-         CRC32c and CRC32 CRC algorithms
-
-         Architecture: s390
-
-         It is available with IBM z13 or later.
-
 config CRYPTO_SHA512_S390
        tristate "Hash functions: SHA-384 and SHA-512"
        depends on S390
index a0cb96937c3de249cd66a17d6e17a68968a72e3e..14dafadbcbed4718af6abf5e4a93677830564440 100644 (file)
@@ -14,9 +14,7 @@ obj-$(CONFIG_CRYPTO_PAES_S390) += paes_s390.o
 obj-$(CONFIG_CRYPTO_CHACHA_S390) += chacha_s390.o
 obj-$(CONFIG_S390_PRNG) += prng.o
 obj-$(CONFIG_CRYPTO_GHASH_S390) += ghash_s390.o
-obj-$(CONFIG_CRYPTO_CRC32_S390) += crc32-vx_s390.o
 obj-$(CONFIG_CRYPTO_HMAC_S390) += hmac_s390.o
 obj-y += arch_random.o
 
-crc32-vx_s390-y := crc32-vx.o crc32le-vx.o crc32be-vx.o
 chacha_s390-y := chacha-glue.o chacha-s390.o
diff --git a/arch/s390/crypto/crc32-vx.c b/arch/s390/crypto/crc32-vx.c
deleted file mode 100644 (file)
index 89a1033..0000000
+++ /dev/null
@@ -1,306 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Crypto-API module for CRC-32 algorithms implemented with the
- * z/Architecture Vector Extension Facility.
- *
- * Copyright IBM Corp. 2015
- * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
- */
-#define KMSG_COMPONENT "crc32-vx"
-#define pr_fmt(fmt)    KMSG_COMPONENT ": " fmt
-
-#include <linux/module.h>
-#include <linux/cpufeature.h>
-#include <linux/crc32.h>
-#include <crypto/internal/hash.h>
-#include <asm/fpu.h>
-#include "crc32-vx.h"
-
-#define CRC32_BLOCK_SIZE       1
-#define CRC32_DIGEST_SIZE      4
-
-#define VX_MIN_LEN             64
-#define VX_ALIGNMENT           16L
-#define VX_ALIGN_MASK          (VX_ALIGNMENT - 1)
-
-struct crc_ctx {
-       u32 key;
-};
-
-struct crc_desc_ctx {
-       u32 crc;
-};
-
-/*
- * DEFINE_CRC32_VX() - Define a CRC-32 function using the vector extension
- *
- * Creates a function to perform a particular CRC-32 computation. Depending
- * on the message buffer, the hardware-accelerated or software implementation
- * is used.   Note that the message buffer is aligned to improve fetch
- * operations of VECTOR LOAD MULTIPLE instructions.
- *
- */
-#define DEFINE_CRC32_VX(___fname, ___crc32_vx, ___crc32_sw)                \
-       static u32 __pure ___fname(u32 crc,                                 \
-                               unsigned char const *data, size_t datalen)  \
-       {                                                                   \
-               unsigned long prealign, aligned, remaining;                 \
-               DECLARE_KERNEL_FPU_ONSTACK16(vxstate);                      \
-                                                                           \
-               if (datalen < VX_MIN_LEN + VX_ALIGN_MASK)                   \
-                       return ___crc32_sw(crc, data, datalen);             \
-                                                                           \
-               if ((unsigned long)data & VX_ALIGN_MASK) {                  \
-                       prealign = VX_ALIGNMENT -                           \
-                                 ((unsigned long)data & VX_ALIGN_MASK);    \
-                       datalen -= prealign;                                \
-                       crc = ___crc32_sw(crc, data, prealign);             \
-                       data = (void *)((unsigned long)data + prealign);    \
-               }                                                           \
-                                                                           \
-               aligned = datalen & ~VX_ALIGN_MASK;                         \
-               remaining = datalen & VX_ALIGN_MASK;                        \
-                                                                           \
-               kernel_fpu_begin(&vxstate, KERNEL_VXR_LOW);                 \
-               crc = ___crc32_vx(crc, data, aligned);                      \
-               kernel_fpu_end(&vxstate, KERNEL_VXR_LOW);                   \
-                                                                           \
-               if (remaining)                                              \
-                       crc = ___crc32_sw(crc, data + aligned, remaining);  \
-                                                                           \
-               return crc;                                                 \
-       }
-
-DEFINE_CRC32_VX(crc32_le_vx, crc32_le_vgfm_16, crc32_le)
-DEFINE_CRC32_VX(crc32_be_vx, crc32_be_vgfm_16, crc32_be)
-DEFINE_CRC32_VX(crc32c_le_vx, crc32c_le_vgfm_16, __crc32c_le)
-
-
-static int crc32_vx_cra_init_zero(struct crypto_tfm *tfm)
-{
-       struct crc_ctx *mctx = crypto_tfm_ctx(tfm);
-
-       mctx->key = 0;
-       return 0;
-}
-
-static int crc32_vx_cra_init_invert(struct crypto_tfm *tfm)
-{
-       struct crc_ctx *mctx = crypto_tfm_ctx(tfm);
-
-       mctx->key = ~0;
-       return 0;
-}
-
-static int crc32_vx_init(struct shash_desc *desc)
-{
-       struct crc_ctx *mctx = crypto_shash_ctx(desc->tfm);
-       struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
-
-       ctx->crc = mctx->key;
-       return 0;
-}
-
-static int crc32_vx_setkey(struct crypto_shash *tfm, const u8 *newkey,
-                          unsigned int newkeylen)
-{
-       struct crc_ctx *mctx = crypto_shash_ctx(tfm);
-
-       if (newkeylen != sizeof(mctx->key))
-               return -EINVAL;
-       mctx->key = le32_to_cpu(*(__le32 *)newkey);
-       return 0;
-}
-
-static int crc32be_vx_setkey(struct crypto_shash *tfm, const u8 *newkey,
-                            unsigned int newkeylen)
-{
-       struct crc_ctx *mctx = crypto_shash_ctx(tfm);
-
-       if (newkeylen != sizeof(mctx->key))
-               return -EINVAL;
-       mctx->key = be32_to_cpu(*(__be32 *)newkey);
-       return 0;
-}
-
-static int crc32le_vx_final(struct shash_desc *desc, u8 *out)
-{
-       struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
-
-       *(__le32 *)out = cpu_to_le32p(&ctx->crc);
-       return 0;
-}
-
-static int crc32be_vx_final(struct shash_desc *desc, u8 *out)
-{
-       struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
-
-       *(__be32 *)out = cpu_to_be32p(&ctx->crc);
-       return 0;
-}
-
-static int crc32c_vx_final(struct shash_desc *desc, u8 *out)
-{
-       struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
-
-       /*
-        * Perform a final XOR with 0xFFFFFFFF to be in sync
-        * with the generic crc32c shash implementation.
-        */
-       *(__le32 *)out = ~cpu_to_le32p(&ctx->crc);
-       return 0;
-}
-
-static int __crc32le_vx_finup(u32 *crc, const u8 *data, unsigned int len,
-                             u8 *out)
-{
-       *(__le32 *)out = cpu_to_le32(crc32_le_vx(*crc, data, len));
-       return 0;
-}
-
-static int __crc32be_vx_finup(u32 *crc, const u8 *data, unsigned int len,
-                             u8 *out)
-{
-       *(__be32 *)out = cpu_to_be32(crc32_be_vx(*crc, data, len));
-       return 0;
-}
-
-static int __crc32c_vx_finup(u32 *crc, const u8 *data, unsigned int len,
-                            u8 *out)
-{
-       /*
-        * Perform a final XOR with 0xFFFFFFFF to be in sync
-        * with the generic crc32c shash implementation.
-        */
-       *(__le32 *)out = ~cpu_to_le32(crc32c_le_vx(*crc, data, len));
-       return 0;
-}
-
-
-#define CRC32_VX_FINUP(alg, func)                                            \
-       static int alg ## _vx_finup(struct shash_desc *desc, const u8 *data,  \
-                                  unsigned int datalen, u8 *out)             \
-       {                                                                     \
-               return __ ## alg ## _vx_finup(shash_desc_ctx(desc),           \
-                                             data, datalen, out);            \
-       }
-
-CRC32_VX_FINUP(crc32le, crc32_le_vx)
-CRC32_VX_FINUP(crc32be, crc32_be_vx)
-CRC32_VX_FINUP(crc32c, crc32c_le_vx)
-
-#define CRC32_VX_DIGEST(alg, func)                                           \
-       static int alg ## _vx_digest(struct shash_desc *desc, const u8 *data, \
-                                    unsigned int len, u8 *out)               \
-       {                                                                     \
-               return __ ## alg ## _vx_finup(crypto_shash_ctx(desc->tfm),    \
-                                             data, len, out);                \
-       }
-
-CRC32_VX_DIGEST(crc32le, crc32_le_vx)
-CRC32_VX_DIGEST(crc32be, crc32_be_vx)
-CRC32_VX_DIGEST(crc32c, crc32c_le_vx)
-
-#define CRC32_VX_UPDATE(alg, func)                                           \
-       static int alg ## _vx_update(struct shash_desc *desc, const u8 *data, \
-                                    unsigned int datalen)                    \
-       {                                                                     \
-               struct crc_desc_ctx *ctx = shash_desc_ctx(desc);              \
-               ctx->crc = func(ctx->crc, data, datalen);                     \
-               return 0;                                                     \
-       }
-
-CRC32_VX_UPDATE(crc32le, crc32_le_vx)
-CRC32_VX_UPDATE(crc32be, crc32_be_vx)
-CRC32_VX_UPDATE(crc32c, crc32c_le_vx)
-
-
-static struct shash_alg crc32_vx_algs[] = {
-       /* CRC-32 LE */
-       {
-               .init           =       crc32_vx_init,
-               .setkey         =       crc32_vx_setkey,
-               .update         =       crc32le_vx_update,
-               .final          =       crc32le_vx_final,
-               .finup          =       crc32le_vx_finup,
-               .digest         =       crc32le_vx_digest,
-               .descsize       =       sizeof(struct crc_desc_ctx),
-               .digestsize     =       CRC32_DIGEST_SIZE,
-               .base           =       {
-                       .cra_name        = "crc32",
-                       .cra_driver_name = "crc32-vx",
-                       .cra_priority    = 200,
-                       .cra_flags       = CRYPTO_ALG_OPTIONAL_KEY,
-                       .cra_blocksize   = CRC32_BLOCK_SIZE,
-                       .cra_ctxsize     = sizeof(struct crc_ctx),
-                       .cra_module      = THIS_MODULE,
-                       .cra_init        = crc32_vx_cra_init_zero,
-               },
-       },
-       /* CRC-32 BE */
-       {
-               .init           =       crc32_vx_init,
-               .setkey         =       crc32be_vx_setkey,
-               .update         =       crc32be_vx_update,
-               .final          =       crc32be_vx_final,
-               .finup          =       crc32be_vx_finup,
-               .digest         =       crc32be_vx_digest,
-               .descsize       =       sizeof(struct crc_desc_ctx),
-               .digestsize     =       CRC32_DIGEST_SIZE,
-               .base           =       {
-                       .cra_name        = "crc32be",
-                       .cra_driver_name = "crc32be-vx",
-                       .cra_priority    = 200,
-                       .cra_flags       = CRYPTO_ALG_OPTIONAL_KEY,
-                       .cra_blocksize   = CRC32_BLOCK_SIZE,
-                       .cra_ctxsize     = sizeof(struct crc_ctx),
-                       .cra_module      = THIS_MODULE,
-                       .cra_init        = crc32_vx_cra_init_zero,
-               },
-       },
-       /* CRC-32C LE */
-       {
-               .init           =       crc32_vx_init,
-               .setkey         =       crc32_vx_setkey,
-               .update         =       crc32c_vx_update,
-               .final          =       crc32c_vx_final,
-               .finup          =       crc32c_vx_finup,
-               .digest         =       crc32c_vx_digest,
-               .descsize       =       sizeof(struct crc_desc_ctx),
-               .digestsize     =       CRC32_DIGEST_SIZE,
-               .base           =       {
-                       .cra_name        = "crc32c",
-                       .cra_driver_name = "crc32c-vx",
-                       .cra_priority    = 200,
-                       .cra_flags       = CRYPTO_ALG_OPTIONAL_KEY,
-                       .cra_blocksize   = CRC32_BLOCK_SIZE,
-                       .cra_ctxsize     = sizeof(struct crc_ctx),
-                       .cra_module      = THIS_MODULE,
-                       .cra_init        = crc32_vx_cra_init_invert,
-               },
-       },
-};
-
-
-static int __init crc_vx_mod_init(void)
-{
-       return crypto_register_shashes(crc32_vx_algs,
-                                      ARRAY_SIZE(crc32_vx_algs));
-}
-
-static void __exit crc_vx_mod_exit(void)
-{
-       crypto_unregister_shashes(crc32_vx_algs, ARRAY_SIZE(crc32_vx_algs));
-}
-
-module_cpu_feature_match(S390_CPU_FEATURE_VXRS, crc_vx_mod_init);
-module_exit(crc_vx_mod_exit);
-
-MODULE_AUTHOR("Hendrik Brueckner <brueckner@linux.vnet.ibm.com>");
-MODULE_DESCRIPTION("CRC-32 algorithms using z/Architecture Vector Extension Facility");
-MODULE_LICENSE("GPL");
-
-MODULE_ALIAS_CRYPTO("crc32");
-MODULE_ALIAS_CRYPTO("crc32-vx");
-MODULE_ALIAS_CRYPTO("crc32c");
-MODULE_ALIAS_CRYPTO("crc32c-vx");
index f43f897d3fc027ad2f5cbf52ba2abc343b6e31ea..14bbfe50033c7c0591bf699608cb410975c31621 100644 (file)
@@ -24,3 +24,6 @@ obj-$(CONFIG_S390_MODULES_SANITY_TEST_HELPERS) += test_modules_helpers.o
 lib-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
 
 obj-$(CONFIG_EXPOLINE_EXTERN) += expoline.o
+
+obj-$(CONFIG_CRC32_ARCH) += crc32-s390.o
+crc32-s390-y := crc32-glue.o crc32le-vx.o crc32be-vx.o
diff --git a/arch/s390/lib/crc32-glue.c b/arch/s390/lib/crc32-glue.c
new file mode 100644 (file)
index 0000000..137080e
--- /dev/null
@@ -0,0 +1,92 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * CRC-32 implemented with the z/Architecture Vector Extension Facility.
+ *
+ * Copyright IBM Corp. 2015
+ * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
+ */
+#define KMSG_COMPONENT "crc32-vx"
+#define pr_fmt(fmt)    KMSG_COMPONENT ": " fmt
+
+#include <linux/module.h>
+#include <linux/cpufeature.h>
+#include <linux/crc32.h>
+#include <asm/fpu.h>
+#include "crc32-vx.h"
+
+#define VX_MIN_LEN             64
+#define VX_ALIGNMENT           16L
+#define VX_ALIGN_MASK          (VX_ALIGNMENT - 1)
+
+static DEFINE_STATIC_KEY_FALSE(have_vxrs);
+
+/*
+ * DEFINE_CRC32_VX() - Define a CRC-32 function using the vector extension
+ *
+ * Creates a function to perform a particular CRC-32 computation. Depending
+ * on the message buffer, the hardware-accelerated or software implementation
+ * is used.   Note that the message buffer is aligned to improve fetch
+ * operations of VECTOR LOAD MULTIPLE instructions.
+ */
+#define DEFINE_CRC32_VX(___fname, ___crc32_vx, ___crc32_sw)                \
+       u32 ___fname(u32 crc, const u8 *data, size_t datalen)               \
+       {                                                                   \
+               unsigned long prealign, aligned, remaining;                 \
+               DECLARE_KERNEL_FPU_ONSTACK16(vxstate);                      \
+                                                                           \
+               if (datalen < VX_MIN_LEN + VX_ALIGN_MASK ||                 \
+                   !static_branch_likely(&have_vxrs))                      \
+                       return ___crc32_sw(crc, data, datalen);             \
+                                                                           \
+               if ((unsigned long)data & VX_ALIGN_MASK) {                  \
+                       prealign = VX_ALIGNMENT -                           \
+                                 ((unsigned long)data & VX_ALIGN_MASK);    \
+                       datalen -= prealign;                                \
+                       crc = ___crc32_sw(crc, data, prealign);             \
+                       data = (void *)((unsigned long)data + prealign);    \
+               }                                                           \
+                                                                           \
+               aligned = datalen & ~VX_ALIGN_MASK;                         \
+               remaining = datalen & VX_ALIGN_MASK;                        \
+                                                                           \
+               kernel_fpu_begin(&vxstate, KERNEL_VXR_LOW);                 \
+               crc = ___crc32_vx(crc, data, aligned);                      \
+               kernel_fpu_end(&vxstate, KERNEL_VXR_LOW);                   \
+                                                                           \
+               if (remaining)                                              \
+                       crc = ___crc32_sw(crc, data + aligned, remaining);  \
+                                                                           \
+               return crc;                                                 \
+       }                                                                   \
+       EXPORT_SYMBOL(___fname);
+
+DEFINE_CRC32_VX(crc32_le_arch, crc32_le_vgfm_16, crc32_le_base)
+DEFINE_CRC32_VX(crc32_be_arch, crc32_be_vgfm_16, crc32_be_base)
+DEFINE_CRC32_VX(crc32c_le_arch, crc32c_le_vgfm_16, crc32c_le_base)
+
+static int __init crc32_s390_init(void)
+{
+       if (cpu_have_feature(S390_CPU_FEATURE_VXRS))
+               static_branch_enable(&have_vxrs);
+       return 0;
+}
+arch_initcall(crc32_s390_init);
+
+static void __exit crc32_s390_exit(void)
+{
+}
+module_exit(crc32_s390_exit);
+
+u32 crc32_optimizations(void)
+{
+       if (static_key_enabled(&have_vxrs))
+               return CRC32_LE_OPTIMIZATION |
+                      CRC32_BE_OPTIMIZATION |
+                      CRC32C_OPTIMIZATION;
+       return 0;
+}
+EXPORT_SYMBOL(crc32_optimizations);
+
+MODULE_AUTHOR("Hendrik Brueckner <brueckner@linux.vnet.ibm.com>");
+MODULE_DESCRIPTION("CRC-32 algorithms using z/Architecture Vector Extension Facility");
+MODULE_LICENSE("GPL");