lib/crypto: riscv/sha512: Migrate optimized SHA-512 code to library

Instead of exposing the riscv-optimized SHA-512 code via riscv-specific
crypto_shash algorithms, instead just implement the sha512_blocks()
library function.  This is much simpler, it makes the SHA-512 (and
SHA-384) library functions be riscv-optimized, and it fixes the
longstanding issue where the riscv-optimized SHA-512 code was disabled
by default.  SHA-512 still remains available through crypto_shash, but
individual architectures no longer need to handle it.

To match sha512_blocks(), change the type of the nblocks parameter of
the assembly function from int to size_t.  The assembly function
actually already treated it as size_t.

Note: to see the diff from arch/riscv/crypto/sha512-riscv64-glue.c to
lib/crypto/riscv/sha512.h, view this commit with 'git show -M10'.

Acked-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20250630160320.2888-12-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@kernel.org>

diff --git a/arch/riscv/crypto/Kconfig b/arch/riscv/crypto/Kconfig
index 53e4e1eacf554ffb60d2d398e1d2f086507cf242..a75d6325607b4c46184cc61a40b76121a44ad320 100644 (file)
--- a/arch/riscv/crypto/Kconfig
+++ b/arch/riscv/crypto/Kconfig
@@ -28,18 +28,6 @@ config CRYPTO_GHASH_RISCV64
          Architecture: riscv64 using:
          - Zvkg vector crypto extension
 
-config CRYPTO_SHA512_RISCV64
-       tristate "Hash functions: SHA-384 and SHA-512"
-       depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO
-       select CRYPTO_LIB_SHA512
-       select CRYPTO_SHA512
-       help
-         SHA-384 and SHA-512 secure hash algorithm (FIPS 180)
-
-         Architecture: riscv64 using:
-         - Zvknhb vector crypto extension
-         - Zvkb vector crypto extension
-
 config CRYPTO_SM3_RISCV64
        tristate "Hash functions: SM3 (ShangMi 3)"
        depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO

diff --git a/arch/riscv/crypto/Makefile b/arch/riscv/crypto/Makefile
index e10e8257734e3f4a7a3fa7b0679daf822fe3008c..183495a95cc0e1fc979e101c90388a092b576fc2 100644 (file)
--- a/arch/riscv/crypto/Makefile
+++ b/arch/riscv/crypto/Makefile
@@ -7,9 +7,6 @@ aes-riscv64-y := aes-riscv64-glue.o aes-riscv64-zvkned.o \
 obj-$(CONFIG_CRYPTO_GHASH_RISCV64) += ghash-riscv64.o
 ghash-riscv64-y := ghash-riscv64-glue.o ghash-riscv64-zvkg.o
 
-obj-$(CONFIG_CRYPTO_SHA512_RISCV64) += sha512-riscv64.o
-sha512-riscv64-y := sha512-riscv64-glue.o sha512-riscv64-zvknhb-zvkb.o
-
 obj-$(CONFIG_CRYPTO_SM3_RISCV64) += sm3-riscv64.o
 sm3-riscv64-y := sm3-riscv64-glue.o sm3-riscv64-zvksh-zvkb.o
 

diff --git a/arch/riscv/crypto/sha512-riscv64-glue.c b/arch/riscv/crypto/sha512-riscv64-glue.c
deleted file mode 100644 (file)
index b3dbc71..0000000
--- a/arch/riscv/crypto/sha512-riscv64-glue.c
+++ /dev/null
@@ -1,130 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * SHA-512 and SHA-384 using the RISC-V vector crypto extensions
- *
- * Copyright (C) 2023 VRULL GmbH
- * Author: Heiko Stuebner <heiko.stuebner@vrull.eu>
- *
- * Copyright (C) 2023 SiFive, Inc.
- * Author: Jerry Shih <jerry.shih@sifive.com>
- */
-
-#include <asm/simd.h>
-#include <asm/vector.h>
-#include <crypto/internal/hash.h>
-#include <crypto/internal/simd.h>
-#include <crypto/sha512_base.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-
-/*
- * Note: the asm function only uses the 'state' field of struct sha512_state.
- * It is assumed to be the first field.
- */
-asmlinkage void sha512_transform_zvknhb_zvkb(
-       struct sha512_state *state, const u8 *data, int num_blocks);
-
-static void sha512_block(struct sha512_state *state, const u8 *data,
-                        int num_blocks)
-{
-       /*
-        * Ensure struct sha512_state begins directly with the SHA-512
-        * 512-bit internal state, as this is what the asm function expects.
-        */
-       BUILD_BUG_ON(offsetof(struct sha512_state, state) != 0);
-
-       if (crypto_simd_usable()) {
-               kernel_vector_begin();
-               sha512_transform_zvknhb_zvkb(state, data, num_blocks);
-               kernel_vector_end();
-       } else {
-               struct __sha512_ctx ctx = {};
-
-               static_assert(sizeof(ctx.state) == sizeof(state->state));
-               memcpy(&ctx.state, state->state, sizeof(ctx.state));
-               __sha512_update(&ctx, data,
-                               (size_t)num_blocks * SHA512_BLOCK_SIZE);
-               memcpy(state->state, &ctx.state, sizeof(state->state));
-       }
-}
-
-static int riscv64_sha512_update(struct shash_desc *desc, const u8 *data,
-                                unsigned int len)
-{
-       return sha512_base_do_update_blocks(desc, data, len, sha512_block);
-}
-
-static int riscv64_sha512_finup(struct shash_desc *desc, const u8 *data,
-                               unsigned int len, u8 *out)
-{
-       sha512_base_do_finup(desc, data, len, sha512_block);
-       return sha512_base_finish(desc, out);
-}
-
-static int riscv64_sha512_digest(struct shash_desc *desc, const u8 *data,
-                                unsigned int len, u8 *out)
-{
-       return sha512_base_init(desc) ?:
-              riscv64_sha512_finup(desc, data, len, out);
-}
-
-static struct shash_alg riscv64_sha512_algs[] = {
-       {
-               .init = sha512_base_init,
-               .update = riscv64_sha512_update,
-               .finup = riscv64_sha512_finup,
-               .digest = riscv64_sha512_digest,
-               .descsize = SHA512_STATE_SIZE,
-               .digestsize = SHA512_DIGEST_SIZE,
-               .base = {
-                       .cra_blocksize = SHA512_BLOCK_SIZE,
-                       .cra_priority = 300,
-                       .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
-                                    CRYPTO_AHASH_ALG_FINUP_MAX,
-                       .cra_name = "sha512",
-                       .cra_driver_name = "sha512-riscv64-zvknhb-zvkb",
-                       .cra_module = THIS_MODULE,
-               },
-       }, {
-               .init = sha384_base_init,
-               .update = riscv64_sha512_update,
-               .finup = riscv64_sha512_finup,
-               .descsize = SHA512_STATE_SIZE,
-               .digestsize = SHA384_DIGEST_SIZE,
-               .base = {
-                       .cra_blocksize = SHA384_BLOCK_SIZE,
-                       .cra_priority = 300,
-                       .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
-                                    CRYPTO_AHASH_ALG_FINUP_MAX,
-                       .cra_name = "sha384",
-                       .cra_driver_name = "sha384-riscv64-zvknhb-zvkb",
-                       .cra_module = THIS_MODULE,
-               },
-       },
-};
-
-static int __init riscv64_sha512_mod_init(void)
-{
-       if (riscv_isa_extension_available(NULL, ZVKNHB) &&
-           riscv_isa_extension_available(NULL, ZVKB) &&
-           riscv_vector_vlen() >= 128)
-               return crypto_register_shashes(riscv64_sha512_algs,
-                                              ARRAY_SIZE(riscv64_sha512_algs));
-
-       return -ENODEV;
-}
-
-static void __exit riscv64_sha512_mod_exit(void)
-{
-       crypto_unregister_shashes(riscv64_sha512_algs,
-                                 ARRAY_SIZE(riscv64_sha512_algs));
-}
-
-module_init(riscv64_sha512_mod_init);
-module_exit(riscv64_sha512_mod_exit);
-
-MODULE_DESCRIPTION("SHA-512 (RISC-V accelerated)");
-MODULE_AUTHOR("Heiko Stuebner <heiko.stuebner@vrull.eu>");
-MODULE_LICENSE("GPL");
-MODULE_ALIAS_CRYPTO("sha512");
-MODULE_ALIAS_CRYPTO("sha384");

diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig
index 303ea15e3e900993acafa2bfb7a13ca4824cac93..656f2c08c46cd189f4bd8c85ceaf3fc8348ad785 100644 (file)
--- a/lib/crypto/Kconfig
+++ b/lib/crypto/Kconfig
@@ -180,6 +180,7 @@ config CRYPTO_LIB_SHA512_ARCH
        default y if ARM && !CPU_V7M
        default y if ARM64
        default y if MIPS && CPU_CAVIUM_OCTEON
+       default y if RISCV && 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO
 
 config CRYPTO_LIB_SM3
        tristate

diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile
index 22269ab06d70089488ff6c95ce5313d9c6db3c5c..bc42464a279b8862022465f6830e5814f8716eb4 100644 (file)
--- a/lib/crypto/Makefile
+++ b/lib/crypto/Makefile
@@ -92,6 +92,8 @@ $(obj)/arm64/sha512-core.S: $(src)/../../arch/arm64/lib/crypto/sha2-armv8.pl
 clean-files += arm64/sha512-core.S
 libsha512-$(CONFIG_KERNEL_MODE_NEON) += arm64/sha512-ce-core.o
 endif
+
+libsha512-$(CONFIG_RISCV) += riscv/sha512-riscv64-zvknhb-zvkb.o
 endif # CONFIG_CRYPTO_LIB_SHA512_ARCH
 
 obj-$(CONFIG_MPILIB) += mpi/

diff --git a/arch/riscv/crypto/sha512-riscv64-zvknhb-zvkb.S b/lib/crypto/riscv/sha512-riscv64-zvknhb-zvkb.S
similarity index 98%
rename from arch/riscv/crypto/sha512-riscv64-zvknhb-zvkb.S
rename to lib/crypto/riscv/sha512-riscv64-zvknhb-zvkb.S
index 89f4a10d12dd678100b7f057ff3fad2b22db334e..b41eebf605462c160b4f292c5cb240c0f4ba09b7 100644 (file)
--- a/arch/riscv/crypto/sha512-riscv64-zvknhb-zvkb.S
+++ b/lib/crypto/riscv/sha512-riscv64-zvknhb-zvkb.S
@@ -93,8 +93,8 @@
        sha512_4rounds  \last, W3, W0, W1, W2
 .endm
 
-// void sha512_transform_zvknhb_zvkb(u64 state[8], const u8 *data,
-//                                  int num_blocks);
+// void sha512_transform_zvknhb_zvkb(struct sha512_block_state *state,
+//                                  const u8 *data, size_t nblocks);
 SYM_FUNC_START(sha512_transform_zvknhb_zvkb)
 
        // Setup mask for the vmerge to replace the first word (idx==0) in

diff --git a/lib/crypto/riscv/sha512.h b/lib/crypto/riscv/sha512.h
new file mode 100644 (file)
index 0000000..9d0abed
--- /dev/null
+++ b/lib/crypto/riscv/sha512.h
@@ -0,0 +1,41 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * SHA-512 and SHA-384 using the RISC-V vector crypto extensions
+ *
+ * Copyright (C) 2023 VRULL GmbH
+ * Author: Heiko Stuebner <heiko.stuebner@vrull.eu>
+ *
+ * Copyright (C) 2023 SiFive, Inc.
+ * Author: Jerry Shih <jerry.shih@sifive.com>
+ */
+
+#include <asm/simd.h>
+#include <asm/vector.h>
+#include <crypto/internal/simd.h>
+
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_extensions);
+
+asmlinkage void sha512_transform_zvknhb_zvkb(struct sha512_block_state *state,
+                                            const u8 *data, size_t nblocks);
+
+static void sha512_blocks(struct sha512_block_state *state,
+                         const u8 *data, size_t nblocks)
+{
+       if (static_branch_likely(&have_extensions) &&
+           likely(crypto_simd_usable())) {
+               kernel_vector_begin();
+               sha512_transform_zvknhb_zvkb(state, data, nblocks);
+               kernel_vector_end();
+       } else {
+               sha512_blocks_generic(state, data, nblocks);
+       }
+}
+
+#define sha512_mod_init_arch sha512_mod_init_arch
+static inline void sha512_mod_init_arch(void)
+{
+       if (riscv_isa_extension_available(NULL, ZVKNHB) &&
+           riscv_isa_extension_available(NULL, ZVKB) &&
+           riscv_vector_vlen() >= 128)
+               static_branch_enable(&have_extensions);
+}