lib/crypto: riscv/sm3: Migrate optimized code into library

Instead of exposing the riscv-optimized SM3 code via a riscv-specific
crypto_shash algorithm, instead just implement the sm3_blocks() library
function.  This is much simpler, it makes the SM3 library functions be
riscv-optimized, and it fixes the longstanding issue where the
riscv-optimized SM3 code was disabled by default.  SM3 still remains
available through crypto_shash, but individual architectures no longer
need to handle it.

Tweak the prototype of sm3_transform_zvksh_zvkb() to match what the
library expects, including changing the block count to size_t.
Note that the assembly code already treated it as size_t.

Note: to see the diff from arch/riscv/crypto/sm3-riscv64-glue.c to
lib/crypto/riscv/sm3.h, view this commit with 'git show -M10'.

Acked-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20260321040935.410034-9-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@kernel.org>

diff --git a/arch/riscv/crypto/Kconfig b/arch/riscv/crypto/Kconfig
index c208f54afbcd781d6d820bdb8fe8fdce19dcc522..6905232ddb03cb512091ca803d739b1b55fe003f 100644 (file)
--- a/arch/riscv/crypto/Kconfig
+++ b/arch/riscv/crypto/Kconfig
@@ -17,19 +17,6 @@ config CRYPTO_AES_RISCV64
          - Zvkb vector crypto extension (CTR)
          - Zvkg vector crypto extension (XTS)
 
-config CRYPTO_SM3_RISCV64
-       tristate "Hash functions: SM3 (ShangMi 3)"
-       depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \
-                  RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS
-       select CRYPTO_HASH
-       select CRYPTO_LIB_SM3
-       help
-         SM3 (ShangMi 3) secure hash function (OSCCA GM/T 0004-2012)
-
-         Architecture: riscv64 using:
-         - Zvksh vector crypto extension
-         - Zvkb vector crypto extension
-
 config CRYPTO_SM4_RISCV64
        tristate "Ciphers: SM4 (ShangMi 4)"
        depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \

diff --git a/arch/riscv/crypto/Makefile b/arch/riscv/crypto/Makefile
index 5c9ee1b876fa0ad7050e1956f2fa0d699dc9ac3e..8cf31db57fc488121f7a146435b2b9786c9f2126 100644 (file)
--- a/arch/riscv/crypto/Makefile
+++ b/arch/riscv/crypto/Makefile
@@ -4,8 +4,5 @@ obj-$(CONFIG_CRYPTO_AES_RISCV64) += aes-riscv64.o
 aes-riscv64-y := aes-riscv64-glue.o aes-riscv64-zvkned.o \
                 aes-riscv64-zvkned-zvbb-zvkg.o aes-riscv64-zvkned-zvkb.o
 
-obj-$(CONFIG_CRYPTO_SM3_RISCV64) += sm3-riscv64.o
-sm3-riscv64-y := sm3-riscv64-glue.o sm3-riscv64-zvksh-zvkb.o
-
 obj-$(CONFIG_CRYPTO_SM4_RISCV64) += sm4-riscv64.o
 sm4-riscv64-y := sm4-riscv64-glue.o sm4-riscv64-zvksed-zvkb.o

diff --git a/arch/riscv/crypto/sm3-riscv64-glue.c b/arch/riscv/crypto/sm3-riscv64-glue.c
deleted file mode 100644 (file)
index abdfe4a..0000000
--- a/arch/riscv/crypto/sm3-riscv64-glue.c
+++ /dev/null
@@ -1,97 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * SM3 using the RISC-V vector crypto extensions
- *
- * Copyright (C) 2023 VRULL GmbH
- * Author: Heiko Stuebner <heiko.stuebner@vrull.eu>
- *
- * Copyright (C) 2023 SiFive, Inc.
- * Author: Jerry Shih <jerry.shih@sifive.com>
- */
-
-#include <asm/simd.h>
-#include <asm/vector.h>
-#include <crypto/internal/hash.h>
-#include <crypto/internal/simd.h>
-#include <crypto/sm3.h>
-#include <crypto/sm3_base.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-
-/*
- * Note: the asm function only uses the 'state' field of struct sm3_state.
- * It is assumed to be the first field.
- */
-asmlinkage void sm3_transform_zvksh_zvkb(
-       struct sm3_state *state, const u8 *data, int num_blocks);
-
-static void sm3_block(struct sm3_state *state, const u8 *data,
-                     int num_blocks)
-{
-       /*
-        * Ensure struct sm3_state begins directly with the SM3
-        * 256-bit internal state, as this is what the asm function expects.
-        */
-       BUILD_BUG_ON(offsetof(struct sm3_state, state) != 0);
-
-       if (crypto_simd_usable()) {
-               kernel_vector_begin();
-               sm3_transform_zvksh_zvkb(state, data, num_blocks);
-               kernel_vector_end();
-       } else {
-               sm3_block_generic(state, data, num_blocks);
-       }
-}
-
-static int riscv64_sm3_update(struct shash_desc *desc, const u8 *data,
-                             unsigned int len)
-{
-       return sm3_base_do_update_blocks(desc, data, len, sm3_block);
-}
-
-static int riscv64_sm3_finup(struct shash_desc *desc, const u8 *data,
-                            unsigned int len, u8 *out)
-{
-       sm3_base_do_finup(desc, data, len, sm3_block);
-       return sm3_base_finish(desc, out);
-}
-
-static struct shash_alg riscv64_sm3_alg = {
-       .init = sm3_base_init,
-       .update = riscv64_sm3_update,
-       .finup = riscv64_sm3_finup,
-       .descsize = SM3_STATE_SIZE,
-       .digestsize = SM3_DIGEST_SIZE,
-       .base = {
-               .cra_blocksize = SM3_BLOCK_SIZE,
-               .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
-                            CRYPTO_AHASH_ALG_FINUP_MAX,
-               .cra_priority = 300,
-               .cra_name = "sm3",
-               .cra_driver_name = "sm3-riscv64-zvksh-zvkb",
-               .cra_module = THIS_MODULE,
-       },
-};
-
-static int __init riscv64_sm3_mod_init(void)
-{
-       if (riscv_isa_extension_available(NULL, ZVKSH) &&
-           riscv_isa_extension_available(NULL, ZVKB) &&
-           riscv_vector_vlen() >= 128)
-               return crypto_register_shash(&riscv64_sm3_alg);
-
-       return -ENODEV;
-}
-
-static void __exit riscv64_sm3_mod_exit(void)
-{
-       crypto_unregister_shash(&riscv64_sm3_alg);
-}
-
-module_init(riscv64_sm3_mod_init);
-module_exit(riscv64_sm3_mod_exit);
-
-MODULE_DESCRIPTION("SM3 (RISC-V accelerated)");
-MODULE_AUTHOR("Heiko Stuebner <heiko.stuebner@vrull.eu>");
-MODULE_LICENSE("GPL");
-MODULE_ALIAS_CRYPTO("sm3");

diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig
index c85956e443a2d33e0c530187ab8718db650f5d36..b209597de5ffa76a4240d64efa3f6eda581f3541 100644 (file)
--- a/lib/crypto/Kconfig
+++ b/lib/crypto/Kconfig
@@ -280,6 +280,8 @@ config CRYPTO_LIB_SM3_ARCH
        bool
        depends on CRYPTO_LIB_SM3 && !UML
        default y if ARM64
+       default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \
+                    RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS
 
 source "lib/crypto/tests/Kconfig"
 

diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile
index 9c2718012428090a30dfc5d31bec83d14d7aa664..ad8da7f3af7825dde54b666b85ea42035a091276 100644 (file)
--- a/lib/crypto/Makefile
+++ b/lib/crypto/Makefile
@@ -374,6 +374,7 @@ ifeq ($(CONFIG_CRYPTO_LIB_SM3_ARCH),y)
 CFLAGS_sm3.o += -I$(src)/$(SRCARCH)
 libsm3-$(CONFIG_ARM64) += arm64/sm3-ce-core.o \
                          arm64/sm3-neon-core.o
+libsm3-$(CONFIG_RISCV) += riscv/sm3-riscv64-zvksh-zvkb.o
 endif # CONFIG_CRYPTO_LIB_SM3_ARCH
 
 ################################################################################

diff --git a/arch/riscv/crypto/sm3-riscv64-zvksh-zvkb.S b/lib/crypto/riscv/sm3-riscv64-zvksh-zvkb.S
similarity index 97%
rename from arch/riscv/crypto/sm3-riscv64-zvksh-zvkb.S
rename to lib/crypto/riscv/sm3-riscv64-zvksh-zvkb.S
index 4fe754846f657edfc95e9703d3a77a4cb1b245e3..a1d4468b0485cf3088e6f475ea96dd2e3bd09a4e 100644 (file)
--- a/arch/riscv/crypto/sm3-riscv64-zvksh-zvkb.S
+++ b/lib/crypto/riscv/sm3-riscv64-zvksh-zvkb.S
@@ -80,7 +80,8 @@
        // For the next 8 rounds, w0 and w1 are swapped.
 .endm
 
-// void sm3_transform_zvksh_zvkb(u32 state[8], const u8 *data, int num_blocks);
+// void sm3_transform_zvksh_zvkb(struct sm3_block_state *state,
+//                              const u8 *data, size_t nblocks);
 SYM_FUNC_START(sm3_transform_zvksh_zvkb)
 
        // Load the state and endian-swap each 32-bit word.

diff --git a/lib/crypto/riscv/sm3.h b/lib/crypto/riscv/sm3.h
new file mode 100644 (file)
index 0000000..c1fbee7
--- /dev/null
+++ b/lib/crypto/riscv/sm3.h
@@ -0,0 +1,39 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * SM3 using the RISC-V vector crypto extensions
+ *
+ * Copyright (C) 2023 VRULL GmbH
+ * Author: Heiko Stuebner <heiko.stuebner@vrull.eu>
+ *
+ * Copyright (C) 2023 SiFive, Inc.
+ * Author: Jerry Shih <jerry.shih@sifive.com>
+ */
+
+#include <asm/simd.h>
+#include <asm/vector.h>
+
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_extensions);
+
+asmlinkage void sm3_transform_zvksh_zvkb(struct sm3_block_state *state,
+                                        const u8 *data, size_t nblocks);
+
+static void sm3_blocks(struct sm3_block_state *state,
+                      const u8 *data, size_t nblocks)
+{
+       if (static_branch_likely(&have_extensions) && likely(may_use_simd())) {
+               kernel_vector_begin();
+               sm3_transform_zvksh_zvkb(state, data, nblocks);
+               kernel_vector_end();
+       } else {
+               sm3_blocks_generic(state, data, nblocks);
+       }
+}
+
+#define sm3_mod_init_arch sm3_mod_init_arch
+static void sm3_mod_init_arch(void)
+{
+       if (riscv_isa_extension_available(NULL, ZVKSH) &&
+           riscv_isa_extension_available(NULL, ZVKB) &&
+           riscv_vector_vlen() >= 128)
+               static_branch_enable(&have_extensions);
+}