From: Eric Biggers Date: Sat, 21 Mar 2026 04:09:31 +0000 (-0700) Subject: lib/crypto: riscv/sm3: Migrate optimized code into library X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=5f6bbba5e9bb7f271557513d0ed77bb7b5a92698;p=thirdparty%2Fkernel%2Fstable.git lib/crypto: riscv/sm3: Migrate optimized code into library Instead of exposing the riscv-optimized SM3 code via a riscv-specific crypto_shash algorithm, instead just implement the sm3_blocks() library function. This is much simpler, it makes the SM3 library functions be riscv-optimized, and it fixes the longstanding issue where the riscv-optimized SM3 code was disabled by default. SM3 still remains available through crypto_shash, but individual architectures no longer need to handle it. Tweak the prototype of sm3_transform_zvksh_zvkb() to match what the library expects, including changing the block count to size_t. Note that the assembly code already treated it as size_t. Note: to see the diff from arch/riscv/crypto/sm3-riscv64-glue.c to lib/crypto/riscv/sm3.h, view this commit with 'git show -M10'. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260321040935.410034-9-ebiggers@kernel.org Signed-off-by: Eric Biggers --- diff --git a/arch/riscv/crypto/Kconfig b/arch/riscv/crypto/Kconfig index c208f54afbcd7..6905232ddb03c 100644 --- a/arch/riscv/crypto/Kconfig +++ b/arch/riscv/crypto/Kconfig @@ -17,19 +17,6 @@ config CRYPTO_AES_RISCV64 - Zvkb vector crypto extension (CTR) - Zvkg vector crypto extension (XTS) -config CRYPTO_SM3_RISCV64 - tristate "Hash functions: SM3 (ShangMi 3)" - depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ - RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS - select CRYPTO_HASH - select CRYPTO_LIB_SM3 - help - SM3 (ShangMi 3) secure hash function (OSCCA GM/T 0004-2012) - - Architecture: riscv64 using: - - Zvksh vector crypto extension - - Zvkb vector crypto extension - config CRYPTO_SM4_RISCV64 tristate "Ciphers: SM4 (ShangMi 4)" depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ diff --git a/arch/riscv/crypto/Makefile b/arch/riscv/crypto/Makefile index 5c9ee1b876fa0..8cf31db57fc48 100644 --- a/arch/riscv/crypto/Makefile +++ b/arch/riscv/crypto/Makefile @@ -4,8 +4,5 @@ obj-$(CONFIG_CRYPTO_AES_RISCV64) += aes-riscv64.o aes-riscv64-y := aes-riscv64-glue.o aes-riscv64-zvkned.o \ aes-riscv64-zvkned-zvbb-zvkg.o aes-riscv64-zvkned-zvkb.o -obj-$(CONFIG_CRYPTO_SM3_RISCV64) += sm3-riscv64.o -sm3-riscv64-y := sm3-riscv64-glue.o sm3-riscv64-zvksh-zvkb.o - obj-$(CONFIG_CRYPTO_SM4_RISCV64) += sm4-riscv64.o sm4-riscv64-y := sm4-riscv64-glue.o sm4-riscv64-zvksed-zvkb.o diff --git a/arch/riscv/crypto/sm3-riscv64-glue.c b/arch/riscv/crypto/sm3-riscv64-glue.c deleted file mode 100644 index abdfe4a63a27f..0000000000000 --- a/arch/riscv/crypto/sm3-riscv64-glue.c +++ /dev/null @@ -1,97 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * SM3 using the RISC-V vector crypto extensions - * - * Copyright (C) 2023 VRULL GmbH - * Author: Heiko Stuebner - * - * Copyright (C) 2023 SiFive, Inc. - * Author: Jerry Shih - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -/* - * Note: the asm function only uses the 'state' field of struct sm3_state. - * It is assumed to be the first field. - */ -asmlinkage void sm3_transform_zvksh_zvkb( - struct sm3_state *state, const u8 *data, int num_blocks); - -static void sm3_block(struct sm3_state *state, const u8 *data, - int num_blocks) -{ - /* - * Ensure struct sm3_state begins directly with the SM3 - * 256-bit internal state, as this is what the asm function expects. - */ - BUILD_BUG_ON(offsetof(struct sm3_state, state) != 0); - - if (crypto_simd_usable()) { - kernel_vector_begin(); - sm3_transform_zvksh_zvkb(state, data, num_blocks); - kernel_vector_end(); - } else { - sm3_block_generic(state, data, num_blocks); - } -} - -static int riscv64_sm3_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return sm3_base_do_update_blocks(desc, data, len, sm3_block); -} - -static int riscv64_sm3_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - sm3_base_do_finup(desc, data, len, sm3_block); - return sm3_base_finish(desc, out); -} - -static struct shash_alg riscv64_sm3_alg = { - .init = sm3_base_init, - .update = riscv64_sm3_update, - .finup = riscv64_sm3_finup, - .descsize = SM3_STATE_SIZE, - .digestsize = SM3_DIGEST_SIZE, - .base = { - .cra_blocksize = SM3_BLOCK_SIZE, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_priority = 300, - .cra_name = "sm3", - .cra_driver_name = "sm3-riscv64-zvksh-zvkb", - .cra_module = THIS_MODULE, - }, -}; - -static int __init riscv64_sm3_mod_init(void) -{ - if (riscv_isa_extension_available(NULL, ZVKSH) && - riscv_isa_extension_available(NULL, ZVKB) && - riscv_vector_vlen() >= 128) - return crypto_register_shash(&riscv64_sm3_alg); - - return -ENODEV; -} - -static void __exit riscv64_sm3_mod_exit(void) -{ - crypto_unregister_shash(&riscv64_sm3_alg); -} - -module_init(riscv64_sm3_mod_init); -module_exit(riscv64_sm3_mod_exit); - -MODULE_DESCRIPTION("SM3 (RISC-V accelerated)"); -MODULE_AUTHOR("Heiko Stuebner "); -MODULE_LICENSE("GPL"); -MODULE_ALIAS_CRYPTO("sm3"); diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index c85956e443a2d..b209597de5ffa 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -280,6 +280,8 @@ config CRYPTO_LIB_SM3_ARCH bool depends on CRYPTO_LIB_SM3 && !UML default y if ARM64 + default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ + RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS source "lib/crypto/tests/Kconfig" diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 9c27180124280..ad8da7f3af782 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -374,6 +374,7 @@ ifeq ($(CONFIG_CRYPTO_LIB_SM3_ARCH),y) CFLAGS_sm3.o += -I$(src)/$(SRCARCH) libsm3-$(CONFIG_ARM64) += arm64/sm3-ce-core.o \ arm64/sm3-neon-core.o +libsm3-$(CONFIG_RISCV) += riscv/sm3-riscv64-zvksh-zvkb.o endif # CONFIG_CRYPTO_LIB_SM3_ARCH ################################################################################ diff --git a/arch/riscv/crypto/sm3-riscv64-zvksh-zvkb.S b/lib/crypto/riscv/sm3-riscv64-zvksh-zvkb.S similarity index 97% rename from arch/riscv/crypto/sm3-riscv64-zvksh-zvkb.S rename to lib/crypto/riscv/sm3-riscv64-zvksh-zvkb.S index 4fe754846f657..a1d4468b0485c 100644 --- a/arch/riscv/crypto/sm3-riscv64-zvksh-zvkb.S +++ b/lib/crypto/riscv/sm3-riscv64-zvksh-zvkb.S @@ -80,7 +80,8 @@ // For the next 8 rounds, w0 and w1 are swapped. .endm -// void sm3_transform_zvksh_zvkb(u32 state[8], const u8 *data, int num_blocks); +// void sm3_transform_zvksh_zvkb(struct sm3_block_state *state, +// const u8 *data, size_t nblocks); SYM_FUNC_START(sm3_transform_zvksh_zvkb) // Load the state and endian-swap each 32-bit word. diff --git a/lib/crypto/riscv/sm3.h b/lib/crypto/riscv/sm3.h new file mode 100644 index 0000000000000..c1fbee7094e60 --- /dev/null +++ b/lib/crypto/riscv/sm3.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * SM3 using the RISC-V vector crypto extensions + * + * Copyright (C) 2023 VRULL GmbH + * Author: Heiko Stuebner + * + * Copyright (C) 2023 SiFive, Inc. + * Author: Jerry Shih + */ + +#include +#include + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_extensions); + +asmlinkage void sm3_transform_zvksh_zvkb(struct sm3_block_state *state, + const u8 *data, size_t nblocks); + +static void sm3_blocks(struct sm3_block_state *state, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_extensions) && likely(may_use_simd())) { + kernel_vector_begin(); + sm3_transform_zvksh_zvkb(state, data, nblocks); + kernel_vector_end(); + } else { + sm3_blocks_generic(state, data, nblocks); + } +} + +#define sm3_mod_init_arch sm3_mod_init_arch +static void sm3_mod_init_arch(void) +{ + if (riscv_isa_extension_available(NULL, ZVKSH) && + riscv_isa_extension_available(NULL, ZVKB) && + riscv_vector_vlen() >= 128) + static_branch_enable(&have_extensions); +}