From: Eric Biggers Date: Sat, 21 Mar 2026 04:09:32 +0000 (-0700) Subject: lib/crypto: x86/sm3: Migrate optimized code into library X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=17ba6108d3e084652807826cc49c851c00976f1a;p=thirdparty%2Fkernel%2Fstable.git lib/crypto: x86/sm3: Migrate optimized code into library Instead of exposing the x86-optimized SM3 code via an x86-specific crypto_shash algorithm, instead just implement the sm3_blocks() library function. This is much simpler, it makes the SM3 library functions be x86-optimized, and it fixes the longstanding issue where the x86-optimized SM3 code was disabled by default. SM3 still remains available through crypto_shash, but individual architectures no longer need to handle it. Tweak the prototype of sm3_transform_avx() to match what the library expects, including changing the block count to size_t. Note that the assembly code actually already treated this argument as size_t. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260321040935.410034-10-ebiggers@kernel.org Signed-off-by: Eric Biggers --- diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig index 905e8a23cec3a..822cbf4142696 100644 --- a/arch/x86/crypto/Kconfig +++ b/arch/x86/crypto/Kconfig @@ -331,17 +331,4 @@ config CRYPTO_AEGIS128_AESNI_SSE2 - AES-NI (AES New Instructions) - SSE4.1 (Streaming SIMD Extensions 4.1) -config CRYPTO_SM3_AVX_X86_64 - tristate "Hash functions: SM3 (AVX)" - depends on 64BIT - select CRYPTO_HASH - select CRYPTO_LIB_SM3 - help - SM3 secure hash function as defined by OSCCA GM/T 0004-2012 SM3 - - Architecture: x86_64 using: - - AVX (Advanced Vector Extensions) - - If unsure, say N. - endmenu diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile index d562f4341da69..3d6d5087a65e2 100644 --- a/arch/x86/crypto/Makefile +++ b/arch/x86/crypto/Makefile @@ -50,9 +50,6 @@ aesni-intel-$(CONFIG_64BIT) += aes-ctr-avx-x86_64.o \ aes-gcm-vaes-avx512.o \ aes-xts-avx-x86_64.o -obj-$(CONFIG_CRYPTO_SM3_AVX_X86_64) += sm3-avx-x86_64.o -sm3-avx-x86_64-y := sm3-avx-asm_64.o sm3_avx_glue.o - obj-$(CONFIG_CRYPTO_SM4_AESNI_AVX_X86_64) += sm4-aesni-avx-x86_64.o sm4-aesni-avx-x86_64-y := sm4-aesni-avx-asm_64.o sm4_aesni_avx_glue.o diff --git a/arch/x86/crypto/sm3_avx_glue.c b/arch/x86/crypto/sm3_avx_glue.c deleted file mode 100644 index 6e8c42b9dc8ed..0000000000000 --- a/arch/x86/crypto/sm3_avx_glue.c +++ /dev/null @@ -1,100 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * SM3 Secure Hash Algorithm, AVX assembler accelerated. - * specified in: https://datatracker.ietf.org/doc/html/draft-sca-cfrg-sm3-02 - * - * Copyright (C) 2021 Tianjia Zhang - */ - -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - -#include -#include -#include -#include -#include -#include -#include - -asmlinkage void sm3_transform_avx(struct sm3_state *state, - const u8 *data, int nblocks); - -static int sm3_avx_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - int remain; - - /* - * Make sure struct sm3_state begins directly with the SM3 - * 256-bit internal state, as this is what the asm functions expect. - */ - BUILD_BUG_ON(offsetof(struct sm3_state, state) != 0); - - kernel_fpu_begin(); - remain = sm3_base_do_update_blocks(desc, data, len, sm3_transform_avx); - kernel_fpu_end(); - return remain; -} - -static int sm3_avx_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - kernel_fpu_begin(); - sm3_base_do_finup(desc, data, len, sm3_transform_avx); - kernel_fpu_end(); - return sm3_base_finish(desc, out); -} - -static struct shash_alg sm3_avx_alg = { - .digestsize = SM3_DIGEST_SIZE, - .init = sm3_base_init, - .update = sm3_avx_update, - .finup = sm3_avx_finup, - .descsize = SM3_STATE_SIZE, - .base = { - .cra_name = "sm3", - .cra_driver_name = "sm3-avx", - .cra_priority = 300, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SM3_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static int __init sm3_avx_mod_init(void) -{ - const char *feature_name; - - if (!boot_cpu_has(X86_FEATURE_AVX)) { - pr_info("AVX instruction are not detected.\n"); - return -ENODEV; - } - - if (!boot_cpu_has(X86_FEATURE_BMI2)) { - pr_info("BMI2 instruction are not detected.\n"); - return -ENODEV; - } - - if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, - &feature_name)) { - pr_info("CPU feature '%s' is not supported.\n", feature_name); - return -ENODEV; - } - - return crypto_register_shash(&sm3_avx_alg); -} - -static void __exit sm3_avx_mod_exit(void) -{ - crypto_unregister_shash(&sm3_avx_alg); -} - -module_init(sm3_avx_mod_init); -module_exit(sm3_avx_mod_exit); - -MODULE_LICENSE("GPL v2"); -MODULE_AUTHOR("Tianjia Zhang "); -MODULE_DESCRIPTION("SM3 Secure Hash Algorithm, AVX assembler accelerated"); -MODULE_ALIAS_CRYPTO("sm3"); -MODULE_ALIAS_CRYPTO("sm3-avx"); diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index b209597de5ffa..91b1d0eb13b09 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -282,6 +282,7 @@ config CRYPTO_LIB_SM3_ARCH default y if ARM64 default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS + default y if X86_64 source "lib/crypto/tests/Kconfig" diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index ad8da7f3af782..ec1747f51d07e 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -375,6 +375,7 @@ CFLAGS_sm3.o += -I$(src)/$(SRCARCH) libsm3-$(CONFIG_ARM64) += arm64/sm3-ce-core.o \ arm64/sm3-neon-core.o libsm3-$(CONFIG_RISCV) += riscv/sm3-riscv64-zvksh-zvkb.o +libsm3-$(CONFIG_X86) += x86/sm3-avx-asm_64.o endif # CONFIG_CRYPTO_LIB_SM3_ARCH ################################################################################ diff --git a/arch/x86/crypto/sm3-avx-asm_64.S b/lib/crypto/x86/sm3-avx-asm_64.S similarity index 98% rename from arch/x86/crypto/sm3-avx-asm_64.S rename to lib/crypto/x86/sm3-avx-asm_64.S index 503bab450a915..a1925b1360101 100644 --- a/arch/x86/crypto/sm3-avx-asm_64.S +++ b/lib/crypto/x86/sm3-avx-asm_64.S @@ -12,10 +12,9 @@ */ #include -#include #include -/* Context structure */ +/* State structure */ #define state_h0 0 #define state_h1 4 @@ -325,13 +324,13 @@ /* * Transform nblocks*64 bytes (nblocks*16 32-bit words) at DATA. * - * void sm3_transform_avx(struct sm3_state *state, - * const u8 *data, int nblocks); + * void sm3_transform_avx(struct sm3_block_state *state, + * const u8 *data, size_t nblocks); */ -SYM_TYPED_FUNC_START(sm3_transform_avx) +SYM_FUNC_START(sm3_transform_avx) /* input: - * %rdi: ctx, CTX - * %rsi: data (64*nblks bytes) + * %rdi: state + * %rsi: data * %rdx: nblocks */ vzeroupper; diff --git a/lib/crypto/x86/sm3.h b/lib/crypto/x86/sm3.h new file mode 100644 index 0000000000000..3834780f2f6a3 --- /dev/null +++ b/lib/crypto/x86/sm3.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * SM3 optimized for x86_64 + * + * Copyright 2026 Google LLC + */ +#include +#include + +asmlinkage void sm3_transform_avx(struct sm3_block_state *state, + const u8 *data, size_t nblocks); + +static void sm3_blocks_avx(struct sm3_block_state *state, + const u8 *data, size_t nblocks) +{ + if (likely(irq_fpu_usable())) { + kernel_fpu_begin(); + sm3_transform_avx(state, data, nblocks); + kernel_fpu_end(); + } else { + sm3_blocks_generic(state, data, nblocks); + } +} + +DEFINE_STATIC_CALL(sm3_blocks_x86, sm3_blocks_generic); + +static void sm3_blocks(struct sm3_block_state *state, + const u8 *data, size_t nblocks) +{ + static_call(sm3_blocks_x86)(state, data, nblocks); +} + +#define sm3_mod_init_arch sm3_mod_init_arch +static void sm3_mod_init_arch(void) +{ + if (boot_cpu_has(X86_FEATURE_AVX) && boot_cpu_has(X86_FEATURE_BMI2) && + cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) + static_call_update(sm3_blocks_x86, sm3_blocks_avx); +}