lib/crypto: x86/sm3: Migrate optimized code into library

Instead of exposing the x86-optimized SM3 code via an x86-specific
crypto_shash algorithm, instead just implement the sm3_blocks() library
function.  This is much simpler, it makes the SM3 library functions be
x86-optimized, and it fixes the longstanding issue where the
x86-optimized SM3 code was disabled by default.  SM3 still remains
available through crypto_shash, but individual architectures no longer
need to handle it.

Tweak the prototype of sm3_transform_avx() to match what the library
expects, including changing the block count to size_t.  Note that the
assembly code actually already treated this argument as size_t.

Acked-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20260321040935.410034-10-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@kernel.org>

diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig
index 905e8a23cec3abf67026e0c21f24909bb0e2acff..822cbf4142696592e2bce3ea6bf5e4818fcdebde 100644 (file)
--- a/arch/x86/crypto/Kconfig
+++ b/arch/x86/crypto/Kconfig
@@ -331,17 +331,4 @@ config CRYPTO_AEGIS128_AESNI_SSE2
          - AES-NI (AES New Instructions)
          - SSE4.1 (Streaming SIMD Extensions 4.1)
 
-config CRYPTO_SM3_AVX_X86_64
-       tristate "Hash functions: SM3 (AVX)"
-       depends on 64BIT
-       select CRYPTO_HASH
-       select CRYPTO_LIB_SM3
-       help
-         SM3 secure hash function as defined by OSCCA GM/T 0004-2012 SM3
-
-         Architecture: x86_64 using:
-         - AVX (Advanced Vector Extensions)
-
-         If unsure, say N.
-
 endmenu

diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile
index d562f4341da69d6d51c01e467c7af395529d1a8a..3d6d5087a65e2d12241b172e2aedd84326ea76cd 100644 (file)
--- a/arch/x86/crypto/Makefile
+++ b/arch/x86/crypto/Makefile
@@ -50,9 +50,6 @@ aesni-intel-$(CONFIG_64BIT) += aes-ctr-avx-x86_64.o \
                               aes-gcm-vaes-avx512.o \
                               aes-xts-avx-x86_64.o
 
-obj-$(CONFIG_CRYPTO_SM3_AVX_X86_64) += sm3-avx-x86_64.o
-sm3-avx-x86_64-y := sm3-avx-asm_64.o sm3_avx_glue.o
-
 obj-$(CONFIG_CRYPTO_SM4_AESNI_AVX_X86_64) += sm4-aesni-avx-x86_64.o
 sm4-aesni-avx-x86_64-y := sm4-aesni-avx-asm_64.o sm4_aesni_avx_glue.o
 

diff --git a/arch/x86/crypto/sm3_avx_glue.c b/arch/x86/crypto/sm3_avx_glue.c
deleted file mode 100644 (file)
index 6e8c42b..0000000
--- a/arch/x86/crypto/sm3_avx_glue.c
+++ /dev/null
@@ -1,100 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * SM3 Secure Hash Algorithm, AVX assembler accelerated.
- * specified in: https://datatracker.ietf.org/doc/html/draft-sca-cfrg-sm3-02
- *
- * Copyright (C) 2021 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
- */
-
-#define pr_fmt(fmt)    KBUILD_MODNAME ": " fmt
-
-#include <crypto/internal/hash.h>
-#include <crypto/internal/simd.h>
-#include <crypto/sm3.h>
-#include <crypto/sm3_base.h>
-#include <linux/cpufeature.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-
-asmlinkage void sm3_transform_avx(struct sm3_state *state,
-                       const u8 *data, int nblocks);
-
-static int sm3_avx_update(struct shash_desc *desc, const u8 *data,
-                        unsigned int len)
-{
-       int remain;
-
-       /*
-        * Make sure struct sm3_state begins directly with the SM3
-        * 256-bit internal state, as this is what the asm functions expect.
-        */
-       BUILD_BUG_ON(offsetof(struct sm3_state, state) != 0);
-
-       kernel_fpu_begin();
-       remain = sm3_base_do_update_blocks(desc, data, len, sm3_transform_avx);
-       kernel_fpu_end();
-       return remain;
-}
-
-static int sm3_avx_finup(struct shash_desc *desc, const u8 *data,
-                     unsigned int len, u8 *out)
-{
-       kernel_fpu_begin();
-       sm3_base_do_finup(desc, data, len, sm3_transform_avx);
-       kernel_fpu_end();
-       return sm3_base_finish(desc, out);
-}
-
-static struct shash_alg sm3_avx_alg = {
-       .digestsize     =       SM3_DIGEST_SIZE,
-       .init           =       sm3_base_init,
-       .update         =       sm3_avx_update,
-       .finup          =       sm3_avx_finup,
-       .descsize       =       SM3_STATE_SIZE,
-       .base           =       {
-               .cra_name       =       "sm3",
-               .cra_driver_name =      "sm3-avx",
-               .cra_priority   =       300,
-               .cra_flags       =      CRYPTO_AHASH_ALG_BLOCK_ONLY |
-                                       CRYPTO_AHASH_ALG_FINUP_MAX,
-               .cra_blocksize  =       SM3_BLOCK_SIZE,
-               .cra_module     =       THIS_MODULE,
-       }
-};
-
-static int __init sm3_avx_mod_init(void)
-{
-       const char *feature_name;
-
-       if (!boot_cpu_has(X86_FEATURE_AVX)) {
-               pr_info("AVX instruction are not detected.\n");
-               return -ENODEV;
-       }
-
-       if (!boot_cpu_has(X86_FEATURE_BMI2)) {
-               pr_info("BMI2 instruction are not detected.\n");
-               return -ENODEV;
-       }
-
-       if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
-                               &feature_name)) {
-               pr_info("CPU feature '%s' is not supported.\n", feature_name);
-               return -ENODEV;
-       }
-
-       return crypto_register_shash(&sm3_avx_alg);
-}
-
-static void __exit sm3_avx_mod_exit(void)
-{
-       crypto_unregister_shash(&sm3_avx_alg);
-}
-
-module_init(sm3_avx_mod_init);
-module_exit(sm3_avx_mod_exit);
-
-MODULE_LICENSE("GPL v2");
-MODULE_AUTHOR("Tianjia Zhang <tianjia.zhang@linux.alibaba.com>");
-MODULE_DESCRIPTION("SM3 Secure Hash Algorithm, AVX assembler accelerated");
-MODULE_ALIAS_CRYPTO("sm3");
-MODULE_ALIAS_CRYPTO("sm3-avx");

diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig
index b209597de5ffa76a4240d64efa3f6eda581f3541..91b1d0eb13b0913140e31e4bfee8f0c2796222d2 100644 (file)
--- a/lib/crypto/Kconfig
+++ b/lib/crypto/Kconfig
@@ -282,6 +282,7 @@ config CRYPTO_LIB_SM3_ARCH
        default y if ARM64
        default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \
                     RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS
+       default y if X86_64
 
 source "lib/crypto/tests/Kconfig"
 

diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile
index ad8da7f3af7825dde54b666b85ea42035a091276..ec1747f51d07ed447e764f515eb53e8c0ed00b9b 100644 (file)
--- a/lib/crypto/Makefile
+++ b/lib/crypto/Makefile
@@ -375,6 +375,7 @@ CFLAGS_sm3.o += -I$(src)/$(SRCARCH)
 libsm3-$(CONFIG_ARM64) += arm64/sm3-ce-core.o \
                          arm64/sm3-neon-core.o
 libsm3-$(CONFIG_RISCV) += riscv/sm3-riscv64-zvksh-zvkb.o
+libsm3-$(CONFIG_X86) += x86/sm3-avx-asm_64.o
 endif # CONFIG_CRYPTO_LIB_SM3_ARCH
 
 ################################################################################

diff --git a/arch/x86/crypto/sm3-avx-asm_64.S b/lib/crypto/x86/sm3-avx-asm_64.S
similarity index 98%
rename from arch/x86/crypto/sm3-avx-asm_64.S
rename to lib/crypto/x86/sm3-avx-asm_64.S
index 503bab450a9157200bc752dceaf03b9bbd020886..a1925b1360101c95651ce0c507d3987589732f38 100644 (file)
--- a/arch/x86/crypto/sm3-avx-asm_64.S
+++ b/lib/crypto/x86/sm3-avx-asm_64.S
@@ -12,10 +12,9 @@
  */
 
 #include <linux/linkage.h>
-#include <linux/cfi_types.h>
 #include <asm/frame.h>
 
-/* Context structure */
+/* State structure */
 
 #define state_h0 0
 #define state_h1 4
@@ -325,13 +324,13 @@
 /*
  * Transform nblocks*64 bytes (nblocks*16 32-bit words) at DATA.
  *
- * void sm3_transform_avx(struct sm3_state *state,
- *                        const u8 *data, int nblocks);
+ * void sm3_transform_avx(struct sm3_block_state *state,
+ *                        const u8 *data, size_t nblocks);
  */
-SYM_TYPED_FUNC_START(sm3_transform_avx)
+SYM_FUNC_START(sm3_transform_avx)
        /* input:
-        *      %rdi: ctx, CTX
-        *      %rsi: data (64*nblks bytes)
+        *      %rdi: state
+        *      %rsi: data
         *      %rdx: nblocks
         */
        vzeroupper;

diff --git a/lib/crypto/x86/sm3.h b/lib/crypto/x86/sm3.h
new file mode 100644 (file)
index 0000000..3834780
--- /dev/null
+++ b/lib/crypto/x86/sm3.h
@@ -0,0 +1,39 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * SM3 optimized for x86_64
+ *
+ * Copyright 2026 Google LLC
+ */
+#include <asm/fpu/api.h>
+#include <linux/static_call.h>
+
+asmlinkage void sm3_transform_avx(struct sm3_block_state *state,
+                                 const u8 *data, size_t nblocks);
+
+static void sm3_blocks_avx(struct sm3_block_state *state,
+                          const u8 *data, size_t nblocks)
+{
+       if (likely(irq_fpu_usable())) {
+               kernel_fpu_begin();
+               sm3_transform_avx(state, data, nblocks);
+               kernel_fpu_end();
+       } else {
+               sm3_blocks_generic(state, data, nblocks);
+       }
+}
+
+DEFINE_STATIC_CALL(sm3_blocks_x86, sm3_blocks_generic);
+
+static void sm3_blocks(struct sm3_block_state *state,
+                      const u8 *data, size_t nblocks)
+{
+       static_call(sm3_blocks_x86)(state, data, nblocks);
+}
+
+#define sm3_mod_init_arch sm3_mod_init_arch
+static void sm3_mod_init_arch(void)
+{
+       if (boot_cpu_has(X86_FEATURE_AVX) && boot_cpu_has(X86_FEATURE_BMI2) &&
+           cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL))
+               static_call_update(sm3_blocks_x86, sm3_blocks_avx);
+}