lib/crypto: sha256: Remove sha256_blocks_simd()

Instead of having both sha256_blocks_arch() and sha256_blocks_simd(),
instead have just sha256_blocks_arch() which uses the most efficient
implementation that is available in the calling context.

This is simpler, as it reduces the API surface.  It's also safer, since
sha256_blocks_arch() just works in all contexts, including contexts
where the FPU/SIMD/vector registers cannot be used.  This doesn't mean
that SHA-256 computations *should* be done in such contexts, but rather
we should just do the right thing instead of corrupting a random task's
registers.  Eliminating this footgun and simplifying the code is well
worth the very small performance cost of doing the check.

Note: in the case of arm and arm64, what used to be sha256_blocks_arch()
is renamed back to its original name of sha256_block_data_order().
sha256_blocks_arch() is now used for the higher-level dispatch function.
This renaming also required an update to lib/crypto/arm64/sha512.h,
since sha2-armv8.pl is shared by both SHA-256 and SHA-512.

Acked-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20250630160645.3198-5-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@kernel.org>

diff --git a/include/crypto/internal/sha2.h b/include/crypto/internal/sha2.h
index 21a27fd5e198f84d24d0955739f6d1c903bd9e52..5a25ccc493886ccaca4b0b858824f9baa238bb15 100644 (file)
--- a/include/crypto/internal/sha2.h
+++ b/include/crypto/internal/sha2.h
@@ -3,7 +3,6 @@
 #ifndef _CRYPTO_INTERNAL_SHA2_H
 #define _CRYPTO_INTERNAL_SHA2_H
 
-#include <crypto/internal/simd.h>
 #include <crypto/sha2.h>
 #include <linux/compiler_attributes.h>
 #include <linux/string.h>
@@ -22,8 +21,6 @@ void sha256_blocks_generic(u32 state[SHA256_STATE_WORDS],
                           const u8 *data, size_t nblocks);
 void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS],
                        const u8 *data, size_t nblocks);
-void sha256_blocks_simd(u32 state[SHA256_STATE_WORDS],
-                       const u8 *data, size_t nblocks);
 
 static __always_inline void sha256_choose_blocks(
        u32 state[SHA256_STATE_WORDS], const u8 *data, size_t nblocks,
@@ -31,9 +28,6 @@ static __always_inline void sha256_choose_blocks(
 {
        if (!IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_SHA256) || force_generic)
                sha256_blocks_generic(state, data, nblocks);
-       else if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_SHA256_SIMD) &&
-                (force_simd || crypto_simd_usable()))
-               sha256_blocks_simd(state, data, nblocks);
        else
                sha256_blocks_arch(state, data, nblocks);
 }

diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig
index 2460ddff967fce21663baa4dc2ab11e99af40464..9bd740475a898f596dd404998fc5baeada6b51dd 100644 (file)
--- a/lib/crypto/Kconfig
+++ b/lib/crypto/Kconfig
@@ -150,14 +150,6 @@ config CRYPTO_ARCH_HAVE_LIB_SHA256
          Declares whether the architecture provides an arch-specific
          accelerated implementation of the SHA-256 library interface.
 
-config CRYPTO_ARCH_HAVE_LIB_SHA256_SIMD
-       bool
-       help
-         Declares whether the architecture provides an arch-specific
-         accelerated implementation of the SHA-256 library interface
-         that is SIMD-based and therefore not usable in hardirq
-         context.
-
 config CRYPTO_LIB_SHA256_GENERIC
        tristate
        default CRYPTO_LIB_SHA256 if !CRYPTO_ARCH_HAVE_LIB_SHA256

diff --git a/lib/crypto/arm/Kconfig b/lib/crypto/arm/Kconfig
index d1ad664f0c67469adb5c5d07b3cfb6b984e28924..9f3ff30f4032868d0c6327396026477f6a53a0f2 100644 (file)
--- a/lib/crypto/arm/Kconfig
+++ b/lib/crypto/arm/Kconfig
@@ -28,4 +28,3 @@ config CRYPTO_SHA256_ARM
        depends on !CPU_V7M
        default CRYPTO_LIB_SHA256
        select CRYPTO_ARCH_HAVE_LIB_SHA256
-       select CRYPTO_ARCH_HAVE_LIB_SHA256_SIMD

diff --git a/lib/crypto/arm/sha256-armv4.pl b/lib/crypto/arm/sha256-armv4.pl
index 8122db7fd5990256eda13d8c2a99b713b41c1cff..f3a2b54efd4ee39fbeaefc87ffd850e97915233b 100644 (file)
--- a/lib/crypto/arm/sha256-armv4.pl
+++ b/lib/crypto/arm/sha256-armv4.pl
@@ -204,18 +204,18 @@ K256:
 .word  0                               @ terminator
 #if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
 .LOPENSSL_armcap:
-.word  OPENSSL_armcap_P-sha256_blocks_arch
+.word  OPENSSL_armcap_P-sha256_block_data_order
 #endif
 .align 5
 
-.global        sha256_blocks_arch
-.type  sha256_blocks_arch,%function
-sha256_blocks_arch:
-.Lsha256_blocks_arch:
+.global        sha256_block_data_order
+.type  sha256_block_data_order,%function
+sha256_block_data_order:
+.Lsha256_block_data_order:
 #if __ARM_ARCH__<7
-       sub     r3,pc,#8                @ sha256_blocks_arch
+       sub     r3,pc,#8                @ sha256_block_data_order
 #else
-       adr     r3,.Lsha256_blocks_arch
+       adr     r3,.Lsha256_block_data_order
 #endif
 #if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
        ldr     r12,.LOPENSSL_armcap
@@ -282,7 +282,7 @@ $code.=<<___;
        moveq   pc,lr                   @ be binary compatible with V4, yet
        bx      lr                      @ interoperable with Thumb ISA:-)
 #endif
-.size  sha256_blocks_arch,.-sha256_blocks_arch
+.size  sha256_block_data_order,.-sha256_block_data_order
 ___
 ######################################################################
 # NEON stuff
@@ -470,8 +470,8 @@ sha256_block_data_order_neon:
        stmdb   sp!,{r4-r12,lr}
 
        sub     $H,sp,#16*4+16
-       adr     $Ktbl,.Lsha256_blocks_arch
-       sub     $Ktbl,$Ktbl,#.Lsha256_blocks_arch-K256
+       adr     $Ktbl,.Lsha256_block_data_order
+       sub     $Ktbl,$Ktbl,#.Lsha256_block_data_order-K256
        bic     $H,$H,#15               @ align for 128-bit stores
        mov     $t2,sp
        mov     sp,$H                   @ alloca

diff --git a/lib/crypto/arm/sha256.c b/lib/crypto/arm/sha256.c
index 109192e54b0f058ea2f84cb9f633c65a52239311..2c9cfdaaa0691ad334dfeb9fd360cc24368503af 100644 (file)
--- a/lib/crypto/arm/sha256.c
+++ b/lib/crypto/arm/sha256.c
@@ -6,12 +6,12 @@
  */
 #include <asm/neon.h>
 #include <crypto/internal/sha2.h>
+#include <crypto/internal/simd.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 
-asmlinkage void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS],
-                                  const u8 *data, size_t nblocks);
-EXPORT_SYMBOL_GPL(sha256_blocks_arch);
+asmlinkage void sha256_block_data_order(u32 state[SHA256_STATE_WORDS],
+                                       const u8 *data, size_t nblocks);
 asmlinkage void sha256_block_data_order_neon(u32 state[SHA256_STATE_WORDS],
                                             const u8 *data, size_t nblocks);
 asmlinkage void sha256_ce_transform(u32 state[SHA256_STATE_WORDS],
@@ -20,11 +20,11 @@ asmlinkage void sha256_ce_transform(u32 state[SHA256_STATE_WORDS],
 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce);
 
-void sha256_blocks_simd(u32 state[SHA256_STATE_WORDS],
+void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS],
                        const u8 *data, size_t nblocks)
 {
        if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
-           static_branch_likely(&have_neon)) {
+           static_branch_likely(&have_neon) && crypto_simd_usable()) {
                kernel_neon_begin();
                if (static_branch_likely(&have_ce))
                        sha256_ce_transform(state, data, nblocks);
@@ -32,10 +32,10 @@ void sha256_blocks_simd(u32 state[SHA256_STATE_WORDS],
                        sha256_block_data_order_neon(state, data, nblocks);
                kernel_neon_end();
        } else {
-               sha256_blocks_arch(state, data, nblocks);
+               sha256_block_data_order(state, data, nblocks);
        }
 }
-EXPORT_SYMBOL_GPL(sha256_blocks_simd);
+EXPORT_SYMBOL_GPL(sha256_blocks_arch);
 
 bool sha256_is_arch_optimized(void)
 {

diff --git a/lib/crypto/arm64/Kconfig b/lib/crypto/arm64/Kconfig
index 129a7685cb4c1e7ad3811be98e7bdc9022dc85a0..49e57bfdb5b529015f95b36ce1357eeab2caadbb 100644 (file)
--- a/lib/crypto/arm64/Kconfig
+++ b/lib/crypto/arm64/Kconfig
@@ -17,4 +17,3 @@ config CRYPTO_SHA256_ARM64
        tristate
        default CRYPTO_LIB_SHA256
        select CRYPTO_ARCH_HAVE_LIB_SHA256
-       select CRYPTO_ARCH_HAVE_LIB_SHA256_SIMD

diff --git a/lib/crypto/arm64/sha2-armv8.pl b/lib/crypto/arm64/sha2-armv8.pl
index 4aebd20c498bc8f28e7336add4785f379f0d9705..35ec9ae99fe169b5a7117271bc4ff7ef928b3c27 100644 (file)
--- a/lib/crypto/arm64/sha2-armv8.pl
+++ b/lib/crypto/arm64/sha2-armv8.pl
@@ -95,7 +95,7 @@ if ($output =~ /512/) {
        $reg_t="w";
 }
 
-$func="sha${BITS}_blocks_arch";
+$func="sha${BITS}_block_data_order";
 
 ($ctx,$inp,$num,$Ktbl)=map("x$_",(0..2,30));
 

diff --git a/lib/crypto/arm64/sha256.c b/lib/crypto/arm64/sha256.c
index bcf7a3adc0c460b1c6b2d943d3664e8b4947064f..fb9bff40357be20eb912a64afce2e1c6875a060b 100644 (file)
--- a/lib/crypto/arm64/sha256.c
+++ b/lib/crypto/arm64/sha256.c
@@ -6,12 +6,12 @@
  */
 #include <asm/neon.h>
 #include <crypto/internal/sha2.h>
+#include <crypto/internal/simd.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 
-asmlinkage void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS],
-                                  const u8 *data, size_t nblocks);
-EXPORT_SYMBOL_GPL(sha256_blocks_arch);
+asmlinkage void sha256_block_data_order(u32 state[SHA256_STATE_WORDS],
+                                       const u8 *data, size_t nblocks);
 asmlinkage void sha256_block_neon(u32 state[SHA256_STATE_WORDS],
                                  const u8 *data, size_t nblocks);
 asmlinkage size_t __sha256_ce_transform(u32 state[SHA256_STATE_WORDS],
@@ -20,11 +20,11 @@ asmlinkage size_t __sha256_ce_transform(u32 state[SHA256_STATE_WORDS],
 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce);
 
-void sha256_blocks_simd(u32 state[SHA256_STATE_WORDS],
+void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS],
                        const u8 *data, size_t nblocks)
 {
        if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
-           static_branch_likely(&have_neon)) {
+           static_branch_likely(&have_neon) && crypto_simd_usable()) {
                if (static_branch_likely(&have_ce)) {
                        do {
                                size_t rem;
@@ -42,10 +42,10 @@ void sha256_blocks_simd(u32 state[SHA256_STATE_WORDS],
                        kernel_neon_end();
                }
        } else {
-               sha256_blocks_arch(state, data, nblocks);
+               sha256_block_data_order(state, data, nblocks);
        }
 }
-EXPORT_SYMBOL_GPL(sha256_blocks_simd);
+EXPORT_SYMBOL_GPL(sha256_blocks_arch);
 
 bool sha256_is_arch_optimized(void)
 {

diff --git a/lib/crypto/arm64/sha512.h b/lib/crypto/arm64/sha512.h
index eae14f9752e0b1284e908920d000c4758c8bb60b..6abb40b467f2ec6c52d57ede9359f09c6a5f28a5 100644 (file)
--- a/lib/crypto/arm64/sha512.h
+++ b/lib/crypto/arm64/sha512.h
@@ -11,8 +11,8 @@
 
 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha512_insns);
 
-asmlinkage void sha512_blocks_arch(struct sha512_block_state *state,
-                                  const u8 *data, size_t nblocks);
+asmlinkage void sha512_block_data_order(struct sha512_block_state *state,
+                                       const u8 *data, size_t nblocks);
 asmlinkage size_t __sha512_ce_transform(struct sha512_block_state *state,
                                        const u8 *data, size_t nblocks);
 
@@ -32,7 +32,7 @@ static void sha512_blocks(struct sha512_block_state *state,
                        nblocks = rem;
                } while (nblocks);
        } else {
-               sha512_blocks_arch(state, data, nblocks);
+               sha512_block_data_order(state, data, nblocks);
        }
 }
 

diff --git a/lib/crypto/riscv/Kconfig b/lib/crypto/riscv/Kconfig
index 47c99ea97ce2c888f424033857ef86f13af01c7f..c100571feb7e8331a5349f73eeed3bbc826c69d0 100644 (file)
--- a/lib/crypto/riscv/Kconfig
+++ b/lib/crypto/riscv/Kconfig
@@ -12,5 +12,4 @@ config CRYPTO_SHA256_RISCV64
        depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO
        default CRYPTO_LIB_SHA256
        select CRYPTO_ARCH_HAVE_LIB_SHA256
-       select CRYPTO_ARCH_HAVE_LIB_SHA256_SIMD
        select CRYPTO_LIB_SHA256_GENERIC

diff --git a/lib/crypto/riscv/sha256.c b/lib/crypto/riscv/sha256.c
index 71808397dff4c54335bdaa14b1a1f8ccb8ce8bb3..aa77349d08f300d397c00f9b26dbe53903326249 100644 (file)
--- a/lib/crypto/riscv/sha256.c
+++ b/lib/crypto/riscv/sha256.c
@@ -11,6 +11,7 @@
 
 #include <asm/vector.h>
 #include <crypto/internal/sha2.h>
+#include <crypto/internal/simd.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 
@@ -19,10 +20,10 @@ asmlinkage void sha256_transform_zvknha_or_zvknhb_zvkb(
 
 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_extensions);
 
-void sha256_blocks_simd(u32 state[SHA256_STATE_WORDS],
+void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS],
                        const u8 *data, size_t nblocks)
 {
-       if (static_branch_likely(&have_extensions)) {
+       if (static_branch_likely(&have_extensions) && crypto_simd_usable()) {
                kernel_vector_begin();
                sha256_transform_zvknha_or_zvknhb_zvkb(state, data, nblocks);
                kernel_vector_end();
@@ -30,13 +31,6 @@ void sha256_blocks_simd(u32 state[SHA256_STATE_WORDS],
                sha256_blocks_generic(state, data, nblocks);
        }
 }
-EXPORT_SYMBOL_GPL(sha256_blocks_simd);
-
-void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS],
-                       const u8 *data, size_t nblocks)
-{
-       sha256_blocks_generic(state, data, nblocks);
-}
 EXPORT_SYMBOL_GPL(sha256_blocks_arch);
 
 bool sha256_is_arch_optimized(void)

diff --git a/lib/crypto/x86/Kconfig b/lib/crypto/x86/Kconfig
index 5e94cdee492c2cd9cc0322215471513bf94ac3d0..e344579db3d855e157d7a14dc73496d0f8dda535 100644 (file)
--- a/lib/crypto/x86/Kconfig
+++ b/lib/crypto/x86/Kconfig
@@ -30,5 +30,4 @@ config CRYPTO_SHA256_X86_64
        depends on 64BIT
        default CRYPTO_LIB_SHA256
        select CRYPTO_ARCH_HAVE_LIB_SHA256
-       select CRYPTO_ARCH_HAVE_LIB_SHA256_SIMD
        select CRYPTO_LIB_SHA256_GENERIC

diff --git a/lib/crypto/x86/sha256.c b/lib/crypto/x86/sha256.c
index 80380f8fdcee443a45350e2bec6ebbe87189a3a7..baba74d7d26f2e440e423526db80ff77cd32d051 100644 (file)
--- a/lib/crypto/x86/sha256.c
+++ b/lib/crypto/x86/sha256.c
@@ -6,6 +6,7 @@
  */
 #include <asm/fpu/api.h>
 #include <crypto/internal/sha2.h>
+#include <crypto/internal/simd.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/static_call.h>
@@ -23,10 +24,10 @@ static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha256_x86);
 
 DEFINE_STATIC_CALL(sha256_blocks_x86, sha256_transform_ssse3);
 
-void sha256_blocks_simd(u32 state[SHA256_STATE_WORDS],
+void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS],
                        const u8 *data, size_t nblocks)
 {
-       if (static_branch_likely(&have_sha256_x86)) {
+       if (static_branch_likely(&have_sha256_x86) && crypto_simd_usable()) {
                kernel_fpu_begin();
                static_call(sha256_blocks_x86)(state, data, nblocks);
                kernel_fpu_end();
@@ -34,13 +35,6 @@ void sha256_blocks_simd(u32 state[SHA256_STATE_WORDS],
                sha256_blocks_generic(state, data, nblocks);
        }
 }
-EXPORT_SYMBOL_GPL(sha256_blocks_simd);
-
-void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS],
-                       const u8 *data, size_t nblocks)
-{
-       sha256_blocks_generic(state, data, nblocks);
-}
 EXPORT_SYMBOL_GPL(sha256_blocks_arch);
 
 bool sha256_is_arch_optimized(void)