Merge tag 'fsverity-for-linus' of git://git.kernel.org/pub/scm/fs/fsverity/linux

Pull interleaved SHA-256 hashing support from Eric Biggers:
 "Optimize fsverity with 2-way interleaved hashing

  Add support for 2-way interleaved SHA-256 hashing to lib/crypto/, and
  make fsverity use it for faster file data verification. This improves
  fsverity performance on many x86_64 and arm64 processors.

  Later, I plan to make dm-verity use this too"

* tag 'fsverity-for-linus' of git://git.kernel.org/pub/scm/fs/fsverity/linux:
  fsverity: Use 2-way interleaved SHA-256 hashing when supported
  fsverity: Remove inode parameter from fsverity_hash_block()
  lib/crypto: tests: Add tests and benchmark for sha256_finup_2x()
  lib/crypto: x86/sha256: Add support for 2-way interleaved hashing
  lib/crypto: arm64/sha256: Add support for 2-way interleaved hashing
  lib/crypto: sha256: Add support for 2-way interleaved hashing

diff --cc fs/verity/verify.c
Simple merge

diff --cc lib/crypto/arm64/sha256.h
index be4aeda9d0e6ee5720e4fe764893fb48c0fbfba6,a7bc3a90ada6b2cfa290fa7081529f8984450f64..80d06df27d3a3973d3141d08895dcc0aab41bd82
--- 1/lib/crypto/arm64/sha256.h
--- 2/lib/crypto/arm64/sha256.h
+++ b/lib/crypto/arm64/sha256.h
@@@ -44,9 -44,46 +44,46 @@@ static void sha256_blocks(struct sha256
        }
  }
  
+ static_assert(offsetof(struct __sha256_ctx, state) == 0);
+ static_assert(offsetof(struct __sha256_ctx, bytecount) == 32);
+ static_assert(offsetof(struct __sha256_ctx, buf) == 40);
+ asmlinkage void sha256_ce_finup2x(const struct __sha256_ctx *ctx,
+                                 const u8 *data1, const u8 *data2, int len,
+                                 u8 out1[SHA256_DIGEST_SIZE],
+                                 u8 out2[SHA256_DIGEST_SIZE]);
+ 
+ #define sha256_finup_2x_arch sha256_finup_2x_arch
+ static bool sha256_finup_2x_arch(const struct __sha256_ctx *ctx,
+                                const u8 *data1, const u8 *data2, size_t len,
+                                u8 out1[SHA256_DIGEST_SIZE],
+                                u8 out2[SHA256_DIGEST_SIZE])
+ {
+       /*
+        * The assembly requires len >= SHA256_BLOCK_SIZE && len <= INT_MAX.
+        * Further limit len to 65536 to avoid spending too long with preemption
+        * disabled.  (Of course, in practice len is nearly always 4096 anyway.)
+        */
+       if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
+           static_branch_likely(&have_ce) && len >= SHA256_BLOCK_SIZE &&
+           len <= 65536 && likely(may_use_simd())) {
+               kernel_neon_begin();
+               sha256_ce_finup2x(ctx, data1, data2, len, out1, out2);
+               kernel_neon_end();
+               kmsan_unpoison_memory(out1, SHA256_DIGEST_SIZE);
+               kmsan_unpoison_memory(out2, SHA256_DIGEST_SIZE);
+               return true;
+       }
+       return false;
+ }
+ 
+ static bool sha256_finup_2x_is_optimized_arch(void)
+ {
+       return static_key_enabled(&have_ce);
+ }
+ 
  #ifdef CONFIG_KERNEL_MODE_NEON
  #define sha256_mod_init_arch sha256_mod_init_arch
 -static inline void sha256_mod_init_arch(void)
 +static void sha256_mod_init_arch(void)
  {
        if (cpu_have_named_feature(ASIMD)) {
                static_branch_enable(&have_neon);

diff --cc lib/crypto/x86/sha256.h
index 41fa95fbc3bf8552b13c053940dcfa5a67752bfa,ecea65b608db33a482b2c5a433ceb0e878367461..38e33b22a092774a884f741d24174c78427fb86d
--- 1/lib/crypto/x86/sha256.h
--- 2/lib/crypto/x86/sha256.h
+++ b/lib/crypto/x86/sha256.h
@@@ -5,8 -5,11 +5,10 @@@
   * Copyright 2025 Google LLC
   */
  #include <asm/fpu/api.h>
 -#include <crypto/internal/simd.h>
  #include <linux/static_call.h>
  
+ static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha_ni);
+ 
  DEFINE_STATIC_CALL(sha256_blocks_x86, sha256_blocks_generic);
  
  #define DEFINE_X86_SHA256_FN(c_fn, asm_fn)                                 \
@@@ -35,8 -38,44 +37,44 @@@ static void sha256_blocks(struct sha256
        static_call(sha256_blocks_x86)(state, data, nblocks);
  }
  
+ static_assert(offsetof(struct __sha256_ctx, state) == 0);
+ static_assert(offsetof(struct __sha256_ctx, bytecount) == 32);
+ static_assert(offsetof(struct __sha256_ctx, buf) == 40);
+ asmlinkage void sha256_ni_finup2x(const struct __sha256_ctx *ctx,
+                                 const u8 *data1, const u8 *data2, int len,
+                                 u8 out1[SHA256_DIGEST_SIZE],
+                                 u8 out2[SHA256_DIGEST_SIZE]);
+ 
+ #define sha256_finup_2x_arch sha256_finup_2x_arch
+ static bool sha256_finup_2x_arch(const struct __sha256_ctx *ctx,
+                                const u8 *data1, const u8 *data2, size_t len,
+                                u8 out1[SHA256_DIGEST_SIZE],
+                                u8 out2[SHA256_DIGEST_SIZE])
+ {
+       /*
+        * The assembly requires len >= SHA256_BLOCK_SIZE && len <= INT_MAX.
+        * Further limit len to 65536 to avoid spending too long with preemption
+        * disabled.  (Of course, in practice len is nearly always 4096 anyway.)
+        */
+       if (static_branch_likely(&have_sha_ni) && len >= SHA256_BLOCK_SIZE &&
+           len <= 65536 && likely(irq_fpu_usable())) {
+               kernel_fpu_begin();
+               sha256_ni_finup2x(ctx, data1, data2, len, out1, out2);
+               kernel_fpu_end();
+               kmsan_unpoison_memory(out1, SHA256_DIGEST_SIZE);
+               kmsan_unpoison_memory(out2, SHA256_DIGEST_SIZE);
+               return true;
+       }
+       return false;
+ }
+ 
+ static bool sha256_finup_2x_is_optimized_arch(void)
+ {
+       return static_key_enabled(&have_sha_ni);
+ }
+ 
  #define sha256_mod_init_arch sha256_mod_init_arch
 -static inline void sha256_mod_init_arch(void)
 +static void sha256_mod_init_arch(void)
  {
        if (boot_cpu_has(X86_FEATURE_SHA_NI)) {
                static_call_update(sha256_blocks_x86, sha256_blocks_ni);