lib/crypto: s390/sha3: Add optimized Keccak functions

Implement sha3_absorb_blocks() and sha3_keccakf() using the hardware-
accelerated SHA-3 support in Message-Security-Assist Extension 6.

This accelerates the SHA3-224, SHA3-256, SHA3-384, SHA3-512, and
SHAKE256 library functions.

Note that arch/s390/crypto/ already has SHA-3 code that uses this
extension, but it is exposed only via crypto_shash.  This commit brings
the same acceleration to the SHA-3 library.  The arch/s390/crypto/
version will become redundant and be removed in later changes.

Tested-by: Harald Freudenberger <freude@linux.ibm.com>
Reviewed-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20251026055032.1413733-11-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@kernel.org>

diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig
index 587490ca65654940c85370f57dae1a7577a5c9ae..7445054fc0ad4fe6896079b8bb09de429865fe44 100644 (file)
--- a/lib/crypto/Kconfig
+++ b/lib/crypto/Kconfig
@@ -206,6 +206,7 @@ config CRYPTO_LIB_SHA3_ARCH
        bool
        depends on CRYPTO_LIB_SHA3 && !UML
        default y if ARM64 && KERNEL_MODE_NEON
+       default y if S390
 
 config CRYPTO_LIB_SM3
        tristate

diff --git a/lib/crypto/s390/sha3.h b/lib/crypto/s390/sha3.h
new file mode 100644 (file)
index 0000000..668e53d
--- /dev/null
+++ b/lib/crypto/s390/sha3.h
@@ -0,0 +1,88 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * SHA-3 optimized using the CP Assist for Cryptographic Functions (CPACF)
+ *
+ * Copyright 2025 Google LLC
+ */
+#include <asm/cpacf.h>
+#include <linux/cpufeature.h>
+
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha3);
+
+static void sha3_absorb_blocks(struct sha3_state *state, const u8 *data,
+                              size_t nblocks, size_t block_size)
+{
+       if (static_branch_likely(&have_sha3)) {
+               /*
+                * Note that KIMD assumes little-endian order of the state
+                * words.  sha3_state already uses that order, though, so
+                * there's no need for a byteswap.
+                */
+               switch (block_size) {
+               case SHA3_224_BLOCK_SIZE:
+                       cpacf_kimd(CPACF_KIMD_SHA3_224, state,
+                                  data, nblocks * block_size);
+                       return;
+               case SHA3_256_BLOCK_SIZE:
+                       /*
+                        * This case handles both SHA3-256 and SHAKE256, since
+                        * they have the same block size.
+                        */
+                       cpacf_kimd(CPACF_KIMD_SHA3_256, state,
+                                  data, nblocks * block_size);
+                       return;
+               case SHA3_384_BLOCK_SIZE:
+                       cpacf_kimd(CPACF_KIMD_SHA3_384, state,
+                                  data, nblocks * block_size);
+                       return;
+               case SHA3_512_BLOCK_SIZE:
+                       cpacf_kimd(CPACF_KIMD_SHA3_512, state,
+                                  data, nblocks * block_size);
+                       return;
+               }
+       }
+       sha3_absorb_blocks_generic(state, data, nblocks, block_size);
+}
+
+static void sha3_keccakf(struct sha3_state *state)
+{
+       if (static_branch_likely(&have_sha3)) {
+               /*
+                * Passing zeroes into any of CPACF_KIMD_SHA3_* gives the plain
+                * Keccak-f permutation, which is what we want here.  Use
+                * SHA3-512 since it has the smallest block size.
+                */
+               static const u8 zeroes[SHA3_512_BLOCK_SIZE];
+
+               cpacf_kimd(CPACF_KIMD_SHA3_512, state, zeroes, sizeof(zeroes));
+       } else {
+               sha3_keccakf_generic(state);
+       }
+}
+
+#define sha3_mod_init_arch sha3_mod_init_arch
+static void sha3_mod_init_arch(void)
+{
+       int num_present = 0;
+       int num_possible = 0;
+
+       if (!cpu_have_feature(S390_CPU_FEATURE_MSA))
+               return;
+       /*
+        * Since all the SHA-3 functions are in Message-Security-Assist
+        * Extension 6, just treat them as all or nothing.  This way we need
+        * only one static_key.
+        */
+#define QUERY(opcode, func) \
+       ({ num_present += !!cpacf_query_func(opcode, func); num_possible++; })
+       QUERY(CPACF_KIMD, CPACF_KIMD_SHA3_224);
+       QUERY(CPACF_KIMD, CPACF_KIMD_SHA3_256);
+       QUERY(CPACF_KIMD, CPACF_KIMD_SHA3_384);
+       QUERY(CPACF_KIMD, CPACF_KIMD_SHA3_512);
+#undef QUERY
+
+       if (num_present == num_possible)
+               static_branch_enable(&have_sha3);
+       else if (num_present != 0)
+               pr_warn("Unsupported combination of SHA-3 facilities\n");
+}