lib/crypto: gf128hash: Support GF128HASH_ARCH without all POLYVAL functions

Currently, some architectures (arm64 and x86) have optimized code for
both GHASH and POLYVAL.  Others (arm, powerpc, riscv, and s390) have
optimized code only for GHASH.  While POLYVAL support could be
implemented on these other architectures, until then we need to support
the case where arch-optimized functions are present only for GHASH.

Therefore, update the support for arch-optimized POLYVAL functions to
allow architectures to opt into supporting these functions individually.

The new meaning of CONFIG_CRYPTO_LIB_GF128HASH_ARCH is that some level
of GHASH and/or POLYVAL acceleration is provided.

Also provide an implementation of polyval_mul() based on
polyval_blocks_arch(), for when polyval_mul_arch() isn't implemented.

Acked-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20260319061723.1140720-3-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@kernel.org>

diff --git a/include/crypto/gf128hash.h b/include/crypto/gf128hash.h
index 5ffa86f5c13fd0b69e4b339715bc89efe2bfb76e..1052041e3499bd2d0a9f4baad61c954c8dc92fd8 100644 (file)
--- a/include/crypto/gf128hash.h
+++ b/include/crypto/gf128hash.h
@@ -44,20 +44,14 @@ struct polyval_elem {
  * exponentiation repeats the POLYVAL dot operation, with its "extra" x^-128.
  */
 struct polyval_key {
-#ifdef CONFIG_CRYPTO_LIB_GF128HASH_ARCH
-#ifdef CONFIG_ARM64
-       /** @h_powers: Powers of the hash key H^8 through H^1 */
-       struct polyval_elem h_powers[8];
-#elif defined(CONFIG_X86)
+#if defined(CONFIG_CRYPTO_LIB_GF128HASH_ARCH) && \
+       (defined(CONFIG_ARM64) || defined(CONFIG_X86))
        /** @h_powers: Powers of the hash key H^8 through H^1 */
        struct polyval_elem h_powers[8];
 #else
-#error "Unhandled arch"
-#endif
-#else /* CONFIG_CRYPTO_LIB_GF128HASH_ARCH */
        /** @h: The hash key H */
        struct polyval_elem h;
-#endif /* !CONFIG_CRYPTO_LIB_GF128HASH_ARCH */
+#endif
 };
 
 /**
@@ -84,19 +78,9 @@ struct polyval_ctx {
  *
  * Context: Any context.
  */
-#ifdef CONFIG_CRYPTO_LIB_GF128HASH_ARCH
 void polyval_preparekey(struct polyval_key *key,
                        const u8 raw_key[POLYVAL_BLOCK_SIZE]);
 
-#else
-static inline void polyval_preparekey(struct polyval_key *key,
-                                     const u8 raw_key[POLYVAL_BLOCK_SIZE])
-{
-       /* Just a simple copy, so inline it. */
-       memcpy(key->h.bytes, raw_key, POLYVAL_BLOCK_SIZE);
-}
-#endif
-
 /**
  * polyval_init() - Initialize a POLYVAL context for a new message
  * @ctx: The context to initialize

diff --git a/lib/crypto/arm64/gf128hash.h b/lib/crypto/arm64/gf128hash.h
index c1012007adcf3babf3f1a41e7f553d88572d40ec..796c36804dda4271e0713874738f8cfac6fd2c27 100644 (file)
--- a/lib/crypto/arm64/gf128hash.h
+++ b/lib/crypto/arm64/gf128hash.h
@@ -17,6 +17,7 @@ asmlinkage void polyval_blocks_pmull(struct polyval_elem *acc,
                                     const struct polyval_key *key,
                                     const u8 *data, size_t nblocks);
 
+#define polyval_preparekey_arch polyval_preparekey_arch
 static void polyval_preparekey_arch(struct polyval_key *key,
                                    const u8 raw_key[POLYVAL_BLOCK_SIZE])
 {
@@ -40,6 +41,7 @@ static void polyval_preparekey_arch(struct polyval_key *key,
        }
 }
 
+#define polyval_mul_arch polyval_mul_arch
 static void polyval_mul_arch(struct polyval_elem *acc,
                             const struct polyval_key *key)
 {
@@ -51,6 +53,7 @@ static void polyval_mul_arch(struct polyval_elem *acc,
        }
 }
 
+#define polyval_blocks_arch polyval_blocks_arch
 static void polyval_blocks_arch(struct polyval_elem *acc,
                                const struct polyval_key *key,
                                const u8 *data, size_t nblocks)

diff --git a/lib/crypto/gf128hash.c b/lib/crypto/gf128hash.c
index 8bb848bf26b7092881a2c6f1542710da82a7c00c..05f44a9193f791c85c9df43c4a5abcadc4798b1f 100644 (file)
--- a/lib/crypto/gf128hash.c
+++ b/lib/crypto/gf128hash.c
@@ -217,16 +217,20 @@ polyval_blocks_generic(struct polyval_elem *acc, const struct polyval_elem *key,
        } while (--nblocks);
 }
 
-/* Include the arch-optimized implementation of POLYVAL, if one is available. */
 #ifdef CONFIG_CRYPTO_LIB_GF128HASH_ARCH
 #include "gf128hash.h" /* $(SRCARCH)/gf128hash.h */
+#endif
+
 void polyval_preparekey(struct polyval_key *key,
                        const u8 raw_key[POLYVAL_BLOCK_SIZE])
 {
+#ifdef polyval_preparekey_arch
        polyval_preparekey_arch(key, raw_key);
+#else
+       memcpy(key->h.bytes, raw_key, POLYVAL_BLOCK_SIZE);
+#endif
 }
 EXPORT_SYMBOL_GPL(polyval_preparekey);
-#endif /* Else, polyval_preparekey() is an inline function. */
 
 /*
  * polyval_mul_generic() and polyval_blocks_generic() take the key as a
@@ -238,8 +242,12 @@ EXPORT_SYMBOL_GPL(polyval_preparekey);
 
 static void polyval_mul(struct polyval_ctx *ctx)
 {
-#ifdef CONFIG_CRYPTO_LIB_GF128HASH_ARCH
+#ifdef polyval_mul_arch
        polyval_mul_arch(&ctx->acc, ctx->key);
+#elif defined(polyval_blocks_arch)
+       static const u8 zeroes[POLYVAL_BLOCK_SIZE];
+
+       polyval_blocks_arch(&ctx->acc, ctx->key, zeroes, 1);
 #else
        polyval_mul_generic(&ctx->acc, &ctx->key->h);
 #endif
@@ -248,7 +256,7 @@ static void polyval_mul(struct polyval_ctx *ctx)
 static void polyval_blocks(struct polyval_ctx *ctx,
                           const u8 *data, size_t nblocks)
 {
-#ifdef CONFIG_CRYPTO_LIB_GF128HASH_ARCH
+#ifdef polyval_blocks_arch
        polyval_blocks_arch(&ctx->acc, ctx->key, data, nblocks);
 #else
        polyval_blocks_generic(&ctx->acc, &ctx->key->h, data, nblocks);

diff --git a/lib/crypto/x86/gf128hash.h b/lib/crypto/x86/gf128hash.h
index fe506cf6431b2ab53ea00339a2b7d573fc39d5f8..adf6147ea6779f673e8173b06f40029ad2d72037 100644 (file)
--- a/lib/crypto/x86/gf128hash.h
+++ b/lib/crypto/x86/gf128hash.h
@@ -17,6 +17,7 @@ asmlinkage void polyval_blocks_pclmul_avx(struct polyval_elem *acc,
                                          const struct polyval_key *key,
                                          const u8 *data, size_t nblocks);
 
+#define polyval_preparekey_arch polyval_preparekey_arch
 static void polyval_preparekey_arch(struct polyval_key *key,
                                    const u8 raw_key[POLYVAL_BLOCK_SIZE])
 {
@@ -40,6 +41,7 @@ static void polyval_preparekey_arch(struct polyval_key *key,
        }
 }
 
+#define polyval_mul_arch polyval_mul_arch
 static void polyval_mul_arch(struct polyval_elem *acc,
                             const struct polyval_key *key)
 {
@@ -52,6 +54,7 @@ static void polyval_mul_arch(struct polyval_elem *acc,
        }
 }
 
+#define polyval_blocks_arch polyval_blocks_arch
 static void polyval_blocks_arch(struct polyval_elem *acc,
                                const struct polyval_key *key,
                                const u8 *data, size_t nblocks)