From: Eric Biggers <ebiggers@kernel.org>
Date: Thu, 19 Mar 2026 06:17:06 +0000 (-0700)
Subject: crypto: arm/ghash - Make the "ghash" crypto_shash NEON-only
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=39afaff983fe64ea749aafb3ebd63d44212d2860;p=thirdparty%2Fkernel%2Fstable.git

crypto: arm/ghash - Make the "ghash" crypto_shash NEON-only

arch/arm/crypto/ghash-ce-glue.c originally provided only a "ghash"
crypto_shash algorithm using PMULL if available, else NEON.

Significantly later, it was updated to also provide a full AES-GCM
implementation using PMULL.

This made the PMULL support in the "ghash" crypto_shash largely
obsolete.  Indeed, the arm64 equivalent of this file unconditionally
uses only ASIMD in its "ghash" crypto_shash.

Given that inconsistency and the fact that the NEON-only code is more
easily separable into the GHASH library than the PMULL based code is,
let's align with arm64 and just support NEON-only for the pure GHASH.

Acked-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20260319061723.1140720-6-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---

diff --git a/arch/arm/crypto/ghash-ce-glue.c b/arch/arm/crypto/ghash-ce-glue.c
index 454adcc62cc69..d7d787de7dd31 100644
--- a/arch/arm/crypto/ghash-ce-glue.c
+++ b/arch/arm/crypto/ghash-ce-glue.c
@@ -36,7 +36,7 @@ MODULE_ALIAS_CRYPTO("rfc4106(gcm(aes))");
 
 struct ghash_key {
 	be128	k;
-	u64	h[][2];
+	u64	h[1][2];
 };
 
 struct gcm_key {
@@ -51,12 +51,10 @@ struct arm_ghash_desc_ctx {
 };
 
 asmlinkage void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src,
-				       u64 const h[][2], const char *head);
+				       u64 const h[4][2], const char *head);
 
 asmlinkage void pmull_ghash_update_p8(int blocks, u64 dg[], const char *src,
-				      u64 const h[][2], const char *head);
-
-static __ro_after_init DEFINE_STATIC_KEY_FALSE(use_p64);
+				      u64 const h[1][2], const char *head);
 
 static int ghash_init(struct shash_desc *desc)
 {
@@ -70,10 +68,7 @@ static void ghash_do_update(int blocks, u64 dg[], const char *src,
 			    struct ghash_key *key, const char *head)
 {
 	kernel_neon_begin();
-	if (static_branch_likely(&use_p64))
-		pmull_ghash_update_p64(blocks, dg, src, key->h, head);
-	else
-		pmull_ghash_update_p8(blocks, dg, src, key->h, head);
+	pmull_ghash_update_p8(blocks, dg, src, key->h, head);
 	kernel_neon_end();
 }
 
@@ -147,19 +142,6 @@ static int ghash_setkey(struct crypto_shash *tfm,
 	/* needed for the fallback */
 	memcpy(&key->k, inkey, GHASH_BLOCK_SIZE);
 	ghash_reflect(key->h[0], &key->k);
-
-	if (static_branch_likely(&use_p64)) {
-		be128 h = key->k;
-
-		gf128mul_lle(&h, &key->k);
-		ghash_reflect(key->h[1], &h);
-
-		gf128mul_lle(&h, &key->k);
-		ghash_reflect(key->h[2], &h);
-
-		gf128mul_lle(&h, &key->k);
-		ghash_reflect(key->h[3], &h);
-	}
 	return 0;
 }
 
@@ -175,11 +157,11 @@ static struct shash_alg ghash_alg = {
 	.statesize		= sizeof(struct ghash_desc_ctx),
 
 	.base.cra_name		= "ghash",
-	.base.cra_driver_name	= "ghash-ce",
+	.base.cra_driver_name	= "ghash-neon",
 	.base.cra_priority	= 300,
 	.base.cra_flags		= CRYPTO_AHASH_ALG_BLOCK_ONLY,
 	.base.cra_blocksize	= GHASH_BLOCK_SIZE,
-	.base.cra_ctxsize	= sizeof(struct ghash_key) + sizeof(u64[2]),
+	.base.cra_ctxsize	= sizeof(struct ghash_key),
 	.base.cra_module	= THIS_MODULE,
 };
 
@@ -571,8 +553,6 @@ static int __init ghash_ce_mod_init(void)
 					    ARRAY_SIZE(gcm_aes_algs));
 		if (err)
 			return err;
-		ghash_alg.base.cra_ctxsize += 3 * sizeof(u64[2]);
-		static_branch_enable(&use_p64);
 	}
 
 	err = crypto_register_shash(&ghash_alg);