From 29e39a11f541d068ed7155368f4a79aa0ddf9c52 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 10 Dec 2025 17:18:35 -0800 Subject: [PATCH] lib/crypto: arm/nh: Migrate optimized code into library Migrate the arm32 NEON implementation of NH into lib/crypto/. This makes the nh() function be optimized on arm32 kernels. Note: this temporarily makes the adiantum template not utilize the arm32 optimized NH code. This is resolved in a later commit that converts the adiantum template to use nh() instead of "nhpoly1305". Link: https://lore.kernel.org/r/20251211011846.8179-4-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/arm/crypto/Kconfig | 10 --- arch/arm/crypto/Makefile | 2 - arch/arm/crypto/nhpoly1305-neon-glue.c | 80 ------------------- lib/crypto/Kconfig | 1 + lib/crypto/Makefile | 1 + .../crypto => lib/crypto/arm}/nh-neon-core.S | 0 lib/crypto/arm/nh.h | 33 ++++++++ 7 files changed, 35 insertions(+), 92 deletions(-) delete mode 100644 arch/arm/crypto/nhpoly1305-neon-glue.c rename {arch/arm/crypto => lib/crypto/arm}/nh-neon-core.S (100%) create mode 100644 lib/crypto/arm/nh.h diff --git a/arch/arm/crypto/Kconfig b/arch/arm/crypto/Kconfig index f30d743df2643..3eb5071bea144 100644 --- a/arch/arm/crypto/Kconfig +++ b/arch/arm/crypto/Kconfig @@ -23,16 +23,6 @@ config CRYPTO_GHASH_ARM_CE that is part of the ARMv8 Crypto Extensions, or a slower variant that uses the vmull.p8 instruction that is part of the basic NEON ISA. -config CRYPTO_NHPOLY1305_NEON - tristate "Hash functions: NHPoly1305 (NEON)" - depends on KERNEL_MODE_NEON - select CRYPTO_NHPOLY1305 - help - NHPoly1305 hash function (Adiantum) - - Architecture: arm using: - - NEON (Advanced SIMD) extensions - config CRYPTO_AES_ARM tristate "Ciphers: AES" select CRYPTO_ALGAPI diff --git a/arch/arm/crypto/Makefile b/arch/arm/crypto/Makefile index 86dd43313dbfd..d6683e9d49928 100644 --- a/arch/arm/crypto/Makefile +++ b/arch/arm/crypto/Makefile @@ -5,7 +5,6 @@ obj-$(CONFIG_CRYPTO_AES_ARM) += aes-arm.o obj-$(CONFIG_CRYPTO_AES_ARM_BS) += aes-arm-bs.o -obj-$(CONFIG_CRYPTO_NHPOLY1305_NEON) += nhpoly1305-neon.o obj-$(CONFIG_CRYPTO_AES_ARM_CE) += aes-arm-ce.o obj-$(CONFIG_CRYPTO_GHASH_ARM_CE) += ghash-arm-ce.o @@ -14,4 +13,3 @@ aes-arm-y := aes-cipher-core.o aes-cipher-glue.o aes-arm-bs-y := aes-neonbs-core.o aes-neonbs-glue.o aes-arm-ce-y := aes-ce-core.o aes-ce-glue.o ghash-arm-ce-y := ghash-ce-core.o ghash-ce-glue.o -nhpoly1305-neon-y := nh-neon-core.o nhpoly1305-neon-glue.o diff --git a/arch/arm/crypto/nhpoly1305-neon-glue.c b/arch/arm/crypto/nhpoly1305-neon-glue.c deleted file mode 100644 index 62cf7ccdde736..0000000000000 --- a/arch/arm/crypto/nhpoly1305-neon-glue.c +++ /dev/null @@ -1,80 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * NHPoly1305 - ε-almost-∆-universal hash function for Adiantum - * (NEON accelerated version) - * - * Copyright 2018 Google LLC - */ - -#include -#include -#include -#include -#include -#include - -asmlinkage void nh_neon(const u32 *key, const u8 *message, size_t message_len, - __le64 hash[NH_NUM_PASSES]); - -static int nhpoly1305_neon_update(struct shash_desc *desc, - const u8 *src, unsigned int srclen) -{ - if (srclen < 64 || !crypto_simd_usable()) - return crypto_nhpoly1305_update(desc, src, srclen); - - do { - unsigned int n = min_t(unsigned int, srclen, SZ_4K); - - kernel_neon_begin(); - crypto_nhpoly1305_update_helper(desc, src, n, nh_neon); - kernel_neon_end(); - src += n; - srclen -= n; - } while (srclen); - return 0; -} - -static int nhpoly1305_neon_digest(struct shash_desc *desc, - const u8 *src, unsigned int srclen, u8 *out) -{ - return crypto_nhpoly1305_init(desc) ?: - nhpoly1305_neon_update(desc, src, srclen) ?: - crypto_nhpoly1305_final(desc, out); -} - -static struct shash_alg nhpoly1305_alg = { - .base.cra_name = "nhpoly1305", - .base.cra_driver_name = "nhpoly1305-neon", - .base.cra_priority = 200, - .base.cra_ctxsize = sizeof(struct nhpoly1305_key), - .base.cra_module = THIS_MODULE, - .digestsize = POLY1305_DIGEST_SIZE, - .init = crypto_nhpoly1305_init, - .update = nhpoly1305_neon_update, - .final = crypto_nhpoly1305_final, - .digest = nhpoly1305_neon_digest, - .setkey = crypto_nhpoly1305_setkey, - .descsize = sizeof(struct nhpoly1305_state), -}; - -static int __init nhpoly1305_mod_init(void) -{ - if (!(elf_hwcap & HWCAP_NEON)) - return -ENODEV; - - return crypto_register_shash(&nhpoly1305_alg); -} - -static void __exit nhpoly1305_mod_exit(void) -{ - crypto_unregister_shash(&nhpoly1305_alg); -} - -module_init(nhpoly1305_mod_init); -module_exit(nhpoly1305_mod_exit); - -MODULE_DESCRIPTION("NHPoly1305 ε-almost-∆-universal hash function (NEON-accelerated)"); -MODULE_LICENSE("GPL v2"); -MODULE_AUTHOR("Eric Biggers "); -MODULE_ALIAS_CRYPTO("nhpoly1305"); -MODULE_ALIAS_CRYPTO("nhpoly1305-neon"); diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index f14c9f5974d8f..c6ee7ca776320 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -117,6 +117,7 @@ config CRYPTO_LIB_NH config CRYPTO_LIB_NH_ARCH bool depends on CRYPTO_LIB_NH && !UML + default y if ARM && KERNEL_MODE_NEON config CRYPTO_LIB_POLY1305 tristate diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 929b845688096..6dae7e1828470 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -135,6 +135,7 @@ obj-$(CONFIG_CRYPTO_LIB_NH) += libnh.o libnh-y := nh.o ifeq ($(CONFIG_CRYPTO_LIB_NH_ARCH),y) CFLAGS_nh.o += -I$(src)/$(SRCARCH) +libnh-$(CONFIG_ARM) += arm/nh-neon-core.o endif ################################################################################ diff --git a/arch/arm/crypto/nh-neon-core.S b/lib/crypto/arm/nh-neon-core.S similarity index 100% rename from arch/arm/crypto/nh-neon-core.S rename to lib/crypto/arm/nh-neon-core.S diff --git a/lib/crypto/arm/nh.h b/lib/crypto/arm/nh.h new file mode 100644 index 0000000000000..c9f39d8193362 --- /dev/null +++ b/lib/crypto/arm/nh.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * ARM32 accelerated implementation of NH + * + * Copyright 2018 Google LLC + */ + +#include +#include + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); + +asmlinkage void nh_neon(const u32 *key, const u8 *message, size_t message_len, + __le64 hash[NH_NUM_PASSES]); + +static bool nh_arch(const u32 *key, const u8 *message, size_t message_len, + __le64 hash[NH_NUM_PASSES]) +{ + if (static_branch_likely(&have_neon) && message_len >= 64 && + may_use_simd()) { + scoped_ksimd() + nh_neon(key, message, message_len, hash); + return true; + } + return false; +} + +#define nh_mod_init_arch nh_mod_init_arch +static void nh_mod_init_arch(void) +{ + if (elf_hwcap & HWCAP_NEON) + static_branch_enable(&have_neon); +} -- 2.47.3