From: Eric Biggers Date: Thu, 19 Mar 2026 06:17:13 +0000 (-0700) Subject: lib/crypto: riscv/ghash: Migrate optimized code into library X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=af413d71f09d4dde28277319926c1c3a6ec8b8d4;p=thirdparty%2Fkernel%2Fstable.git lib/crypto: riscv/ghash: Migrate optimized code into library Remove the "ghash-riscv64-zvkg" crypto_shash algorithm. Move the corresponding assembly code into lib/crypto/, modify it to take the length in blocks instead of bytes, and wire it up to the GHASH library. This makes the GHASH library be optimized with the RISC-V Vector Cryptography Extension. It also greatly reduces the amount of riscv-specific glue code that is needed, and it fixes the issue where this optimized GHASH code was disabled by default. Note that this RISC-V code has multiple opportunities for improvement, such as adding more parallelism, providing an optimized multiplication function, and directly supporting POLYVAL. But for now, this commit simply tweaks ghash_zvkg() slightly to make it compatible with the library, then wires it up to ghash_blocks_arch(). ghash_preparekey_arch() is also implemented to store the copy of the raw key needed by the vghsh.vv instruction. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260319061723.1140720-13-ebiggers@kernel.org Signed-off-by: Eric Biggers --- diff --git a/arch/riscv/crypto/Kconfig b/arch/riscv/crypto/Kconfig index 22d4eaab15f38..c208f54afbcd7 100644 --- a/arch/riscv/crypto/Kconfig +++ b/arch/riscv/crypto/Kconfig @@ -17,17 +17,6 @@ config CRYPTO_AES_RISCV64 - Zvkb vector crypto extension (CTR) - Zvkg vector crypto extension (XTS) -config CRYPTO_GHASH_RISCV64 - tristate "Hash functions: GHASH" - depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ - RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS - select CRYPTO_GCM - help - GCM GHASH function (NIST SP 800-38D) - - Architecture: riscv64 using: - - Zvkg vector crypto extension - config CRYPTO_SM3_RISCV64 tristate "Hash functions: SM3 (ShangMi 3)" depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ diff --git a/arch/riscv/crypto/Makefile b/arch/riscv/crypto/Makefile index 183495a95cc0e..5c9ee1b876fa0 100644 --- a/arch/riscv/crypto/Makefile +++ b/arch/riscv/crypto/Makefile @@ -4,9 +4,6 @@ obj-$(CONFIG_CRYPTO_AES_RISCV64) += aes-riscv64.o aes-riscv64-y := aes-riscv64-glue.o aes-riscv64-zvkned.o \ aes-riscv64-zvkned-zvbb-zvkg.o aes-riscv64-zvkned-zvkb.o -obj-$(CONFIG_CRYPTO_GHASH_RISCV64) += ghash-riscv64.o -ghash-riscv64-y := ghash-riscv64-glue.o ghash-riscv64-zvkg.o - obj-$(CONFIG_CRYPTO_SM3_RISCV64) += sm3-riscv64.o sm3-riscv64-y := sm3-riscv64-glue.o sm3-riscv64-zvksh-zvkb.o diff --git a/arch/riscv/crypto/ghash-riscv64-glue.c b/arch/riscv/crypto/ghash-riscv64-glue.c deleted file mode 100644 index d86073d253872..0000000000000 --- a/arch/riscv/crypto/ghash-riscv64-glue.c +++ /dev/null @@ -1,146 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * GHASH using the RISC-V vector crypto extensions - * - * Copyright (C) 2023 VRULL GmbH - * Author: Heiko Stuebner - * - * Copyright (C) 2023 SiFive, Inc. - * Author: Jerry Shih - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -asmlinkage void ghash_zvkg(be128 *accumulator, const be128 *key, const u8 *data, - size_t len); - -struct riscv64_ghash_tfm_ctx { - be128 key; -}; - -struct riscv64_ghash_desc_ctx { - be128 accumulator; -}; - -static int riscv64_ghash_setkey(struct crypto_shash *tfm, const u8 *key, - unsigned int keylen) -{ - struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(tfm); - - if (keylen != GHASH_BLOCK_SIZE) - return -EINVAL; - - memcpy(&tctx->key, key, GHASH_BLOCK_SIZE); - - return 0; -} - -static int riscv64_ghash_init(struct shash_desc *desc) -{ - struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - *dctx = (struct riscv64_ghash_desc_ctx){}; - - return 0; -} - -static inline void -riscv64_ghash_blocks(const struct riscv64_ghash_tfm_ctx *tctx, - struct riscv64_ghash_desc_ctx *dctx, - const u8 *src, size_t srclen) -{ - /* The srclen is nonzero and a multiple of 16. */ - if (crypto_simd_usable()) { - kernel_vector_begin(); - ghash_zvkg(&dctx->accumulator, &tctx->key, src, srclen); - kernel_vector_end(); - } else { - do { - crypto_xor((u8 *)&dctx->accumulator, src, - GHASH_BLOCK_SIZE); - gf128mul_lle(&dctx->accumulator, &tctx->key); - src += GHASH_BLOCK_SIZE; - srclen -= GHASH_BLOCK_SIZE; - } while (srclen); - } -} - -static int riscv64_ghash_update(struct shash_desc *desc, const u8 *src, - unsigned int srclen) -{ - const struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); - struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - riscv64_ghash_blocks(tctx, dctx, src, - round_down(srclen, GHASH_BLOCK_SIZE)); - return srclen - round_down(srclen, GHASH_BLOCK_SIZE); -} - -static int riscv64_ghash_finup(struct shash_desc *desc, const u8 *src, - unsigned int len, u8 *out) -{ - const struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); - struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - if (len) { - u8 buf[GHASH_BLOCK_SIZE] = {}; - - memcpy(buf, src, len); - riscv64_ghash_blocks(tctx, dctx, buf, GHASH_BLOCK_SIZE); - memzero_explicit(buf, sizeof(buf)); - } - - memcpy(out, &dctx->accumulator, GHASH_DIGEST_SIZE); - return 0; -} - -static struct shash_alg riscv64_ghash_alg = { - .init = riscv64_ghash_init, - .update = riscv64_ghash_update, - .finup = riscv64_ghash_finup, - .setkey = riscv64_ghash_setkey, - .descsize = sizeof(struct riscv64_ghash_desc_ctx), - .digestsize = GHASH_DIGEST_SIZE, - .base = { - .cra_blocksize = GHASH_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct riscv64_ghash_tfm_ctx), - .cra_priority = 300, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, - .cra_name = "ghash", - .cra_driver_name = "ghash-riscv64-zvkg", - .cra_module = THIS_MODULE, - }, -}; - -static int __init riscv64_ghash_mod_init(void) -{ - if (riscv_isa_extension_available(NULL, ZVKG) && - riscv_vector_vlen() >= 128) - return crypto_register_shash(&riscv64_ghash_alg); - - return -ENODEV; -} - -static void __exit riscv64_ghash_mod_exit(void) -{ - crypto_unregister_shash(&riscv64_ghash_alg); -} - -module_init(riscv64_ghash_mod_init); -module_exit(riscv64_ghash_mod_exit); - -MODULE_DESCRIPTION("GHASH (RISC-V accelerated)"); -MODULE_AUTHOR("Heiko Stuebner "); -MODULE_LICENSE("GPL"); -MODULE_ALIAS_CRYPTO("ghash"); diff --git a/include/crypto/gf128hash.h b/include/crypto/gf128hash.h index 650652dd60034..b798438cce238 100644 --- a/include/crypto/gf128hash.h +++ b/include/crypto/gf128hash.h @@ -44,6 +44,9 @@ struct ghash_key { #if defined(CONFIG_CRYPTO_LIB_GF128HASH_ARCH) && defined(CONFIG_PPC64) /** @htable: GHASH key format used by the POWER8 assembly code */ u64 htable[4][2]; +#elif defined(CONFIG_CRYPTO_LIB_GF128HASH_ARCH) && defined(CONFIG_RISCV) + /** @h_raw: The hash key H, in GHASH format */ + u8 h_raw[GHASH_BLOCK_SIZE]; #endif /** @h: The hash key H, in POLYVAL format */ struct polyval_elem h; diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index f54add7d90707..027802e0de332 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -122,6 +122,8 @@ config CRYPTO_LIB_GF128HASH_ARCH default y if ARM && KERNEL_MODE_NEON default y if ARM64 default y if PPC64 && VSX + default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ + RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS default y if X86_64 config CRYPTO_LIB_MD5 diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 8a90841887789..8950509833afe 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -173,6 +173,7 @@ targets += powerpc/ghashp8-ppc.S OBJECT_FILES_NON_STANDARD_powerpc/ghashp8-ppc.o := y endif +libgf128hash-$(CONFIG_RISCV) += riscv/ghash-riscv64-zvkg.o libgf128hash-$(CONFIG_X86) += x86/polyval-pclmul-avx.o endif # CONFIG_CRYPTO_LIB_GF128HASH_ARCH diff --git a/lib/crypto/riscv/gf128hash.h b/lib/crypto/riscv/gf128hash.h new file mode 100644 index 0000000000000..4301a0384f609 --- /dev/null +++ b/lib/crypto/riscv/gf128hash.h @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * GHASH, RISC-V optimized + * + * Copyright (C) 2023 VRULL GmbH + * Copyright (C) 2023 SiFive, Inc. + * Copyright 2026 Google LLC + */ + +#include +#include + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_zvkg); + +asmlinkage void ghash_zvkg(u8 accumulator[GHASH_BLOCK_SIZE], + const u8 key[GHASH_BLOCK_SIZE], + const u8 *data, size_t nblocks); + +#define ghash_preparekey_arch ghash_preparekey_arch +static void ghash_preparekey_arch(struct ghash_key *key, + const u8 raw_key[GHASH_BLOCK_SIZE]) +{ + /* Save key in POLYVAL format for fallback */ + ghash_key_to_polyval(raw_key, &key->h); + + /* Save key in GHASH format for zvkg */ + memcpy(key->h_raw, raw_key, GHASH_BLOCK_SIZE); +} + +#define ghash_blocks_arch ghash_blocks_arch +static void ghash_blocks_arch(struct polyval_elem *acc, + const struct ghash_key *key, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_zvkg) && likely(may_use_simd())) { + u8 ghash_acc[GHASH_BLOCK_SIZE]; + + polyval_acc_to_ghash(acc, ghash_acc); + + kernel_vector_begin(); + ghash_zvkg(ghash_acc, key->h_raw, data, nblocks); + kernel_vector_end(); + + ghash_acc_to_polyval(ghash_acc, acc); + memzero_explicit(ghash_acc, sizeof(ghash_acc)); + } else { + ghash_blocks_generic(acc, &key->h, data, nblocks); + } +} + +#define gf128hash_mod_init_arch gf128hash_mod_init_arch +static void gf128hash_mod_init_arch(void) +{ + if (riscv_isa_extension_available(NULL, ZVKG) && + riscv_vector_vlen() >= 128) + static_branch_enable(&have_zvkg); +} diff --git a/arch/riscv/crypto/ghash-riscv64-zvkg.S b/lib/crypto/riscv/ghash-riscv64-zvkg.S similarity index 91% rename from arch/riscv/crypto/ghash-riscv64-zvkg.S rename to lib/crypto/riscv/ghash-riscv64-zvkg.S index f2b43fb4d434f..6a2a2f2bc7c80 100644 --- a/arch/riscv/crypto/ghash-riscv64-zvkg.S +++ b/lib/crypto/riscv/ghash-riscv64-zvkg.S @@ -50,12 +50,13 @@ #define ACCUMULATOR a0 #define KEY a1 #define DATA a2 -#define LEN a3 +#define NBLOCKS a3 -// void ghash_zvkg(be128 *accumulator, const be128 *key, const u8 *data, -// size_t len); +// void ghash_zvkg(u8 accumulator[GHASH_BLOCK_SIZE], +// const u8 key[GHASH_BLOCK_SIZE], +// const u8 *data, size_t nblocks); // -// |len| must be nonzero and a multiple of 16 (GHASH_BLOCK_SIZE). +// |nblocks| must be nonzero. SYM_FUNC_START(ghash_zvkg) vsetivli zero, 4, e32, m1, ta, ma vle32.v v1, (ACCUMULATOR) @@ -64,8 +65,8 @@ SYM_FUNC_START(ghash_zvkg) vle32.v v3, (DATA) vghsh.vv v1, v2, v3 addi DATA, DATA, 16 - addi LEN, LEN, -16 - bnez LEN, .Lnext_block + addi NBLOCKS, NBLOCKS, -1 + bnez NBLOCKS, .Lnext_block vse32.v v1, (ACCUMULATOR) ret