lib/crypto: riscv/ghash: Migrate optimized code into library

Remove the "ghash-riscv64-zvkg" crypto_shash algorithm.  Move the
corresponding assembly code into lib/crypto/, modify it to take the
length in blocks instead of bytes, and wire it up to the GHASH library.

This makes the GHASH library be optimized with the RISC-V Vector
Cryptography Extension.  It also greatly reduces the amount of
riscv-specific glue code that is needed, and it fixes the issue where
this optimized GHASH code was disabled by default.

Note that this RISC-V code has multiple opportunities for improvement,
such as adding more parallelism, providing an optimized multiplication
function, and directly supporting POLYVAL.  But for now, this commit
simply tweaks ghash_zvkg() slightly to make it compatible with the
library, then wires it up to ghash_blocks_arch().

ghash_preparekey_arch() is also implemented to store the copy of the raw
key needed by the vghsh.vv instruction.

Acked-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20260319061723.1140720-13-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@kernel.org>

diff --git a/arch/riscv/crypto/Kconfig b/arch/riscv/crypto/Kconfig
index 22d4eaab15f38ededfb97abddd65abbd55d7887b..c208f54afbcd781d6d820bdb8fe8fdce19dcc522 100644 (file)
--- a/arch/riscv/crypto/Kconfig
+++ b/arch/riscv/crypto/Kconfig
@@ -17,17 +17,6 @@ config CRYPTO_AES_RISCV64
          - Zvkb vector crypto extension (CTR)
          - Zvkg vector crypto extension (XTS)
 
-config CRYPTO_GHASH_RISCV64
-       tristate "Hash functions: GHASH"
-       depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \
-                  RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS
-       select CRYPTO_GCM
-       help
-         GCM GHASH function (NIST SP 800-38D)
-
-         Architecture: riscv64 using:
-         - Zvkg vector crypto extension
-
 config CRYPTO_SM3_RISCV64
        tristate "Hash functions: SM3 (ShangMi 3)"
        depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \

diff --git a/arch/riscv/crypto/Makefile b/arch/riscv/crypto/Makefile
index 183495a95cc0e1fc979e101c90388a092b576fc2..5c9ee1b876fa0ad7050e1956f2fa0d699dc9ac3e 100644 (file)
--- a/arch/riscv/crypto/Makefile
+++ b/arch/riscv/crypto/Makefile
@@ -4,9 +4,6 @@ obj-$(CONFIG_CRYPTO_AES_RISCV64) += aes-riscv64.o
 aes-riscv64-y := aes-riscv64-glue.o aes-riscv64-zvkned.o \
                 aes-riscv64-zvkned-zvbb-zvkg.o aes-riscv64-zvkned-zvkb.o
 
-obj-$(CONFIG_CRYPTO_GHASH_RISCV64) += ghash-riscv64.o
-ghash-riscv64-y := ghash-riscv64-glue.o ghash-riscv64-zvkg.o
-
 obj-$(CONFIG_CRYPTO_SM3_RISCV64) += sm3-riscv64.o
 sm3-riscv64-y := sm3-riscv64-glue.o sm3-riscv64-zvksh-zvkb.o
 

diff --git a/arch/riscv/crypto/ghash-riscv64-glue.c b/arch/riscv/crypto/ghash-riscv64-glue.c
deleted file mode 100644 (file)
index d86073d..0000000
--- a/arch/riscv/crypto/ghash-riscv64-glue.c
+++ /dev/null
@@ -1,146 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * GHASH using the RISC-V vector crypto extensions
- *
- * Copyright (C) 2023 VRULL GmbH
- * Author: Heiko Stuebner <heiko.stuebner@vrull.eu>
- *
- * Copyright (C) 2023 SiFive, Inc.
- * Author: Jerry Shih <jerry.shih@sifive.com>
- */
-
-#include <asm/simd.h>
-#include <asm/vector.h>
-#include <crypto/b128ops.h>
-#include <crypto/gf128mul.h>
-#include <crypto/ghash.h>
-#include <crypto/internal/hash.h>
-#include <crypto/internal/simd.h>
-#include <crypto/utils.h>
-#include <linux/errno.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/string.h>
-
-asmlinkage void ghash_zvkg(be128 *accumulator, const be128 *key, const u8 *data,
-                          size_t len);
-
-struct riscv64_ghash_tfm_ctx {
-       be128 key;
-};
-
-struct riscv64_ghash_desc_ctx {
-       be128 accumulator;
-};
-
-static int riscv64_ghash_setkey(struct crypto_shash *tfm, const u8 *key,
-                               unsigned int keylen)
-{
-       struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(tfm);
-
-       if (keylen != GHASH_BLOCK_SIZE)
-               return -EINVAL;
-
-       memcpy(&tctx->key, key, GHASH_BLOCK_SIZE);
-
-       return 0;
-}
-
-static int riscv64_ghash_init(struct shash_desc *desc)
-{
-       struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
-
-       *dctx = (struct riscv64_ghash_desc_ctx){};
-
-       return 0;
-}
-
-static inline void
-riscv64_ghash_blocks(const struct riscv64_ghash_tfm_ctx *tctx,
-                    struct riscv64_ghash_desc_ctx *dctx,
-                    const u8 *src, size_t srclen)
-{
-       /* The srclen is nonzero and a multiple of 16. */
-       if (crypto_simd_usable()) {
-               kernel_vector_begin();
-               ghash_zvkg(&dctx->accumulator, &tctx->key, src, srclen);
-               kernel_vector_end();
-       } else {
-               do {
-                       crypto_xor((u8 *)&dctx->accumulator, src,
-                                  GHASH_BLOCK_SIZE);
-                       gf128mul_lle(&dctx->accumulator, &tctx->key);
-                       src += GHASH_BLOCK_SIZE;
-                       srclen -= GHASH_BLOCK_SIZE;
-               } while (srclen);
-       }
-}
-
-static int riscv64_ghash_update(struct shash_desc *desc, const u8 *src,
-                               unsigned int srclen)
-{
-       const struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
-       struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
-
-       riscv64_ghash_blocks(tctx, dctx, src,
-                            round_down(srclen, GHASH_BLOCK_SIZE));
-       return srclen - round_down(srclen, GHASH_BLOCK_SIZE);
-}
-
-static int riscv64_ghash_finup(struct shash_desc *desc, const u8 *src,
-                              unsigned int len, u8 *out)
-{
-       const struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
-       struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
-
-       if (len) {
-               u8 buf[GHASH_BLOCK_SIZE] = {};
-
-               memcpy(buf, src, len);
-               riscv64_ghash_blocks(tctx, dctx, buf, GHASH_BLOCK_SIZE);
-               memzero_explicit(buf, sizeof(buf));
-       }
-
-       memcpy(out, &dctx->accumulator, GHASH_DIGEST_SIZE);
-       return 0;
-}
-
-static struct shash_alg riscv64_ghash_alg = {
-       .init = riscv64_ghash_init,
-       .update = riscv64_ghash_update,
-       .finup = riscv64_ghash_finup,
-       .setkey = riscv64_ghash_setkey,
-       .descsize = sizeof(struct riscv64_ghash_desc_ctx),
-       .digestsize = GHASH_DIGEST_SIZE,
-       .base = {
-               .cra_blocksize = GHASH_BLOCK_SIZE,
-               .cra_ctxsize = sizeof(struct riscv64_ghash_tfm_ctx),
-               .cra_priority = 300,
-               .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY,
-               .cra_name = "ghash",
-               .cra_driver_name = "ghash-riscv64-zvkg",
-               .cra_module = THIS_MODULE,
-       },
-};
-
-static int __init riscv64_ghash_mod_init(void)
-{
-       if (riscv_isa_extension_available(NULL, ZVKG) &&
-           riscv_vector_vlen() >= 128)
-               return crypto_register_shash(&riscv64_ghash_alg);
-
-       return -ENODEV;
-}
-
-static void __exit riscv64_ghash_mod_exit(void)
-{
-       crypto_unregister_shash(&riscv64_ghash_alg);
-}
-
-module_init(riscv64_ghash_mod_init);
-module_exit(riscv64_ghash_mod_exit);
-
-MODULE_DESCRIPTION("GHASH (RISC-V accelerated)");
-MODULE_AUTHOR("Heiko Stuebner <heiko.stuebner@vrull.eu>");
-MODULE_LICENSE("GPL");
-MODULE_ALIAS_CRYPTO("ghash");

diff --git a/include/crypto/gf128hash.h b/include/crypto/gf128hash.h
index 650652dd600347d42e8895fa532753415951e732..b798438cce238e6e79261d489d563209242e906e 100644 (file)
--- a/include/crypto/gf128hash.h
+++ b/include/crypto/gf128hash.h
@@ -44,6 +44,9 @@ struct ghash_key {
 #if defined(CONFIG_CRYPTO_LIB_GF128HASH_ARCH) && defined(CONFIG_PPC64)
        /** @htable: GHASH key format used by the POWER8 assembly code */
        u64 htable[4][2];
+#elif defined(CONFIG_CRYPTO_LIB_GF128HASH_ARCH) && defined(CONFIG_RISCV)
+       /** @h_raw: The hash key H, in GHASH format */
+       u8 h_raw[GHASH_BLOCK_SIZE];
 #endif
        /** @h: The hash key H, in POLYVAL format */
        struct polyval_elem h;

diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig
index f54add7d907075be4fd6cb26d4eccff9adbdd5e5..027802e0de332a7460c6b98e48dcf612b2f81d73 100644 (file)
--- a/lib/crypto/Kconfig
+++ b/lib/crypto/Kconfig
@@ -122,6 +122,8 @@ config CRYPTO_LIB_GF128HASH_ARCH
        default y if ARM && KERNEL_MODE_NEON
        default y if ARM64
        default y if PPC64 && VSX
+       default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \
+                    RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS
        default y if X86_64
 
 config CRYPTO_LIB_MD5

diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile
index 8a9084188778962a79c63876c1b2f47ec8e05efe..8950509833afe3133647bb2045bf4e43b83856b3 100644 (file)
--- a/lib/crypto/Makefile
+++ b/lib/crypto/Makefile
@@ -173,6 +173,7 @@ targets += powerpc/ghashp8-ppc.S
 OBJECT_FILES_NON_STANDARD_powerpc/ghashp8-ppc.o := y
 endif
 
+libgf128hash-$(CONFIG_RISCV) += riscv/ghash-riscv64-zvkg.o
 libgf128hash-$(CONFIG_X86) += x86/polyval-pclmul-avx.o
 endif # CONFIG_CRYPTO_LIB_GF128HASH_ARCH
 

diff --git a/lib/crypto/riscv/gf128hash.h b/lib/crypto/riscv/gf128hash.h
new file mode 100644 (file)
index 0000000..4301a03
--- /dev/null
+++ b/lib/crypto/riscv/gf128hash.h
@@ -0,0 +1,57 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * GHASH, RISC-V optimized
+ *
+ * Copyright (C) 2023 VRULL GmbH
+ * Copyright (C) 2023 SiFive, Inc.
+ * Copyright 2026 Google LLC
+ */
+
+#include <asm/simd.h>
+#include <asm/vector.h>
+
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_zvkg);
+
+asmlinkage void ghash_zvkg(u8 accumulator[GHASH_BLOCK_SIZE],
+                          const u8 key[GHASH_BLOCK_SIZE],
+                          const u8 *data, size_t nblocks);
+
+#define ghash_preparekey_arch ghash_preparekey_arch
+static void ghash_preparekey_arch(struct ghash_key *key,
+                                 const u8 raw_key[GHASH_BLOCK_SIZE])
+{
+       /* Save key in POLYVAL format for fallback */
+       ghash_key_to_polyval(raw_key, &key->h);
+
+       /* Save key in GHASH format for zvkg */
+       memcpy(key->h_raw, raw_key, GHASH_BLOCK_SIZE);
+}
+
+#define ghash_blocks_arch ghash_blocks_arch
+static void ghash_blocks_arch(struct polyval_elem *acc,
+                             const struct ghash_key *key,
+                             const u8 *data, size_t nblocks)
+{
+       if (static_branch_likely(&have_zvkg) && likely(may_use_simd())) {
+               u8 ghash_acc[GHASH_BLOCK_SIZE];
+
+               polyval_acc_to_ghash(acc, ghash_acc);
+
+               kernel_vector_begin();
+               ghash_zvkg(ghash_acc, key->h_raw, data, nblocks);
+               kernel_vector_end();
+
+               ghash_acc_to_polyval(ghash_acc, acc);
+               memzero_explicit(ghash_acc, sizeof(ghash_acc));
+       } else {
+               ghash_blocks_generic(acc, &key->h, data, nblocks);
+       }
+}
+
+#define gf128hash_mod_init_arch gf128hash_mod_init_arch
+static void gf128hash_mod_init_arch(void)
+{
+       if (riscv_isa_extension_available(NULL, ZVKG) &&
+           riscv_vector_vlen() >= 128)
+               static_branch_enable(&have_zvkg);
+}

diff --git a/arch/riscv/crypto/ghash-riscv64-zvkg.S b/lib/crypto/riscv/ghash-riscv64-zvkg.S
similarity index 91%
rename from arch/riscv/crypto/ghash-riscv64-zvkg.S
rename to lib/crypto/riscv/ghash-riscv64-zvkg.S
index f2b43fb4d434fc11e3e24e1f8e29242d4533828e..6a2a2f2bc7c80e36824305ec8185dd1d0f836d39 100644 (file)
--- a/arch/riscv/crypto/ghash-riscv64-zvkg.S
+++ b/lib/crypto/riscv/ghash-riscv64-zvkg.S
@@ -50,12 +50,13 @@
 #define ACCUMULATOR    a0
 #define KEY            a1
 #define DATA           a2
-#define LEN            a3
+#define NBLOCKS                a3
 
-// void ghash_zvkg(be128 *accumulator, const be128 *key, const u8 *data,
-//                size_t len);
+// void ghash_zvkg(u8 accumulator[GHASH_BLOCK_SIZE],
+//                const u8 key[GHASH_BLOCK_SIZE],
+//                const u8 *data, size_t nblocks);
 //
-// |len| must be nonzero and a multiple of 16 (GHASH_BLOCK_SIZE).
+// |nblocks| must be nonzero.
 SYM_FUNC_START(ghash_zvkg)
        vsetivli        zero, 4, e32, m1, ta, ma
        vle32.v         v1, (ACCUMULATOR)
@@ -64,8 +65,8 @@ SYM_FUNC_START(ghash_zvkg)
        vle32.v         v3, (DATA)
        vghsh.vv        v1, v2, v3
        addi            DATA, DATA, 16
-       addi            LEN, LEN, -16
-       bnez            LEN, .Lnext_block
+       addi            NBLOCKS, NBLOCKS, -1
+       bnez            NBLOCKS, .Lnext_block
 
        vse32.v         v1, (ACCUMULATOR)
        ret