// SPDX-License-Identifier: GPL-2.0-only
/*
- * Accelerated GHASH implementation with ARMv8 PMULL instructions.
+ * AES-GCM using ARMv8 Crypto Extensions
*
* Copyright (C) 2014 - 2018 Linaro Ltd. <ard.biesheuvel@linaro.org>
*/
#include <crypto/ghash.h>
#include <crypto/gf128mul.h>
#include <crypto/internal/aead.h>
-#include <crypto/internal/hash.h>
#include <crypto/internal/skcipher.h>
#include <crypto/scatterwalk.h>
#include <linux/cpufeature.h>
#include <asm/simd.h>
-MODULE_DESCRIPTION("GHASH and AES-GCM using ARMv8 Crypto Extensions");
+MODULE_DESCRIPTION("AES-GCM using ARMv8 Crypto Extensions");
MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
MODULE_LICENSE("GPL v2");
-MODULE_ALIAS_CRYPTO("ghash");
+MODULE_ALIAS_CRYPTO("gcm(aes)");
+MODULE_ALIAS_CRYPTO("rfc4106(gcm(aes))");
#define RFC4106_NONCE_SIZE 4
u64 h[][2];
};
-struct arm_ghash_desc_ctx {
- u64 digest[GHASH_DIGEST_SIZE/sizeof(u64)];
-};
-
struct gcm_aes_ctx {
struct aes_enckey aes_key;
u8 nonce[RFC4106_NONCE_SIZE];
asmlinkage void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src,
u64 const h[][2], const char *head);
-asmlinkage void pmull_ghash_update_p8(int blocks, u64 dg[], const char *src,
- u64 const h[][2], const char *head);
-
asmlinkage void pmull_gcm_encrypt(int bytes, u8 dst[], const u8 src[],
u64 const h[][2], u64 dg[], u8 ctr[],
u32 const rk[], int rounds, u8 tag[]);
u32 const rk[], int rounds, const u8 l[],
const u8 tag[], u64 authsize);
-static int ghash_init(struct shash_desc *desc)
-{
- struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc);
-
- *ctx = (struct arm_ghash_desc_ctx){};
- return 0;
-}
-
-static __always_inline
-void ghash_do_simd_update(int blocks, u64 dg[], const char *src,
- struct ghash_key *key, const char *head,
- void (*simd_update)(int blocks, u64 dg[],
- const char *src,
- u64 const h[][2],
- const char *head))
+static void ghash_do_simd_update(int blocks, u64 dg[], const char *src,
+ struct ghash_key *key, const char *head)
{
scoped_ksimd()
- simd_update(blocks, dg, src, key->h, head);
-}
-
-/* avoid hogging the CPU for too long */
-#define MAX_BLOCKS (SZ_64K / GHASH_BLOCK_SIZE)
-
-static int ghash_update(struct shash_desc *desc, const u8 *src,
- unsigned int len)
-{
- struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc);
- struct ghash_key *key = crypto_shash_ctx(desc->tfm);
- int blocks;
-
- blocks = len / GHASH_BLOCK_SIZE;
- len -= blocks * GHASH_BLOCK_SIZE;
-
- do {
- int chunk = min(blocks, MAX_BLOCKS);
-
- ghash_do_simd_update(chunk, ctx->digest, src, key, NULL,
- pmull_ghash_update_p8);
- blocks -= chunk;
- src += chunk * GHASH_BLOCK_SIZE;
- } while (unlikely(blocks > 0));
- return len;
-}
-
-static int ghash_export(struct shash_desc *desc, void *out)
-{
- struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc);
- u8 *dst = out;
-
- put_unaligned_be64(ctx->digest[1], dst);
- put_unaligned_be64(ctx->digest[0], dst + 8);
- return 0;
-}
-
-static int ghash_import(struct shash_desc *desc, const void *in)
-{
- struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc);
- const u8 *src = in;
-
- ctx->digest[1] = get_unaligned_be64(src);
- ctx->digest[0] = get_unaligned_be64(src + 8);
- return 0;
-}
-
-static int ghash_finup(struct shash_desc *desc, const u8 *src,
- unsigned int len, u8 *dst)
-{
- struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc);
- struct ghash_key *key = crypto_shash_ctx(desc->tfm);
-
- if (len) {
- u8 buf[GHASH_BLOCK_SIZE] = {};
-
- memcpy(buf, src, len);
- ghash_do_simd_update(1, ctx->digest, buf, key, NULL,
- pmull_ghash_update_p8);
- memzero_explicit(buf, sizeof(buf));
- }
- return ghash_export(desc, dst);
+ pmull_ghash_update_p64(blocks, dg, src, key->h, head);
}
static void ghash_reflect(u64 h[], const be128 *k)
h[1] ^= 0xc200000000000000UL;
}
-static int ghash_setkey(struct crypto_shash *tfm,
- const u8 *inkey, unsigned int keylen)
-{
- struct ghash_key *key = crypto_shash_ctx(tfm);
-
- if (keylen != GHASH_BLOCK_SIZE)
- return -EINVAL;
-
- /* needed for the fallback */
- memcpy(&key->k, inkey, GHASH_BLOCK_SIZE);
-
- ghash_reflect(key->h[0], &key->k);
- return 0;
-}
-
-static struct shash_alg ghash_alg = {
- .base.cra_name = "ghash",
- .base.cra_driver_name = "ghash-neon",
- .base.cra_priority = 150,
- .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY,
- .base.cra_blocksize = GHASH_BLOCK_SIZE,
- .base.cra_ctxsize = sizeof(struct ghash_key) + sizeof(u64[2]),
- .base.cra_module = THIS_MODULE,
-
- .digestsize = GHASH_DIGEST_SIZE,
- .init = ghash_init,
- .update = ghash_update,
- .finup = ghash_finup,
- .setkey = ghash_setkey,
- .export = ghash_export,
- .import = ghash_import,
- .descsize = sizeof(struct arm_ghash_desc_ctx),
- .statesize = sizeof(struct ghash_desc_ctx),
-};
-
static int gcm_aes_setkey(struct crypto_aead *tfm, const u8 *inkey,
unsigned int keylen)
{
int blocks = count / GHASH_BLOCK_SIZE;
ghash_do_simd_update(blocks, dg, src, &ctx->ghash_key,
- *buf_count ? buf : NULL,
- pmull_ghash_update_p64);
-
+ *buf_count ? buf : NULL);
src += blocks * GHASH_BLOCK_SIZE;
count %= GHASH_BLOCK_SIZE;
*buf_count = 0;
if (buf_count) {
memset(&buf[buf_count], 0, GHASH_BLOCK_SIZE - buf_count);
- ghash_do_simd_update(1, dg, buf, &ctx->ghash_key, NULL,
- pmull_ghash_update_p64);
+ ghash_do_simd_update(1, dg, buf, &ctx->ghash_key, NULL);
}
}
static int __init ghash_ce_mod_init(void)
{
- if (!cpu_have_named_feature(ASIMD))
+ if (!cpu_have_named_feature(ASIMD) || !cpu_have_named_feature(PMULL))
return -ENODEV;
- if (cpu_have_named_feature(PMULL))
- return crypto_register_aeads(gcm_aes_algs,
- ARRAY_SIZE(gcm_aes_algs));
-
- return crypto_register_shash(&ghash_alg);
+ return crypto_register_aeads(gcm_aes_algs, ARRAY_SIZE(gcm_aes_algs));
}
static void __exit ghash_ce_mod_exit(void)
{
- if (cpu_have_named_feature(PMULL))
- crypto_unregister_aeads(gcm_aes_algs, ARRAY_SIZE(gcm_aes_algs));
- else
- crypto_unregister_shash(&ghash_alg);
+ crypto_unregister_aeads(gcm_aes_algs, ARRAY_SIZE(gcm_aes_algs));
}
static const struct cpu_feature __maybe_unused ghash_cpu_feature[] = {
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
- * POLYVAL library functions, arm64 optimized
+ * GHASH and POLYVAL, arm64 optimized
*
* Copyright 2025 Google LLC
*/
#define NUM_H_POWERS 8
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_asimd);
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull);
+asmlinkage void pmull_ghash_update_p8(size_t blocks, struct polyval_elem *dg,
+ const u8 *src,
+ const struct polyval_elem *h);
asmlinkage void polyval_mul_pmull(struct polyval_elem *a,
const struct polyval_elem *b);
asmlinkage void polyval_blocks_pmull(struct polyval_elem *acc,
}
}
+static void polyval_mul_arm64(struct polyval_elem *a,
+ const struct polyval_elem *b)
+{
+ if (static_branch_likely(&have_asimd) && may_use_simd()) {
+ static const u8 zeroes[GHASH_BLOCK_SIZE];
+
+ scoped_ksimd() {
+ if (static_branch_likely(&have_pmull)) {
+ polyval_mul_pmull(a, b);
+ } else {
+ /*
+ * Note that this is indeed equivalent to a
+ * POLYVAL multiplication, since it takes the
+ * accumulator and key in POLYVAL format, and
+ * byte-swapping a block of zeroes is a no-op.
+ */
+ pmull_ghash_update_p8(1, a, zeroes, b);
+ }
+ }
+ } else {
+ polyval_mul_generic(a, b);
+ }
+}
+
+#define ghash_mul_arch ghash_mul_arch
+static void ghash_mul_arch(struct polyval_elem *acc,
+ const struct ghash_key *key)
+{
+ polyval_mul_arm64(acc, &key->h);
+}
+
#define polyval_mul_arch polyval_mul_arch
static void polyval_mul_arch(struct polyval_elem *acc,
const struct polyval_key *key)
{
- if (static_branch_likely(&have_pmull) && may_use_simd()) {
- scoped_ksimd()
- polyval_mul_pmull(acc, &key->h_powers[NUM_H_POWERS - 1]);
+ polyval_mul_arm64(acc, &key->h_powers[NUM_H_POWERS - 1]);
+}
+
+#define ghash_blocks_arch ghash_blocks_arch
+static void ghash_blocks_arch(struct polyval_elem *acc,
+ const struct ghash_key *key,
+ const u8 *data, size_t nblocks)
+{
+ if (static_branch_likely(&have_asimd) && may_use_simd()) {
+ do {
+ /* Allow rescheduling every 4 KiB. */
+ size_t n = min_t(size_t, nblocks,
+ 4096 / GHASH_BLOCK_SIZE);
+
+ scoped_ksimd()
+ pmull_ghash_update_p8(n, acc, data, &key->h);
+ data += n * GHASH_BLOCK_SIZE;
+ nblocks -= n;
+ } while (nblocks);
} else {
- polyval_mul_generic(acc, &key->h_powers[NUM_H_POWERS - 1]);
+ ghash_blocks_generic(acc, &key->h, data, nblocks);
}
}
#define gf128hash_mod_init_arch gf128hash_mod_init_arch
static void gf128hash_mod_init_arch(void)
{
- if (cpu_have_named_feature(PMULL))
- static_branch_enable(&have_pmull);
+ if (cpu_have_named_feature(ASIMD)) {
+ static_branch_enable(&have_asimd);
+ if (cpu_have_named_feature(PMULL))
+ static_branch_enable(&have_pmull);
+ }
}
projects / thirdparty / linux.git / commitdiff
