CONFIG_CRYPTO_USER_API_RNG=m
CONFIG_CRYPTO_GHASH_ARM64_CE=y
CONFIG_CRYPTO_SHA1_ARM64_CE=y
-CONFIG_CRYPTO_SHA512_ARM64_CE=m
CONFIG_CRYPTO_SHA3_ARM64=m
CONFIG_CRYPTO_SM3_ARM64_CE=m
CONFIG_CRYPTO_AES_ARM64_CE_BLK=y
Architecture: arm64 using:
- ARMv8 Crypto Extensions
-config CRYPTO_SHA512_ARM64
- tristate "Hash functions: SHA-384 and SHA-512"
- select CRYPTO_HASH
- help
- SHA-384 and SHA-512 secure hash algorithms (FIPS 180)
-
- Architecture: arm64
-
-config CRYPTO_SHA512_ARM64_CE
- tristate "Hash functions: SHA-384 and SHA-512 (ARMv8 Crypto Extensions)"
- depends on KERNEL_MODE_NEON
- select CRYPTO_HASH
- select CRYPTO_SHA512_ARM64
- help
- SHA-384 and SHA-512 secure hash algorithms (FIPS 180)
-
- Architecture: arm64 using:
- - ARMv8 Crypto Extensions
-
config CRYPTO_SHA3_ARM64
tristate "Hash functions: SHA-3 (ARMv8.2 Crypto Extensions)"
depends on KERNEL_MODE_NEON
obj-$(CONFIG_CRYPTO_SHA1_ARM64_CE) += sha1-ce.o
sha1-ce-y := sha1-ce-glue.o sha1-ce-core.o
-obj-$(CONFIG_CRYPTO_SHA512_ARM64_CE) += sha512-ce.o
-sha512-ce-y := sha512-ce-glue.o sha512-ce-core.o
-
obj-$(CONFIG_CRYPTO_SHA3_ARM64) += sha3-ce.o
sha3-ce-y := sha3-ce-glue.o sha3-ce-core.o
obj-$(CONFIG_CRYPTO_AES_ARM64_NEON_BLK) += aes-neon-blk.o
aes-neon-blk-y := aes-glue-neon.o aes-neon.o
-obj-$(CONFIG_CRYPTO_SHA512_ARM64) += sha512-arm64.o
-sha512-arm64-y := sha512-glue.o sha512-core.o
-
obj-$(CONFIG_CRYPTO_NHPOLY1305_NEON) += nhpoly1305-neon.o
nhpoly1305-neon-y := nh-neon-core.o nhpoly1305-neon-glue.o
obj-$(CONFIG_CRYPTO_AES_ARM64_BS) += aes-neon-bs.o
aes-neon-bs-y := aes-neonbs-core.o aes-neonbs-glue.o
-
-quiet_cmd_perlasm = PERLASM $@
- cmd_perlasm = $(PERL) $(<) void $(@)
-
-$(obj)/sha512-core.S: $(src)/../lib/crypto/sha2-armv8.pl
- $(call cmd,perlasm)
-
-clean-files += sha512-core.S
+++ /dev/null
-// SPDX-License-Identifier: GPL-2.0
-/*
- * sha512-ce-glue.c - SHA-384/SHA-512 using ARMv8 Crypto Extensions
- *
- * Copyright (C) 2018 Linaro Ltd <ard.biesheuvel@linaro.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-#include <asm/neon.h>
-#include <crypto/internal/hash.h>
-#include <crypto/sha2.h>
-#include <crypto/sha512_base.h>
-#include <linux/cpufeature.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-
-MODULE_DESCRIPTION("SHA-384/SHA-512 secure hash using ARMv8 Crypto Extensions");
-MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
-MODULE_LICENSE("GPL v2");
-MODULE_ALIAS_CRYPTO("sha384");
-MODULE_ALIAS_CRYPTO("sha512");
-
-asmlinkage int __sha512_ce_transform(struct sha512_state *sst, u8 const *src,
- int blocks);
-
-static void sha512_ce_transform(struct sha512_state *sst, u8 const *src,
- int blocks)
-{
- do {
- int rem;
-
- kernel_neon_begin();
- rem = __sha512_ce_transform(sst, src, blocks);
- kernel_neon_end();
- src += (blocks - rem) * SHA512_BLOCK_SIZE;
- blocks = rem;
- } while (blocks);
-}
-
-static int sha512_ce_update(struct shash_desc *desc, const u8 *data,
- unsigned int len)
-{
- return sha512_base_do_update_blocks(desc, data, len,
- sha512_ce_transform);
-}
-
-static int sha512_ce_finup(struct shash_desc *desc, const u8 *data,
- unsigned int len, u8 *out)
-{
- sha512_base_do_finup(desc, data, len, sha512_ce_transform);
- return sha512_base_finish(desc, out);
-}
-
-static struct shash_alg algs[] = { {
- .init = sha384_base_init,
- .update = sha512_ce_update,
- .finup = sha512_ce_finup,
- .descsize = SHA512_STATE_SIZE,
- .digestsize = SHA384_DIGEST_SIZE,
- .base.cra_name = "sha384",
- .base.cra_driver_name = "sha384-ce",
- .base.cra_priority = 200,
- .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
- CRYPTO_AHASH_ALG_FINUP_MAX,
- .base.cra_blocksize = SHA512_BLOCK_SIZE,
- .base.cra_module = THIS_MODULE,
-}, {
- .init = sha512_base_init,
- .update = sha512_ce_update,
- .finup = sha512_ce_finup,
- .descsize = SHA512_STATE_SIZE,
- .digestsize = SHA512_DIGEST_SIZE,
- .base.cra_name = "sha512",
- .base.cra_driver_name = "sha512-ce",
- .base.cra_priority = 200,
- .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
- CRYPTO_AHASH_ALG_FINUP_MAX,
- .base.cra_blocksize = SHA512_BLOCK_SIZE,
- .base.cra_module = THIS_MODULE,
-} };
-
-static int __init sha512_ce_mod_init(void)
-{
- return crypto_register_shashes(algs, ARRAY_SIZE(algs));
-}
-
-static void __exit sha512_ce_mod_fini(void)
-{
- crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
-}
-
-module_cpu_feature_match(SHA512, sha512_ce_mod_init);
-module_exit(sha512_ce_mod_fini);
+++ /dev/null
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * Linux/arm64 port of the OpenSSL SHA512 implementation for AArch64
- *
- * Copyright (c) 2016 Linaro Ltd. <ard.biesheuvel@linaro.org>
- */
-
-#include <crypto/internal/hash.h>
-#include <crypto/sha2.h>
-#include <crypto/sha512_base.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-
-MODULE_DESCRIPTION("SHA-384/SHA-512 secure hash for arm64");
-MODULE_AUTHOR("Andy Polyakov <appro@openssl.org>");
-MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
-MODULE_LICENSE("GPL v2");
-MODULE_ALIAS_CRYPTO("sha384");
-MODULE_ALIAS_CRYPTO("sha512");
-
-asmlinkage void sha512_blocks_arch(u64 *digest, const void *data,
- unsigned int num_blks);
-
-static void sha512_arm64_transform(struct sha512_state *sst, u8 const *src,
- int blocks)
-{
- sha512_blocks_arch(sst->state, src, blocks);
-}
-
-static int sha512_update_arm64(struct shash_desc *desc, const u8 *data,
- unsigned int len)
-{
- return sha512_base_do_update_blocks(desc, data, len,
- sha512_arm64_transform);
-}
-
-static int sha512_finup(struct shash_desc *desc, const u8 *data,
- unsigned int len, u8 *out)
-{
- sha512_base_do_finup(desc, data, len, sha512_arm64_transform);
- return sha512_base_finish(desc, out);
-}
-
-static struct shash_alg algs[] = { {
- .digestsize = SHA512_DIGEST_SIZE,
- .init = sha512_base_init,
- .update = sha512_update_arm64,
- .finup = sha512_finup,
- .descsize = SHA512_STATE_SIZE,
- .base.cra_name = "sha512",
- .base.cra_driver_name = "sha512-arm64",
- .base.cra_priority = 150,
- .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
- CRYPTO_AHASH_ALG_FINUP_MAX,
- .base.cra_blocksize = SHA512_BLOCK_SIZE,
- .base.cra_module = THIS_MODULE,
-}, {
- .digestsize = SHA384_DIGEST_SIZE,
- .init = sha384_base_init,
- .update = sha512_update_arm64,
- .finup = sha512_finup,
- .descsize = SHA512_STATE_SIZE,
- .base.cra_name = "sha384",
- .base.cra_driver_name = "sha384-arm64",
- .base.cra_priority = 150,
- .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
- CRYPTO_AHASH_ALG_FINUP_MAX,
- .base.cra_blocksize = SHA384_BLOCK_SIZE,
- .base.cra_module = THIS_MODULE,
-} };
-
-static int __init sha512_mod_init(void)
-{
- return crypto_register_shashes(algs, ARRAY_SIZE(algs));
-}
-
-static void __exit sha512_mod_fini(void)
-{
- crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
-}
-
-module_init(sha512_mod_init);
-module_exit(sha512_mod_fini);
bool
depends on CRYPTO_LIB_SHA512 && !UML
default y if ARM && !CPU_V7M
+ default y if ARM64
config CRYPTO_LIB_SM3
tristate
quiet_cmd_perlasm = PERLASM $@
cmd_perlasm = $(PERL) $(<) > $(@)
+quiet_cmd_perlasm_with_args = PERLASM $@
+ cmd_perlasm_with_args = $(PERL) $(<) void $(@)
+
obj-$(CONFIG_CRYPTO_LIB_UTILS) += libcryptoutils.o
libcryptoutils-y := memneq.o utils.o
AFLAGS_arm/sha512-core.o += $(aflags-thumb2-y)
endif
+ifeq ($(CONFIG_ARM64),y)
+libsha512-y += arm64/sha512-core.o
+$(obj)/arm64/sha512-core.S: $(src)/../../arch/arm64/lib/crypto/sha2-armv8.pl
+ $(call cmd,perlasm_with_args)
+clean-files += arm64/sha512-core.S
+libsha512-$(CONFIG_KERNEL_MODE_NEON) += arm64/sha512-ce-core.o
+endif
endif # CONFIG_CRYPTO_LIB_SHA512_ARCH
obj-$(CONFIG_MPILIB) += mpi/
--- /dev/null
+# SPDX-License-Identifier: GPL-2.0-only
+sha512-core.S
.endm
/*
- * int __sha512_ce_transform(struct sha512_state *sst, u8 const *src,
- * int blocks)
+ * size_t __sha512_ce_transform(struct sha512_block_state *state,
+ * const u8 *data, size_t nblocks);
*/
.text
SYM_FUNC_START(__sha512_ce_transform)
/* load input */
0: ld1 {v12.2d-v15.2d}, [x1], #64
ld1 {v16.2d-v19.2d}, [x1], #64
- sub w2, w2, #1
+ sub x2, x2, #1
CPU_LE( rev64 v12.16b, v12.16b )
CPU_LE( rev64 v13.16b, v13.16b )
cond_yield 3f, x4, x5
/* handled all input blocks? */
- cbnz w2, 0b
+ cbnz x2, 0b
/* store new state */
3: st1 {v8.2d-v11.2d}, [x0]
- mov w0, w2
+ mov x0, x2
ret
SYM_FUNC_END(__sha512_ce_transform)
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * arm64-optimized SHA-512 block function
+ *
+ * Copyright 2025 Google LLC
+ */
+
+#include <asm/neon.h>
+#include <crypto/internal/simd.h>
+#include <linux/cpufeature.h>
+
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha512_insns);
+
+asmlinkage void sha512_blocks_arch(struct sha512_block_state *state,
+ const u8 *data, size_t nblocks);
+asmlinkage size_t __sha512_ce_transform(struct sha512_block_state *state,
+ const u8 *data, size_t nblocks);
+
+static void sha512_blocks(struct sha512_block_state *state,
+ const u8 *data, size_t nblocks)
+{
+ if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
+ static_branch_likely(&have_sha512_insns) &&
+ likely(crypto_simd_usable())) {
+ do {
+ size_t rem;
+
+ kernel_neon_begin();
+ rem = __sha512_ce_transform(state, data, nblocks);
+ kernel_neon_end();
+ data += (nblocks - rem) * SHA512_BLOCK_SIZE;
+ nblocks = rem;
+ } while (nblocks);
+ } else {
+ sha512_blocks_arch(state, data, nblocks);
+ }
+}
+
+#ifdef CONFIG_KERNEL_MODE_NEON
+#define sha512_mod_init_arch sha512_mod_init_arch
+static inline void sha512_mod_init_arch(void)
+{
+ if (cpu_have_named_feature(SHA512))
+ static_branch_enable(&have_sha512_insns);
+}
+#endif /* CONFIG_KERNEL_MODE_NEON */
projects / thirdparty / linux.git / commitdiff
