CONFIG_CRYPTO_ECHAINIV=y
CONFIG_CRYPTO_MICHAEL_MIC=m
CONFIG_CRYPTO_SHA3=m
+CONFIG_CRYPTO_SM3=m
CONFIG_CRYPTO_USER_API_RNG=m
CONFIG_CRYPTO_GHASH_ARM64_CE=y
-CONFIG_CRYPTO_SM3_ARM64_CE=m
CONFIG_CRYPTO_AES_ARM64_CE_BLK=y
CONFIG_CRYPTO_AES_ARM64_BS=m
CONFIG_CRYPTO_AES_ARM64_CE_CCM=y
Architecture: arm64 using:
- ARMv8 Crypto Extensions
-config CRYPTO_SM3_NEON
- tristate "Hash functions: SM3 (NEON)"
- depends on KERNEL_MODE_NEON
- select CRYPTO_HASH
- select CRYPTO_LIB_SM3
- help
- SM3 (ShangMi 3) secure hash function (OSCCA GM/T 0004-2012)
-
- Architecture: arm64 using:
- - NEON (Advanced SIMD) extensions
-
-config CRYPTO_SM3_ARM64_CE
- tristate "Hash functions: SM3 (ARMv8.2 Crypto Extensions)"
- depends on KERNEL_MODE_NEON
- select CRYPTO_HASH
- select CRYPTO_LIB_SM3
- help
- SM3 (ShangMi 3) secure hash function (OSCCA GM/T 0004-2012)
-
- Architecture: arm64 using:
- - ARMv8.2 Crypto Extensions
-
config CRYPTO_AES_ARM64_CE_BLK
tristate "Ciphers: AES, modes: ECB/CBC/CTR/XTS (ARMv8 Crypto Extensions)"
depends on KERNEL_MODE_NEON
# Copyright (C) 2014 Linaro Ltd <ard.biesheuvel@linaro.org>
#
-obj-$(CONFIG_CRYPTO_SM3_NEON) += sm3-neon.o
-sm3-neon-y := sm3-neon-glue.o sm3-neon-core.o
-
-obj-$(CONFIG_CRYPTO_SM3_ARM64_CE) += sm3-ce.o
-sm3-ce-y := sm3-ce-glue.o sm3-ce-core.o
-
obj-$(CONFIG_CRYPTO_SM4_ARM64_CE) += sm4-ce-cipher.o
sm4-ce-cipher-y := sm4-ce-cipher-glue.o sm4-ce-cipher-core.o
+++ /dev/null
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * sm3-ce-glue.c - SM3 secure hash using ARMv8.2 Crypto Extensions
- *
- * Copyright (C) 2018 Linaro Ltd <ard.biesheuvel@linaro.org>
- */
-
-#include <crypto/internal/hash.h>
-#include <crypto/sm3.h>
-#include <crypto/sm3_base.h>
-#include <linux/cpufeature.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-
-#include <asm/simd.h>
-
-MODULE_DESCRIPTION("SM3 secure hash using ARMv8 Crypto Extensions");
-MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
-MODULE_LICENSE("GPL v2");
-
-asmlinkage void sm3_ce_transform(struct sm3_state *sst, u8 const *src,
- int blocks);
-
-static int sm3_ce_update(struct shash_desc *desc, const u8 *data,
- unsigned int len)
-{
- int remain;
-
- scoped_ksimd() {
- remain = sm3_base_do_update_blocks(desc, data, len, sm3_ce_transform);
- }
- return remain;
-}
-
-static int sm3_ce_finup(struct shash_desc *desc, const u8 *data,
- unsigned int len, u8 *out)
-{
- scoped_ksimd() {
- sm3_base_do_finup(desc, data, len, sm3_ce_transform);
- }
- return sm3_base_finish(desc, out);
-}
-
-static struct shash_alg sm3_alg = {
- .digestsize = SM3_DIGEST_SIZE,
- .init = sm3_base_init,
- .update = sm3_ce_update,
- .finup = sm3_ce_finup,
- .descsize = SM3_STATE_SIZE,
- .base.cra_name = "sm3",
- .base.cra_driver_name = "sm3-ce",
- .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
- CRYPTO_AHASH_ALG_FINUP_MAX,
- .base.cra_blocksize = SM3_BLOCK_SIZE,
- .base.cra_module = THIS_MODULE,
- .base.cra_priority = 400,
-};
-
-static int __init sm3_ce_mod_init(void)
-{
- return crypto_register_shash(&sm3_alg);
-}
-
-static void __exit sm3_ce_mod_fini(void)
-{
- crypto_unregister_shash(&sm3_alg);
-}
-
-module_cpu_feature_match(SM3, sm3_ce_mod_init);
-module_exit(sm3_ce_mod_fini);
+++ /dev/null
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * sm3-neon-glue.c - SM3 secure hash using NEON instructions
- *
- * Copyright (C) 2022 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
- */
-
-#include <asm/simd.h>
-#include <crypto/internal/hash.h>
-#include <crypto/sm3.h>
-#include <crypto/sm3_base.h>
-#include <linux/cpufeature.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-
-
-asmlinkage void sm3_neon_transform(struct sm3_state *sst, u8 const *src,
- int blocks);
-
-static int sm3_neon_update(struct shash_desc *desc, const u8 *data,
- unsigned int len)
-{
- scoped_ksimd()
- return sm3_base_do_update_blocks(desc, data, len,
- sm3_neon_transform);
-}
-
-static int sm3_neon_finup(struct shash_desc *desc, const u8 *data,
- unsigned int len, u8 *out)
-{
- scoped_ksimd()
- sm3_base_do_finup(desc, data, len, sm3_neon_transform);
- return sm3_base_finish(desc, out);
-}
-
-static struct shash_alg sm3_alg = {
- .digestsize = SM3_DIGEST_SIZE,
- .init = sm3_base_init,
- .update = sm3_neon_update,
- .finup = sm3_neon_finup,
- .descsize = SM3_STATE_SIZE,
- .base.cra_name = "sm3",
- .base.cra_driver_name = "sm3-neon",
- .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
- CRYPTO_AHASH_ALG_FINUP_MAX,
- .base.cra_blocksize = SM3_BLOCK_SIZE,
- .base.cra_module = THIS_MODULE,
- .base.cra_priority = 200,
-};
-
-static int __init sm3_neon_init(void)
-{
- return crypto_register_shash(&sm3_alg);
-}
-
-static void __exit sm3_neon_fini(void)
-{
- crypto_unregister_shash(&sm3_alg);
-}
-
-module_init(sm3_neon_init);
-module_exit(sm3_neon_fini);
-
-MODULE_DESCRIPTION("SM3 secure hash using NEON instructions");
-MODULE_AUTHOR("Jussi Kivilinna <jussi.kivilinna@iki.fi>");
-MODULE_AUTHOR("Tianjia Zhang <tianjia.zhang@linux.alibaba.com>");
-MODULE_LICENSE("GPL v2");
config CRYPTO_LIB_SM3_ARCH
bool
depends on CRYPTO_LIB_SM3 && !UML
+ default y if ARM64
source "lib/crypto/tests/Kconfig"
################################################################################
+obj-$(CONFIG_CRYPTO_LIB_SM3) += libsm3.o
+libsm3-y := sm3.o
+ifeq ($(CONFIG_CRYPTO_LIB_SM3_ARCH),y)
+CFLAGS_sm3.o += -I$(src)/$(SRCARCH)
+libsm3-$(CONFIG_ARM64) += arm64/sm3-ce-core.o \
+ arm64/sm3-neon-core.o
+endif # CONFIG_CRYPTO_LIB_SM3_ARCH
+
+################################################################################
+
obj-$(CONFIG_MPILIB) += mpi/
obj-$(CONFIG_CRYPTO_SELFTESTS_FULL) += simd.o
-obj-$(CONFIG_CRYPTO_LIB_SM3) += libsm3.o
-libsm3-y := sm3.o
-
# clean-files must be defined unconditionally
clean-files += arm/sha256-core.S arm/sha512-core.S
clean-files += arm64/sha256-core.S arm64/sha512-core.S
*/
#include <linux/linkage.h>
-#include <linux/cfi_types.h>
#include <asm/assembler.h>
.irp b, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12
.endm
/*
- * void sm3_ce_transform(struct sm3_state *sst, u8 const *src,
- * int blocks)
+ * void sm3_ce_transform(struct sm3_block_state *state,
+ * const u8 *data, size_t nblocks)
*/
.text
-SYM_TYPED_FUNC_START(sm3_ce_transform)
+SYM_FUNC_START(sm3_ce_transform)
/* load state */
ld1 {v8.4s-v9.4s}, [x0]
rev64 v8.4s, v8.4s
/* load input */
0: ld1 {v0.16b-v3.16b}, [x1], #64
- sub w2, w2, #1
+ sub x2, x2, #1
mov v15.16b, v8.16b
mov v16.16b, v9.16b
eor v9.16b, v9.16b, v16.16b
/* handled all input blocks? */
- cbnz w2, 0b
+ cbnz x2, 0b
/* save state */
rev64 v8.4s, v8.4s
*/
#include <linux/linkage.h>
-#include <linux/cfi_types.h>
#include <asm/assembler.h>
/* Context structure */
/*
- * Transform blocks*64 bytes (blocks*16 32-bit words) at 'src'.
+ * Transform nblocks*64 bytes (nblocks*16 32-bit words) at 'data'.
*
- * void sm3_neon_transform(struct sm3_state *sst, u8 const *src,
- * int blocks)
+ * void sm3_neon_transform(struct sm3_block_state *state,
+ * const u8 *data, size_t nblocks)
*/
.text
.align 3
-SYM_TYPED_FUNC_START(sm3_neon_transform)
+SYM_FUNC_START(sm3_neon_transform)
ldp ra, rb, [RSTATE, #0]
ldp rc, rd, [RSTATE, #8]
ldp re, rf, [RSTATE, #16]
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * SM3 optimized for ARM64
+ *
+ * Copyright 2026 Google LLC
+ */
+#include <asm/simd.h>
+#include <linux/cpufeature.h>
+
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce);
+
+asmlinkage void sm3_neon_transform(struct sm3_block_state *state,
+ const u8 *data, size_t nblocks);
+asmlinkage void sm3_ce_transform(struct sm3_block_state *state,
+ const u8 *data, size_t nblocks);
+
+static void sm3_blocks(struct sm3_block_state *state,
+ const u8 *data, size_t nblocks)
+{
+ if (static_branch_likely(&have_neon) && likely(may_use_simd())) {
+ scoped_ksimd() {
+ if (static_branch_likely(&have_ce))
+ sm3_ce_transform(state, data, nblocks);
+ else
+ sm3_neon_transform(state, data, nblocks);
+ }
+ } else {
+ sm3_blocks_generic(state, data, nblocks);
+ }
+}
+
+#define sm3_mod_init_arch sm3_mod_init_arch
+static void sm3_mod_init_arch(void)
+{
+ if (cpu_have_named_feature(ASIMD)) {
+ static_branch_enable(&have_neon);
+ if (cpu_have_named_feature(SM3))
+ static_branch_enable(&have_ce);
+ }
+}
projects / thirdparty / kernel / stable.git / commitdiff
