- AES-NI (AES New Instructions)
- SSE4.1 (Streaming SIMD Extensions 4.1)
-config CRYPTO_NHPOLY1305_SSE2
- tristate "Hash functions: NHPoly1305 (SSE2)"
- depends on 64BIT
- select CRYPTO_NHPOLY1305
- help
- NHPoly1305 hash function for Adiantum
-
- Architecture: x86_64 using:
- - SSE2 (Streaming SIMD Extensions 2)
-
-config CRYPTO_NHPOLY1305_AVX2
- tristate "Hash functions: NHPoly1305 (AVX2)"
- depends on 64BIT
- select CRYPTO_NHPOLY1305
- help
- NHPoly1305 hash function for Adiantum
-
- Architecture: x86_64 using:
- - AVX2 (Advanced Vector Extensions 2)
-
config CRYPTO_SM3_AVX_X86_64
tristate "Hash functions: SM3 (AVX)"
depends on 64BIT
obj-$(CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL) += ghash-clmulni-intel.o
ghash-clmulni-intel-y := ghash-clmulni-intel_asm.o ghash-clmulni-intel_glue.o
-obj-$(CONFIG_CRYPTO_NHPOLY1305_SSE2) += nhpoly1305-sse2.o
-nhpoly1305-sse2-y := nh-sse2-x86_64.o nhpoly1305-sse2-glue.o
-obj-$(CONFIG_CRYPTO_NHPOLY1305_AVX2) += nhpoly1305-avx2.o
-nhpoly1305-avx2-y := nh-avx2-x86_64.o nhpoly1305-avx2-glue.o
-
obj-$(CONFIG_CRYPTO_SM3_AVX_X86_64) += sm3-avx-x86_64.o
sm3-avx-x86_64-y := sm3-avx-asm_64.o sm3_avx_glue.o
+++ /dev/null
-// SPDX-License-Identifier: GPL-2.0
-/*
- * NHPoly1305 - ε-almost-∆-universal hash function for Adiantum
- * (AVX2 accelerated version)
- *
- * Copyright 2018 Google LLC
- */
-
-#include <crypto/internal/hash.h>
-#include <crypto/internal/simd.h>
-#include <crypto/nhpoly1305.h>
-#include <linux/module.h>
-#include <linux/sizes.h>
-#include <asm/simd.h>
-
-asmlinkage void nh_avx2(const u32 *key, const u8 *message, size_t message_len,
- __le64 hash[NH_NUM_PASSES]);
-
-static int nhpoly1305_avx2_update(struct shash_desc *desc,
- const u8 *src, unsigned int srclen)
-{
- if (srclen < 64 || !crypto_simd_usable())
- return crypto_nhpoly1305_update(desc, src, srclen);
-
- do {
- unsigned int n = min_t(unsigned int, srclen, SZ_4K);
-
- kernel_fpu_begin();
- crypto_nhpoly1305_update_helper(desc, src, n, nh_avx2);
- kernel_fpu_end();
- src += n;
- srclen -= n;
- } while (srclen);
- return 0;
-}
-
-static int nhpoly1305_avx2_digest(struct shash_desc *desc,
- const u8 *src, unsigned int srclen, u8 *out)
-{
- return crypto_nhpoly1305_init(desc) ?:
- nhpoly1305_avx2_update(desc, src, srclen) ?:
- crypto_nhpoly1305_final(desc, out);
-}
-
-static struct shash_alg nhpoly1305_alg = {
- .base.cra_name = "nhpoly1305",
- .base.cra_driver_name = "nhpoly1305-avx2",
- .base.cra_priority = 300,
- .base.cra_ctxsize = sizeof(struct nhpoly1305_key),
- .base.cra_module = THIS_MODULE,
- .digestsize = POLY1305_DIGEST_SIZE,
- .init = crypto_nhpoly1305_init,
- .update = nhpoly1305_avx2_update,
- .final = crypto_nhpoly1305_final,
- .digest = nhpoly1305_avx2_digest,
- .setkey = crypto_nhpoly1305_setkey,
- .descsize = sizeof(struct nhpoly1305_state),
-};
-
-static int __init nhpoly1305_mod_init(void)
-{
- if (!boot_cpu_has(X86_FEATURE_AVX2) ||
- !boot_cpu_has(X86_FEATURE_OSXSAVE))
- return -ENODEV;
-
- return crypto_register_shash(&nhpoly1305_alg);
-}
-
-static void __exit nhpoly1305_mod_exit(void)
-{
- crypto_unregister_shash(&nhpoly1305_alg);
-}
-
-module_init(nhpoly1305_mod_init);
-module_exit(nhpoly1305_mod_exit);
-
-MODULE_DESCRIPTION("NHPoly1305 ε-almost-∆-universal hash function (AVX2-accelerated)");
-MODULE_LICENSE("GPL v2");
-MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>");
-MODULE_ALIAS_CRYPTO("nhpoly1305");
-MODULE_ALIAS_CRYPTO("nhpoly1305-avx2");
+++ /dev/null
-// SPDX-License-Identifier: GPL-2.0
-/*
- * NHPoly1305 - ε-almost-∆-universal hash function for Adiantum
- * (SSE2 accelerated version)
- *
- * Copyright 2018 Google LLC
- */
-
-#include <crypto/internal/hash.h>
-#include <crypto/internal/simd.h>
-#include <crypto/nhpoly1305.h>
-#include <linux/module.h>
-#include <linux/sizes.h>
-#include <asm/simd.h>
-
-asmlinkage void nh_sse2(const u32 *key, const u8 *message, size_t message_len,
- __le64 hash[NH_NUM_PASSES]);
-
-static int nhpoly1305_sse2_update(struct shash_desc *desc,
- const u8 *src, unsigned int srclen)
-{
- if (srclen < 64 || !crypto_simd_usable())
- return crypto_nhpoly1305_update(desc, src, srclen);
-
- do {
- unsigned int n = min_t(unsigned int, srclen, SZ_4K);
-
- kernel_fpu_begin();
- crypto_nhpoly1305_update_helper(desc, src, n, nh_sse2);
- kernel_fpu_end();
- src += n;
- srclen -= n;
- } while (srclen);
- return 0;
-}
-
-static int nhpoly1305_sse2_digest(struct shash_desc *desc,
- const u8 *src, unsigned int srclen, u8 *out)
-{
- return crypto_nhpoly1305_init(desc) ?:
- nhpoly1305_sse2_update(desc, src, srclen) ?:
- crypto_nhpoly1305_final(desc, out);
-}
-
-static struct shash_alg nhpoly1305_alg = {
- .base.cra_name = "nhpoly1305",
- .base.cra_driver_name = "nhpoly1305-sse2",
- .base.cra_priority = 200,
- .base.cra_ctxsize = sizeof(struct nhpoly1305_key),
- .base.cra_module = THIS_MODULE,
- .digestsize = POLY1305_DIGEST_SIZE,
- .init = crypto_nhpoly1305_init,
- .update = nhpoly1305_sse2_update,
- .final = crypto_nhpoly1305_final,
- .digest = nhpoly1305_sse2_digest,
- .setkey = crypto_nhpoly1305_setkey,
- .descsize = sizeof(struct nhpoly1305_state),
-};
-
-static int __init nhpoly1305_mod_init(void)
-{
- if (!boot_cpu_has(X86_FEATURE_XMM2))
- return -ENODEV;
-
- return crypto_register_shash(&nhpoly1305_alg);
-}
-
-static void __exit nhpoly1305_mod_exit(void)
-{
- crypto_unregister_shash(&nhpoly1305_alg);
-}
-
-module_init(nhpoly1305_mod_init);
-module_exit(nhpoly1305_mod_exit);
-
-MODULE_DESCRIPTION("NHPoly1305 ε-almost-∆-universal hash function (SSE2-accelerated)");
-MODULE_LICENSE("GPL v2");
-MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>");
-MODULE_ALIAS_CRYPTO("nhpoly1305");
-MODULE_ALIAS_CRYPTO("nhpoly1305-sse2");
depends on CRYPTO_LIB_NH && !UML
default y if ARM && KERNEL_MODE_NEON
default y if ARM64 && KERNEL_MODE_NEON
+ default y if X86_64
config CRYPTO_LIB_POLY1305
tristate
CFLAGS_nh.o += -I$(src)/$(SRCARCH)
libnh-$(CONFIG_ARM) += arm/nh-neon-core.o
libnh-$(CONFIG_ARM64) += arm64/nh-neon-core.o
+libnh-$(CONFIG_X86) += x86/nh-sse2.o x86/nh-avx2.o
endif
################################################################################
*/
#include <linux/linkage.h>
-#include <linux/cfi_types.h>
#define PASS0_SUMS %ymm0
#define PASS1_SUMS %ymm1
*
* It's guaranteed that message_len % 16 == 0.
*/
-SYM_TYPED_FUNC_START(nh_avx2)
+SYM_FUNC_START(nh_avx2)
vmovdqu 0x00(KEY), K0
vmovdqu 0x10(KEY), K1
*/
#include <linux/linkage.h>
-#include <linux/cfi_types.h>
#define PASS0_SUMS %xmm0
#define PASS1_SUMS %xmm1
*
* It's guaranteed that message_len % 16 == 0.
*/
-SYM_TYPED_FUNC_START(nh_sse2)
+SYM_FUNC_START(nh_sse2)
movdqu 0x00(KEY), K0
movdqu 0x10(KEY), K1
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * x86_64 accelerated implementation of NH
+ *
+ * Copyright 2018 Google LLC
+ */
+
+#include <asm/fpu/api.h>
+#include <linux/static_call.h>
+
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sse2);
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_avx2);
+
+asmlinkage void nh_sse2(const u32 *key, const u8 *message, size_t message_len,
+ __le64 hash[NH_NUM_PASSES]);
+asmlinkage void nh_avx2(const u32 *key, const u8 *message, size_t message_len,
+ __le64 hash[NH_NUM_PASSES]);
+
+static bool nh_arch(const u32 *key, const u8 *message, size_t message_len,
+ __le64 hash[NH_NUM_PASSES])
+{
+ if (message_len >= 64 && static_branch_likely(&have_sse2) &&
+ irq_fpu_usable()) {
+ kernel_fpu_begin();
+ if (static_branch_likely(&have_avx2))
+ nh_avx2(key, message, message_len, hash);
+ else
+ nh_sse2(key, message, message_len, hash);
+ kernel_fpu_end();
+ return true;
+ }
+ return false;
+}
+
+#define nh_mod_init_arch nh_mod_init_arch
+static void nh_mod_init_arch(void)
+{
+ if (boot_cpu_has(X86_FEATURE_XMM2)) {
+ static_branch_enable(&have_sse2);
+ if (boot_cpu_has(X86_FEATURE_AVX2) &&
+ cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
+ NULL))
+ static_branch_enable(&have_avx2);
+ }
+}
projects / thirdparty / linux.git / commitdiff
