From: Jeremy Allison Date: Wed, 6 Sep 2017 18:40:02 +0000 (-0700) Subject: lib: crypto: Plumb in the Intel AES instructions. X-Git-Tag: samba-4.7.0rc6~54 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=9d193a942fdbcab135e40d06422eab640d33d00b;p=thirdparty%2Fsamba.git lib: crypto: Plumb in the Intel AES instructions. Causes: AES_set_encrypt_key() AES_set_decrypt_key() AES_encrypt() AES_decrypt() to probe for the Intel AES instructions at runtime (only once) and then call the hardware implementations if so, otherwise fall back to the software implementations. BUG: https://bugzilla.samba.org/show_bug.cgi?id=13008 Based on original work by Justin Maggard Signed-off-by: Jeremy Allison Reviewed-by: Stefan Metzmacher (cherry picked from commit 53ac0f7c59a9192e4fa88bf3d257ad80379ded23) --- diff --git a/lib/crypto/aes.c b/lib/crypto/aes.c index 8e6d8418f16..c226ac1b3df 100644 --- a/lib/crypto/aes.c +++ b/lib/crypto/aes.c @@ -37,6 +37,146 @@ #ifdef SAMBA_RIJNDAEL #include "rijndael-alg-fst.h" +#if defined(HAVE_AESNI_INTEL) + +/* + * NB. HAVE_AESNI_INTEL is only defined if -lang-asm is + * available. + */ + +static inline void __cpuid(unsigned int where[4], unsigned int leaf) +{ + asm volatile("cpuid" : + "=a" (where[0]), + "=b" (where[1]), + "=c" (where[2]), + "=d" (where[3]): "a" (leaf)); +} + +/* + * has_intel_aes_instructions() + * return true if supports AES-NI and false if doesn't + */ +static bool has_intel_aes_instructions(void) +{ + static int has_aes_instructions = -1; + unsigned int cpuid_results[4]; + + if (has_aes_instructions != -1) { + return (bool)has_aes_instructions; + } + + __cpuid(cpuid_results, 0); + /* + * MSB LSB + * EBX = 'u' 'n' 'e' 'G' + * EDX = 'I' 'e' 'n' 'i' + * ECX = 'l' 'e' 't' 'n' + */ + if (memcmp((unsigned char *)&cpuid_results[1], "Genu", 4) != 0 || + memcmp((unsigned char *)&cpuid_results[3], + "ineI", 4) != 0 || + memcmp((unsigned char *)&cpuid_results[2], + "ntel", 4) != 0) { + has_aes_instructions = 0; + return (bool)has_aes_instructions; + } + + __cpuid(cpuid_results, 1); + has_aes_instructions = !!(cpuid_results[2] & (1 << 25)); + return (bool)has_aes_instructions; +} + +/* + * Macro to ensure the AES key schedule starts on a 16 byte boundary. + */ + +#define SET_ACC_CTX(k) \ + do { \ + (k)->u.aes_ni.acc_ctx = \ + (struct crypto_aes_ctx *)(((unsigned long)(k)->u.aes_ni._acc_ctx + 15) & ~0xfUL); \ + } while (0) + +/* + * The next 4 functions call the Intel AES hardware implementations + * of: + * + * AES_set_encrypt_key() + * AES_set_decrypt_key() + * AES_encrypt() + * AES_decrypt() + */ + +static int AES_set_encrypt_key_aesni(const unsigned char *userkey, + const int bits, + AES_KEY *key) +{ + SET_ACC_CTX(key); + return aesni_set_key(key->u.aes_ni.acc_ctx, userkey, bits/8); +} + +static int AES_set_decrypt_key_aesni(const unsigned char *userkey, + const int bits, + AES_KEY *key) +{ + SET_ACC_CTX(key); + return aesni_set_key(key->u.aes_ni.acc_ctx, userkey, bits/8); +} + +static void AES_encrypt_aesni(const unsigned char *in, + unsigned char *out, + const AES_KEY *key) +{ + aesni_enc(key->u.aes_ni.acc_ctx, out, in); +} + +static void AES_decrypt_aesni(const unsigned char *in, + unsigned char *out, + const AES_KEY *key) +{ + aesni_dec(key->u.aes_ni.acc_ctx, out, in); +} +#else /* defined(HAVE_AESNI_INTEL) */ + +/* + * Dummy implementations if no Intel AES instructions present. + * Only has_intel_aes_instructions() will ever be called. +*/ + +static bool has_intel_aes_instructions(void) +{ + return false; +} + +static int AES_set_encrypt_key_aesni(const unsigned char *userkey, + const int bits, + AES_KEY *key) +{ + return -1; +} + +static int AES_set_decrypt_key_aesni(const unsigned char *userkey, + const int bits, + AES_KEY *key) +{ + return -1; +} + +static void AES_encrypt_aesni(const unsigned char *in, + unsigned char *out, + const AES_KEY *key) +{ + abort(); +} + +static void AES_decrypt_aesni(const unsigned char *in, + unsigned char *out, + const AES_KEY *key) +{ + abort(); +} +#endif /* defined(HAVE_AENI_INTEL) */ + /* * The next 4 functions are the pure software implementations * of: @@ -88,31 +228,41 @@ AES_decrypt_rj(const unsigned char *in, unsigned char *out, const AES_KEY *key) * * If the hardware instructions don't exist, fall back to the software * versions. - * - * Currently only use the software implementations. */ int AES_set_encrypt_key(const unsigned char *userkey, const int bits, AES_KEY *key) { + if (has_intel_aes_instructions()) { + return AES_set_encrypt_key_aesni(userkey, bits, key); + } return AES_set_encrypt_key_rj(userkey, bits, key); } int AES_set_decrypt_key(const unsigned char *userkey, const int bits, AES_KEY *key) { + if (has_intel_aes_instructions()) { + return AES_set_decrypt_key_aesni(userkey, bits, key); + } return AES_set_decrypt_key_rj(userkey, bits, key); } void AES_encrypt(const unsigned char *in, unsigned char *out, const AES_KEY *key) { + if (has_intel_aes_instructions()) { + return AES_encrypt_aesni(in, out, key); + } return AES_encrypt_rj(in, out, key); } void AES_decrypt(const unsigned char *in, unsigned char *out, const AES_KEY *key) { + if (has_intel_aes_instructions()) { + return AES_decrypt_aesni(in, out, key); + } return AES_decrypt_rj(in, out, key); } diff --git a/lib/crypto/aes.h b/lib/crypto/aes.h index 0ab0a376757..00bfa3e26ce 100644 --- a/lib/crypto/aes.h +++ b/lib/crypto/aes.h @@ -36,6 +36,8 @@ #ifndef LIB_CRYPTO_AES_H #define LIB_CRYPTO_AES_H 1 +#include "aesni.h" + #define SAMBA_RIJNDAEL 1 #define SAMBA_AES_CBC_ENCRYPT 1 #define SAMBA_AES_CFB8_ENCRYPT 1 @@ -67,6 +69,7 @@ struct aes_key_rj { typedef struct aes_key { union { struct aes_key_rj aes_rj; + struct crypto_aesni_ctx aes_ni; } u; } AES_KEY; diff --git a/lib/crypto/aesni.h b/lib/crypto/aesni.h new file mode 100644 index 00000000000..13d09d2d676 --- /dev/null +++ b/lib/crypto/aesni.h @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2008, Intel Corp. + * Author: Huang Ying + * Vinodh Gopal + * Kahraman Akdemir + * + * Ported x86_64 version to x86: + * Author: Mathias Krause + * + * Modified for use in Samba by Justin Maggard + * and Jeremy Allison + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + */ + +#ifndef LIB_CRYPTO_AESNI_H +#define LIB_CRYPTO_AESNI_H 1 + +#if defined(HAVE_AESNI_INTEL) + +#define AES_MAX_KEYLENGTH (15 * 16) +#define AES_MAX_KEYLENGTH_U32 (AES_MAX_KEYLENGTH / sizeof(uint32_t)) + +/* + * Please ensure that the first two fields are 16-byte aligned + * relative to the start of the structure, i.e., don't move them! + */ +struct crypto_aes_ctx { + uint32_t key_enc[AES_MAX_KEYLENGTH_U32]; + uint32_t key_dec[AES_MAX_KEYLENGTH_U32]; + uint32_t key_length; +}; + +struct crypto_aesni_ctx { + uint8_t _acc_ctx[sizeof(struct crypto_aes_ctx) + 16]; + struct crypto_aes_ctx *acc_ctx; +}; + +/* + * These next 4 functions are actually implemented + * in the assembly language file: + * third_party/aesni-intel/aesni-intel_asm.c + */ + +int aesni_set_key(struct crypto_aes_ctx *ctx, + const uint8_t *in_key, + unsigned int key_len); +void aesni_enc(struct crypto_aes_ctx *ctx, uint8_t *dst, const uint8_t *src); +void aesni_dec(struct crypto_aes_ctx *ctx, uint8_t *dst, const uint8_t *src); + +#else /* #if defined(HAVE_AESNI_INTEL) */ + +/* + * We need a dummy definition of struct crypto_aesni_ctx to allow compiles. + */ + +struct crypto_aesni_ctx { + int dummy; +}; + +#endif /* #if defined(HAVE_AESNI_INTEL) */ + +#endif /* LIB_CRYPTO_AESNI_H */ diff --git a/lib/crypto/wscript_build b/lib/crypto/wscript_build index d1f152ebcf1..f3257d8d1ed 100644 --- a/lib/crypto/wscript_build +++ b/lib/crypto/wscript_build @@ -11,6 +11,9 @@ elif bld.CONFIG_SET('HAVE_SYS_MD5_H') and bld.CONFIG_SET('HAVE_LIBMD'): elif not bld.CONFIG_SET('HAVE_SYS_MD5_H') and not bld.CONFIG_SET('HAVE_COMMONCRYPTO_COMMONDIGEST_H'): extra_source += ' md5.c' +if bld.CONFIG_SET("HAVE_AESNI_INTEL"): + extra_deps += ' aesni-intel' + bld.SAMBA_SUBSYSTEM('LIBCRYPTO', source='''crc32.c hmacmd5.c md4.c arcfour.c sha256.c sha512.c hmacsha256.c aes.c rijndael-alg-fst.c aes_cmac_128.c aes_ccm_128.c aes_gcm_128.c