From: Simo Sorce Date: Thu, 11 Jun 2026 21:22:17 +0000 (-0400) Subject: Consolidate ARMv8 AES hardware implementations X-Git-Url: http://git.ipfire.org/gitweb/?a=commitdiff_plain;h=d1004c62ace7bc550b755d5bea356473e6d2df32;p=thirdparty%2Fopenssl.git Consolidate ARMv8 AES hardware implementations Move the ARMv8-specific hardware implementations for AES GCM into a single consolidated file (`cipher_aes_hw_armv8.c`). This groups architecture-specific optimizations together to improve code organization and maintainability. Additionally, remove the unused `keybits` parameter from the ARMv8 provider functions to simplify the function signatures. Signed-off-by: Simo Sorce Reviewed-by: Dmitry Belyavskiy Reviewed-by: Norbert Pocs Reviewed-by: Shane Lontis MergeDate: Sat Jun 27 09:05:50 2026 (Merged from https://github.com/openssl/openssl/pull/31472) --- diff --git a/providers/implementations/ciphers/build.info b/providers/implementations/ciphers/build.info index 720874ba784..9dcf738946e 100644 --- a/providers/implementations/ciphers/build.info +++ b/providers/implementations/ciphers/build.info @@ -108,7 +108,7 @@ SOURCE[$AES_GOAL]=\ cipher_aes_hw_s390x.c cipher_aes_hw_t4.c \ cipher_aes_xts.c cipher_aes_xts_hw.c \ cipher_aes_gcm.c cipher_aes_gcm_hw.c \ - cipher_aes_gcm_hw_armv8.c cipher_aes_gcm_hw_ppc.c \ + cipher_aes_gcm_hw_ppc.c \ cipher_aes_gcm_hw_rv32i.c cipher_aes_gcm_hw_rv64i.c \ cipher_aes_gcm_hw_s390x.c cipher_aes_gcm_hw_t4.c \ cipher_aes_ccm.c cipher_aes_ccm_hw.c \ diff --git a/providers/implementations/ciphers/cipher_aes.h b/providers/implementations/ciphers/cipher_aes.h index 313c91ec9c1..0f0aff48917 100644 --- a/providers/implementations/ciphers/cipher_aes.h +++ b/providers/implementations/ciphers/cipher_aes.h @@ -88,8 +88,7 @@ void ossl_cipher_aes_copyctx(PROV_CIPHER_CTX *dst, const PROV_CIPHER_CTX *src); #if defined(AESNI_CAPABLE) const PROV_CIPHER_HW *ossl_prov_cipher_hw_aesni(enum aes_modes mode); #elif defined(ARMv8_HWAES_CAPABLE) -const PROV_CIPHER_HW *ossl_prov_cipher_hw_arm(enum aes_modes mode, - size_t keybits); +const PROV_CIPHER_HW *ossl_prov_cipher_hw_arm(enum aes_modes mode); #elif defined(OPENSSL_CPUID_OBJ) && defined(__riscv) && __riscv_xlen == 32 const PROV_CIPHER_HW *ossl_prov_cipher_hw_rv32i(enum aes_modes mode, size_t keybits); diff --git a/providers/implementations/ciphers/cipher_aes_gcm.h b/providers/implementations/ciphers/cipher_aes_gcm.h index 71ee4c97eb8..0b8d23fcb96 100644 --- a/providers/implementations/ciphers/cipher_aes_gcm.h +++ b/providers/implementations/ciphers/cipher_aes_gcm.h @@ -57,7 +57,7 @@ const PROV_GCM_HW *ossl_prov_aes_hw_gcm(size_t keybits); const PROV_GCM_HW *ossl_prov_aes_hw_gcm_aesni(void); #endif #if defined(AES_PMULL_CAPABLE) && defined(AES_GCM_ASM) -const PROV_GCM_HW *ossl_prov_aes_hw_gcm_armv8(size_t keybits); +const PROV_GCM_HW *ossl_prov_aes_hw_gcm_armv8(void); #endif #if defined(PPC_AES_GCM_CAPABLE) && defined(_ARCH_PPC64) const PROV_GCM_HW *ossl_prov_aes_hw_gcm_ppc(size_t keybits); diff --git a/providers/implementations/ciphers/cipher_aes_gcm_hw.c b/providers/implementations/ciphers/cipher_aes_gcm_hw.c index 41ff71cd218..0d166b545a1 100644 --- a/providers/implementations/ciphers/cipher_aes_gcm_hw.c +++ b/providers/implementations/ciphers/cipher_aes_gcm_hw.c @@ -158,7 +158,7 @@ const PROV_GCM_HW *ossl_prov_aes_hw_gcm(size_t keybits) #if defined(AESNI_CAPABLE) aes_gcm_hw = ossl_prov_aes_hw_gcm_aesni(); #elif defined(AES_PMULL_CAPABLE) && defined(AES_GCM_ASM) - aes_gcm_hw = ossl_prov_aes_hw_gcm_armv8(keybits); + aes_gcm_hw = ossl_prov_aes_hw_gcm_armv8(); #elif defined(PPC_AES_GCM_CAPABLE) && defined(_ARCH_PPC64) aes_gcm_hw = ossl_prov_aes_hw_gcm_ppc(keybits); #elif defined(OPENSSL_CPUID_OBJ) && defined(__riscv) && __riscv_xlen == 64 diff --git a/providers/implementations/ciphers/cipher_aes_gcm_hw_armv8.c b/providers/implementations/ciphers/cipher_aes_gcm_hw_armv8.c deleted file mode 100644 index 7d50de2a2f7..00000000000 --- a/providers/implementations/ciphers/cipher_aes_gcm_hw_armv8.c +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright 2019-2025 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * Crypto extension support for AES GCM. - * This file is used by cipher_aes_gcm_hw.c - */ -#include "internal/deprecated.h" -#include "cipher_aes_gcm.h" - -#if defined(AES_PMULL_CAPABLE) && defined(AES_GCM_ASM) - -size_t armv8_aes_gcm_encrypt(const unsigned char *in, unsigned char *out, size_t len, - const void *key, unsigned char ivec[16], uint64_t *Xi) -{ - AES_KEY *aes_key = (AES_KEY *)key; - size_t align_bytes = len - len % 16; - - switch (aes_key->rounds) { - case 10: - if (IS_CPU_SUPPORT_UNROLL8_EOR3()) { - unroll8_eor3_aes_gcm_enc_128_kernel(in, align_bytes * 8, out, (uint64_t *)Xi, ivec, key); - } else { - aes_gcm_enc_128_kernel(in, align_bytes * 8, out, (uint64_t *)Xi, ivec, key); - } - break; - case 12: - if (IS_CPU_SUPPORT_UNROLL8_EOR3()) { - unroll8_eor3_aes_gcm_enc_192_kernel(in, align_bytes * 8, out, (uint64_t *)Xi, ivec, key); - } else { - aes_gcm_enc_192_kernel(in, align_bytes * 8, out, (uint64_t *)Xi, ivec, key); - } - break; - case 14: - if (IS_CPU_SUPPORT_UNROLL8_EOR3()) { - unroll8_eor3_aes_gcm_enc_256_kernel(in, align_bytes * 8, out, (uint64_t *)Xi, ivec, key); - } else { - aes_gcm_enc_256_kernel(in, align_bytes * 8, out, (uint64_t *)Xi, ivec, key); - } - break; - } - return align_bytes; -} - -size_t armv8_aes_gcm_decrypt(const unsigned char *in, unsigned char *out, size_t len, - const void *key, unsigned char ivec[16], uint64_t *Xi) -{ - AES_KEY *aes_key = (AES_KEY *)key; - size_t align_bytes = len - len % 16; - - switch (aes_key->rounds) { - case 10: - if (IS_CPU_SUPPORT_UNROLL8_EOR3()) { - unroll8_eor3_aes_gcm_dec_128_kernel(in, align_bytes * 8, out, (uint64_t *)Xi, ivec, key); - } else { - aes_gcm_dec_128_kernel(in, align_bytes * 8, out, (uint64_t *)Xi, ivec, key); - } - break; - case 12: - if (IS_CPU_SUPPORT_UNROLL8_EOR3()) { - unroll8_eor3_aes_gcm_dec_192_kernel(in, align_bytes * 8, out, (uint64_t *)Xi, ivec, key); - } else { - aes_gcm_dec_192_kernel(in, align_bytes * 8, out, (uint64_t *)Xi, ivec, key); - } - break; - case 14: - if (IS_CPU_SUPPORT_UNROLL8_EOR3()) { - unroll8_eor3_aes_gcm_dec_256_kernel(in, align_bytes * 8, out, (uint64_t *)Xi, ivec, key); - } else { - aes_gcm_dec_256_kernel(in, align_bytes * 8, out, (uint64_t *)Xi, ivec, key); - } - break; - } - return align_bytes; -} - -static int armv8_aes_gcm_initkey(PROV_GCM_CTX *ctx, const unsigned char *key, - size_t keylen) -{ - if (AES_UNROLL12_EOR3_CAPABLE) { - return aes_gcm_hw_initkey(ctx, key, keylen, - aes_v8_set_encrypt_key, aes_v8_encrypt, - aes_v8_ctr32_encrypt_blocks_unroll12_eor3); - } else { - return aes_gcm_hw_initkey(ctx, key, keylen, - aes_v8_set_encrypt_key, aes_v8_encrypt, - aes_v8_ctr32_encrypt_blocks); - } -} - -static const PROV_GCM_HW armv8_aes_gcm = { - armv8_aes_gcm_initkey, - ossl_gcm_setiv, - ossl_gcm_aad_update, - generic_aes_gcm_cipher_update, - ossl_gcm_cipher_final, - ossl_gcm_one_shot -}; - -const PROV_GCM_HW *ossl_prov_aes_hw_gcm_armv8(size_t keybits) -{ - return AES_PMULL_CAPABLE ? &armv8_aes_gcm : NULL; -} - -#endif diff --git a/providers/implementations/ciphers/cipher_aes_hw.c b/providers/implementations/ciphers/cipher_aes_hw.c index 60abfd11655..e71a15cc005 100644 --- a/providers/implementations/ciphers/cipher_aes_hw.c +++ b/providers/implementations/ciphers/cipher_aes_hw.c @@ -219,7 +219,7 @@ static const PROV_CIPHER_HW *ossl_prov_cipher_hw_aes_mode(enum aes_modes mode, #if defined(AESNI_CAPABLE) aes_hw_mode = ossl_prov_cipher_hw_aesni(mode); #elif defined(ARMv8_HWAES_CAPABLE) - aes_hw_mode = ossl_prov_cipher_hw_arm(mode, keybits); + aes_hw_mode = ossl_prov_cipher_hw_arm(mode); #elif defined(OPENSSL_CPUID_OBJ) && defined(__riscv) && __riscv_xlen == 32 aes_hw_mode = ossl_prov_cipher_hw_rv32i(mode, keybits); #elif defined(OPENSSL_CPUID_OBJ) && defined(__riscv) && __riscv_xlen == 64 diff --git a/providers/implementations/ciphers/cipher_aes_hw_aesni.c b/providers/implementations/ciphers/cipher_aes_hw_aesni.c index 6264c88426d..15e2d71b3e5 100644 --- a/providers/implementations/ciphers/cipher_aes_hw_aesni.c +++ b/providers/implementations/ciphers/cipher_aes_hw_aesni.c @@ -8,8 +8,7 @@ */ /*- - * AES-NI support for AES modes ecb, cbc, ofb, cfb, ctr. - * This file is used by cipher_aes_hw.c + * AES-NI support for all hardware accelerated AES modes. */ #include "internal/deprecated.h" diff --git a/providers/implementations/ciphers/cipher_aes_hw_armv8.c b/providers/implementations/ciphers/cipher_aes_hw_armv8.c index 773e40a411d..da4f9fd0e3c 100644 --- a/providers/implementations/ciphers/cipher_aes_hw_armv8.c +++ b/providers/implementations/ciphers/cipher_aes_hw_armv8.c @@ -1,5 +1,5 @@ /* - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2026 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -8,15 +8,17 @@ */ /* - * Crypto extension support for AES modes ecb, cbc, ofb, cfb, ctr. - * This file is used by cipher_aes_hw.c + * ARMv8 support for all hardware accelerated AES modes. */ #include "internal/deprecated.h" #include "cipher_aes.h" +#include "cipher_aes_gcm.h" #if defined(ARMv8_HWAES_CAPABLE) +/* MODES: ctr */ + static int cipher_hw_aes_arm_initkey(PROV_CIPHER_CTX *ctx, const unsigned char *key, size_t keylen) { @@ -35,8 +37,7 @@ static const PROV_CIPHER_HW arm_ctr = { ossl_cipher_aes_copyctx }; -const PROV_CIPHER_HW *ossl_prov_cipher_hw_arm(enum aes_modes mode, - size_t keybits) +const PROV_CIPHER_HW *ossl_prov_cipher_hw_arm(enum aes_modes mode) { if (ARMv8_HWAES_CAPABLE && mode == AES_MODE_CTR) return &arm_ctr; @@ -44,3 +45,101 @@ const PROV_CIPHER_HW *ossl_prov_cipher_hw_arm(enum aes_modes mode, } #endif + +/* MODES: GCM */ + +#if defined(AES_PMULL_CAPABLE) && defined(AES_GCM_ASM) + +size_t armv8_aes_gcm_encrypt(const unsigned char *in, unsigned char *out, size_t len, + const void *key, unsigned char ivec[16], uint64_t *Xi) +{ + AES_KEY *aes_key = (AES_KEY *)key; + size_t align_bytes = len - len % 16; + + switch (aes_key->rounds) { + case 10: + if (IS_CPU_SUPPORT_UNROLL8_EOR3()) { + unroll8_eor3_aes_gcm_enc_128_kernel(in, align_bytes * 8, out, (uint64_t *)Xi, ivec, key); + } else { + aes_gcm_enc_128_kernel(in, align_bytes * 8, out, (uint64_t *)Xi, ivec, key); + } + break; + case 12: + if (IS_CPU_SUPPORT_UNROLL8_EOR3()) { + unroll8_eor3_aes_gcm_enc_192_kernel(in, align_bytes * 8, out, (uint64_t *)Xi, ivec, key); + } else { + aes_gcm_enc_192_kernel(in, align_bytes * 8, out, (uint64_t *)Xi, ivec, key); + } + break; + case 14: + if (IS_CPU_SUPPORT_UNROLL8_EOR3()) { + unroll8_eor3_aes_gcm_enc_256_kernel(in, align_bytes * 8, out, (uint64_t *)Xi, ivec, key); + } else { + aes_gcm_enc_256_kernel(in, align_bytes * 8, out, (uint64_t *)Xi, ivec, key); + } + break; + } + return align_bytes; +} + +size_t armv8_aes_gcm_decrypt(const unsigned char *in, unsigned char *out, size_t len, + const void *key, unsigned char ivec[16], uint64_t *Xi) +{ + AES_KEY *aes_key = (AES_KEY *)key; + size_t align_bytes = len - len % 16; + + switch (aes_key->rounds) { + case 10: + if (IS_CPU_SUPPORT_UNROLL8_EOR3()) { + unroll8_eor3_aes_gcm_dec_128_kernel(in, align_bytes * 8, out, (uint64_t *)Xi, ivec, key); + } else { + aes_gcm_dec_128_kernel(in, align_bytes * 8, out, (uint64_t *)Xi, ivec, key); + } + break; + case 12: + if (IS_CPU_SUPPORT_UNROLL8_EOR3()) { + unroll8_eor3_aes_gcm_dec_192_kernel(in, align_bytes * 8, out, (uint64_t *)Xi, ivec, key); + } else { + aes_gcm_dec_192_kernel(in, align_bytes * 8, out, (uint64_t *)Xi, ivec, key); + } + break; + case 14: + if (IS_CPU_SUPPORT_UNROLL8_EOR3()) { + unroll8_eor3_aes_gcm_dec_256_kernel(in, align_bytes * 8, out, (uint64_t *)Xi, ivec, key); + } else { + aes_gcm_dec_256_kernel(in, align_bytes * 8, out, (uint64_t *)Xi, ivec, key); + } + break; + } + return align_bytes; +} + +static int armv8_aes_gcm_initkey(PROV_GCM_CTX *ctx, const unsigned char *key, + size_t keylen) +{ + if (AES_UNROLL12_EOR3_CAPABLE) { + return aes_gcm_hw_initkey(ctx, key, keylen, + aes_v8_set_encrypt_key, aes_v8_encrypt, + aes_v8_ctr32_encrypt_blocks_unroll12_eor3); + } else { + return aes_gcm_hw_initkey(ctx, key, keylen, + aes_v8_set_encrypt_key, aes_v8_encrypt, + aes_v8_ctr32_encrypt_blocks); + } +} + +static const PROV_GCM_HW armv8_aes_gcm = { + armv8_aes_gcm_initkey, + ossl_gcm_setiv, + ossl_gcm_aad_update, + generic_aes_gcm_cipher_update, + ossl_gcm_cipher_final, + ossl_gcm_one_shot +}; + +const PROV_GCM_HW *ossl_prov_aes_hw_gcm_armv8(void) +{ + return AES_PMULL_CAPABLE ? &armv8_aes_gcm : NULL; +} + +#endif