From: Madan mohan Manokar Date: Wed, 29 Jul 2026 12:42:22 +0000 (+0530) Subject: Add AVX-512/VAES optimized AES-CBC decryption X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=62fe048fbe324faaf8e11ea9ce3c76be8a085ae7;p=thirdparty%2Fopenssl.git Add AVX-512/VAES optimized AES-CBC decryption Implement an AVX-512 VAES-based bulk path for AES-CBC decryption that processes multiple blocks in parallel, improving decryption performance for large inputs (1024 bytes or more) by 3.5x to 3.8x. - Add the VAES CBC decrypt intrinsic implementation (crypto/aes/aes_cbc_vaes_intrinsic.c) and wire it into the AES-NI provider cipher path. - Centralize the VAES_CBC_ELIGIBLE eligibility guard in include/crypto/aes_platform.h to keep the header and implementation in sync, and drop the redundant AES_BLOCK_SIZE define. - Guard the intrinsics with refined gcc, clang, and msvc compiler-version checks, ensuring clean builds in no-asm, no-deprecated, and Windows (shared/minimal) configurations. - Add an EVP-level AES-CBC decrypt test exercising the VAES bulk path across AES-128/192/256 and a range of input lengths. Reviewed-by: Simo Sorce Reviewed-by: Andrew Dinh MergeDate: Fri Jul 31 16:33:35 2026 (Merged from https://github.com/openssl/openssl/pull/30902) --- diff --git a/crypto/aes/aes_cbc_vaes_intrinsic.c b/crypto/aes/aes_cbc_vaes_intrinsic.c new file mode 100644 index 00000000000..91862f02f94 --- /dev/null +++ b/crypto/aes/aes_cbc_vaes_intrinsic.c @@ -0,0 +1,389 @@ +/* + * Copyright 2025 The OpenSSL Project Authors. All Rights Reserved. + * Copyright (C) 2026, Advanced Micro Devices, all rights reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + * + * Implements AES-CBC128/192/256 decryption with VAES (AVX-512) + * + * CBC encryption is inherently serial (each ciphertext block depends + * on the previous one), so VAES provides no benefit there -- the + * encrypt path falls back to the aesni_cbc_encrypt assembly routine. + * + * CBC decryption IS parallel: all blocks can be independently decrypted, + * then XORed with the preceding ciphertext block (or IV for the first). + * This implementation processes 4x4=16 blocks per iteration using four + * ZMM registers, falling back to 8, 4, then single-block processing. + */ + +#include "internal/deprecated.h" + +#include +#include "internal/cryptlib.h" +#include +#include "crypto/modes.h" +#include "crypto/aes_platform.h" + +#if VAES_CBC_ELIGIBLE + +/* Function prototypes */ +void ossl_aes_cbc_vaes_decrypt(const unsigned char *in, unsigned char *out, + size_t len, const void *key, + unsigned char ivec[16], int enc); +int ossl_aes_cbc_vaes_eligible(void); + +#include + +/* Forward declarations -- defined in aesni-x86_64.pl assembly */ +void aesni_cbc_encrypt(const unsigned char *in, unsigned char *out, + size_t len, const AES_KEY *key, + unsigned char *ivec, int enc); +void aesni_decrypt(const unsigned char *in, unsigned char *out, + const AES_KEY *key); + +/* Portable compiler abstractions for inlining and ISA target selection */ +#define STRINGIFY_IMPL_(a) #a +#define STRINGIFY_(a) STRINGIFY_IMPL_(a) + +#ifdef __clang__ +#define OPENSSL_TARGET_VAES512 \ + _Pragma(STRINGIFY_(clang attribute push( \ + __attribute__((target("avx512f,avx512dq,avx512bw,vaes,aes"))), \ + apply_to = function))) +#define OPENSSL_UNTARGET_VAES512 _Pragma("clang attribute pop") +#elif defined(__GNUC__) +#define OPENSSL_TARGET_VAES512 \ + _Pragma("GCC push_options") \ + _Pragma(STRINGIFY_(GCC target("avx512f,avx512dq,avx512bw,vaes,aes"))) +#define OPENSSL_UNTARGET_VAES512 _Pragma("GCC pop_options") +#else +/* MSVC: all intrinsics are always available via . */ +#define OPENSSL_TARGET_VAES512 +#define OPENSSL_UNTARGET_VAES512 +#endif + +#if defined(__GNUC__) || defined(__clang__) +#define OSSL_FUNC_ALWAYS_INLINE static inline __attribute__((always_inline)) +#define OSSL_FUNC_NOINLINE __attribute__((noinline)) +#elif defined(_MSC_VER) +#define OSSL_FUNC_ALWAYS_INLINE static __forceinline +#define OSSL_FUNC_NOINLINE __declspec(noinline) +#else +#define OSSL_FUNC_ALWAYS_INLINE static inline +#define OSSL_FUNC_NOINLINE +#endif + +#include + +OPENSSL_TARGET_VAES512 + +/* ------------------------------------------------------------------ */ +/* AES decryption helpers: 1x, 2x, 4x parallel 512-bit blocks */ +/* Each 512-bit register holds 4 independent 128-bit AES blocks. */ +/* always_inline guarantees the compiler keeps keys in ZMM regs. */ +/* ------------------------------------------------------------------ */ + +#define DEFINE_AES_DECRYPT_FUNCS(ROUNDS) \ + OSSL_FUNC_ALWAYS_INLINE \ + void AesDec_4x512_##ROUNDS( \ + __m512i *b1, __m512i *b2, __m512i *b3, __m512i *b4, \ + const __m512i *rk) \ + { \ + *b1 = _mm512_xor_si512(*b1, rk[0]); \ + *b2 = _mm512_xor_si512(*b2, rk[0]); \ + *b3 = _mm512_xor_si512(*b3, rk[0]); \ + *b4 = _mm512_xor_si512(*b4, rk[0]); \ + for (int i = 1; i < ROUNDS; i++) { \ + *b1 = _mm512_aesdec_epi128(*b1, rk[i]); \ + *b2 = _mm512_aesdec_epi128(*b2, rk[i]); \ + *b3 = _mm512_aesdec_epi128(*b3, rk[i]); \ + *b4 = _mm512_aesdec_epi128(*b4, rk[i]); \ + } \ + *b1 = _mm512_aesdeclast_epi128(*b1, rk[ROUNDS]); \ + *b2 = _mm512_aesdeclast_epi128(*b2, rk[ROUNDS]); \ + *b3 = _mm512_aesdeclast_epi128(*b3, rk[ROUNDS]); \ + *b4 = _mm512_aesdeclast_epi128(*b4, rk[ROUNDS]); \ + } \ + \ + OSSL_FUNC_ALWAYS_INLINE \ + void AesDec_2x512_##ROUNDS( \ + __m512i *b1, __m512i *b2, const __m512i *rk) \ + { \ + *b1 = _mm512_xor_si512(*b1, rk[0]); \ + *b2 = _mm512_xor_si512(*b2, rk[0]); \ + for (int i = 1; i < ROUNDS; i++) { \ + *b1 = _mm512_aesdec_epi128(*b1, rk[i]); \ + *b2 = _mm512_aesdec_epi128(*b2, rk[i]); \ + } \ + *b1 = _mm512_aesdeclast_epi128(*b1, rk[ROUNDS]); \ + *b2 = _mm512_aesdeclast_epi128(*b2, rk[ROUNDS]); \ + } \ + \ + OSSL_FUNC_ALWAYS_INLINE \ + void AesDec_1x512_##ROUNDS( \ + __m512i *b1, const __m512i *rk) \ + { \ + *b1 = _mm512_xor_si512(*b1, rk[0]); \ + for (int i = 1; i < ROUNDS; i++) \ + *b1 = _mm512_aesdec_epi128(*b1, rk[i]); \ + *b1 = _mm512_aesdeclast_epi128(*b1, rk[ROUNDS]); \ + } + +DEFINE_AES_DECRYPT_FUNCS(10) /* AES-128 */ +DEFINE_AES_DECRYPT_FUNCS(12) /* AES-192 */ +DEFINE_AES_DECRYPT_FUNCS(14) /* AES-256 */ + +/* ------------------------------------------------------------------ */ +/* CBC-mode decryption -- templated per round count */ +/* */ +/* Processes as many full blocks as possible: */ +/* 16 blocks at a time (4 x zmm = 4 x 4 = 16 blocks) */ +/* 8 blocks at a time (2 x zmm) */ +/* 4 blocks at a time (1 x zmm) */ +/* 1 block at a time for the remaining 0-3 blocks */ +/* */ +/* The chaining vector b1 packs [prev_ct[last] | ct[0] | ct[1] */ +/* | ct[2]] so that a single XOR after decryption applies the CBC */ +/* feedback to all four lanes simultaneously. */ +/* ------------------------------------------------------------------ */ + +#define DEFINE_CBC_DECRYPT(NR) \ + OSSL_FUNC_NOINLINE \ + static void cbc_decrypt_##NR( \ + const unsigned char *in, unsigned char *out, size_t len, \ + const AES_KEY *key, unsigned char *iv) \ + { \ + const unsigned char *rk_bytes = (const unsigned char *)key->rd_key; \ + __m512i rk[NR + 1]; \ + for (int i = 0; i <= NR; i++) { \ + __m128i t = _mm_loadu_si128((const __m128i *)(rk_bytes + i * 16)); \ + rk[i] = _mm512_broadcast_i32x4(t); \ + } \ + \ + __m512i a1, a2, a3, a4; \ + __m512i b1, b2, b3, b4; \ + \ + const __m128i *pa = (const __m128i *)in; \ + __m512i *po = (__m512i *)out; \ + size_t blocks = len / AES_BLOCK_SIZE; \ + \ + /* Save last ciphertext block for IV update (in-place safe) */ \ + __m128i saved_iv = _mm_setzero_si128(); \ + int has_blocks = (blocks > 0); \ + if (has_blocks) \ + saved_iv = _mm_loadu_si128(pa + blocks - 1); \ + \ + if (blocks >= 4) { \ + /* Build b1 = [IV | ct[0] | ct[1] | ct[2]] */ \ + __m512i idx = _mm512_set_epi64(5, 4, 3, 2, 1, 0, 0, 0); \ + __m512i ct0; \ + \ + /* CBC C[0]=IV; 0x03 loads one 128-bit block (two 64-bit lanes). */ \ + b1 = _mm512_maskz_loadu_epi64(0x03, iv); \ + ct0 = _mm512_loadu_si512(pa); \ + ct0 = _mm512_permutexvar_epi64(idx, ct0); \ + b1 = _mm512_mask_blend_epi64(0xFC, b1, ct0); \ + \ + /* --- 16-block (4 x zmm) main loop --- */ \ + while (blocks >= 16) { \ + __m128i last; \ + \ + a1 = _mm512_loadu_si512(pa); \ + a2 = _mm512_loadu_si512(pa + 4); \ + a3 = _mm512_loadu_si512(pa + 8); \ + a4 = _mm512_loadu_si512(pa + 12); \ + \ + b2 = _mm512_loadu_si512(pa + 3); \ + b3 = _mm512_loadu_si512(pa + 7); \ + b4 = _mm512_loadu_si512(pa + 11); \ + \ + last = _mm_loadu_si128(pa + 15); \ + \ + AesDec_4x512_##NR(&a1, &a2, &a3, &a4, rk); \ + \ + a1 = _mm512_xor_si512(a1, b1); \ + a2 = _mm512_xor_si512(a2, b2); \ + a3 = _mm512_xor_si512(a3, b3); \ + a4 = _mm512_xor_si512(a4, b4); \ + \ + _mm512_storeu_si512(po, a1); \ + _mm512_storeu_si512(po + 1, a2); \ + _mm512_storeu_si512(po + 2, a3); \ + _mm512_storeu_si512(po + 3, a4); \ + \ + /* Build next b1 from last ciphertext block */ \ + b1 = _mm512_maskz_loadu_epi64(0x03, &last); \ + if (blocks > 16) { \ + size_t rem = blocks - 16; \ + /* Load only available lookahead blocks to avoid OOB read. */ \ + /* One AES block is 16 bytes, it maps to two 64-bit lanes. */ \ + /* 0x03 (00000011) for 1 block (2 lanes), */ \ + /* 0x0F (00001111) for 2 blocks (4 lanes), */ \ + /* 0x3F (00111111) for 3 or more blocks (6 lanes). */ \ + __mmask8 nxmask = (rem >= 3) ? 0x3F : (rem == 2 ? 0x0F : 0x03); \ + __m512i nx = _mm512_maskz_loadu_epi64(nxmask, pa + 16); \ + nx = _mm512_permutexvar_epi64(idx, nx); \ + b1 = _mm512_mask_blend_epi64(0xFC, b1, nx); \ + } \ + \ + pa += 16; \ + po += 4; \ + blocks -= 16; \ + } \ + \ + /* --- 8-block (2 x zmm) --- */ \ + if (blocks >= 8) { \ + __m128i last8; \ + \ + a1 = _mm512_loadu_si512(pa); \ + a2 = _mm512_loadu_si512(pa + 4); \ + b2 = _mm512_loadu_si512(pa + 3); \ + last8 = _mm_loadu_si128(pa + 7); \ + \ + AesDec_2x512_##NR(&a1, &a2, rk); \ + a1 = _mm512_xor_si512(a1, b1); \ + a2 = _mm512_xor_si512(a2, b2); \ + \ + _mm512_storeu_si512(po, a1); \ + _mm512_storeu_si512(po + 1, a2); \ + \ + b1 = _mm512_maskz_loadu_epi64(0x03, &last8); \ + pa += 8; \ + po += 2; \ + blocks -= 8; \ + \ + if (blocks >= 4) { \ + __m512i nx = _mm512_loadu_si512(pa); \ + nx = _mm512_permutexvar_epi64(idx, nx); \ + b1 = _mm512_mask_blend_epi64(0xFC, b1, nx); \ + } \ + } \ + \ + /* --- 4-block (1 x zmm) --- */ \ + if (blocks >= 4) { \ + __m128i last4; \ + \ + a1 = _mm512_loadu_si512(pa); \ + last4 = _mm_loadu_si128(pa + 3); \ + \ + AesDec_1x512_##NR(&a1, rk); \ + a1 = _mm512_xor_si512(a1, b1); \ + _mm512_storeu_si512(po, a1); \ + \ + b1 = _mm512_maskz_loadu_epi64(0x03, &last4); \ + pa += 4; \ + po += 1; \ + blocks -= 4; \ + } \ + \ + /* --- Remaining 1-3 blocks --- */ \ + { \ + __m128i *po128 = (__m128i *)po; \ + while (blocks > 0) { \ + __m128i ct = _mm_loadu_si128(pa); \ + a1 = _mm512_maskz_loadu_epi64(0x03, pa); \ + AesDec_1x512_##NR(&a1, rk); \ + a1 = _mm512_xor_si512(a1, b1); \ + _mm512_mask_storeu_epi64(po128, 0x03, a1); \ + b1 = _mm512_maskz_loadu_epi64(0x03, &ct); \ + pa++; \ + po128++; \ + blocks--; \ + } \ + } \ + } else { \ + /* Less than 4 blocks -- process individually */ \ + __m128i *po128 = (__m128i *)po; \ + b1 = _mm512_maskz_loadu_epi64(0x03, iv); \ + while (blocks > 0) { \ + __m128i ct = _mm_loadu_si128(pa); \ + a1 = _mm512_maskz_loadu_epi64(0x03, pa); \ + AesDec_1x512_##NR(&a1, rk); \ + a1 = _mm512_xor_si512(a1, b1); \ + _mm512_mask_storeu_epi64(po128, 0x03, a1); \ + b1 = _mm512_maskz_loadu_epi64(0x03, &ct); \ + pa++; \ + po128++; \ + blocks--; \ + } \ + } \ + \ + if (has_blocks) \ + _mm_storeu_si128((__m128i *)iv, saved_iv); \ + \ + /* Clear round-key material from the stack */ \ + { \ + /* Use of volatile prevents dead-store elimination by compilers. */ \ + volatile __m512i *vrk = (volatile __m512i *)(volatile void *)rk; \ + for (int i = 0; i <= NR; i++) \ + vrk[i] = _mm512_setzero_si512(); \ + } \ + } + +DEFINE_CBC_DECRYPT(10) /* AES-128 */ +DEFINE_CBC_DECRYPT(12) /* AES-192 */ +DEFINE_CBC_DECRYPT(14) /* AES-256 */ + +/* ------------------------------------------------------------------ */ +/* Public entry point */ +/* ------------------------------------------------------------------ */ + +void ossl_aes_cbc_vaes_decrypt(const unsigned char *in, unsigned char *out, + size_t len, const void *key, + unsigned char ivec[16], int enc) +{ + size_t full_bytes; + int nr = ((const AES_KEY *)key)->rounds + 1; + + if (len == 0) + return; + + /* VAES path only optimises decryption; encrypt falls back to asm */ + if (enc) { + aesni_cbc_encrypt(in, out, len, (const AES_KEY *)key, ivec, enc); + return; + } + + full_bytes = (len / AES_BLOCK_SIZE) * AES_BLOCK_SIZE; + if (full_bytes > 0) { + switch (nr) { + case 10: + cbc_decrypt_10(in, out, full_bytes, (const AES_KEY *)key, ivec); + break; + case 12: + cbc_decrypt_12(in, out, full_bytes, (const AES_KEY *)key, ivec); + break; + case 14: + cbc_decrypt_14(in, out, full_bytes, (const AES_KEY *)key, ivec); + break; + default: /* invalid key size */ + aesni_cbc_encrypt(in, out, len, (const AES_KEY *)key, ivec, 0); + break; + } + } +} + +/* ------------------------------------------------------------------ */ +/* CPU feature check */ +/* ------------------------------------------------------------------ */ + +int ossl_aes_cbc_vaes_eligible(void) +{ + return (OPENSSL_ia32cap_P[2] & (1 << 16)) /* AVX512F */ + && (OPENSSL_ia32cap_P[2] & (1 << 17)) /* AVX512DQ */ + && (OPENSSL_ia32cap_P[2] & (1 << 30)) /* AVX512BW */ + && (OPENSSL_ia32cap_P[3] & (1 << 9)); /* AVX512VAES */ +} + +OPENSSL_UNTARGET_VAES512 + +#undef OPENSSL_TARGET_VAES512 +#undef OPENSSL_UNTARGET_VAES512 +#undef STRINGIFY_IMPL_ +#undef STRINGIFY_ +#undef OSSL_FUNC_ALWAYS_INLINE +#undef OSSL_FUNC_NOINLINE +#endif /* VAES_CBC_ELIGIBLE */ diff --git a/crypto/aes/build.info b/crypto/aes/build.info index cb3bb5f2deb..9df6fcc7903 100644 --- a/crypto/aes/build.info +++ b/crypto/aes/build.info @@ -74,11 +74,11 @@ IF[{- !$disabled{asm} -}] ENDIF $COMMON=aes_misc.c aes_ecb.c $AESASM -SOURCE[../../libcrypto]=$COMMON aes_cfb.c aes_ofb.c aes_wrap.c +SOURCE[../../libcrypto]=$COMMON aes_cfb.c aes_ofb.c aes_wrap.c aes_cbc_vaes_intrinsic.c IF[{- !$disabled{'deprecated-3.0'} -}] SOURCE[../../libcrypto]=aes_ige.c ENDIF -SOURCE[../../providers/libfips.a]=$COMMON +SOURCE[../../providers/libfips.a]=$COMMON aes_cbc_vaes_intrinsic.c # Implementations are now spread across several libraries, so the defines # need to be applied to all affected libraries and modules. diff --git a/include/crypto/aes_platform.h b/include/crypto/aes_platform.h index 36ba2665ee8..6c0d0258e4b 100644 --- a/include/crypto/aes_platform.h +++ b/include/crypto/aes_platform.h @@ -184,6 +184,22 @@ void gcm_ghash_v8(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp, siz #endif #endif +#if (defined(__x86_64) || defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64)) \ + && !defined(OPENSSL_NO_ASM) \ + && ((defined(__GNUC__) && !defined(__clang__) && (__GNUC__ >= 8)) \ + || (defined(__clang__) && (__clang_major__ >= 7)) \ + || (defined(_MSC_VER) && (_MSC_VER >= 1927))) +#define VAES_CBC_ELIGIBLE 1 +#else +#define VAES_CBC_ELIGIBLE 0 +#endif + +#if VAES_CBC_ELIGIBLE +void ossl_aes_cbc_vaes_decrypt(const unsigned char *in, unsigned char *out, + size_t len, const void *key, unsigned char ivec[16], int enc); +int ossl_aes_cbc_vaes_eligible(void); +#endif + #if defined(AES_ASM) && !defined(I386_ONLY) && (((defined(__i386) || defined(__i386__) || defined(_M_IX86)) && defined(OPENSSL_IA32_SSE2)) || defined(__x86_64) || defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64)) /* AES-NI section */ @@ -210,6 +226,7 @@ void ossl_aes_cfb128_vaes_enc(const unsigned char *in, unsigned char *out, void ossl_aes_cfb128_vaes_dec(const unsigned char *in, unsigned char *out, size_t len, const AES_KEY *ks, const unsigned char ivec[16], ossl_ssize_t *num); + int ossl_aes_cfb128_vaes_eligible(void); void aesni_encrypt(const unsigned char *in, unsigned char *out, diff --git a/providers/implementations/ciphers/cipher_aes_hw_aesni.c b/providers/implementations/ciphers/cipher_aes_hw_aesni.c index f8ae45129bf..db4a9e3e77a 100644 --- a/providers/implementations/ciphers/cipher_aes_hw_aesni.c +++ b/providers/implementations/ciphers/cipher_aes_hw_aesni.c @@ -68,6 +68,35 @@ static const PROV_CIPHER_HW aesni_cbc = { ossl_cipher_aes_copyctx }; +#if VAES_CBC_ELIGIBLE +/* + * CBC decryption is fully parallel, so VAES accelerates it; CBC encryption is + * inherently serial and stays on the aesni_cbc_encrypt assembly routine. VAES + * is only worthwhile once the payload is large enough to amortize the per-call + * key broadcast, so small buffers also fall back to the assembly routine. + */ +static int aes_cbc_vaes_wrapper( + PROV_CIPHER_CTX *ctx, + unsigned char *out, + const unsigned char *in, + size_t len) +{ + if (!ctx->enc && len >= 256) { + ossl_aes_cbc_vaes_decrypt(in, out, len, ctx->ks, ctx->iv, ctx->enc); + return 1; + } + + aesni_cbc_encrypt(in, out, len, ctx->ks, ctx->iv, ctx->enc); + return 1; +} + +static const PROV_CIPHER_HW aesni_vaes_cbc = { + cipher_hw_aesni_initkey, + aes_cbc_vaes_wrapper, + ossl_cipher_aes_copyctx +}; +#endif /* VAES_CBC_ELIGIBLE */ + #if (defined(__x86_64) || defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64)) /* active in 64-bit builds when AES-NI, AVX512F, and VAES are detected */ #define VAES_CFB128_ELIGIBLE 1 @@ -146,6 +175,10 @@ const PROV_CIPHER_HW *ossl_prov_cipher_hw_aesni(enum aes_modes mode) case AES_MODE_ECB: return &aesni_ecb; case AES_MODE_CBC: +#if VAES_CBC_ELIGIBLE + if (ossl_aes_cbc_vaes_eligible()) + return &aesni_vaes_cbc; +#endif return &aesni_cbc; case AES_MODE_CFB128: #if VAES_CFB128_ELIGIBLE diff --git a/test/evp_extra_test.c b/test/evp_extra_test.c index 00ed62f264f..af729d13f2f 100644 --- a/test/evp_extra_test.c +++ b/test/evp_extra_test.c @@ -6993,6 +6993,102 @@ err: return testresult; } +static const char *const aes_cbc_decrypt_ciphers[] = { + "AES-128-CBC", "AES-192-CBC", "AES-256-CBC" +}; + +/* + * Lengths (in bytes, all block-aligned) chosen so that the block count modulo + * the 16-block main loop hits every tail path in the bulk CBC decrypt routine: + * exact multiple of 16 blocks, the 1/2/3-block lookahead variants, and the + * 8-block, 4-block and 1-3 block remainder paths. + */ +static const int aes_cbc_decrypt_lengths[] = { + 256, /* 16 blocks: main loop once, no lookahead */ + 272, /* 17 blocks: lookahead rem==1, 1-block tail */ + 288, /* 18 blocks: lookahead rem==2, 2-block tail */ + 304, /* 19 blocks: lookahead rem==3, 3-block tail */ + 320, /* 20 blocks: 4-block path */ + 384, /* 24 blocks: 8-block path */ + 448, /* 28 blocks: 8-block + nested lookahead + 4-block */ + 496, /* 31 blocks: 8 + 4 + 3-block tail */ + 512 /* 32 blocks: main loop twice */ +}; + +#define AES_CBC_DECRYPT_MAXLEN 512 + +/* + * For each length, decrypt the ciphertext in a single call (the >= 256 byte + * length exercises the bulk/VAES CBC decrypt path) and again one block at a + * time (keeping every call below the bulk threshold, i.e. an independent + * reference decrypt). The bulk output must match both the original plaintext + * and the reference, for AES-128/192/256 and across all length branches. + */ +static int test_aes_cbc_decrypt(int idx) +{ + const char *ciphername = aes_cbc_decrypt_ciphers[idx]; + unsigned char key[32], iv[16], pt[AES_CBC_DECRYPT_MAXLEN]; + unsigned char ct[AES_CBC_DECRYPT_MAXLEN]; + unsigned char bulk_out[AES_CBC_DECRYPT_MAXLEN]; + unsigned char ref_out[AES_CBC_DECRYPT_MAXLEN]; + int testresult = 0, i, outl, tmpl, off; + size_t li; + EVP_CIPHER *cipher = NULL; + EVP_CIPHER_CTX *ctx = NULL; + + for (i = 0; i < (int)sizeof(key); i++) + key[i] = (unsigned char)(i + 1); + for (i = 0; i < (int)sizeof(iv); i++) + iv[i] = (unsigned char)(0xf0 ^ i); + for (i = 0; i < AES_CBC_DECRYPT_MAXLEN; i++) + pt[i] = (unsigned char)(i * 7 + 3); + + if (!TEST_ptr(cipher = EVP_CIPHER_fetch(testctx, ciphername, testpropq)) + || !TEST_ptr(ctx = EVP_CIPHER_CTX_new())) + goto err; + + for (li = 0; li < OSSL_NELEM(aes_cbc_decrypt_lengths); li++) { + int buflen = aes_cbc_decrypt_lengths[li]; + + /* Reference encrypt (block-aligned input, padding disabled). */ + if (!TEST_true(EVP_EncryptInit_ex(ctx, cipher, NULL, key, iv)) + || !TEST_true(EVP_CIPHER_CTX_set_padding(ctx, 0)) + || !TEST_true(EVP_EncryptUpdate(ctx, ct, &outl, pt, buflen)) + || !TEST_int_eq(outl, buflen)) + goto err; + + /* One-shot decrypt: exercises the bulk path for this length. */ + if (!TEST_true(EVP_DecryptInit_ex(ctx, cipher, NULL, key, iv)) + || !TEST_true(EVP_CIPHER_CTX_set_padding(ctx, 0)) + || !TEST_true(EVP_DecryptUpdate(ctx, bulk_out, &outl, ct, buflen)) + || !TEST_int_eq(outl, buflen)) + goto err; + + /* Reference decrypt: one block per call bypasses the bulk path. */ + if (!TEST_true(EVP_DecryptInit_ex(ctx, cipher, NULL, key, iv)) + || !TEST_true(EVP_CIPHER_CTX_set_padding(ctx, 0))) + goto err; + for (off = 0; off < buflen; off += 16) { + if (!TEST_true(EVP_DecryptUpdate(ctx, ref_out + off, &tmpl, + ct + off, 16)) + || !TEST_int_eq(tmpl, 16)) + goto err; + } + + if (!TEST_mem_eq(bulk_out, buflen, pt, buflen) + || !TEST_mem_eq(bulk_out, buflen, ref_out, buflen)) { + TEST_info("%s failed at length %d", ciphername, buflen); + goto err; + } + } + + testresult = 1; +err: + EVP_CIPHER_CTX_free(ctx); + EVP_CIPHER_free(cipher); + return testresult; +} + typedef struct { const char *cipher; int enc; @@ -9362,6 +9458,7 @@ int setup_tests(void) ADD_ALL_TESTS(test_evp_init_seq, OSSL_NELEM(evp_init_tests)); ADD_ALL_TESTS(test_evp_reset, OSSL_NELEM(evp_reset_tests)); + ADD_ALL_TESTS(test_aes_cbc_decrypt, OSSL_NELEM(aes_cbc_decrypt_ciphers)); ADD_ALL_TESTS(test_evp_reinit_seq, OSSL_NELEM(evp_reinit_tests)); ADD_ALL_TESTS(test_gcm_reinit, OSSL_NELEM(gcm_reinit_tests)); ADD_ALL_TESTS(test_evp_updated_iv, OSSL_NELEM(evp_updated_iv_tests));