From: eclipse07077 Date: Mon, 9 Mar 2026 12:35:43 +0000 (+0900) Subject: Fix integer overflow in EVP_ENCODE_LENGTH and base64 encoding paths X-Git-Tag: openssl-4.0.0~70 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=8c9e0909d832cb4bb93bf097870e9020d6d28a62;p=thirdparty%2Fopenssl.git Fix integer overflow in EVP_ENCODE_LENGTH and base64 encoding paths The EVP_ENCODE_LENGTH macro performs all arithmetic in the type of its argument. When the argument is int and exceeds approximately 1.6 billion, intermediate results overflow signed int, potentially wrapping to a smaller positive value rather than a negative one. In b64_write() (crypto/evp/bio_b64.c), this causes OPENSSL_malloc to allocate a buffer smaller than the actual encoded output size. EVP_EncodeUpdate then writes past the end of the undersized buffer. Changes: - Cast macro argument to size_t in EVP_ENCODE_LENGTH to prevent signed integer overflow - Change encoded_length in b64_write() from int to size_t and add an explicit overflow sanity check before allocation - Change return type of evp_encodeblock_int() and encode_base64_avx2() from int to size_t so that large encoded output lengths are not truncated - Update EVP_EncodeUpdate() to use size_t for the encoder return value accumulator (j), consistent with the existing size_t total - Add explicit (int) casts in EVP_EncodeBlock() and EVP_EncodeFinal() where the public API requires int return values Reviewed-by: Saša Nedvědický Reviewed-by: Tomas Mraz MergeDate: Fri Apr 3 14:55:29 2026 (Merged from https://github.com/openssl/openssl/pull/30321) (cherry picked from commit 3741076a34f4953ccbb6e1e131756ac206c8f27e) --- diff --git a/crypto/evp/bio_b64.c b/crypto/evp/bio_b64.c index 33ddb6334d7..7f32d7a5d0c 100644 --- a/crypto/evp/bio_b64.c +++ b/crypto/evp/bio_b64.c @@ -334,7 +334,7 @@ static int b64_write(BIO *b, const char *in, int inl) int i; BIO_B64_CTX *ctx; BIO *next; - int encoded_length; + size_t encoded_length; unsigned char *encoded; int n_bytes_enc; @@ -393,7 +393,12 @@ static int b64_write(BIO *b, const char *in, int inl) encoded_length = EVP_ENCODE_LENGTH(inl); - if (ctx->encoded_buf == NULL || (size_t)encoded_length > ctx->encoded_buf_len) { + if (encoded_length > SIZE_MAX / 2) { + ERR_raise(ERR_LIB_BIO, BIO_R_LENGTH_TOO_LONG); + return -1; + } + + if (ctx->encoded_buf == NULL || encoded_length > ctx->encoded_buf_len) { OPENSSL_free(ctx->encoded_buf); ctx->encoded_buf = OPENSSL_malloc(encoded_length); if (ctx->encoded_buf == NULL) { diff --git a/crypto/evp/enc_b64_avx2.c b/crypto/evp/enc_b64_avx2.c index e6f571dd8e3..acf35739b5c 100644 --- a/crypto/evp/enc_b64_avx2.c +++ b/crypto/evp/enc_b64_avx2.c @@ -477,7 +477,7 @@ static inline size_t insert_nl_str8(const __m256i v0, uint8_t *output) OPENSSL_UNTARGET_AVX2 OPENSSL_TARGET_AVX2 -int encode_base64_avx2(EVP_ENCODE_CTX *ctx, unsigned char *dst, +size_t encode_base64_avx2(EVP_ENCODE_CTX *ctx, unsigned char *dst, const unsigned char *src, int srclen, int ctx_length, int *final_wrap_cnt) { @@ -665,7 +665,7 @@ int encode_base64_avx2(EVP_ENCODE_CTX *ctx, unsigned char *dst, *out++ = '\n'; } - return (int)(out - (uint8_t *)dst) + +evp_encodeblock_int(ctx, out, src + i, srclen - i, final_wrap_cnt); + return (size_t)(out - (uint8_t *)dst) + evp_encodeblock_int(ctx, out, src + i, srclen - i, final_wrap_cnt); } OPENSSL_UNTARGET_AVX2 #endif /* !defined(_M_ARM64EC) */ diff --git a/crypto/evp/enc_b64_avx2.h b/crypto/evp/enc_b64_avx2.h index db67b5cf8ad..89dba6285d5 100644 --- a/crypto/evp/enc_b64_avx2.h +++ b/crypto/evp/enc_b64_avx2.h @@ -2,10 +2,11 @@ #define OSSL_CRYPTO_EVP_B64_AVX2_H #include +#include #if defined(__x86_64) || defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64) #if !defined(_M_ARM64EC) -int encode_base64_avx2(EVP_ENCODE_CTX *ctx, +size_t encode_base64_avx2(EVP_ENCODE_CTX *ctx, unsigned char *out, const unsigned char *src, int srclen, int newlines, int *wrap_cnt); #endif /* !defined(_M_ARM64EC) */ diff --git a/crypto/evp/enc_b64_scalar.c b/crypto/evp/enc_b64_scalar.c index 1ca46a8357e..4bdadcea0ca 100644 --- a/crypto/evp/enc_b64_scalar.c +++ b/crypto/evp/enc_b64_scalar.c @@ -124,11 +124,11 @@ static const unsigned char base64_std_bin2ascii_2[256] = { '/' }; -int evp_encodeblock_int(EVP_ENCODE_CTX *ctx, unsigned char *t, +size_t evp_encodeblock_int(EVP_ENCODE_CTX *ctx, unsigned char *t, const unsigned char *f, int dlen, int *wrap_cnt) { int i = 0; - int ret = 0; + size_t ret = 0; uint8_t t1, t2, t3; const unsigned char *e0, *e1, *e2; int srp = (ctx != NULL diff --git a/crypto/evp/enc_b64_scalar.h b/crypto/evp/enc_b64_scalar.h index 91d416f7586..43059938ccf 100644 --- a/crypto/evp/enc_b64_scalar.h +++ b/crypto/evp/enc_b64_scalar.h @@ -1,8 +1,9 @@ #ifndef OSSL_CRYPTO_EVP_B64_SCALAR_H #define OSSL_CRYPTO_EVP_B64_SCALAR_H #include +#include -int evp_encodeblock_int(EVP_ENCODE_CTX *ctx, unsigned char *t, +size_t evp_encodeblock_int(EVP_ENCODE_CTX *ctx, unsigned char *t, const unsigned char *f, int dlen, int *wrap_cnt); #endif diff --git a/crypto/evp/encode.c b/crypto/evp/encode.c index eacc68bb965..faf9a3887d5 100644 --- a/crypto/evp/encode.c +++ b/crypto/evp/encode.c @@ -26,7 +26,7 @@ static unsigned char conv_ascii2bin(unsigned char a, const unsigned char *table); -int evp_encodeblock_int(EVP_ENCODE_CTX *ctx, unsigned char *t, +size_t evp_encodeblock_int(EVP_ENCODE_CTX *ctx, unsigned char *t, const unsigned char *f, int dlen, int *wrap_cnt); static int evp_decodeblock_int(EVP_ENCODE_CTX *ctx, unsigned char *t, const unsigned char *f, int n, int eof); @@ -374,7 +374,8 @@ void EVP_EncodeInit(EVP_ENCODE_CTX *ctx) int EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl, const unsigned char *in, int inl) { - int i, j; + int i; + size_t j; size_t total = 0; *outl = 0; @@ -452,20 +453,20 @@ int EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl, void EVP_EncodeFinal(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl) { - int ret = 0; + size_t ret = 0; int wrap_cnt = 0; if (ctx->num != 0) { ret = evp_encodeblock_int(ctx, out, ctx->enc_data, ctx->num, &wrap_cnt); - if (ossl_assert(ret >= 0)) { + if (ret > 0) { if ((ctx->flags & EVP_ENCODE_CTX_NO_NEWLINES) == 0) out[ret++] = '\n'; out[ret] = '\0'; ctx->num = 0; } } - *outl = ret; + *outl = (int)ret; } int EVP_EncodeBlock(unsigned char *t, const unsigned char *f, int dlen) @@ -473,14 +474,14 @@ int EVP_EncodeBlock(unsigned char *t, const unsigned char *f, int dlen) int wrap_cnt = 0; #if defined(__AVX2__) - return encode_base64_avx2(NULL, t, f, dlen, 0, &wrap_cnt); + return (int)encode_base64_avx2(NULL, t, f, dlen, 0, &wrap_cnt); #elif defined(HAS_IA32CAP_IS_64) if ((OPENSSL_ia32cap_P[2] & (1u << 5)) != 0) - return encode_base64_avx2(NULL, t, f, dlen, 0, &wrap_cnt); + return (int)encode_base64_avx2(NULL, t, f, dlen, 0, &wrap_cnt); else - return evp_encodeblock_int(NULL, t, f, dlen, &wrap_cnt); + return (int)evp_encodeblock_int(NULL, t, f, dlen, &wrap_cnt); #else - return evp_encodeblock_int(NULL, t, f, dlen, &wrap_cnt); + return (int)evp_encodeblock_int(NULL, t, f, dlen, &wrap_cnt); #endif } diff --git a/include/openssl/evp.h b/include/openssl/evp.h index 464a9f1efa5..39a6204545d 100644 --- a/include/openssl/evp.h +++ b/include/openssl/evp.h @@ -570,7 +570,7 @@ void *EVP_CIPHER_CTX_set_cipher_data(EVP_CIPHER_CTX *ctx, void *cipher_data); #define EVP_CIPHER_CTX_get_mode(c) EVP_CIPHER_get_mode(EVP_CIPHER_CTX_get0_cipher(c)) #define EVP_CIPHER_CTX_mode EVP_CIPHER_CTX_get_mode -#define EVP_ENCODE_LENGTH(l) ((((l) + 2) / 3 * 4) + ((l) / 48 + 1) * 2 + 80) +#define EVP_ENCODE_LENGTH(l) (((((size_t)(l)) + 2) / 3 * 4) + (((size_t)(l)) / 48 + 1) * 2 + 80) #define EVP_DECODE_LENGTH(l) (((l) + 3) / 4 * 3 + 80) #define EVP_SignInit_ex(a, b, c) EVP_DigestInit_ex(a, b, c)