Fix integer overflow in EVP_ENCODE_LENGTH and base64 encoding paths

The EVP_ENCODE_LENGTH macro performs all arithmetic in the type of
its argument. When the argument is int and exceeds approximately
1.6 billion, intermediate results overflow signed int, potentially
wrapping to a smaller positive value rather than a negative one.

In b64_write() (crypto/evp/bio_b64.c), this causes OPENSSL_malloc
to allocate a buffer smaller than the actual encoded output size.
EVP_EncodeUpdate then writes past the end of the undersized buffer.

Changes:
- Cast macro argument to size_t in EVP_ENCODE_LENGTH to prevent
  signed integer overflow
- Change encoded_length in b64_write() from int to size_t and add
  an explicit overflow sanity check before allocation
- Change return type of evp_encodeblock_int() and
  encode_base64_avx2() from int to size_t so that large encoded
  output lengths are not truncated
- Update EVP_EncodeUpdate() to use size_t for the encoder return
  value accumulator (j), consistent with the existing size_t total
- Add explicit (int) casts in EVP_EncodeBlock() and EVP_EncodeFinal()
  where the public API requires int return values

Reviewed-by: Saša Nedvědický <sashan@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.foundation>
MergeDate: Fri Apr  3 14:55:29 2026
(Merged from https://github.com/openssl/openssl/pull/30321)

(cherry picked from commit 3741076a34f4953ccbb6e1e131756ac206c8f27e)

diff --git a/crypto/evp/bio_b64.c b/crypto/evp/bio_b64.c
index 33ddb6334d7e10fcc66b70a8a21163bc8291db06..7f32d7a5d0c50e018b30f81c79346c6a9d256246 100644 (file)
--- a/crypto/evp/bio_b64.c
+++ b/crypto/evp/bio_b64.c
@@ -334,7 +334,7 @@ static int b64_write(BIO *b, const char *in, int inl)
     int i;
     BIO_B64_CTX *ctx;
     BIO *next;
-    int encoded_length;
+    size_t encoded_length;
     unsigned char *encoded;
     int n_bytes_enc;
 
@@ -393,7 +393,12 @@ static int b64_write(BIO *b, const char *in, int inl)
 
     encoded_length = EVP_ENCODE_LENGTH(inl);
 
-    if (ctx->encoded_buf == NULL || (size_t)encoded_length > ctx->encoded_buf_len) {
+    if (encoded_length > SIZE_MAX / 2) {
+        ERR_raise(ERR_LIB_BIO, BIO_R_LENGTH_TOO_LONG);
+        return -1;
+    }
+
+    if (ctx->encoded_buf == NULL || encoded_length > ctx->encoded_buf_len) {
         OPENSSL_free(ctx->encoded_buf);
         ctx->encoded_buf = OPENSSL_malloc(encoded_length);
         if (ctx->encoded_buf == NULL) {

diff --git a/crypto/evp/enc_b64_avx2.c b/crypto/evp/enc_b64_avx2.c
index e6f571dd8e3a795bab0660eb4152c9a713fb2dba..acf35739b5cdcc4a91bc0a563e90c3da38d37218 100644 (file)
--- a/crypto/evp/enc_b64_avx2.c
+++ b/crypto/evp/enc_b64_avx2.c
@@ -477,7 +477,7 @@ static inline size_t insert_nl_str8(const __m256i v0, uint8_t *output)
 OPENSSL_UNTARGET_AVX2
 
 OPENSSL_TARGET_AVX2
-int encode_base64_avx2(EVP_ENCODE_CTX *ctx, unsigned char *dst,
+size_t encode_base64_avx2(EVP_ENCODE_CTX *ctx, unsigned char *dst,
     const unsigned char *src, int srclen, int ctx_length,
     int *final_wrap_cnt)
 {
@@ -665,7 +665,7 @@ int encode_base64_avx2(EVP_ENCODE_CTX *ctx, unsigned char *dst,
         *out++ = '\n';
     }
 
-    return (int)(out - (uint8_t *)dst) + +evp_encodeblock_int(ctx, out, src + i, srclen - i, final_wrap_cnt);
+    return (size_t)(out - (uint8_t *)dst) + evp_encodeblock_int(ctx, out, src + i, srclen - i, final_wrap_cnt);
 }
 OPENSSL_UNTARGET_AVX2
 #endif /* !defined(_M_ARM64EC) */

diff --git a/crypto/evp/enc_b64_avx2.h b/crypto/evp/enc_b64_avx2.h
index db67b5cf8ad065ddc1962c432f435e14892cf3d7..89dba6285d54ab5ab72216148533bf5ac7bcee18 100644 (file)
--- a/crypto/evp/enc_b64_avx2.h
+++ b/crypto/evp/enc_b64_avx2.h
@@ -2,10 +2,11 @@
 #define OSSL_CRYPTO_EVP_B64_AVX2_H
 
 #include <openssl/evp.h>
+#include <stddef.h>
 
 #if defined(__x86_64) || defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64)
 #if !defined(_M_ARM64EC)
-int encode_base64_avx2(EVP_ENCODE_CTX *ctx,
+size_t encode_base64_avx2(EVP_ENCODE_CTX *ctx,
     unsigned char *out, const unsigned char *src, int srclen,
     int newlines, int *wrap_cnt);
 #endif /* !defined(_M_ARM64EC) */

diff --git a/crypto/evp/enc_b64_scalar.c b/crypto/evp/enc_b64_scalar.c
index 1ca46a8357e8532b69a9aa3c230b5d834e2748ab..4bdadcea0cae65a0fe73e54dcc800c74c360cdd5 100644 (file)
--- a/crypto/evp/enc_b64_scalar.c
+++ b/crypto/evp/enc_b64_scalar.c
@@ -124,11 +124,11 @@ static const unsigned char base64_std_bin2ascii_2[256] = {
     '/'
 };
 
-int evp_encodeblock_int(EVP_ENCODE_CTX *ctx, unsigned char *t,
+size_t evp_encodeblock_int(EVP_ENCODE_CTX *ctx, unsigned char *t,
     const unsigned char *f, int dlen, int *wrap_cnt)
 {
     int i = 0;
-    int ret = 0;
+    size_t ret = 0;
     uint8_t t1, t2, t3;
     const unsigned char *e0, *e1, *e2;
     int srp = (ctx != NULL

diff --git a/crypto/evp/enc_b64_scalar.h b/crypto/evp/enc_b64_scalar.h
index 91d416f7586ca66b0229846812c97e635617712c..43059938ccf66eac0bbbcf0c181d2d5ef9ddbb4b 100644 (file)
--- a/crypto/evp/enc_b64_scalar.h
+++ b/crypto/evp/enc_b64_scalar.h
@@ -1,8 +1,9 @@
 #ifndef OSSL_CRYPTO_EVP_B64_SCALAR_H
 #define OSSL_CRYPTO_EVP_B64_SCALAR_H
 #include <openssl/evp.h>
+#include <stddef.h>
 
-int evp_encodeblock_int(EVP_ENCODE_CTX *ctx, unsigned char *t,
+size_t evp_encodeblock_int(EVP_ENCODE_CTX *ctx, unsigned char *t,
     const unsigned char *f, int dlen, int *wrap_cnt);
 
 #endif

diff --git a/crypto/evp/encode.c b/crypto/evp/encode.c
index eacc68bb96551cc0934bff981813cd53f8e2ccf9..faf9a3887d556400f79535c15fe4a9a7700b83fc 100644 (file)
--- a/crypto/evp/encode.c
+++ b/crypto/evp/encode.c
@@ -26,7 +26,7 @@
 
 static unsigned char conv_ascii2bin(unsigned char a,
     const unsigned char *table);
-int evp_encodeblock_int(EVP_ENCODE_CTX *ctx, unsigned char *t,
+size_t evp_encodeblock_int(EVP_ENCODE_CTX *ctx, unsigned char *t,
     const unsigned char *f, int dlen, int *wrap_cnt);
 static int evp_decodeblock_int(EVP_ENCODE_CTX *ctx, unsigned char *t,
     const unsigned char *f, int n, int eof);
@@ -374,7 +374,8 @@ void EVP_EncodeInit(EVP_ENCODE_CTX *ctx)
 int EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,
     const unsigned char *in, int inl)
 {
-    int i, j;
+    int i;
+    size_t j;
     size_t total = 0;
 
     *outl = 0;
@@ -452,20 +453,20 @@ int EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,
 
 void EVP_EncodeFinal(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl)
 {
-    int ret = 0;
+    size_t ret = 0;
     int wrap_cnt = 0;
 
     if (ctx->num != 0) {
         ret = evp_encodeblock_int(ctx, out, ctx->enc_data, ctx->num,
             &wrap_cnt);
-        if (ossl_assert(ret >= 0)) {
+        if (ret > 0) {
             if ((ctx->flags & EVP_ENCODE_CTX_NO_NEWLINES) == 0)
                 out[ret++] = '\n';
             out[ret] = '\0';
             ctx->num = 0;
         }
     }
-    *outl = ret;
+    *outl = (int)ret;
 }
 
 int EVP_EncodeBlock(unsigned char *t, const unsigned char *f, int dlen)
@@ -473,14 +474,14 @@ int EVP_EncodeBlock(unsigned char *t, const unsigned char *f, int dlen)
     int wrap_cnt = 0;
 
 #if defined(__AVX2__)
-    return encode_base64_avx2(NULL, t, f, dlen, 0, &wrap_cnt);
+    return (int)encode_base64_avx2(NULL, t, f, dlen, 0, &wrap_cnt);
 #elif defined(HAS_IA32CAP_IS_64)
     if ((OPENSSL_ia32cap_P[2] & (1u << 5)) != 0)
-        return encode_base64_avx2(NULL, t, f, dlen, 0, &wrap_cnt);
+        return (int)encode_base64_avx2(NULL, t, f, dlen, 0, &wrap_cnt);
     else
-        return evp_encodeblock_int(NULL, t, f, dlen, &wrap_cnt);
+        return (int)evp_encodeblock_int(NULL, t, f, dlen, &wrap_cnt);
 #else
-    return evp_encodeblock_int(NULL, t, f, dlen, &wrap_cnt);
+    return (int)evp_encodeblock_int(NULL, t, f, dlen, &wrap_cnt);
 #endif
 }
 

diff --git a/include/openssl/evp.h b/include/openssl/evp.h
index 464a9f1efa50843bfcc57a29ba847d0ec38302c9..39a6204545d313ff7cdfa49c07ac02a1962971be 100644 (file)
--- a/include/openssl/evp.h
+++ b/include/openssl/evp.h
@@ -570,7 +570,7 @@ void *EVP_CIPHER_CTX_set_cipher_data(EVP_CIPHER_CTX *ctx, void *cipher_data);
 #define EVP_CIPHER_CTX_get_mode(c) EVP_CIPHER_get_mode(EVP_CIPHER_CTX_get0_cipher(c))
 #define EVP_CIPHER_CTX_mode EVP_CIPHER_CTX_get_mode
 
-#define EVP_ENCODE_LENGTH(l) ((((l) + 2) / 3 * 4) + ((l) / 48 + 1) * 2 + 80)
+#define EVP_ENCODE_LENGTH(l) (((((size_t)(l)) + 2) / 3 * 4) + (((size_t)(l)) / 48 + 1) * 2 + 80)
 #define EVP_DECODE_LENGTH(l) (((l) + 3) / 4 * 3 + 80)
 
 #define EVP_SignInit_ex(a, b, c) EVP_DigestInit_ex(a, b, c)