Fix integer overflow in evp_EncryptDecryptUpdate

Fixes #17871.

Reviewed-by: Paul Dale <pauli@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/17872)

diff --git a/crypto/evp/evp_enc.c b/crypto/evp/evp_enc.c
index d835968f253ce24c93e2c5af9d9c911f29dae8da..b8b9d90d3652f8bc809789de950f1fff5ceea496 100644 (file)
--- a/crypto/evp/evp_enc.c
+++ b/crypto/evp/evp_enc.c
@@ -281,7 +281,7 @@ int EVP_DecryptInit_ex(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *cipher,
 # define PTRDIFF_T size_t
 #endif
 
-int is_partially_overlapping(const void *ptr1, const void *ptr2, int len)
+int is_partially_overlapping(const void *ptr1, const void *ptr2, size_t len)
 {
     PTRDIFF_T diff = (PTRDIFF_T)ptr1-(PTRDIFF_T)ptr2;
     /*
@@ -299,7 +299,8 @@ static int evp_EncryptDecryptUpdate(EVP_CIPHER_CTX *ctx,
                                     unsigned char *out, int *outl,
                                     const unsigned char *in, int inl)
 {
-    int i, j, bl, cmpl = inl;
+    int i, j, bl;
+    size_t cmpl = (size_t)inl;
 
     if (EVP_CIPHER_CTX_test_flags(ctx, EVP_CIPH_FLAG_LENGTH_BITS))
         cmpl = (cmpl + 7) / 8;
@@ -464,8 +465,9 @@ int EVP_EncryptFinal_ex(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl)
 int EVP_DecryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl,
                       const unsigned char *in, int inl)
 {
-    int fix_len, cmpl = inl;
+    int fix_len;
     unsigned int b;
+    size_t cmpl = (size_t)inl;
 
     /* Prevent accidental use of encryption context when decrypting */
     if (ctx->encrypt) {

diff --git a/crypto/evp/evp_local.h b/crypto/evp/evp_local.h
index f1589d68289428ab6f80d497cc36bc8b0098e82f..cd3c1cf148fd32fd96071b457b6473a8c64f4611 100644 (file)
--- a/crypto/evp/evp_local.h
+++ b/crypto/evp/evp_local.h
@@ -65,4 +65,4 @@ struct evp_Encode_Ctx_st {
 typedef struct evp_pbe_st EVP_PBE_CTL;
 DEFINE_STACK_OF(EVP_PBE_CTL)
 
-int is_partially_overlapping(const void *ptr1, const void *ptr2, int len);
+int is_partially_overlapping(const void *ptr1, const void *ptr2, size_t len);