i2d_name_canon(): Check overflow in len accumulation

Fixes Coverity 1604638

Reviewed-by: Neil Horman <nhorman@openssl.org>
Reviewed-by: Paul Dale <ppzgs1@gmail.com>
Reviewed-by: Todd Short <todd.short@me.com>
(Merged from https://github.com/openssl/openssl/pull/24930)

(cherry picked from commit b2deefb9d262f0f9eae6964006df98c2fa24daac)

diff --git a/crypto/x509/x_name.c b/crypto/x509/x_name.c
index 4568833f81b94876216681c4ba5d182a6f95c859..79c166ca6a92116438615f852b63fc2733ccfef9 100644 (file)
--- a/crypto/x509/x_name.c
+++ b/crypto/x509/x_name.c
@@ -480,8 +480,8 @@ static int i2d_name_canon(const STACK_OF(STACK_OF_X509_NAME_ENTRY) * _intname,
         v = sk_ASN1_VALUE_value(intname, i);
         ltmp = ASN1_item_ex_i2d(&v, in,
                                 ASN1_ITEM_rptr(X509_NAME_ENTRIES), -1, -1);
-        if (ltmp < 0)
-            return ltmp;
+        if (ltmp < 0 || len > INT_MAX - ltmp)
+            return -1;
         len += ltmp;
     }
     return len;