i2d_name_canon(): Check overflow in len accumulation

Fixes Coverity 1604638

Reviewed-by: Neil Horman <nhorman@openssl.org>
Reviewed-by: Paul Dale <ppzgs1@gmail.com>
Reviewed-by: Todd Short <todd.short@me.com>
(Merged from https://github.com/openssl/openssl/pull/24930)

(cherry picked from commit b2deefb9d262f0f9eae6964006df98c2fa24daac)
(cherry picked from commit dd744cd19b3ff2bdc320c8a77b5c32ff543eaeb3)
(cherry picked from commit a3bfc4fd5b5641b05d6611073146627cf9114436)

diff --git a/crypto/x509/x_name.c b/crypto/x509/x_name.c
index 944eb9992486d39552dfc7374354fe82f95d6097..5d3a4f92004075edfb5131836d9fd04473442bf8 100644 (file)
--- a/crypto/x509/x_name.c
+++ b/crypto/x509/x_name.c
@@ -476,8 +476,8 @@ static int i2d_name_canon(const STACK_OF(STACK_OF_X509_NAME_ENTRY) * _intname,
         v = sk_ASN1_VALUE_value(intname, i);
         ltmp = ASN1_item_ex_i2d(&v, in,
                                 ASN1_ITEM_rptr(X509_NAME_ENTRIES), -1, -1);
-        if (ltmp < 0)
-            return ltmp;
+        if (ltmp < 0 || len > INT_MAX - ltmp)
+            return -1;
         len += ltmp;
     }
     return len;