do_print_ex(): Avoid possible integer overflow

Fixes Coverity 1604657
Fixes openssl/project#780

Reviewed-by: Neil Horman <nhorman@openssl.org>
Reviewed-by: Tom Cosgrove <tom.cosgrove@arm.com>
Reviewed-by: Paul Dale <ppzgs1@gmail.com>
(Merged from https://github.com/openssl/openssl/pull/25084)

(cherry picked from commit e3e15e77f14cc4026fd456cc8a2b5190b2d79610)

diff --git a/crypto/asn1/a_strex.c b/crypto/asn1/a_strex.c
index b31761aae6f52c1f9e75acea646b24a0f71942bd..f61612737a90f8b50ee85379a0aab6b709017a2e 100644 (file)
--- a/crypto/asn1/a_strex.c
+++ b/crypto/asn1/a_strex.c
@@ -10,6 +10,7 @@
 #include <stdio.h>
 #include <string.h>
 #include "internal/cryptlib.h"
+#include "internal/sizes.h"
 #include "crypto/asn1.h"
 #include <openssl/crypto.h>
 #include <openssl/x509.h>
@@ -345,8 +346,10 @@ static int do_print_ex(char_io *io_ch, void *arg, unsigned long lflags,
 
     if (lflags & ASN1_STRFLGS_SHOW_TYPE) {
         const char *tagname;
+
         tagname = ASN1_tag2str(type);
-        outlen += strlen(tagname);
+        /* We can directly cast here as tagname will never be too large. */
+        outlen += (int)strlen(tagname);
         if (!io_ch(arg, tagname, outlen) || !io_ch(arg, ":", 1))
             return -1;
         outlen++;
@@ -372,7 +375,7 @@ static int do_print_ex(char_io *io_ch, void *arg, unsigned long lflags,
 
     if (type == -1) {
         len = do_dump(lflags, io_ch, arg, str);
-        if (len < 0)
+        if (len < 0 || len > INT_MAX - outlen)
             return -1;
         outlen += len;
         return outlen;
@@ -391,7 +394,7 @@ static int do_print_ex(char_io *io_ch, void *arg, unsigned long lflags,
     }
 
     len = do_buf(str->data, str->length, type, flags, &quotes, io_ch, NULL);
-    if (len < 0)
+    if (len < 0 || len > INT_MAX - 2 - outlen)
         return -1;
     outlen += len;
     if (quotes)