archive_string: Check values before casts

The size_t to int conversion is especially required on Windows systems
to support their int-based functions. These variables should be properly
checked before casts. This avoids integer truncations with large
strings.

I prefer size_t over int for sizes and adjusted variables to size_t
where possible to avoid casts.

Signed-off-by: Tobias Stoeckmann <tobias@stoeckmann.org>

diff --git a/libarchive/archive_string.c b/libarchive/archive_string.c
index a996334d641fc69b6650d494c6b6643753962ba6..7437715f91226f758b0ead507115adcd9cf6f8f0 100644 (file)
--- a/libarchive/archive_string.c
+++ b/libarchive/archive_string.c
@@ -472,7 +472,8 @@ static int
 archive_wstring_append_from_mbs_in_codepage(struct archive_wstring *dest,
     const char *s, size_t length, struct archive_string_conv *sc)
 {
-       int count, ret = 0;
+       int ret = 0;
+       size_t count;
        UINT from_cp;
 
        if (sc != NULL)
@@ -494,7 +495,7 @@ archive_wstring_append_from_mbs_in_codepage(struct archive_wstring *dest,
                ws = dest->s + dest->length;
                mp = (const unsigned char *)s;
                count = 0;
-               while (count < (int)length && *mp) {
+               while (count < length && *mp) {
                        *ws++ = (wchar_t)*mp++;
                        count++;
                }
@@ -517,13 +518,13 @@ archive_wstring_append_from_mbs_in_codepage(struct archive_wstring *dest,
                         *  UTF-16BE/LE NFD ===> UTF-16 NFC
                         *  UTF-16BE/LE NFC ===> UTF-16 NFD
                         */
-                       count = (int)utf16nbytes(s, length);
+                       count = utf16nbytes(s, length);
                } else {
                        /*
                         *  UTF-8 NFD ===> UTF-16 NFC
                         *  UTF-8 NFC ===> UTF-16 NFD
                         */
-                       count = (int)mbsnbytes(s, length);
+                       count = mbsnbytes(s, length);
                }
                u16.s = (char *)dest->s;
                u16.length = dest->length << 1;
@@ -538,7 +539,7 @@ archive_wstring_append_from_mbs_in_codepage(struct archive_wstring *dest,
                sc->flag = saved_flag;/* restore the saved flag. */
                return (ret);
        } else if (sc != NULL && (sc->flag & SCONV_FROM_UTF16)) {
-               count = (int)utf16nbytes(s, length);
+               count = utf16nbytes(s, length);
                count >>= 1; /* to be WCS length */
                /* Allocate memory for WCS. */
                if (NULL == archive_wstring_ensure(dest,
@@ -547,14 +548,14 @@ archive_wstring_append_from_mbs_in_codepage(struct archive_wstring *dest,
                wmemcpy(dest->s + dest->length, (const wchar_t *)s, count);
                if ((sc->flag & SCONV_FROM_UTF16BE) && !IS_BIG_ENDIAN) {
                        uint16_t *u16 = (uint16_t *)(dest->s + dest->length);
-                       int b;
+                       size_t b;
                        for (b = 0; b < count; b++) {
                                uint16_t val = archive_le16dec(u16+b);
                                archive_be16enc(u16+b, val);
                        }
                } else if ((sc->flag & SCONV_FROM_UTF16LE) && IS_BIG_ENDIAN) {
                        uint16_t *u16 = (uint16_t *)(dest->s + dest->length);
-                       int b;
+                       size_t b;
                        for (b = 0; b < count; b++) {
                                uint16_t val = archive_be16dec(u16+b);
                                archive_le16enc(u16+b, val);
@@ -578,21 +579,28 @@ archive_wstring_append_from_mbs_in_codepage(struct archive_wstring *dest,
 
                buffsize = dest->length + length + 1;
                do {
+                       int r;
+
+                       /* MultiByteToWideChar is limited to int. */
+                       if (length > (size_t)INT_MAX ||
+                               (dest->buffer_length >> 1) > (size_t)INT_MAX)
+                               return (-1);
                        /* Allocate memory for WCS. */
                        if (NULL == archive_wstring_ensure(dest, buffsize))
                                return (-1);
                        /* Convert MBS to WCS. */
-                       count = MultiByteToWideChar(from_cp,
+                       r = MultiByteToWideChar(from_cp,
                            mbflag, s, (int)length, dest->s + dest->length,
                            (int)(dest->buffer_length >> 1) -1);
-                       if (count == 0 &&
+                       if (r == 0 &&
                            GetLastError() == ERROR_INSUFFICIENT_BUFFER) {
                                /* Expand the WCS buffer. */
                                buffsize = dest->buffer_length << 1;
                                continue;
                        }
-                       if (count == 0 && length != 0)
+                       if (r == 0 && length != 0)
                                ret = -1;
+                       count = (size_t)r;
                        break;
                } while (1);
        }
@@ -701,9 +709,9 @@ archive_string_append_from_wcs_in_codepage(struct archive_string *as,
     const wchar_t *ws, size_t len, struct archive_string_conv *sc)
 {
        BOOL defchar_used, *dp;
-       int count, ret = 0;
+       int ret = 0;
        UINT to_cp;
-       int wslen = (int)len;
+       size_t count, wslen = len;
 
        if (sc != NULL)
                to_cp = sc->to_cp;
@@ -742,13 +750,13 @@ archive_string_append_from_wcs_in_codepage(struct archive_string *as,
                count = 0;
                defchar_used = 0;
                if (sc->flag & SCONV_TO_UTF16BE) {
-                       while (count < (int)len && *ws) {
+                       while (count < len && *ws) {
                                archive_be16enc(u16+count, *ws);
                                ws++;
                                count++;
                        }
                } else {
-                       while (count < (int)len && *ws) {
+                       while (count < len && *ws) {
                                archive_le16enc(u16+count, *ws);
                                ws++;
                                count++;
@@ -761,15 +769,21 @@ archive_string_append_from_wcs_in_codepage(struct archive_string *as,
                    archive_string_ensure(as, as->length + len * 2 + 1))
                        return (-1);
                do {
+                       int r;
+
                        defchar_used = 0;
                        if (to_cp == CP_UTF8 || sc == NULL)
                                dp = NULL;
                        else
                                dp = &defchar_used;
-                       count = WideCharToMultiByte(to_cp, 0, ws, wslen,
+                       /* WideCharToMultiByte is limited to int. */
+                       if (as->buffer_length - as->length - 1 > (size_t)INT_MAX ||
+                               wslen > (size_t)INT_MAX)
+                               return (-1);
+                       r = WideCharToMultiByte(to_cp, 0, ws, (int)wslen,
                            as->s + as->length,
-                           (int)as->buffer_length - (int)as->length - 1, NULL, dp);
-                       if (count == 0 &&
+                           (int)(as->buffer_length - as->length - 1), NULL, dp);
+                       if (r == 0 &&
                            GetLastError() == ERROR_INSUFFICIENT_BUFFER) {
                                /* Expand the MBS buffer and retry. */
                                if (NULL == archive_string_ensure(as,
@@ -777,8 +791,9 @@ archive_string_append_from_wcs_in_codepage(struct archive_string *as,
                                        return (-1);
                                continue;
                        }
-                       if (count == 0)
+                       if (r == 0)
                                ret = -1;
+                       count = (size_t)r;
                        break;
                } while (1);
        }
@@ -2054,7 +2069,7 @@ iconv_strncat_in_locale(struct archive_string *as, const void *_p,
        char *outp;
        size_t avail, bs;
        int return_value = 0; /* success */
-       int to_size, from_size;
+       size_t to_size, from_size;
 
        if (sc->flag & SCONV_TO_UTF16)
                to_size = 2;
@@ -2073,7 +2088,7 @@ iconv_strncat_in_locale(struct archive_string *as, const void *_p,
        remaining = length;
        outp = as->s + as->length;
        avail = as->buffer_length - as->length - to_size;
-       while (remaining >= (size_t)from_size) {
+       while (remaining >= from_size) {
                size_t result = iconv(cd, &itp, &remaining, &outp, &avail);
 
                if (result != (size_t)-1)
@@ -2196,6 +2211,8 @@ invalid_mbs(const void *_p, size_t n, struct archive_string_conv *sc)
        if (codepage != CP_UTF8)
                mbflag |= MB_PRECOMPOSED;
 
+       if (n > (size_t)INT_MAX)
+               return (-1); /* Invalid */
        if (MultiByteToWideChar(codepage, mbflag, p, (int)n, NULL, 0) == 0)
                return (-1); /* Invalid */
        return (0); /* Okay */
@@ -2349,7 +2366,7 @@ _utf8_to_unicode(uint32_t *pwc, const char *s, size_t n)
        cnt = utf8_count[ch];
 
        /* Invalid sequence or there are not plenty bytes. */
-       if ((int)n < cnt) {
+       if (n < (size_t)cnt) {
                cnt = (int)n;
                for (i = 1; i < cnt; i++) {
                        if ((s[i] & 0xc0) != 0x80) {
@@ -2418,7 +2435,7 @@ _utf8_to_unicode(uint32_t *pwc, const char *s, size_t n)
                        cnt = 6;
                else
                        cnt = 1;
-               if ((int)n < cnt)
+               if (n < (size_t)cnt)
                        cnt = (int)n;
                for (i = 1; i < cnt; i++) {
                        if ((s[i] & 0xc0) != 0x80) {
@@ -3521,10 +3538,9 @@ win_strncat_from_utf16(struct archive_string *as, const void *_p, size_t bytes,
 {
        struct archive_string tmp;
        const char *u16;
-       int ll;
        BOOL defchar;
        char *mbs;
-       size_t mbs_size, b;
+       size_t mbs_size, b, ll;
        int ret = 0;
 
        bytes &= ~1;
@@ -3588,18 +3604,24 @@ win_strncat_from_utf16(struct archive_string *as, const void *_p, size_t bytes,
        }
 
        do {
+               int r;
                defchar = 0;
-               ll = WideCharToMultiByte(sc->to_cp, 0,
+               /* WideCharToMultiByte is limited to int. */
+               if (bytes > (size_t)INT_MAX || mbs_size > (size_t)INT_MAX)
+                       return (-1);
+               r = WideCharToMultiByte(sc->to_cp, 0,
                    (LPCWSTR)u16, (int)bytes>>1, mbs, (int)mbs_size,
                        NULL, &defchar);
                /* Exit loop if we succeeded */
-               if (ll != 0 ||
+               if (r != 0 ||
                    GetLastError() != ERROR_INSUFFICIENT_BUFFER) {
+                       ll = (size_t)r;
                        break;
                }
                /* Else expand buffer and loop to try again. */
-               ll = WideCharToMultiByte(sc->to_cp, 0,
+               r = WideCharToMultiByte(sc->to_cp, 0,
                    (LPCWSTR)u16, (int)bytes, NULL, 0, NULL, NULL);
+               ll = (size_t)r;
                if (archive_string_ensure(as, ll +1) == NULL)
                        return (-1);
                mbs = as->s + as->length;
@@ -3665,16 +3687,21 @@ win_strncat_to_utf16(struct archive_string *as16, const void *_p,
                return (0);
        }
        do {
-               count = MultiByteToWideChar(sc->from_cp,
+               int r;
+               if (length > (size_t)INT_MAX || (avail >> 1) > (size_t)INT_MAX)
+                       return (-1);
+               r = MultiByteToWideChar(sc->from_cp,
                    MB_PRECOMPOSED, s, (int)length, (LPWSTR)u16, (int)avail>>1);
                /* Exit loop if we succeeded */
-               if (count != 0 ||
+               if (r != 0 ||
                    GetLastError() != ERROR_INSUFFICIENT_BUFFER) {
+                       count = (size_t)r;
                        break;
                }
                /* Expand buffer and try again */
-               count = MultiByteToWideChar(sc->from_cp,
+               r = MultiByteToWideChar(sc->from_cp,
                    MB_PRECOMPOSED, s, (int)length, NULL, 0);
+               count = (size_t)r;
                if (archive_string_ensure(as16, (count +1) * 2)
                    == NULL)
                        return (-1);