unicode_case.c: defend against truncated UTF8.

Reviewed-by: Chao Li <li.evan.chao@gmail.com>
Discussion: https://postgr.es/m/c355354e6c3f4a7aafb047361b73db247260fca0.camel@j-davis.com
Backpatch-through: 17

diff --git a/src/backend/utils/adt/pg_locale_builtin.c b/src/backend/utils/adt/pg_locale_builtin.c
index df5612b5a2d472e7b42f71106a10b76a64281ba9..faf71d129d7b7eb635a8d7ac602c36b379182f4c 100644 (file)
--- a/src/backend/utils/adt/pg_locale_builtin.c
+++ b/src/backend/utils/adt/pg_locale_builtin.c
@@ -57,21 +57,33 @@ initcap_wbnext(void *state)
        while (wbstate->offset < wbstate->len &&
                   wbstate->str[wbstate->offset] != '\0')
        {
-               pg_wchar        u = utf8_to_unicode((unsigned char *) wbstate->str +
+               int                     ulen = pg_utf_mblen((const unsigned char *) wbstate->str +
                                                                                wbstate->offset);
-               bool            curr_alnum = pg_u_isalnum(u, wbstate->posix);
+               pg_wchar        u;
+               bool            curr_alnum;
+               size_t          prev_offset = wbstate->offset;
 
-               if (!wbstate->init || curr_alnum != wbstate->prev_alnum)
+               /* invalid UTF8 */
+               if (wbstate->offset + ulen > wbstate->len)
                {
-                       size_t          prev_offset = wbstate->offset;
+                       wbstate->init = true;
+                       wbstate->offset = wbstate->len;
+                       return prev_offset;
+               }
 
+               u = utf8_to_unicode((const unsigned char *) wbstate->str +
+                                                       wbstate->offset);
+               curr_alnum = pg_u_isalnum(u, wbstate->posix);
+
+               if (!wbstate->init || curr_alnum != wbstate->prev_alnum)
+               {
                        wbstate->init = true;
-                       wbstate->offset += unicode_utf8len(u);
+                       wbstate->offset += ulen;
                        wbstate->prev_alnum = curr_alnum;
                        return prev_offset;
                }
 
-               wbstate->offset += unicode_utf8len(u);
+               wbstate->offset += ulen;
        }
 
        return wbstate->len;

diff --git a/src/common/unicode/case_test.c b/src/common/unicode/case_test.c
index 6dfc4d130eac27ca19cd08b7b0a59d69297d087c..d98384e6141abad28ec527a27189e87dc0c747fe 100644 (file)
--- a/src/common/unicode/case_test.c
+++ b/src/common/unicode/case_test.c
@@ -330,6 +330,8 @@ tfunc_fold(char *dst, size_t dstsize, const char *src,
 static void
 test_convert_case()
 {
+       size_t          needed;
+
        /* test string with no case changes */
        test_convert(tfunc_lower, "√∞", "√∞");
        /* test adjust-to-cased behavior */
@@ -354,6 +356,12 @@ test_convert_case()
        /* U+FF11 FULLWIDTH ONE is alphanumeric for full case mapping */
        test_convert(tfunc_title, "\uFF11a", "\uFF11a");
 
+       /* invalid UTF8: truncated multibyte sequence */
+       needed = unicode_strfold(NULL, 0, "abc\xCE", 4, false);
+       Assert(needed == 3);
+       /* invalid UTF8: invalid byte */
+       needed = unicode_strfold(NULL, 0, "abc\xF8xyz", 7, false);
+       Assert(needed == 3);
 
 #ifdef USE_ICU
        icu_test_full("");

diff --git a/src/common/unicode_case.c b/src/common/unicode_case.c
index 073faf6a0d58b9c5c36e7ab1cfe471f4f42e15f2..448a4f9d08a186e7fea3dc00d1d40ee6e84d6989 100644 (file)
--- a/src/common/unicode_case.c
+++ b/src/common/unicode_case.c
@@ -193,6 +193,22 @@ unicode_strfold(char *dst, size_t dstsize, const char *src, ssize_t srclen,
                                                NULL);
 }
 
+/* local version of pg_utf_mblen() to be inlinable */
+static int
+utf8_mblen(const unsigned char *s)
+{
+       if ((*s & 0x80) == 0)
+               return 1;
+       else if ((*s & 0xe0) == 0xc0)
+               return 2;
+       else if ((*s & 0xf0) == 0xe0)
+               return 3;
+       else if ((*s & 0xf8) == 0xf0)
+               return 4;
+       else
+               return -1;
+}
+
 /*
  * Implement Unicode Default Case Conversion algorithm.
  *
@@ -229,14 +245,21 @@ convert_case(char *dst, size_t dstsize, const char *src, ssize_t srclen,
                Assert(boundary == 0);  /* start of text is always a boundary */
        }
 
-       while ((srclen < 0 || srcoff < srclen) && src[srcoff] != '\0')
+       srclen = (srclen < 0) ? strlen(src) : srclen;
+       while (srcoff < srclen && src[srcoff] != '\0')
        {
-               pg_wchar        u1 = utf8_to_unicode((unsigned char *) src + srcoff);
-               int                     u1len = unicode_utf8len(u1);
+               int                     u1len = utf8_mblen((const unsigned char *) src + srcoff);
+               pg_wchar        u1;
                pg_wchar        simple = 0;
                const pg_wchar *special = NULL;
                enum CaseMapResult casemap_result;
 
+               /* invalid UTF8 */
+               if (u1len < 0 || srcoff + u1len > srclen)
+                       break;
+
+               u1 = utf8_to_unicode((const unsigned char *) src + srcoff);
+
                if (str_casekind == CaseTitle)
                {
                        if (srcoff == boundary)
@@ -320,7 +343,14 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset)
        {
                if ((str[i] & 0x80) == 0 || (str[i] & 0xC0) == 0xC0)
                {
-                       pg_wchar        curr = utf8_to_unicode(str + i);
+                       int                     u1len = utf8_mblen((const unsigned char *) str + i);
+                       pg_wchar        curr;
+
+                       /* invalid UTF8 */
+                       if (u1len < 0 || i + u1len > len)
+                               return false;
+
+                       curr = utf8_to_unicode(str + i);
 
                        if (pg_u_prop_case_ignorable(curr))
                                continue;
@@ -331,8 +361,8 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset)
                }
                else if ((str[i] & 0xC0) == 0x80)
                        continue;
-
-               Assert(false);                  /* invalid UTF-8 */
+               else
+                       return false;                   /* invalid UTF8 */
        }
 
        /* end of string is not followed by a Cased character */
@@ -344,7 +374,14 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset)
        {
                if ((str[i] & 0x80) == 0 || (str[i] & 0xC0) == 0xC0)
                {
-                       pg_wchar        curr = utf8_to_unicode(str + i);
+                       int                     u1len = utf8_mblen((const unsigned char *) str + i);
+                       pg_wchar        curr;
+
+                       /* invalid UTF8 */
+                       if (u1len < 0 || i + u1len > len)
+                               return false;
+
+                       curr = utf8_to_unicode(str + i);
 
                        if (pg_u_prop_case_ignorable(curr))
                                continue;
@@ -355,8 +392,8 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset)
                }
                else if ((str[i] & 0xC0) == 0x80)
                        continue;
-
-               Assert(false);                  /* invalid UTF-8 */
+               else
+                       return false;                   /* invalid UTF8 */
        }
 
        return true;