unicode_case.c: defend against truncated UTF8.

Reviewed-by: Chao Li <li.evan.chao@gmail.com>
Discussion: https://postgr.es/m/c355354e6c3f4a7aafb047361b73db247260fca0.camel@j-davis.com
Backpatch-through: 17

diff --git a/src/backend/utils/adt/pg_locale_builtin.c b/src/backend/utils/adt/pg_locale_builtin.c
index 01d4f55b07e5e06e6434058b0c87170834ab596b..7c36fd5091b821c97fa356a4a82df1b07a454d83 100644 (file)
--- a/src/backend/utils/adt/pg_locale_builtin.c
+++ b/src/backend/utils/adt/pg_locale_builtin.c
@@ -62,21 +62,33 @@ initcap_wbnext(void *state)
 
        while (wbstate->offset < wbstate->len)
        {
-               char32_t        u = utf8_to_unicode((const unsigned char *) wbstate->str +
+               int                     ulen = pg_utf_mblen((const unsigned char *) wbstate->str +
                                                                                wbstate->offset);
-               bool            curr_alnum = pg_u_isalnum(u, wbstate->posix);
+               char32_t        u;
+               bool            curr_alnum;
+               size_t          prev_offset = wbstate->offset;
 
-               if (!wbstate->init || curr_alnum != wbstate->prev_alnum)
+               /* invalid UTF8 */
+               if (wbstate->offset + ulen > wbstate->len)
                {
-                       size_t          prev_offset = wbstate->offset;
+                       wbstate->init = true;
+                       wbstate->offset = wbstate->len;
+                       return prev_offset;
+               }
 
+               u = utf8_to_unicode((const unsigned char *) wbstate->str +
+                                                       wbstate->offset);
+               curr_alnum = pg_u_isalnum(u, wbstate->posix);
+
+               if (!wbstate->init || curr_alnum != wbstate->prev_alnum)
+               {
                        wbstate->init = true;
-                       wbstate->offset += unicode_utf8len(u);
+                       wbstate->offset += ulen;
                        wbstate->prev_alnum = curr_alnum;
                        return prev_offset;
                }
 
-               wbstate->offset += unicode_utf8len(u);
+               wbstate->offset += ulen;
        }
 
        return wbstate->len;

diff --git a/src/common/unicode/case_test.c b/src/common/unicode/case_test.c
index a0dbf00b671fdbf1fff0ee34d492f3dad2b4f821..31ea94513bfe37eed4eebfa0aba577eefe3c3bac 100644 (file)
--- a/src/common/unicode/case_test.c
+++ b/src/common/unicode/case_test.c
@@ -296,6 +296,8 @@ tfunc_fold(char *dst, size_t dstsize, const char *src,
 static void
 test_convert_case(void)
 {
+       size_t          needed;
+
        /* test string with no case changes */
        test_convert(tfunc_lower, "√∞", "√∞");
        /* test adjust-to-cased behavior */
@@ -320,6 +322,12 @@ test_convert_case(void)
        /* U+FF11 FULLWIDTH ONE is alphanumeric for full case mapping */
        test_convert(tfunc_title, "\uFF11a", "\uFF11a");
 
+       /* invalid UTF8: truncated multibyte sequence */
+       needed = unicode_strfold(NULL, 0, "abc\xCE", 4, false);
+       Assert(needed == 3);
+       /* invalid UTF8: invalid byte */
+       needed = unicode_strfold(NULL, 0, "abc\xF8xyz", 7, false);
+       Assert(needed == 3);
 
 #ifdef USE_ICU
        icu_test_full("");

diff --git a/src/common/unicode_case.c b/src/common/unicode_case.c
index d6ee00b7d9c7d8367c6034068c520a1e3335c9f2..42eb7d22211ca933129ad664f61e3da02f251274 100644 (file)
--- a/src/common/unicode_case.c
+++ b/src/common/unicode_case.c
@@ -189,6 +189,22 @@ unicode_strfold(char *dst, size_t dstsize, const char *src, size_t srclen,
                                                NULL);
 }
 
+/* local version of pg_utf_mblen() to be inlinable */
+static int
+utf8_mblen(const unsigned char *s)
+{
+       if ((*s & 0x80) == 0)
+               return 1;
+       else if ((*s & 0xe0) == 0xc0)
+               return 2;
+       else if ((*s & 0xf0) == 0xe0)
+               return 3;
+       else if ((*s & 0xf8) == 0xf0)
+               return 4;
+       else
+               return -1;
+}
+
 /*
  * Implement Unicode Default Case Conversion algorithm.
  *
@@ -227,12 +243,18 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
 
        while (srcoff < srclen)
        {
-               char32_t        u1 = utf8_to_unicode((const unsigned char *) src + srcoff);
-               int                     u1len = unicode_utf8len(u1);
+               int                     u1len = utf8_mblen((const unsigned char *) src + srcoff);
+               char32_t        u1;
                char32_t        simple = 0;
                const char32_t *special = NULL;
                enum CaseMapResult casemap_result;
 
+               /* invalid UTF8 */
+               if (u1len < 0 || srcoff + u1len > srclen)
+                       break;
+
+               u1 = utf8_to_unicode((const unsigned char *) src + srcoff);
+
                if (str_casekind == CaseTitle)
                {
                        if (srcoff == boundary)
@@ -316,7 +338,14 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset)
        {
                if ((str[i] & 0x80) == 0 || (str[i] & 0xC0) == 0xC0)
                {
-                       char32_t        curr = utf8_to_unicode(str + i);
+                       int                     u1len = utf8_mblen((const unsigned char *) str + i);
+                       char32_t        curr;
+
+                       /* invalid UTF8 */
+                       if (u1len < 0 || i + u1len > len)
+                               return false;
+
+                       curr = utf8_to_unicode(str + i);
 
                        if (pg_u_prop_case_ignorable(curr))
                                continue;
@@ -327,8 +356,8 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset)
                }
                else if ((str[i] & 0xC0) == 0x80)
                        continue;
-
-               Assert(false);                  /* invalid UTF-8 */
+               else
+                       return false;                   /* invalid UTF8 */
        }
 
        /* end of string is not followed by a Cased character */
@@ -340,7 +369,14 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset)
        {
                if ((str[i] & 0x80) == 0 || (str[i] & 0xC0) == 0xC0)
                {
-                       char32_t        curr = utf8_to_unicode(str + i);
+                       int                     u1len = utf8_mblen((const unsigned char *) str + i);
+                       char32_t        curr;
+
+                       /* invalid UTF8 */
+                       if (u1len < 0 || i + u1len > len)
+                               return false;
+
+                       curr = utf8_to_unicode(str + i);
 
                        if (pg_u_prop_case_ignorable(curr))
                                continue;
@@ -351,8 +387,8 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset)
                }
                else if ((str[i] & 0xC0) == 0x80)
                        continue;
-
-               Assert(false);                  /* invalid UTF-8 */
+               else
+                       return false;                   /* invalid UTF8 */
        }
 
        return true;