utf8: let's update utf16_to_utf8() a bit

Let's change utf16_to_utf8() prototype to refer to utf16 chars with char16_t rather than void

Let's not cast away a "const" needlessly.

Let's add a few comments.

Let's fix the calculations of the buffer size to allocate, and how long
to run the loop in case of uneven byte numbers

diff --git a/src/basic/utf8.c b/src/basic/utf8.c
index f6ab26d826414107c9bc333c85679fa6e20abb45..7dc84c1157ec06829764fefee20ae5e24818f6b2 100644 (file)
--- a/src/basic/utf8.c
+++ b/src/basic/utf8.c
@@ -314,18 +314,25 @@ size_t utf8_encode_unichar(char *out_utf8, char32_t g) {
         return 0;
 }
 
-char *utf16_to_utf8(const void *s, size_t length) {
+char *utf16_to_utf8(const char16_t *s, size_t length /* bytes! */) {
         const uint8_t *f;
         char *r, *t;
 
-        r = new(char, (length * 4 + 1) / 2 + 1);
+        assert(s);
+
+        /* Input length is in bytes, i.e. the shortest possible character takes 2 bytes. Each unicode character may
+         * take up to 4 bytes in UTF-8. Let's also account for a trailing NUL byte. */
+        if (length * 2 < length)
+                return NULL; /* overflow */
+
+        r = new(char, length * 2 + 1);
         if (!r)
                 return NULL;
 
-        f = s;
+        f = (const uint8_t*) s;
         t = r;
 
-        while (f < (const uint8_t*) s + length) {
+        while (f + 1 < (const uint8_t*) s + length) {
                 char16_t w1, w2;
 
                 /* see RFC 2781 section 2.2 */
@@ -335,13 +342,13 @@ char *utf16_to_utf8(const void *s, size_t length) {
 
                 if (!utf16_is_surrogate(w1)) {
                         t += utf8_encode_unichar(t, w1);
-
                         continue;
                 }
 
                 if (utf16_is_trailing_surrogate(w1))
-                        continue;
-                else if (f >= (const uint8_t*) s + length)
+                        continue; /* spurious trailing surrogate, ignore */
+
+                if (f + 1 >= (const uint8_t*) s + length)
                         break;
 
                 w2 = f[1] << 8 | f[0];
@@ -349,7 +356,7 @@ char *utf16_to_utf8(const void *s, size_t length) {
 
                 if (!utf16_is_trailing_surrogate(w2)) {
                         f -= 2;
-                        continue;
+                        continue; /* surrogate missing its trailing surrogate, ignore */
                 }
 
                 t += utf8_encode_unichar(t, utf16_surrogate_pair_to_unichar(w1, w2));

diff --git a/src/basic/utf8.h b/src/basic/utf8.h
index 63991bf8c89fcbcee9d37b42900371e6df8b06fe..13c48b0978cbe6947e0763f67d6f06626f84a473 100644 (file)
--- a/src/basic/utf8.h
+++ b/src/basic/utf8.h
@@ -25,7 +25,7 @@ char *utf8_escape_invalid(const char *s);
 char *utf8_escape_non_printable(const char *str);
 
 size_t utf8_encode_unichar(char *out_utf8, char32_t g);
-char *utf16_to_utf8(const void *s, size_t length);
+char *utf16_to_utf8(const char16_t *s, size_t length /* bytes! */);
 
 int utf8_encoded_valid_unichar(const char *str);
 int utf8_encoded_to_unichar(const char *str, char32_t *ret_unichar);