lib: json-parser - Insert properly escaped unicode

diff --git a/src/lib/json-parser.c b/src/lib/json-parser.c
index 2dc3eacd98f19390b28c5cafcd2aa4a2176e9fb4..09964741ce91cf4302b6fc55df9cb9fc7a6a0332 100644 (file)
--- a/src/lib/json-parser.c
+++ b/src/lib/json-parser.c
@@ -755,7 +755,7 @@ static void json_append_escaped_char(string_t *dest, unsigned char src)
                str_append(dest, "\\\\");
                break;
        default:
-               if (src < 32)
+               if (src < 0x20 || src >= 0x80)
                        str_printfa(dest, "\\u%04x", src);
                else
                        str_append_c(dest, src);
@@ -763,6 +763,19 @@ static void json_append_escaped_char(string_t *dest, unsigned char src)
        }
 }
 
+void json_append_escaped_ucs4(string_t *dest, unichar_t chr)
+{
+       unichar_t high,low;
+       if (chr < 0x80)
+               json_append_escaped_char(dest, (unsigned char)chr);
+       else if (chr >= UTF16_SURROGATE_BASE) {
+               uni_split_surrogate(chr, &high, &low);
+               str_printfa(dest, "\\u%04x\\u%04x", high, low);
+       } else {
+               str_printfa(dest, "\\u%04x", chr);
+       }
+}
+
 void ostream_escaped_json_format(string_t *dest, unsigned char src)
 {
        json_append_escaped_char(dest, src);
@@ -770,14 +783,24 @@ void ostream_escaped_json_format(string_t *dest, unsigned char src)
 
 void json_append_escaped(string_t *dest, const char *src)
 {
-       for (; *src != '\0'; src++)
-               json_append_escaped_char(dest, *src);
+       json_append_escaped_data(dest, (const unsigned char*)src, strlen(src));
 }
 
 void json_append_escaped_data(string_t *dest, const unsigned char *src, size_t size)
 {
        size_t i;
-
-       for (i = 0; i < size; i++)
-               json_append_escaped_char(dest, src[i]);
+       int bytes = 0;
+       unichar_t chr;
+
+       for (i = 0; i < size;) {
+               bytes = uni_utf8_get_char_n(src+i, size-i, &chr);
+               /* if it was valid unichar, encode + move forward by bytes */
+               if (bytes > 0) {
+                       json_append_escaped_ucs4(dest, chr);
+                       i += bytes;
+               /* encode as byte data */
+               } else {
+                       json_append_escaped_char(dest, src[i++]);
+               }
+       }
 }

diff --git a/src/lib/json-parser.h b/src/lib/json-parser.h
index 551422dcc3b9dd6f88f89b9a030451fc366797f0..78fb448f7e4ee2710e1d10c1f42d3546d803e94f 100644 (file)
--- a/src/lib/json-parser.h
+++ b/src/lib/json-parser.h
@@ -1,6 +1,8 @@
 #ifndef JSON_PARSER_H
 #define JSON_PARSER_H
 
+#include "unichar.h"
+
 enum json_type {
        /* { key: */
        JSON_TYPE_OBJECT_KEY,
@@ -46,6 +48,8 @@ void json_parse_skip_next(struct json_parser *parser);
 int json_parse_next_stream(struct json_parser *parser,
                           struct istream **input_r);
 
+/* Append UCS4 to already opened JSON string. */
+void json_append_escaped_ucs4(string_t *dest, unichar_t chr);
 /* Append data to already opened JSON string. src should be valid UTF-8 data. */
 void json_append_escaped(string_t *dest, const char *src);
 /* Same as json_append_escaped(), but append non-\0 terminated input. */

diff --git a/src/lib/test-json-parser.c b/src/lib/test-json-parser.c
index 1635bb848b43e52651d39f724f5b9d3842f447f4..eedeb8440a7b919f7d26c0b94d39d58683a9b894 100644 (file)
--- a/src/lib/test-json-parser.c
+++ b/src/lib/test-json-parser.c
@@ -266,20 +266,20 @@ static void test_json_append_escaped(void)
        string_t *str = t_str_new(32);
 
        test_begin("json_append_escaped()");
-       json_append_escaped(str, "\b\f\r\n\t\"\\\001\002-\xC3\xA4");
-       test_assert(strcmp(str_c(str), "\\b\\f\\r\\n\\t\\\"\\\\\\u0001\\u0002-\xC3\xA4") == 0);
+       json_append_escaped(str, "\b\f\r\n\t\"\\\001\002-\xC3\xA4\xf0\x90\x90\xb7");
+       test_assert(strcmp(str_c(str), "\\b\\f\\r\\n\\t\\\"\\\\\\u0001\\u0002-\\u00e4\\ud801\\udc37") == 0);
        test_end();
 }
 
 static void test_json_append_escaped_data(void)
 {
        static const unsigned char test_input[] =
-               "\b\f\r\n\t\"\\\000\001\002-\xC3\xA4";
+               "\b\f\r\n\t\"\\\000\001\002-\xC3\xA4\xf0\x90\x90\xb7";
        string_t *str = t_str_new(32);
 
        test_begin("json_append_escaped()");
        json_append_escaped_data(str, test_input, sizeof(test_input)-1);
-       test_assert(strcmp(str_c(str), "\\b\\f\\r\\n\\t\\\"\\\\\\u0000\\u0001\\u0002-\xC3\xA4") == 0);
+       test_assert(strcmp(str_c(str), "\\b\\f\\r\\n\\t\\\"\\\\\\u0000\\u0001\\u0002-\\u00e4\\ud801\\udc37") == 0);
        test_end();
 }