lib-mail: Change NUL -> 0x80 replacement to use unicode replacement char instead

Using 0x80 produces invalid UTF-8 output, which we should avoid. Some places
in Dovecot code already requires that input is valid UTF-8, so it's also
safer and easier to use the unicode replacement character.

Fixes: Panic: file fts-tokenizer-generic.c: line 210 (fts_tokenizer_generic_simple_next): assertion failed: (char_size > 0)

diff --git a/src/lib-mail/message-part-data.c b/src/lib-mail/message-part-data.c
index ca06462c467378bdb2651603eef6721eb2f7e8c9..08c1f0af76ea316f1dc8225904dc2e387addca76 100644 (file)
--- a/src/lib-mail/message-part-data.c
+++ b/src/lib-mail/message-part-data.c
@@ -173,18 +173,23 @@ envelope_get_field(const char *name)
 static const char *
 hdr_strdup(pool_t pool, const unsigned char *data, size_t size)
 {
-       char *dest = p_malloc(pool, size+1);
-
        if (memchr(data, '\0', size) == NULL) {
                /* fast path */
+               char *dest = p_malloc(pool, size+1);
                memcpy(dest, data, size);
-       } else {
-               /* slow path - this could be made faster, but it should be
-                  rare so keep it simple */
-               for (size_t i = 0; i < size; i++)
-                       dest[i] = data[i] == '\0' ? 0x80 : data[i];
+               return dest;
+       }
+
+       /* slow path - this could be made faster, but it should be
+          rare so keep it simple */
+       string_t *str = str_new(pool, size+2);
+       for (size_t i = 0; i < size; i++) {
+               if (data[i] != '\0')
+                       str_append_c(str, data[i]);
+               else
+                       str_append(str, UNICODE_REPLACEMENT_CHAR_UTF8);
        }
-       return dest;
+       return str_c(str);
 }
 
 void message_part_envelope_parse_from_header(pool_t pool,

diff --git a/src/lib-mail/rfc2231-parser.h b/src/lib-mail/rfc2231-parser.h
index 0db536fd4e0b3406bc84f1c030c10f86449c301e..95cadc2e6d9dbfb1e2ba0a8d3e95b8bceea52f2f 100644 (file)
--- a/src/lib-mail/rfc2231-parser.h
+++ b/src/lib-mail/rfc2231-parser.h
@@ -3,9 +3,9 @@
 
 /* Parse all content parameters using rfc822_parse_content_param() and return
    them as a NULL-terminated [key, value] array. RFC 2231-style continuations
-   are merged to a single key. NULs are converted into 0x80. Returns -1 if some
-   of the input was invalid (but valid key/value pairs are still returned), 0
-   if everything looked ok. */
+   are merged to a single key. NULs are converted into unicode replacement
+   character (U+FFFD). Returns -1 if some of the input was invalid (but valid
+   key/value pairs are still returned), 0 if everything looked ok. */
 int ATTR_NOWARN_UNUSED_RESULT
 rfc2231_parse(struct rfc822_parser_context *ctx,
              const char *const **result_r);

diff --git a/src/lib-mail/rfc822-parser.h b/src/lib-mail/rfc822-parser.h
index fa41a6e21bec9cb4c1a5a1a6d4590a9400ae44ae..c001f761f0aef2b522323b92ab43aa231ebf75bf 100644 (file)
--- a/src/lib-mail/rfc822-parser.h
+++ b/src/lib-mail/rfc822-parser.h
@@ -1,8 +1,10 @@
 #ifndef RFC822_PARSER_H
 #define RFC822_PARSER_H
 
+#include "unichar.h"
+
 /* This can be used as a common NUL replacement character */
-#define RFC822_NUL_REPLACEMENT_STR "\x80"
+#define RFC822_NUL_REPLACEMENT_STR UNICODE_REPLACEMENT_CHAR_UTF8
 
 struct rfc822_parser_context {
        const unsigned char *data, *end;

diff --git a/src/lib-mail/test-message-address.c b/src/lib-mail/test-message-address.c
index 90baac5e343e65247f2fab06b7c0cceff6418012..0f5cae2a4d8770f6a4b20e1cba528ad2a3d555b6 100644 (file)
--- a/src/lib-mail/test-message-address.c
+++ b/src/lib-mail/test-message-address.c
@@ -324,8 +324,8 @@ static void test_message_address_nuls(void)
        const unsigned char input[] =
                "\"user\0nuls\"@[domain\0nuls] (comment\0nuls)";
        const struct message_address output = {
-               NULL, "comment\x80nuls", NULL, "user\x80nuls",
-               "[domain\x80nuls]", FALSE
+               NULL, "comment\xEF\xBF\xBDnuls", NULL, "user\xEF\xBF\xBDnuls",
+               "[domain\xEF\xBF\xBDnuls]", FALSE
        };
        const struct message_address *addr;
 

diff --git a/src/lib-mail/test-rfc2231-parser.c b/src/lib-mail/test-rfc2231-parser.c
index 3c7e1b61f79be0b6521e8478ed7ab83007e9dbc4..f54910adff1bc3ef93814d238558d0857d9e9b7b 100644 (file)
--- a/src/lib-mail/test-rfc2231-parser.c
+++ b/src/lib-mail/test-rfc2231-parser.c
@@ -18,7 +18,7 @@ static void test_rfc2231_parser(void)
                "; key*1=baz";
        const char *output[] = {
                "key",
-               "f\x80oobazba%",
+               "f\xEF\xBF\xBDoobazba%",
                "key2*",
                "''ab%25",
                "key3*",