[Fix] message: bound per-part newline metadata

Newline normalization recorded every newline as a pointer array entry
plus a mempool exception object and a GList node, then sorted the
whole exceptions list: a newline-dense body turned a few MiB of input
into hundreds of MiB of heap and millions of allocations. Cap the
recorded newline positions at 100k per part (text normalization and
line counters are unaffected), set a part flag on truncation and
expose it as newlines_truncated in textpart:get_stats().

diff --git a/src/libmime/message.c b/src/libmime/message.c
index 0c5c8073508a9543d858365ba1f6025a08677442..7722ffc22fc2b3c8814f489830834e60d7c11a2e 100644 (file)
--- a/src/libmime/message.c
+++ b/src/libmime/message.c
@@ -231,6 +231,21 @@ rspamd_mime_part_detect_language(struct rspamd_task *task,
        }
 }
 
        }
 }
 

+/* Bound per-part newline metadata; the text itself is still normalized fully */
+static const unsigned int max_part_newlines = 100000;
+
+static inline void
+rspamd_text_part_maybe_add_newline(struct rspamd_mime_text_part *part)
+{
+       if (G_LIKELY(part->newlines->len < max_part_newlines)) {
+               g_ptr_array_add(part->newlines,
+                                               (((gpointer) (goffset) (part->utf_stripped_content->len))));
+       }
+       else {
+               part->flags |= RSPAMD_MIME_TEXT_PART_FLAG_NEWLINES_TRUNCATED;
+       }
+}
+

 static void
 rspamd_strip_newlines_parse(struct rspamd_task *task,
                                                        const char *begin, const char *pe,
 static void
 rspamd_strip_newlines_parse(struct rspamd_task *task,
                                                        const char *begin, const char *pe,


@@ -310,8 +325,7 @@ rspamd_strip_newlines_parse(struct rspamd_task *task,
                                        g_byte_array_append(part->utf_stripped_content,
                                                                                (const uint8_t *) " ", 1);
                                        crlf_added = TRUE;
                                        g_byte_array_append(part->utf_stripped_content,
                                                                                (const uint8_t *) " ", 1);
                                        crlf_added = TRUE;

-                                       g_ptr_array_add(part->newlines,
-                                                                       (((gpointer) (goffset) (part->utf_stripped_content->len))));
+                                       rspamd_text_part_maybe_add_newline(part);

                                }
 
                                part->nlines++;
                                }
 
                                part->nlines++;


@@ -344,8 +358,7 @@ rspamd_strip_newlines_parse(struct rspamd_task *task,
                                if (IS_TEXT_PART_HTML(part) || !url_open_bracket) {
                                        g_byte_array_append(part->utf_stripped_content,
                                                                                (const uint8_t *) " ", 1);
                                if (IS_TEXT_PART_HTML(part) || !url_open_bracket) {
                                        g_byte_array_append(part->utf_stripped_content,
                                                                                (const uint8_t *) " ", 1);

-                                       g_ptr_array_add(part->newlines,
-                                                                       (((gpointer) (goffset) (part->utf_stripped_content->len))));
+                                       rspamd_text_part_maybe_add_newline(part);

                                        crlf_added = TRUE;
                                }
                                else {
                                        crlf_added = TRUE;
                                }
                                else {


@@ -362,8 +375,7 @@ rspamd_strip_newlines_parse(struct rspamd_task *task,
                                                crlf_added = TRUE;
                                        }
 
                                                crlf_added = TRUE;
                                        }
 

-                                       g_ptr_array_add(part->newlines,
-                                                                       (((gpointer) (goffset) (part->utf_stripped_content->len))));
+                                       rspamd_text_part_maybe_add_newline(part);

                                }
 
                                c = p + 1;
                                }
 
                                c = p + 1;


@@ -376,8 +388,7 @@ rspamd_strip_newlines_parse(struct rspamd_task *task,
                                        g_byte_array_append(part->utf_stripped_content,
                                                                                (const uint8_t *) " ", 1);
                                        crlf_added = TRUE;
                                        g_byte_array_append(part->utf_stripped_content,
                                                                                (const uint8_t *) " ", 1);
                                        crlf_added = TRUE;

-                                       g_ptr_array_add(part->newlines,
-                                                                       (((gpointer) (goffset) (part->utf_stripped_content->len))));
+                                       rspamd_text_part_maybe_add_newline(part);

                                }
 
                                part->nlines++;
                                }
 
                                part->nlines++;


@@ -430,8 +441,7 @@ rspamd_strip_newlines_parse(struct rspamd_task *task,
                                part->nlines++;
 
                                if (!crlf_added) {
                                part->nlines++;
 
                                if (!crlf_added) {

-                                       g_ptr_array_add(part->newlines,
-                                                                       (((gpointer) (goffset) (part->utf_stripped_content->len))));
+                                       rspamd_text_part_maybe_add_newline(part);

                                }
 
                                /* Skip initial spaces */
                                }
 
                                /* Skip initial spaces */


@@ -498,8 +508,7 @@ rspamd_strip_newlines_parse(struct rspamd_task *task,
                        if (!crlf_added) {
                                g_byte_array_append(part->utf_stripped_content,
                                                                        (const uint8_t *) " ", 1);
                        if (!crlf_added) {
                                g_byte_array_append(part->utf_stripped_content,
                                                                        (const uint8_t *) " ", 1);

-                               g_ptr_array_add(part->newlines,
-                                                               (((gpointer) (goffset) (part->utf_stripped_content->len))));
+                               rspamd_text_part_maybe_add_newline(part);

                        }
 
                        part->nlines++;
                        }
 
                        part->nlines++;


@@ -537,6 +546,12 @@ rspamd_normalize_text_part(struct rspamd_task *task,
 
                rspamd_strip_newlines_parse(task, p, end, part);
 
 
                rspamd_strip_newlines_parse(task, p, end, part);
 

+               if (part->flags & RSPAMD_MIME_TEXT_PART_FLAG_NEWLINES_TRUNCATED) {
+                       msg_warn_task("too many newlines in text part; only %ud newline "
+                                                 "positions are recorded",
+                                                 max_part_newlines);
+               }
+

                for (i = 0; i < part->newlines->len; i++) {
                        ex = rspamd_mempool_alloc(task->task_pool, sizeof(*ex));
                        off = (goffset) g_ptr_array_index(part->newlines, i);
                for (i = 0; i < part->newlines->len; i++) {
                        ex = rspamd_mempool_alloc(task->task_pool, sizeof(*ex));
                        off = (goffset) g_ptr_array_index(part->newlines, i);

diff --git a/src/libmime/message.h b/src/libmime/message.h
index b69c809f23d220c64dc7141a8c5c3d2cbf06b130..34cf99c97d0c3f89a64f3494d502a59c95851d9f 100644 (file)
--- a/src/libmime/message.h
+++ b/src/libmime/message.h
@@ -121,6 +121,7 @@ struct rspamd_mime_part {
 #define RSPAMD_MIME_TEXT_PART_FLAG_HTML (1 << 2)
 #define RSPAMD_MIME_TEXT_PART_FLAG_8BIT_RAW (1 << 3)
 #define RSPAMD_MIME_TEXT_PART_FLAG_8BIT_ENCODED (1 << 4)
 #define RSPAMD_MIME_TEXT_PART_FLAG_HTML (1 << 2)
 #define RSPAMD_MIME_TEXT_PART_FLAG_8BIT_RAW (1 << 3)
 #define RSPAMD_MIME_TEXT_PART_FLAG_8BIT_ENCODED (1 << 4)

+#define RSPAMD_MIME_TEXT_PART_FLAG_NEWLINES_TRUNCATED (1 << 5)

 #define RSPAMD_MIME_TEXT_PART_ATTACHMENT (1 << 5)
 
 #define IS_TEXT_PART_EMPTY(part) ((part)->flags & RSPAMD_MIME_TEXT_PART_FLAG_EMPTY)
 #define RSPAMD_MIME_TEXT_PART_ATTACHMENT (1 << 5)
 
 #define IS_TEXT_PART_EMPTY(part) ((part)->flags & RSPAMD_MIME_TEXT_PART_FLAG_EMPTY)


@@ -200,6 +201,8 @@ struct rspamd_message {
        gboolean headers_limit_reached;
        unsigned int narchive_files; /**< total archive file metadata entries */
        gboolean archive_files_limit_reached;
        gboolean headers_limit_reached;
        unsigned int narchive_files; /**< total archive file metadata entries */
        gboolean archive_files_limit_reached;

+       unsigned int ntext_newlines; /**< total tracked text newline positions */
+       gboolean text_newlines_limit_reached;

        struct rspamd_task *task;
        GPtrArray *rcpt_mime;
        GPtrArray *from_mime;
        struct rspamd_task *task;
        GPtrArray *rcpt_mime;
        GPtrArray *from_mime;

diff --git a/src/lua/lua_mimepart.c b/src/lua/lua_mimepart.c
index 0dec35f891acd555c61b7258f86b0ec90d19b18d..54dc722088a0af605d867419213d84f5314c0a01 100644 (file)
--- a/src/lua/lua_mimepart.c
+++ b/src/lua/lua_mimepart.c
@@ -125,6 +125,7 @@ LUA_FUNCTION_DEF(textpart, get_lines_count);
  * - `empty_lines`: number of empty lines
  * - `non_ascii_characters`: number of non ascii characters
  * - `ascii_characters`: number of ascii characters
  * - `empty_lines`: number of empty lines
  * - `non_ascii_characters`: number of non ascii characters
  * - `ascii_characters`: number of ascii characters

+ * - `newlines_truncated`: whether newline metadata was capped

  * @return {table} table of stats
  */
 LUA_FUNCTION_DEF(textpart, get_stats);
  * @return {table} table of stats
  */
 LUA_FUNCTION_DEF(textpart, get_stats);


@@ -1648,13 +1649,13 @@ lua_textpart_get_alt_part(lua_State *L)
 /***
  * @method mime_part:get_stats()
  * Returns a table with the following data:
 /***
  * @method mime_part:get_stats()
  * Returns a table with the following data:

- * -

  * - `lines`: number of lines
  * - `spaces`: number of spaces
  * - `double_spaces`: double spaces
  * - `empty_lines`: number of empty lines
  * - `non_ascii_characters`: number of non ascii characters
  * - `ascii_characters`: number of ascii characters
  * - `lines`: number of lines
  * - `spaces`: number of spaces
  * - `double_spaces`: double spaces
  * - `empty_lines`: number of empty lines
  * - `non_ascii_characters`: number of non ascii characters
  * - `ascii_characters`: number of ascii characters

+ * - `newlines_truncated`: whether newline metadata was capped

  * @return {table} table of stats
  */
 static int
  * @return {table} table of stats
  */
 static int


@@ -1664,7 +1665,7 @@ lua_textpart_get_stats(lua_State *L)
        struct rspamd_mime_text_part *part = lua_check_textpart(L);
 
        if (part != NULL) {
        struct rspamd_mime_text_part *part = lua_check_textpart(L);
 
        if (part != NULL) {

-               lua_createtable(L, 0, 9);
+               lua_createtable(L, 0, 10);

 
                lua_pushstring(L, "lines");
                lua_pushinteger(L, part->nlines);
 
                lua_pushstring(L, "lines");
                lua_pushinteger(L, part->nlines);


@@ -1693,6 +1694,9 @@ lua_textpart_get_stats(lua_State *L)
                lua_pushstring(L, "numeric_characters");
                lua_pushinteger(L, part->numeric_characters);
                lua_settable(L, -3);
                lua_pushstring(L, "numeric_characters");
                lua_pushinteger(L, part->numeric_characters);
                lua_settable(L, -3);

+               lua_pushstring(L, "newlines_truncated");
+               lua_pushboolean(L, part->flags & RSPAMD_MIME_TEXT_PART_FLAG_NEWLINES_TRUNCATED);
+               lua_settable(L, -3);

        }
        else {
                return luaL_error(L, "invalid arguments");
        }
        else {
                return luaL_error(L, "invalid arguments");

diff --git a/test/lua/unit/task.lua b/test/lua/unit/task.lua
index 60679dcade07ed7a7584918fbc3dd123256f3635..9d623b943a2d8bae5823b72477cc58f1fcb329c5 100644 (file)
--- a/test/lua/unit/task.lua
+++ b/test/lua/unit/task.lua
@@ -323,6 +323,25 @@ Thank you,
     task:destroy()
   end)
 
     task:destroy()
   end)
 

+  test("Text newline metadata is bounded", function()
+    local msg = table.concat {
+      hdrs,
+      "Content-Type: text/plain; charset=utf-8\n",
+      "\n",
+      string.rep("x\n", 100001),
+    }
+    local res, task = rspamd_task.load_from_string(msg, rspamd_config)
+    assert_true(res, "failed to load message")
+    task:process_message()
+
+    local text_parts = task:get_text_parts()
+    assert_equal(1, #text_parts)
+    local stats = text_parts[1]:get_stats()
+    assert_true(stats.newlines_truncated,
+      "newline metadata limit was not reported")
+    task:destroy()
+  end)
+

   test("MIME header count is bounded", function()
     local msg = string.rep('X:\n', 100001) .. '\nbody\n'
     local res, task = rspamd_task.load_from_string(msg, rspamd_config)
   test("MIME header count is bounded", function()
     local msg = string.rep('X:\n', 100001) .. '\nbody\n'
     local res, task = rspamd_task.load_from_string(msg, rspamd_config)