From: Vsevolod Stakhov Date: Wed, 22 Jul 2026 21:38:08 +0000 (+0100) Subject: [Fix] message: bound per-part newline metadata X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=HEAD;p=thirdparty%2Frspamd.git [Fix] message: bound per-part newline metadata Newline normalization recorded every newline as a pointer array entry plus a mempool exception object and a GList node, then sorted the whole exceptions list: a newline-dense body turned a few MiB of input into hundreds of MiB of heap and millions of allocations. Cap the recorded newline positions at 100k per part (text normalization and line counters are unaffected), set a part flag on truncation and expose it as newlines_truncated in textpart:get_stats(). --- diff --git a/src/libmime/message.c b/src/libmime/message.c index 0c5c807350..7722ffc22f 100644 --- a/src/libmime/message.c +++ b/src/libmime/message.c @@ -231,6 +231,21 @@ rspamd_mime_part_detect_language(struct rspamd_task *task, } } +/* Bound per-part newline metadata; the text itself is still normalized fully */ +static const unsigned int max_part_newlines = 100000; + +static inline void +rspamd_text_part_maybe_add_newline(struct rspamd_mime_text_part *part) +{ + if (G_LIKELY(part->newlines->len < max_part_newlines)) { + g_ptr_array_add(part->newlines, + (((gpointer) (goffset) (part->utf_stripped_content->len)))); + } + else { + part->flags |= RSPAMD_MIME_TEXT_PART_FLAG_NEWLINES_TRUNCATED; + } +} + static void rspamd_strip_newlines_parse(struct rspamd_task *task, const char *begin, const char *pe, @@ -310,8 +325,7 @@ rspamd_strip_newlines_parse(struct rspamd_task *task, g_byte_array_append(part->utf_stripped_content, (const uint8_t *) " ", 1); crlf_added = TRUE; - g_ptr_array_add(part->newlines, - (((gpointer) (goffset) (part->utf_stripped_content->len)))); + rspamd_text_part_maybe_add_newline(part); } part->nlines++; @@ -344,8 +358,7 @@ rspamd_strip_newlines_parse(struct rspamd_task *task, if (IS_TEXT_PART_HTML(part) || !url_open_bracket) { g_byte_array_append(part->utf_stripped_content, (const uint8_t *) " ", 1); - g_ptr_array_add(part->newlines, - (((gpointer) (goffset) (part->utf_stripped_content->len)))); + rspamd_text_part_maybe_add_newline(part); crlf_added = TRUE; } else { @@ -362,8 +375,7 @@ rspamd_strip_newlines_parse(struct rspamd_task *task, crlf_added = TRUE; } - g_ptr_array_add(part->newlines, - (((gpointer) (goffset) (part->utf_stripped_content->len)))); + rspamd_text_part_maybe_add_newline(part); } c = p + 1; @@ -376,8 +388,7 @@ rspamd_strip_newlines_parse(struct rspamd_task *task, g_byte_array_append(part->utf_stripped_content, (const uint8_t *) " ", 1); crlf_added = TRUE; - g_ptr_array_add(part->newlines, - (((gpointer) (goffset) (part->utf_stripped_content->len)))); + rspamd_text_part_maybe_add_newline(part); } part->nlines++; @@ -430,8 +441,7 @@ rspamd_strip_newlines_parse(struct rspamd_task *task, part->nlines++; if (!crlf_added) { - g_ptr_array_add(part->newlines, - (((gpointer) (goffset) (part->utf_stripped_content->len)))); + rspamd_text_part_maybe_add_newline(part); } /* Skip initial spaces */ @@ -498,8 +508,7 @@ rspamd_strip_newlines_parse(struct rspamd_task *task, if (!crlf_added) { g_byte_array_append(part->utf_stripped_content, (const uint8_t *) " ", 1); - g_ptr_array_add(part->newlines, - (((gpointer) (goffset) (part->utf_stripped_content->len)))); + rspamd_text_part_maybe_add_newline(part); } part->nlines++; @@ -537,6 +546,12 @@ rspamd_normalize_text_part(struct rspamd_task *task, rspamd_strip_newlines_parse(task, p, end, part); + if (part->flags & RSPAMD_MIME_TEXT_PART_FLAG_NEWLINES_TRUNCATED) { + msg_warn_task("too many newlines in text part; only %ud newline " + "positions are recorded", + max_part_newlines); + } + for (i = 0; i < part->newlines->len; i++) { ex = rspamd_mempool_alloc(task->task_pool, sizeof(*ex)); off = (goffset) g_ptr_array_index(part->newlines, i); diff --git a/src/libmime/message.h b/src/libmime/message.h index b69c809f23..34cf99c97d 100644 --- a/src/libmime/message.h +++ b/src/libmime/message.h @@ -121,6 +121,7 @@ struct rspamd_mime_part { #define RSPAMD_MIME_TEXT_PART_FLAG_HTML (1 << 2) #define RSPAMD_MIME_TEXT_PART_FLAG_8BIT_RAW (1 << 3) #define RSPAMD_MIME_TEXT_PART_FLAG_8BIT_ENCODED (1 << 4) +#define RSPAMD_MIME_TEXT_PART_FLAG_NEWLINES_TRUNCATED (1 << 5) #define RSPAMD_MIME_TEXT_PART_ATTACHMENT (1 << 5) #define IS_TEXT_PART_EMPTY(part) ((part)->flags & RSPAMD_MIME_TEXT_PART_FLAG_EMPTY) @@ -200,6 +201,8 @@ struct rspamd_message { gboolean headers_limit_reached; unsigned int narchive_files; /**< total archive file metadata entries */ gboolean archive_files_limit_reached; + unsigned int ntext_newlines; /**< total tracked text newline positions */ + gboolean text_newlines_limit_reached; struct rspamd_task *task; GPtrArray *rcpt_mime; GPtrArray *from_mime; diff --git a/src/lua/lua_mimepart.c b/src/lua/lua_mimepart.c index 0dec35f891..54dc722088 100644 --- a/src/lua/lua_mimepart.c +++ b/src/lua/lua_mimepart.c @@ -125,6 +125,7 @@ LUA_FUNCTION_DEF(textpart, get_lines_count); * - `empty_lines`: number of empty lines * - `non_ascii_characters`: number of non ascii characters * - `ascii_characters`: number of ascii characters + * - `newlines_truncated`: whether newline metadata was capped * @return {table} table of stats */ LUA_FUNCTION_DEF(textpart, get_stats); @@ -1648,13 +1649,13 @@ lua_textpart_get_alt_part(lua_State *L) /*** * @method mime_part:get_stats() * Returns a table with the following data: - * - * - `lines`: number of lines * - `spaces`: number of spaces * - `double_spaces`: double spaces * - `empty_lines`: number of empty lines * - `non_ascii_characters`: number of non ascii characters * - `ascii_characters`: number of ascii characters + * - `newlines_truncated`: whether newline metadata was capped * @return {table} table of stats */ static int @@ -1664,7 +1665,7 @@ lua_textpart_get_stats(lua_State *L) struct rspamd_mime_text_part *part = lua_check_textpart(L); if (part != NULL) { - lua_createtable(L, 0, 9); + lua_createtable(L, 0, 10); lua_pushstring(L, "lines"); lua_pushinteger(L, part->nlines); @@ -1693,6 +1694,9 @@ lua_textpart_get_stats(lua_State *L) lua_pushstring(L, "numeric_characters"); lua_pushinteger(L, part->numeric_characters); lua_settable(L, -3); + lua_pushstring(L, "newlines_truncated"); + lua_pushboolean(L, part->flags & RSPAMD_MIME_TEXT_PART_FLAG_NEWLINES_TRUNCATED); + lua_settable(L, -3); } else { return luaL_error(L, "invalid arguments"); diff --git a/test/lua/unit/task.lua b/test/lua/unit/task.lua index 60679dcade..9d623b943a 100644 --- a/test/lua/unit/task.lua +++ b/test/lua/unit/task.lua @@ -323,6 +323,25 @@ Thank you, task:destroy() end) + test("Text newline metadata is bounded", function() + local msg = table.concat { + hdrs, + "Content-Type: text/plain; charset=utf-8\n", + "\n", + string.rep("x\n", 100001), + } + local res, task = rspamd_task.load_from_string(msg, rspamd_config) + assert_true(res, "failed to load message") + task:process_message() + + local text_parts = task:get_text_parts() + assert_equal(1, #text_parts) + local stats = text_parts[1]:get_stats() + assert_true(stats.newlines_truncated, + "newline metadata limit was not reported") + task:destroy() + end) + test("MIME header count is bounded", function() local msg = string.rep('X:\n', 100001) .. '\nbody\n' local res, task = rspamd_task.load_from_string(msg, rspamd_config)