From: Vsevolod Stakhov Date: Wed, 22 Jul 2026 13:37:46 +0000 (+0100) Subject: [Fix] Bound alt-part linking and fasttext langdet cost X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=285d201df9e4141ca0fda8ff155e0329fc20e419;p=thirdparty%2Frspamd.git [Fix] Bound alt-part linking and fasttext langdet cost Alternative text part linking used one subtree scan per text part, so a crafted multipart/alternative with thousands of sibling text parts made linking quadratic (~100M sibling visits at the MIME part cap). All searches of a task now share a single visit budget; when it is exhausted the remaining parts are left unlinked with a warning. Fasttext language detection fed up to 1M words into the model with no bound on word length or on the resulting token count. As the UTF tokenizer intentionally does not enforce max_word_len, a crafted body made of huge unspaced words expanded into 3-4 subword ngram ids per character: hundreds of megabytes of heap and matching vector-add work in predict() from one message. Skip overlong words, cap the total tokens fed to the model, and refuse OOV subword expansion in the shim for words longer than any real dictionary entry. --- diff --git a/src/libmime/lang_detection_fasttext.cxx b/src/libmime/lang_detection_fasttext.cxx index e05526b608..9dd353d6cd 100644 --- a/src/libmime/lang_detection_fasttext.cxx +++ b/src/libmime/lang_detection_fasttext.cxx @@ -335,6 +335,10 @@ rspamd_fasttext_predict_result_t rspamd_lang_detection_fasttext_detect(void *ud, { /* Avoid too long inputs */ static const size_t max_fasttext_input_len = 1024 * 1024; + /* Longer "words" are garbage that only inflates subword ngrams */ + static const size_t max_fasttext_word_len = 256; + /* Bound the total number of tokens fed into the model */ + static const size_t max_fasttext_tokens = 128 * 1024; auto *real_model = FASTTEXT_MODEL_TO_C_API(ud); std::vector words_vec; @@ -343,13 +347,19 @@ rspamd_fasttext_predict_result_t rspamd_lang_detection_fasttext_detect(void *ud, } auto words_count = kv_size(*utf_words); - words_vec.reserve(words_count); + words_vec.reserve(std::min(words_count, max_fasttext_tokens)); - for (auto i = 0; i < std::min(words_count, max_fasttext_input_len); i++) { + for (size_t i = 0; i < std::min(words_count, max_fasttext_input_len); i++) { const auto *w = &kv_A(*utf_words, i); - if (w->original.len > 0) { + if (w->original.len > 0 && w->original.len <= max_fasttext_word_len) { real_model->word2vec(w->original.begin, w->original.len, words_vec); } + + if (words_vec.size() >= max_fasttext_tokens) { + msg_debug_lang_det("fasttext: tokens limit of %z is reached after %z words", + max_fasttext_tokens, i); + break; + } } msg_debug_lang_det("fasttext: got %z word tokens from %z words", words_vec.size(), words_count); diff --git a/src/libmime/message.c b/src/libmime/message.c index 2a493983c6..0c5c807350 100644 --- a/src/libmime/message.c +++ b/src/libmime/message.c @@ -1014,18 +1014,23 @@ rspamd_mime_part_is_ancestor(struct rspamd_mime_part *part, return FALSE; } +/* Total children visited when linking all alternative text parts of a task */ +static const unsigned int max_alt_search_iterations = 1000000; + /* * Recursively search for a text part in a subtree. * - `root` is the multipart to search in * - `exclude` is the subtree to skip (the branch containing the original part) * - `want_html` TRUE means we want HTML part, FALSE means we want plain text + * - `budget` bounds the total number of children visited across all searches * - Returns the first matching text part, or NULL * - Stops recursion if we hit a message/rfc822 part */ static struct rspamd_mime_text_part * rspamd_mime_part_find_text_in_subtree(struct rspamd_mime_part *root, struct rspamd_mime_part *exclude, - gboolean want_html) + gboolean want_html, + unsigned int *budget) { if (!root || !IS_PART_MULTIPART(root) || !root->specific.mp) { return NULL; @@ -1040,6 +1045,11 @@ rspamd_mime_part_find_text_in_subtree(struct rspamd_mime_part *root, for (unsigned int i = 0; i < mp->children->len; i++) { struct rspamd_mime_part *child = g_ptr_array_index(mp->children, i); + if (*budget == 0) { + return NULL; + } + (*budget)--; + /* Skip the excluded subtree */ if (exclude && rspamd_mime_part_is_ancestor(exclude, child)) { continue; @@ -1068,7 +1078,7 @@ rspamd_mime_part_find_text_in_subtree(struct rspamd_mime_part *root, /* Recurse into multiparts */ if (IS_PART_MULTIPART(child)) { struct rspamd_mime_text_part *found = - rspamd_mime_part_find_text_in_subtree(child, NULL, want_html); + rspamd_mime_part_find_text_in_subtree(child, NULL, want_html, budget); if (found) { return found; } @@ -1088,7 +1098,8 @@ rspamd_mime_part_find_text_in_subtree(struct rspamd_mime_part *root, * 4. Stop if we hit message/rfc822 (embedded message) */ static struct rspamd_mime_text_part * -rspamd_mime_text_part_find_alternative(struct rspamd_mime_text_part *text_part) +rspamd_mime_text_part_find_alternative(struct rspamd_mime_text_part *text_part, + unsigned int *budget) { if (!text_part || !text_part->mime_part) { return NULL; @@ -1114,7 +1125,8 @@ rspamd_mime_text_part_find_alternative(struct rspamd_mime_text_part *text_part) } /* Search other branches for a text part of the opposite type */ - return rspamd_mime_part_find_text_in_subtree(alt_parent, our_branch, !is_html); + return rspamd_mime_part_find_text_in_subtree(alt_parent, our_branch, !is_html, + budget); } static enum rspamd_message_part_is_text_result @@ -1977,9 +1989,21 @@ void rspamd_message_process(struct rspamd_task *task) } /* Compute alternative text parts for each text part */ + unsigned int alt_budget = max_alt_search_iterations; + PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, text_parts), i, text_part) { - text_part->alt_text_part = rspamd_mime_text_part_find_alternative(text_part); + if (alt_budget == 0) { + msg_warn_task("alternative text parts linking limit of %ud iterations " + "is reached; %ud of %ud text parts are not linked", + max_alt_search_iterations, + MESSAGE_FIELD(task, text_parts)->len - i, + MESSAGE_FIELD(task, text_parts)->len); + break; + } + + text_part->alt_text_part = rspamd_mime_text_part_find_alternative(text_part, + &alt_budget); } /* Calculate distance for alternative parts */ diff --git a/src/libserver/fasttext/fasttext_shim.cxx b/src/libserver/fasttext/fasttext_shim.cxx index acbaa569ec..834b4bfd50 100644 --- a/src/libserver/fasttext/fasttext_shim.cxx +++ b/src/libserver/fasttext/fasttext_shim.cxx @@ -620,6 +620,8 @@ public: void word2vec(std::string_view word, std::vector &ngrams) const { + /* No real dictionary word is that long; bound OOV subword expansion */ + static constexpr std::size_t max_oov_word_len = 1024; auto wid = dict.find(word); if (wid >= 0) { @@ -636,7 +638,7 @@ public: } else { /* OOV: compute subwords on the fly */ - if (args.maxn > 0) { + if (args.maxn > 0 && word.size() <= max_oov_word_len) { std::string wrapped = "<" + std::string(word) + ">"; dict.compute_subwords(wrapped, ngrams); } diff --git a/test/lua/unit/task.lua b/test/lua/unit/task.lua index 478be0e433..60679dcade 100644 --- a/test/lua/unit/task.lua +++ b/test/lua/unit/task.lua @@ -280,6 +280,49 @@ Thank you, task:destroy() end) + test("Alternative text part linking is bounded", function() + -- One multipart/alternative with many HTML branches and a single plain + -- branch at the end: every HTML part scans almost all siblings to find + -- the plain one, so unbounded linking is quadratic in the part count. + local nparts = 9000 + local body_parts = {} + + for i = 1, nparts do + body_parts[i] = '--ALT\nContent-Type: text/html\n\nx\n' + end + + local msg = table.concat { + hdrs, + 'Content-Type: multipart/alternative; boundary=ALT\n', + '\n', + table.concat(body_parts), + '--ALT\n', + 'Content-Type: text/plain\n', + '\n', + 'plain text\n', + '--ALT--\n', + } + local res, task = rspamd_task.load_from_string(msg, rspamd_config) + assert_true(res, "failed to load message") + task:process_message() + local tp = task:get_text_parts() + assert_equal(#tp, nparts + 1, + string.format("expected %d text parts, got %d", nparts + 1, #tp)) + + local linked = 0 + for _, p in ipairs(tp) do + if p:get_alt_part() then + linked = linked + 1 + end + end + -- Linking works while the budget lasts, then it is cut off; without the + -- bound every HTML part would be linked at a quadratic cost + assert_true(linked >= 1, "no text part has an alternative linked") + assert_true(linked < nparts / 2, + string.format("alternative linking was not bounded: %d parts linked", linked)) + task:destroy() + end) + test("MIME header count is bounded", function() local msg = string.rep('X:\n', 100001) .. '\nbody\n' local res, task = rspamd_task.load_from_string(msg, rspamd_config)