{
/* Avoid too long inputs */
static const size_t max_fasttext_input_len = 1024 * 1024;
+ /* Longer "words" are garbage that only inflates subword ngrams */
+ static const size_t max_fasttext_word_len = 256;
+ /* Bound the total number of tokens fed into the model */
+ static const size_t max_fasttext_tokens = 128 * 1024;
auto *real_model = FASTTEXT_MODEL_TO_C_API(ud);
std::vector<std::int32_t> words_vec;
}
auto words_count = kv_size(*utf_words);
- words_vec.reserve(words_count);
+ words_vec.reserve(std::min(words_count, max_fasttext_tokens));
- for (auto i = 0; i < std::min(words_count, max_fasttext_input_len); i++) {
+ for (size_t i = 0; i < std::min(words_count, max_fasttext_input_len); i++) {
const auto *w = &kv_A(*utf_words, i);
- if (w->original.len > 0) {
+ if (w->original.len > 0 && w->original.len <= max_fasttext_word_len) {
real_model->word2vec(w->original.begin, w->original.len, words_vec);
}
+
+ if (words_vec.size() >= max_fasttext_tokens) {
+ msg_debug_lang_det("fasttext: tokens limit of %z is reached after %z words",
+ max_fasttext_tokens, i);
+ break;
+ }
}
msg_debug_lang_det("fasttext: got %z word tokens from %z words", words_vec.size(), words_count);
return FALSE;
}
+/* Total children visited when linking all alternative text parts of a task */
+static const unsigned int max_alt_search_iterations = 1000000;
+
/*
* Recursively search for a text part in a subtree.
* - `root` is the multipart to search in
* - `exclude` is the subtree to skip (the branch containing the original part)
* - `want_html` TRUE means we want HTML part, FALSE means we want plain text
+ * - `budget` bounds the total number of children visited across all searches
* - Returns the first matching text part, or NULL
* - Stops recursion if we hit a message/rfc822 part
*/
static struct rspamd_mime_text_part *
rspamd_mime_part_find_text_in_subtree(struct rspamd_mime_part *root,
struct rspamd_mime_part *exclude,
- gboolean want_html)
+ gboolean want_html,
+ unsigned int *budget)
{
if (!root || !IS_PART_MULTIPART(root) || !root->specific.mp) {
return NULL;
for (unsigned int i = 0; i < mp->children->len; i++) {
struct rspamd_mime_part *child = g_ptr_array_index(mp->children, i);
+ if (*budget == 0) {
+ return NULL;
+ }
+ (*budget)--;
+
/* Skip the excluded subtree */
if (exclude && rspamd_mime_part_is_ancestor(exclude, child)) {
continue;
/* Recurse into multiparts */
if (IS_PART_MULTIPART(child)) {
struct rspamd_mime_text_part *found =
- rspamd_mime_part_find_text_in_subtree(child, NULL, want_html);
+ rspamd_mime_part_find_text_in_subtree(child, NULL, want_html, budget);
if (found) {
return found;
}
* 4. Stop if we hit message/rfc822 (embedded message)
*/
static struct rspamd_mime_text_part *
-rspamd_mime_text_part_find_alternative(struct rspamd_mime_text_part *text_part)
+rspamd_mime_text_part_find_alternative(struct rspamd_mime_text_part *text_part,
+ unsigned int *budget)
{
if (!text_part || !text_part->mime_part) {
return NULL;
}
/* Search other branches for a text part of the opposite type */
- return rspamd_mime_part_find_text_in_subtree(alt_parent, our_branch, !is_html);
+ return rspamd_mime_part_find_text_in_subtree(alt_parent, our_branch, !is_html,
+ budget);
}
static enum rspamd_message_part_is_text_result
}
/* Compute alternative text parts for each text part */
+ unsigned int alt_budget = max_alt_search_iterations;
+
PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, text_parts), i, text_part)
{
- text_part->alt_text_part = rspamd_mime_text_part_find_alternative(text_part);
+ if (alt_budget == 0) {
+ msg_warn_task("alternative text parts linking limit of %ud iterations "
+ "is reached; %ud of %ud text parts are not linked",
+ max_alt_search_iterations,
+ MESSAGE_FIELD(task, text_parts)->len - i,
+ MESSAGE_FIELD(task, text_parts)->len);
+ break;
+ }
+
+ text_part->alt_text_part = rspamd_mime_text_part_find_alternative(text_part,
+ &alt_budget);
}
/* Calculate distance for alternative parts */
void word2vec(std::string_view word, std::vector<std::int32_t> &ngrams) const
{
+ /* No real dictionary word is that long; bound OOV subword expansion */
+ static constexpr std::size_t max_oov_word_len = 1024;
auto wid = dict.find(word);
if (wid >= 0) {
}
else {
/* OOV: compute subwords on the fly */
- if (args.maxn > 0) {
+ if (args.maxn > 0 && word.size() <= max_oov_word_len) {
std::string wrapped = "<" + std::string(word) + ">";
dict.compute_subwords(wrapped, ngrams);
}
task:destroy()
end)
+ test("Alternative text part linking is bounded", function()
+ -- One multipart/alternative with many HTML branches and a single plain
+ -- branch at the end: every HTML part scans almost all siblings to find
+ -- the plain one, so unbounded linking is quadratic in the part count.
+ local nparts = 9000
+ local body_parts = {}
+
+ for i = 1, nparts do
+ body_parts[i] = '--ALT\nContent-Type: text/html\n\n<b>x</b>\n'
+ end
+
+ local msg = table.concat {
+ hdrs,
+ 'Content-Type: multipart/alternative; boundary=ALT\n',
+ '\n',
+ table.concat(body_parts),
+ '--ALT\n',
+ 'Content-Type: text/plain\n',
+ '\n',
+ 'plain text\n',
+ '--ALT--\n',
+ }
+ local res, task = rspamd_task.load_from_string(msg, rspamd_config)
+ assert_true(res, "failed to load message")
+ task:process_message()
+ local tp = task:get_text_parts()
+ assert_equal(#tp, nparts + 1,
+ string.format("expected %d text parts, got %d", nparts + 1, #tp))
+
+ local linked = 0
+ for _, p in ipairs(tp) do
+ if p:get_alt_part() then
+ linked = linked + 1
+ end
+ end
+ -- Linking works while the budget lasts, then it is cut off; without the
+ -- bound every HTML part would be linked at a quadratic cost
+ assert_true(linked >= 1, "no text part has an alternative linked")
+ assert_true(linked < nparts / 2,
+ string.format("alternative linking was not bounded: %d parts linked", linked))
+ task:destroy()
+ end)
+
test("MIME header count is bounded", function()
local msg = string.rep('X:\n', 100001) .. '\nbody\n'
local res, task = rspamd_task.load_from_string(msg, rspamd_config)
projects / thirdparty / rspamd.git / commitdiff
