}
}
+/*
+ * Bound the words retained for the whole message: the per part decay merely
+ * samples words within a single part, so a message built of many small parts
+ * can still produce a huge amount of words, and each of them is normalized and
+ * stemmed with several allocations from the task pool afterwards
+ */
+static const uint64_t max_message_words = 100000;
+static const uint64_t max_message_words_bytes = 1024 * 1024;
+
static void
rspamd_mime_part_create_words(struct rspamd_task *task,
struct rspamd_mime_text_part *part)
part->exceptions,
NULL,
&part->utf_words,
+ &MESSAGE_FIELD(task, words_budget),
task->task_pool);
+ if (MESSAGE_FIELD(task, words_budget).exceeded) {
+ part->flags |= RSPAMD_MIME_TEXT_PART_FLAG_WORDS_TRUNCATED;
+
+ if (!MESSAGE_FIELD(task, text_words_limit_reached)) {
+ msg_warn_task("words limit reached: %uL words of %uL bytes are "
+ "tokenized for the whole message, the remaining text "
+ "is not tokenized",
+ MESSAGE_FIELD(task, words_budget).words,
+ MESSAGE_FIELD(task, words_budget).bytes);
+ MESSAGE_FIELD(task, text_words_limit_reached) = TRUE;
+ }
+ }
if (part->utf_words.a) {
part->normalized_hashes = g_array_sized_new(FALSE, FALSE,
msg->parts = g_ptr_array_sized_new(4);
msg->text_parts = g_ptr_array_sized_new(2);
msg->task = task;
+ msg->words_budget.max_words = max_message_words;
+ msg->words_budget.max_bytes = max_message_words_bytes;
REF_INIT_RETAIN(msg, rspamd_message_dtor);
#define RSPAMD_MIME_TEXT_PART_FLAG_8BIT_ENCODED (1 << 4)
#define RSPAMD_MIME_TEXT_PART_ATTACHMENT (1 << 5)
#define RSPAMD_MIME_TEXT_PART_FLAG_NEWLINES_TRUNCATED (1 << 6)
+#define RSPAMD_MIME_TEXT_PART_FLAG_WORDS_TRUNCATED (1 << 7)
#define IS_TEXT_PART_EMPTY(part) ((part)->flags & RSPAMD_MIME_TEXT_PART_FLAG_EMPTY)
#define IS_TEXT_PART_UTF(part) ((part)->flags & RSPAMD_MIME_TEXT_PART_FLAG_UTF)
gboolean archive_files_limit_reached;
unsigned int ntext_newlines; /**< total tracked text newline positions */
gboolean text_newlines_limit_reached;
+ struct rspamd_tokenize_budget words_budget; /**< message wide words budget */
+ gboolean text_words_limit_reached;
struct rspamd_task *task;
GPtrArray *rcpt_mime;
GPtrArray *from_mime;
*/
typedef kvec_t(rspamd_word_t) rspamd_words_t;
+/**
+ * Bounds tokenization for a single message: both the number of the retained
+ * words and the amount of text behind them, as every retained word is
+ * normalized and stemmed afterwards, and that costs several allocations from
+ * the task pool. The same budget is used for all text parts of a message, so
+ * that many small parts cannot bypass the per-part limits.
+ * Zero limits mean 'unlimited'.
+ */
+struct rspamd_tokenize_budget {
+ uint64_t max_words; /**< maximum number of the retained words */
+ uint64_t max_bytes; /**< maximum total length of the retained words */
+ uint64_t words; /**< words retained so far */
+ uint64_t bytes; /**< bytes retained so far */
+ gboolean exceeded; /**< set when one of the limits has been hit */
+};
+
/* Legacy typedefs for backward compatibility */
typedef rspamd_word_t rspamd_stat_token_t;
} \
} while (0)
+/*
+ * Account a token that is about to be retained against the message wide budget.
+ * Returns FALSE when the budget is exhausted, meaning that tokenization must
+ * stop: the token must not be added in that case.
+ */
+static inline gboolean
+rspamd_tokenize_budget_take(struct rspamd_tokenize_budget *budget, gsize len)
+{
+ if (budget == NULL) {
+ return TRUE;
+ }
+
+ if ((budget->max_words > 0 && budget->words >= budget->max_words) ||
+ (budget->max_bytes > 0 && budget->bytes + len > budget->max_bytes)) {
+ budget->exceeded = TRUE;
+
+ return FALSE;
+ }
+
+ budget->words++;
+ budget->bytes += len;
+
+ return TRUE;
+}
+
+/*
+ * Time checks are not free, so they are performed for the inputs that are
+ * likely to be slow: large parts and, regardless of the part size, the inputs
+ * that have already produced a lot of tokens (e.g. many small parts of the
+ * same message)
+ */
+static inline gboolean
+rspamd_tokenize_needs_time_check(gboolean long_text_mode,
+ const struct rspamd_tokenize_budget *budget,
+ gsize nwords)
+{
+ static const gsize long_text_words = 4096;
+
+ if (long_text_mode || nwords >= long_text_words) {
+ return TRUE;
+ }
+
+ return budget != NULL && budget->words >= long_text_words;
+}
+
static inline void
-rspamd_tokenize_exception(struct rspamd_process_exception *ex, rspamd_words_t *res)
+rspamd_tokenize_exception(struct rspamd_process_exception *ex, rspamd_words_t *res,
+ struct rspamd_tokenize_budget *budget)
{
rspamd_word_t token;
token.original.len = sizeof("!!EX!!") - 1;
token.flags = RSPAMD_STAT_TOKEN_FLAG_EXCEPTION;
+ if (!rspamd_tokenize_budget_take(budget, token.original.len)) {
+ return;
+ }
+
kv_push_safe(rspamd_word_t, *res, token, exception_error);
token.flags = 0;
}
}
token.flags = RSPAMD_STAT_TOKEN_FLAG_EXCEPTION;
+
+ if (!rspamd_tokenize_budget_take(budget, token.original.len)) {
+ return;
+ }
+
kv_push_safe(rspamd_word_t, *res, token, exception_error);
token.flags = 0;
}
GList *exceptions,
uint64_t *hash,
rspamd_words_t *output_kvec,
+ struct rspamd_tokenize_budget *budget,
rspamd_mempool_t *pool)
{
rspamd_word_t token, buf;
* In this mode we do additional checks to avoid performance issues
*/
long_text_mode = TRUE;
- start = ev_time();
}
+ /*
+ * Time checks can also be enabled by the number of the produced tokens, so
+ * the start time is always needed
+ */
+ start = ev_time();
+
buf.original.begin = text;
buf.original.len = len;
buf.flags = 0;
/* Copy custom tokenizer results to output kvec */
for (unsigned int i = 0; i < kv_size(*custom_res); i++) {
+ if (!rspamd_tokenize_budget_take(budget,
+ kv_A(*custom_res, i).original.len)) {
+ msg_debug_pool_check(
+ "words budget is exhausted, stop tokenization: "
+ "%z words of %z are copied from the custom tokenizer",
+ (gsize) i, kv_size(*custom_res));
+
+ break;
+ }
+
kv_push_safe(rspamd_word_t, *res, kv_A(*custom_res, i), custom_tokenizer_error);
}
}
}
- if (long_text_mode) {
+ if (rspamd_tokenize_needs_time_check(long_text_mode, budget,
+ kv_size(*res))) {
if ((kv_size(*res) + 1) % 16 == 0) {
ev_tstamp now = ev_time();
}
}
+ if (!rspamd_tokenize_budget_take(budget, token.original.len)) {
+ msg_debug_pool_check(
+ "words budget is exhausted: %uL words, %uL bytes; "
+ "stop tokenization",
+ budget->words, budget->bytes);
+
+ goto end;
+ }
+
kv_push_safe(rspamd_word_t, *res, token, tokenize_error);
if (kv_size(*res) * sizeof(token) > (0x1ull << 30u)) {
while (cur && ex->pos <= last) {
/* We have an exception at the beginning, skip those */
last += ex->len;
- rspamd_tokenize_exception(ex, res);
+ rspamd_tokenize_exception(ex, res, budget);
if (last > p) {
/* Exception spread over the boundaries */
/* Process the current exception */
last += ex->len + (ex->pos - last);
- rspamd_tokenize_exception(ex, res);
+ rspamd_tokenize_exception(ex, res, budget);
if (last > p) {
/* Exception spread over the boundaries */
decay = TRUE;
}
else {
- token.flags |= RSPAMD_STAT_TOKEN_FLAG_SKIPPED;
+ /*
+ * Drop the decayed token as the raw tokenizer does:
+ * keeping it in the vector defeats the decay, as all
+ * retained tokens are normalized and stemmed later on
+ */
+ token.original.len = 0;
+ token.flags = 0;
}
}
}
goto end;
}
+ if (!rspamd_tokenize_budget_take(budget, token.original.len)) {
+ msg_debug_pool_check(
+ "words budget is exhausted: %uL words, %uL bytes; "
+ "stop tokenization",
+ budget->words, budget->bytes);
+
+ goto end;
+ }
+
kv_push_safe(rspamd_word_t, *res, token, tokenize_error);
}
- /* Also check for long text mode */
- if (long_text_mode) {
+ /* Also check for long text mode or a large number of words */
+ if (rspamd_tokenize_needs_time_check(long_text_mode, budget,
+ kv_size(*res))) {
/* Check time each 128 words added */
const int words_check_mask = 0x7F;
unsigned int i = 0;
UChar32 uc;
gboolean valid_utf = TRUE;
+ /* Meta words share the message wide budget with the text parts */
+ struct rspamd_tokenize_budget *budget = task->message ? &MESSAGE_FIELD(task, words_budget) : NULL;
while (i < len) {
U8_NEXT(beg, i, len, uc);
&utxt, RSPAMD_TOKENIZE_UTF,
task->cfg, NULL, NULL,
&task->meta_words,
+ budget,
task->task_pool);
utext_close(&utxt);
NULL, RSPAMD_TOKENIZE_RAW,
task->cfg, NULL, NULL,
&task->meta_words,
+ budget,
task->task_pool);
}
}
for (i = 0; i < kv_size(*words); i++) {
tok = &kv_A(*words, i);
+
+ /*
+ * Statistics and shingles ignore skipped words, so there is no point
+ * in spending allocations on normalizing them
+ */
+ if (tok->flags & RSPAMD_WORD_FLAG_SKIPPED) {
+ continue;
+ }
+
rspamd_normalize_single_word(tok, pool);
}
}
for (i = 0; i < kv_size(*words); i++) {
tok = &kv_A(*words, i);
+ /* Skipped words are not normalized, so there is nothing to stem */
+ if (tok->flags & RSPAMD_WORD_FLAG_SKIPPED) {
+ continue;
+ }
+
/* Skip stemming if token has already been stemmed by custom tokenizer */
if (tok->flags & RSPAMD_STAT_TOKEN_FLAG_STEMMED) {
/* Already stemmed, just check for stop words */
GList *exceptions,
uint64_t *hash,
rspamd_words_t *output_kvec,
+ struct rspamd_tokenize_budget *budget,
rspamd_mempool_t *pool);
/* OSB tokenize function */
&utxt,
RSPAMD_TOKENIZE_UTF, NULL,
exceptions,
- NULL, NULL, NULL);
+ NULL, NULL, NULL, NULL);
if (res == NULL) {
lua_pushnil(L);
#include "rspamd_cxx_unit_text_stats.hxx"
#include "rspamd_cxx_unit_multipattern.hxx"
#include "rspamd_cxx_unit_compression.hxx"
+#include "rspamd_cxx_unit_tokenizer.hxx"
#include "rspamd_cxx_unit_http_timeout.hxx"
static gboolean verbose = false;
--- /dev/null
+/*
+ * Copyright 2026 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef RSPAMD_RSPAMD_CXX_UNIT_TOKENIZER_HXX
+#define RSPAMD_RSPAMD_CXX_UNIT_TOKENIZER_HXX
+
+#define DOCTEST_CONFIG_IMPLEMENTATION_IN_DLL
+#include "doctest/doctest.h"
+
+/* Both headers provide their own C++ linkage guards */
+#include "libstat/tokenizers/tokenizers.h"
+#include <unicode/utext.h>
+
+#include <memory>
+#include <string>
+
+/*
+ * Bounds of the text tokenizer: the per part words decay and the message wide
+ * words budget. Both must limit the words that are actually retained, as every
+ * retained word is normalized and stemmed later on, with several allocations
+ * from the task pool each.
+ */
+TEST_SUITE("tokenizer limits")
+{
+ /* Distinct words, so that nothing is deduplicated on the way */
+ static auto make_words_text(unsigned int nwords) -> std::string
+ {
+ std::string text;
+
+ text.reserve(nwords * 9);
+
+ for (unsigned int i = 0; i < nwords; i++) {
+ char buf[16];
+
+ rspamd_snprintf(buf, sizeof(buf), "word%04d ", i % 10000);
+ text += buf;
+ }
+
+ return text;
+ }
+
+ static auto tokenize(const std::string &text,
+ enum rspamd_tokenize_type how,
+ struct rspamd_config *cfg,
+ struct rspamd_tokenize_budget *budget,
+ rspamd_words_t *words,
+ rspamd_mempool_t *pool) -> void
+ {
+ UText utxt = UTEXT_INITIALIZER;
+ UErrorCode uc_err = U_ZERO_ERROR;
+
+ utext_openUTF8(&utxt, text.data(), text.size(), &uc_err);
+ REQUIRE(U_SUCCESS(uc_err));
+
+ rspamd_tokenize_text(text.data(), text.size(),
+ how == RSPAMD_TOKENIZE_UTF ? &utxt : nullptr,
+ how, cfg, nullptr, nullptr, words, budget, pool);
+
+ utext_close(&utxt);
+ }
+
+ static auto count_skipped(const rspamd_words_t &words) -> unsigned int
+ {
+ unsigned int nskipped = 0;
+
+ for (unsigned int i = 0; i < kv_size(words); i++) {
+ if (kv_A(words, i).flags & RSPAMD_WORD_FLAG_SKIPPED) {
+ nskipped++;
+ }
+ }
+
+ return nskipped;
+ }
+
+ TEST_CASE("words decay bounds the retained words")
+ {
+ constexpr auto nwords = 5000;
+ auto cfg = std::make_unique<struct rspamd_config>();
+ auto *pool = rspamd_mempool_new(rspamd_mempool_suggest_size(), "tok", 0);
+ auto text = make_words_text(nwords);
+
+ cfg->words_decay = 50;
+
+ for (auto how: {RSPAMD_TOKENIZE_UTF, RSPAMD_TOKENIZE_RAW}) {
+ rspamd_words_t words;
+
+ kv_init(words);
+ tokenize(text, how, cfg.get(), nullptr, &words, pool);
+
+ /*
+ * Decay keeps roughly words_decay more words after it starts, so
+ * the retained amount must be nowhere near the words in the text
+ */
+ CHECK(kv_size(words) > 0);
+ CHECK(kv_size(words) < nwords / 4);
+ /* Decayed words must be dropped, not retained and normalized */
+ CHECK(count_skipped(words) == 0);
+
+ kv_destroy(words);
+ }
+
+ rspamd_mempool_delete(pool);
+ }
+
+ TEST_CASE("words budget bounds a single part")
+ {
+ auto cfg = std::make_unique<struct rspamd_config>();
+ auto *pool = rspamd_mempool_new(rspamd_mempool_suggest_size(), "tok", 0);
+ auto text = make_words_text(1000);
+ struct rspamd_tokenize_budget budget = {};
+ rspamd_words_t words;
+
+ budget.max_words = 100;
+ kv_init(words);
+
+ tokenize(text, RSPAMD_TOKENIZE_UTF, cfg.get(), &budget, &words, pool);
+
+ CHECK(kv_size(words) == 100);
+ CHECK(budget.words == 100);
+ CHECK(budget.exceeded);
+
+ kv_destroy(words);
+ rspamd_mempool_delete(pool);
+ }
+
+ TEST_CASE("words budget is shared by all parts of a message")
+ {
+ auto cfg = std::make_unique<struct rspamd_config>();
+ auto *pool = rspamd_mempool_new(rspamd_mempool_suggest_size(), "tok", 0);
+ /* Each part on its own is far too small to hit any per part limit */
+ auto text = make_words_text(50);
+ struct rspamd_tokenize_budget budget = {};
+ uint64_t total = 0;
+
+ budget.max_words = 120;
+
+ for (auto i = 0; i < 10; i++) {
+ rspamd_words_t words;
+
+ kv_init(words);
+ tokenize(text, RSPAMD_TOKENIZE_UTF, cfg.get(), &budget, &words, pool);
+ total += kv_size(words);
+ kv_destroy(words);
+ }
+
+ CHECK(total == 120);
+ CHECK(budget.words == 120);
+ CHECK(budget.exceeded);
+
+ rspamd_mempool_delete(pool);
+ }
+
+ TEST_CASE("words budget bounds the retained bytes")
+ {
+ auto cfg = std::make_unique<struct rspamd_config>();
+ auto *pool = rspamd_mempool_new(rspamd_mempool_suggest_size(), "tok", 0);
+ auto text = make_words_text(1000);
+ struct rspamd_tokenize_budget budget = {};
+ rspamd_words_t words;
+ uint64_t bytes = 0;
+
+ budget.max_bytes = 256;
+ kv_init(words);
+
+ tokenize(text, RSPAMD_TOKENIZE_UTF, cfg.get(), &budget, &words, pool);
+
+ for (unsigned int i = 0; i < kv_size(words); i++) {
+ bytes += kv_A(words, i).original.len;
+ }
+
+ CHECK(bytes <= 256);
+ CHECK(budget.bytes == bytes);
+ CHECK(budget.exceeded);
+
+ kv_destroy(words);
+ rspamd_mempool_delete(pool);
+ }
+
+ TEST_CASE("no budget means no limits")
+ {
+ constexpr auto nwords = 1000;
+ auto cfg = std::make_unique<struct rspamd_config>();
+ auto *pool = rspamd_mempool_new(rspamd_mempool_suggest_size(), "tok", 0);
+ auto text = make_words_text(nwords);
+ rspamd_words_t words;
+
+ kv_init(words);
+ tokenize(text, RSPAMD_TOKENIZE_UTF, cfg.get(), nullptr, &words, pool);
+
+ CHECK(kv_size(words) == nwords);
+
+ kv_destroy(words);
+ rspamd_mempool_delete(pool);
+ }
+}
+
+#endif
projects / thirdparty / rspamd.git / commitdiff
