From: Vsevolod Stakhov <vsevolod@highsecure.ru>
Date: Thu, 2 Apr 2015 13:10:09 +0000 (+0100)
Subject: Fix tokenization of the last token in a message.
X-Git-Tag: 0.9.0~344
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=cfc1e965d51ed2fc2c6462e979094e47e47ba2f4;p=thirdparty%2Frspamd.git

Fix tokenization of the last token in a message.
---

diff --git a/src/libstat/tokenizers/tokenizers.c b/src/libstat/tokenizers/tokenizers.c
index eebc57c22d..4b8f0cfb39 100644
--- a/src/libstat/tokenizers/tokenizers.c
+++ b/src/libstat/tokenizers/tokenizers.c
@@ -295,7 +295,7 @@ rspamd_tokenize_text (gchar *text, gsize len, gboolean is_utf,
 	res = g_array_sized_new (FALSE, FALSE, sizeof (rspamd_fstring_t), 128);
 
 	while (func (&buf, &pos, &token, &cur, is_utf, &l)) {
-		if (min_len > 0 && l < min_len) {
+		if (l == 0 || (min_len > 0 && l < min_len)) {
 			token.begin = pos;
 			continue;
 		}