From: Timo Sirainen <tss@iki.fi>
Date: Mon, 1 Jun 2015 18:48:59 +0000 (+0300)
Subject: lib-fts: tokenizers - don't include removed apostrophes as part of the token size
X-Git-Tag: 2.2.19.rc1~416
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=202468f94e6c6c8b5d3d98ee74e01bb0d0bb04aa;p=thirdparty%2Fdovecot%2Fcore.git

lib-fts: tokenizers - don't include removed apostrophes as part of the token size
---

diff --git a/src/lib-fts/fts-tokenizer-generic.c b/src/lib-fts/fts-tokenizer-generic.c
index d9d0d65b69..78c850e6ff 100644
--- a/src/lib-fts/fts-tokenizer-generic.c
+++ b/src/lib-fts/fts-tokenizer-generic.c
@@ -180,6 +180,19 @@ static void tok_append_truncated(struct generic_fts_tokenizer *tok,
 	size_t append_len, pos = 0, appended = 0;
 	unichar_t c;
 
+	if (size == 0)
+		return;
+	if (data[0] == '\'' && tok->token->used == 0) {
+		/* Skip apostrophes in the beginning of the token.
+		   We need to do it here so that we don't truncate the
+		   token too early. */
+		data++;
+		size--;
+		if (size == 0)
+			return;
+		i_assert(data[0] != '\'');
+	}
+
 	i_assert(tok->max_length >= tok->token->used);
 	append_len = I_MIN(size, tok->max_length - tok->token->used);
 
diff --git a/src/lib-fts/test-fts-tokenizer.c b/src/lib-fts/test-fts-tokenizer.c
index 3aaf49f232..8ee4484b70 100644
--- a/src/lib-fts/test-fts-tokenizer.c
+++ b/src/lib-fts/test-fts-tokenizer.c
@@ -31,6 +31,8 @@ static const char *test_inputs[] = {
 
 	"' ' '' ''' 'quoted text' 'word' 'hlo words' you're bad'''word '''pre post'''",
 
+	"'1234567890123456789012345678ä,"
+
 	/* whitespace: with Unicode(utf8) U+FF01(ef bc 81)(U+2000(e2 80 80) and
 	   U+205A(e2 81 9a) and U+205F(e2 81 9f) */
 	"hello\xEF\xBC\x81world\r\nAnd\xE2\x80\x80there\twas: text "
@@ -136,6 +138,8 @@ static void test_fts_tokenizer_generic_only(void)
 		"quoted", "text", "word", "hlo", "words", "you're", "bad",
 		"word", "pre", "post", NULL,
 
+		"1234567890123456789012345678ä",
+
 		"hello", "world", "And",
 		"there", "was", "text", "galore",
 		"and", "more", NULL,
@@ -178,6 +182,8 @@ static void test_fts_tokenizer_generic_tr29_only(void)
 		"quoted", "text", "word", "hlo", "words", "you're", "bad",
 		"word", "pre", "post", NULL,
 
+		"1234567890123456789012345678ä",
+
 		"hello", "world", "And",
 		"there", "was", "text", "galore",
 		"and", "more", NULL,