lib-fts: tokenizers - don't include removed apostrophes as part of the token size

diff --git a/src/lib-fts/fts-tokenizer-generic.c b/src/lib-fts/fts-tokenizer-generic.c
index d9d0d65b69d402f1f96baa7c2feed9ef3cf57995..78c850e6ffaa3a43343bf920da925126ef5c853d 100644 (file)
--- a/src/lib-fts/fts-tokenizer-generic.c
+++ b/src/lib-fts/fts-tokenizer-generic.c
@@ -180,6 +180,19 @@ static void tok_append_truncated(struct generic_fts_tokenizer *tok,
        size_t append_len, pos = 0, appended = 0;
        unichar_t c;
 
+       if (size == 0)
+               return;
+       if (data[0] == '\'' && tok->token->used == 0) {
+               /* Skip apostrophes in the beginning of the token.
+                  We need to do it here so that we don't truncate the
+                  token too early. */
+               data++;
+               size--;
+               if (size == 0)
+                       return;
+               i_assert(data[0] != '\'');
+       }
+
        i_assert(tok->max_length >= tok->token->used);
        append_len = I_MIN(size, tok->max_length - tok->token->used);
 

diff --git a/src/lib-fts/test-fts-tokenizer.c b/src/lib-fts/test-fts-tokenizer.c
index 3aaf49f2325bb3d845cdf044628078d2d64738f0..8ee4484b706b8517d0302731d5ed628d17bd5508 100644 (file)
--- a/src/lib-fts/test-fts-tokenizer.c
+++ b/src/lib-fts/test-fts-tokenizer.c
@@ -31,6 +31,8 @@ static const char *test_inputs[] = {
 
        "' ' '' ''' 'quoted text' 'word' 'hlo words' you're bad'''word '''pre post'''",
 
+       "'1234567890123456789012345678ä,"
+
        /* whitespace: with Unicode(utf8) U+FF01(ef bc 81)(U+2000(e2 80 80) and
           U+205A(e2 81 9a) and U+205F(e2 81 9f) */
        "hello\xEF\xBC\x81world\r\nAnd\xE2\x80\x80there\twas: text "
@@ -136,6 +138,8 @@ static void test_fts_tokenizer_generic_only(void)
                "quoted", "text", "word", "hlo", "words", "you're", "bad",
                "word", "pre", "post", NULL,
 
+               "1234567890123456789012345678ä",
+
                "hello", "world", "And",
                "there", "was", "text", "galore",
                "and", "more", NULL,
@@ -178,6 +182,8 @@ static void test_fts_tokenizer_generic_tr29_only(void)
                "quoted", "text", "word", "hlo", "words", "you're", "bad",
                "word", "pre", "post", NULL,
 
+               "1234567890123456789012345678ä",
+
                "hello", "world", "And",
                "there", "was", "text", "galore",
                "and", "more", NULL,