lib-fts: Fixed using max_length setting in simple tokenizer

diff --git a/src/lib-fts/fts-tokenizer-generic.c b/src/lib-fts/fts-tokenizer-generic.c
index d1754b61d44f29fbf6dea48cd8e31c9d4d0432b0..2867afe9c79af481839a58d8d921bd0bde35494f 100644 (file)
--- a/src/lib-fts/fts-tokenizer-generic.c
+++ b/src/lib-fts/fts-tokenizer-generic.c
@@ -86,7 +86,7 @@ static int
 fts_tokenizer_generic_simple_current_token(struct generic_fts_tokenizer *tok,
                                            const char **token_r)
 {
-       *token_r = t_strndup(tok->token->data, tok->token->used);
+       *token_r = t_strndup(tok->token->data, I_MIN(tok->token->used, tok->max_length));
        buffer_set_used_size(tok->token, 0);
        return 1;
 }
@@ -147,8 +147,7 @@ fts_tokenizer_generic_next_simple(struct fts_tokenizer *_tok,
                char_start_i = i;
                if (data_is_word_boundary(data, size, &i)) {
                        len = char_start_i - start;
-                       buffer_append(tok->token, data + start,
-                                     I_MIN(len, tok->max_length));
+                       buffer_append(tok->token, data + start, len);
                        if (tok->token->used == 0) {
                                /* no text read yet */
                                start = i + 1;
@@ -161,14 +160,16 @@ fts_tokenizer_generic_next_simple(struct fts_tokenizer *_tok,
        }
        /* word boundary not found yet */
        len = i - start;
-       buffer_append(tok->token, data + start, I_MIN(len, tok->max_length));
-
+       buffer_append(tok->token, data + start, len);
        *skip_r = i;
 
-       if (size == 0 && tok->token->used > 0) {
-               /* return the last token */
+       /* return the last token */
+       if (size == 0 && tok->token->used > 0)
+               return fts_tokenizer_generic_simple_current_token(tok, token_r);
+
+       /* token too long */
+       if (tok->token->used > tok->max_length)
                return fts_tokenizer_generic_simple_current_token(tok, token_r);
-       }
        return 0;
 }
 

diff --git a/src/lib-fts/test-fts-tokenizer.c b/src/lib-fts/test-fts-tokenizer.c
index 5df34a0f7598c09afe2148b6310c18f1ded7bb74..162ce2ae6e8996c7a1dc97e44e110e74f411c60f 100644 (file)
--- a/src/lib-fts/test-fts-tokenizer.c
+++ b/src/lib-fts/test-fts-tokenizer.c
@@ -6,6 +6,8 @@
 #include "test-common.h"
 #include "fts-tokenizer.h"
 #include "fts-tokenizer-private.h"
+/* TODO: fix including and linking of this. */
+/* #include "fts-tokenizer-generic-private.h" */
 
 #include <stdlib.h>
 
@@ -13,11 +15,12 @@ static void test_fts_tokenizer_generic_only(void)
 {
        static const unsigned char input[] =
                "hello world\r\nAnd there\twas: text "
-               "galore, and more.\n\n (\"Hello world\")last ";
+               "galore, and longlonglongabcdefghijklmnopqrstuvwxyz more.\n\n (\"Hello world\")last ";
        static const char *const expected_output[] = {
                "hello", "world", "And",
                "there", "was", "text", "galore",
-               "and", "more", "Hello", "world", "last", NULL
+               "and", "longlonglongabcdefghijklmnopqr",
+               "more", "Hello", "world", "last", NULL
        };
        const struct fts_tokenizer *tok_class;
        struct fts_tokenizer *tok;
@@ -28,6 +31,8 @@ static void test_fts_tokenizer_generic_only(void)
        fts_tokenizers_init();
        tok_class = fts_tokenizer_find(FTS_TOKENIZER_GENERIC_NAME);
        test_assert(fts_tokenizer_create(tok_class, NULL, NULL, &tok, &error) == 0);
+/*TODO: Uncomment when fts-tokenizer-generic-private.h inclusion is fixed */
+/*test_assert(((struct generic_fts_tokenizer *) tok)->algorithm ==  BOUNDARY_ALGORITHM_SIMPLE);*/
        while (fts_tokenizer_next(tok, input, sizeof(input)-1, &token) > 0) {
                test_assert(strcmp(token, *eopp) == 0);
                eopp++;