fts: Reset tokenizers before using them

This is mainly needed if the previous tokenization had failed.

diff --git a/src/plugins/fts/fts-build-mail.c b/src/plugins/fts/fts-build-mail.c
index 6ee9f46a5b710e2ecb62aff9c2dce6be1297ff86..731fe37c1b129b14d8701c4ad0d4aced258df45b 100644 (file)
--- a/src/plugins/fts/fts-build-mail.c
+++ b/src/plugins/fts/fts-build-mail.c
@@ -456,8 +456,16 @@ fts_build_mail_real(struct fts_backend_update_context *update_ctx,
        memset(&ctx, 0, sizeof(ctx));
        ctx.update_ctx = update_ctx;
        ctx.mail = mail;
-       if ((update_ctx->backend->flags & FTS_BACKEND_FLAG_TOKENIZED_INPUT) != 0)
+       if ((update_ctx->backend->flags & FTS_BACKEND_FLAG_TOKENIZED_INPUT) != 0) {
                ctx.pending_input = buffer_create_dynamic(default_pool, 128);
+               /* reset tokenizer between mails - just to be sure no state
+                  leaks between mails (especially if previous indexing had
+                  failed) */
+               struct fts_tokenizer *tokenizer;
+
+               tokenizer = fts_user_get_index_tokenizer(update_ctx->backend->ns->user);
+               fts_tokenizer_reset(tokenizer);
+       }
 
        prev_part = NULL;
        parser = message_parser_init(pool_datastack_create(), input,

diff --git a/src/plugins/fts/fts-search-args.c b/src/plugins/fts/fts-search-args.c
index 76eed0505ef872f45eb7b699cbeed778f89eaf07..a72107aedb02d607f10aa3e4883541f6b82e1742 100644 (file)
--- a/src/plugins/fts/fts-search-args.c
+++ b/src/plugins/fts/fts-search-args.c
@@ -114,6 +114,9 @@ static int fts_search_arg_expand(struct fts_backend *backend, pool_t pool,
        and_arg->match_not = orig_arg->match_not;
        and_arg->next = orig_arg->next;
 
+       /* reset tokenizer between search args in case there's any state left
+          from some previous failure */
+       fts_tokenizer_reset(tokenizer);
        while (fts_tokenizer_next(tokenizer,
                                  (const void *)orig_token,
                                  orig_token_len, &token) > 0) {