fts: Keep track of every backend mailbox fts index status for virtual folders.

diff --git a/src/plugins/fts/fts-search.c b/src/plugins/fts/fts-search.c
index 110308bb7d57538ec975d3dfa90a8e8a2e57e3bc..f25078c2948cf99b325fc816c6a0c61b6826bf1f 100644 (file)
--- a/src/plugins/fts/fts-search.c
+++ b/src/plugins/fts/fts-search.c
@@ -9,6 +9,7 @@
 #include "fts-search-args.h"
 #include "fts-search-serialize.h"
 #include "fts-storage.h"
+#include "hash.h"
 
 static void
 uid_range_to_seqs(struct fts_search_context *fctx,
@@ -351,6 +352,11 @@ static void fts_search_try_lookup(struct fts_search_context *fctx)
                              &seq1, &seq2);
        fctx->first_unindexed_seq = seq1 != 0 ? seq1 : (uint32_t)-1;
 
+       if (fctx->virtual_mailbox) {
+               hash_table_clear(fctx->last_indexed_virtual_uids, TRUE);
+               fctx->next_unindexed_seq = fctx->first_unindexed_seq;
+       }
+
        if ((fctx->backend->flags & FTS_BACKEND_FLAG_TOKENIZED_INPUT) != 0) {
                if (fts_search_args_expand(fctx->backend, fctx->args) < 0)
                        return;

diff --git a/src/plugins/fts/fts-storage.c b/src/plugins/fts/fts-storage.c
index abf56f87782edc3e791b09f40bbff54bbd7c0c0c..8024e20bb1c9a22059f27ae59634d0bb1a71c200 100644 (file)
--- a/src/plugins/fts/fts-storage.c
+++ b/src/plugins/fts/fts-storage.c
@@ -16,6 +16,7 @@
 #include "fts-plugin.h"
 #include "fts-user.h"
 #include "fts-storage.h"
+#include "hash.h"
 
 
 #define FTS_CONTEXT(obj) \
@@ -231,6 +232,10 @@ fts_mailbox_search_init(struct mailbox_transaction_context *t,
        fctx->result_pool = pool_alloconly_create("fts results", 1024*64);
        fctx->orig_matches = buffer_create_dynamic(default_pool, 64);
        fctx->virtual_mailbox = t->box->virtual_vfuncs != NULL;
+       if (fctx->virtual_mailbox) {
+               hash_table_create(&fctx->last_indexed_virtual_uids,
+                                 default_pool, 0, str_hash, strcmp);
+       }
        fctx->enforced = fts_enforced_parse(
                mail_user_plugin_getenv(t->box->storage->user, "fts_enforced"));
        i_array_init(&fctx->levels, 8);
@@ -378,6 +383,8 @@ static int fts_mailbox_search_deinit(struct mail_search_context *ctx)
        int ret = 0;
 
        if (fctx != NULL) {
+               if (fctx->virtual_mailbox)
+                       hash_table_destroy(&fctx->last_indexed_virtual_uids);
                if (fctx->indexer_ctx != NULL) {
                        if (fts_indexer_deinit(&fctx->indexer_ctx) < 0)
                                ft->failure_reason = "FTS indexing failed";
@@ -796,6 +803,67 @@ static int fts_copy(struct mail_save_context *ctx, struct mail *mail)
        return 0;
 }
 
+static void fts_mailbox_virtual_match_mail(struct mail_search_context *ctx,
+                                          struct mail *mail)
+{
+       struct fts_search_context *fctx = FTS_CONTEXT(ctx);
+       unsigned int idx, be_last_uid;
+
+       if (fctx == NULL || !fctx->fts_lookup_success || !fctx->virtual_mailbox ||
+           ctx->seq < fctx->first_unindexed_seq)
+               return;
+       /* Table of last indexed UID per backend mailbox */
+       HASH_TABLE_TYPE(virtual_last_indexed) hash_tbl =
+               fctx->last_indexed_virtual_uids;
+
+       struct mail *backend_mail;
+       if (mail->box->mail_vfuncs->get_backend_mail(mail, &backend_mail) < 0)
+               return;
+       const char *box_name = backend_mail->box->vname;
+       /* Get the last indexed UID in the backend mailbox */
+       void *uid_value =
+               hash_table_lookup(fctx->last_indexed_virtual_uids, box_name);
+       if (uid_value == NULL) {
+               /* This backend's last indexed uid is not yet inserted to the table */
+               pool_t p = pool_alloconly_create("be mailbox names", 1024);
+               if (mailbox_open(backend_mail->box) < 0 ||
+                   fts_backend_get_last_uid(fctx->backend, backend_mail->box,
+                                            &be_last_uid) < 0) {
+                       be_last_uid = 0;
+               } else {
+                       const char *vname_copy =
+                               p_strdup(p, backend_mail->box->vname);
+                       hash_table_insert(hash_tbl, vname_copy,
+                                         POINTER_CAST(be_last_uid + 1));
+               }
+       } else {
+               be_last_uid = POINTER_CAST_TO(uid_value, uint32_t) - 1;
+       }
+       if (backend_mail->uid <= be_last_uid) {
+               /* Mail was already indexed in the backend mailbox.
+                  Apply [non]matches based on the FTS lookup results */
+               struct fts_transaction_context *ft = FTS_CONTEXT_REQUIRE(ctx->transaction);
+
+               if (fctx->next_unindexed_seq == mail->seq) {
+                       fctx->next_unindexed_seq++;
+                       ft->highest_virtual_uid = mail->uid;
+               }
+               idx = 0;
+               fts_search_apply_results_level(ctx, ctx->args->args, &idx);
+       } else {
+               fctx->virtual_seen_unindexed_gaps = TRUE;
+       }
+}
+
+static int fts_mailbox_search_next_match_mail(struct mail_search_context *ctx,
+                                             struct mail *mail)
+{
+       struct fts_mailbox *fbox = FTS_CONTEXT_REQUIRE(ctx->transaction->box);
+
+       fts_mailbox_virtual_match_mail(ctx, mail);
+       return fbox->module_ctx.super.search_next_match_mail(ctx, mail);
+}
+
 void fts_mailbox_allocated(struct mailbox *box)
 {
        struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(box->list);
@@ -822,6 +890,7 @@ void fts_mailbox_allocated(struct mailbox *box)
        v->sync_deinit = fts_sync_deinit;
        v->save_finish = fts_save_finish;
        v->copy = fts_copy;
+       v->search_next_match_mail = fts_mailbox_search_next_match_mail;
 
        MODULE_CONTEXT_SET(box, fts_storage_module, fbox);
 }

diff --git a/src/plugins/fts/fts-storage.h b/src/plugins/fts/fts-storage.h
index 3776dddf6abc585750d760033da955ff4424dd78..890b8e70b0908f092f41a325e1d234c26535aa4a 100644 (file)
--- a/src/plugins/fts/fts-storage.h
+++ b/src/plugins/fts/fts-storage.h
@@ -21,6 +21,8 @@ struct fts_search_level {
        ARRAY_TYPE(fts_score_map) score_map;
 };
 
+HASH_TABLE_DEFINE_TYPE(virtual_last_indexed, const char *, void *);
+
 struct fts_search_context {
        union mail_search_module_context module_ctx;
 
@@ -36,6 +38,8 @@ struct fts_search_context {
        buffer_t *orig_matches;
 
        uint32_t first_unindexed_seq;
+       uint32_t next_unindexed_seq;
+       HASH_TABLE_TYPE(virtual_last_indexed) last_indexed_virtual_uids;
 
        /* final scores, combined from all levels */
        struct fts_scores *scores;
@@ -45,6 +49,7 @@ struct fts_search_context {
        bool virtual_mailbox:1;
        bool fts_lookup_success:1;
        bool indexing_timed_out:1;
+       bool virtual_seen_unindexed_gaps:1;
 };
 
 /* Figure out if we want to use full text search indexes and update