From: sergey.kitov Date: Thu, 16 Sep 2021 08:24:31 +0000 (+0300) Subject: fts: Keep track of every backend mailbox fts index status for virtual folders. X-Git-Tag: 2.3.18~179 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=9705b81fb51b5bdeaba12932a390ced2cc9dcad7;p=thirdparty%2Fdovecot%2Fcore.git fts: Keep track of every backend mailbox fts index status for virtual folders. --- diff --git a/src/plugins/fts/fts-search.c b/src/plugins/fts/fts-search.c index 110308bb7d..f25078c294 100644 --- a/src/plugins/fts/fts-search.c +++ b/src/plugins/fts/fts-search.c @@ -9,6 +9,7 @@ #include "fts-search-args.h" #include "fts-search-serialize.h" #include "fts-storage.h" +#include "hash.h" static void uid_range_to_seqs(struct fts_search_context *fctx, @@ -351,6 +352,11 @@ static void fts_search_try_lookup(struct fts_search_context *fctx) &seq1, &seq2); fctx->first_unindexed_seq = seq1 != 0 ? seq1 : (uint32_t)-1; + if (fctx->virtual_mailbox) { + hash_table_clear(fctx->last_indexed_virtual_uids, TRUE); + fctx->next_unindexed_seq = fctx->first_unindexed_seq; + } + if ((fctx->backend->flags & FTS_BACKEND_FLAG_TOKENIZED_INPUT) != 0) { if (fts_search_args_expand(fctx->backend, fctx->args) < 0) return; diff --git a/src/plugins/fts/fts-storage.c b/src/plugins/fts/fts-storage.c index abf56f8778..8024e20bb1 100644 --- a/src/plugins/fts/fts-storage.c +++ b/src/plugins/fts/fts-storage.c @@ -16,6 +16,7 @@ #include "fts-plugin.h" #include "fts-user.h" #include "fts-storage.h" +#include "hash.h" #define FTS_CONTEXT(obj) \ @@ -231,6 +232,10 @@ fts_mailbox_search_init(struct mailbox_transaction_context *t, fctx->result_pool = pool_alloconly_create("fts results", 1024*64); fctx->orig_matches = buffer_create_dynamic(default_pool, 64); fctx->virtual_mailbox = t->box->virtual_vfuncs != NULL; + if (fctx->virtual_mailbox) { + hash_table_create(&fctx->last_indexed_virtual_uids, + default_pool, 0, str_hash, strcmp); + } fctx->enforced = fts_enforced_parse( mail_user_plugin_getenv(t->box->storage->user, "fts_enforced")); i_array_init(&fctx->levels, 8); @@ -378,6 +383,8 @@ static int fts_mailbox_search_deinit(struct mail_search_context *ctx) int ret = 0; if (fctx != NULL) { + if (fctx->virtual_mailbox) + hash_table_destroy(&fctx->last_indexed_virtual_uids); if (fctx->indexer_ctx != NULL) { if (fts_indexer_deinit(&fctx->indexer_ctx) < 0) ft->failure_reason = "FTS indexing failed"; @@ -796,6 +803,67 @@ static int fts_copy(struct mail_save_context *ctx, struct mail *mail) return 0; } +static void fts_mailbox_virtual_match_mail(struct mail_search_context *ctx, + struct mail *mail) +{ + struct fts_search_context *fctx = FTS_CONTEXT(ctx); + unsigned int idx, be_last_uid; + + if (fctx == NULL || !fctx->fts_lookup_success || !fctx->virtual_mailbox || + ctx->seq < fctx->first_unindexed_seq) + return; + /* Table of last indexed UID per backend mailbox */ + HASH_TABLE_TYPE(virtual_last_indexed) hash_tbl = + fctx->last_indexed_virtual_uids; + + struct mail *backend_mail; + if (mail->box->mail_vfuncs->get_backend_mail(mail, &backend_mail) < 0) + return; + const char *box_name = backend_mail->box->vname; + /* Get the last indexed UID in the backend mailbox */ + void *uid_value = + hash_table_lookup(fctx->last_indexed_virtual_uids, box_name); + if (uid_value == NULL) { + /* This backend's last indexed uid is not yet inserted to the table */ + pool_t p = pool_alloconly_create("be mailbox names", 1024); + if (mailbox_open(backend_mail->box) < 0 || + fts_backend_get_last_uid(fctx->backend, backend_mail->box, + &be_last_uid) < 0) { + be_last_uid = 0; + } else { + const char *vname_copy = + p_strdup(p, backend_mail->box->vname); + hash_table_insert(hash_tbl, vname_copy, + POINTER_CAST(be_last_uid + 1)); + } + } else { + be_last_uid = POINTER_CAST_TO(uid_value, uint32_t) - 1; + } + if (backend_mail->uid <= be_last_uid) { + /* Mail was already indexed in the backend mailbox. + Apply [non]matches based on the FTS lookup results */ + struct fts_transaction_context *ft = FTS_CONTEXT_REQUIRE(ctx->transaction); + + if (fctx->next_unindexed_seq == mail->seq) { + fctx->next_unindexed_seq++; + ft->highest_virtual_uid = mail->uid; + } + idx = 0; + fts_search_apply_results_level(ctx, ctx->args->args, &idx); + } else { + fctx->virtual_seen_unindexed_gaps = TRUE; + } +} + +static int fts_mailbox_search_next_match_mail(struct mail_search_context *ctx, + struct mail *mail) +{ + struct fts_mailbox *fbox = FTS_CONTEXT_REQUIRE(ctx->transaction->box); + + fts_mailbox_virtual_match_mail(ctx, mail); + return fbox->module_ctx.super.search_next_match_mail(ctx, mail); +} + void fts_mailbox_allocated(struct mailbox *box) { struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(box->list); @@ -822,6 +890,7 @@ void fts_mailbox_allocated(struct mailbox *box) v->sync_deinit = fts_sync_deinit; v->save_finish = fts_save_finish; v->copy = fts_copy; + v->search_next_match_mail = fts_mailbox_search_next_match_mail; MODULE_CONTEXT_SET(box, fts_storage_module, fbox); } diff --git a/src/plugins/fts/fts-storage.h b/src/plugins/fts/fts-storage.h index 3776dddf6a..890b8e70b0 100644 --- a/src/plugins/fts/fts-storage.h +++ b/src/plugins/fts/fts-storage.h @@ -21,6 +21,8 @@ struct fts_search_level { ARRAY_TYPE(fts_score_map) score_map; }; +HASH_TABLE_DEFINE_TYPE(virtual_last_indexed, const char *, void *); + struct fts_search_context { union mail_search_module_context module_ctx; @@ -36,6 +38,8 @@ struct fts_search_context { buffer_t *orig_matches; uint32_t first_unindexed_seq; + uint32_t next_unindexed_seq; + HASH_TABLE_TYPE(virtual_last_indexed) last_indexed_virtual_uids; /* final scores, combined from all levels */ struct fts_scores *scores; @@ -45,6 +49,7 @@ struct fts_search_context { bool virtual_mailbox:1; bool fts_lookup_success:1; bool indexing_timed_out:1; + bool virtual_seen_unindexed_gaps:1; }; /* Figure out if we want to use full text search indexes and update