From: Marco Bettini Date: Mon, 31 Jul 2023 09:09:35 +0000 (+0000) Subject: lib-storage: struct mailbox_status - Replace last_cached_seq with fts_last_indexed_uid X-Git-Tag: 2.4.0~2532 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=c632d886a4011cb4273f8c9d9848e47e215dbd53;p=thirdparty%2Fdovecot%2Fcore.git lib-storage: struct mailbox_status - Replace last_cached_seq with fts_last_indexed_uid Doveadm index behavior is changed so that now it is intended to be used only for adding missing mails to fts, and no longer adds missing mails to dovecot.index.cache. STATUS_FIRST_RECENT_UID is replaced by STATUS_FTS_LAST_INDEXED_UID last_cached_seq s replaced by fts_last_indexed_uid --- diff --git a/src/doveadm/doveadm-mail-index.c b/src/doveadm/doveadm-mail-index.c index f518a5883a..42446f59d2 100644 --- a/src/doveadm/doveadm-mail-index.c +++ b/src/doveadm/doveadm-mail-index.c @@ -38,7 +38,6 @@ static int cmd_index_box_precache(struct doveadm_mail_cmd_context *dctx, struct mail_search_context *ctx; struct mail *mail; struct mailbox_metadata metadata; - uint32_t seq; unsigned int counter = 0, max; int ret = 0; @@ -49,7 +48,7 @@ static int cmd_index_box_precache(struct doveadm_mail_cmd_context *dctx, mailbox_get_last_internal_error(box, NULL)); return -1; } - if (mailbox_get_status(box, STATUS_MESSAGES | STATUS_LAST_CACHED_SEQ, + if (mailbox_get_status(box, STATUS_MESSAGES | STATUS_FTS_LAST_INDEXED_UID, &status) < 0) { e_error(event, "Mailbox %s: Status lookup failed: %s", mailbox_get_vname(box), @@ -57,7 +56,12 @@ static int cmd_index_box_precache(struct doveadm_mail_cmd_context *dctx, return -1; } - seq = status.last_cached_seq + 1; + uint32_t seq = 0, unused ATTR_UNUSED; + if (status.fts_last_indexed_uid > 0) + mailbox_get_seq_range(box, 1, status.fts_last_indexed_uid, + &unused, &seq); + seq++; + if (seq > status.messages) { if (doveadm_verbose) { e_info(event, "%s: Cache is already up to date", @@ -78,7 +82,7 @@ static int cmd_index_box_precache(struct doveadm_mail_cmd_context *dctx, metadata.precache_fields, NULL); mail_search_args_unref(&search_args); - max = status.messages - seq + 1; + max = status.messages + 1 - seq; while (mailbox_search_next(ctx, &mail)) { if (mail_precache(mail) < 0) { e_error(event, diff --git a/src/indexer/master-connection.c b/src/indexer/master-connection.c index f5e0c2c0af..c1e18d1aa7 100644 --- a/src/indexer/master-connection.c +++ b/src/indexer/master-connection.c @@ -77,7 +77,6 @@ index_mailbox_precache_real(struct master_connection *conn, struct mailbox *box) struct mail_search_context *ctx; struct mail *mail; struct mailbox_metadata metadata; - uint32_t seq, first_uid = 0, last_uid = 0; int ret = 0; struct event *index_event = event_create(box->event); event_add_category(index_event, &event_category_indexer_worker); @@ -89,14 +88,19 @@ index_mailbox_precache_real(struct master_connection *conn, struct mailbox *box) event_unref(&index_event); return -1; } - if (mailbox_get_status(box, STATUS_MESSAGES | STATUS_LAST_CACHED_SEQ, + if (mailbox_get_status(box, STATUS_MESSAGES | STATUS_FTS_LAST_INDEXED_UID, &status) < 0) { e_error(index_event, "Status lookup failed: %s", mailbox_get_last_internal_error(box, NULL)); event_unref(&index_event); return -1; } - seq = status.last_cached_seq + 1; + + uint32_t seq = 0, unused ATTR_UNUSED; + if (status.fts_last_indexed_uid > 0) + mailbox_get_seq_range(box, 1, status.fts_last_indexed_uid, + &unused, &seq); + seq++; trans = mailbox_transaction_begin(box, MAILBOX_TRANSACTION_FLAG_NO_CACHE_DEC, "indexing"); @@ -112,6 +116,7 @@ index_mailbox_precache_real(struct master_connection *conn, struct mailbox *box) /* otherwise the client doesn't receive the updates timely */ o_stream_uncork(conn->conn.output); + uint32_t first_uid = 0, last_uid = 0; unsigned int counter = 0; unsigned int percentage_sent = 0; unsigned int goal = status.messages + 1 - seq; diff --git a/src/lib-storage/index/index-status.c b/src/lib-storage/index/index-status.c index 1685339ca7..0c9064af28 100644 --- a/src/lib-storage/index/index-status.c +++ b/src/lib-storage/index/index-status.c @@ -7,28 +7,6 @@ #include "mailbox-recent-flags.h" #include "index-storage.h" -static void -get_last_cached_seq(struct mailbox *box, uint32_t *last_cached_seq_r) -{ - const struct mail_index_header *hdr; - struct mail_cache_view *cache_view; - uint32_t seq; - - *last_cached_seq_r = 0; - if (!mail_cache_exists(box->cache)) - return; - - cache_view = mail_cache_view_open(box->cache, box->view); - hdr = mail_index_get_header(box->view); - for (seq = hdr->messages_count; seq > 0; seq--) { - if (mail_cache_field_exists_any(cache_view, seq)) { - *last_cached_seq_r = seq; - break; - } - } - mail_cache_view_close(&cache_view); -} - int index_storage_get_status(struct mailbox *box, enum mailbox_status_items items, struct mailbox_status *status_r) @@ -173,8 +151,13 @@ void index_storage_get_open_status(struct mailbox *box, index_storage_find_first_pvt_unseen_seq(box); } } - if ((items & STATUS_LAST_CACHED_SEQ) != 0) - get_last_cached_seq(box, &status_r->last_cached_seq); + if ((items & STATUS_FTS_LAST_INDEXED_UID) != 0) { + /* This is called only when FTS plugin isn't loaded, + in which case it returns "all mails are FTS indexed" + to avoid "doveadm index" constantly trying to precache + all mails. */ + status_r->fts_last_indexed_uid = status_r->uidnext - 1; + } if ((items & STATUS_KEYWORDS) != 0) status_r->keywords = mail_index_get_keywords(box->index); diff --git a/src/lib-storage/mail-storage.h b/src/lib-storage/mail-storage.h index 9afd32a453..8adda4cb94 100644 --- a/src/lib-storage/mail-storage.h +++ b/src/lib-storage/mail-storage.h @@ -93,23 +93,23 @@ enum mailbox_existence { }; enum mailbox_status_items { - STATUS_MESSAGES = 0x01, - STATUS_RECENT = 0x02, - STATUS_UIDNEXT = 0x04, - STATUS_UIDVALIDITY = 0x08, - STATUS_UNSEEN = 0x10, - STATUS_FIRST_UNSEEN_SEQ = 0x20, - STATUS_KEYWORDS = 0x40, - STATUS_HIGHESTMODSEQ = 0x80, - STATUS_PERMANENT_FLAGS = 0x200, - STATUS_FIRST_RECENT_UID = 0x400, - STATUS_LAST_CACHED_SEQ = 0x800, - STATUS_CHECK_OVER_QUOTA = 0x1000, /* return error if over quota */ - STATUS_HIGHESTPVTMODSEQ = 0x2000, + STATUS_MESSAGES = 0x01, + STATUS_RECENT = 0x02, + STATUS_UIDNEXT = 0x04, + STATUS_UIDVALIDITY = 0x08, + STATUS_UNSEEN = 0x10, + STATUS_FIRST_UNSEEN_SEQ = 0x20, + STATUS_KEYWORDS = 0x40, + STATUS_HIGHESTMODSEQ = 0x80, + STATUS_PERMANENT_FLAGS = 0x200, + STATUS_FIRST_RECENT_UID = 0x400, + STATUS_FTS_LAST_INDEXED_UID = 0x800, + STATUS_CHECK_OVER_QUOTA = 0x1000, /* return error if over quota */ + STATUS_HIGHESTPVTMODSEQ = 0x2000, /* status items that must not be looked up with mailbox_get_open_status(), because they can return failure. */ #define MAILBOX_STATUS_FAILING_ITEMS \ - (STATUS_LAST_CACHED_SEQ | STATUS_CHECK_OVER_QUOTA) + (STATUS_FTS_LAST_INDEXED_UID | STATUS_CHECK_OVER_QUOTA) }; enum mailbox_metadata_items { @@ -261,7 +261,7 @@ struct mailbox_status { uint32_t first_unseen_seq; /* STATUS_FIRST_UNSEEN_SEQ */ uint32_t first_recent_uid; /* STATUS_FIRST_RECENT_UID */ - uint32_t last_cached_seq; /* STATUS_LAST_CACHED_SEQ */ + uint32_t fts_last_indexed_uid; /* STATUS_FTS_LAST_INDEXED_UID */ uint64_t highest_modseq; /* STATUS_HIGHESTMODSEQ */ /* 0 if no private index (STATUS_HIGHESTPVTMODSEQ) */ uint64_t highest_pvt_modseq; diff --git a/src/plugins/fts/fts-storage.c b/src/plugins/fts/fts-storage.c index f2d91fced7..f6be09911e 100644 --- a/src/plugins/fts/fts-storage.c +++ b/src/plugins/fts/fts-storage.c @@ -77,29 +77,14 @@ static MODULE_CONTEXT_DEFINE_INIT(fts_mail_module, &mail_module_register); static MODULE_CONTEXT_DEFINE_INIT(fts_mailbox_list_module, &mailbox_list_module_register); -static int fts_mailbox_get_last_cached_seq(struct mailbox *box, uint32_t *seq_r) +static int fts_mailbox_get_last_indexed_uid(struct mailbox *box, uint32_t *uid_r) { struct fts_mailbox_list *flist = FTS_LIST_CONTEXT_REQUIRE(box->list); - uint32_t seq1, seq2, last_uid; - int ret; - - ret = fts_search_get_first_missing_uid(flist->backend, box, &last_uid); + int ret = fts_search_get_first_missing_uid(flist->backend, box, uid_r); if (ret < 0) { mail_storage_set_internal_error(box->storage); return -1; } - - if (ret == 0 && last_uid == 0) { - /* nothing is indexed. */ - *seq_r = 0; - } else { - if (ret > 0) { - /* everything is indexed */ - last_uid = (uint32_t)-1; - } - mailbox_get_seq_range(box, 1, last_uid, &seq1, &seq2); - *seq_r = seq2; - } return 0; } @@ -108,20 +93,16 @@ fts_mailbox_get_status(struct mailbox *box, enum mailbox_status_items items, struct mailbox_status *status_r) { struct fts_mailbox *fbox = FTS_CONTEXT_REQUIRE(box); - uint32_t seq; - - if (fbox->module_ctx.super.get_status(box, items, status_r) < 0) + if (fbox->module_ctx.super.get_status( + box, items & ENUM_NEGATE(STATUS_FTS_LAST_INDEXED_UID), + status_r) < 0) return -1; - if ((items & STATUS_LAST_CACHED_SEQ) != 0) { - if (fts_mailbox_get_last_cached_seq(box, &seq) < 0) - return -1; + if ((items & STATUS_FTS_LAST_INDEXED_UID) != 0 && + fts_mailbox_get_last_indexed_uid( + box, &status_r->fts_last_indexed_uid) < 0) + return -1; - /* Always use the FTS's last_cached_seq. This is because we - don't want to reindex all mails to FTS if .cache file is - deleted. */ - status_r->last_cached_seq = seq; - } return 0; } @@ -486,13 +467,17 @@ static int fts_mail_precache_init(struct mail *_mail) { struct fts_transaction_context *ft = FTS_CONTEXT_REQUIRE(_mail->transaction); struct fts_mailbox_list *flist = FTS_LIST_CONTEXT_REQUIRE(_mail->box->list); - uint32_t last_seq; - if (fts_mailbox_get_last_cached_seq(_mail->box, &last_seq) < 0) { + uint32_t last_uid; + if (fts_mailbox_get_last_indexed_uid(_mail->box,&last_uid) < 0) { ft->failure_reason = "Failed to lookup last indexed FTS mail"; return -1; } + uint32_t last_seq = 0, unused ATTR_UNUSED; + if (last_uid > 0) + mailbox_get_seq_range(_mail->box, 1, last_uid, &unused, &last_seq); + ft->precached = TRUE; ft->next_index_seq = last_seq + 1; if (flist->update_ctx == NULL) diff --git a/src/plugins/virtual/virtual-storage.c b/src/plugins/virtual/virtual-storage.c index bac3bc6c33..90b59900a9 100644 --- a/src/plugins/virtual/virtual-storage.c +++ b/src/plugins/virtual/virtual-storage.c @@ -634,21 +634,9 @@ virtual_storage_get_status(struct mailbox *box, struct virtual_mailbox *mbox = container_of(box, struct virtual_mailbox, box); - if ((items & STATUS_LAST_CACHED_SEQ) != 0) - items |= STATUS_MESSAGES; - if (index_storage_get_status(box, items, status_r) < 0) return -1; - if ((items & STATUS_LAST_CACHED_SEQ) != 0) { - /* Virtual mailboxes have no cached data of their own, so the - current value is always 0. The most important use for this - functionality is for "doveadm index" to do FTS indexing and - it doesn't really matter there if we set this value - correctly or not. So for now just assume that everything is - indexed. */ - status_r->last_cached_seq = status_r->messages; - } if (!mbox->have_guid_flags_set) { if (virtual_storage_set_have_guid_flags(mbox) < 0) return -1;