lib-index: Optimize modseq scanning by using the modseq header in index

View is very commonly set to start reading new changes since dovecot.index
was last written. When reading the newer records in dovecot.index.log, the
modseq of each change is needed to be known. However, the initial modseq
calculation was usually done inefficiently by reading the whole
dovecot.index.log from the beginning of the file. This change prevents
that by using instead the "modseq" header in dovecot.index to get the
initial modseq.

diff --git a/src/lib-index/mail-index-map-read.c b/src/lib-index/mail-index-map-read.c
index 9a8b3341c71c742d075e37b4bfd3def8e987d3ad..6999bf151100e9c0f342731ecade0216c8c95080 100644 (file)
--- a/src/lib-index/mail-index-map-read.c
+++ b/src/lib-index/mail-index-map-read.c
@@ -8,6 +8,7 @@
 #include "mail-index-private.h"
 #include "mail-index-sync-private.h"
 #include "mail-transaction-log-private.h"
+#include "mail-index-modseq.h"
 #include "ioloop.h"
 
 static void mail_index_map_copy_hdr(struct mail_index_map *map,
@@ -393,6 +394,7 @@ mail_index_map_latest_file(struct mail_index *index, const char **reason_r)
        index->main_index_hdr_log_file_seq = new_map->hdr.log_file_seq;
        index->main_index_hdr_log_file_tail_offset =
                new_map->hdr.log_file_tail_offset;
+       mail_index_modseq_hdr_snapshot_update(new_map);
 
        mail_index_unmap(&index->map);
        index->map = new_map;

diff --git a/src/lib-index/mail-index-modseq.c b/src/lib-index/mail-index-modseq.c
index 5f61d3368a909e7e707da7677f81e2aa43c453d5..4285983658e0cea560385fdc0e029b8ea884e45d 100644 (file)
--- a/src/lib-index/mail-index-modseq.c
+++ b/src/lib-index/mail-index-modseq.c
@@ -91,6 +91,16 @@ bool mail_index_have_modseq_tracking(struct mail_index *index)
        return mail_index_map_get_modseq_header(index->map) != NULL;
 }
 
+void mail_index_modseq_hdr_snapshot_update(struct mail_index_map *map)
+{
+       const struct mail_index_modseq_header *hdr =
+               mail_index_map_get_modseq_header(map);
+       if (hdr != NULL)
+               map->modseq_hdr_snapshot = *hdr;
+       else
+               i_zero(&map->modseq_hdr_snapshot);
+}
+
 const struct mail_index_modseq_header *
 mail_index_map_get_modseq_header(struct mail_index_map *map)
 {

diff --git a/src/lib-index/mail-index-modseq.h b/src/lib-index/mail-index-modseq.h
index bef44a1a78e50e93dc4437d7b229fb2379698396..f0fe48a1ace80ac6392aa237c981778ed263163a 100644 (file)
--- a/src/lib-index/mail-index-modseq.h
+++ b/src/lib-index/mail-index-modseq.h
@@ -13,16 +13,13 @@ struct mail_index_modseq;
 struct mail_index_map_modseq;
 struct mail_index_sync_map_ctx;
 
-struct mail_index_modseq_header {
-       /* highest used modseq */
-       uint64_t highest_modseq;
-       /* last tracked log file position */
-       uint32_t log_seq;
-       uint32_t log_offset;
-};
-
 void mail_index_modseq_init(struct mail_index *index);
 
+/* Save a copy of the current modseq header to map->modseq_hdr_snapshot. This
+   is expected to be called when reading the dovecot.index header before any
+   changes are applied on top of it from dovecot.index.log. */
+void mail_index_modseq_hdr_snapshot_update(struct mail_index_map *map);
+
 const struct mail_index_modseq_header *
 mail_index_map_get_modseq_header(struct mail_index_map *map);
 uint64_t mail_index_map_modseq_get_highest(struct mail_index_map *map);

diff --git a/src/lib-index/mail-index-private.h b/src/lib-index/mail-index-private.h
index e0736daf5c096a56e3af03f1e8ab52b472a87aad..26f7e392a02e658d8d0b72eade4e6a0c9111f860 100644 (file)
--- a/src/lib-index/mail-index-private.h
+++ b/src/lib-index/mail-index-private.h
@@ -113,6 +113,14 @@ struct mail_index_registered_ext {
        mail_index_expunge_handler_t *expunge_handler;
 };
 
+struct mail_index_modseq_header {
+       /* highest used modseq */
+       uint64_t highest_modseq;
+       /* last tracked log file position */
+       uint32_t log_seq;
+       uint32_t log_offset;
+};
+
 struct mail_index_record_map {
        ARRAY(struct mail_index_map *) maps;
 
@@ -147,6 +155,8 @@ struct mail_index_map {
 
        ARRAY(unsigned int) keyword_idx_map; /* file -> index */
 
+       struct mail_index_modseq_header modseq_hdr_snapshot;
+
        struct mail_index_record_map *rec_map;
 };
 

diff --git a/src/lib-index/mail-transaction-log-modseq.c b/src/lib-index/mail-transaction-log-modseq.c
index 919f855a47e561bb71a3f726091bcd7d6587414c..eb2b5330a062c89acf3f0b169874aed7ab50a431 100644 (file)
--- a/src/lib-index/mail-transaction-log-modseq.c
+++ b/src/lib-index/mail-transaction-log-modseq.c
@@ -2,6 +2,7 @@
 
 #include "lib.h"
 #include "mail-index-private.h"
+#include "mail-index-modseq.h"
 #include "mail-transaction-log-private.h"
 
 static struct modseq_cache *
@@ -135,6 +136,19 @@ int mail_transaction_log_file_get_highest_modseq_at(
                cur_modseq = cache->highest_modseq;
        }
 
+       /* See if we can use the "modseq" header in dovecot.index to further
+          reduce how much we have to scan. */
+       const struct mail_index_modseq_header *modseq_hdr =
+               file->log->index->map == NULL ? NULL :
+               &file->log->index->map->modseq_hdr_snapshot;
+       if (modseq_hdr != NULL &&
+           modseq_hdr->log_seq == file->hdr.file_seq &&
+           modseq_hdr->log_offset <= offset &&
+           modseq_hdr->log_offset >= cur_offset) {
+               cur_offset = modseq_hdr->log_offset;
+               cur_modseq = modseq_hdr->highest_modseq;
+       }
+
        ret = mail_transaction_log_file_map(file, cur_offset, offset, &reason);
        if (ret <= 0) {
                *error_r = t_strdup_printf(