From: Timo Sirainen <timo.sirainen@open-xchange.com>
Date: Fri, 8 Apr 2022 12:10:53 +0000 (+0300)
Subject: lib-index: Optimize modseq scanning by using the modseq header in index
X-Git-Tag: 2.3.19~13
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=aa7d67e6d6eeca798f29b68d11cd2ef1f26ef21d;p=thirdparty%2Fdovecot%2Fcore.git

lib-index: Optimize modseq scanning by using the modseq header in index

View is very commonly set to start reading new changes since dovecot.index
was last written. When reading the newer records in dovecot.index.log, the
modseq of each change is needed to be known. However, the initial modseq
calculation was usually done inefficiently by reading the whole
dovecot.index.log from the beginning of the file. This change prevents
that by using instead the "modseq" header in dovecot.index to get the
initial modseq.
---

diff --git a/src/lib-index/mail-index-map-read.c b/src/lib-index/mail-index-map-read.c
index 9a8b3341c7..6999bf1511 100644
--- a/src/lib-index/mail-index-map-read.c
+++ b/src/lib-index/mail-index-map-read.c
@@ -8,6 +8,7 @@
 #include "mail-index-private.h"
 #include "mail-index-sync-private.h"
 #include "mail-transaction-log-private.h"
+#include "mail-index-modseq.h"
 #include "ioloop.h"
 
 static void mail_index_map_copy_hdr(struct mail_index_map *map,
@@ -393,6 +394,7 @@ mail_index_map_latest_file(struct mail_index *index, const char **reason_r)
 	index->main_index_hdr_log_file_seq = new_map->hdr.log_file_seq;
 	index->main_index_hdr_log_file_tail_offset =
 		new_map->hdr.log_file_tail_offset;
+	mail_index_modseq_hdr_snapshot_update(new_map);
 
 	mail_index_unmap(&index->map);
 	index->map = new_map;
diff --git a/src/lib-index/mail-index-modseq.c b/src/lib-index/mail-index-modseq.c
index 5f61d3368a..4285983658 100644
--- a/src/lib-index/mail-index-modseq.c
+++ b/src/lib-index/mail-index-modseq.c
@@ -91,6 +91,16 @@ bool mail_index_have_modseq_tracking(struct mail_index *index)
 	return mail_index_map_get_modseq_header(index->map) != NULL;
 }
 
+void mail_index_modseq_hdr_snapshot_update(struct mail_index_map *map)
+{
+	const struct mail_index_modseq_header *hdr =
+		mail_index_map_get_modseq_header(map);
+	if (hdr != NULL)
+		map->modseq_hdr_snapshot = *hdr;
+	else
+		i_zero(&map->modseq_hdr_snapshot);
+}
+
 const struct mail_index_modseq_header *
 mail_index_map_get_modseq_header(struct mail_index_map *map)
 {
diff --git a/src/lib-index/mail-index-modseq.h b/src/lib-index/mail-index-modseq.h
index bef44a1a78..f0fe48a1ac 100644
--- a/src/lib-index/mail-index-modseq.h
+++ b/src/lib-index/mail-index-modseq.h
@@ -13,16 +13,13 @@ struct mail_index_modseq;
 struct mail_index_map_modseq;
 struct mail_index_sync_map_ctx;
 
-struct mail_index_modseq_header {
-	/* highest used modseq */
-	uint64_t highest_modseq;
-	/* last tracked log file position */
-	uint32_t log_seq;
-	uint32_t log_offset;
-};
-
 void mail_index_modseq_init(struct mail_index *index);
 
+/* Save a copy of the current modseq header to map->modseq_hdr_snapshot. This
+   is expected to be called when reading the dovecot.index header before any
+   changes are applied on top of it from dovecot.index.log. */
+void mail_index_modseq_hdr_snapshot_update(struct mail_index_map *map);
+
 const struct mail_index_modseq_header *
 mail_index_map_get_modseq_header(struct mail_index_map *map);
 uint64_t mail_index_map_modseq_get_highest(struct mail_index_map *map);
diff --git a/src/lib-index/mail-index-private.h b/src/lib-index/mail-index-private.h
index e0736daf5c..26f7e392a0 100644
--- a/src/lib-index/mail-index-private.h
+++ b/src/lib-index/mail-index-private.h
@@ -113,6 +113,14 @@ struct mail_index_registered_ext {
 	mail_index_expunge_handler_t *expunge_handler;
 };
 
+struct mail_index_modseq_header {
+	/* highest used modseq */
+	uint64_t highest_modseq;
+	/* last tracked log file position */
+	uint32_t log_seq;
+	uint32_t log_offset;
+};
+
 struct mail_index_record_map {
 	ARRAY(struct mail_index_map *) maps;
 
@@ -147,6 +155,8 @@ struct mail_index_map {
 
 	ARRAY(unsigned int) keyword_idx_map; /* file -> index */
 
+	struct mail_index_modseq_header modseq_hdr_snapshot;
+
 	struct mail_index_record_map *rec_map;
 };
 
diff --git a/src/lib-index/mail-transaction-log-modseq.c b/src/lib-index/mail-transaction-log-modseq.c
index 919f855a47..eb2b5330a0 100644
--- a/src/lib-index/mail-transaction-log-modseq.c
+++ b/src/lib-index/mail-transaction-log-modseq.c
@@ -2,6 +2,7 @@
 
 #include "lib.h"
 #include "mail-index-private.h"
+#include "mail-index-modseq.h"
 #include "mail-transaction-log-private.h"
 
 static struct modseq_cache *
@@ -135,6 +136,19 @@ int mail_transaction_log_file_get_highest_modseq_at(
 		cur_modseq = cache->highest_modseq;
 	}
 
+	/* See if we can use the "modseq" header in dovecot.index to further
+	   reduce how much we have to scan. */
+	const struct mail_index_modseq_header *modseq_hdr =
+		file->log->index->map == NULL ? NULL :
+		&file->log->index->map->modseq_hdr_snapshot;
+	if (modseq_hdr != NULL &&
+	    modseq_hdr->log_seq == file->hdr.file_seq &&
+	    modseq_hdr->log_offset <= offset &&
+	    modseq_hdr->log_offset >= cur_offset) {
+		cur_offset = modseq_hdr->log_offset;
+		cur_modseq = modseq_hdr->highest_modseq;
+	}
+
 	ret = mail_transaction_log_file_map(file, cur_offset, offset, &reason);
 	if (ret <= 0) {
 		*error_r = t_strdup_printf(