From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 27 Mar 2017 15:05:29 +0000 (+0300)
Subject: lib-index: Don't increase modseq for backend/dirty flag changes
X-Git-Tag: 2.2.29.rc1~32
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=b6a369a3dc1ceb4e9f328e642df35f55cbc6ccbc;p=thirdparty%2Fdovecot%2Fcore.git

lib-index: Don't increase modseq for backend/dirty flag changes

These flags are used only for internal changes and they shouldn't be
triggering any modseq changes.

To avoid modseqs from unexpectedly shrinking, the new modseq counting
behavior is enabled only for newly rotated transaction log files that have
a new minor_version.
---

diff --git a/src/doveadm/doveadm-dump-log.c b/src/doveadm/doveadm-dump-log.c
index e762225373..b2a6e15dea 100644
--- a/src/doveadm/doveadm-dump-log.c
+++ b/src/doveadm/doveadm-dump-log.c
@@ -11,7 +11,8 @@
 
 static struct mail_transaction_ext_intro prev_intro;
 
-static void dump_hdr(struct istream *input, uint64_t *modseq_r)
+static void dump_hdr(struct istream *input, uint64_t *modseq_r,
+		     unsigned int *version_r)
 {
 	struct mail_transaction_log_header hdr;
 	const unsigned char *data;
@@ -42,6 +43,7 @@ static void dump_hdr(struct istream *input, uint64_t *modseq_r)
 	       (unsigned long long)hdr.initial_modseq);
 	printf("compat flags = %x\n", hdr.compat_flags);
 	*modseq_r = hdr.initial_modseq;
+	*version_r = MAIL_TRANSACTION_LOG_HDR_VERSION(&hdr);
 }
 
 static const char *log_record_type(unsigned int type)
@@ -464,7 +466,8 @@ static void log_record_print(const struct mail_transaction_header *hdr,
 	}
 }
 
-static int dump_record(struct istream *input, uint64_t *modseq)
+static int dump_record(struct istream *input, uint64_t *modseq,
+		       unsigned int version)
 {
 	struct mail_transaction_header hdr;
 	unsigned int hdr_size;
@@ -505,7 +508,7 @@ static int dump_record(struct istream *input, uint64_t *modseq)
 	}
 
 	uint64_t prev_modseq = *modseq;
-	mail_transaction_update_modseq(&hdr, data, modseq);
+	mail_transaction_update_modseq(&hdr, data, modseq, version);
 	if (*modseq > prev_modseq)
 		printf(", modseq=%llu", (unsigned long long)*modseq);
 	printf("\n");
@@ -519,13 +522,14 @@ static void cmd_dump_log(int argc ATTR_UNUSED, char *argv[])
 {
 	struct istream *input;
 	uint64_t modseq;
+	unsigned int version;
 	int ret;
 
 	input = i_stream_create_file(argv[1], (size_t)-1);
-	dump_hdr(input, &modseq);
+	dump_hdr(input, &modseq, &version);
 	do {
 		T_BEGIN {
-			ret = dump_record(input, &modseq);
+			ret = dump_record(input, &modseq, version);
 		} T_END;
 	} while (ret > 0);
 	i_stream_unref(&input);
diff --git a/src/lib-index/mail-index-transaction-export.c b/src/lib-index/mail-index-transaction-export.c
index 9bdf1bb779..19d32d90ff 100644
--- a/src/lib-index/mail-index-transaction-export.c
+++ b/src/lib-index/mail-index-transaction-export.c
@@ -536,6 +536,32 @@ mail_index_transaction_keywords_count_modseq_incs(struct mail_index_transaction
 	return count;
 }
 
+static bool
+transaction_flag_updates_have_non_internal(struct mail_index_transaction *t)
+{
+	struct mail_transaction_log_file *file = t->view->index->log->head;
+	const uint8_t internal_flags =
+		MAIL_INDEX_MAIL_FLAG_BACKEND | MAIL_INDEX_MAIL_FLAG_DIRTY;
+	const struct mail_index_flag_update *u;
+	const unsigned int hdr_version =
+		MAIL_TRANSACTION_LOG_HDR_VERSION(&file->hdr);
+
+	if (!MAIL_TRANSACTION_LOG_VERSION_HAVE(hdr_version, HIDE_INTERNAL_MODSEQS)) {
+		/* this check can be a bit racy if the call isn't done while
+		   transaction log is locked. practically it won't matter
+		   now though. */
+		return array_count(&t->updates) > 0;
+	}
+
+	array_foreach(&t->updates, u) {
+		uint8_t changed_flags = u->add_flags | u->remove_flags;
+
+		if ((changed_flags & ~internal_flags) != 0)
+			return TRUE;
+	}
+	return FALSE;
+}
+
 uint64_t mail_index_transaction_get_highest_modseq(struct mail_index_transaction *t)
 {
 	struct mail_transaction_log_file *file = t->view->index->log->head;
@@ -565,7 +591,8 @@ uint64_t mail_index_transaction_get_highest_modseq(struct mail_index_transaction
 		/* sorting may change the order of keyword_updates,  */
 		new_highest_modseq++;
 	}
-	if (array_is_created(&t->updates) && array_count(&t->updates) > 0)
+	if (array_is_created(&t->updates) &&
+	    transaction_flag_updates_have_non_internal(t) > 0)
 		new_highest_modseq++;
 	if (array_is_created(&t->keyword_updates)) {
 		new_highest_modseq +=
diff --git a/src/lib-index/mail-transaction-log-append.c b/src/lib-index/mail-transaction-log-append.c
index d05ae5becb..04031a5b31 100644
--- a/src/lib-index/mail-transaction-log-append.c
+++ b/src/lib-index/mail-transaction-log-append.c
@@ -31,7 +31,8 @@ void mail_transaction_log_append_add(struct mail_transaction_log_append_ctx *ctx
 	buffer_append(ctx->output, &hdr, sizeof(hdr));
 	buffer_append(ctx->output, data, size);
 
-	mail_transaction_update_modseq(&hdr, data, &ctx->new_highest_modseq);
+	mail_transaction_update_modseq(&hdr, data, &ctx->new_highest_modseq,
+		MAIL_TRANSACTION_LOG_HDR_VERSION(&ctx->log->head->hdr));
 	ctx->transaction_count++;
 }
 
diff --git a/src/lib-index/mail-transaction-log-file.c b/src/lib-index/mail-transaction-log-file.c
index f388846b43..4395bb10e2 100644
--- a/src/lib-index/mail-transaction-log-file.c
+++ b/src/lib-index/mail-transaction-log-file.c
@@ -1007,8 +1007,31 @@ log_file_track_mailbox_sync_offset_hdr(struct mail_transaction_log_file *file,
 	return 0;
 }
 
+static bool
+flag_updates_have_non_internal(const struct mail_transaction_flag_update *u,
+			       unsigned int count, unsigned int version)
+{
+	const uint8_t internal_flags =
+		MAIL_INDEX_MAIL_FLAG_BACKEND | MAIL_INDEX_MAIL_FLAG_DIRTY;
+
+	/* Hide internal flags from modseqs if the log file's version
+	   is new enough. This allows upgrading without the modseqs suddenly
+	   shrinking. */
+	if (!MAIL_TRANSACTION_LOG_VERSION_HAVE(version, HIDE_INTERNAL_MODSEQS))
+		return TRUE;
+
+	for (unsigned int i = 0; i < count; i++) {
+		uint8_t changed_flags = u->add_flags | u->remove_flags;
+
+		if ((changed_flags & ~internal_flags) != 0)
+			return TRUE;
+	}
+	return FALSE;
+}
+
 void mail_transaction_update_modseq(const struct mail_transaction_header *hdr,
-				    const void *data, uint64_t *cur_modseq)
+				    const void *data, uint64_t *cur_modseq,
+				    unsigned int version)
 {
 	uint32_t trans_size;
 
@@ -1046,13 +1069,21 @@ void mail_transaction_update_modseq(const struct mail_transaction_header *hdr,
 			break;
 		}
 	case MAIL_TRANSACTION_APPEND:
-	case MAIL_TRANSACTION_FLAG_UPDATE:
 	case MAIL_TRANSACTION_KEYWORD_UPDATE:
 	case MAIL_TRANSACTION_KEYWORD_RESET:
 	case MAIL_TRANSACTION_ATTRIBUTE_UPDATE:
 		/* these changes increase modseq */
 		*cur_modseq += 1;
 		break;
+	case MAIL_TRANSACTION_FLAG_UPDATE: {
+		const struct mail_transaction_flag_update *rec = data;
+		unsigned int count;
+
+		count = (trans_size - sizeof(*hdr)) / sizeof(*rec);
+		if (flag_updates_have_non_internal(rec, count, version))
+			*cur_modseq += 1;
+		break;
+	}
 	case MAIL_TRANSACTION_MODSEQ_UPDATE: {
 		const struct mail_transaction_modseq_update *rec, *end;
 
@@ -1209,7 +1240,8 @@ int mail_transaction_log_file_get_highest_modseq_at(
 	while (cur_offset < offset) {
 		if (log_get_synced_record(file, &cur_offset, &hdr) < 0)
 			return- 1;
-		mail_transaction_update_modseq(hdr, hdr + 1, &cur_modseq);
+		mail_transaction_update_modseq(hdr, hdr + 1, &cur_modseq,
+			MAIL_TRANSACTION_LOG_HDR_VERSION(&file->hdr));
 	}
 
 	/* @UNSAFE: cache the value */
@@ -1280,7 +1312,8 @@ int mail_transaction_log_file_get_modseq_next_offset(
 	while (cur_offset < file->sync_offset) {
 		if (log_get_synced_record(file, &cur_offset, &hdr) < 0)
 			return -1;
-		mail_transaction_update_modseq(hdr, hdr + 1, &cur_modseq);
+		mail_transaction_update_modseq(hdr, hdr + 1, &cur_modseq,
+			MAIL_TRANSACTION_LOG_HDR_VERSION(&file->hdr));
 		if (cur_modseq >= modseq)
 			break;
 	}
@@ -1312,8 +1345,8 @@ log_file_track_sync(struct mail_transaction_log_file *file,
 	const void *data = hdr + 1;
 	int ret;
 
-	mail_transaction_update_modseq(hdr, hdr + 1,
-				       &file->sync_highest_modseq);
+	mail_transaction_update_modseq(hdr, hdr + 1, &file->sync_highest_modseq,
+		MAIL_TRANSACTION_LOG_HDR_VERSION(&file->hdr));
 	if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0)
 		return 1;
 
diff --git a/src/lib-index/mail-transaction-log-private.h b/src/lib-index/mail-transaction-log-private.h
index 74166a8771..cc6c1804bd 100644
--- a/src/lib-index/mail-transaction-log-private.h
+++ b/src/lib-index/mail-transaction-log-private.h
@@ -148,7 +148,8 @@ void mail_transaction_log_file_unlock(struct mail_transaction_log_file *file,
 				      const char *lock_reason);
 
 void mail_transaction_update_modseq(const struct mail_transaction_header *hdr,
-				    const void *data, uint64_t *cur_modseq);
+				    const void *data, uint64_t *cur_modseq,
+				    unsigned int version);
 int mail_transaction_log_file_get_highest_modseq_at(
 		struct mail_transaction_log_file *file,
 		uoff_t offset, uint64_t *highest_modseq_r);
diff --git a/src/lib-index/mail-transaction-log-view.c b/src/lib-index/mail-transaction-log-view.c
index 8f2cd3831e..4837b5eaea 100644
--- a/src/lib-index/mail-transaction-log-view.c
+++ b/src/lib-index/mail-transaction-log-view.c
@@ -792,7 +792,8 @@ log_view_get_next(struct mail_transaction_log_view *view,
 		ret = log_view_is_record_valid(file, hdr, data) ? 1 : -1;
 	} T_END;
 	if (ret > 0) {
-		mail_transaction_update_modseq(hdr, data, &view->prev_modseq);
+		mail_transaction_update_modseq(hdr, data, &view->prev_modseq,
+			MAIL_TRANSACTION_LOG_HDR_VERSION(&file->hdr));
 		*hdr_r = hdr;
 		*data_r = data;
 		view->cur_offset += full_size;
diff --git a/src/lib-index/mail-transaction-log.c b/src/lib-index/mail-transaction-log.c
index 61668c8510..2507bef22f 100644
--- a/src/lib-index/mail-transaction-log.c
+++ b/src/lib-index/mail-transaction-log.c
@@ -218,6 +218,13 @@ bool mail_transaction_log_want_rotate(struct mail_transaction_log *log)
 {
 	struct mail_transaction_log_file *file = log->head;
 
+	if (file->hdr.major_version < MAIL_TRANSACTION_LOG_MAJOR_VERSION ||
+	    (file->hdr.major_version == MAIL_TRANSACTION_LOG_MAJOR_VERSION &&
+	     file->hdr.minor_version < MAIL_TRANSACTION_LOG_MINOR_VERSION)) {
+		/* upgrade immediately to a new log file format */
+		return TRUE;
+	}
+
 	if (file->sync_offset > log->index->log_rotate_max_size) {
 		/* file is too large, definitely rotate */
 		return TRUE;
diff --git a/src/lib-index/mail-transaction-log.h b/src/lib-index/mail-transaction-log.h
index b180467480..41a635c17e 100644
--- a/src/lib-index/mail-transaction-log.h
+++ b/src/lib-index/mail-transaction-log.h
@@ -6,7 +6,7 @@
 #define MAIL_TRANSACTION_LOG_SUFFIX ".log"
 
 #define MAIL_TRANSACTION_LOG_MAJOR_VERSION 1
-#define MAIL_TRANSACTION_LOG_MINOR_VERSION 2
+#define MAIL_TRANSACTION_LOG_MINOR_VERSION 3
 #define MAIL_TRANSACTION_LOG_HEADER_MIN_SIZE 24
 
 #define MAIL_TRANSACTION_LOG_VERSION_FULL(major, minor) \
@@ -18,6 +18,8 @@
 
 #define MAIL_TRANSACTION_LOG_VERSION_COMPAT_FLAGS \
 	MAIL_TRANSACTION_LOG_VERSION_FULL(1, 2)
+#define MAIL_TRANSACTION_LOG_VERSION_HIDE_INTERNAL_MODSEQS \
+	MAIL_TRANSACTION_LOG_VERSION_FULL(1, 3)
 
 struct mail_transaction_log_header {
 	uint8_t major_version;
diff --git a/src/lib-index/test-mail-transaction-log-append.c b/src/lib-index/test-mail-transaction-log-append.c
index 5cf4c9bad5..310dd4b562 100644
--- a/src/lib-index/test-mail-transaction-log-append.c
+++ b/src/lib-index/test-mail-transaction-log-append.c
@@ -27,7 +27,8 @@ void mail_transaction_log_file_unlock(struct mail_transaction_log_file *file ATT
 
 void mail_transaction_update_modseq(const struct mail_transaction_header *hdr,
 				    const void *data ATTR_UNUSED,
-				    uint64_t *cur_modseq)
+				    uint64_t *cur_modseq,
+				    unsigned int version ATTR_UNUSED)
 {
 	if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) != 0)
 		*cur_modseq += 1;
diff --git a/src/lib-index/test-mail-transaction-log-view.c b/src/lib-index/test-mail-transaction-log-view.c
index 31bc96b3d3..c0ee81485b 100644
--- a/src/lib-index/test-mail-transaction-log-view.c
+++ b/src/lib-index/test-mail-transaction-log-view.c
@@ -51,7 +51,8 @@ int mail_transaction_log_file_get_highest_modseq_at(
 
 void mail_transaction_update_modseq(const struct mail_transaction_header *hdr ATTR_UNUSED,
 				    const void *data ATTR_UNUSED,
-				    uint64_t *cur_modseq)
+				    uint64_t *cur_modseq,
+				    unsigned int version ATTR_UNUSED)
 {
 	*cur_modseq += 1;
 }