lib-index: Don't increase modseq for backend/dirty flag changes

These flags are used only for internal changes and they shouldn't be
triggering any modseq changes.

To avoid modseqs from unexpectedly shrinking, the new modseq counting
behavior is enabled only for newly rotated transaction log files that have
a new minor_version.

diff --git a/src/doveadm/doveadm-dump-log.c b/src/doveadm/doveadm-dump-log.c
index e762225373a39267d8f64956b807aff1b187fab6..b2a6e15dea23d8572e2ed2d6f682ad5b6d502e4b 100644 (file)
--- a/src/doveadm/doveadm-dump-log.c
+++ b/src/doveadm/doveadm-dump-log.c
@@ -11,7 +11,8 @@
 
 static struct mail_transaction_ext_intro prev_intro;
 
-static void dump_hdr(struct istream *input, uint64_t *modseq_r)
+static void dump_hdr(struct istream *input, uint64_t *modseq_r,
+                    unsigned int *version_r)
 {
        struct mail_transaction_log_header hdr;
        const unsigned char *data;
@@ -42,6 +43,7 @@ static void dump_hdr(struct istream *input, uint64_t *modseq_r)
               (unsigned long long)hdr.initial_modseq);
        printf("compat flags = %x\n", hdr.compat_flags);
        *modseq_r = hdr.initial_modseq;
+       *version_r = MAIL_TRANSACTION_LOG_HDR_VERSION(&hdr);
 }
 
 static const char *log_record_type(unsigned int type)
@@ -464,7 +466,8 @@ static void log_record_print(const struct mail_transaction_header *hdr,
        }
 }
 
-static int dump_record(struct istream *input, uint64_t *modseq)
+static int dump_record(struct istream *input, uint64_t *modseq,
+                      unsigned int version)
 {
        struct mail_transaction_header hdr;
        unsigned int hdr_size;
@@ -505,7 +508,7 @@ static int dump_record(struct istream *input, uint64_t *modseq)
        }
 
        uint64_t prev_modseq = *modseq;
-       mail_transaction_update_modseq(&hdr, data, modseq);
+       mail_transaction_update_modseq(&hdr, data, modseq, version);
        if (*modseq > prev_modseq)
                printf(", modseq=%llu", (unsigned long long)*modseq);
        printf("\n");
@@ -519,13 +522,14 @@ static void cmd_dump_log(int argc ATTR_UNUSED, char *argv[])
 {
        struct istream *input;
        uint64_t modseq;
+       unsigned int version;
        int ret;
 
        input = i_stream_create_file(argv[1], (size_t)-1);
-       dump_hdr(input, &modseq);
+       dump_hdr(input, &modseq, &version);
        do {
                T_BEGIN {
-                       ret = dump_record(input, &modseq);
+                       ret = dump_record(input, &modseq, version);
                } T_END;
        } while (ret > 0);
        i_stream_unref(&input);

diff --git a/src/lib-index/mail-index-transaction-export.c b/src/lib-index/mail-index-transaction-export.c
index 9bdf1bb77971134dbbbd46846137dc42dd6fd47a..19d32d90ff3e50bc9ee8d95339d59bd724502c1d 100644 (file)
--- a/src/lib-index/mail-index-transaction-export.c
+++ b/src/lib-index/mail-index-transaction-export.c
@@ -536,6 +536,32 @@ mail_index_transaction_keywords_count_modseq_incs(struct mail_index_transaction
        return count;
 }
 
+static bool
+transaction_flag_updates_have_non_internal(struct mail_index_transaction *t)
+{
+       struct mail_transaction_log_file *file = t->view->index->log->head;
+       const uint8_t internal_flags =
+               MAIL_INDEX_MAIL_FLAG_BACKEND | MAIL_INDEX_MAIL_FLAG_DIRTY;
+       const struct mail_index_flag_update *u;
+       const unsigned int hdr_version =
+               MAIL_TRANSACTION_LOG_HDR_VERSION(&file->hdr);
+
+       if (!MAIL_TRANSACTION_LOG_VERSION_HAVE(hdr_version, HIDE_INTERNAL_MODSEQS)) {
+               /* this check can be a bit racy if the call isn't done while
+                  transaction log is locked. practically it won't matter
+                  now though. */
+               return array_count(&t->updates) > 0;
+       }
+
+       array_foreach(&t->updates, u) {
+               uint8_t changed_flags = u->add_flags | u->remove_flags;
+
+               if ((changed_flags & ~internal_flags) != 0)
+                       return TRUE;
+       }
+       return FALSE;
+}
+
 uint64_t mail_index_transaction_get_highest_modseq(struct mail_index_transaction *t)
 {
        struct mail_transaction_log_file *file = t->view->index->log->head;
@@ -565,7 +591,8 @@ uint64_t mail_index_transaction_get_highest_modseq(struct mail_index_transaction
                /* sorting may change the order of keyword_updates,  */
                new_highest_modseq++;
        }
-       if (array_is_created(&t->updates) && array_count(&t->updates) > 0)
+       if (array_is_created(&t->updates) &&
+           transaction_flag_updates_have_non_internal(t) > 0)
                new_highest_modseq++;
        if (array_is_created(&t->keyword_updates)) {
                new_highest_modseq +=

diff --git a/src/lib-index/mail-transaction-log-append.c b/src/lib-index/mail-transaction-log-append.c
index d05ae5becbdc9b139eb7f35a5140b8ac2aae9b53..04031a5b31c9ac24605af437d0f7ac18c71f5923 100644 (file)
--- a/src/lib-index/mail-transaction-log-append.c
+++ b/src/lib-index/mail-transaction-log-append.c
@@ -31,7 +31,8 @@ void mail_transaction_log_append_add(struct mail_transaction_log_append_ctx *ctx
        buffer_append(ctx->output, &hdr, sizeof(hdr));
        buffer_append(ctx->output, data, size);
 
-       mail_transaction_update_modseq(&hdr, data, &ctx->new_highest_modseq);
+       mail_transaction_update_modseq(&hdr, data, &ctx->new_highest_modseq,
+               MAIL_TRANSACTION_LOG_HDR_VERSION(&ctx->log->head->hdr));
        ctx->transaction_count++;
 }
 

diff --git a/src/lib-index/mail-transaction-log-file.c b/src/lib-index/mail-transaction-log-file.c
index f388846b43e36cdf152c817cc47e598677b3a729..4395bb10e28c2e49ef3dac19c0a2750d14b4f03f 100644 (file)
--- a/src/lib-index/mail-transaction-log-file.c
+++ b/src/lib-index/mail-transaction-log-file.c
@@ -1007,8 +1007,31 @@ log_file_track_mailbox_sync_offset_hdr(struct mail_transaction_log_file *file,
        return 0;
 }
 
+static bool
+flag_updates_have_non_internal(const struct mail_transaction_flag_update *u,
+                              unsigned int count, unsigned int version)
+{
+       const uint8_t internal_flags =
+               MAIL_INDEX_MAIL_FLAG_BACKEND | MAIL_INDEX_MAIL_FLAG_DIRTY;
+
+       /* Hide internal flags from modseqs if the log file's version
+          is new enough. This allows upgrading without the modseqs suddenly
+          shrinking. */
+       if (!MAIL_TRANSACTION_LOG_VERSION_HAVE(version, HIDE_INTERNAL_MODSEQS))
+               return TRUE;
+
+       for (unsigned int i = 0; i < count; i++) {
+               uint8_t changed_flags = u->add_flags | u->remove_flags;
+
+               if ((changed_flags & ~internal_flags) != 0)
+                       return TRUE;
+       }
+       return FALSE;
+}
+
 void mail_transaction_update_modseq(const struct mail_transaction_header *hdr,
-                                   const void *data, uint64_t *cur_modseq)
+                                   const void *data, uint64_t *cur_modseq,
+                                   unsigned int version)
 {
        uint32_t trans_size;
 
@@ -1046,13 +1069,21 @@ void mail_transaction_update_modseq(const struct mail_transaction_header *hdr,
                        break;
                }
        case MAIL_TRANSACTION_APPEND:
-       case MAIL_TRANSACTION_FLAG_UPDATE:
        case MAIL_TRANSACTION_KEYWORD_UPDATE:
        case MAIL_TRANSACTION_KEYWORD_RESET:
        case MAIL_TRANSACTION_ATTRIBUTE_UPDATE:
                /* these changes increase modseq */
                *cur_modseq += 1;
                break;
+       case MAIL_TRANSACTION_FLAG_UPDATE: {
+               const struct mail_transaction_flag_update *rec = data;
+               unsigned int count;
+
+               count = (trans_size - sizeof(*hdr)) / sizeof(*rec);
+               if (flag_updates_have_non_internal(rec, count, version))
+                       *cur_modseq += 1;
+               break;
+       }
        case MAIL_TRANSACTION_MODSEQ_UPDATE: {
                const struct mail_transaction_modseq_update *rec, *end;
 
@@ -1209,7 +1240,8 @@ int mail_transaction_log_file_get_highest_modseq_at(
        while (cur_offset < offset) {
                if (log_get_synced_record(file, &cur_offset, &hdr) < 0)
                        return- 1;
-               mail_transaction_update_modseq(hdr, hdr + 1, &cur_modseq);
+               mail_transaction_update_modseq(hdr, hdr + 1, &cur_modseq,
+                       MAIL_TRANSACTION_LOG_HDR_VERSION(&file->hdr));
        }
 
        /* @UNSAFE: cache the value */
@@ -1280,7 +1312,8 @@ int mail_transaction_log_file_get_modseq_next_offset(
        while (cur_offset < file->sync_offset) {
                if (log_get_synced_record(file, &cur_offset, &hdr) < 0)
                        return -1;
-               mail_transaction_update_modseq(hdr, hdr + 1, &cur_modseq);
+               mail_transaction_update_modseq(hdr, hdr + 1, &cur_modseq,
+                       MAIL_TRANSACTION_LOG_HDR_VERSION(&file->hdr));
                if (cur_modseq >= modseq)
                        break;
        }
@@ -1312,8 +1345,8 @@ log_file_track_sync(struct mail_transaction_log_file *file,
        const void *data = hdr + 1;
        int ret;
 
-       mail_transaction_update_modseq(hdr, hdr + 1,
-                                      &file->sync_highest_modseq);
+       mail_transaction_update_modseq(hdr, hdr + 1, &file->sync_highest_modseq,
+               MAIL_TRANSACTION_LOG_HDR_VERSION(&file->hdr));
        if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0)
                return 1;
 

diff --git a/src/lib-index/mail-transaction-log-private.h b/src/lib-index/mail-transaction-log-private.h
index 74166a877181a470437b4f2e28ec73c17058fb10..cc6c1804bdbefbaa277ff12508f0276f4e34f477 100644 (file)
--- a/src/lib-index/mail-transaction-log-private.h
+++ b/src/lib-index/mail-transaction-log-private.h
@@ -148,7 +148,8 @@ void mail_transaction_log_file_unlock(struct mail_transaction_log_file *file,
                                      const char *lock_reason);
 
 void mail_transaction_update_modseq(const struct mail_transaction_header *hdr,
-                                   const void *data, uint64_t *cur_modseq);
+                                   const void *data, uint64_t *cur_modseq,
+                                   unsigned int version);
 int mail_transaction_log_file_get_highest_modseq_at(
                struct mail_transaction_log_file *file,
                uoff_t offset, uint64_t *highest_modseq_r);

diff --git a/src/lib-index/mail-transaction-log-view.c b/src/lib-index/mail-transaction-log-view.c
index 8f2cd3831ee12f5cc163774f3c009dbadc54d254..4837b5eaea316b56b506eebeb3bb4037098bdb2e 100644 (file)
--- a/src/lib-index/mail-transaction-log-view.c
+++ b/src/lib-index/mail-transaction-log-view.c
@@ -792,7 +792,8 @@ log_view_get_next(struct mail_transaction_log_view *view,
                ret = log_view_is_record_valid(file, hdr, data) ? 1 : -1;
        } T_END;
        if (ret > 0) {
-               mail_transaction_update_modseq(hdr, data, &view->prev_modseq);
+               mail_transaction_update_modseq(hdr, data, &view->prev_modseq,
+                       MAIL_TRANSACTION_LOG_HDR_VERSION(&file->hdr));
                *hdr_r = hdr;
                *data_r = data;
                view->cur_offset += full_size;

diff --git a/src/lib-index/mail-transaction-log.c b/src/lib-index/mail-transaction-log.c
index 61668c85109bf4d1bf1376bf856fe11496f71a77..2507bef22f5de8667bdd63cd598292fb643e2afc 100644 (file)
--- a/src/lib-index/mail-transaction-log.c
+++ b/src/lib-index/mail-transaction-log.c
@@ -218,6 +218,13 @@ bool mail_transaction_log_want_rotate(struct mail_transaction_log *log)
 {
        struct mail_transaction_log_file *file = log->head;
 
+       if (file->hdr.major_version < MAIL_TRANSACTION_LOG_MAJOR_VERSION ||
+           (file->hdr.major_version == MAIL_TRANSACTION_LOG_MAJOR_VERSION &&
+            file->hdr.minor_version < MAIL_TRANSACTION_LOG_MINOR_VERSION)) {
+               /* upgrade immediately to a new log file format */
+               return TRUE;
+       }
+
        if (file->sync_offset > log->index->log_rotate_max_size) {
                /* file is too large, definitely rotate */
                return TRUE;

diff --git a/src/lib-index/mail-transaction-log.h b/src/lib-index/mail-transaction-log.h
index b18046748028bd41cd51952a9b13b421bc5641b7..41a635c17ea33416cb4f33efa393ecfe51cc5e31 100644 (file)
--- a/src/lib-index/mail-transaction-log.h
+++ b/src/lib-index/mail-transaction-log.h
@@ -6,7 +6,7 @@
 #define MAIL_TRANSACTION_LOG_SUFFIX ".log"
 
 #define MAIL_TRANSACTION_LOG_MAJOR_VERSION 1
-#define MAIL_TRANSACTION_LOG_MINOR_VERSION 2
+#define MAIL_TRANSACTION_LOG_MINOR_VERSION 3
 #define MAIL_TRANSACTION_LOG_HEADER_MIN_SIZE 24
 
 #define MAIL_TRANSACTION_LOG_VERSION_FULL(major, minor) \
@@ -18,6 +18,8 @@
 
 #define MAIL_TRANSACTION_LOG_VERSION_COMPAT_FLAGS \
        MAIL_TRANSACTION_LOG_VERSION_FULL(1, 2)
+#define MAIL_TRANSACTION_LOG_VERSION_HIDE_INTERNAL_MODSEQS \
+       MAIL_TRANSACTION_LOG_VERSION_FULL(1, 3)
 
 struct mail_transaction_log_header {
        uint8_t major_version;

diff --git a/src/lib-index/test-mail-transaction-log-append.c b/src/lib-index/test-mail-transaction-log-append.c
index 5cf4c9bad5b219d646ea81c7cbbb4cbce7f21f86..310dd4b5624fd46c39237efcbff934a8d3dc9f47 100644 (file)
--- a/src/lib-index/test-mail-transaction-log-append.c
+++ b/src/lib-index/test-mail-transaction-log-append.c
@@ -27,7 +27,8 @@ void mail_transaction_log_file_unlock(struct mail_transaction_log_file *file ATT
 
 void mail_transaction_update_modseq(const struct mail_transaction_header *hdr,
                                    const void *data ATTR_UNUSED,
-                                   uint64_t *cur_modseq)
+                                   uint64_t *cur_modseq,
+                                   unsigned int version ATTR_UNUSED)
 {
        if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) != 0)
                *cur_modseq += 1;

diff --git a/src/lib-index/test-mail-transaction-log-view.c b/src/lib-index/test-mail-transaction-log-view.c
index 31bc96b3d36ca5f801ef0ed19f202ff0f60217b8..c0ee81485b2b29d1ff511168d14d491ee0055629 100644 (file)
--- a/src/lib-index/test-mail-transaction-log-view.c
+++ b/src/lib-index/test-mail-transaction-log-view.c
@@ -51,7 +51,8 @@ int mail_transaction_log_file_get_highest_modseq_at(
 
 void mail_transaction_update_modseq(const struct mail_transaction_header *hdr ATTR_UNUSED,
                                    const void *data ATTR_UNUSED,
-                                   uint64_t *cur_modseq)
+                                   uint64_t *cur_modseq,
+                                   unsigned int version ATTR_UNUSED)
 {
        *cur_modseq += 1;
 }