lib-index: Don't increase modseq for backend/dirty flag changes

These flags are used only for internal changes and they shouldn't be
triggering any modseq changes.

To avoid modseqs from unexpectedly shrinking, the new modseq counting
behavior is enabled only for newly rotated transaction log files that have
a new minor_version.

diff --git a/src/doveadm/doveadm-dump-log.c b/src/doveadm/doveadm-dump-log.c
index e762225373a39267d8f64956b807aff1b187fab6..b2a6e15dea23d8572e2ed2d6f682ad5b6d502e4b 100644 (file)
--- a/src/doveadm/doveadm-dump-log.c
+++ b/src/doveadm/doveadm-dump-log.c
@@ -11,7 +11,8 @@
 
 static struct mail_transaction_ext_intro prev_intro;
 
-static void dump_hdr(struct istream *input, uint64_t *modseq_r)
+static void dump_hdr(struct istream *input, uint64_t *modseq_r,
+                    unsigned int *version_r)
 {
        struct mail_transaction_log_header hdr;
        const unsigned char *data;
@@ -42,6 +43,7 @@ static void dump_hdr(struct istream *input, uint64_t *modseq_r)
               (unsigned long long)hdr.initial_modseq);
        printf("compat flags = %x\n", hdr.compat_flags);
        *modseq_r = hdr.initial_modseq;
+       *version_r = MAIL_TRANSACTION_LOG_HDR_VERSION(&hdr);
 }
 
 static const char *log_record_type(unsigned int type)
@@ -464,7 +466,8 @@ static void log_record_print(const struct mail_transaction_header *hdr,
        }
 }
 
-static int dump_record(struct istream *input, uint64_t *modseq)
+static int dump_record(struct istream *input, uint64_t *modseq,
+                      unsigned int version)
 {
        struct mail_transaction_header hdr;
        unsigned int hdr_size;
@@ -505,7 +508,7 @@ static int dump_record(struct istream *input, uint64_t *modseq)
        }
 
        uint64_t prev_modseq = *modseq;
-       mail_transaction_update_modseq(&hdr, data, modseq);
+       mail_transaction_update_modseq(&hdr, data, modseq, version);
        if (*modseq > prev_modseq)
                printf(", modseq=%llu", (unsigned long long)*modseq);
        printf("\n");
@@ -519,13 +522,14 @@ static void cmd_dump_log(int argc ATTR_UNUSED, char *argv[])
 {
        struct istream *input;
        uint64_t modseq;
+       unsigned int version;
        int ret;
 
        input = i_stream_create_file(argv[1], (size_t)-1);
-       dump_hdr(input, &modseq);
+       dump_hdr(input, &modseq, &version);
        do {
                T_BEGIN {
-                       ret = dump_record(input, &modseq);
+                       ret = dump_record(input, &modseq, version);
                } T_END;
        } while (ret > 0);
        i_stream_unref(&input);

diff --git a/src/lib-index/mail-index-transaction-export.c b/src/lib-index/mail-index-transaction-export.c
index 328c42f31379ade997ecbab97995d20e600cb8cd..5e519a1b9bcb18cf6156b3dd87bfd87ce0c1025a 100644 (file)
--- a/src/lib-index/mail-index-transaction-export.c
+++ b/src/lib-index/mail-index-transaction-export.c
@@ -536,6 +536,32 @@ mail_index_transaction_keywords_count_modseq_incs(struct mail_index_transaction
        return count;
 }
 
+static bool
+transaction_flag_updates_have_non_internal(struct mail_index_transaction *t)
+{
+       struct mail_transaction_log_file *file = t->view->index->log->head;
+       const uint8_t internal_flags =
+               MAIL_INDEX_MAIL_FLAG_BACKEND | MAIL_INDEX_MAIL_FLAG_DIRTY;
+       const struct mail_index_flag_update *u;
+       const unsigned int hdr_version =
+               MAIL_TRANSACTION_LOG_HDR_VERSION(&file->hdr);
+
+       if (!MAIL_TRANSACTION_LOG_VERSION_HAVE(hdr_version, HIDE_INTERNAL_MODSEQS)) {
+               /* this check can be a bit racy if the call isn't done while
+                  transaction log is locked. practically it won't matter
+                  now though. */
+               return array_count(&t->updates) > 0;
+       }
+
+       array_foreach(&t->updates, u) {
+               uint8_t changed_flags = u->add_flags | u->remove_flags;
+
+               if ((changed_flags & ~internal_flags) != 0)
+                       return TRUE;
+       }
+       return FALSE;
+}
+
 uint64_t mail_index_transaction_get_highest_modseq(struct mail_index_transaction *t)
 {
        struct mail_transaction_log_file *file = t->view->index->log->head;
@@ -565,7 +591,8 @@ uint64_t mail_index_transaction_get_highest_modseq(struct mail_index_transaction
                /* sorting may change the order of keyword_updates,  */
                new_highest_modseq++;
        }
-       if (array_is_created(&t->updates) && array_count(&t->updates) > 0)
+       if (array_is_created(&t->updates) &&
+           transaction_flag_updates_have_non_internal(t) > 0)
                new_highest_modseq++;
        if (array_is_created(&t->keyword_updates)) {
                new_highest_modseq +=

diff --git a/src/lib-index/mail-transaction-log-append.c b/src/lib-index/mail-transaction-log-append.c
index d05ae5becbdc9b139eb7f35a5140b8ac2aae9b53..04031a5b31c9ac24605af437d0f7ac18c71f5923 100644 (file)
--- a/src/lib-index/mail-transaction-log-append.c
+++ b/src/lib-index/mail-transaction-log-append.c
@@ -31,7 +31,8 @@ void mail_transaction_log_append_add(struct mail_transaction_log_append_ctx *ctx
        buffer_append(ctx->output, &hdr, sizeof(hdr));
        buffer_append(ctx->output, data, size);
 
-       mail_transaction_update_modseq(&hdr, data, &ctx->new_highest_modseq);
+       mail_transaction_update_modseq(&hdr, data, &ctx->new_highest_modseq,
+               MAIL_TRANSACTION_LOG_HDR_VERSION(&ctx->log->head->hdr));
        ctx->transaction_count++;
 }
 

diff --git a/src/lib-index/mail-transaction-log-file.c b/src/lib-index/mail-transaction-log-file.c
index 9753b9806bd90b138955bce294f23a088195593e..bc66331e9fec9315f21f34d532885a8228161f0b 100644 (file)
--- a/src/lib-index/mail-transaction-log-file.c
+++ b/src/lib-index/mail-transaction-log-file.c
@@ -1007,8 +1007,31 @@ log_file_track_mailbox_sync_offset_hdr(struct mail_transaction_log_file *file,
        return 0;
 }
 
+static bool
+flag_updates_have_non_internal(const struct mail_transaction_flag_update *u,
+                              unsigned int count, unsigned int version)
+{
+       const uint8_t internal_flags =
+               MAIL_INDEX_MAIL_FLAG_BACKEND | MAIL_INDEX_MAIL_FLAG_DIRTY;
+
+       /* Hide internal flags from modseqs if the log file's version
+          is new enough. This allows upgrading without the modseqs suddenly
+          shrinking. */
+       if (!MAIL_TRANSACTION_LOG_VERSION_HAVE(version, HIDE_INTERNAL_MODSEQS))
+               return TRUE;
+
+       for (unsigned int i = 0; i < count; i++) {
+               uint8_t changed_flags = u->add_flags | u->remove_flags;
+
+               if ((changed_flags & ~internal_flags) != 0)
+                       return TRUE;
+       }
+       return FALSE;
+}
+
 void mail_transaction_update_modseq(const struct mail_transaction_header *hdr,
-                                   const void *data, uint64_t *cur_modseq)
+                                   const void *data, uint64_t *cur_modseq,
+                                   unsigned int version)
 {
        uint32_t trans_size;
 
@@ -1046,13 +1069,21 @@ void mail_transaction_update_modseq(const struct mail_transaction_header *hdr,
                        break;
                }
        case MAIL_TRANSACTION_APPEND:
-       case MAIL_TRANSACTION_FLAG_UPDATE:
        case MAIL_TRANSACTION_KEYWORD_UPDATE:
        case MAIL_TRANSACTION_KEYWORD_RESET:
        case MAIL_TRANSACTION_ATTRIBUTE_UPDATE:
                /* these changes increase modseq */
                *cur_modseq += 1;
                break;
+       case MAIL_TRANSACTION_FLAG_UPDATE: {
+               const struct mail_transaction_flag_update *rec = data;
+               unsigned int count;
+
+               count = (trans_size - sizeof(*hdr)) / sizeof(*rec);
+               if (flag_updates_have_non_internal(rec, count, version))
+                       *cur_modseq += 1;
+               break;
+       }
        case MAIL_TRANSACTION_MODSEQ_UPDATE: {
                const struct mail_transaction_modseq_update *rec, *end;
 
@@ -1209,7 +1240,8 @@ int mail_transaction_log_file_get_highest_modseq_at(
        while (cur_offset < offset) {
                if (log_get_synced_record(file, &cur_offset, &hdr) < 0)
                        return- 1;
-               mail_transaction_update_modseq(hdr, hdr + 1, &cur_modseq);
+               mail_transaction_update_modseq(hdr, hdr + 1, &cur_modseq,
+                       MAIL_TRANSACTION_LOG_HDR_VERSION(&file->hdr));
        }
 
        /* @UNSAFE: cache the value */
@@ -1280,7 +1312,8 @@ int mail_transaction_log_file_get_modseq_next_offset(
        while (cur_offset < file->sync_offset) {
                if (log_get_synced_record(file, &cur_offset, &hdr) < 0)
                        return -1;
-               mail_transaction_update_modseq(hdr, hdr + 1, &cur_modseq);
+               mail_transaction_update_modseq(hdr, hdr + 1, &cur_modseq,
+                       MAIL_TRANSACTION_LOG_HDR_VERSION(&file->hdr));
                if (cur_modseq >= modseq)
                        break;
        }
@@ -1312,8 +1345,8 @@ log_file_track_sync(struct mail_transaction_log_file *file,
        const void *data = hdr + 1;
        int ret;
 
-       mail_transaction_update_modseq(hdr, hdr + 1,
-                                      &file->sync_highest_modseq);
+       mail_transaction_update_modseq(hdr, hdr + 1, &file->sync_highest_modseq,
+               MAIL_TRANSACTION_LOG_HDR_VERSION(&file->hdr));
        if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0)
                return 1;
 

diff --git a/src/lib-index/mail-transaction-log-private.h b/src/lib-index/mail-transaction-log-private.h
index b002d687bb99a5faef8e4c21a0e6cf0c8d586951..18b9d00e5b952a94085e7eeff3e52b04af6c9c49 100644 (file)
--- a/src/lib-index/mail-transaction-log-private.h
+++ b/src/lib-index/mail-transaction-log-private.h
@@ -148,7 +148,8 @@ void mail_transaction_log_file_unlock(struct mail_transaction_log_file *file,
                                      const char *lock_reason);
 
 void mail_transaction_update_modseq(const struct mail_transaction_header *hdr,
-                                   const void *data, uint64_t *cur_modseq);
+                                   const void *data, uint64_t *cur_modseq,
+                                   unsigned int version);
 int mail_transaction_log_file_get_highest_modseq_at(
                struct mail_transaction_log_file *file,
                uoff_t offset, uint64_t *highest_modseq_r);

diff --git a/src/lib-index/mail-transaction-log-view.c b/src/lib-index/mail-transaction-log-view.c
index 2ce9b4f7b5f685bd0ca612f77eeab29e3623e8be..c2d7c14f8df4d78c0af41a009374f282b045623c 100644 (file)
--- a/src/lib-index/mail-transaction-log-view.c
+++ b/src/lib-index/mail-transaction-log-view.c
@@ -792,7 +792,8 @@ log_view_get_next(struct mail_transaction_log_view *view,
                ret = log_view_is_record_valid(file, hdr, data) ? 1 : -1;
        } T_END;
        if (ret > 0) {
-               mail_transaction_update_modseq(hdr, data, &view->prev_modseq);
+               mail_transaction_update_modseq(hdr, data, &view->prev_modseq,
+                       MAIL_TRANSACTION_LOG_HDR_VERSION(&file->hdr));
                *hdr_r = hdr;
                *data_r = data;
                view->cur_offset += full_size;

diff --git a/src/lib-index/mail-transaction-log.c b/src/lib-index/mail-transaction-log.c
index 61668c85109bf4d1bf1376bf856fe11496f71a77..2507bef22f5de8667bdd63cd598292fb643e2afc 100644 (file)
--- a/src/lib-index/mail-transaction-log.c
+++ b/src/lib-index/mail-transaction-log.c
@@ -218,6 +218,13 @@ bool mail_transaction_log_want_rotate(struct mail_transaction_log *log)
 {
        struct mail_transaction_log_file *file = log->head;
 
+       if (file->hdr.major_version < MAIL_TRANSACTION_LOG_MAJOR_VERSION ||
+           (file->hdr.major_version == MAIL_TRANSACTION_LOG_MAJOR_VERSION &&
+            file->hdr.minor_version < MAIL_TRANSACTION_LOG_MINOR_VERSION)) {
+               /* upgrade immediately to a new log file format */
+               return TRUE;
+       }
+
        if (file->sync_offset > log->index->log_rotate_max_size) {
                /* file is too large, definitely rotate */
                return TRUE;

diff --git a/src/lib-index/mail-transaction-log.h b/src/lib-index/mail-transaction-log.h
index 8bdca3929aa782828acaeaca325bbf46ba2260bb..794f7c8df38d5e26460ad27c570f353d002765fe 100644 (file)
--- a/src/lib-index/mail-transaction-log.h
+++ b/src/lib-index/mail-transaction-log.h
@@ -6,7 +6,7 @@
 #define MAIL_TRANSACTION_LOG_SUFFIX ".log"
 
 #define MAIL_TRANSACTION_LOG_MAJOR_VERSION 1
-#define MAIL_TRANSACTION_LOG_MINOR_VERSION 2
+#define MAIL_TRANSACTION_LOG_MINOR_VERSION 3
 #define MAIL_TRANSACTION_LOG_HEADER_MIN_SIZE 24
 
 #define MAIL_TRANSACTION_LOG_VERSION_FULL(major, minor) \
@@ -18,6 +18,8 @@
 
 #define MAIL_TRANSACTION_LOG_VERSION_COMPAT_FLAGS \
        MAIL_TRANSACTION_LOG_VERSION_FULL(1, 2)
+#define MAIL_TRANSACTION_LOG_VERSION_HIDE_INTERNAL_MODSEQS \
+       MAIL_TRANSACTION_LOG_VERSION_FULL(1, 3)
 
 struct mail_transaction_log_header {
        uint8_t major_version;

diff --git a/src/lib-index/test-mail-transaction-log-append.c b/src/lib-index/test-mail-transaction-log-append.c
index 4cade880136e3f15afa0b1863bf12af5750c2f6c..e950d8ff9a41bc1a1945a7dac55f799fa21845ff 100644 (file)
--- a/src/lib-index/test-mail-transaction-log-append.c
+++ b/src/lib-index/test-mail-transaction-log-append.c
@@ -27,7 +27,8 @@ void mail_transaction_log_file_unlock(struct mail_transaction_log_file *file ATT
 
 void mail_transaction_update_modseq(const struct mail_transaction_header *hdr,
                                    const void *data ATTR_UNUSED,
-                                   uint64_t *cur_modseq)
+                                   uint64_t *cur_modseq,
+                                   unsigned int version ATTR_UNUSED)
 {
        if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) != 0)
                *cur_modseq += 1;

diff --git a/src/lib-index/test-mail-transaction-log-view.c b/src/lib-index/test-mail-transaction-log-view.c
index 3507c8bc31294eb8994809124dc05602406e1d85..5fbc2f5fb06b50464ab82609de0d10409d8d65d0 100644 (file)
--- a/src/lib-index/test-mail-transaction-log-view.c
+++ b/src/lib-index/test-mail-transaction-log-view.c
@@ -51,7 +51,8 @@ int mail_transaction_log_file_get_highest_modseq_at(
 
 void mail_transaction_update_modseq(const struct mail_transaction_header *hdr ATTR_UNUSED,
                                    const void *data ATTR_UNUSED,
-                                   uint64_t *cur_modseq)
+                                   uint64_t *cur_modseq,
+                                   unsigned int version ATTR_UNUSED)
 {
        *cur_modseq += 1;
 }