lib-index: Don't update log_file_tail_offset unnecessarily.

Update it only if we're already writing to transaction log anyway or if
we're required to update the offset because mail_index_sync_commit() has
increased it past non-external transactions (this is especially important
with mdbox map index).

diff --git a/src/lib-index/mail-index-sync.c b/src/lib-index/mail-index-sync.c
index b5263b6a7d4e707ceead1737234b83be6348e7da..42a9b1388d01997edb5eaf6398bdc1974d33e345 100644 (file)
--- a/src/lib-index/mail-index-sync.c
+++ b/src/lib-index/mail-index-sync.c
@@ -751,9 +751,12 @@ mail_index_sync_update_mailbox_offset(struct mail_index_sync_ctx *ctx)
        mail_transaction_log_set_mailbox_sync_pos(ctx->index->log, seq, offset);
 
        /* If tail offset has changed, make sure it gets written to
-          transaction log. */
-       if (ctx->last_tail_offset != offset)
+          transaction log. do this only if we're required to changes. */
+       if (ctx->last_tail_seq != seq ||
+           ctx->last_tail_offset < offset) {
                ctx->ext_trans->log_updates = TRUE;
+               ctx->ext_trans->tail_offset_changed = TRUE;
+       }
 }
 
 static bool mail_index_sync_want_index_write(struct mail_index *index)

diff --git a/src/lib-index/mail-index-transaction-export.c b/src/lib-index/mail-index-transaction-export.c
index 2ad856595012a19d2c349ce60395b2e23ab51336..fd628eb3166eff9b08291401207d9583db4920ef 100644 (file)
--- a/src/lib-index/mail-index-transaction-export.c
+++ b/src/lib-index/mail-index-transaction-export.c
@@ -476,12 +476,8 @@ void mail_index_transaction_export(struct mail_index_transaction *t,
                                                &null4, 4);
        }
 
-       /* Update the tail offsets only when committing the sync transaction.
-          Other transactions may not know the latest tail offset and might
-          end up shrinking it. (Alternatively the shrinking tail offsets could
-          just be ignored, which would probably work fine too.) */
-       append_ctx->append_sync_offset = t->sync_transaction;
-
+       append_ctx->index_sync_transaction = t->sync_transaction;
+       append_ctx->tail_offset_changed = t->tail_offset_changed;
        append_ctx->want_fsync =
                (t->view->index->fsync_mask & change_mask) != 0 ||
                (t->flags & MAIL_INDEX_TRANSACTION_FLAG_FSYNC) != 0;

diff --git a/src/lib-index/mail-index-transaction-private.h b/src/lib-index/mail-index-transaction-private.h
index 421c7ff6ba6412b6631e81f07b3395ee8fa21da6..8b51e7f6557420d841abbce3d32e129f9873da74 100644 (file)
--- a/src/lib-index/mail-index-transaction-private.h
+++ b/src/lib-index/mail-index-transaction-private.h
@@ -93,6 +93,7 @@ struct mail_index_transaction {
        unsigned int index_deleted:1;
        unsigned int index_undeleted:1;
        unsigned int commit_deleted_index:1;
+       unsigned int tail_offset_changed:1;
        /* non-extension updates. flag updates don't change this because
           they may be added and removed, so be sure to check that the updates
           array is non-empty also. */

diff --git a/src/lib-index/mail-index-transaction-update.c b/src/lib-index/mail-index-transaction-update.c
index 354829d3a425649641ff0dcccef895ca5b00a576..1b14d149ef31094ce25da9d99656d510f2d58e28 100644 (file)
--- a/src/lib-index/mail-index-transaction-update.c
+++ b/src/lib-index/mail-index-transaction-update.c
@@ -101,6 +101,7 @@ void mail_index_transaction_reset_v(struct mail_index_transaction *t)
        t->index_undeleted = FALSE;
        t->log_updates = FALSE;
        t->log_ext_updates = FALSE;
+       t->tail_offset_changed = FALSE;
 }
 
 void mail_index_transaction_set_log_updates(struct mail_index_transaction *t)

diff --git a/src/lib-index/mail-transaction-log-append.c b/src/lib-index/mail-transaction-log-append.c
index aeb2fc53a385ec28403ef99f2a7133d2c9e26de7..20eb6b0e2d62d3fe3481f770ce24a8688fe6c3c2 100644 (file)
--- a/src/lib-index/mail-transaction-log-append.c
+++ b/src/lib-index/mail-transaction-log-append.c
@@ -122,9 +122,22 @@ log_append_sync_offset_if_needed(struct mail_transaction_log_append_ctx *ctx)
        buffer_t buf;
        unsigned char update_data[sizeof(*u) + sizeof(offset)];
 
-       if (file->max_tail_offset == file->sync_offset) {
-               if (ctx->output->used == 0 &&
-                   file->saved_tail_offset == file->max_tail_offset) {
+       if (!ctx->index_sync_transaction) {
+               /* this is a non-syncing transaction. update the tail offset
+                  only if we're already writing something else to transaction
+                  log anyway. */
+               i_assert(!ctx->tail_offset_changed);
+               /* FIXME: For now we never do this update, because it would
+                  cause errors about shrinking tail offsets with old Dovecot
+                  versions. This is anyway just an optimization, so it doesn't
+                  matter all that much if we don't do it here. Finish this
+                  in v2.3. */
+               /*if (ctx->output->used == 0)*/
+                       return;
+       } else if (file->max_tail_offset == file->sync_offset) {
+               /* we're synced all the way to tail offset, so this sync
+                  transaction can also be included in the same tail offset. */
+               if (ctx->output->used == 0 && !ctx->tail_offset_changed) {
                        /* nothing to write here after all (e.g. all unchanged
                           flag updates were dropped by export) */
                        return;
@@ -137,6 +150,10 @@ log_append_sync_offset_if_needed(struct mail_transaction_log_append_ctx *ctx)
                file->max_tail_offset += ctx->output->used +
                        sizeof(*hdr) + sizeof(*u) + sizeof(offset);
                ctx->sync_includes_this = TRUE;
+       } else {
+               /* This is a syncing transaction. Since we're finishing a sync,
+                  we may need to update the tail offset even if we don't have
+                  anything else to do. */
        }
        offset = file->max_tail_offset;
 
@@ -189,9 +206,7 @@ mail_transaction_log_append_locked(struct mail_transaction_log_append_ctx *ctx)
                buffer_delete(ctx->output, 0, boundary_size);
        }
 
-       if (ctx->append_sync_offset)
-               log_append_sync_offset_if_needed(ctx);
-
+       log_append_sync_offset_if_needed(ctx);
        if (log_buffer_write(ctx) < 0)
                return -1;
        file->sync_highest_modseq = ctx->new_highest_modseq;

diff --git a/src/lib-index/mail-transaction-log-file.c b/src/lib-index/mail-transaction-log-file.c
index f32b735ac6d2874766fe0228613322ff6b871e65..161d0665a63abf70bfa34a4704c5de13ff62b6db 100644 (file)
--- a/src/lib-index/mail-transaction-log-file.c
+++ b/src/lib-index/mail-transaction-log-file.c
@@ -957,20 +957,8 @@ log_file_track_mailbox_sync_offset_hdr(struct mail_transaction_log_file *file,
                       sizeof(tail_offset));
 
                if (tail_offset < file->saved_tail_offset) {
-                       if (file->sync_offset < file->saved_tail_sync_offset) {
-                               /* saved_tail_offset was already set in header,
-                                  but we still had to resync the file to find
-                                  modseqs. ignore this record. */
-                               return 1;
-                       }
-                       mail_index_set_error(file->log->index,
-                               "Transaction log file %s seq %u: "
-                               "log_file_tail_offset update shrank it "
-                               "(%u vs %"PRIuUOFF_T" "
-                               "sync_offset=%"PRIuUOFF_T")",
-                               file->filepath, file->hdr.file_seq,
-                               tail_offset, file->saved_tail_offset,
-                               file->sync_offset);
+                       /* ignore shrinking tail offsets */
+                       return 1;
                } else {
                        file->saved_tail_offset = tail_offset;
                        if (tail_offset > file->max_tail_offset)

diff --git a/src/lib-index/mail-transaction-log.h b/src/lib-index/mail-transaction-log.h
index 69e9b7139d46221569b8789de933d0bda59e6df6..96f3fd278ec7e220a1818630773210fe2d6ec63c 100644 (file)
--- a/src/lib-index/mail-transaction-log.h
+++ b/src/lib-index/mail-transaction-log.h
@@ -176,7 +176,10 @@ struct mail_transaction_log_append_ctx {
        uint64_t new_highest_modseq;
        unsigned int transaction_count;
 
-       unsigned int append_sync_offset:1;
+       /* same as mail_index_transaction->sync_transaction */
+       unsigned int index_sync_transaction:1;
+       /* same as mail_index_transaction->tail_offset_changed */
+       unsigned int tail_offset_changed:1;
        unsigned int sync_includes_this:1;
        unsigned int want_fsync:1;
 };

diff --git a/src/lib-index/test-mail-transaction-log-append.c b/src/lib-index/test-mail-transaction-log-append.c
index 9d10d649b4f56c37ca680dc2246f8b2ac94727e2..4e29a9aa831eab8a7c5eae387b449efe0163221b 100644 (file)
--- a/src/lib-index/test-mail-transaction-log-append.c
+++ b/src/lib-index/test-mail-transaction-log-append.c
@@ -97,7 +97,7 @@ static void test_append_sync_offset(struct mail_transaction_log *log)
 
        test_begin("transaction log append: append_sync_offset only");
        test_assert(mail_transaction_log_append_begin(log->index, 0, &ctx) == 0);
-       ctx->append_sync_offset = TRUE;
+       ctx->index_sync_transaction = TRUE;
        file->max_tail_offset = 123;
        test_assert(mail_transaction_log_append_commit(&ctx) == 0);