View syncing: Keep track of highest modseq in views. If we lose transaction

log, return all messages with higher modseq than the old highest-modseq as
having changed. This works better than the previous code which compared the
old modseq which may have already changed after the previous view sync.
Return changes based on modseq being larger than

--HG--
branch : HEAD

diff --git a/src/lib-index/mail-index-modseq.c b/src/lib-index/mail-index-modseq.c
index c051e38bacc9176236f15fb95c0f01d5289a3aa3..ed74d84df5b237d5d9cb05208dca1887a1a52907 100644 (file)
--- a/src/lib-index/mail-index-modseq.c
+++ b/src/lib-index/mail-index-modseq.c
@@ -114,8 +114,10 @@ uint64_t mail_index_map_modseq_get_highest(struct mail_index_map *map)
        if (modseq_hdr != NULL && modseq_hdr->highest_modseq != 0)
                return modseq_hdr->highest_modseq;
        else {
-               /* fallback to returning the log head */
-               return mail_index_modseq_get_head(map->index);
+               /* fallback to returning the log head. if modseqs aren't
+                  enabled, we return 0. */
+               return map->index->log->head == NULL ? 0 :
+                       map->index->log->head->sync_highest_modseq;
        }
 }
 

diff --git a/src/lib-index/mail-index-view-private.h b/src/lib-index/mail-index-view-private.h
index 8102eeef057b77586cb3a7414815ef70a7bb0e9d..af27c24986901d13e4ec2bd43b6ba1eda87ddb35 100644 (file)
--- a/src/lib-index/mail-index-view-private.h
+++ b/src/lib-index/mail-index-view-private.h
@@ -50,6 +50,7 @@ struct mail_index_view {
 
        uint32_t indexid;
        unsigned int inconsistency_id;
+       uint64_t highest_modseq;
 
        struct mail_index_map *map;
        /* All mappings where we have returned records. They need to be kept

diff --git a/src/lib-index/mail-index-view-sync.c b/src/lib-index/mail-index-view-sync.c
index a2d3e9369ee9205e2701fca318ecc3e7b975b038..ac1e60039224692461a6a838a229f0405f09e56f 100644 (file)
--- a/src/lib-index/mail-index-view-sync.c
+++ b/src/lib-index/mail-index-view-sync.c
@@ -27,7 +27,7 @@ struct mail_index_view_sync_ctx {
        /* temporary variables while handling lost transaction logs: */
        ARRAY_TYPE(keyword_indexes) lost_old_kw, lost_new_kw;
        buffer_t *lost_kw_buf;
-       uint32_t lost_old_ext_idx, lost_new_ext_idx;
+       uint32_t lost_new_ext_idx;
        /* result of lost transaction logs: */
        ARRAY_TYPE(seq_range) lost_flags;
        unsigned int lost_flag_idx;
@@ -332,7 +332,7 @@ static int view_sync_apply_lost_changes(struct mail_index_view_sync_ctx *ctx,
        struct mail_transaction_header thdr;
        const struct mail_index_ext *ext;
        const uint64_t *modseqp;
-       uint64_t old_modseq, new_modseq;
+       uint64_t new_modseq;
        bool changed = FALSE;
 
        old_rec = MAIL_INDEX_MAP_IDX(old_map, old_seq - 1);
@@ -383,24 +383,20 @@ static int view_sync_apply_lost_changes(struct mail_index_view_sync_ctx *ctx,
 
        if (changed) {
                /* flags or keywords changed */
-       } else if (ctx->lost_old_ext_idx != (uint32_t)-1 &&
+       } else if (ctx->view->highest_modseq != 0 &&
                   ctx->lost_new_ext_idx != (uint32_t)-1) {
                /* if modseq has changed include this message in changed flags
                   list, even if we didn't see any changes above. */
-               ext = array_idx(&old_map->extensions, ctx->lost_old_ext_idx);
-               modseqp = CONST_PTR_OFFSET(old_rec, ext->record_offset);
-               old_modseq = *modseqp;
-
                ext = array_idx(&new_map->extensions, ctx->lost_new_ext_idx);
                modseqp = CONST_PTR_OFFSET(new_rec, ext->record_offset);
                new_modseq = *modseqp;
 
-               if (old_modseq != new_modseq)
+               if (new_modseq > ctx->view->highest_modseq)
                        changed = TRUE;
        }
 
-       /* lost_flags isn't updated perfectly correctly, because by the time
-          we're comparing old flags (or modseqs) it may have changed from what
+       /* without modseqs lost_flags isn't updated perfectly correctly, because
+          by the time we're comparing old flags it may have changed from what
           we last sent to the client (because the map is shared). This could
           be avoided by always keeping a private copy of the map in the view,
           but that's a waste of memory for as rare of a problem as this. */
@@ -427,9 +423,6 @@ view_sync_get_log_lost_changes(struct mail_index_view_sync_ctx *ctx,
           want. get an atomic map to make sure these get removed. */
        (void)mail_index_sync_get_atomic_map(&ctx->sync_map_ctx);
 
-       if (!mail_index_map_get_ext_idx(old_map, view->index->modseq_ext_id,
-                                       &ctx->lost_old_ext_idx))
-               ctx->lost_old_ext_idx = (uint32_t)-1;
        if (!mail_index_map_get_ext_idx(new_map, view->index->modseq_ext_id,
                                        &ctx->lost_new_ext_idx))
                ctx->lost_new_ext_idx = (uint32_t)-1;
@@ -958,6 +951,7 @@ int mail_index_view_sync_commit(struct mail_index_view_sync_ctx **_ctx,
        if (array_is_created(&ctx->lost_flags))
                array_free(&ctx->lost_flags);
 
+       view->highest_modseq = mail_index_map_modseq_get_highest(view->map);
        view->syncing = FALSE;
        i_free(ctx);
        return ret;

diff --git a/src/lib-index/mail-transaction-log-file.c b/src/lib-index/mail-transaction-log-file.c
index 1df482e770acfd4c95a90d86476a7edc64e1856b..d03fc77805df4ec397fb4e5f8bb9b7202cae6457 100644 (file)
--- a/src/lib-index/mail-transaction-log-file.c
+++ b/src/lib-index/mail-transaction-log-file.c
@@ -196,8 +196,7 @@ mail_transaction_log_init_hdr(struct mail_transaction_log *log,
                hdr->prev_file_seq = index->map->hdr.log_file_seq;
                hdr->prev_file_offset = index->map->hdr.log_file_head_offset;
                hdr->file_seq = index->map->hdr.log_file_seq + 1;
-               hdr->initial_modseq = log->head == NULL ||
-                       log->head->sync_highest_modseq == 0 ? 0 :
+               hdr->initial_modseq =
                        mail_index_map_modseq_get_highest(index->map);
        } else {
                hdr->file_seq = 1;

diff --git a/src/lib-storage/index/index-status.c b/src/lib-storage/index/index-status.c
index dde9cbb5ed52da1a1d572cf630cd86683ce5a1ac..57db9bf1ac419e4cc789f1f0188f006a56461e77 100644 (file)
--- a/src/lib-storage/index/index-status.c
+++ b/src/lib-storage/index/index-status.c
@@ -30,6 +30,10 @@ void index_storage_get_status(struct mailbox *box,
        if ((items & STATUS_HIGHESTMODSEQ) != 0) {
                status_r->highest_modseq =
                        mail_index_modseq_get_highest(ibox->view);
+               if (status_r->highest_modseq == 0) {
+                       /* modseqs not enabled yet, but we can't return 0 */
+                       status_r->highest_modseq = 1;
+               }
        }
 
        if (items & STATUS_FIRST_UNSEEN_SEQ) {