Keep modseqs as 1 until the first modseq ext intro record enables them.

Performance should be better again when modseqs are disabled.

--HG--
branch : HEAD

diff --git a/src/lib-index/mail-index-modseq.c b/src/lib-index/mail-index-modseq.c
index acc0bd8303d2f49b8b5ae79213069a4b23811235..c051e38bacc9176236f15fb95c0f01d5289a3aa3 100644 (file)
--- a/src/lib-index/mail-index-modseq.c
+++ b/src/lib-index/mail-index-modseq.c
@@ -7,8 +7,6 @@
 #include "mail-index-sync-private.h"
 #include "mail-index-modseq.h"
 
-#define MAIL_INDEX_MODSEQ_EXT_NAME "modseq"
-
 ARRAY_DEFINE_TYPE(modseqs, uint64_t);
 
 enum modseq_metadata_idx {
@@ -51,7 +49,7 @@ void mail_index_modseq_init(struct mail_index *index)
 static uint64_t mail_index_modseq_get_head(struct mail_index *index)
 {
        return index->log->head == NULL ? 1 :
-               index->log->head->sync_highest_modseq;
+               I_MAX(index->log->head->sync_highest_modseq, 1);
 }
 
 void mail_index_modseq_enable(struct mail_index *index)

diff --git a/src/lib-index/mail-index-modseq.h b/src/lib-index/mail-index-modseq.h
index a1e9faa7cd5dc1fd2903a857b380cccec38ae5ea..58151171067838344d3eed6d7743ad2e938d7c30 100644 (file)
--- a/src/lib-index/mail-index-modseq.h
+++ b/src/lib-index/mail-index-modseq.h
@@ -1,6 +1,8 @@
 #ifndef MAIL_INDEX_MODSEQ_H
 #define MAIL_INDEX_MODSEQ_H
 
+#define MAIL_INDEX_MODSEQ_EXT_NAME "modseq"
+
 enum mail_flags;
 struct mail_keywords;
 struct mail_index;

diff --git a/src/lib-index/mail-transaction-log-append.c b/src/lib-index/mail-transaction-log-append.c
index e69f84ba2c6aac7639804693406394aeac2c7a5b..83184d86e012e190d185b4ab308061d7c448559a 100644 (file)
--- a/src/lib-index/mail-transaction-log-append.c
+++ b/src/lib-index/mail-transaction-log-append.c
@@ -6,6 +6,7 @@
 #include "write-full.h"
 #include "mail-index-private.h"
 #include "mail-index-view-private.h"
+#include "mail-index-modseq.h"
 #include "mail-index-transaction-private.h"
 #include "mail-transaction-log-private.h"
 
@@ -14,7 +15,7 @@ struct log_append_context {
        struct mail_index_transaction *trans;
        buffer_t *output;
 
-       unsigned int modseq_change_count;
+       uint64_t modseq;
        uint32_t first_append_size;
        bool sync_includes_this;
 };
@@ -25,6 +26,7 @@ static void log_append_buffer(struct log_append_context *ctx,
 {
        struct mail_transaction_header hdr;
        uint32_t hdr_size;
+       size_t hdr_pos;
 
        i_assert((type & MAIL_TRANSACTION_TYPE_MASK) != 0);
        i_assert((buf->used % 4) == 0);
@@ -39,26 +41,31 @@ static void log_append_buffer(struct log_append_context *ctx,
                hdr.type |= MAIL_TRANSACTION_EXPUNGE_PROT;
        if ((ctx->trans->flags & MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL) != 0)
                hdr.type |= MAIL_TRANSACTION_EXTERNAL;
+       hdr.size = sizeof(hdr) + buf->used +
+               (hdr_buf == NULL ? 0 : hdr_buf->used);
 
-       if (mail_transaction_header_has_modseq(&hdr))
-               ctx->modseq_change_count++;
+       hdr_pos = ctx->output->used;
+       buffer_append(ctx->output, &hdr, sizeof(hdr));
+       if (hdr_buf != NULL)
+               buffer_append(ctx->output, hdr_buf->data, hdr_buf->used);
+       buffer_append(ctx->output, buf->data, buf->used);
 
-       hdr_size = mail_index_uint32_to_offset(sizeof(hdr) + buf->used +
-                                              (hdr_buf == NULL ? 0 :
-                                               hdr_buf->used));
+       if (mail_transaction_header_has_modseq(buf->data,
+                       CONST_PTR_OFFSET(buf->data, sizeof(hdr)), ctx->modseq))
+               ctx->modseq++;
+
+       /* update the size */
+       hdr_size = mail_index_uint32_to_offset(hdr.size);
        if (!MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(ctx->file) &&
            ctx->first_append_size == 0) {
                /* size will be written later once everything
                   is in disk */
                ctx->first_append_size = hdr_size;
+               hdr.size = 0;
        } else {
                hdr.size = hdr_size;
        }
-
-       buffer_append(ctx->output, &hdr, sizeof(hdr));
-       if (hdr_buf != NULL)
-               buffer_append(ctx->output, hdr_buf->data, hdr_buf->used);
-       buffer_append(ctx->output, buf->data, buf->used);
+       buffer_write(ctx->output, hdr_pos, &hdr, sizeof(hdr));
 }
 
 static int log_buffer_move_to_memory(struct log_append_context *ctx)
@@ -280,10 +287,15 @@ static void log_append_ext_intro(struct log_append_context *ctx,
                /* new extension, reset_id defaults to 0 */
        }
        buffer_append(buf, rext->name, intro->name_size);
-
        if ((buf->used % 4) != 0)
                buffer_append_zero(buf, 4 - (buf->used % 4));
 
+       if (ctx->file->sync_highest_modseq == 0 &&
+           strcmp(rext->name, MAIL_INDEX_MODSEQ_EXT_NAME) == 0) {
+               /* modseq tracking started */
+               ctx->file->sync_highest_modseq = 1;
+       }
+
        log_append_buffer(ctx, buf, NULL, MAIL_TRANSACTION_EXT_INTRO);
 }
 
@@ -601,6 +613,7 @@ mail_transaction_log_append_locked(struct mail_index_transaction *t,
        ctx.file = file;
        ctx.trans = t;
        ctx.output = buffer_create_dynamic(default_pool, 1024);
+       ctx.modseq = file->sync_highest_modseq;
 
        /* send all extension introductions and resizes before appends
           to avoid resize overhead as much as possible */
@@ -674,7 +687,7 @@ mail_transaction_log_append_locked(struct mail_index_transaction *t,
                buffer_free(&ctx.output);
                return -1;
        }
-       file->sync_highest_modseq += ctx.modseq_change_count;
+       file->sync_highest_modseq = ctx.modseq;
        buffer_free(&ctx.output);
 
        if ((t->flags & MAIL_INDEX_TRANSACTION_FLAG_HIDE) != 0) {

diff --git a/src/lib-index/mail-transaction-log-file.c b/src/lib-index/mail-transaction-log-file.c
index 039679d2e1055da77dd94f6e5b5fd5bb2307ca7d..1df482e770acfd4c95a90d86476a7edc64e1856b 100644 (file)
--- a/src/lib-index/mail-transaction-log-file.c
+++ b/src/lib-index/mail-transaction-log-file.c
@@ -102,45 +102,62 @@ void mail_transaction_log_file_free(struct mail_transaction_log_file **_file)
 }
 
 static void
-mail_transaction_log_file_add_to_list(struct mail_transaction_log_file *file)
+mail_transaction_log_file_skip_to_head(struct mail_transaction_log_file *file)
 {
        struct mail_transaction_log *log = file->log;
-       struct mail_transaction_log_file **p;
        struct mail_index_map *map = log->index->map;
        const struct mail_index_modseq_header *modseq_hdr;
+       uoff_t head_offset;
 
-       if (map != NULL && file->hdr.file_seq == map->hdr.log_file_seq &&
-           map->hdr.log_file_head_offset != 0) {
-               /* we can get a valid log offset from index file. initialize
-                  sync_offset from it so we don't have to read the whole log
-                  file from beginning. */
-               uoff_t head_offset = map->hdr.log_file_head_offset;
-
-               modseq_hdr = mail_index_map_get_modseq_header(map);
-               if (head_offset < file->hdr.hdr_size) {
-                       mail_index_set_error(log->index,
-                               "%s: log_file_head_offset too small",
-                               log->index->filepath);
-                       file->sync_offset = file->hdr.hdr_size;
-                       file->sync_highest_modseq = file->hdr.initial_modseq;
-               } else if (modseq_hdr == NULL ||
-                          modseq_hdr->log_seq != file->hdr.file_seq ||
-                          modseq_hdr->log_offset != head_offset) {
-                       /* highest_modseq not synced, start from beginning */
-                       file->sync_offset = file->hdr.hdr_size;
-                       file->sync_highest_modseq = file->hdr.initial_modseq;
-               } else {
-                       file->sync_offset = head_offset;
-                       file->sync_highest_modseq = modseq_hdr->highest_modseq;
-               }
-               file->saved_tail_offset = map->hdr.log_file_tail_offset;
-       } else {
+       if (map == NULL || file->hdr.file_seq != map->hdr.log_file_seq ||
+           map->hdr.log_file_head_offset == 0)
+               return;
+
+       /* we can get a valid log offset from index file. initialize
+          sync_offset from it so we don't have to read the whole log
+          file from beginning. */
+       head_offset = map->hdr.log_file_head_offset;
+
+       modseq_hdr = mail_index_map_get_modseq_header(map);
+       if (head_offset < file->hdr.hdr_size) {
+               mail_index_set_error(log->index,
+                                    "%s: log_file_head_offset too small",
+                                    log->index->filepath);
                file->sync_offset = file->hdr.hdr_size;
                file->sync_highest_modseq = file->hdr.initial_modseq;
+       } else if (modseq_hdr == NULL && file->hdr.initial_modseq == 0) {
+               /* modseqs not used yet */
+               file->sync_offset = head_offset;
+               file->sync_highest_modseq = 0;
+       } else if (modseq_hdr->log_seq != file->hdr.file_seq) {
+               /* highest_modseq not synced, start from beginning */
+               file->sync_offset = file->hdr.hdr_size;
+               file->sync_highest_modseq = file->hdr.initial_modseq;
+       } else if (modseq_hdr->log_offset > head_offset) {
+               mail_index_set_error(log->index,
+                                    "%s: modseq_hdr.log_offset too large",
+                                    log->index->filepath);
+               file->sync_offset = file->hdr.hdr_size;
+               file->sync_highest_modseq = file->hdr.initial_modseq;
+       } else {
+               /* start from where we last stopped tracking modseqs */
+               file->sync_offset = modseq_hdr->log_offset;
+               file->sync_highest_modseq = modseq_hdr->highest_modseq;
        }
+       file->saved_tail_offset = log->index->map->hdr.log_file_tail_offset;
+}
+
+static void
+mail_transaction_log_file_add_to_list(struct mail_transaction_log_file *file)
+{
+       struct mail_transaction_log_file **p;
+
+       file->sync_offset = file->hdr.hdr_size;
+       file->sync_highest_modseq = file->hdr.initial_modseq;
+       mail_transaction_log_file_skip_to_head(file);
 
        /* insert it to correct position */
-       for (p = &log->files; *p != NULL; p = &(*p)->next) {
+       for (p = &file->log->files; *p != NULL; p = &(*p)->next) {
                if ((*p)->hdr.file_seq > file->hdr.file_seq)
                        break;
                i_assert((*p)->hdr.file_seq < file->hdr.file_seq);
@@ -179,7 +196,8 @@ mail_transaction_log_init_hdr(struct mail_transaction_log *log,
                hdr->prev_file_seq = index->map->hdr.log_file_seq;
                hdr->prev_file_offset = index->map->hdr.log_file_head_offset;
                hdr->file_seq = index->map->hdr.log_file_seq + 1;
-               hdr->initial_modseq =
+               hdr->initial_modseq = log->head == NULL ||
+                       log->head->sync_highest_modseq == 0 ? 0 :
                        mail_index_map_modseq_get_highest(index->map);
        } else {
                hdr->file_seq = 1;
@@ -368,8 +386,6 @@ mail_transaction_log_file_read_hdr(struct mail_transaction_log_file *file,
                   shouldn't have filled */
                memset(PTR_OFFSET(&file->hdr, file->hdr.hdr_size), 0,
                       sizeof(file->hdr) - file->hdr.hdr_size);
-               if (file->hdr.minor_version == 0)
-                       file->hdr.initial_modseq = 1;
        }
 
        if (file->hdr.indexid == 0) {
@@ -524,7 +540,7 @@ mail_transaction_log_file_create2(struct mail_transaction_log_file *file,
        if (reset) {
                file->hdr.prev_file_seq = 0;
                file->hdr.prev_file_offset = 0;
-               file->hdr.initial_modseq = 1;
+               file->hdr.initial_modseq = 0;
        }
 
        if (write_full(new_fd, &file->hdr, sizeof(file->hdr)) < 0) {
@@ -724,8 +740,31 @@ log_file_track_mailbox_sync_offset_hdr(struct mail_transaction_log_file *file,
 }
 
 bool
-mail_transaction_header_has_modseq(const struct mail_transaction_header *hdr)
+mail_transaction_header_has_modseq(const struct mail_transaction_header *hdr,
+                                  const void *data,
+                                  uint64_t cur_modseq)
 {
+       if (cur_modseq != 0) {
+               /* tracking modseqs */
+       } else if ((hdr->type & MAIL_TRANSACTION_TYPE_MASK) ==
+                  MAIL_TRANSACTION_EXT_INTRO) {
+               /* modseqs not tracked yet. see if this is a modseq
+                  extension introduction. */
+               const struct mail_transaction_ext_intro *intro = data;
+               const unsigned int modseq_ext_len =
+                       strlen(MAIL_INDEX_MODSEQ_EXT_NAME);
+
+               if (intro->name_size == modseq_ext_len &&
+                   memcmp(intro + 1, MAIL_INDEX_MODSEQ_EXT_NAME,
+                          modseq_ext_len) == 0) {
+                       /* modseq tracking started */
+                       return TRUE;
+               }
+       } else {
+               /* not tracking modseqs */
+               return FALSE;
+       }
+
        switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
        case MAIL_TRANSACTION_EXPUNGE | MAIL_TRANSACTION_EXPUNGE_PROT:
                if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
@@ -882,7 +921,8 @@ int mail_transaction_log_file_get_highest_modseq_at(
        while (cur_offset < offset) {
                if (log_get_synced_record(file, &cur_offset, &hdr) < 0)
                        return- 1;
-               if (mail_transaction_header_has_modseq(hdr))
+               if (mail_transaction_header_has_modseq(hdr, hdr + 1,
+                                                      cur_modseq))
                        cur_modseq++;
        }
 
@@ -943,7 +983,8 @@ int mail_transaction_log_file_get_modseq_next_offset(
                prev_offset = cur_offset;
                if (log_get_synced_record(file, &cur_offset, &hdr) < 0)
                        return -1;
-               if (mail_transaction_header_has_modseq(hdr)) {
+               if (mail_transaction_header_has_modseq(hdr, hdr + 1,
+                                                      cur_modseq)) {
                        if (++cur_modseq == modseq)
                                break;
                }
@@ -973,11 +1014,12 @@ log_file_track_sync(struct mail_transaction_log_file *file,
                    const struct mail_transaction_header *hdr,
                    unsigned int trans_size)
 {
+       const void *data = hdr + 1;
        int ret;
 
-       if (mail_transaction_header_has_modseq(hdr))
+       if (mail_transaction_header_has_modseq(hdr, hdr + 1,
+                                              file->sync_highest_modseq))
                file->sync_highest_modseq++;
-
        if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0)
                return 0;
 
@@ -985,7 +1027,7 @@ log_file_track_sync(struct mail_transaction_log_file *file,
        if ((hdr->type & MAIL_TRANSACTION_TYPE_MASK) ==
            MAIL_TRANSACTION_HEADER_UPDATE) {
                /* see if this updates mailbox_sync_offset */
-               ret = log_file_track_mailbox_sync_offset_hdr(file, hdr + 1,
+               ret = log_file_track_mailbox_sync_offset_hdr(file, data,
                                                             trans_size -
                                                             sizeof(*hdr));
                if (ret != 0)
@@ -1209,13 +1251,6 @@ mail_transaction_log_file_read(struct mail_transaction_log_file *file,
                }
        }
 
-       if (start_offset > file->sync_offset) {
-               /* although we could just skip over the unwanted data, we have
-                  to sync everything so that modseqs are calculated
-                  correctly */
-               start_offset = file->sync_offset;
-       }
-
        if (file->buffer != NULL && file->buffer_offset > start_offset) {
                /* we have to insert missing data to beginning of buffer */
                ret = mail_transaction_log_file_insert_read(file, start_offset);
@@ -1419,6 +1454,15 @@ int mail_transaction_log_file_map(struct mail_transaction_log_file *file,
                                                  end_offset);
        }
 
+       if (start_offset > file->sync_offset)
+               mail_transaction_log_file_skip_to_head(file);
+       if (start_offset > file->sync_offset) {
+               /* although we could just skip over the unwanted data, we have
+                  to sync everything so that modseqs are calculated
+                  correctly */
+               start_offset = file->sync_offset;
+       }
+
        if (!index->mmap_disable)
                ret = mail_transaction_log_file_map_mmap(file, start_offset);
        else {

diff --git a/src/lib-index/mail-transaction-log-private.h b/src/lib-index/mail-transaction-log-private.h
index fb4608a8231ae6e88e4136f7243ba2628a66e951..552ab49d1f3f5c48064bb794cdbb7138afc78c58 100644 (file)
--- a/src/lib-index/mail-transaction-log-private.h
+++ b/src/lib-index/mail-transaction-log-private.h
@@ -122,7 +122,9 @@ int mail_transaction_log_lock_head(struct mail_transaction_log *log);
 void mail_transaction_log_file_unlock(struct mail_transaction_log_file *file);
 
 bool
-mail_transaction_header_has_modseq(const struct mail_transaction_header *hdr);
+mail_transaction_header_has_modseq(const struct mail_transaction_header *hdr,
+                                  const void *data,
+                                  uint64_t cur_modseq);
 int mail_transaction_log_file_get_highest_modseq_at(
                struct mail_transaction_log_file *file,
                uoff_t offset, uint64_t *highest_modseq_r);

diff --git a/src/lib-index/mail-transaction-log-view.c b/src/lib-index/mail-transaction-log-view.c
index 80c943de72694da4d7821625dbb5ebfcd425cb6c..0f48adcdb531448947087ef5c19e3f70689022b6 100644 (file)
--- a/src/lib-index/mail-transaction-log-view.c
+++ b/src/lib-index/mail-transaction-log-view.c
@@ -21,12 +21,7 @@ struct mail_transaction_log_view {
         struct mail_transaction_log_file *cur, *head, *tail;
        uoff_t cur_offset;
 
-       /* prev_modseq doesn't contain correct values until we know that
-          caller is really interested in modseqs. so the prev_modseq begins
-          from 0 and it's relative to prev_modseq_start_offset. when
-          prev_modseq_initialized=TRUE prev_modseq contains a correct value */
        uint64_t prev_modseq;
-
        uint32_t prev_file_seq;
        uoff_t prev_file_offset;
 
@@ -614,7 +609,8 @@ log_view_get_next(struct mail_transaction_log_view *view,
                ret = log_view_is_record_valid(file, hdr, data) ? 1 : -1;
        } T_END;
        if (ret > 0) {
-               if (mail_transaction_header_has_modseq(hdr))
+               if (mail_transaction_header_has_modseq(hdr, data,
+                                                      view->prev_modseq))
                        view->prev_modseq++;
                *hdr_r = hdr;
                *data_r = data;

diff --git a/src/util/logview.c b/src/util/logview.c
index 00cd8646c26be9febcf11931b0cb545a99bdcb50..fb7bcc3480abbdb00ea7868b8273437940553294 100644 (file)
--- a/src/util/logview.c
+++ b/src/util/logview.c
@@ -45,7 +45,7 @@ static void dump_hdr(int fd, uint64_t *modseq_r)
        printf("create stamp = %u\n", hdr.create_stamp);
        printf("initial modseq = %llu\n",
               (unsigned long long)hdr.initial_modseq);
-       *modseq_r = I_MAX(hdr.initial_modseq, 1);
+       *modseq_r = hdr.initial_modseq;
 }
 
 static bool
@@ -220,7 +220,7 @@ static void log_header_update(const struct mail_transaction_header_update *u)
 }
 
 static void log_record_print(const struct mail_transaction_header *hdr,
-                            const void *data)
+                            const void *data, uint64_t *modseq)
 {
        unsigned int size = hdr->size - sizeof(*hdr);
 
@@ -275,8 +275,12 @@ static void log_record_print(const struct mail_transaction_header *hdr,
                printf(" - flags = %u\n", intro->flags);
                printf(" - name_size = %u\n", intro->name_size);
                if (intro->name_size > 0) {
-                       printf(" - name = '%.*s'\n",
-                              intro->name_size, (const char *)(intro+1));
+                       const char *name = (const char *)(intro+1);
+
+                       printf(" - name = '%.*s'\n", intro->name_size, name);
+                       if (*modseq == 0 && intro->name_size == 6 &&
+                           memcmp(name, "modseq", 6) == 0)
+                               *modseq = 1;
                }
                break;
        }
@@ -372,7 +376,7 @@ static int dump_record(int fd, uint64_t *modseq)
 
        printf("record: offset=%"PRIuUOFF_T", type=%s, size=%u",
               offset, log_record_type(hdr.type), hdr.size);
-       if (mail_transaction_header_has_modseq(&hdr)) {
+       if (*modseq > 0 && mail_transaction_header_has_modseq(&hdr)) {
                *modseq += 1;
                printf(", modseq=%llu", (unsigned long long)*modseq);
        }
@@ -386,7 +390,7 @@ static int dump_record(int fd, uint64_t *modseq)
                        i_fatal("rec data read() %"PRIuSIZE_T" != %"PRIuSIZE_T,
                                ret, hdr.size - sizeof(hdr));
                }
-               log_record_print(&hdr, buf);
+               log_record_print(&hdr, buf, modseq);
        } else {
                lseek(fd, hdr.size - sizeof(hdr), SEEK_CUR);
        }