]> git.ipfire.org Git - thirdparty/dovecot/core.git/commitdiff
Transaction log contains only UIDs now, no more sequences which just mess up
authorTimo Sirainen <tss@iki.fi>
Sat, 22 May 2004 00:48:45 +0000 (03:48 +0300)
committerTimo Sirainen <tss@iki.fi>
Sat, 22 May 2004 00:48:45 +0000 (03:48 +0300)
everything.

--HG--
branch : HEAD

17 files changed:
src/lib-index/mail-index-sync-private.h
src/lib-index/mail-index-sync-update.c
src/lib-index/mail-index-sync.c
src/lib-index/mail-index-transaction.c
src/lib-index/mail-index-view-sync.c
src/lib-index/mail-index.h
src/lib-index/mail-transaction-log-view.c
src/lib-index/mail-transaction-log.c
src/lib-index/mail-transaction-log.h
src/lib-index/mail-transaction-util.c
src/lib-index/mail-transaction-util.h
src/lib-storage/index/index-storage.c
src/lib-storage/index/index-sync.c
src/lib-storage/index/maildir/maildir-sync.c
src/lib-storage/index/mbox/mbox-sync-parse.c
src/lib-storage/index/mbox/mbox-sync-update.c
src/lib-storage/index/mbox/mbox-sync.c

index 1cf82fdcc3df41f0ade9fdc7ba920809312f6416..7513f23aaf544598c761338697e81a7a315c4c1a 100644 (file)
@@ -15,7 +15,7 @@ struct mail_index_sync_ctx {
        const void *data;
 
        size_t expunge_idx, update_idx;
-       uint32_t next_seq;
+       uint32_t next_uid;
 
        unsigned int lock_id, dirty_lock_id;
 
index dc987920ad9669b9e20750fcf197849d300bf65b..2cf0014cc56d96c1daef8801d37bce9cdeb63f15 100644 (file)
@@ -10,6 +10,7 @@
 
 struct mail_index_update_ctx {
        struct mail_index *index;
+       struct mail_index_view *view;
        struct mail_index_header hdr;
        struct mail_transaction_log_view *log_view;
 };
@@ -64,7 +65,15 @@ static void mail_index_sync_update_flags(struct mail_index_update_ctx *ctx,
        struct mail_index_record *rec, *end;
        uint8_t flag_mask, old_flags;
        keywords_mask_t keyword_mask;
-       int i, update_keywords;
+       uint32_t seq1, seq2;
+       int i, update_keywords, ret;
+
+       ret = mail_index_lookup_uid_range(ctx->view, syncrec->uid1,
+                                         syncrec->uid2, &seq1, &seq2);
+       i_assert(ret == 0);
+
+       if (seq1 == 0)
+               return;
 
        update_keywords = FALSE;
        for (i = 0; i < INDEX_KEYWORDS_BYTE_COUNT; i++) {
@@ -76,8 +85,8 @@ static void mail_index_sync_update_flags(struct mail_index_update_ctx *ctx,
        }
 
        flag_mask = ~syncrec->remove_flags;
-       rec = &ctx->index->map->records[syncrec->seq1-1];
-       end = rec + (syncrec->seq2 - syncrec->seq1) + 1;
+       rec = &ctx->index->map->records[seq1-1];
+       end = rec + (seq2 - seq1) + 1;
        for (; rec != end; rec++) {
                old_flags = rec->flags;
                rec->flags = (rec->flags & flag_mask) | syncrec->add_flags;
@@ -177,6 +186,7 @@ int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx,
        const struct mail_index_record *appends;
        unsigned int append_count;
        uint32_t count, file_seq, src_idx, dest_idx, dirty_flag;
+       uint32_t seq1, seq2;
        uoff_t file_offset;
        unsigned int lock_id;
        int ret, changed;
@@ -190,6 +200,7 @@ int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx,
 
        memset(&ctx, 0, sizeof(ctx));
        ctx.index = index;
+       ctx.view = sync_ctx->view;
        ctx.hdr = *index->hdr;
        ctx.log_view = sync_ctx->view->log_view;
 
@@ -227,10 +238,18 @@ int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx,
                switch (rec.type) {
                case MAIL_INDEX_SYNC_TYPE_APPEND:
                        i_assert(appends == NULL);
-                       append_count = rec.seq2 - rec.seq1 + 1;
                        appends = rec.appends;
+                       append_count = rec.appends_count;
                        break;
                case MAIL_INDEX_SYNC_TYPE_EXPUNGE:
+                       ret = mail_index_lookup_uid_range(sync_ctx->view,
+                                                         rec.uid1, rec.uid2,
+                                                         &seq1, &seq2);
+                       i_assert(ret == 0);
+
+                       if (seq1 == 0)
+                               break;
+
                        if (src_idx == 0) {
                                /* expunges have to be atomic. so we'll have
                                   to copy the mapping, do the changes there
@@ -244,18 +263,17 @@ int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx,
                                index->hdr = map->hdr;
                                map->write_to_disk = TRUE;
 
-                               dest_idx = rec.seq1-1;
+                               dest_idx = seq1-1;
                        } else {
-                               count = (rec.seq1-1) - src_idx;
+                               count = (seq1-1) - src_idx;
                                memmove(map->records + dest_idx,
                                        map->records + src_idx,
                                        count * sizeof(*map->records));
                                dest_idx += count;
                        }
 
-                       mail_index_sync_update_expunges(&ctx, rec.seq1,
-                                                       rec.seq2);
-                       src_idx = rec.seq2;
+                       mail_index_sync_update_expunges(&ctx, seq1, seq2);
+                       src_idx = seq2;
                        break;
                case MAIL_INDEX_SYNC_TYPE_FLAGS:
                        mail_index_sync_update_flags(&ctx, &rec);
index 6a3bb12e8b0bcf6d006083e02b191480b497bf1b..967e691d12b3ee0ad31af46ba31cf6b679bbe3f9 100644 (file)
@@ -14,7 +14,6 @@ static void mail_index_sync_sort_flags(struct mail_index_sync_ctx *ctx)
        const struct mail_transaction_flag_update *src, *src_end;
        const struct mail_transaction_flag_update *dest;
        struct mail_transaction_flag_update new_update;
-       struct mail_transaction_expunge_iter_ctx *exp_ctx;
        uint32_t last;
        size_t i, dest_count;
 
@@ -26,41 +25,29 @@ static void mail_index_sync_sort_flags(struct mail_index_sync_ctx *ctx)
        dest = buffer_get_data(ctx->updates_buf, &dest_count);
        dest_count /= sizeof(*dest);
 
-       exp_ctx = mail_transaction_expunge_iter_init(ctx->expunges_buf);
-       mail_transaction_expunge_iter_seek(exp_ctx, src->seq1, src->seq2);
-
        for (i = 0; src != src_end; ) {
                new_update = *src;
 
-               if (!mail_transaction_expunge_iter_get(exp_ctx,
-                                                      &new_update.seq1,
-                                                      &new_update.seq2)) {
-                       mail_transaction_expunge_iter_seek(exp_ctx, src->seq1,
-                                                          src->seq2);
-                       src++;
-                       continue;
-               }
-
                /* insert it into buffer, split it in multiple parts if needed
                   to make sure the ordering stays the same */
                for (; i < dest_count; i++) {
-                       if (dest[i].seq1 <= new_update.seq1)
+                       if (dest[i].uid1 <= new_update.uid1)
                                continue;
 
-                       if (dest[i].seq1 > new_update.seq2)
+                       if (dest[i].uid1 > new_update.uid2)
                                break;
 
                        /* partial */
-                       last = new_update.seq2;
-                       new_update.seq2 = dest[i].seq1-1;
+                       last = new_update.uid2;
+                       new_update.uid2 = dest[i].uid1-1;
 
                        buffer_insert(ctx->updates_buf, i * sizeof(new_update),
                                      &new_update, sizeof(new_update));
                        dest = buffer_get_data(ctx->updates_buf, NULL);
                        dest_count++;
 
-                       new_update.seq1 = new_update.seq2+1;
-                       new_update.seq2 = last;
+                       new_update.uid1 = new_update.uid2+1;
+                       new_update.uid2 = last;
                }
 
                buffer_insert(ctx->updates_buf, i * sizeof(new_update),
@@ -68,7 +55,6 @@ static void mail_index_sync_sort_flags(struct mail_index_sync_ctx *ctx)
                dest = buffer_get_data(ctx->updates_buf, NULL);
                dest_count++;
        }
-       mail_transaction_expunge_iter_deinit(exp_ctx);
 }
 
 static void mail_index_sync_sort_transaction(struct mail_index_sync_ctx *ctx)
@@ -205,8 +191,8 @@ mail_index_sync_get_expunge(struct mail_index_sync_rec *rec,
                            const struct mail_transaction_expunge *exp)
 {
        rec->type = MAIL_INDEX_SYNC_TYPE_EXPUNGE;
-       rec->seq1 = exp->seq1;
-       rec->seq2 = exp->seq2;
+       rec->uid1 = exp->uid1;
+       rec->uid2 = exp->uid2;
 }
 
 void
@@ -214,8 +200,8 @@ mail_index_sync_get_update(struct mail_index_sync_rec *rec,
                           const struct mail_transaction_flag_update *update)
 {
        rec->type = MAIL_INDEX_SYNC_TYPE_FLAGS;
-       rec->seq1 = update->seq1;
-       rec->seq2 = update->seq2;
+       rec->uid1 = update->uid1;
+       rec->uid2 = update->uid2;
 
        rec->add_flags = update->add_flags;
        memcpy(rec->add_keywords, update->add_keywords,
@@ -228,23 +214,13 @@ mail_index_sync_get_update(struct mail_index_sync_rec *rec,
 static int mail_index_sync_rec_check(struct mail_index_view *view,
                                     struct mail_index_sync_rec *rec)
 {
-       uint32_t message_count;
-
        switch (rec->type) {
        case MAIL_INDEX_SYNC_TYPE_EXPUNGE:
        case MAIL_INDEX_SYNC_TYPE_FLAGS:
-               if (rec->seq1 > rec->seq2 || rec->seq1 == 0) {
-                       mail_transaction_log_view_set_corrupted(view->log_view,
-                               "Broken sequence: %u..%u (type 0x%x)",
-                               rec->seq1, rec->seq2, rec->type);
-                       return FALSE;
-               }
-
-               message_count = mail_index_view_get_message_count(view);
-               if (rec->seq2 > message_count) {
+               if (rec->uid1 > rec->uid2 || rec->uid1 == 0) {
                        mail_transaction_log_view_set_corrupted(view->log_view,
-                               "Sequence out of range: %u > %u (type 0x%x)",
-                               rec->seq2, message_count, rec->type);
+                               "Broken UID range: %u..%u (type 0x%x)",
+                               rec->uid1, rec->uid2, rec->type);
                        return FALSE;
                }
                break;
@@ -281,19 +257,19 @@ int mail_index_sync_next(struct mail_index_sync_ctx *ctx,
           update A: 7, 7
        */
        while (next_update != NULL &&
-              (next_exp == NULL || next_update->seq1 < next_exp->seq1)) {
-               if (next_update->seq2 >= ctx->next_seq) {
+              (next_exp == NULL || next_update->uid1 < next_exp->uid1)) {
+               if (next_update->uid2 >= ctx->next_uid) {
                        mail_index_sync_get_update(sync_rec, next_update);
                        if (next_exp != NULL &&
-                           next_exp->seq1 <= next_update->seq2) {
+                           next_exp->uid1 <= next_update->uid2) {
                                /* it's overlapping.. */
-                               sync_rec->seq2 = next_exp->seq1-1;
+                               sync_rec->uid2 = next_exp->uid1-1;
                        }
 
-                       if (sync_rec->seq1 < ctx->next_seq)
-                               sync_rec->seq1 = ctx->next_seq;
+                       if (sync_rec->uid1 < ctx->next_uid)
+                               sync_rec->uid1 = ctx->next_uid;
 
-                       i_assert(sync_rec->seq1 <= sync_rec->seq2);
+                       i_assert(sync_rec->uid1 <= sync_rec->uid2);
                        ctx->update_idx++;
                        return mail_index_sync_rec_check(ctx->view, sync_rec);
                }
@@ -304,44 +280,25 @@ int mail_index_sync_next(struct mail_index_sync_ctx *ctx,
        }
 
        if (next_exp != NULL) {
-               /* a few sanity checks here, we really don't ever want to
-                  accidentally expunge a message. If sequence and UID matches,
-                  it's quite unlikely this expunge was caused by some bug. */
-               uint32_t uid1, uid2;
-
                mail_index_sync_get_expunge(sync_rec, next_exp);
                if (mail_index_sync_rec_check(ctx->view, sync_rec) < 0)
                        return -1;
 
-               if (mail_index_lookup_uid(ctx->view, next_exp->seq1, &uid1) < 0)
-                       return -1;
-               if (mail_index_lookup_uid(ctx->view, next_exp->seq2, &uid2) < 0)
-                       return -1;
-               if (next_exp->uid1 != uid1 || next_exp->uid2 != uid2) {
-                       mail_transaction_log_view_set_corrupted(
-                               ctx->view->log_view, "Expunge range %u..%u: "
-                               "UIDs %u..%u doesn't match real UIDs %u..%u",
-                               next_exp->seq1, next_exp->seq2,
-                               next_exp->uid1, next_exp->uid2, uid1, uid2);
-                       return -1;
-               }
-
                ctx->expunge_idx++;
 
                /* scan updates again from the beginning */
                ctx->update_idx = 0;
-               ctx->next_seq = next_exp->seq2;          
+               ctx->next_uid = next_exp->uid2+1;
                return 1;
        }
 
        if (ctx->sync_appends) {
                ctx->sync_appends = FALSE;
                sync_rec->type = MAIL_INDEX_SYNC_TYPE_APPEND;
-               sync_rec->seq1 = ctx->index->map->records_count+1;
-               sync_rec->seq2 = sync_rec->seq1-1 +
-                       buffer_get_used_size(ctx->appends_buf) /
-                       sizeof(struct mail_index_record);
-               sync_rec->appends = buffer_get_data(ctx->appends_buf, NULL);
+               sync_rec->appends = buffer_get_data(ctx->appends_buf,
+                                                   &sync_rec->appends_count);
+               sync_rec->appends_count /= sizeof(*sync_rec->appends);
+               sync_rec->uid1 = sync_rec->uid2 = 0;
                return 1;
        }
 
@@ -363,6 +320,8 @@ int mail_index_sync_set_dirty(struct mail_index_sync_ctx *ctx, uint32_t seq)
                        return -1;
        }
 
+       /* FIXME: maybe this should go through transaction log anyway?
+          doesn't work well with non-mmaped indexes.. */
        i_assert(seq <= ctx->view->map->records_count);
        ctx->view->map->records[seq-1].flags |= MAIL_INDEX_MAIL_FLAG_DIRTY;
        ctx->have_dirty = TRUE;
index ca5c757ddf775240fdd06993f8597463c5c160c5..2f4cef45e7ecd78d0ca0f8e3cc673fe5da707173 100644 (file)
@@ -1,5 +1,9 @@
 /* Copyright (C) 2003-2004 Timo Sirainen */
 
+/* Inside transaction we keep messages stored in sequences in uid fields.
+   Before they're written to transaction log the sequences are changed to
+   UIDs. This is because we're able to compress sequence ranges better. */
+
 #include "lib.h"
 #include "buffer.h"
 #include "mail-index-view-private.h"
@@ -34,6 +38,43 @@ static void mail_index_transaction_free(struct mail_index_transaction *t)
        i_free(t);
 }
 
+static void
+mail_index_buffer_convert_to_uids(struct mail_index_view *view,
+                                 buffer_t *buf, size_t record_size, int range)
+{
+       unsigned char *data;
+       size_t size, i;
+       uint32_t *seq;
+
+       if (buf == NULL)
+               return;
+
+       /* @UNSAFE */
+       data = buffer_get_modifyable_data(buf, &size);
+       for (i = 0; i < size; i += record_size) {
+               seq = (uint32_t *)&data[i];
+
+               seq[0] = view->map->records[seq[0]-1].uid;
+               if (range)
+                       seq[1] = view->map->records[seq[1]-1].uid;
+       }
+}
+
+static int
+mail_index_transaction_convert_to_uids(struct mail_index_transaction *t)
+{
+       if (mail_index_view_lock(t->view) < 0)
+               return -1;
+
+       mail_index_buffer_convert_to_uids(t->view, t->expunges,
+               sizeof(struct mail_transaction_expunge), TRUE);
+       mail_index_buffer_convert_to_uids(t->view, t->updates,
+               sizeof(struct mail_transaction_flag_update), TRUE);
+       mail_index_buffer_convert_to_uids(t->view, t->cache_updates,
+               sizeof(struct mail_transaction_cache_update), TRUE);
+       return 0;
+}
+
 int mail_index_transaction_commit(struct mail_index_transaction *t,
                                  uint32_t *log_file_seq_r,
                                  uoff_t *log_file_offset_r)
@@ -45,10 +86,15 @@ int mail_index_transaction_commit(struct mail_index_transaction *t,
                return -1;
        }
 
-       if (t->last_update.seq1 != 0)
+       if (t->last_update.uid1 != 0)
                mail_index_transaction_add_last(t);
 
-       ret = mail_transaction_log_append(t, log_file_seq_r, log_file_offset_r);
+       if (mail_index_transaction_convert_to_uids(t) < 0)
+               ret = -1;
+       else {
+               ret = mail_transaction_log_append(t, log_file_seq_r,
+                                                 log_file_offset_r);
+       }
 
        mail_index_transaction_free(t);
        return ret;
@@ -87,14 +133,11 @@ void mail_index_expunge(struct mail_index_transaction *t, uint32_t seq)
 {
         struct mail_transaction_expunge exp, *data;
        unsigned int idx, left_idx, right_idx;
-       uint32_t uid;
        size_t size;
 
        i_assert(seq > 0 && seq <= mail_index_view_get_message_count(t->view));
 
-       uid = t->view->map->records[seq-1].uid;
-       exp.seq1 = exp.seq2 = seq;
-       exp.uid1 = exp.uid2 = uid;
+       exp.uid1 = exp.uid2 = seq;
 
        /* expunges is a sorted array of {seq1, seq2, ..}, .. */
 
@@ -110,23 +153,21 @@ void mail_index_expunge(struct mail_index_transaction *t, uint32_t seq)
        i_assert(size > 0);
 
        /* quick checks */
-       if (data[size-1].seq2 == seq-1) {
+       if (data[size-1].uid2 == seq-1) {
                /* grow last range */
-               data[size-1].seq2 = seq;
-               data[size-1].uid2 = uid;
+               data[size-1].uid2 = seq;
                return;
        }
-       if (data[size-1].seq2 < seq) {
+       if (data[size-1].uid2 < seq) {
                buffer_append(t->expunges, &exp, sizeof(exp));
                return;
        }
-       if (data[0].seq1 == seq+1) {
+       if (data[0].uid1 == seq+1) {
                /* grow down first range */
-               data[0].seq1 = seq;
-               data[0].uid1 = uid;
+               data[0].uid1 = seq;
                return;
        }
-       if (data[0].seq1 > seq) {
+       if (data[0].uid1 > seq) {
                buffer_insert(t->expunges, 0, &exp, sizeof(exp));
                return;
        }
@@ -137,42 +178,38 @@ void mail_index_expunge(struct mail_index_transaction *t, uint32_t seq)
        while (left_idx < right_idx) {
                idx = (left_idx + right_idx) / 2;
 
-               if (data[idx].seq1 < seq)
+               if (data[idx].uid1 < seq)
                        left_idx = idx+1;
-               else if (data[idx].seq1 > seq)
+               else if (data[idx].uid1 > seq)
                        right_idx = idx;
                else
                        break;
        }
 
-       if (data[idx].seq2 < seq)
+       if (data[idx].uid2 < seq)
                idx++;
 
         /* idx == size couldn't happen because we already handle it above */
-       i_assert(idx < size && data[idx].seq1 >= seq);
+       i_assert(idx < size && data[idx].uid1 >= seq);
 
-       if (data[idx].seq1 <= seq && data[idx].seq2 >= seq) {
+       if (data[idx].uid1 <= seq && data[idx].uid2 >= seq) {
                /* already expunged */
                return;
        }
 
-       if (data[idx].seq1 == seq+1) {
-               data[idx].seq1 = seq;
-               data[idx].uid1 = uid;
-               if (idx > 0 && data[idx-1].seq2 == seq-1) {
+       if (data[idx].uid1 == seq+1) {
+               data[idx].uid1 = seq;
+               if (idx > 0 && data[idx-1].uid2 == seq-1) {
                        /* merge */
-                       data[idx-1].seq2 = data[idx].seq2;
                        data[idx-1].uid2 = data[idx].uid2;
                        buffer_delete(t->expunges, idx * sizeof(*data),
                                      sizeof(*data));
                }
-       } else if (data[idx].seq2 == seq-1) {
+       } else if (data[idx].uid2 == seq-1) {
                i_assert(idx+1 < size); /* already handled above */
-               data[idx].seq2 = seq;
-               data[idx].uid2 = uid;
-               if (data[idx+1].seq1 == seq+1) {
+               data[idx].uid2 = seq;
+               if (data[idx+1].uid1 == seq+1) {
                        /* merge */
-                       data[idx+1].seq1 = data[idx].seq1;
                        data[idx+1].uid1 = data[idx].uid1;
                        buffer_delete(t->expunges, idx * sizeof(*data),
                                      sizeof(*data));
@@ -237,25 +274,25 @@ void mail_index_update_flags(struct mail_index_transaction *t, uint32_t seq,
        /* first get group updates into same structure. this allows faster
           updates if same mails have multiple flag updates during same
           transaction (eg. 1:10 +seen, 1:10 +deleted) */
-       if (t->last_update.seq2 == seq-1) {
-               if (t->last_update.seq1 != 0 &&
+       if (t->last_update.uid2 == seq-1) {
+               if (t->last_update.uid1 != 0 &&
                    IS_COMPATIBLE_UPDATE(t, modify_type, flags, keywords)) {
-                       t->last_update.seq2 = seq;
+                       t->last_update.uid2 = seq;
                        return;
                }
-       } else if (t->last_update.seq1 == seq+1) {
-               if (t->last_update.seq1 != 0 &&
+       } else if (t->last_update.uid1 == seq+1) {
+               if (t->last_update.uid1 != 0 &&
                    IS_COMPATIBLE_UPDATE(t, modify_type, flags, keywords)) {
-                       t->last_update.seq1 = seq;
+                       t->last_update.uid1 = seq;
                        return;
                }
        }
 
-       if (t->last_update.seq1 != 0)
+       if (t->last_update.uid1 != 0)
                mail_index_transaction_add_last(t);
 
        t->last_update_modify_type = modify_type;
-       t->last_update.seq1 = t->last_update.seq2 = seq;
+       t->last_update.uid1 = t->last_update.uid2 = seq;
        t->last_update.add_flags = flags;
        memcpy(t->last_update.add_keywords, keywords,
               INDEX_KEYWORDS_BYTE_COUNT);
@@ -312,35 +349,35 @@ static void mail_index_transaction_add_last(struct mail_index_transaction *t)
        while (left_idx < right_idx) {
                idx = (left_idx + right_idx) / 2;
 
-               if (data[idx].seq1 < update.seq1)
+               if (data[idx].uid1 < update.uid1)
                        left_idx = idx+1;
-               else if (data[idx].seq1 > update.seq1)
+               else if (data[idx].uid1 > update.uid1)
                        right_idx = idx;
                else
                        break;
        }
-       if (idx < size && data[idx].seq2 < update.seq1)
+       if (idx < size && data[idx].uid2 < update.uid1)
                idx++;
 
-       i_assert(idx == size || data[idx].seq1 < update.seq1);
+       i_assert(idx == size || data[idx].uid1 < update.uid1);
 
        /* insert it into buffer, split it in multiple parts if needed
           to make sure the ordering stays the same */
        for (; idx < size; idx++) {
-               if (data[idx].seq1 > update.seq2)
+               if (data[idx].uid1 > update.uid2)
                        break;
 
                /* partial */
-               last = update.seq2;
-               update.seq2 = data[idx].seq1-1;
+               last = update.uid2;
+               update.uid2 = data[idx].uid1-1;
 
                buffer_insert(t->updates, idx * sizeof(update),
                              &update, sizeof(update));
                data = buffer_get_modifyable_data(t->updates, NULL);
                size++;
 
-               update.seq1 = update.seq2+1;
-               update.seq2 = last;
+               update.uid1 = update.uid2+1;
+               update.uid2 = last;
        }
 
        buffer_insert(t->updates, idx * sizeof(update),
@@ -363,16 +400,16 @@ void mail_index_update_cache(struct mail_index_transaction *t,
        size /= sizeof(*data);
 
        /* we're probably appending it, check */
-       if (size == 0 || data[size-1].seq < seq)
+       if (size == 0 || data[size-1].uid < seq)
                idx = size;
        else {
                idx = 0; left_idx = 0; right_idx = size;
                while (left_idx < right_idx) {
                        idx = (left_idx + right_idx) / 2;
 
-                       if (data[idx].seq < seq)
+                       if (data[idx].uid < seq)
                                left_idx = idx+1;
-                       else if (data[idx].seq > seq)
+                       else if (data[idx].uid > seq)
                                right_idx = idx;
                        else {
                                /* already there, update */
@@ -382,7 +419,7 @@ void mail_index_update_cache(struct mail_index_transaction *t,
                }
        }
 
-       update.seq = seq;
+       update.uid = seq;
        update.cache_offset = offset;
        buffer_insert(t->updates, idx * sizeof(update),
                      &update, sizeof(update));
index 69cce524ef0d85eb4632dddd80b418fd6003ce65..2c391c8d51362c83c83ba99545b6c0a8c6f853e5 100644 (file)
@@ -25,9 +25,13 @@ struct mail_index_view_sync_ctx {
 static int
 view_sync_get_expunges(struct mail_index_view *view, buffer_t **expunges_r)
 {
-       const struct mail_transaction_expunge *exp, *end;
-       buffer_t *expunges;
+       const struct mail_transaction_header *hdr;
+       struct mail_transaction_expunge *src, *src_end, *dest;
+       const void *data;
        size_t size;
+       int ret;
+
+       *expunges_r = buffer_create_dynamic(default_pool, 512, (size_t)-1);
 
        /* with mask 0 we don't get anything, we'll just read the expunges
           while seeking to end */
@@ -36,23 +40,37 @@ view_sync_get_expunges(struct mail_index_view *view, buffer_t **expunges_r)
                                          view->log_file_offset,
                                          view->index->hdr->log_file_seq,
                                          view->index->hdr->log_file_offset,
-                                         0) < 0)
-               return -1;
-       if (mail_transaction_log_view_next(view->log_view,
-                                          NULL, NULL, NULL) < 0)
+                                         MAIL_TRANSACTION_EXPUNGE) < 0)
                return -1;
+       while ((ret = mail_transaction_log_view_next(view->log_view,
+                                                    &hdr, &data, NULL)) > 0) {
+               mail_transaction_log_sort_expunges(*expunges_r,
+                                                  data, hdr->size);
+       }
 
-       expunges = mail_transaction_log_view_get_expunges(view->log_view);
-       exp = buffer_get_data(expunges, &size);
-       end = CONST_PTR_OFFSET(exp, size);
-
-       *expunges_r = buffer_create_dynamic(default_pool, size, (size_t)-1);
-       for (; exp != end; exp++) {
-               buffer_append(*expunges_r, &exp->seq1, sizeof(exp->seq1));
-               buffer_append(*expunges_r, &exp->seq2, sizeof(exp->seq2));
+       if (ret == 0) {
+               /* convert to sequences */
+               src = dest = buffer_get_modifyable_data(*expunges_r, &size);
+               src_end = src + size / sizeof(*src);
+               for (; src != src_end; src++) {
+                       ret = mail_index_lookup_uid_range(view, src->uid1,
+                                                         src->uid2,
+                                                         &dest->uid1,
+                                                         &dest->uid2);
+                       i_assert(ret == 0);
+
+                       if (dest->uid1 == 0)
+                               size -= sizeof(*dest);
+                       else
+                               dest++;
+               }
+               buffer_set_used_size(*expunges_r, size);
+       } else {
+               buffer_set_used_size(*expunges_r, 0);
        }
-        mail_transaction_log_view_unset(view->log_view);
-       return 0;
+
+       mail_transaction_log_view_unset(view->log_view);
+       return ret;
 }
 
 int mail_index_view_sync_begin(struct mail_index_view *view,
@@ -135,17 +153,26 @@ static int view_is_transaction_synced(struct mail_index_view *view,
 
 static int sync_expunge(const struct mail_transaction_expunge *e, void *context)
 {
-       struct mail_index_map *map = context;
-       unsigned int idx, count;
+        struct mail_index_view_sync_ctx *ctx = context;
+       struct mail_index_map *map = ctx->sync_map;
+       uint32_t idx, count, seq1, seq2;
+       int ret;
+
+       ret = mail_index_lookup_uid_range(ctx->view, e->uid1, e->uid2,
+                                         &seq1, &seq2);
+       i_assert(ret == 0);
+
+       if (seq1 == 0)
+               return 1;
 
-       for (idx = e->seq1-1; idx < e->seq2; idx++) {
+       for (idx = seq1-1; idx < seq2; idx++) {
                mail_index_header_update_counts(&map->hdr_copy,
                                                map->records[idx].flags, 0);
        }
 
-       count = e->seq2 - e->seq1 + 1;
+       count = seq2 - seq1 + 1;
        buffer_delete(map->buffer,
-                     (e->seq1-1) * sizeof(struct mail_index_record),
+                     (seq1-1) * sizeof(struct mail_index_record),
                      count * sizeof(struct mail_index_record));
        map->records = buffer_get_modifyable_data(map->buffer, NULL);
 
@@ -156,7 +183,8 @@ static int sync_expunge(const struct mail_transaction_expunge *e, void *context)
 
 static int sync_append(const struct mail_index_record *rec, void *context)
 {
-       struct mail_index_map *map = context;
+        struct mail_index_view_sync_ctx *ctx = context;
+       struct mail_index_map *map = ctx->sync_map;
 
        buffer_append(map->buffer, rec, sizeof(*rec));
        map->records = buffer_get_modifyable_data(map->buffer, NULL);
@@ -173,12 +201,21 @@ static int sync_append(const struct mail_index_record *rec, void *context)
 static int sync_flag_update(const struct mail_transaction_flag_update *u,
                            void *context)
 {
-       struct mail_index_map *map = context;
+        struct mail_index_view_sync_ctx *ctx = context;
+       struct mail_index_map *map = ctx->sync_map;
        struct mail_index_record *rec;
-       unsigned int i, idx;
+       uint32_t i, idx, seq1, seq2;
        uint8_t old_flags;
+       int ret;
+
+       ret = mail_index_lookup_uid_range(ctx->view, u->uid1, u->uid2,
+                                         &seq1, &seq2);
+       i_assert(ret == 0);
 
-       for (idx = u->seq1-1; idx < u->seq2; idx++) {
+       if (seq1 == 0)
+               return 1;
+
+       for (idx = seq1-1; idx < seq2; idx++) {
                rec = &map->records[idx];
 
                old_flags = rec->flags;
@@ -198,9 +235,16 @@ static int sync_flag_update(const struct mail_transaction_flag_update *u,
 static int sync_cache_update(const struct mail_transaction_cache_update *u,
                             void *context)
 {
-       struct mail_index_map *map = context;
+        struct mail_index_view_sync_ctx *ctx = context;
+       uint32_t seq;
+       int ret;
+
+       ret = mail_index_lookup_uid_range(ctx->view, u->uid, u->uid,
+                                         &seq, &seq);
+       i_assert(ret == 0);
 
-       map->records[u->seq-1].cache_offset = u->cache_offset;
+       if (seq != 0)
+               ctx->sync_map->records[seq-1].cache_offset = u->cache_offset;
        return 1;
 }
 
@@ -211,7 +255,7 @@ static int mail_index_view_sync_map(struct mail_index_view_sync_ctx *ctx)
        };
 
        return mail_transaction_map(ctx->hdr, ctx->data,
-                                   &map_funcs, ctx->sync_map);
+                                   &map_funcs, ctx);
 }
 
 static int mail_index_view_sync_next_trans(struct mail_index_view_sync_ctx *ctx,
@@ -258,12 +302,7 @@ mail_index_view_sync_get_rec(struct mail_index_view_sync_ctx *ctx,
        switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
        case MAIL_TRANSACTION_APPEND: {
                rec->type = MAIL_INDEX_SYNC_TYPE_APPEND;
-               rec->seq1 = ctx->messages_count + 1;
-               ctx->messages_count +=
-                       hdr->size / sizeof(struct mail_index_record);
-               rec->seq2 = ctx->messages_count;
-               rec->appends = NULL;
-
+               rec->uid1 = rec->uid2 = 0;
                ctx->data_offset += hdr->size;
                break;
        }
index 160227e93a4cb31876c3e234ad728c1b952defc4..5e0362231233b78c8cba67dca8e9e448c19ca6d0 100644 (file)
@@ -109,7 +109,7 @@ enum mail_index_sync_type {
 #define MAIL_INDEX_SYNC_MASK_ALL 0xff
 
 struct mail_index_sync_rec {
-       uint32_t seq1, seq2;
+       uint32_t uid1, uid2;
        enum mail_index_sync_type type;
 
        /* MAIL_INDEX_SYNC_TYPE_FLAGS: */
@@ -119,7 +119,8 @@ struct mail_index_sync_rec {
        keywords_mask_t remove_keywords;
 
        /* MAIL_INDEX_SYNC_TYPE_APPEND: */
-        const struct mail_index_record *appends;
+       const struct mail_index_record *appends;
+       size_t appends_count;
 };
 
 struct mail_index;
index 36be7c7e8d58d6362446dd039167a77777ffa14c..ec296502d657eb95fbbf36371ca84a5fd0651529 100644 (file)
@@ -14,8 +14,7 @@ struct mail_transaction_log_view {
        uoff_t min_file_offset, max_file_offset;
 
        enum mail_transaction_type type_mask;
-       buffer_t *expunges_buf, *data_buf;
-        struct mail_transaction_expunge_iter_ctx *exp_ctx;
+       buffer_t *expunges_buf;
        struct mail_transaction_header tmp_hdr;
 
         struct mail_transaction_log_file *file;
@@ -55,8 +54,6 @@ void mail_transaction_log_view_close(struct mail_transaction_log_view *view)
        }
 
        mail_transaction_log_view_unset(view);
-       if (view->data_buf != NULL)
-               buffer_free(view->data_buf);
        buffer_free(view->expunges_buf);
        i_free(view);
 }
@@ -314,74 +311,10 @@ static int log_view_get_next(struct mail_transaction_log_view *view,
        return 1;
 }
 
-static int seqfix_expunge(const struct mail_transaction_expunge *e,
-                         void *context)
-{
-       struct mail_transaction_log_view *view = context;
-       struct mail_transaction_expunge new_e;
-
-       if (!mail_transaction_expunge_iter_seek(view->exp_ctx,
-                                               e->seq1, e->seq2)) {
-               new_e = *e;
-               /*FIXME:buffer_append(view->data_buf, e, sizeof(*e));
-               return 1;*/
-       }
-
-       new_e.uid1 = new_e.uid2 = 0; // FIXME: this breaks anyway
-
-       while (mail_transaction_expunge_iter_get(view->exp_ctx,
-                                                &new_e.seq1, &new_e.seq2)) {
-               i_assert(new_e.seq1 != 0);
-               buffer_append(view->data_buf, &new_e, sizeof(new_e));
-       }
-       return 1;
-}
-
-static int seqfix_flag_update(const struct mail_transaction_flag_update *u,
-                             void *context)
-{
-       struct mail_transaction_log_view *view = context;
-       struct mail_transaction_flag_update new_u;
-
-       if (!mail_transaction_expunge_iter_seek(view->exp_ctx,
-                                               u->seq1, u->seq2)) {
-               buffer_append(view->data_buf, u, sizeof(*u));
-               return 1;
-       }
-
-       new_u = *u;
-       while (mail_transaction_expunge_iter_get(view->exp_ctx,
-                                                &new_u.seq1, &new_u.seq2))
-               buffer_append(view->data_buf, &new_u, sizeof(new_u));
-       return 1;
-}
-
-static int seqfix_cache_update(const struct mail_transaction_cache_update *u,
-                              void *context)
-{
-       struct mail_transaction_log_view *view = context;
-       struct mail_transaction_cache_update new_u;
-
-       if (!mail_transaction_expunge_iter_seek(view->exp_ctx,
-                                               u->seq, u->seq)) {
-               buffer_append(view->data_buf, u, sizeof(*u));
-               return 1;
-       }
-
-       new_u = *u;
-       if (mail_transaction_expunge_iter_get(view->exp_ctx,
-                                             &new_u.seq, &new_u.seq))
-               buffer_append(view->data_buf, &new_u, sizeof(new_u));
-       return 1;
-}
-
 int mail_transaction_log_view_next(struct mail_transaction_log_view *view,
                                   const struct mail_transaction_header **hdr_r,
                                   const void **data_r, int *skipped_r)
 {
-       struct mail_transaction_map_functions seqfix_funcs = {
-               seqfix_expunge, NULL, seqfix_flag_update, seqfix_cache_update
-       };
        const struct mail_transaction_header *hdr;
        const void *data;
        int ret = 0;
@@ -399,11 +332,6 @@ int mail_transaction_log_view_next(struct mail_transaction_log_view *view,
                if (skipped_r != NULL)
                        *skipped_r = TRUE;
 
-               if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) != 0) {
-                       mail_transaction_log_sort_expunges(view->expunges_buf,
-                                                          data, hdr->size);
-               }
-
                /* FIXME: hide flag/cache updates for appends if
                   append isn't in mask */
        }
@@ -414,38 +342,7 @@ int mail_transaction_log_view_next(struct mail_transaction_log_view *view,
        *hdr_r = hdr;
        *data_r = data;
 
-       if (buffer_get_used_size(view->expunges_buf) > 0) {
-               /* we have to fix sequences in the data */
-               if (view->data_buf == NULL) {
-                       view->data_buf =
-                               buffer_create_dynamic(default_pool,
-                                                     hdr->size, (size_t)-1);
-               } else {
-                       buffer_set_used_size(view->data_buf, 0);
-               }
-
-               view->exp_ctx =
-                       mail_transaction_expunge_iter_init(view->expunges_buf);
-               ret = mail_transaction_map(hdr, data, &seqfix_funcs, view);
-               mail_transaction_expunge_iter_deinit(view->exp_ctx);
-
-               if (ret > 0) {
-                       /* modified - size may have changed, so update header */
-                       view->tmp_hdr = *hdr;
-                       view->tmp_hdr.size =
-                               buffer_get_used_size(view->data_buf);
-                       *hdr_r = &view->tmp_hdr;
-
-                       *data_r = buffer_get_data(view->data_buf, NULL);
-               } else {
-                       i_assert(buffer_get_used_size(view->data_buf) == 0);
-               }
-       }
-
        if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) != 0) {
-               mail_transaction_log_sort_expunges(view->expunges_buf,
-                                                  data, hdr->size);
-
                /* hide expunge protection */
                if (*hdr_r != &view->tmp_hdr) {
                        view->tmp_hdr = *hdr;
@@ -456,9 +353,3 @@ int mail_transaction_log_view_next(struct mail_transaction_log_view *view,
 
        return 1;
 }
-
-buffer_t *
-mail_transaction_log_view_get_expunges(struct mail_transaction_log_view *view)
-{
-       return view->expunges_buf;
-}
index eb9202d4352d1b46a9bb52cdaad7dfc4032e3df6..f64b820cbc8b6bbdc1f40ba09237e50676f856fd 100644 (file)
@@ -853,172 +853,6 @@ static int mail_transaction_log_lock_head(struct mail_transaction_log *log)
        return ret;
 }
 
-static int get_expunge_buf(struct mail_transaction_log *log,
-                          struct mail_index_view *view, buffer_t *expunges)
-{
-       struct mail_transaction_log_view *sync_view;
-       const struct mail_transaction_header *hdr;
-       const void *data;
-       int ret;
-
-       sync_view = mail_transaction_log_view_open(log);
-       ret = mail_transaction_log_view_set(sync_view, view->log_file_seq,
-                                           view->log_file_offset,
-                                           log->head->hdr.file_seq,
-                                           log->head->hdr.used_size,
-                                           MAIL_TRANSACTION_TYPE_MASK);
-       while ((ret = mail_transaction_log_view_next(sync_view,
-                                                    &hdr, &data, NULL)) == 1) {
-               if ((hdr->type & MAIL_TRANSACTION_TYPE_MASK) ==
-                   MAIL_TRANSACTION_EXPUNGE) {
-                       mail_transaction_log_sort_expunges(expunges,
-                                                          data, hdr->size);
-               }
-       }
-       mail_transaction_log_view_close(sync_view);
-       return ret;
-}
-
-static void
-log_view_fix_sequences(struct mail_index_view *view, buffer_t *view_expunges,
-                      buffer_t *buf, size_t record_size, int two, int uids)
-{
-       // FIXME: make sure this function works correctly
-       const struct mail_transaction_expunge *exp, *exp_end, *exp2;
-       unsigned char *data;
-       uint32_t *seq, expunges_before, count;
-       uint32_t last_exp, last_nonexp, last_nonexp_count;
-       size_t src_idx, dest_idx, size;
-       int ret;
-
-       if (buf == NULL)
-               return;
-
-       exp = buffer_get_data(view_expunges, &size);
-       exp_end = exp + (size / sizeof(*exp));
-       if (exp == exp_end)
-               return;
-
-       data = buffer_get_modifyable_data(buf, &size);
-
-       expunges_before = 0;
-       for (src_idx = dest_idx = 0; src_idx < size; src_idx += record_size) {
-               seq = (uint32_t *)&data[src_idx];
-
-               i_assert(src_idx + record_size == size ||
-                        *seq <= *((uint32_t *) &data[src_idx+record_size]));
-
-               while (exp != exp_end && exp->seq2 < seq[0]) {
-                       expunges_before += exp->seq2 - exp->seq1 + 1;
-                       exp++;
-               }
-               if (exp != exp_end && exp->seq1 <= seq[0]) {
-                       /* this sequence was expunged at least partially */
-                       if (!two)
-                               continue;
-
-                       exp2 = exp;
-                       count = 0;
-                       do {
-                               /* we point to next non-expunged message */
-                               seq[0] = exp2->seq2 + 1;
-                               count += exp->seq2 - exp->seq1 + 1;
-                               exp2++;
-                       } while (exp2 != exp_end && exp2->seq1 == seq[0]);
-
-                       if (seq[0] > seq[1] ||
-                           seq[0] > view->map->records_count) {
-                               /* it's all expunged */
-                               continue;
-                       }
-
-                       if (uids) {
-                               /* get new first UID */
-                               ret = mail_index_lookup_uid(view, seq[0],
-                                                           &seq[2]);
-                               i_assert(ret == 0);
-                       }
-                       seq[0] -= count;
-               }
-               seq[0] -= expunges_before;
-
-               if (two) {
-                       count = expunges_before;
-                       last_exp = 0;
-                       last_nonexp = seq[0];
-                       last_nonexp_count = count;
-
-                       exp2 = exp;
-                       while (exp2 != exp_end && exp2->seq1 <= seq[1]) {
-                               if (exp2->seq1-1 != last_exp) {
-                                       last_nonexp = exp2->seq1-1;
-                                       last_nonexp_count = count;
-                               }
-
-                               count += exp2->seq2 - exp2->seq1 + 1;
-                               last_exp = exp2->seq2;
-                               exp2++;
-                       }
-
-                       if (last_exp >= seq[1]) {
-                               seq[1] = last_nonexp;
-                               count = last_nonexp_count;
-                               if (uids) {
-                                       /* ending of the range was expunged,
-                                          we need to get last UID */
-                                       ret = mail_index_lookup_uid(view,
-                                                                   seq[1],
-                                                                   &seq[3]);
-                                       i_assert(ret == 0);
-                               }
-                       }
-                       seq[1] -= count;
-               }
-
-               if (src_idx != dest_idx) {
-                       memcpy(&data[dest_idx], &data[src_idx], record_size);
-                       i_assert(dest_idx == 0 ||
-                                *((uint32_t *) &data[dest_idx]) >=
-                                *((uint32_t *) &data[dest_idx-record_size]));
-               }
-               dest_idx += record_size;
-       }
-       buffer_set_used_size(buf, dest_idx);
-}
-
-static int
-mail_transaction_log_fix_sequences(struct mail_transaction_log *log,
-                                   struct mail_index_transaction *t)
-{
-       buffer_t *view_expunges;
-
-       if (t->updates == NULL && t->cache_updates == NULL &&
-           t->expunges == NULL)
-               return 0;
-
-       /* all sequences are currently relative to given view. we have to
-          find out all the expunges since then, even the ones that aren't
-          yet synchronized to index file. */
-       view_expunges = buffer_create_dynamic(default_pool, 1024, (size_t)-1);
-       if (get_expunge_buf(log, t->view, view_expunges) < 0) {
-               buffer_free(view_expunges);
-               return -1;
-       }
-
-       log_view_fix_sequences(t->view, view_expunges, t->updates,
-                              sizeof(struct mail_transaction_flag_update),
-                              TRUE, FALSE);
-       log_view_fix_sequences(t->view, view_expunges, t->cache_updates,
-                              sizeof(struct mail_transaction_cache_update),
-                              FALSE, FALSE);
-       log_view_fix_sequences(t->view, view_expunges, t->expunges,
-                              sizeof(struct mail_transaction_expunge),
-                              TRUE, TRUE);
-
-       buffer_free(view_expunges);
-       return 0;
-}
-
 static int mail_transaction_log_fix_appends(struct mail_transaction_log *log,
                                            struct mail_index_transaction *t)
 {
@@ -1155,6 +989,8 @@ int mail_transaction_log_append(struct mail_index_transaction *t,
                        return -1;
        }
 
+       /* FIXME: index->hdr may not be up-to-date and so log_file_seq check
+          might go wrong! sync header before we get here. */
        if (log->head->hdr.file_seq == index->hdr->log_file_seq &&
            log->head->hdr.used_size > MAIL_TRANSACTION_LOG_ROTATE_SIZE &&
            log->head->last_mtime <
@@ -1172,8 +1008,7 @@ int mail_transaction_log_append(struct mail_index_transaction *t,
        file = log->head;
        append_offset = file->hdr.used_size;
 
-       if (mail_transaction_log_fix_sequences(log, t) < 0 ||
-           mail_transaction_log_fix_appends(log, t) < 0) {
+       if (mail_transaction_log_fix_appends(log, t) < 0) {
                if (!log->index->log_locked)
                        (void)mail_transaction_log_file_lock(file, F_UNLCK);
                return -1;
index 3727f0d142f054392194150d5e9004bd0a22358e..d8b04b331b24ebc45fcd2d20a601e75bc156e366 100644 (file)
@@ -37,17 +37,16 @@ struct mail_transaction_header {
 };
 
 struct mail_transaction_expunge {
-       uint32_t seq1, seq2;
-       uint32_t uid1, uid2; /* only to avoid accidental expunges due to bugs */
+       uint32_t uid1, uid2;
 };
 
 struct mail_transaction_cache_update {
-       uint32_t seq;
+       uint32_t uid;
        uint32_t cache_offset;
 };
 
 struct mail_transaction_flag_update {
-       uint32_t seq1, seq2;
+       uint32_t uid1, uid2;
        uint8_t add_flags;
        keywords_mask_t add_keywords;
        uint8_t remove_flags;
@@ -84,9 +83,6 @@ mail_transaction_log_view_get_prev_pos(struct mail_transaction_log_view *view,
                                       uint32_t *file_seq_r,
                                       uoff_t *file_offset_r);
 
-buffer_t *
-mail_transaction_log_view_get_expunges(struct mail_transaction_log_view *view);
-
 /* Marks the log file in current position to be corrupted. */
 void
 mail_transaction_log_view_set_corrupted(struct mail_transaction_log_view *view,
index d69306bce9610729d8592aac2119a6ab814db5e2..043f7f3025e8781e45c290d4471c2c8e36047802 100644 (file)
@@ -130,7 +130,6 @@ mail_transaction_log_sort_expunges(buffer_t *expunges_buf,
        const struct mail_transaction_expunge *src_end;
        struct mail_transaction_expunge *dest;
        struct mail_transaction_expunge new_exp;
-       uint32_t expunges_before, count;
        size_t first, i, dest_count;
 
        i_assert(src_buf_size % sizeof(*src) == 0);
@@ -140,43 +139,27 @@ mail_transaction_log_sort_expunges(buffer_t *expunges_buf,
        dest = buffer_get_modifyable_data(expunges_buf, &dest_count);
        dest_count /= sizeof(*dest);
 
-       expunges_before = 0;
        for (i = 0; src != src_end; src++) {
                /* src[] must be sorted. */
-               i_assert(src+1 == src_end || src->seq1 < src[1].seq1);
+               i_assert(src+1 == src_end || src->uid1 < src[1].uid1);
 
                for (; i < dest_count; i++) {
-                       if (src->seq1 + expunges_before < dest[i].seq1)
+                       if (src->uid1 < dest[i].uid1)
                                break;
-
-                       i_assert(src->uid2 == 0 || src->uid2 > dest[i].uid1);
-                       expunges_before += dest[i].seq2 - dest[i].seq1 + 1;
                }
 
                new_exp = *src;
-               new_exp.seq1 += expunges_before;
-               new_exp.seq2 += expunges_before;
-
-               /* if src[] is in format {1,2}{1,2} rather than {1,2}{3,4}:
-                  expunges_before += new_exp.seq2 - new_exp.seq1 + 1;*/
 
                first = i;
-               while (i < dest_count && new_exp.seq2 >= dest[i].seq1-1) {
+               while (i < dest_count && src->uid2 >= dest[i].uid1-1) {
                        /* we can/must merge with next record */
-                       count = dest[i].seq2 - dest[i].seq1 + 1;
-                       expunges_before += count;
-
-                       new_exp.seq2 += count;
-                       if (new_exp.seq2 == dest[i].seq2)
+                       if (new_exp.uid2 < dest[i].uid2)
                                new_exp.uid2 = dest[i].uid2;
-                       i_assert(new_exp.uid2 == 0 ||
-                                new_exp.uid2 >= dest[i].uid2);
                        i++;
                }
 
-               if (first > 0 && new_exp.seq1 == dest[first-1].seq2+1) {
+               if (first > 0 && new_exp.uid1 <= dest[first-1].uid2+1) {
                        /* continue previous record */
-                       dest[first-1].seq2 = new_exp.seq2;
                        dest[first-1].uid2 = new_exp.uid2;
                } else if (i == first) {
                        buffer_insert(expunges_buf, i * sizeof(new_exp),
@@ -201,86 +184,3 @@ mail_transaction_log_sort_expunges(buffer_t *expunges_buf,
                }
        }
 }
-
-struct mail_transaction_expunge_iter_ctx *
-mail_transaction_expunge_iter_init(const buffer_t *expunges_buf)
-{
-       struct mail_transaction_expunge_iter_ctx *ctx;
-
-       ctx = i_new(struct mail_transaction_expunge_iter_ctx, 1);
-       ctx->cur_seq = 1;
-
-       if (expunges_buf != NULL) {
-               ctx->expunges =
-                       buffer_get_data(expunges_buf, &ctx->expunges_count);
-               ctx->expunges_count /= sizeof(*ctx->expunges);
-       }
-       return ctx;
-}
-
-void mail_transaction_expunge_iter_deinit(
-       struct mail_transaction_expunge_iter_ctx *ctx)
-{
-       i_free(ctx);
-}
-
-int mail_transaction_expunge_iter_seek(
-       struct mail_transaction_expunge_iter_ctx *ctx,
-       uint32_t seq1, uint32_t seq2)
-{
-       uint32_t idx, count, last_seq;
-
-       i_assert(seq1 >= ctx->cur_seq);
-
-       idx = ctx->cur_idx;
-       last_seq = idx == 0 ? 1 : ctx->expunges[idx-1].seq2 + 1;
-       for (; idx < ctx->expunges_count; idx++) {
-               count = ctx->expunges[idx].seq1 - last_seq;
-               if (ctx->cur_seq + count > seq1)
-                       break;
-               ctx->cur_seq += count;
-
-               ctx->expunges_before += ctx->expunges[idx].seq2 -
-                       ctx->expunges[idx].seq1 + 1;
-               last_seq = ctx->expunges[idx].seq2+1;
-       }
-
-       ctx->iter_idx = idx;
-       ctx->iter_seq = seq1 + ctx->expunges_before;
-       ctx->iter_count = seq2 - seq1 + 1;
-
-       ctx->cur_idx = idx;
-       return ctx->expunges_before != 0 ||
-               (idx != ctx->expunges_count && ctx->expunges[idx].seq1 <= seq2);
-}
-
-int mail_transaction_expunge_iter_get(
-       struct mail_transaction_expunge_iter_ctx *ctx,
-       uint32_t *seq1_r, uint32_t *seq2_r)
-{
-       if (ctx->iter_count == 0)
-               return 0;
-
-       *seq1_r = ctx->iter_seq;
-
-       if (ctx->iter_idx == ctx->expunges_count ||
-           ctx->expunges[ctx->iter_idx].seq1 >=
-           ctx->iter_seq + ctx->iter_count) {
-               /* last one */
-               *seq2_r = ctx->iter_seq + ctx->iter_count - 1;
-               ctx->iter_count = 0;
-               return 1;
-       }
-
-       ctx->iter_count -= ctx->expunges[ctx->iter_idx].seq1 - ctx->iter_seq;
-       i_assert(ctx->iter_count > 0);
-
-       /* have to split this one */
-       *seq2_r = ctx->expunges[ctx->iter_idx].seq1-1;
-       do {
-               ctx->iter_seq = ctx->expunges[ctx->iter_idx].seq2+1;
-               ctx->iter_idx++;
-       } while (ctx->iter_idx != ctx->expunges_count &&
-                ctx->expunges[ctx->iter_idx].seq1 == ctx->iter_seq);
-       return 1;
-}
index 6e30c2bc70923f135479856cbf57a410631289b5..e95d825a78dda5bb199d250a3cb5be15e57eac16 100644 (file)
@@ -32,22 +32,4 @@ mail_transaction_log_sort_expunges(buffer_t *expunges_buf,
                                   const struct mail_transaction_expunge *src,
                                   size_t src_buf_size);
 
-/* Iterate through expunges buffer. iter_seek()'s seq1/seq2 is assumed to be
-   in post-expunge view, iter_get() updates them to pre-expunge view. Some
-   post-expunge sequence arrays may go through expunges, we split them so it
-   won't be visible. */
-struct mail_transaction_expunge_iter_ctx *
-mail_transaction_expunge_iter_init(const buffer_t *expunges_buf);
-void mail_transaction_expunge_iter_deinit(
-       struct mail_transaction_expunge_iter_ctx *ctx);
-/* Returns TRUE if seq1 or seq2 will be modified by iter_get(). If FALSE is
-   returned calling iter_get() is a bit pointless. */
-int mail_transaction_expunge_iter_seek(
-       struct mail_transaction_expunge_iter_ctx *ctx,
-       uint32_t seq1, uint32_t seq2);
-/* Returns TRUE while sequences are returned. */
-int mail_transaction_expunge_iter_get(
-       struct mail_transaction_expunge_iter_ctx *ctx,
-       uint32_t *seq1_r, uint32_t *seq2_r);
-
 #endif
index ee3e33f7b875b5b8caf0ba45afe8d002ece66550..48fbfab7b25ac8783c0317092ed2d82bf2d6d2c3 100644 (file)
@@ -448,27 +448,3 @@ int index_mailbox_fix_keywords(struct index_mailbox *ibox,
                return mail_storage_set_index_error(ibox);
        }*/
 }
-
-unsigned int index_storage_get_recent_count(struct mail_index_view *view)
-{
-#if 0
-       struct mail_index_header *hdr;
-       struct mail_index_record *rec;
-       unsigned int seq;
-
-       hdr = mail_index_get_header(view);
-       if (index->first_recent_uid <= 1) {
-               /* all are recent */
-               return hdr->messages_count;
-       }
-
-       /* get the first recent message */
-       if (index->first_recent_uid >= hdr->next_uid)
-               return 0;
-
-       rec = mail_index_lookup_uid_range(view, index->first_recent_uid,
-                                         hdr->next_uid - 1, &seq);
-       return rec == NULL ? 0 : hdr->messages_count+1 - seq;
-#endif
-       return 0;
-}
index 259f6c3eda5544395bc33b0db7d752c2785d9d84..0747c1c9704cfe0c023f7ce7e6cd389510504b44 100644 (file)
@@ -15,7 +15,7 @@ int index_storage_sync(struct mailbox *box, enum mailbox_sync_flags flags)
        size_t i, expunges_count;
        void *sc_context;
        enum mail_index_sync_type sync_mask;
-       uint32_t seq, messages_count, recent_count;
+       uint32_t seq, seq1, seq2, messages_count, recent_count;
        int ret, appends;
 
        sync_mask = MAIL_INDEX_SYNC_MASK_ALL;
@@ -54,11 +54,18 @@ int index_storage_sync(struct mailbox *box, enum mailbox_sync_flags flags)
                        if (sc->update_flags == NULL)
                                break;
 
-                       if (sync.seq2 > messages_count)
-                               sync.seq2 = messages_count;
+                       if (mail_index_lookup_uid_range(ibox->view,
+                                                       sync.uid1, sync.uid2,
+                                                       &seq1, &seq2) < 0) {
+                               ret = -1;
+                               break;
+                       }
+
+                       if (seq1 == 0)
+                               break;
 
                        /* FIXME: hide the flag updates for expunged messages */
-                       for (seq = sync.seq1; seq <= sync.seq2; seq++) {
+                       for (seq = seq1; seq <= seq2; seq++) {
                                if (mail_index_lookup(ibox->view,
                                                      seq, &rec) < 0) {
                                        ret = -1;
@@ -76,6 +83,7 @@ int index_storage_sync(struct mailbox *box, enum mailbox_sync_flags flags)
                mail_storage_set_index_error(ibox);
 
        if (sc->expunge != NULL) {
+               // FIXME: these are UIDs now..
                for (i = expunges_count*2; i > 0; i -= 2) {
                        seq = expunges[i-1];
                        if (seq > messages_count)
index e7124701867333b67dd0387bd6a78f3a9f450942..0cbd73819b0da97116ad1e5c06bf123326a57a40 100644 (file)
@@ -257,13 +257,23 @@ static int maildir_sync_record(struct index_mailbox *ibox,
 {
        struct mail_index_sync_rec *sync_rec = &ctx->sync_rec;
        struct mail_index_view *view = ctx->view;
-       uint32_t seq, uid;
+       uint32_t seq, seq1, seq2, uid;
 
        switch (sync_rec->type) {
        case MAIL_INDEX_SYNC_TYPE_APPEND:
                break;
        case MAIL_INDEX_SYNC_TYPE_EXPUNGE:
-               for (seq = sync_rec->seq1; seq <= sync_rec->seq2; seq++) {
+               /* make it go through sequences to avoid looping through huge
+                  holes in UID range */
+               if (mail_index_lookup_uid_range(view, sync_rec->uid1,
+                                               sync_rec->uid2,
+                                               &seq1, &seq2) < 0)
+                       return -1;
+
+               if (seq1 == 0)
+                       break;
+
+               for (seq = seq1; seq <= seq2; seq++) {
                        if (mail_index_lookup_uid(view, seq, &uid) < 0)
                                return -1;
                        if (maildir_file_do(ibox, uid, maildir_expunge,
@@ -272,8 +282,15 @@ static int maildir_sync_record(struct index_mailbox *ibox,
                }
                break;
        case MAIL_INDEX_SYNC_TYPE_FLAGS:
-                ctx->seq = sync_rec->seq1;
-               for (; ctx->seq <= sync_rec->seq2; ctx->seq++) {
+               if (mail_index_lookup_uid_range(view, sync_rec->uid1,
+                                               sync_rec->uid2,
+                                               &seq1, &seq2) < 0)
+                       return -1;
+
+               if (seq1 == 0)
+                       break;
+
+               for (ctx->seq = seq1; ctx->seq <= seq2; ctx->seq++) {
                        if (mail_index_lookup_uid(view, ctx->seq, &uid) < 0)
                                return -1;
                        if (maildir_file_do(ibox, uid, maildir_sync_flags,
index f9bcb907ec07321a6e27f9dc914aa0efd3d7eefe..90c620ccc1a444b79e7d1ca943e85a0f961242b6 100644 (file)
@@ -299,6 +299,7 @@ void mbox_sync_parse_next_mail(struct istream *input,
        if (ctx->mail.uid == 0) {
                /* missing X-UID */
                ctx->need_rewrite = TRUE;
+               ctx->mail.uid = ctx->sync_ctx->next_uid++;
        }
 
        ctx->body_offset = input->v_offset;
index 3f2718ecb039f9a04aeadacdcb108123ede33763..e3ed62f4817dc3a22554f70b299795d4daa1ac5d 100644 (file)
@@ -98,12 +98,9 @@ static void mbox_sync_add_missing_headers(struct mbox_sync_mail_context *ctx)
        i_assert(ctx->sync_ctx->base_uid_validity != 0);
 
        if (ctx->hdr_pos[MBOX_HDR_X_UID] == (size_t)-1) {
-               if (ctx->mail.uid == 0)
-                       ctx->mail.uid = ctx->sync_ctx->next_uid++;
                ctx->hdr_pos[MBOX_HDR_X_UID] = str_len(ctx->header);
                str_printfa(ctx->header, "X-UID: %u\n", ctx->mail.uid);
        }
-       i_assert(ctx->mail.uid != 0);
 
        if (ctx->hdr_pos[MBOX_HDR_STATUS] == (size_t)-1 &&
            (ctx->mail.flags & STATUS_FLAGS_MASK) != 0) {
index 879fd706211604ea05b5cf6966bce910470102aa..5b56e2020126fcbf15aaa299f6c6de77a548dc41 100644 (file)
@@ -82,7 +82,7 @@ static int mbox_sync_grow_file(struct mbox_sync_context *sync_ctx,
        return 0;
 }
 
-static void mbox_sync_buffer_delete_old(buffer_t *syncs_buf, uint32_t seq)
+static void mbox_sync_buffer_delete_old(buffer_t *syncs_buf, uint32_t uid)
 {
        struct mail_index_sync_rec *sync;
        size_t size, src, dest;
@@ -91,7 +91,7 @@ static void mbox_sync_buffer_delete_old(buffer_t *syncs_buf, uint32_t seq)
        size /= sizeof(*sync);
 
        for (src = dest = 0; src < size; src++) {
-               if (sync[src].seq2 >= seq) {
+               if (sync[src].uid2 >= uid) {
                        if (src != dest)
                                sync[dest] = sync[src];
                        dest++;
@@ -199,12 +199,14 @@ static int mbox_sync_do(struct index_mailbox *ibox,
                if (input->eof)
                        break;
 
+               mbox_sync_next_mail(&sync_ctx, &mail_ctx, seq);
+
                /* get all sync records related to this message */
                ret = 1;
-               mbox_sync_buffer_delete_old(syncs, seq);
-               while (seq >= sync_rec.seq1 && ret > 0) {
-                       if (sync_rec.seq1 != 0) {
-                               i_assert(seq <= sync_rec.seq2);
+               mbox_sync_buffer_delete_old(syncs, mail_ctx.mail.uid);
+               while (mail_ctx.mail.uid >= sync_rec.uid1 && ret > 0) {
+                       if (sync_rec.uid1 != 0) {
+                               i_assert(mail_ctx.mail.uid <= sync_rec.uid2);
                                buffer_append(syncs, &sync_rec,
                                              sizeof(sync_rec));
                        }
@@ -215,8 +217,6 @@ static int mbox_sync_do(struct index_mailbox *ibox,
                if (ret < 0)
                        break;
 
-               mbox_sync_next_mail(&sync_ctx, &mail_ctx, seq);
-
                if (seq == 1 && sync_ctx.base_uid_validity == 0) {
                        if (mail_index_get_header(sync_view, &hdr) < 0) {
                                ret = -1;