dbox fixes.

--HG--
branch : HEAD

diff --git a/src/lib-storage/index/dbox/dbox-file.c b/src/lib-storage/index/dbox/dbox-file.c
index 7e5f02e1de9bb17abfc8691b6ba2ee439118aef0..ce0f3c8ed76bbeefe049c42abbdc56fdf8a8b2fc 100644 (file)
--- a/src/lib-storage/index/dbox/dbox-file.c
+++ b/src/lib-storage/index/dbox/dbox-file.c
@@ -634,8 +634,9 @@ int dbox_file_seek_next(struct dbox_file *file, uoff_t *offset_r, bool *last_r)
        int ret;
 
        if (file->cur_offset == (uoff_t)-1) {
-               /* first mail */
-               offset = file->file_header_size;
+               /* first mail. we may not have read the file at all yet,
+                  so set the offset afterwards. */
+               offset = 0;
        } else {
                offset = file->cur_offset + file->msg_header_size +
                        file->cur_physical_size;
@@ -650,7 +651,13 @@ int dbox_file_seek_next(struct dbox_file *file, uoff_t *offset_r, bool *last_r)
        }
        *last_r = FALSE;
 
-       return dbox_file_get_mail_stream(file, offset, &size, NULL, &expunged);
+       ret = dbox_file_get_mail_stream(file, offset, &size, NULL, &expunged);
+       if (ret <= 0)
+               return ret;
+
+       if (*offset_r == 0)
+               *offset_r = file->file_header_size;
+       return 1;
 }
 
 static int

diff --git a/src/lib-storage/index/dbox/dbox-mail.c b/src/lib-storage/index/dbox/dbox-mail.c
index 252fe2aa791ebc7469613d9320bbaa1d14d50fec..9e3af8ca91c9c3426a65d66bb3f11248a92bc067 100644 (file)
--- a/src/lib-storage/index/dbox/dbox-mail.c
+++ b/src/lib-storage/index/dbox/dbox-mail.c
@@ -44,16 +44,50 @@ static void dbox_mail_close(struct mail *_mail)
        index_mail_close(_mail);
 }
 
-uint32_t dbox_mail_lookup(struct dbox_mailbox *mbox,
-                         struct mail_index_view *view, uint32_t seq)
+int dbox_mail_lookup(struct dbox_mailbox *mbox, struct mail_index_view *view,
+                    uint32_t seq, uint32_t *map_uid_r)
 {
        const struct dbox_mail_index_record *dbox_rec;
+       const struct dbox_index_header *hdr;
        const void *data;
+       size_t data_size;
+       uint32_t cur_map_uid_validity;
        bool expunged;
 
        mail_index_lookup_ext(view, seq, mbox->dbox_ext_id, &data, &expunged);
        dbox_rec = data;
-       return dbox_rec == NULL ? 0 : dbox_rec->map_uid;
+       if (dbox_rec == NULL || dbox_rec->map_uid == 0) {
+               *map_uid_r = 0;
+               return 0;
+       }
+
+       if (mbox->map_uid_validity == 0) {
+               mail_index_get_header_ext(mbox->ibox.view,
+                                         mbox->dbox_hdr_ext_id,
+                                         &data, &data_size);
+               if (data_size != sizeof(*hdr)) {
+                       mail_storage_set_critical(&mbox->storage->storage,
+                               "dbox %s: Invalid dbox header size", mbox->path);
+                       mbox->storage->sync_rebuild = TRUE;
+                       return -1;
+               }
+               hdr = data;
+               mbox->map_uid_validity = hdr->map_uid_validity;
+       }
+       if (dbox_map_open(mbox->storage->map) < 0)
+               return -1;
+
+       cur_map_uid_validity = dbox_map_get_uid_validity(mbox->storage->map);
+       if (cur_map_uid_validity != mbox->map_uid_validity) {
+               mail_storage_set_critical(&mbox->storage->storage,
+                       "dbox %s: map uidvalidity mismatch (%u vs %u)",
+                       mbox->path, mbox->map_uid_validity,
+                       cur_map_uid_validity);
+               mbox->storage->sync_rebuild = TRUE;
+               return -1;
+       }
+       *map_uid_r = dbox_rec->map_uid;
+       return 0;
 }
 
 static int dbox_mail_open(struct dbox_mail *mail,
@@ -65,7 +99,9 @@ static int dbox_mail_open(struct dbox_mail *mail,
        int ret;
 
        if (mail->open_file == NULL) {
-               map_uid = dbox_mail_lookup(mbox, mbox->ibox.view, _mail->seq);
+               if (dbox_mail_lookup(mbox, mbox->ibox.view, _mail->seq,
+                                    &map_uid) < 0)
+                       return -1;
                if (map_uid == 0) {
                        mail->open_file =
                                dbox_file_init_single(mbox, _mail->uid);

diff --git a/src/lib-storage/index/dbox/dbox-map-private.h b/src/lib-storage/index/dbox/dbox-map-private.h
index f95f3cc0da4a361609fc690146d169f3221d24f1..398f4ba5ee90e490974c19a98b9e3fd744fed373 100644 (file)
--- a/src/lib-storage/index/dbox/dbox-map-private.h
+++ b/src/lib-storage/index/dbox/dbox-map-private.h
@@ -13,6 +13,7 @@ struct dbox_map {
        struct dbox_storage *storage;
        struct mail_index *index;
        struct mail_index_view *view;
+       uint32_t created_uid_validity;
 
        uint32_t map_ext_id, ref_ext_id;
        ARRAY_TYPE(seq_range) ref0_file_ids;
@@ -43,7 +44,6 @@ struct dbox_map_append_context {
        unsigned int committed:1;
 };
 
-int dbox_map_open(struct dbox_map *map);
 int dbox_map_refresh(struct dbox_map *map);
 int dbox_map_view_lookup_rec(struct dbox_map *map, struct mail_index_view *view,
                             uint32_t seq, struct dbox_mail_lookup_rec *rec_r);

diff --git a/src/lib-storage/index/dbox/dbox-map.c b/src/lib-storage/index/dbox/dbox-map.c
index af7d7a843446617f175f62087f7760d911c343dc..add25240cb874c17de9b5ac8d943af9c04cb9012 100644 (file)
--- a/src/lib-storage/index/dbox/dbox-map.c
+++ b/src/lib-storage/index/dbox/dbox-map.c
@@ -41,6 +41,7 @@ struct dbox_map *dbox_map_init(struct dbox_storage *storage)
                                sizeof(uint32_t));
        map->ref_ext_id = mail_index_ext_register(map->index, "ref", 0,
                                sizeof(uint16_t), sizeof(uint16_t));
+       map->created_uid_validity = ioloop_time;
        return map;
 }
 
@@ -888,7 +889,7 @@ int dbox_map_append_move(struct dbox_map_append_context *ctx,
        while (seq_range_array_iter_nth(&iter, i++, &uid)) {
                if (!mail_index_lookup_seq(ctx->sync_view, uid, &seq))
                        i_unreached();
-               mail_index_expunge(ctx->trans, seq);
+               mail_index_expunge(ctx->sync_trans, seq);
        }
        return 0;
 }
@@ -976,3 +977,14 @@ void dbox_map_append_free(struct dbox_map_append_context **_ctx)
        array_free(&ctx->files);
        i_free(ctx);
 }
+
+uint32_t dbox_map_get_uid_validity(struct dbox_map *map)
+{
+       const struct mail_index_header *hdr;
+
+       i_assert(map->view != NULL);
+
+       hdr = mail_index_get_header(map->view);
+       return hdr->uid_validity != 0 ? hdr->uid_validity :
+               map->created_uid_validity;
+}

diff --git a/src/lib-storage/index/dbox/dbox-map.h b/src/lib-storage/index/dbox/dbox-map.h
index 0d9369ac1851d8cf1dbcddb4b96860d8e7615c94..8147ac117e357322ea3eaff08ba9a972a0e20915 100644 (file)
--- a/src/lib-storage/index/dbox/dbox-map.h
+++ b/src/lib-storage/index/dbox/dbox-map.h
@@ -16,7 +16,7 @@ struct dbox_mail_index_map_header {
 struct dbox_mail_index_map_record {
        uint32_t file_id;
        uint32_t offset;
-       uint32_t size;
+       uint32_t size; /* including pre/post metadata */
 };
 
 struct dbox_map_file_msg {
@@ -29,6 +29,10 @@ ARRAY_DEFINE_TYPE(dbox_map_file_msg, struct dbox_map_file_msg);
 struct dbox_map *dbox_map_init(struct dbox_storage *storage);
 void dbox_map_deinit(struct dbox_map **map);
 
+/* Open the map. This is done automatically for most operations.
+   Returns 0 if ok, -1 if error. */
+int dbox_map_open(struct dbox_map *map);
+
 /* Look up file_id and offset for given map UID. Returns 1 if ok, 0 if UID
    is already expunged, -1 if error. */
 int dbox_map_lookup(struct dbox_map *map, uint32_t map_uid,
@@ -77,6 +81,10 @@ int dbox_map_append_move(struct dbox_map_append_context *ctx,
 int dbox_map_append_commit(struct dbox_map_append_context *ctx);
 void dbox_map_append_free(struct dbox_map_append_context **ctx);
 
+/* Get either existing uidvalidity or create a new one if map was
+   just created. */
+uint32_t dbox_map_get_uid_validity(struct dbox_map *map);
+
 void dbox_map_set_corrupted(struct dbox_map *map, const char *format, ...)
        ATTR_FORMAT(2, 3);
 

diff --git a/src/lib-storage/index/dbox/dbox-storage-rebuild.c b/src/lib-storage/index/dbox/dbox-storage-rebuild.c
index 207d8ed589cb41717dbc5a27c742fbabfc06a90e..5c4a45e43141638574ae362fe742da3cf316ebf1 100644 (file)
--- a/src/lib-storage/index/dbox/dbox-storage-rebuild.c
+++ b/src/lib-storage/index/dbox/dbox-storage-rebuild.c
@@ -2,6 +2,7 @@
 
 #include "lib.h"
 #include "array.h"
+#include "istream.h"
 #include "hash.h"
 #include "hex-binary.h"
 #include "str.h"
@@ -180,13 +181,14 @@ static int rebuild_add_file(struct dbox_storage_rebuild_context *ctx,
                rec = p_new(ctx->pool, struct dbox_rebuild_msg, 1);
                rec->file_id = file_id;
                rec->offset = offset;
-               rec->size = file->cur_physical_size;
+               rec->size = file->input->v_offset - offset;
                memcpy(rec->guid_128, guid_buf->data, sizeof(rec->guid_128));
                array_append(&ctx->msgs, &rec, 1);
 
                if (hash_table_lookup(ctx->guid_hash, guid_buf->data) != NULL) {
                        /* duplicate. save this as a refcount=0 to map,
                           so it will eventually be deleted. */
+                       rec->seen_zero_ref_in_map = TRUE;
                } else {
                        hash_table_insert(ctx->guid_hash, rec->guid_128, rec);
                }
@@ -255,7 +257,8 @@ static int rebuild_apply_map(struct dbox_storage_rebuild_context *ctx)
                        mail_index_expunge(ctx->trans, seq);
                } else {
                        (*pos)->map_uid = rec.map_uid;
-                       (*pos)->seen_zero_ref_in_map = rec.refcount == 0;
+                       if (rec.refcount == 0)
+                               (*pos)->seen_zero_ref_in_map = TRUE;
                }
        }
        rebuild_add_missing_map_uids(ctx, hdr->next_uid);
@@ -386,7 +389,7 @@ rebuild_mailbox(struct dbox_storage_rebuild_context *ctx, const char *name)
                return -1;
        }
 
-       rebuild_ctx = dbox_sync_index_rebuild_init(mbox, view, trans);
+       rebuild_ctx = dbox_sync_index_rebuild_init(mbox, view, trans, TRUE);
        ret = dbox_sync_index_rebuild_singles(rebuild_ctx);
        if (ret == 0)
                rebuild_mailbox_multi(ctx, rebuild_ctx, mbox, view, trans);
@@ -456,7 +459,7 @@ static int rebuild_restore_msg(struct dbox_storage_rebuild_context *ctx,
        file = dbox_file_init_multi(ctx->storage, msg->file_id);
        ret = dbox_file_get_mail_stream(file, msg->offset, &size, NULL,
                                        &expunged);
-       if (ret > 0 && !expunged && dbox_file_metadata_read(file) == 0) {
+       if (ret > 0 && !expunged && dbox_file_metadata_read(file) > 0) {
                mailbox = dbox_file_metadata_get(file,
                                                 DBOX_METADATA_ORIG_MAILBOX);
        }
@@ -533,6 +536,8 @@ static int rebuild_restore_msg(struct dbox_storage_rebuild_context *ctx,
        mail_index_append(ctx->prev_msg.trans, ctx->prev_msg.next_uid++, &seq);
        mail_index_update_ext(ctx->prev_msg.trans, seq, mbox->dbox_ext_id,
                              &dbox_rec, NULL);
+       mail_index_update_ext(ctx->prev_msg.trans, seq, mbox->guid_ext_id,
+                             msg->guid_128, NULL);
 
        msg->refcount++;
        return 0;
@@ -619,10 +624,12 @@ static int rebuild_finish(struct dbox_storage_rebuild_context *ctx)
 
 static int dbox_storage_rebuild_scan(struct dbox_storage_rebuild_context *ctx)
 {
+       const struct mail_index_header *hdr;
        DIR *dir;
        struct dirent *d;
        string_t *path;
        unsigned int dir_len;
+       uint32_t uid_validity;
        int ret = 0;
 
        if (dbox_map_open(ctx->storage->map) < 0)
@@ -639,6 +646,14 @@ static int dbox_storage_rebuild_scan(struct dbox_storage_rebuild_context *ctx)
                return -1;
        }
 
+       uid_validity = dbox_map_get_uid_validity(ctx->storage->map);
+       hdr = mail_index_get_header(ctx->sync_view);
+       if (hdr->uid_validity != uid_validity) {
+               mail_index_update_header(ctx->trans,
+                       offsetof(struct mail_index_header, uid_validity),
+                       &uid_validity, sizeof(uid_validity), TRUE);
+       }
+
        dir = opendir(ctx->storage->storage_dir);
        if (dir == NULL) {
                mail_storage_set_critical(&ctx->storage->storage,
@@ -700,6 +715,7 @@ int dbox_storage_rebuild(struct dbox_storage *storage)
                        "stat(%s) failed: %m", storage->storage_dir);
                return -1;
        }
+       storage->have_multi_msgs = TRUE;
 
        if (storage->sync_rebuild)
                i_warning("dbox %s: rebuilding indexes", storage->storage_dir);

diff --git a/src/lib-storage/index/dbox/dbox-storage.h b/src/lib-storage/index/dbox/dbox-storage.h
index 87e9699c917679ad60744ba68d63b31772229261..17d62856f3ecea9915c403da951d305e8b7d2e22 100644 (file)
--- a/src/lib-storage/index/dbox/dbox-storage.h
+++ b/src/lib-storage/index/dbox/dbox-storage.h
@@ -33,7 +33,7 @@
 #define DBOX_INDEX_FLAG_ALT MAIL_INDEX_MAIL_FLAG_BACKEND
 
 struct dbox_index_header {
-       uint32_t unused; /* for backwards compatibility */
+       uint32_t map_uid_validity;
        uint32_t highest_maildir_uid;
 };
 
@@ -57,6 +57,7 @@ struct dbox_storage {
        ARRAY_DEFINE(open_files, struct dbox_file *);
 
        unsigned int sync_rebuild:1;
+       unsigned int have_multi_msgs:1;
 };
 
 struct dbox_mail_index_record {
@@ -69,6 +70,7 @@ struct dbox_mailbox {
 
        struct maildir_uidlist *maildir_uidlist;
        uint32_t highest_maildir_uid;
+       uint32_t map_uid_validity;
 
        uint32_t dbox_ext_id, dbox_hdr_ext_id, guid_ext_id;
 
@@ -98,8 +100,8 @@ dbox_mail_alloc(struct mailbox_transaction_context *t,
                struct mailbox_header_lookup_ctx *wanted_headers);
 
 /* Get map_uid for wanted message. */
-uint32_t dbox_mail_lookup(struct dbox_mailbox *mbox,
-                         struct mail_index_view *view, uint32_t seq);
+int dbox_mail_lookup(struct dbox_mailbox *mbox, struct mail_index_view *view,
+                    uint32_t seq, uint32_t *map_uid_r);
 
 struct mail_save_context *
 dbox_save_alloc(struct mailbox_transaction_context *_t);

diff --git a/src/lib-storage/index/dbox/dbox-sync-rebuild.c b/src/lib-storage/index/dbox/dbox-sync-rebuild.c
index e39d306eba91b7245fa77dc3e4e18d5faa452740..9d8e23b0b59480859aa1364f1280078df23b2456 100644 (file)
--- a/src/lib-storage/index/dbox/dbox-sync-rebuild.c
+++ b/src/lib-storage/index/dbox/dbox-sync-rebuild.c
@@ -7,6 +7,7 @@
 #include "maildir/maildir-uidlist.h"
 #include "maildir/maildir-keywords.h"
 #include "maildir/maildir-filename.h"
+#include "dbox-map.h"
 #include "dbox-file.h"
 #include "dbox-sync.h"
 
@@ -31,6 +32,7 @@ struct dbox_sync_rebuild_context {
        uint32_t highest_uid;
 
        unsigned int cache_used:1;
+       unsigned int storage_rebuild:1;
 };
 
 static uint32_t dbox_get_uidvalidity_next(struct mail_storage *storage)
@@ -365,16 +367,14 @@ static void dbox_sync_update_header(struct dbox_sync_rebuild_context *ctx)
                                  ctx->mbox->dbox_hdr_ext_id,
                                  &data, &data_size);
        hdr = data;
-       if (data_size == sizeof(*hdr)) {
-               if (hdr->highest_maildir_uid >= ctx->mbox->highest_maildir_uid) {
-                       /* nothing to change */
-                       return;
-               }
+       if (data_size == sizeof(*hdr))
                new_hdr = *hdr;
-       } else {
+       else
                memset(&new_hdr, 0, sizeof(new_hdr));
-       }
-       new_hdr.highest_maildir_uid = ctx->mbox->highest_maildir_uid;
+       if (new_hdr.highest_maildir_uid < ctx->mbox->highest_maildir_uid)
+               new_hdr.highest_maildir_uid = ctx->mbox->highest_maildir_uid;
+       new_hdr.map_uid_validity = !ctx->storage_rebuild ? 0 :
+               dbox_map_get_uid_validity(ctx->mbox->storage->map);
        mail_index_update_header_ext(ctx->trans, ctx->mbox->dbox_hdr_ext_id, 0,
                                     &new_hdr, sizeof(new_hdr));
 }
@@ -382,7 +382,8 @@ static void dbox_sync_update_header(struct dbox_sync_rebuild_context *ctx)
 struct dbox_sync_rebuild_context *
 dbox_sync_index_rebuild_init(struct dbox_mailbox *mbox,
                             struct mail_index_view *view,
-                            struct mail_index_transaction *trans)
+                            struct mail_index_transaction *trans,
+                            bool storage_rebuild)
 {
        struct mailbox *box = &mbox->ibox.box;
        struct dbox_sync_rebuild_context *ctx;
@@ -393,6 +394,7 @@ dbox_sync_index_rebuild_init(struct dbox_mailbox *mbox,
        ctx->mbox = mbox;
        ctx->view = view;
        ctx->trans = trans;
+       ctx->storage_rebuild = storage_rebuild;
        mail_index_reset(ctx->trans);
        index_mailbox_reset_uidvalidity(&mbox->ibox);
        mail_index_ext_lookup(mbox->ibox.index, "cache", &ctx->cache_ext_id);
@@ -468,7 +470,7 @@ int dbox_sync_index_rebuild(struct dbox_mailbox *mbox)
        trans = mail_index_transaction_begin(view,
                                        MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL);
 
-       ctx = dbox_sync_index_rebuild_init(mbox, view, trans);
+       ctx = dbox_sync_index_rebuild_init(mbox, view, trans, FALSE);
        ret = dbox_sync_index_rebuild_singles(ctx);
        dbox_sync_index_rebuild_deinit(&ctx);
 

diff --git a/src/lib-storage/index/dbox/dbox-sync.c b/src/lib-storage/index/dbox/dbox-sync.c
index 3433152eb08ac625a2914f16922e1e58a7ad4d29..e4d6ea8c4d75548992a8f0195084f36fe678d0e9 100644 (file)
--- a/src/lib-storage/index/dbox/dbox-sync.c
+++ b/src/lib-storage/index/dbox/dbox-sync.c
@@ -48,7 +48,8 @@ static int dbox_sync_add_seq(struct dbox_sync_context *ctx,
                 sync_rec->type == MAIL_INDEX_SYNC_TYPE_FLAGS);
 
        memset(&lookup_entry, 0, sizeof(lookup_entry));
-       map_uid = dbox_mail_lookup(ctx->mbox, ctx->sync_view, seq);
+       if (dbox_mail_lookup(ctx->mbox, ctx->sync_view, seq, &map_uid) < 0)
+               return 0;
        if (map_uid == 0)
                mail_index_lookup_uid(ctx->sync_view, seq, &lookup_entry.uid);
        else {
@@ -136,7 +137,6 @@ static int dbox_sync_index(struct dbox_sync_context *ctx)
        hdr = mail_index_get_header(ctx->sync_view);
        if (hdr->uid_validity == 0) {
                /* newly created index file */
-               i_warning("FIXME: newly created");
                return 0;
        }
 
@@ -197,8 +197,7 @@ static int dbox_refresh_header(struct dbox_mailbox *mbox)
                if (data_size != 0 && data_size != 4) {
                        i_warning("dbox %s: Invalid dbox header size",
                                  mbox->path);
-               } else
-                       i_warning("FIXME: initial sync");
+               }
                return -1;
        }
        hdr = data;
@@ -252,17 +251,9 @@ int dbox_sync_begin(struct dbox_mailbox *mbox, enum dbox_sync_flags flags,
                }
 
                /* now that we're locked, check again if we want to rebuild */
-               if (dbox_refresh_header(mbox) < 0) {
-                       if (!storage_rebuilt) {
-                               /* we'll need to rebuild storage too.
-                                  try again from the beginning. */
-                               mail_index_sync_rollback(&ctx->index_sync_ctx);
-                               i_free(ctx);
-                               return dbox_sync_begin(mbox, flags, ctx_r);
-                       }
+               if (dbox_refresh_header(mbox) < 0)
                        ret = 0;
-                       i_warning("FIXME: poop");
-               } else {
+               else {
                        if ((ret = dbox_sync_index(ctx)) > 0)
                                break;
                }
@@ -270,7 +261,19 @@ int dbox_sync_begin(struct dbox_mailbox *mbox, enum dbox_sync_flags flags,
                /* failure. keep the index locked while we're doing a
                   rebuild. */
                if (ret == 0) {
-                       if (i >= DBOX_REBUILD_COUNT) {
+                       if (!storage_rebuilt) {
+                               /* we'll need to rebuild storage too.
+                                  try again from the beginning. */
+                               mail_index_sync_rollback(&ctx->index_sync_ctx);
+                               i_free(ctx);
+                               return dbox_sync_begin(mbox, flags, ctx_r);
+                       }
+                       if (mbox->storage->have_multi_msgs) {
+                               mail_storage_set_critical(storage,
+                                       "dbox %s: Storage keeps breaking",
+                                       ctx->mbox->path);
+                               ret = -1;
+                       } else if (i >= DBOX_REBUILD_COUNT) {
                                mail_storage_set_critical(storage,
                                        "dbox %s: Index keeps breaking",
                                        ctx->mbox->path);

diff --git a/src/lib-storage/index/dbox/dbox-sync.h b/src/lib-storage/index/dbox/dbox-sync.h
index e19d8d8d3888b1f487f0fbe61f12e5e8645c4f7b..febfa4bc35aff2b649ecc69c37e7ef915b5e7751 100644 (file)
--- a/src/lib-storage/index/dbox/dbox-sync.h
+++ b/src/lib-storage/index/dbox/dbox-sync.h
@@ -45,7 +45,8 @@ int dbox_sync_file_cleanup(struct dbox_file *file);
 struct dbox_sync_rebuild_context *
 dbox_sync_index_rebuild_init(struct dbox_mailbox *mbox,
                             struct mail_index_view *view,
-                            struct mail_index_transaction *trans);
+                            struct mail_index_transaction *trans,
+                            bool storage_rebuild);
 int dbox_sync_index_rebuild_singles(struct dbox_sync_rebuild_context *ctx);
 void dbox_sync_rebuild_index_metadata(struct dbox_sync_rebuild_context *ctx,
                                      struct dbox_file *file,