mdbox: Avoid rebuilding storage if another process already did it.

--HG--
branch : HEAD

diff --git a/src/lib-storage/index/dbox-common/dbox-file.c b/src/lib-storage/index/dbox-common/dbox-file.c
index 047df7c3472a38ee6d83f353d322082812197229..39a89812c65c65695ae1325e6d0462d40d05f271 100644 (file)
--- a/src/lib-storage/index/dbox-common/dbox-file.c
+++ b/src/lib-storage/index/dbox-common/dbox-file.c
@@ -54,14 +54,14 @@ void dbox_file_set_corrupted(struct dbox_file *file, const char *reason, ...)
 {
        va_list args;
 
-       file->storage->files_corrupted = TRUE;
-
        va_start(args, reason);
        mail_storage_set_critical(&file->storage->storage,
                "Corrupted dbox file %s (around offset=%"PRIuUOFF_T"): %s",
                file->cur_path, file->input == NULL ? 0 : file->input->v_offset,
                t_strdup_vprintf(reason, args));
        va_end(args);
+
+       file->storage->v.set_file_corrupted(file);
 }
 
 void dbox_file_init(struct dbox_file *file)

diff --git a/src/lib-storage/index/dbox-common/dbox-storage.h b/src/lib-storage/index/dbox-common/dbox-storage.h
index ea59d938ce5158d67b64b8fbb8c75b2bfb693fe1..5f22639552eab741735bf216b59369dfc12ee169 100644 (file)
--- a/src/lib-storage/index/dbox-common/dbox-storage.h
+++ b/src/lib-storage/index/dbox-common/dbox-storage.h
@@ -5,6 +5,7 @@
 
 struct dbox_file;
 struct dbox_mail;
+struct dbox_storage;
 
 #define DBOX_SUBSCRIPTION_FILE_NAME "subscriptions"
 #define DBOX_UIDVALIDITY_FILE_NAME "dovecot-uidvalidity"
@@ -37,13 +38,13 @@ struct dbox_storage_vfuncs {
        /* create/update mailbox indexes */
        int (*mailbox_create_indexes)(struct mailbox *box,
                                      const struct mailbox_update *update);
+       /* mark the file corrupted */
+       void (*set_file_corrupted)(struct dbox_file *file);
 };
 
 struct dbox_storage {
        struct mail_storage storage;
        struct dbox_storage_vfuncs v;
-
-       unsigned int files_corrupted:1;
 };
 
 void dbox_storage_get_list_settings(const struct mail_namespace *ns,

diff --git a/src/lib-storage/index/dbox-multi/mdbox-mail.c b/src/lib-storage/index/dbox-multi/mdbox-mail.c
index c9d8c4e4dae6a02afd9237562f9f1c596f82f5ae..2d5701985fd6adddc482a5edc7006d23ef7bb800 100644 (file)
--- a/src/lib-storage/index/dbox-multi/mdbox-mail.c
+++ b/src/lib-storage/index/dbox-multi/mdbox-mail.c
@@ -30,13 +30,13 @@ int mdbox_mail_lookup(struct mdbox_mailbox *mbox, struct mail_index_view *view,
                mail_storage_set_critical(&mbox->storage->storage.storage,
                        "dbox %s: map uid lost for uid %u",
                        mbox->box.path, uid);
-               mbox->storage->storage.files_corrupted = TRUE;
+               mdbox_storage_set_corrupted(mbox->storage);
                return -1;
        }
 
        if (mbox->map_uid_validity == 0) {
                if (mdbox_read_header(mbox, &hdr) < 0) {
-                       mbox->storage->storage.files_corrupted = TRUE;
+                       mdbox_storage_set_corrupted(mbox->storage);
                        return -1;
                }
                mbox->map_uid_validity = hdr.map_uid_validity;
@@ -50,7 +50,7 @@ int mdbox_mail_lookup(struct mdbox_mailbox *mbox, struct mail_index_view *view,
                        "dbox %s: map uidvalidity mismatch (%u vs %u)",
                        mbox->box.path, mbox->map_uid_validity,
                        cur_map_uid_validity);
-               mbox->storage->storage.files_corrupted = TRUE;
+               mdbox_storage_set_corrupted(mbox->storage);
                return -1;
        }
        *map_uid_r = dbox_rec->map_uid;
@@ -72,7 +72,6 @@ static void dbox_mail_set_expunged(struct dbox_mail *mail, uint32_t map_uid)
                               "Unexpectedly lost %s uid=%u map_uid=%u",
                               mailbox_get_vname(_mail->box),
                               _mail->uid, map_uid);
-       mbox->storage->storage.files_corrupted = TRUE;
 }
 
 static int dbox_mail_open_init(struct dbox_mail *mail, uint32_t map_uid)

diff --git a/src/lib-storage/index/dbox-multi/mdbox-map.c b/src/lib-storage/index/dbox-multi/mdbox-map.c
index 1d827588ec0771d46b95f6927b7c6bfdc21fbdd3..bcacd679e4d3b9b501559800dbcf932814c84710 100644 (file)
--- a/src/lib-storage/index/dbox-multi/mdbox-map.c
+++ b/src/lib-storage/index/dbox-multi/mdbox-map.c
@@ -33,14 +33,14 @@ void dbox_map_set_corrupted(struct dbox_map *map, const char *format, ...)
 {
        va_list args;
 
-       map->storage->storage.files_corrupted = TRUE;
-
        va_start(args, format);
        mail_storage_set_critical(MAP_STORAGE(map),
                                  "dbox map %s corrupted: %s",
                                  map->index->filepath,
                                  t_strdup_vprintf(format, args));
        va_end(args);
+
+       mdbox_storage_set_corrupted(map->storage);
 }
 
 struct dbox_map *
@@ -188,6 +188,26 @@ int dbox_map_refresh(struct dbox_map *map)
        return 0;
 }
 
+static void
+mdbox_map_get_ext_hdr(struct dbox_map *map, struct mail_index_view *view,
+                     struct dbox_map_mail_index_header *hdr_r)
+{
+       const void *data;
+       size_t data_size;
+
+       mail_index_get_header_ext(view, map->map_ext_id, &data, &data_size);
+       memset(hdr_r, 0, sizeof(*hdr_r));
+       memcpy(hdr_r, data, I_MIN(data_size, sizeof(*hdr_r)));
+}
+
+uint32_t mdbox_map_get_rebuild_count(struct dbox_map *map)
+{
+       struct dbox_map_mail_index_header hdr;
+
+       mdbox_map_get_ext_hdr(map, map->view, &hdr);
+       return hdr.rebuild_count;
+}
+
 static int dbox_map_lookup_seq(struct dbox_map *map, uint32_t seq,
                               const struct dbox_map_mail_index_record **rec_r)
 {
@@ -397,7 +417,7 @@ dbox_map_sync_handle(struct dbox_map *map, struct mail_index_sync_ctx *sync_ctx)
                i_warning("dbox %s: Inconsistency in map index "
                          "(%u,%"PRIuUOFF_T" != %u,%"PRIuUOFF_T")",
                          map->path, seq1, offset1, seq2, offset2);
-               map->storage->storage.files_corrupted = TRUE;
+               mdbox_storage_set_corrupted(map->storage);
        } else {
                while (mail_index_sync_next(sync_ctx, &sync_rec)) ;
        }
@@ -962,35 +982,12 @@ void dbox_map_append_finish(struct dbox_map_append_context *ctx)
        appends[count-1].size = cur_offset - appends[count-1].offset;
 }
 
-static int
-dbox_map_get_next_file_id(struct dbox_map *map, struct mail_index_view *view,
-                         uint32_t *file_id_r)
-{
-       const struct dbox_map_mail_index_header *hdr;
-       const void *data;
-       size_t data_size;
-
-       mail_index_get_header_ext(view, map->map_ext_id, &data, &data_size);
-       if (data_size != sizeof(*hdr)) {
-               if (data_size != 0) {
-                       dbox_map_set_corrupted(map, "hdr size=%"PRIuSIZE_T,
-                                              data_size);
-                       return -1;
-               }
-               /* first file */
-               *file_id_r = 1;
-       } else {
-               hdr = data;
-               *file_id_r = hdr->highest_file_id + 1;
-       }
-       return 0;
-}
-
 static int dbox_map_assign_file_ids(struct dbox_map_append_context *ctx,
                                    bool separate_transaction)
 {
        struct dbox_file_append_context *const *file_appends;
        unsigned int i, count;
+       struct dbox_map_mail_index_header hdr;
        uint32_t first_file_id, file_id;
        int ret;
 
@@ -1006,10 +1003,8 @@ static int dbox_map_assign_file_ids(struct dbox_map_append_context *ctx,
        }
        dbox_map_sync_handle(ctx->map, ctx->sync_ctx);
 
-       if (dbox_map_get_next_file_id(ctx->map, ctx->sync_view, &file_id) < 0) {
-               mail_index_sync_rollback(&ctx->sync_ctx);
-               return -1;
-       }
+       mdbox_map_get_ext_hdr(ctx->map, ctx->sync_view, &hdr);
+       file_id = hdr.highest_file_id + 1;
 
        /* assign file_ids for newly created files */
        first_file_id = file_id;

diff --git a/src/lib-storage/index/dbox-multi/mdbox-map.h b/src/lib-storage/index/dbox-multi/mdbox-map.h
index 513c5f838afa09d6c37c2484f782874dc51eae5a..71ceec6af820d2daae6592ec43869af2f80ac554 100644 (file)
--- a/src/lib-storage/index/dbox-multi/mdbox-map.h
+++ b/src/lib-storage/index/dbox-multi/mdbox-map.h
@@ -13,6 +13,8 @@ enum dbox_map_append_flags {
 
 struct dbox_map_mail_index_header {
        uint32_t highest_file_id;
+       /* increased every time storage is rebuilt */
+       uint32_t rebuild_count;
 };
 
 struct dbox_map_mail_index_record {
@@ -41,6 +43,9 @@ int dbox_map_open_or_create(struct dbox_map *map);
 /* Refresh the map. Returns 0 if ok, -1 if error. */
 int dbox_map_refresh(struct dbox_map *map);
 
+/* Return the current rebuild counter */
+uint32_t mdbox_map_get_rebuild_count(struct dbox_map *map);
+
 /* Look up file_id and offset for given map UID. Returns 1 if ok, 0 if UID
    is already expunged, -1 if error. */
 int dbox_map_lookup(struct dbox_map *map, uint32_t map_uid,

diff --git a/src/lib-storage/index/dbox-multi/mdbox-purge.c b/src/lib-storage/index/dbox-multi/mdbox-purge.c
index fd7d238a5e0b5ca59b2203ed03f0f51d89babdd7..90535b436ec07a05c18e89a70ff9a6d09285a508 100644 (file)
--- a/src/lib-storage/index/dbox-multi/mdbox-purge.c
+++ b/src/lib-storage/index/dbox-multi/mdbox-purge.c
@@ -533,7 +533,7 @@ int mdbox_purge(struct mail_storage *_storage)
        } T_END;
        mdbox_purge_free(&ctx);
 
-       if (storage->storage.files_corrupted) {
+       if (storage->corrupted) {
                /* purging found corrupted files */
                (void)mdbox_storage_rebuild(storage);
                ret = -1;

diff --git a/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c b/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c
index 40e549e3dd8f51d99d5d9f432b789b30058ad0c1..9d71e8827344880ceb13de515a94f749cf9b0717 100644 (file)
--- a/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c
+++ b/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c
@@ -43,6 +43,7 @@ struct mdbox_storage_rebuild_context {
        struct hash_table *guid_hash;
        ARRAY_DEFINE(msgs, struct mdbox_rebuild_msg *);
 
+       uint32_t rebuild_count;
        uint32_t prev_file_id;
        uint32_t highest_seen_map_uid;
 
@@ -743,6 +744,11 @@ static int rebuild_finish(struct mdbox_storage_rebuild_context *ctx)
        if (rebuild_handle_zero_refs(ctx) < 0)
                return -1;
        rebuild_update_refcounts(ctx);
+
+       ctx->rebuild_count++;
+       mail_index_update_header_ext(ctx->trans, ctx->storage->map->map_ext_id,
+               offsetof(struct dbox_map_mail_index_header, rebuild_count),
+               &ctx->rebuild_count, sizeof(ctx->rebuild_count));
        return 0;
 }
 
@@ -817,6 +823,16 @@ static int mdbox_storage_rebuild_scan(struct mdbox_storage_rebuild_context *ctx)
                return -1;
        }
 
+       /* get storage rebuild counter after locking */
+       ctx->rebuild_count = mdbox_map_get_rebuild_count(ctx->storage->map);
+       if (ctx->rebuild_count != ctx->storage->corrupted_rebuild_count &&
+           ctx->storage->corrupted) {
+               /* storage was already rebuilt by someone else */
+               return 0;
+       }
+
+       i_warning("dbox %s: rebuilding indexes", ctx->storage->storage_dir);
+
        uid_validity = dbox_map_get_uid_validity(ctx->storage->map);
        hdr = mail_index_get_header(ctx->sync_view);
        if (hdr->uid_validity != uid_validity) {
@@ -859,13 +875,13 @@ int mdbox_storage_rebuild(struct mdbox_storage *storage)
                return -1;
        }
 
-       i_warning("dbox %s: rebuilding indexes", storage->storage_dir);
-
        ctx = mdbox_storage_rebuild_init(storage);
        ret = mdbox_storage_rebuild_scan(ctx);
        mdbox_storage_rebuild_deinit(ctx);
 
-       if (ret == 0)
-               storage->storage.files_corrupted = FALSE;
+       if (ret == 0) {
+               storage->corrupted = FALSE;
+               storage->corrupted_rebuild_count = 0;
+       }
        return ret;
 }

diff --git a/src/lib-storage/index/dbox-multi/mdbox-storage.c b/src/lib-storage/index/dbox-multi/mdbox-storage.c
index 8c95ae29a7342654e62502e9c1cccc5bd8dc9b88..83540da78772232a1803ac7dc6242ac45b93ef70 100644 (file)
--- a/src/lib-storage/index/dbox-multi/mdbox-storage.c
+++ b/src/lib-storage/index/dbox-multi/mdbox-storage.c
@@ -127,8 +127,7 @@ static void mdbox_mailbox_close(struct mailbox *box)
 {
        struct mdbox_storage *mstorage = (struct mdbox_storage *)box->storage;
 
-       if (mstorage->storage.files_corrupted &&
-           !mstorage->rebuilding_storage)
+       if (mstorage->corrupted && !mstorage->rebuilding_storage)
                (void)mdbox_storage_rebuild(mstorage);
 
        index_storage_mailbox_close(box);
@@ -145,8 +144,9 @@ int mdbox_read_header(struct mdbox_mailbox *mbox,
        if (data_size < MDBOX_INDEX_HEADER_MIN_SIZE &&
            (!mbox->creating || data_size != 0)) {
                mail_storage_set_critical(&mbox->storage->storage.storage,
-                       "dbox %s: Invalid dbox header size",
-                       mbox->box.path);
+                       "dbox %s: Invalid dbox header size: %"PRIuSIZE_T,
+                       mbox->box.path, data_size);
+               mdbox_storage_set_corrupted(mbox->storage);
                return -1;
        }
        memset(hdr, 0, sizeof(*hdr));
@@ -244,6 +244,30 @@ static int mdbox_mailbox_create_indexes(struct mailbox *box,
        return ret;
 }
 
+void mdbox_storage_set_corrupted(struct mdbox_storage *storage)
+{
+       if (storage->corrupted) {
+               /* already set it corrupted (possibly recursing back here) */
+               return;
+       }
+
+       storage->corrupted = TRUE;
+       storage->corrupted_rebuild_count = (uint32_t)-1;
+
+       if (dbox_map_open(storage->map) > 0 &&
+           dbox_map_refresh(storage->map) == 0) {
+               storage->corrupted_rebuild_count =
+                       mdbox_map_get_rebuild_count(storage->map);
+       }
+}
+
+static void mdbox_set_file_corrupted(struct dbox_file *file)
+{
+       struct mdbox_storage *mstorage = (struct mdbox_storage *)file->storage;
+
+       mdbox_storage_set_corrupted(mstorage);
+}
+
 static int
 mdbox_mailbox_get_guid(struct mailbox *box, uint8_t guid[MAIL_GUID_128_SIZE])
 {
@@ -388,5 +412,6 @@ struct dbox_storage_vfuncs mdbox_dbox_storage_vfuncs = {
        mdbox_file_unrefed,
        mdbox_file_create_fd,
        mdbox_mail_open,
-       mdbox_mailbox_create_indexes
+       mdbox_mailbox_create_indexes,
+       mdbox_set_file_corrupted
 };

diff --git a/src/lib-storage/index/dbox-multi/mdbox-storage.h b/src/lib-storage/index/dbox-multi/mdbox-storage.h
index 1bb431a173ab6f4da3ffb99f2c1e6d4ab33aa63a..e08bc469f8f80ec618da8128c1b2642c63ef7060 100644 (file)
--- a/src/lib-storage/index/dbox-multi/mdbox-storage.h
+++ b/src/lib-storage/index/dbox-multi/mdbox-storage.h
@@ -35,6 +35,11 @@ struct mdbox_storage {
        ARRAY_TYPE(uint32_t) move_to_alt_map_uids;
        ARRAY_TYPE(uint32_t) move_from_alt_map_uids;
 
+       /* if non-zero, storage should be rebuilt (except if rebuild_count
+          has changed from this value) */
+       uint32_t corrupted_rebuild_count;
+
+       unsigned int corrupted:1;
        unsigned int rebuilding_storage:1;
 };
 
@@ -94,4 +99,6 @@ int mdbox_copy(struct mail_save_context *ctx, struct mail *mail);
 void mdbox_purge_alt_flag_change(struct mail *mail, bool move_to_alt);
 int mdbox_purge(struct mail_storage *storage);
 
+void mdbox_storage_set_corrupted(struct mdbox_storage *storage);
+
 #endif

diff --git a/src/lib-storage/index/dbox-multi/mdbox-sync.c b/src/lib-storage/index/dbox-multi/mdbox-sync.c
index 6557262063e0bd7e4f96eb1c3fddc81975a15b90..863afec69f6aa1264948ad440172db9c43536a31 100644 (file)
--- a/src/lib-storage/index/dbox-multi/mdbox-sync.c
+++ b/src/lib-storage/index/dbox-multi/mdbox-sync.c
@@ -44,7 +44,7 @@ dbox_sync_verify_expunge_guid(struct mdbox_sync_context *ctx, uint32_t seq,
                ctx->mbox->box.vname, uid,
                binary_to_hex(data, MAIL_GUID_128_SIZE),
                binary_to_hex(guid_128, MAIL_GUID_128_SIZE));
-       ctx->mbox->storage->storage.files_corrupted = TRUE;
+       mdbox_storage_set_corrupted(ctx->mbox->storage);
        return -1;
 }
 
@@ -175,7 +175,7 @@ static int mdbox_sync_index(struct mdbox_sync_context *ctx)
 
        array_free(&ctx->expunged_seqs);
        return ret == 0 ? 1 :
-               (ctx->mbox->storage->storage.files_corrupted ? 0 : -1);
+               (ctx->mbox->storage->corrupted ? 0 : -1);
 }
 
 static int mdbox_refresh_header(struct mdbox_mailbox *mbox, bool retry)
@@ -189,7 +189,7 @@ static int mdbox_refresh_header(struct mdbox_mailbox *mbox, bool retry)
        mail_index_view_close(&view);
 
        if (ret == 0) {
-               ret = mbox->storage->storage.files_corrupted ? -1 : 0;
+               ret = mbox->storage->corrupted ? -1 : 0;
        } else if (retry) {
                (void)mail_index_refresh(mbox->box.index);
                return mdbox_refresh_header(mbox, FALSE);
@@ -255,7 +255,7 @@ int mdbox_sync_begin(struct mdbox_mailbox *mbox, enum mdbox_sync_flags flags,
                        if (!storage_rebuilt) {
                                /* we'll need to rebuild storage too.
                                   try again from the beginning. */
-                               mbox->storage->storage.files_corrupted = TRUE;
+                               mdbox_storage_set_corrupted(mbox->storage);
                                mail_index_sync_rollback(&ctx->index_sync_ctx);
                                i_free(ctx);
                                return mdbox_sync_begin(mbox, flags, ctx_r);
@@ -321,7 +321,7 @@ mdbox_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags)
        }
 
        if (ret == 0 && (index_mailbox_want_full_sync(&mbox->box, flags) ||
-                        mbox->storage->storage.files_corrupted)) {
+                        mbox->storage->corrupted)) {
                if ((flags & MAILBOX_SYNC_FLAG_FORCE_RESYNC) != 0)
                        mdbox_sync_flags |= MDBOX_SYNC_FLAG_FORCE_REBUILD;
                ret = mdbox_sync(mbox, mdbox_sync_flags);

diff --git a/src/lib-storage/index/dbox-single/sdbox-storage.c b/src/lib-storage/index/dbox-single/sdbox-storage.c
index 953cca6550422542d83aa238135b71acd10face8..856829885b110a764c0587da476039026b37ded4 100644 (file)
--- a/src/lib-storage/index/dbox-single/sdbox-storage.c
+++ b/src/lib-storage/index/dbox-single/sdbox-storage.c
@@ -189,6 +189,11 @@ static int sdbox_mailbox_create_indexes(struct mailbox *box,
        return ret;
 }
 
+static void sdbox_set_file_corrupted(struct dbox_file *file ATTR_UNUSED)
+{
+       /* FIXME */
+}
+
 static int
 sdbox_mailbox_get_guid(struct mailbox *box, uint8_t guid[MAIL_GUID_128_SIZE])
 {
@@ -294,5 +299,6 @@ struct dbox_storage_vfuncs sdbox_dbox_storage_vfuncs = {
        dbox_file_free,
        sdbox_file_create_fd,
        sdbox_mail_open,
-       sdbox_mailbox_create_indexes
+       sdbox_mailbox_create_indexes,
+       sdbox_set_file_corrupted
 };