From: Timo Sirainen Date: Thu, 29 Apr 2010 15:43:02 +0000 (+0300) Subject: mdbox: Avoid rebuilding storage if another process already did it. X-Git-Tag: 2.0.beta5~58 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=5b62dea2f88165f3f4d87bba9011343f3ff415ff;p=thirdparty%2Fdovecot%2Fcore.git mdbox: Avoid rebuilding storage if another process already did it. --HG-- branch : HEAD --- diff --git a/src/lib-storage/index/dbox-common/dbox-file.c b/src/lib-storage/index/dbox-common/dbox-file.c index 047df7c347..39a89812c6 100644 --- a/src/lib-storage/index/dbox-common/dbox-file.c +++ b/src/lib-storage/index/dbox-common/dbox-file.c @@ -54,14 +54,14 @@ void dbox_file_set_corrupted(struct dbox_file *file, const char *reason, ...) { va_list args; - file->storage->files_corrupted = TRUE; - va_start(args, reason); mail_storage_set_critical(&file->storage->storage, "Corrupted dbox file %s (around offset=%"PRIuUOFF_T"): %s", file->cur_path, file->input == NULL ? 0 : file->input->v_offset, t_strdup_vprintf(reason, args)); va_end(args); + + file->storage->v.set_file_corrupted(file); } void dbox_file_init(struct dbox_file *file) diff --git a/src/lib-storage/index/dbox-common/dbox-storage.h b/src/lib-storage/index/dbox-common/dbox-storage.h index ea59d938ce..5f22639552 100644 --- a/src/lib-storage/index/dbox-common/dbox-storage.h +++ b/src/lib-storage/index/dbox-common/dbox-storage.h @@ -5,6 +5,7 @@ struct dbox_file; struct dbox_mail; +struct dbox_storage; #define DBOX_SUBSCRIPTION_FILE_NAME "subscriptions" #define DBOX_UIDVALIDITY_FILE_NAME "dovecot-uidvalidity" @@ -37,13 +38,13 @@ struct dbox_storage_vfuncs { /* create/update mailbox indexes */ int (*mailbox_create_indexes)(struct mailbox *box, const struct mailbox_update *update); + /* mark the file corrupted */ + void (*set_file_corrupted)(struct dbox_file *file); }; struct dbox_storage { struct mail_storage storage; struct dbox_storage_vfuncs v; - - unsigned int files_corrupted:1; }; void dbox_storage_get_list_settings(const struct mail_namespace *ns, diff --git a/src/lib-storage/index/dbox-multi/mdbox-mail.c b/src/lib-storage/index/dbox-multi/mdbox-mail.c index c9d8c4e4da..2d5701985f 100644 --- a/src/lib-storage/index/dbox-multi/mdbox-mail.c +++ b/src/lib-storage/index/dbox-multi/mdbox-mail.c @@ -30,13 +30,13 @@ int mdbox_mail_lookup(struct mdbox_mailbox *mbox, struct mail_index_view *view, mail_storage_set_critical(&mbox->storage->storage.storage, "dbox %s: map uid lost for uid %u", mbox->box.path, uid); - mbox->storage->storage.files_corrupted = TRUE; + mdbox_storage_set_corrupted(mbox->storage); return -1; } if (mbox->map_uid_validity == 0) { if (mdbox_read_header(mbox, &hdr) < 0) { - mbox->storage->storage.files_corrupted = TRUE; + mdbox_storage_set_corrupted(mbox->storage); return -1; } mbox->map_uid_validity = hdr.map_uid_validity; @@ -50,7 +50,7 @@ int mdbox_mail_lookup(struct mdbox_mailbox *mbox, struct mail_index_view *view, "dbox %s: map uidvalidity mismatch (%u vs %u)", mbox->box.path, mbox->map_uid_validity, cur_map_uid_validity); - mbox->storage->storage.files_corrupted = TRUE; + mdbox_storage_set_corrupted(mbox->storage); return -1; } *map_uid_r = dbox_rec->map_uid; @@ -72,7 +72,6 @@ static void dbox_mail_set_expunged(struct dbox_mail *mail, uint32_t map_uid) "Unexpectedly lost %s uid=%u map_uid=%u", mailbox_get_vname(_mail->box), _mail->uid, map_uid); - mbox->storage->storage.files_corrupted = TRUE; } static int dbox_mail_open_init(struct dbox_mail *mail, uint32_t map_uid) diff --git a/src/lib-storage/index/dbox-multi/mdbox-map.c b/src/lib-storage/index/dbox-multi/mdbox-map.c index 1d827588ec..bcacd679e4 100644 --- a/src/lib-storage/index/dbox-multi/mdbox-map.c +++ b/src/lib-storage/index/dbox-multi/mdbox-map.c @@ -33,14 +33,14 @@ void dbox_map_set_corrupted(struct dbox_map *map, const char *format, ...) { va_list args; - map->storage->storage.files_corrupted = TRUE; - va_start(args, format); mail_storage_set_critical(MAP_STORAGE(map), "dbox map %s corrupted: %s", map->index->filepath, t_strdup_vprintf(format, args)); va_end(args); + + mdbox_storage_set_corrupted(map->storage); } struct dbox_map * @@ -188,6 +188,26 @@ int dbox_map_refresh(struct dbox_map *map) return 0; } +static void +mdbox_map_get_ext_hdr(struct dbox_map *map, struct mail_index_view *view, + struct dbox_map_mail_index_header *hdr_r) +{ + const void *data; + size_t data_size; + + mail_index_get_header_ext(view, map->map_ext_id, &data, &data_size); + memset(hdr_r, 0, sizeof(*hdr_r)); + memcpy(hdr_r, data, I_MIN(data_size, sizeof(*hdr_r))); +} + +uint32_t mdbox_map_get_rebuild_count(struct dbox_map *map) +{ + struct dbox_map_mail_index_header hdr; + + mdbox_map_get_ext_hdr(map, map->view, &hdr); + return hdr.rebuild_count; +} + static int dbox_map_lookup_seq(struct dbox_map *map, uint32_t seq, const struct dbox_map_mail_index_record **rec_r) { @@ -397,7 +417,7 @@ dbox_map_sync_handle(struct dbox_map *map, struct mail_index_sync_ctx *sync_ctx) i_warning("dbox %s: Inconsistency in map index " "(%u,%"PRIuUOFF_T" != %u,%"PRIuUOFF_T")", map->path, seq1, offset1, seq2, offset2); - map->storage->storage.files_corrupted = TRUE; + mdbox_storage_set_corrupted(map->storage); } else { while (mail_index_sync_next(sync_ctx, &sync_rec)) ; } @@ -962,35 +982,12 @@ void dbox_map_append_finish(struct dbox_map_append_context *ctx) appends[count-1].size = cur_offset - appends[count-1].offset; } -static int -dbox_map_get_next_file_id(struct dbox_map *map, struct mail_index_view *view, - uint32_t *file_id_r) -{ - const struct dbox_map_mail_index_header *hdr; - const void *data; - size_t data_size; - - mail_index_get_header_ext(view, map->map_ext_id, &data, &data_size); - if (data_size != sizeof(*hdr)) { - if (data_size != 0) { - dbox_map_set_corrupted(map, "hdr size=%"PRIuSIZE_T, - data_size); - return -1; - } - /* first file */ - *file_id_r = 1; - } else { - hdr = data; - *file_id_r = hdr->highest_file_id + 1; - } - return 0; -} - static int dbox_map_assign_file_ids(struct dbox_map_append_context *ctx, bool separate_transaction) { struct dbox_file_append_context *const *file_appends; unsigned int i, count; + struct dbox_map_mail_index_header hdr; uint32_t first_file_id, file_id; int ret; @@ -1006,10 +1003,8 @@ static int dbox_map_assign_file_ids(struct dbox_map_append_context *ctx, } dbox_map_sync_handle(ctx->map, ctx->sync_ctx); - if (dbox_map_get_next_file_id(ctx->map, ctx->sync_view, &file_id) < 0) { - mail_index_sync_rollback(&ctx->sync_ctx); - return -1; - } + mdbox_map_get_ext_hdr(ctx->map, ctx->sync_view, &hdr); + file_id = hdr.highest_file_id + 1; /* assign file_ids for newly created files */ first_file_id = file_id; diff --git a/src/lib-storage/index/dbox-multi/mdbox-map.h b/src/lib-storage/index/dbox-multi/mdbox-map.h index 513c5f838a..71ceec6af8 100644 --- a/src/lib-storage/index/dbox-multi/mdbox-map.h +++ b/src/lib-storage/index/dbox-multi/mdbox-map.h @@ -13,6 +13,8 @@ enum dbox_map_append_flags { struct dbox_map_mail_index_header { uint32_t highest_file_id; + /* increased every time storage is rebuilt */ + uint32_t rebuild_count; }; struct dbox_map_mail_index_record { @@ -41,6 +43,9 @@ int dbox_map_open_or_create(struct dbox_map *map); /* Refresh the map. Returns 0 if ok, -1 if error. */ int dbox_map_refresh(struct dbox_map *map); +/* Return the current rebuild counter */ +uint32_t mdbox_map_get_rebuild_count(struct dbox_map *map); + /* Look up file_id and offset for given map UID. Returns 1 if ok, 0 if UID is already expunged, -1 if error. */ int dbox_map_lookup(struct dbox_map *map, uint32_t map_uid, diff --git a/src/lib-storage/index/dbox-multi/mdbox-purge.c b/src/lib-storage/index/dbox-multi/mdbox-purge.c index fd7d238a5e..90535b436e 100644 --- a/src/lib-storage/index/dbox-multi/mdbox-purge.c +++ b/src/lib-storage/index/dbox-multi/mdbox-purge.c @@ -533,7 +533,7 @@ int mdbox_purge(struct mail_storage *_storage) } T_END; mdbox_purge_free(&ctx); - if (storage->storage.files_corrupted) { + if (storage->corrupted) { /* purging found corrupted files */ (void)mdbox_storage_rebuild(storage); ret = -1; diff --git a/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c b/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c index 40e549e3dd..9d71e88273 100644 --- a/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c +++ b/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c @@ -43,6 +43,7 @@ struct mdbox_storage_rebuild_context { struct hash_table *guid_hash; ARRAY_DEFINE(msgs, struct mdbox_rebuild_msg *); + uint32_t rebuild_count; uint32_t prev_file_id; uint32_t highest_seen_map_uid; @@ -743,6 +744,11 @@ static int rebuild_finish(struct mdbox_storage_rebuild_context *ctx) if (rebuild_handle_zero_refs(ctx) < 0) return -1; rebuild_update_refcounts(ctx); + + ctx->rebuild_count++; + mail_index_update_header_ext(ctx->trans, ctx->storage->map->map_ext_id, + offsetof(struct dbox_map_mail_index_header, rebuild_count), + &ctx->rebuild_count, sizeof(ctx->rebuild_count)); return 0; } @@ -817,6 +823,16 @@ static int mdbox_storage_rebuild_scan(struct mdbox_storage_rebuild_context *ctx) return -1; } + /* get storage rebuild counter after locking */ + ctx->rebuild_count = mdbox_map_get_rebuild_count(ctx->storage->map); + if (ctx->rebuild_count != ctx->storage->corrupted_rebuild_count && + ctx->storage->corrupted) { + /* storage was already rebuilt by someone else */ + return 0; + } + + i_warning("dbox %s: rebuilding indexes", ctx->storage->storage_dir); + uid_validity = dbox_map_get_uid_validity(ctx->storage->map); hdr = mail_index_get_header(ctx->sync_view); if (hdr->uid_validity != uid_validity) { @@ -859,13 +875,13 @@ int mdbox_storage_rebuild(struct mdbox_storage *storage) return -1; } - i_warning("dbox %s: rebuilding indexes", storage->storage_dir); - ctx = mdbox_storage_rebuild_init(storage); ret = mdbox_storage_rebuild_scan(ctx); mdbox_storage_rebuild_deinit(ctx); - if (ret == 0) - storage->storage.files_corrupted = FALSE; + if (ret == 0) { + storage->corrupted = FALSE; + storage->corrupted_rebuild_count = 0; + } return ret; } diff --git a/src/lib-storage/index/dbox-multi/mdbox-storage.c b/src/lib-storage/index/dbox-multi/mdbox-storage.c index 8c95ae29a7..83540da787 100644 --- a/src/lib-storage/index/dbox-multi/mdbox-storage.c +++ b/src/lib-storage/index/dbox-multi/mdbox-storage.c @@ -127,8 +127,7 @@ static void mdbox_mailbox_close(struct mailbox *box) { struct mdbox_storage *mstorage = (struct mdbox_storage *)box->storage; - if (mstorage->storage.files_corrupted && - !mstorage->rebuilding_storage) + if (mstorage->corrupted && !mstorage->rebuilding_storage) (void)mdbox_storage_rebuild(mstorage); index_storage_mailbox_close(box); @@ -145,8 +144,9 @@ int mdbox_read_header(struct mdbox_mailbox *mbox, if (data_size < MDBOX_INDEX_HEADER_MIN_SIZE && (!mbox->creating || data_size != 0)) { mail_storage_set_critical(&mbox->storage->storage.storage, - "dbox %s: Invalid dbox header size", - mbox->box.path); + "dbox %s: Invalid dbox header size: %"PRIuSIZE_T, + mbox->box.path, data_size); + mdbox_storage_set_corrupted(mbox->storage); return -1; } memset(hdr, 0, sizeof(*hdr)); @@ -244,6 +244,30 @@ static int mdbox_mailbox_create_indexes(struct mailbox *box, return ret; } +void mdbox_storage_set_corrupted(struct mdbox_storage *storage) +{ + if (storage->corrupted) { + /* already set it corrupted (possibly recursing back here) */ + return; + } + + storage->corrupted = TRUE; + storage->corrupted_rebuild_count = (uint32_t)-1; + + if (dbox_map_open(storage->map) > 0 && + dbox_map_refresh(storage->map) == 0) { + storage->corrupted_rebuild_count = + mdbox_map_get_rebuild_count(storage->map); + } +} + +static void mdbox_set_file_corrupted(struct dbox_file *file) +{ + struct mdbox_storage *mstorage = (struct mdbox_storage *)file->storage; + + mdbox_storage_set_corrupted(mstorage); +} + static int mdbox_mailbox_get_guid(struct mailbox *box, uint8_t guid[MAIL_GUID_128_SIZE]) { @@ -388,5 +412,6 @@ struct dbox_storage_vfuncs mdbox_dbox_storage_vfuncs = { mdbox_file_unrefed, mdbox_file_create_fd, mdbox_mail_open, - mdbox_mailbox_create_indexes + mdbox_mailbox_create_indexes, + mdbox_set_file_corrupted }; diff --git a/src/lib-storage/index/dbox-multi/mdbox-storage.h b/src/lib-storage/index/dbox-multi/mdbox-storage.h index 1bb431a173..e08bc469f8 100644 --- a/src/lib-storage/index/dbox-multi/mdbox-storage.h +++ b/src/lib-storage/index/dbox-multi/mdbox-storage.h @@ -35,6 +35,11 @@ struct mdbox_storage { ARRAY_TYPE(uint32_t) move_to_alt_map_uids; ARRAY_TYPE(uint32_t) move_from_alt_map_uids; + /* if non-zero, storage should be rebuilt (except if rebuild_count + has changed from this value) */ + uint32_t corrupted_rebuild_count; + + unsigned int corrupted:1; unsigned int rebuilding_storage:1; }; @@ -94,4 +99,6 @@ int mdbox_copy(struct mail_save_context *ctx, struct mail *mail); void mdbox_purge_alt_flag_change(struct mail *mail, bool move_to_alt); int mdbox_purge(struct mail_storage *storage); +void mdbox_storage_set_corrupted(struct mdbox_storage *storage); + #endif diff --git a/src/lib-storage/index/dbox-multi/mdbox-sync.c b/src/lib-storage/index/dbox-multi/mdbox-sync.c index 6557262063..863afec69f 100644 --- a/src/lib-storage/index/dbox-multi/mdbox-sync.c +++ b/src/lib-storage/index/dbox-multi/mdbox-sync.c @@ -44,7 +44,7 @@ dbox_sync_verify_expunge_guid(struct mdbox_sync_context *ctx, uint32_t seq, ctx->mbox->box.vname, uid, binary_to_hex(data, MAIL_GUID_128_SIZE), binary_to_hex(guid_128, MAIL_GUID_128_SIZE)); - ctx->mbox->storage->storage.files_corrupted = TRUE; + mdbox_storage_set_corrupted(ctx->mbox->storage); return -1; } @@ -175,7 +175,7 @@ static int mdbox_sync_index(struct mdbox_sync_context *ctx) array_free(&ctx->expunged_seqs); return ret == 0 ? 1 : - (ctx->mbox->storage->storage.files_corrupted ? 0 : -1); + (ctx->mbox->storage->corrupted ? 0 : -1); } static int mdbox_refresh_header(struct mdbox_mailbox *mbox, bool retry) @@ -189,7 +189,7 @@ static int mdbox_refresh_header(struct mdbox_mailbox *mbox, bool retry) mail_index_view_close(&view); if (ret == 0) { - ret = mbox->storage->storage.files_corrupted ? -1 : 0; + ret = mbox->storage->corrupted ? -1 : 0; } else if (retry) { (void)mail_index_refresh(mbox->box.index); return mdbox_refresh_header(mbox, FALSE); @@ -255,7 +255,7 @@ int mdbox_sync_begin(struct mdbox_mailbox *mbox, enum mdbox_sync_flags flags, if (!storage_rebuilt) { /* we'll need to rebuild storage too. try again from the beginning. */ - mbox->storage->storage.files_corrupted = TRUE; + mdbox_storage_set_corrupted(mbox->storage); mail_index_sync_rollback(&ctx->index_sync_ctx); i_free(ctx); return mdbox_sync_begin(mbox, flags, ctx_r); @@ -321,7 +321,7 @@ mdbox_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags) } if (ret == 0 && (index_mailbox_want_full_sync(&mbox->box, flags) || - mbox->storage->storage.files_corrupted)) { + mbox->storage->corrupted)) { if ((flags & MAILBOX_SYNC_FLAG_FORCE_RESYNC) != 0) mdbox_sync_flags |= MDBOX_SYNC_FLAG_FORCE_REBUILD; ret = mdbox_sync(mbox, mdbox_sync_flags); diff --git a/src/lib-storage/index/dbox-single/sdbox-storage.c b/src/lib-storage/index/dbox-single/sdbox-storage.c index 953cca6550..856829885b 100644 --- a/src/lib-storage/index/dbox-single/sdbox-storage.c +++ b/src/lib-storage/index/dbox-single/sdbox-storage.c @@ -189,6 +189,11 @@ static int sdbox_mailbox_create_indexes(struct mailbox *box, return ret; } +static void sdbox_set_file_corrupted(struct dbox_file *file ATTR_UNUSED) +{ + /* FIXME */ +} + static int sdbox_mailbox_get_guid(struct mailbox *box, uint8_t guid[MAIL_GUID_128_SIZE]) { @@ -294,5 +299,6 @@ struct dbox_storage_vfuncs sdbox_dbox_storage_vfuncs = { dbox_file_free, sdbox_file_create_fd, sdbox_mail_open, - sdbox_mailbox_create_indexes + sdbox_mailbox_create_indexes, + sdbox_set_file_corrupted };