From: Timo Sirainen Date: Fri, 20 Mar 2009 18:06:15 +0000 (-0400) Subject: dbox: Initial code for handling crashes while writing to map index. X-Git-Tag: 2.0.alpha1~1038^2~36 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=849cc9deab8b1b5219f8578c2c517f9ffe028296;p=thirdparty%2Fdovecot%2Fcore.git dbox: Initial code for handling crashes while writing to map index. --HG-- branch : HEAD --- diff --git a/src/lib-storage/index/dbox/dbox-map-private.h b/src/lib-storage/index/dbox/dbox-map-private.h index e1324396bf..39cf86433a 100644 --- a/src/lib-storage/index/dbox/dbox-map-private.h +++ b/src/lib-storage/index/dbox/dbox-map-private.h @@ -29,7 +29,7 @@ struct dbox_map_append_context { struct mail_index_sync_ctx *sync_ctx; struct mail_index_view *sync_view; - struct mail_index_transaction *trans; + struct mail_index_transaction *sync_trans, *trans; ARRAY_DEFINE(files, struct dbox_file *); ARRAY_DEFINE(appends, struct dbox_map_append); @@ -40,6 +40,7 @@ struct dbox_map_append_context { unsigned int files_nonappendable_count; unsigned int failed:1; + unsigned int committed:1; }; int dbox_map_refresh(struct dbox_map *map); diff --git a/src/lib-storage/index/dbox/dbox-map.c b/src/lib-storage/index/dbox/dbox-map.c index e3262d9138..68dd2d1b42 100644 --- a/src/lib-storage/index/dbox/dbox-map.c +++ b/src/lib-storage/index/dbox/dbox-map.c @@ -9,6 +9,12 @@ #define MAX_BACKWARDS_LOOKUPS 10 +struct dbox_map_transaction_context { + struct dbox_map *map; + struct mail_index_transaction *trans; + unsigned int changed:1; +}; + void dbox_map_set_corrupted(struct dbox_map *map, const char *format, ...) { va_list args; @@ -247,17 +253,86 @@ const ARRAY_TYPE(seq_range) *dbox_map_get_zero_ref_files(struct dbox_map *map) return &map->ref0_file_ids; } -int dbox_map_update_refcounts(struct dbox_map *map, +struct dbox_map_transaction_context * +dbox_map_transaction_begin(struct dbox_map *map) +{ + struct dbox_map_transaction_context *ctx; + + ctx = i_new(struct dbox_map_transaction_context, 1); + ctx->map = map; + ctx->trans = mail_index_transaction_begin(map->view, + MAIL_INDEX_TRANSACTION_FLAG_FSYNC); + return ctx; +} + +int dbox_map_transaction_commit(struct dbox_map_transaction_context **_ctx) +{ + struct dbox_map_transaction_context *ctx = *_ctx; + struct dbox_map *map = ctx->map; + struct mail_index_sync_ctx *sync_ctx; + struct mail_index_view *view; + struct mail_index_transaction *sync_trans, *trans = ctx->trans; + uint32_t seq1, seq2; + uoff_t offset1, offset2; + int ret; + + *_ctx = NULL; + i_free(ctx); + + if (!ctx->changed) { + mail_index_transaction_rollback(&trans); + return 0; + } + + /* use syncing to lock the transaction log, so that we always see + log's head_offset = tail_offset */ + ret = mail_index_sync_begin(map->index, &sync_ctx, + &view, &sync_trans, 0); + if (ret <= 0) { + i_assert(ret != 0); + mail_storage_set_internal_error(&map->storage->storage); + mail_index_reset_error(map->index); + mail_index_transaction_rollback(&trans); + return -1; + } + + mail_index_sync_get_offsets(sync_ctx, &seq1, &offset1, &seq2, &offset2); + if (offset1 != offset2 || seq1 != seq2) { + /* something had crashed. need a full resync. */ + i_warning("dbox %s: Inconsistency in map index", + map->storage->storage_dir); + map->storage->sync_rebuild = TRUE; + } + + if (mail_index_transaction_commit(&trans) < 0 || + mail_index_sync_commit(&sync_ctx) < 0) { + mail_storage_set_internal_error(&map->storage->storage); + mail_index_reset_error(map->index); + if (sync_ctx != NULL) + mail_index_sync_rollback(&sync_ctx); + return -1; + } + return 0; +} + +void dbox_map_transaction_rollback(struct dbox_map_transaction_context **_ctx) +{ + struct dbox_map_transaction_context *ctx = *_ctx; + + *_ctx = NULL; + mail_index_transaction_rollback(&ctx->trans); + i_free(ctx); +} + +int dbox_map_update_refcounts(struct dbox_map_transaction_context *ctx, const ARRAY_TYPE(seq_range) *map_uids, int diff) { - struct mail_index_transaction *trans; + struct dbox_map *map = ctx->map; struct seq_range_iter iter; unsigned int i; uint32_t map_uid, seq; int ret; - trans = mail_index_transaction_begin(map->view, - MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL); seq_range_array_iter_init(&iter, map_uids); i = 0; while (seq_range_array_iter_nth(&iter, i++, &map_uid)) { if ((ret = dbox_map_get_seq(map, map_uid, &seq)) <= 0) { @@ -266,24 +341,20 @@ int dbox_map_update_refcounts(struct dbox_map *map, "refcount update lost map_uid=%u", map_uid); } - mail_index_transaction_rollback(&trans); return -1; } - mail_index_atomic_inc_ext(trans, seq, map->ref_ext_id, diff); - } - - if (mail_index_transaction_commit(&trans) < 0) { - mail_storage_set_internal_error(&map->storage->storage); - mail_index_reset_error(map->index); - return -1; + ctx->changed = TRUE; + mail_index_atomic_inc_ext(ctx->trans, seq, + map->ref_ext_id, diff); } return 0; } -int dbox_map_remove_file_id(struct dbox_map *map, uint32_t file_id) +int dbox_map_remove_file_id(struct dbox_map_transaction_context *ctx, + uint32_t file_id) { - struct mail_index_transaction *trans; + struct dbox_map *map = ctx->map; const struct mail_index_header *hdr; const struct dbox_mail_index_map_record *rec; const void *data; @@ -295,26 +366,20 @@ int dbox_map_remove_file_id(struct dbox_map *map, uint32_t file_id) if (dbox_map_refresh(map) < 0) return -1; - trans = mail_index_transaction_begin(map->view, - MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL); hdr = mail_index_get_header(map->view); for (seq = 1; seq <= hdr->messages_count; seq++) { mail_index_lookup_ext(map->view, seq, map->map_ext_id, &data, &expunged); if (data == NULL) { dbox_map_set_corrupted(map, "missing map extension"); - mail_index_transaction_rollback(&trans); return -1; } rec = data; - if (rec->file_id == file_id) - mail_index_expunge(trans, seq); - } - if (mail_index_transaction_commit(&trans) < 0) { - mail_storage_set_internal_error(&map->storage->storage); - mail_index_reset_error(map->index); - return -1; + if (rec->file_id == file_id) { + ctx->changed = TRUE; + mail_index_expunge(ctx->trans, seq); + } } return 0; } @@ -639,17 +704,19 @@ dbox_map_get_next_file_id(struct dbox_map *map, struct mail_index_view *view, return 0; } -static int dbox_map_assign_file_ids(struct dbox_map_append_context *ctx) +static int dbox_map_assign_file_ids(struct dbox_map_append_context *ctx, + bool separate_transaction) { struct dbox_file *const *files; unsigned int i, count; - uint32_t first_file_id, file_id; + uint32_t first_file_id, file_id, seq1, seq2; + uoff_t offset1, offset2; int ret; /* start the syncing. we'll need it even if there are no file ids to be assigned. */ ret = mail_index_sync_begin(ctx->map->index, &ctx->sync_ctx, - &ctx->sync_view, &ctx->trans, 0); + &ctx->sync_view, &ctx->sync_trans, 0); if (ret <= 0) { i_assert(ret != 0); mail_storage_set_internal_error(&ctx->map->storage->storage); @@ -657,6 +724,15 @@ static int dbox_map_assign_file_ids(struct dbox_map_append_context *ctx) return -1; } + mail_index_sync_get_offsets(ctx->sync_ctx, &seq1, &offset1, + &seq2, &offset2); + if (offset1 != offset2 || seq1 != seq2) { + /* something had crashed. need a full resync. */ + i_warning("dbox %s: Inconsistency in map index", + ctx->map->storage->storage_dir); + ctx->map->storage->sync_rebuild = TRUE; + } + if (dbox_map_get_next_file_id(ctx->map, ctx->sync_view, &file_id) < 0) { mail_index_sync_rollback(&ctx->sync_ctx); return -1; @@ -689,10 +765,16 @@ static int dbox_map_assign_file_ids(struct dbox_map_append_context *ctx) return -1; } + ctx->trans = !separate_transaction ? NULL : + mail_index_transaction_begin(ctx->map->view, + MAIL_INDEX_TRANSACTION_FLAG_FSYNC); + /* update the highest used file_id */ if (first_file_id != file_id) { file_id--; - mail_index_update_header_ext(ctx->trans, ctx->map->map_ext_id, + mail_index_update_header_ext(ctx->trans != NULL ? ctx->trans : + ctx->sync_trans, + ctx->map->map_ext_id, 0, &file_id, sizeof(file_id)); } return 0; @@ -716,7 +798,7 @@ int dbox_map_append_assign_map_uids(struct dbox_map_append_context *ctx, return 0; } - if (dbox_map_assign_file_ids(ctx) < 0) + if (dbox_map_assign_file_ids(ctx, TRUE) < 0) return -1; /* append map records to index */ @@ -752,7 +834,7 @@ int dbox_map_append_assign_map_uids(struct dbox_map_append_context *ctx, &uid_validity, sizeof(uid_validity), TRUE); } - if (mail_index_sync_commit(&ctx->sync_ctx) < 0) { + if (mail_index_transaction_commit(&ctx->trans) < 0) { mail_storage_set_internal_error(&ctx->map->storage->storage); mail_index_reset_error(ctx->map->index); return -1; @@ -774,7 +856,7 @@ int dbox_map_append_move(struct dbox_map_append_context *ctx, unsigned int i, j, map_uids_count, appends_count; uint32_t uid, seq; - if (dbox_map_assign_file_ids(ctx) < 0) + if (dbox_map_assign_file_ids(ctx, FALSE) < 0) return -1; memset(&rec, 0, sizeof(rec)); @@ -790,8 +872,8 @@ int dbox_map_append_move(struct dbox_map_append_context *ctx, if (!mail_index_lookup_seq(ctx->sync_view, uids[i], &seq)) i_unreached(); - mail_index_update_ext(ctx->trans, seq, ctx->map->map_ext_id, - &rec, NULL); + mail_index_update_ext(ctx->sync_trans, seq, + ctx->map->map_ext_id, &rec, NULL); } seq_range_array_iter_init(&iter, expunge_map_uids); i = 0; @@ -800,12 +882,6 @@ int dbox_map_append_move(struct dbox_map_append_context *ctx, i_unreached(); mail_index_expunge(ctx->trans, seq); } - - if (mail_index_sync_commit(&ctx->sync_ctx) < 0) { - mail_storage_set_internal_error(&ctx->map->storage->storage); - mail_index_reset_error(ctx->map->index); - return -1; - } return 0; } @@ -828,62 +904,65 @@ int dbox_map_append_assign_uids(struct dbox_map_append_context *ctx, return 0; } -void dbox_map_append_commit(struct dbox_map_append_context **_ctx) +int dbox_map_append_commit(struct dbox_map_append_context *ctx) { - struct dbox_map_append_context *ctx = *_ctx; - struct dbox_file **files; - unsigned int i, count; + i_assert(ctx->sync_ctx != NULL); + i_assert(ctx->trans == NULL); - *_ctx = NULL; + if (mail_index_sync_commit(&ctx->sync_ctx) < 0) { + mail_storage_set_internal_error(&ctx->map->storage->storage); + mail_index_reset_error(ctx->map->index); + return -1; + } + ctx->committed = TRUE; + return 0; +} - files = array_get_modifiable(&ctx->files, &count); - for (i = 0; i < count; i++) { - files[i]->first_append_offset = 0; - dbox_file_unlock(files[i]); - dbox_file_unref(&files[i]); +static void dbox_map_append_file_rollback(struct dbox_file *file) +{ + struct mail_storage *storage = &file->storage->storage; + + if (file->output != NULL) { + /* flush before truncating */ + (void)o_stream_flush(file->output); } - array_free(&ctx->appends); - array_free(&ctx->files); - i_free(ctx); + if (file->file_id != 0 && + file->first_append_offset > file->file_header_size) { + if (ftruncate(file->fd, file->first_append_offset) < 0) { + mail_storage_set_critical(storage, + "ftruncate(%s, %"PRIuUOFF_T") failed: %m", + file->current_path, file->first_append_offset); + } + } else { + if (unlink(file->current_path) < 0) { + mail_storage_set_critical(storage, + "unlink(%s) failed: %m", file->current_path); + } + } } -void dbox_map_append_rollback(struct dbox_map_append_context **_ctx) +void dbox_map_append_free(struct dbox_map_append_context **_ctx) { struct dbox_map_append_context *ctx = *_ctx; - struct mail_storage *storage = &ctx->map->storage->storage; - struct dbox_file *const *files, *file; + struct dbox_file **files; unsigned int i, count; *_ctx = NULL; - files = array_get(&ctx->files, &count); - for (i = 0; i < count; i++) { - file = files[i]; + if (ctx->trans != NULL) + mail_index_transaction_rollback(&ctx->trans); + if (ctx->sync_ctx != NULL) + mail_index_sync_rollback(&ctx->sync_ctx); - if (file->output != NULL) { - /* flush before truncating */ - (void)o_stream_flush(file->output); - } + files = array_get_modifiable(&ctx->files, &count); + for (i = 0; i < count; i++) { + if (!ctx->committed) + dbox_map_append_file_rollback(files[i]); - if (file->file_id != 0 && - file->first_append_offset > file->file_header_size) { - if (ftruncate(file->fd, file->first_append_offset) < 0) { - mail_storage_set_critical(storage, - "ftruncate(%s, %"PRIuUOFF_T") failed: %m", - file->current_path, - file->first_append_offset); - } - } else { - if (unlink(file->current_path) < 0) { - mail_storage_set_critical(storage, - "unlink(%s) failed: %m", - file->current_path); - } - } - file->first_append_offset = 0; + files[i]->first_append_offset = 0; dbox_file_unlock(files[i]); - dbox_file_unref(&file); + dbox_file_unref(&files[i]); } array_free(&ctx->appends); array_free(&ctx->files); diff --git a/src/lib-storage/index/dbox/dbox-map.h b/src/lib-storage/index/dbox/dbox-map.h index 7f5e6edc94..49af4f26ca 100644 --- a/src/lib-storage/index/dbox/dbox-map.h +++ b/src/lib-storage/index/dbox/dbox-map.h @@ -38,9 +38,15 @@ int dbox_map_lookup(struct dbox_map *map, uint32_t map_uid, int dbox_map_get_file_msgs(struct dbox_map *map, uint32_t file_id, ARRAY_TYPE(dbox_map_file_msg) *recs); -int dbox_map_update_refcounts(struct dbox_map *map, +struct dbox_map_transaction_context * +dbox_map_transaction_begin(struct dbox_map *map); +int dbox_map_transaction_commit(struct dbox_map_transaction_context **ctx); +void dbox_map_transaction_rollback(struct dbox_map_transaction_context **ctx); + +int dbox_map_update_refcounts(struct dbox_map_transaction_context *ctx, const ARRAY_TYPE(seq_range) *map_uids, int diff); -int dbox_map_remove_file_id(struct dbox_map *map, uint32_t file_id); +int dbox_map_remove_file_id(struct dbox_map_transaction_context *ctx, + uint32_t file_id); /* Return all files containing messages with zero refcount. */ const ARRAY_TYPE(seq_range) *dbox_map_get_zero_ref_files(struct dbox_map *map); @@ -69,8 +75,8 @@ int dbox_map_append_move(struct dbox_map_append_context *ctx, const ARRAY_TYPE(uint32_t) *map_uids, const ARRAY_TYPE(seq_range) *expunge_map_uids); /* Returns 0 if ok, -1 if error. */ -void dbox_map_append_commit(struct dbox_map_append_context **ctx); -void dbox_map_append_rollback(struct dbox_map_append_context **ctx); +int dbox_map_append_commit(struct dbox_map_append_context *ctx); +void dbox_map_append_free(struct dbox_map_append_context **ctx); void dbox_map_set_corrupted(struct dbox_map *map, const char *format, ...) ATTR_FORMAT(2, 3); diff --git a/src/lib-storage/index/dbox/dbox-save.c b/src/lib-storage/index/dbox/dbox-save.c index f0d63630c7..31e62b074c 100644 --- a/src/lib-storage/index/dbox/dbox-save.c +++ b/src/lib-storage/index/dbox/dbox-save.c @@ -316,15 +316,18 @@ int dbox_transaction_save_commit_pre(struct dbox_save_context *ctx) i_assert(ctx->finished); - /* get map UIDs for messages saved to multi-files */ - if (dbox_map_append_assign_map_uids(ctx->append_ctx, &first_map_uid, - &last_map_uid) < 0) { + /* lock the mailbox before map to avoid deadlocks */ + if (dbox_sync_begin(ctx->mbox, DBOX_SYNC_FLAG_FORCE | + DBOX_SYNC_FLAG_FSYNC, &ctx->sync_ctx) < 0) { dbox_transaction_save_rollback(ctx); return -1; } - /* lock the mailbox */ - if (dbox_sync_begin(ctx->mbox, TRUE, &ctx->sync_ctx) < 0) { + /* get map UIDs for messages saved to multi-files. they're written + to transaction log immediately within this function, but the map + is left locked. */ + if (dbox_map_append_assign_map_uids(ctx->append_ctx, &first_map_uid, + &last_map_uid) < 0) { dbox_transaction_save_rollback(ctx); return -1; } @@ -366,7 +369,6 @@ int dbox_transaction_save_commit_pre(struct dbox_save_context *ctx) i_assert(next_map_uid == last_map_uid + 1); } - dbox_map_append_commit(&ctx->append_ctx); if (ctx->mail != NULL) mail_free(&ctx->mail); @@ -380,7 +382,13 @@ void dbox_transaction_save_commit_post(struct dbox_save_context *ctx) { ctx->ctx.transaction = NULL; /* transaction is already freed */ - (void)dbox_sync_finish(&ctx->sync_ctx, TRUE); + /* finish writing the mailbox APPENDs */ + if (dbox_sync_finish(&ctx->sync_ctx, TRUE) == 0) { + /* commit only updates the sync tail offset, everything else + was already written at this point. */ + (void)dbox_map_append_commit(ctx->append_ctx); + } + dbox_map_append_free(&ctx->append_ctx); if (!ctx->mbox->ibox.fsync_disable) { if (fdatasync_path(ctx->mbox->path) < 0) { @@ -396,7 +404,7 @@ void dbox_transaction_save_rollback(struct dbox_save_context *ctx) if (!ctx->finished) dbox_save_cancel(&ctx->ctx); if (ctx->append_ctx != NULL) - dbox_map_append_rollback(&ctx->append_ctx); + dbox_map_append_free(&ctx->append_ctx); if (ctx->sync_ctx != NULL) (void)dbox_sync_finish(&ctx->sync_ctx, FALSE); diff --git a/src/lib-storage/index/dbox/dbox-storage-rebuild.c b/src/lib-storage/index/dbox/dbox-storage-rebuild.c index 8cf1262fbe..d1efd1ee88 100644 --- a/src/lib-storage/index/dbox/dbox-storage-rebuild.c +++ b/src/lib-storage/index/dbox/dbox-storage-rebuild.c @@ -694,6 +694,9 @@ int dbox_storage_rebuild(struct dbox_storage *storage) return -1; } + if (storage->sync_rebuild) + i_warning("dbox %s: rebuilding indexes", storage->storage_dir); + ctx = dbox_storage_rebuild_init(storage); ret = dbox_storage_rebuild_scan(ctx); dbox_storage_rebuild_deinit(ctx); diff --git a/src/lib-storage/index/dbox/dbox-sync-file.c b/src/lib-storage/index/dbox/dbox-sync-file.c index 547825bf79..1857ee5f6d 100644 --- a/src/lib-storage/index/dbox/dbox-sync-file.c +++ b/src/lib-storage/index/dbox/dbox-sync-file.c @@ -226,25 +226,25 @@ int dbox_sync_file_cleanup(struct dbox_file *file) array_free(&msgs_arr); msgs = NULL; if (ret <= 0) { - dbox_map_append_rollback(&append_ctx); + dbox_map_append_free(&append_ctx); dbox_file_unlock(file); ret = -1; } else if (array_count(&copied_map_uids) == 0) { /* everything expunged in this file, unlink it */ ret = dbox_sync_file_unlink(file); - dbox_map_append_rollback(&append_ctx); + dbox_map_append_free(&append_ctx); } else { /* assign new file_id + offset to moved messages */ if (dbox_map_append_move(append_ctx, &copied_map_uids, - &expunged_map_uids) < 0) { + &expunged_map_uids) < 0 || + dbox_map_append_commit(append_ctx) < 0) { dbox_file_unlock(file); - dbox_map_append_rollback(&append_ctx); ret = -1; } else { - (void)dbox_sync_file_unlink(file); - dbox_map_append_commit(&append_ctx); ret = 1; + (void)dbox_sync_file_unlink(file); } + dbox_map_append_free(&append_ctx); } array_free(&copied_map_uids); array_free(&expunged_map_uids); @@ -290,12 +290,13 @@ dbox_sync_mark_expunges(struct dbox_sync_context *ctx, int dbox_sync_file(struct dbox_sync_context *ctx, const struct dbox_sync_file_entry *entry) { + struct dbox_mailbox *mbox = ctx->mbox; struct dbox_file *file; int ret = 1; file = entry->file_id != 0 ? - dbox_file_init_multi(ctx->mbox->storage, entry->file_id) : - dbox_file_init_single(ctx->mbox, entry->uid); + dbox_file_init_multi(mbox->storage, entry->file_id) : + dbox_file_init_single(mbox, entry->uid); if (!array_is_created(&entry->expunge_map_uids)) { /* no expunges - we want to move it */ dbox_sync_file_move_if_needed(file, entry); @@ -307,7 +308,11 @@ int dbox_sync_file(struct dbox_sync_context *ctx, ret = 1; } } else { - if (dbox_map_update_refcounts(ctx->mbox->storage->map, + if (ctx->map_trans == NULL) { + ctx->map_trans = + dbox_map_transaction_begin(mbox->storage->map); + } + if (dbox_map_update_refcounts(ctx->map_trans, &entry->expunge_map_uids, -1) < 0) ret = -1; else diff --git a/src/lib-storage/index/dbox/dbox-sync.c b/src/lib-storage/index/dbox/dbox-sync.c index b86c6e4c76..808669c278 100644 --- a/src/lib-storage/index/dbox/dbox-sync.c +++ b/src/lib-storage/index/dbox/dbox-sync.c @@ -171,6 +171,9 @@ static int dbox_sync_index(struct dbox_sync_context *ctx) hash_table_iterate_deinit(&iter); } + if (ret == 0) + ret = dbox_map_transaction_commit(&ctx->map_trans); + if (box->v.sync_notify != NULL) box->v.sync_notify(box, 0, 0); @@ -204,7 +207,7 @@ static int dbox_refresh_header(struct dbox_mailbox *mbox) return 0; } -int dbox_sync_begin(struct dbox_mailbox *mbox, bool force, +int dbox_sync_begin(struct dbox_mailbox *mbox, enum dbox_sync_flags flags, struct dbox_sync_context **ctx_r) { struct mail_storage *storage = mbox->ibox.box.storage; @@ -215,8 +218,7 @@ int dbox_sync_begin(struct dbox_mailbox *mbox, bool force, bool rebuild, storage_rebuilt = FALSE; rebuild = dbox_refresh_header(mbox) < 0; - - if (mbox->storage->sync_rebuild) { + if (rebuild) { if (dbox_storage_rebuild(mbox->storage) < 0) return -1; storage_rebuilt = TRUE; @@ -227,8 +229,10 @@ int dbox_sync_begin(struct dbox_mailbox *mbox, bool force, if (!mbox->ibox.keep_recent) sync_flags |= MAIL_INDEX_SYNC_FLAG_DROP_RECENT; - if (!rebuild && !force) + if (!rebuild && (flags & DBOX_SYNC_FLAG_FORCE) == 0) sync_flags |= MAIL_INDEX_SYNC_FLAG_REQUIRE_CHANGES; + if ((flags & DBOX_SYNC_FLAG_FSYNC) != 0) + sync_flags |= MAIL_INDEX_SYNC_FLAG_FSYNC; /* don't write unnecessary dirty flag updates */ sync_flags |= MAIL_INDEX_SYNC_FLAG_AVOID_FLAG_UPDATES; @@ -252,7 +256,7 @@ int dbox_sync_begin(struct dbox_mailbox *mbox, bool force, try again from the beginning. */ mail_index_sync_rollback(&ctx->index_sync_ctx); i_free(ctx); - return dbox_sync_begin(mbox, force, ctx_r); + return dbox_sync_begin(mbox, flags, ctx_r); } ret = 0; } else { @@ -291,6 +295,9 @@ int dbox_sync_finish(struct dbox_sync_context **_ctx, bool success) *_ctx = NULL; + if (ctx->map_trans != NULL) + dbox_map_transaction_rollback(&ctx->map_trans); + if (success) { if (mail_index_sync_commit(&ctx->index_sync_ctx) < 0) { mail_storage_set_index_error(&ctx->mbox->ibox); @@ -302,14 +309,14 @@ int dbox_sync_finish(struct dbox_sync_context **_ctx, bool success) if (ctx->path != NULL) str_free(&ctx->path); i_free(ctx); - return 0; + return ret; } int dbox_sync(struct dbox_mailbox *mbox) { struct dbox_sync_context *sync_ctx; - if (dbox_sync_begin(mbox, FALSE, &sync_ctx) < 0) + if (dbox_sync_begin(mbox, 0, &sync_ctx) < 0) return -1; if (sync_ctx == NULL) @@ -326,7 +333,8 @@ dbox_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags) if (!box->opened) index_storage_mailbox_open(&mbox->ibox); - if (index_mailbox_want_full_sync(&mbox->ibox, flags)) + if (index_mailbox_want_full_sync(&mbox->ibox, flags) || + mbox->storage->sync_rebuild) ret = dbox_sync(mbox); return index_mailbox_sync_init(box, flags, ret < 0); @@ -335,12 +343,15 @@ dbox_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags) void dbox_sync_cleanup(struct dbox_storage *storage) { const ARRAY_TYPE(seq_range) *ref0_file_ids; + struct dbox_map_transaction_context *map_trans; struct dbox_file *file; struct seq_range_iter iter; unsigned int i = 0; uint32_t file_id; bool deleted; + map_trans = dbox_map_transaction_begin(storage->map); + ref0_file_ids = dbox_map_get_zero_ref_files(storage->map); seq_range_array_iter_init(&iter, ref0_file_ids); i = 0; while (seq_range_array_iter_nth(&iter, i++, &file_id)) T_BEGIN { @@ -348,7 +359,9 @@ void dbox_sync_cleanup(struct dbox_storage *storage) if (dbox_file_open_or_create(file, &deleted) > 0 && !deleted) (void)dbox_sync_file_cleanup(file); else - dbox_map_remove_file_id(storage->map, file_id); + dbox_map_remove_file_id(map_trans, file_id); dbox_file_unref(&file); } T_END; + + (void)dbox_map_transaction_commit(&map_trans); } diff --git a/src/lib-storage/index/dbox/dbox-sync.h b/src/lib-storage/index/dbox/dbox-sync.h index 811ad03df5..e19d8d8d38 100644 --- a/src/lib-storage/index/dbox/dbox-sync.h +++ b/src/lib-storage/index/dbox/dbox-sync.h @@ -4,6 +4,11 @@ struct mailbox; struct dbox_mailbox; +enum dbox_sync_flags { + DBOX_SYNC_FLAG_FORCE = 0x01, + DBOX_SYNC_FLAG_FSYNC = 0x02 +}; + struct dbox_sync_file_entry { uint32_t uid, file_id; @@ -18,6 +23,7 @@ struct dbox_sync_context { struct mail_index_sync_ctx *index_sync_ctx; struct mail_index_view *sync_view; struct mail_index_transaction *trans; + struct dbox_map_transaction_context *map_trans; string_t *path; unsigned int path_dir_prefix_len; @@ -26,7 +32,7 @@ struct dbox_sync_context { struct hash_table *syncs; /* struct dbox_sync_file_entry */ }; -int dbox_sync_begin(struct dbox_mailbox *mbox, bool force, +int dbox_sync_begin(struct dbox_mailbox *mbox, enum dbox_sync_flags flags, struct dbox_sync_context **ctx_r); int dbox_sync_finish(struct dbox_sync_context **ctx, bool success); int dbox_sync(struct dbox_mailbox *mbox);