From f4810655b62b5d3e767d60cf5861750ec9df399f Mon Sep 17 00:00:00 2001 From: Timo Sirainen Date: Fri, 6 Mar 2009 16:17:59 -0500 Subject: [PATCH] dbox: Fixes to save/expunge/cleanup. --HG-- branch : HEAD --- src/lib-storage/index/dbox/dbox-file.c | 38 +++---- src/lib-storage/index/dbox/dbox-map.c | 110 ++++++++++++-------- src/lib-storage/index/dbox/dbox-map.h | 2 + src/lib-storage/index/dbox/dbox-sync-file.c | 51 ++++----- src/lib-storage/index/dbox/dbox-sync.c | 2 + 5 files changed, 116 insertions(+), 87 deletions(-) diff --git a/src/lib-storage/index/dbox/dbox-file.c b/src/lib-storage/index/dbox/dbox-file.c index cf3d3c5b77..57b7d23681 100644 --- a/src/lib-storage/index/dbox/dbox-file.c +++ b/src/lib-storage/index/dbox/dbox-file.c @@ -489,8 +489,7 @@ int dbox_file_open_if_needed(struct dbox_file *file) void dbox_file_close(struct dbox_file *file) { - if (file->lock != NULL) - file_lock_free(&file->lock); + dbox_file_unlock(file); if (file->input != NULL) i_stream_unref(&file->input); if (file->output != NULL) @@ -513,8 +512,16 @@ int dbox_file_try_lock(struct dbox_file *file) void dbox_file_unlock(struct dbox_file *file) { - if (file->lock != NULL) + struct stat st; + + if (file->lock != NULL) { + if (file->output != NULL) { + if (fstat(file->fd, &st) == 0 && + (uoff_t)st.st_size != file->output->offset) + i_panic("dbox file modified while locked"); + } file_unlock(&file->lock); + } if (file->input != NULL) i_stream_sync(file->input); } @@ -661,10 +668,7 @@ static int dbox_file_seek_append_pos(struct dbox_file *file, uoff_t last_msg_offset, uoff_t last_msg_size) { - uoff_t offset, size; struct stat st; - bool last; - int ret; if (file->file_version != DBOX_VERSION || file->msg_header_size != sizeof(struct dbox_message_header)) { @@ -682,25 +686,15 @@ dbox_file_seek_append_pos(struct dbox_file *file, st.st_size = -1; } - offset = last_msg_offset; - if (st.st_size == (off_t)(last_msg_offset + last_msg_size)) { - offset += last_msg_size; - } else { - ret = dbox_file_seek_next(file, &offset, &size, &last); - if (ret <= 0) - return ret; - if (!last) { - /* not end of file? previous write probably crashed. */ - if (ftruncate(file->fd, offset) < 0) { - dbox_file_set_syscall_error(file, "ftruncate()"); - return -1; - } - i_stream_sync(file->input); - } + if (st.st_size != (off_t)(last_msg_offset + last_msg_size)) { + /* not end of file? either previous write crashed or map index + got broken. play it safe and resync everything. */ + i_warning("%d vs %d+%d", (int)st.st_size, (int)last_msg_offset, (int)last_msg_size); + return 0; } file->output = o_stream_create_fd_file(file->fd, 0, FALSE); - o_stream_seek(file->output, offset); + o_stream_seek(file->output, last_msg_offset + last_msg_size); return 1; } diff --git a/src/lib-storage/index/dbox/dbox-map.c b/src/lib-storage/index/dbox/dbox-map.c index f2fbec9eea..cb812401cc 100644 --- a/src/lib-storage/index/dbox/dbox-map.c +++ b/src/lib-storage/index/dbox/dbox-map.c @@ -45,10 +45,9 @@ struct dbox_map_append_context { ARRAY_DEFINE(appends, struct dbox_map_append); uint32_t first_new_file_id; - uint32_t orig_msg_count; + uint32_t orig_next_uid; unsigned int files_nonappendable_count; - uint32_t retry_seq_min, retry_seq_max; unsigned int failed:1; }; @@ -285,7 +284,8 @@ int dbox_map_update_refcounts(struct dbox_map *map, uint32_t map_uid, seq; int ret = 0; - trans = mail_index_transaction_begin(map->view, 0); + trans = mail_index_transaction_begin(map->view, + MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL); seq_range_array_iter_init(&iter, map_uids); i = 0; while (seq_range_array_iter_nth(&iter, i++, &map_uid)) { if ((ret = dbox_map_get_seq(map, map_uid, &seq)) <= 0) { @@ -303,11 +303,48 @@ int dbox_map_update_refcounts(struct dbox_map *map, uint32_t log_seq; uoff_t log_offset; - return mail_index_transaction_commit(&trans, &log_seq, - &log_offset); + if (mail_index_transaction_commit(&trans, &log_seq, + &log_offset) < 0) { + mail_storage_set_internal_error(&map->storage->storage); + mail_index_reset_error(map->index); + return -1; + } + return 0; } } +int dbox_map_remove_file_id(struct dbox_map *map, uint32_t file_id) +{ + struct mail_index_transaction *trans; + const struct mail_index_header *hdr; + const struct dbox_mail_index_map_record *rec; + const void *data; + bool expunged; + uint32_t seq; + uint32_t log_seq; + uoff_t log_offset; + + trans = mail_index_transaction_begin(map->view, + MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL); + hdr = mail_index_get_header(map->view); + for (seq = 1; seq <= hdr->messages_count; seq++) { + mail_index_lookup_ext(map->view, seq, map->map_ext_id, + &data, &expunged); + if (data == NULL) + break; + + rec = data; + if (rec->file_id == file_id) + mail_index_expunge(trans, seq); + } + if (mail_index_transaction_commit(&trans, &log_seq, &log_offset) < 0) { + mail_storage_set_internal_error(&map->storage->storage); + mail_index_reset_error(map->index); + return -1; + } + return 0; +} + struct dbox_map_append_context * dbox_map_append_begin_storage(struct dbox_storage *storage) { @@ -363,7 +400,6 @@ static time_t day_begin_stamp(unsigned int days) static bool dbox_map_file_try_append(struct dbox_map_append_context *ctx, uint32_t file_id, time_t stamp, uoff_t mail_size, - uoff_t last_msg_offset, uoff_t last_msg_size, struct dbox_file **file_r, struct ostream **output_r, bool *retry_later_r) { @@ -372,7 +408,7 @@ dbox_map_file_try_append(struct dbox_map_append_context *ctx, const struct mail_index_header *hdr; struct dbox_file *file; uint32_t seq, tmp_file_id; - uoff_t tmp_offset, tmp_size, new_size; + uoff_t tmp_offset, tmp_size, last_msg_offset, last_msg_size, new_size; bool deleted, file_too_old = FALSE; int ret; @@ -380,10 +416,10 @@ dbox_map_file_try_append(struct dbox_map_append_context *ctx, *retry_later_r = FALSE; file = dbox_file_init_multi(storage, file_id); - if (dbox_file_open_or_create(file, &deleted) <= 0) - return TRUE; - if (deleted) + if (dbox_file_open_or_create(file, &deleted) <= 0 || deleted) { + dbox_file_unref(&file); return TRUE; + } if (file->lock != NULL) { /* already locked, we're possibly in the middle of cleaning it up in which case we really don't want to write there. */ @@ -398,22 +434,24 @@ dbox_map_file_try_append(struct dbox_map_append_context *ctx, *retry_later_r = ret == 0; } else if (dbox_map_refresh(map) == 0) { /* now that the file is locked and map is refreshed, make sure - we still have the last msg's offset. */ + we still have the last msg's offset. we have to go through + the whole map, because existing messages may have already + been appended to this file. */ + last_msg_offset = 0; hdr = mail_index_get_header(map->view); - seq = ctx->orig_msg_count + 1; - for (; seq <= hdr->messages_count; seq++) { + for (seq = 1; seq <= hdr->messages_count; seq++) { if (dbox_map_lookup_seq(map, seq, &tmp_file_id, &tmp_offset, &tmp_size) < 0) break; - if (tmp_file_id == file->file_id) { - i_assert(last_msg_offset < tmp_offset); + if (tmp_file_id == file->file_id && + last_msg_offset < tmp_offset) { last_msg_offset = tmp_offset; last_msg_size = tmp_size; } } new_size = last_msg_offset + last_msg_size + mail_size; - if (seq > hdr->messages_count && + if (seq > hdr->messages_count && last_msg_offset > 0 && new_size <= storage->rotate_size && dbox_file_get_append_stream(file, last_msg_offset, last_msg_size, output_r) > 0) { @@ -454,10 +492,10 @@ dbox_map_find_appendable_file(struct dbox_map_append_context *ctx, struct dbox_file *const *files; const struct mail_index_header *hdr; unsigned int i, count, backwards_lookup_count; - uint32_t seq, file_id, min_seen_file_id; + uint32_t seq, seq1, uid, file_id, min_seen_file_id; uoff_t offset, append_offset, size; time_t stamp; - bool retry_later, updated_retry_seq_min; + bool retry_later; *existing_r = FALSE; @@ -500,15 +538,9 @@ dbox_map_find_appendable_file(struct dbox_map_append_context *ctx, hdr = mail_index_get_header(map->view); min_seen_file_id = (uint32_t)-1; - ctx->orig_msg_count = hdr->messages_count; - updated_retry_seq_min = FALSE; + ctx->orig_next_uid = hdr->next_uid; backwards_lookup_count = 0; for (seq = hdr->messages_count; seq > 0; seq--) { - if (seq == ctx->retry_seq_max) { - /* skip over files that we know won't match */ - seq = ctx->retry_seq_min + 1; - continue; - } if (dbox_map_lookup_seq(map, seq, &file_id, &offset, &size) < 0) return -1; if (file_id >= min_seen_file_id) @@ -521,7 +553,8 @@ dbox_map_find_appendable_file(struct dbox_map_append_context *ctx, } /* first lookup: this should be enough usually, but we can't - be sure until after locking */ + be sure until after locking. also if messages were recently + moved, this message might not be the last one in the file. */ if (offset + size + mail_size >= map->storage->rotate_size) continue; @@ -530,26 +563,21 @@ dbox_map_find_appendable_file(struct dbox_map_append_context *ctx, continue; } + mail_index_lookup_uid(map->view, seq, &uid); if (!dbox_map_file_try_append(ctx, file_id, stamp, mail_size, - offset, size, file_r, output_r, - &retry_later)) { + file_r, output_r, &retry_later)) { /* file is too old. the rest of the files are too. */ break; } - if (*file_r != NULL) { - if (seq > ctx->retry_seq_min) - ctx->retry_seq_min = seq; + /* NOTE: we've now refreshed map view. there are no guarantees + about sequences anymore. */ + if (*file_r != NULL) return 1; - } - if (retry_later) { - ctx->retry_seq_min = seq; - updated_retry_seq_min = TRUE; - } - } - ctx->retry_seq_max = ctx->orig_msg_count; - if (!updated_retry_seq_min) { - /* there are no files that can be appended to */ - ctx->retry_seq_min = 0; + /* FIXME: use retry_later somehow */ + if (!mail_index_lookup_seq_range(map->view, 1, uid-1, + &seq1, &seq)) + break; + seq++; } return 0; } diff --git a/src/lib-storage/index/dbox/dbox-map.h b/src/lib-storage/index/dbox/dbox-map.h index 3213d7e542..538ab4bbe0 100644 --- a/src/lib-storage/index/dbox/dbox-map.h +++ b/src/lib-storage/index/dbox/dbox-map.h @@ -25,6 +25,8 @@ int dbox_map_get_file_msgs(struct dbox_map *map, uint32_t file_id, int dbox_map_update_refcounts(struct dbox_map *map, const ARRAY_TYPE(seq_range) *map_uids, int diff); +/* */ +int dbox_map_remove_file_id(struct dbox_map *map, uint32_t file_id); /* Return all files containing messages with zero refcount. */ const ARRAY_TYPE(seq_range) *dbox_map_get_zero_ref_files(struct dbox_map *map); diff --git a/src/lib-storage/index/dbox/dbox-sync-file.c b/src/lib-storage/index/dbox/dbox-sync-file.c index db44241364..fd1248beea 100644 --- a/src/lib-storage/index/dbox/dbox-sync-file.c +++ b/src/lib-storage/index/dbox/dbox-sync-file.c @@ -65,15 +65,16 @@ int dbox_sync_file_cleanup(struct dbox_file *file) append_ctx = dbox_map_append_begin_storage(file->storage); - t_array_init(&msgs_arr, 128); + i_array_init(&msgs_arr, 128); if (dbox_map_get_file_msgs(file->storage->map, file->file_id, &msgs_arr) < 0) { // FIXME + array_free(&msgs_arr); return -1; } msgs = array_get(&msgs_arr, &count); - t_array_init(&copied_map_uids, I_MIN(count, 1)); - t_array_init(&expunged_map_uids, I_MIN(count, 1)); + i_array_init(&copied_map_uids, I_MIN(count, 1)); + i_array_init(&expunged_map_uids, I_MIN(count, 1)); for (i = 0; i < count; i++) { if (msgs[i].refcount == 0) { seq_range_array_add(&expunged_map_uids, 0, @@ -145,29 +146,31 @@ int dbox_sync_file_cleanup(struct dbox_file *file) dbox_map_append_finish_multi_mail(append_ctx); seq_range_array_add(&copied_map_uids, 0, msgs[i].map_uid); } + array_free(&msgs_arr); msgs = NULL; if (ret < 0) { dbox_map_append_rollback(&append_ctx); - return -1; - } - - if (output == NULL) { + ret = -1; + } else if (output == NULL) { /* everything expunged in this file, unlink it */ dbox_map_append_rollback(&append_ctx); - if (dbox_sync_file_unlink(file) < 0) - return -1; - return 0; - } - - /* assign new file_id + offset to moved messages */ - if (dbox_map_append_move(append_ctx, &copied_map_uids, - &expunged_map_uids) < 0) { - // FIXME - return -1; + ret = dbox_sync_file_unlink(file); + } else { + /* assign new file_id + offset to moved messages */ + if (dbox_map_append_move(append_ctx, &copied_map_uids, + &expunged_map_uids) < 0) { + // FIXME + dbox_map_append_rollback(&append_ctx); + ret = -1; + } else { + dbox_map_append_commit(&append_ctx); + (void)dbox_sync_file_unlink(file); + ret = 1; + } } - dbox_map_append_commit(&append_ctx); - (void)dbox_sync_file_unlink(file); - return 1; + array_free(&copied_map_uids); + array_free(&expunged_map_uids); + return ret; } static void @@ -207,7 +210,7 @@ int dbox_sync_file(struct dbox_sync_context *ctx, const struct dbox_sync_file_entry *entry) { struct dbox_file *file; - int ret; + int ret = 1; file = entry->file_id != 0 ? dbox_file_init_multi(ctx->mbox->storage, entry->file_id) : @@ -225,9 +228,9 @@ int dbox_sync_file(struct dbox_sync_context *ctx, } else { if (dbox_map_update_refcounts(ctx->mbox->storage->map, &entry->expunge_map_uids, -1) < 0) - return -1; - - dbox_sync_mark_expunges(ctx, &entry->expunge_seqs); + ret = -1; + else + dbox_sync_mark_expunges(ctx, &entry->expunge_seqs); } dbox_file_unref(&file); return ret; diff --git a/src/lib-storage/index/dbox/dbox-sync.c b/src/lib-storage/index/dbox/dbox-sync.c index 053a1c677c..8f56e35158 100644 --- a/src/lib-storage/index/dbox/dbox-sync.c +++ b/src/lib-storage/index/dbox/dbox-sync.c @@ -330,6 +330,8 @@ void dbox_sync_cleanup(struct dbox_storage *storage) file = dbox_file_init_multi(storage, file_id); if (dbox_file_open_or_create(file, &deleted) > 0 && !deleted) (void)dbox_sync_file_cleanup(file); + else + dbox_map_remove_file_id(storage->map, file_id); dbox_file_unref(&file); } } -- 2.47.3