From 9963bef626fd9ea227fb606e8b1694cdb1ab39aa Mon Sep 17 00:00:00 2001 From: Timo Sirainen Date: Mon, 21 Sep 2015 16:32:27 +0300 Subject: [PATCH] lib-storage: Update mailbox vsize header on save/copy/expunge. This allows always efficiently looking up maiboxes' vsizes after they're initially calculated. The expunge handling is unfortunately done currently in quota handling code, so it works only if quota is enabled. Ideally this would be solved in v2.3 with some lib-storage core changes. --- src/lib-storage/index/Makefile.am | 1 + src/lib-storage/index/cydir/cydir-sync.c | 10 +- src/lib-storage/index/dbox-multi/mdbox-sync.c | 19 +- .../index/dbox-single/sdbox-sync.c | 12 +- src/lib-storage/index/index-mailbox-size.c | 342 +++++++++++++++--- src/lib-storage/index/index-storage.c | 69 ++++ src/lib-storage/index/index-storage.h | 9 + src/lib-storage/index/index-sync.c | 3 + .../index/maildir/maildir-sync-index.c | 7 +- src/lib-storage/index/mbox/mbox-sync.c | 11 +- src/plugins/quota/quota-storage.c | 9 + 11 files changed, 406 insertions(+), 86 deletions(-) diff --git a/src/lib-storage/index/Makefile.am b/src/lib-storage/index/Makefile.am index 742b1f4da1..6e2fa6f9cd 100644 --- a/src/lib-storage/index/Makefile.am +++ b/src/lib-storage/index/Makefile.am @@ -40,6 +40,7 @@ headers = \ istream-mail.h \ index-attachment.h \ index-mail.h \ + index-mailbox-size.h \ index-rebuild.h \ index-search-private.h \ index-search-result.h \ diff --git a/src/lib-storage/index/cydir/cydir-sync.c b/src/lib-storage/index/cydir/cydir-sync.c index 3e6122858b..246d6f5463 100644 --- a/src/lib-storage/index/cydir/cydir-sync.c +++ b/src/lib-storage/index/cydir/cydir-sync.c @@ -112,18 +112,18 @@ int cydir_sync_begin(struct cydir_mailbox *mbox, if (!force) sync_flags |= MAIL_INDEX_SYNC_FLAG_REQUIRE_CHANGES; - ret = mail_index_sync_begin(mbox->box.index, &ctx->index_sync_ctx, - &ctx->sync_view, &ctx->trans, - sync_flags); + ret = index_storage_expunged_sync_begin(&mbox->box, &ctx->index_sync_ctx, + &ctx->sync_view, &ctx->trans, + sync_flags); if (ret <= 0) { - if (ret < 0) - mailbox_set_index_error(&mbox->box); i_free(ctx); *ctx_r = NULL; return ret; } cydir_sync_index(ctx); + index_storage_expunging_deinit(&mbox->box); + *ctx_r = ctx; return 0; } diff --git a/src/lib-storage/index/dbox-multi/mdbox-sync.c b/src/lib-storage/index/dbox-multi/mdbox-sync.c index 5da1be1ae8..31bc011f2a 100644 --- a/src/lib-storage/index/dbox-multi/mdbox-sync.c +++ b/src/lib-storage/index/dbox-multi/mdbox-sync.c @@ -198,24 +198,20 @@ static int mdbox_sync_try_begin(struct mdbox_sync_context *ctx, struct mdbox_mailbox *mbox = ctx->mbox; int ret; - ret = mail_index_sync_begin(mbox->box.index, &ctx->index_sync_ctx, - &ctx->sync_view, &ctx->trans, sync_flags); + ret = index_storage_expunged_sync_begin(&mbox->box, &ctx->index_sync_ctx, + &ctx->sync_view, &ctx->trans, sync_flags); if (mail_index_reset_fscked(mbox->box.index)) mdbox_storage_set_corrupted(mbox->storage); - if (ret < 0) { - mailbox_set_index_error(&mbox->box); - return -1; - } - if (ret == 0) { - /* nothing to do */ - return 0; - } + if (ret <= 0) + return ret; /* error / nothing to do */ if (!mdbox_map_atomic_is_locked(ctx->atomic) && mail_index_sync_has_expunges(ctx->index_sync_ctx)) { /* we have expunges, so we need to write to map. it needs to be locked before mailbox index. */ mail_index_sync_rollback(&ctx->index_sync_ctx); + index_storage_expunging_deinit(&ctx->mbox->box); + if (mdbox_map_atomic_lock(ctx->atomic) < 0) return -1; return mdbox_sync_try_begin(ctx, sync_flags); @@ -262,12 +258,14 @@ int mdbox_sync_begin(struct mdbox_mailbox *mbox, enum mdbox_sync_flags flags, ret = mdbox_sync_try_begin(ctx, sync_flags); if (ret <= 0) { /* failed / nothing to do */ + index_storage_expunging_deinit(&mbox->box); i_free(ctx); return ret; } if ((ret = mdbox_sync_index(ctx)) <= 0) { mail_index_sync_rollback(&ctx->index_sync_ctx); + index_storage_expunging_deinit(&mbox->box); i_free_and_null(ctx); if (ret < 0) @@ -292,6 +290,7 @@ int mdbox_sync_begin(struct mdbox_mailbox *mbox, enum mdbox_sync_flags flags, } return mdbox_sync_begin(mbox, flags, atomic, ctx_r); } + index_storage_expunging_deinit(&mbox->box); *ctx_r = ctx; return 0; diff --git a/src/lib-storage/index/dbox-single/sdbox-sync.c b/src/lib-storage/index/dbox-single/sdbox-sync.c index 59bd3df40f..f9af5dc0b2 100644 --- a/src/lib-storage/index/dbox-single/sdbox-sync.c +++ b/src/lib-storage/index/dbox-single/sdbox-sync.c @@ -216,15 +216,12 @@ int sdbox_sync_begin(struct sdbox_mailbox *mbox, enum sdbox_sync_flags flags, sync_flags |= MAIL_INDEX_SYNC_FLAG_AVOID_FLAG_UPDATES; for (i = 0;; i++) { - ret = mail_index_sync_begin(mbox->box.index, - &ctx->index_sync_ctx, - &ctx->sync_view, &ctx->trans, - sync_flags); + ret = index_storage_expunged_sync_begin(&mbox->box, + &ctx->index_sync_ctx, &ctx->sync_view, + &ctx->trans, sync_flags); if (mail_index_reset_fscked(mbox->box.index)) sdbox_set_mailbox_corrupted(&mbox->box); if (ret <= 0) { - if (ret < 0) - mailbox_set_index_error(&mbox->box); array_free(&ctx->expunged_uids); i_free(ctx); *ctx_r = NULL; @@ -255,6 +252,7 @@ int sdbox_sync_begin(struct sdbox_mailbox *mbox, enum sdbox_sync_flags flags, } mail_index_sync_rollback(&ctx->index_sync_ctx); if (ret < 0) { + index_storage_expunging_deinit(&ctx->mbox->box); array_free(&ctx->expunged_uids); i_free(ctx); return -1; @@ -274,6 +272,7 @@ int sdbox_sync_finish(struct sdbox_sync_context **_ctx, bool success) if (success) { mail_index_view_ref(ctx->sync_view); + if (mail_index_sync_commit(&ctx->index_sync_ctx) < 0) { mailbox_set_index_error(&ctx->mbox->box); ret = -1; @@ -285,6 +284,7 @@ int sdbox_sync_finish(struct sdbox_sync_context **_ctx, bool success) mail_index_sync_rollback(&ctx->index_sync_ctx); } + index_storage_expunging_deinit(&ctx->mbox->box); array_free(&ctx->expunged_uids); i_free(ctx); return ret; diff --git a/src/lib-storage/index/index-mailbox-size.c b/src/lib-storage/index/index-mailbox-size.c index 6701a2084f..da55ebd958 100644 --- a/src/lib-storage/index/index-mailbox-size.c +++ b/src/lib-storage/index/index-mailbox-size.c @@ -1,53 +1,283 @@ /* Copyright (c) 2002-2015 Dovecot authors, see the included COPYING file */ #include "lib.h" +#include "file-create-locked.h" #include "mail-search-build.h" #include "index-storage.h" +#include "index-mailbox-size.h" + +/* + Saving new mails: After transaction is committed and synced, trigger + vsize updating. Lock vsize updates. Check if the message count + + last-indexed-uid are still valid. If they are, add all the missing new + mails. Unlock. + + Fetching vsize: Lock vsize updates. Check if the message count + + last-indexed-uid are still valid. If not, set them to zero. Add all + the missing mails. Unlock. + + Expunging mails: Check if syncing would expunge any mails. If so, lock the + vsize updates before locking syncing (to avoid deadlocks). Check if the + message count + last-indexed-uid are still valid. If not, unlock vsize and + do nothing else. Otherwise, for each expunged mail whose UID <= + last-indexed-uid, decrease the message count and the vsize in memory. After + syncing is successfully committed, write the changes to header. Unlock. + + Note that the final expunge handling with some mailbox formats is done while + syncing is no longer locked. Because of this we need to have the vsize + locking. The final vsize header update requires committing a transaction, + which internally is the same as a sync lock. So to avoid deadlocks we always + need to lock vsize updates before sync. +*/ + +#define VSIZE_LOCK_SUFFIX ".vsize.lock" +#define VSIZE_UPDATE_MAX_LOCK_SECS 10 + +struct mailbox_vsize_update { + struct mailbox *box; + struct mail_index_view *view; + struct mailbox_index_vsize vsize_hdr, orig_vsize_hdr; + + char *lock_path; + int lock_fd; + struct file_lock *lock; + bool rebuild; + bool written; +}; + +static void vsize_header_refresh(struct mailbox_vsize_update *update) +{ + const void *data; + size_t size; + + if (update->view != NULL) + mail_index_view_close(&update->view); + (void)mail_index_refresh(update->box->index); + update->view = mail_index_view_open(update->box->index); + + mail_index_get_header_ext(update->view, update->box->vsize_hdr_ext_id, + &data, &size); + memcpy(&update->orig_vsize_hdr, data, + I_MIN(size, sizeof(update->orig_vsize_hdr))); + if (size == sizeof(update->vsize_hdr)) + memcpy(&update->vsize_hdr, data, sizeof(update->vsize_hdr)); + else { + if (size != 0) { + mail_storage_set_critical(update->box->storage, + "vsize-hdr has invalid size: %"PRIuSIZE_T, + size); + } + update->rebuild = TRUE; + memset(&update->vsize_hdr, 0, sizeof(update->vsize_hdr)); + } +} + +static void +index_mailbox_vsize_check_rebuild(struct mailbox_vsize_update *update) +{ + uint32_t seq1, seq2; + + if (update->vsize_hdr.highest_uid == 0) + return; + if (!mail_index_lookup_seq_range(update->view, 1, + update->vsize_hdr.highest_uid, + &seq1, &seq2)) + seq2 = 0; + + if (update->vsize_hdr.message_count != seq2) { + if (update->vsize_hdr.message_count < seq2) { + mail_storage_set_critical(update->box->storage, + "vsize-hdr has invalid message-count (%u < %u)", + update->vsize_hdr.message_count, seq2); + } else { + /* some messages have been expunged, rescan */ + } + memset(&update->vsize_hdr, 0, sizeof(update->vsize_hdr)); + update->rebuild = TRUE; + } +} + +struct mailbox_vsize_update * +index_mailbox_vsize_update_init(struct mailbox *box) +{ + struct mailbox_vsize_update *update; + + update = i_new(struct mailbox_vsize_update, 1); + update->box = box; + update->lock_fd = -1; + + vsize_header_refresh(update); + return update; +} + +static bool vsize_update_lock_full(struct mailbox_vsize_update *update, + unsigned int lock_secs) +{ + struct mailbox *box = update->box; + const struct mailbox_permissions *perm; + struct file_create_settings set; + const char *error; + bool created; + + if (update->lock_path != NULL) + return update->lock != NULL; + if (MAIL_INDEX_IS_IN_MEMORY(box->index)) + return FALSE; + + perm = mailbox_get_permissions(box); + memset(&set, 0, sizeof(set)); + set.lock_timeout_secs = + mail_storage_get_lock_timeout(box->storage, lock_secs); + set.lock_method = box->storage->set->parsed_lock_method; + set.mode = perm->file_create_mode; + set.gid = perm->file_create_gid; + set.gid_origin = perm->file_create_gid_origin; + + update->lock_path = i_strdup_printf("%s/"VSIZE_LOCK_SUFFIX, + box->index->dir); + update->lock_fd = file_create_locked(update->lock_path, &set, + &update->lock, &created, &error); + if (update->lock_fd == -1) { + if (errno != EAGAIN) { + i_error("file_create_locked(%s) failed: %m", + update->lock_path); + } + return FALSE; + } + update->rebuild = FALSE; + vsize_header_refresh(update); + index_mailbox_vsize_check_rebuild(update); + return TRUE; +} + +bool index_mailbox_vsize_update_try_lock(struct mailbox_vsize_update *update) +{ + return vsize_update_lock_full(update, 0); +} + +bool index_mailbox_vsize_update_wait_lock(struct mailbox_vsize_update *update) +{ + return vsize_update_lock_full(update, VSIZE_UPDATE_MAX_LOCK_SECS); +} + +bool index_mailbox_vsize_want_updates(struct mailbox_vsize_update *update) +{ + return update->vsize_hdr.highest_uid > 0; +} + +static void +index_mailbox_vsize_update_write(struct mailbox_vsize_update *update) +{ + struct mail_index_transaction *trans; + + if (update->written) + return; + update->written = TRUE; + + if (memcmp(&update->orig_vsize_hdr, &update->vsize_hdr, + sizeof(update->vsize_hdr)) == 0) { + /* no changes */ + return; + } + trans = mail_index_transaction_begin(update->view, + MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL); + mail_index_update_header_ext(trans, update->box->vsize_hdr_ext_id, + 0, &update->vsize_hdr, + sizeof(update->vsize_hdr)); + (void)mail_index_transaction_commit(&trans); +} + +void index_mailbox_vsize_update_deinit(struct mailbox_vsize_update **_update) +{ + struct mailbox_vsize_update *update = *_update; + + *_update = NULL; + + if (update->lock != NULL || update->rebuild) + index_mailbox_vsize_update_write(update); + if (update->lock != NULL) { + if (unlink(update->lock_path) < 0) + i_error("unlink(%s) failed: %m", update->lock_path); + file_lock_free(&update->lock); + i_close_fd(&update->lock_fd); + } + mail_index_view_close(&update->view); + i_free(update->lock_path); + i_free(update); +} + +void index_mailbox_vsize_hdr_expunge(struct mailbox_vsize_update *update, + uint32_t uid, uoff_t vsize) +{ + i_assert(update->lock != NULL); + + if (uid > update->vsize_hdr.highest_uid) + return; + if (update->vsize_hdr.message_count == 0) { + mail_storage_set_critical(update->box->storage, + "vsize-hdr's message_count shrank below 0"); + memset(&update->vsize_hdr, 0, sizeof(update->vsize_hdr)); + return; + } + update->vsize_hdr.message_count--; + if (update->vsize_hdr.vsize < vsize) { + mail_storage_set_critical(update->box->storage, + "vsize-hdr's vsize shrank below 0"); + memset(&update->vsize_hdr, 0, sizeof(update->vsize_hdr)); + return; + } + update->vsize_hdr.vsize -= vsize; +} static int -virtual_size_add_new(struct mailbox *box, - struct mailbox_index_vsize *vsize_hdr) +index_mailbox_vsize_hdr_add_missing(struct mailbox_vsize_update *update, + bool need_result) { - const struct mail_index_header *hdr; + struct mailbox_index_vsize *vsize_hdr = &update->vsize_hdr; struct mailbox_transaction_context *trans; struct mail_search_context *search_ctx; struct mail_search_args *search_args; + struct mailbox_status status; struct mail *mail; uint32_t seq1, seq2; uoff_t vsize; int ret = 0; - hdr = mail_index_get_header(box->view); - if (vsize_hdr->highest_uid == 0) - seq2 = 0; - else if (!mail_index_lookup_seq_range(box->view, 1, - vsize_hdr->highest_uid, - &seq1, &seq2)) - seq2 = 0; - - if (vsize_hdr->message_count != seq2) { - if (vsize_hdr->message_count < seq2) { - mail_storage_set_critical(box->storage, - "vsize-hdr has invalid message-count (%u < %u)", - vsize_hdr->message_count, seq2); - } else { - /* some messages have been expunged, rescan */ - } - memset(vsize_hdr, 0, sizeof(*vsize_hdr)); - seq2 = 0; + mailbox_get_open_status(update->box, STATUS_UIDNEXT, &status); + if (vsize_hdr->highest_uid + 1 >= status.uidnext) { + /* nothing to do - we should have usually caught this already + before locking */ + return 0; } + /* note that update->view may be more up-to-date than box->view. + we'll just add whatever new mails are in box->view. if we'll notice + that some of the new mails are missing, we'll need to stop there + since that expunge will be applied later on to the vsize header. */ search_args = mail_search_build_init(); - mail_search_build_add_seqset(search_args, seq2 + 1, - hdr->messages_count); + if (!mail_index_lookup_seq_range(update->box->view, + vsize_hdr->highest_uid + 1, + status.uidnext-1, &seq1, &seq2)) { + /* nothing existed, but update uidnext */ + vsize_hdr->highest_uid = status.uidnext - 1; + mail_search_args_unref(&search_args); + return 0; + } + mail_search_build_add_seqset(search_args, seq1, seq2); - trans = mailbox_transaction_begin(box, 0); + trans = mailbox_transaction_begin(update->box, 0); search_ctx = mailbox_search_init(trans, search_args, NULL, MAIL_FETCH_VIRTUAL_SIZE, NULL); while (mailbox_search_next(search_ctx, &mail)) { if (mail_get_virtual_size(mail, &vsize) < 0) { - if (mail->expunged) + if (mail->expunged) { + if (!need_result) { + ret = -1; + break; + } + index_mailbox_vsize_update_write(update); continue; + } ret = -1; break; } @@ -61,12 +291,10 @@ virtual_size_add_new(struct mailbox *box, if (ret == 0) { /* success, cache all */ - vsize_hdr->highest_uid = hdr->next_uid - 1; + vsize_hdr->highest_uid = status.uidnext - 1; } else { /* search failed, cache only up to highest seen uid */ } - mail_index_update_header_ext(trans->itrans, box->vsize_hdr_ext_id, - 0, vsize_hdr, sizeof(*vsize_hdr)); (void)mailbox_transaction_commit(&trans); return ret; } @@ -74,40 +302,27 @@ virtual_size_add_new(struct mailbox *box, int index_mailbox_get_virtual_size(struct mailbox *box, struct mailbox_metadata *metadata_r) { - struct mailbox_index_vsize vsize_hdr; + struct mailbox_vsize_update *update; struct mailbox_status status; - const void *data; - size_t size; int ret; mailbox_get_open_status(box, STATUS_MESSAGES | STATUS_UIDNEXT, &status); - mail_index_get_header_ext(box->view, box->vsize_hdr_ext_id, - &data, &size); - if (size == sizeof(vsize_hdr)) - memcpy(&vsize_hdr, data, sizeof(vsize_hdr)); - else { - if (size != 0) { - mail_storage_set_critical(box->storage, - "vsize-hdr has invalid size: %"PRIuSIZE_T, - size); - } - memset(&vsize_hdr, 0, sizeof(vsize_hdr)); - } - - if (vsize_hdr.highest_uid + 1 == status.uidnext && - vsize_hdr.message_count == status.messages) { + update = index_mailbox_vsize_update_init(box); + if (update->vsize_hdr.highest_uid + 1 == status.uidnext && + update->vsize_hdr.message_count == status.messages) { /* up to date */ - metadata_r->virtual_size = vsize_hdr.vsize; + metadata_r->virtual_size = update->vsize_hdr.vsize; + index_mailbox_vsize_update_deinit(&update); return 0; } - if (vsize_hdr.highest_uid >= status.uidnext) { - mail_storage_set_critical(box->storage, - "vsize-hdr has invalid highest-uid (%u >= %u)", - vsize_hdr.highest_uid, status.uidnext); - memset(&vsize_hdr, 0, sizeof(vsize_hdr)); - } - ret = virtual_size_add_new(box, &vsize_hdr); - metadata_r->virtual_size = vsize_hdr.vsize; + + /* we need to update it - lock it if possible. if not, update it + anyway internally even though we won't be saving the result. */ + (void)index_mailbox_vsize_update_wait_lock(update); + + ret = index_mailbox_vsize_hdr_add_missing(update, TRUE); + metadata_r->virtual_size = update->vsize_hdr.vsize; + index_mailbox_vsize_update_deinit(&update); return ret; } @@ -168,3 +383,20 @@ int index_mailbox_get_physical_size(struct mailbox *box, (void)mailbox_transaction_commit(&trans); return ret; } + +void index_mailbox_vsize_update_appends(struct mailbox *box) +{ + struct mailbox_vsize_update *update; + struct mailbox_status status; + + update = index_mailbox_vsize_update_init(box); + + mailbox_get_open_status(update->box, STATUS_UIDNEXT, &status); + /* update here only if we don't need to rebuild the whole vsize. */ + index_mailbox_vsize_check_rebuild(update); + if (update->vsize_hdr.highest_uid + 1 != status.uidnext && + index_mailbox_vsize_want_updates(update) && + index_mailbox_vsize_update_try_lock(update)) + (void)index_mailbox_vsize_hdr_add_missing(update, FALSE); + index_mailbox_vsize_update_deinit(&update); +} diff --git a/src/lib-storage/index/index-storage.c b/src/lib-storage/index/index-storage.c index 9203d3f572..8d59dd6a25 100644 --- a/src/lib-storage/index/index-storage.c +++ b/src/lib-storage/index/index-storage.c @@ -17,6 +17,7 @@ #include "index-mail.h" #include "index-attachment.h" #include "index-thread-private.h" +#include "index-mailbox-size.h" #include #include @@ -947,3 +948,71 @@ void index_storage_destroy(struct mail_storage *storage) dict_deinit(&storage->_shared_attr_dict); } } + +static void index_storage_expunging_init(struct mailbox *box) +{ + struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box); + + if (ibox->vsize_update != NULL) + return; + + ibox->vsize_update = index_mailbox_vsize_update_init(box); + if (!index_mailbox_vsize_want_updates(ibox->vsize_update) || + !index_mailbox_vsize_update_wait_lock(ibox->vsize_update)) + index_mailbox_vsize_update_deinit(&ibox->vsize_update); +} + +void index_storage_expunging_deinit(struct mailbox *box) +{ + struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box); + + if (ibox->vsize_update != NULL) + index_mailbox_vsize_update_deinit(&ibox->vsize_update); +} + +static bool index_storage_expunging_want_updates(struct mailbox *box) +{ + struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box); + bool ret; + + i_assert(ibox->vsize_update == NULL); + + ibox->vsize_update = index_mailbox_vsize_update_init(box); + ret = index_mailbox_vsize_want_updates(ibox->vsize_update); + index_mailbox_vsize_update_deinit(&ibox->vsize_update); + return ret; +} + +int index_storage_expunged_sync_begin(struct mailbox *box, + struct mail_index_sync_ctx **ctx_r, + struct mail_index_view **view_r, + struct mail_index_transaction **trans_r, + enum mail_index_sync_flags flags) +{ + struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box); + int ret; + + /* try to avoid locking vsize updates by checking if we see any + expunges */ + if (mail_index_sync_have_any_expunges(box->index)) + index_storage_expunging_init(box); + + ret = mail_index_sync_begin(box->index, ctx_r, view_r, + trans_r, flags); + if (ret <= 0) { + mailbox_set_index_error(box); + index_storage_expunging_deinit(box); + return ret; + } + if (ibox->vsize_update == NULL && + mail_index_sync_has_expunges(*ctx_r) && + index_storage_expunging_want_updates(box)) { + /* race condition - need to abort the sync and retry with + the vsize locked */ + mail_index_sync_rollback(ctx_r); + index_storage_expunging_init(box); + return index_storage_expunged_sync_begin(box, ctx_r, view_r, + trans_r, flags); + } + return 1; +} diff --git a/src/lib-storage/index/index-storage.h b/src/lib-storage/index/index-storage.h index 08ff0877ad..24bf9f2d17 100644 --- a/src/lib-storage/index/index-storage.h +++ b/src/lib-storage/index/index-storage.h @@ -28,6 +28,8 @@ struct index_mailbox_context { const ARRAY_TYPE(keywords) *keyword_names; struct mail_cache_field *cache_fields; + struct mailbox_vsize_update *vsize_update; + uint32_t recent_flags_last_check_nextuid; time_t sync_last_check; @@ -157,4 +159,11 @@ void index_storage_list_index_update_sync(struct mailbox *box, struct mail_index_transaction *trans, uint32_t seq); +int index_storage_expunged_sync_begin(struct mailbox *box, + struct mail_index_sync_ctx **ctx_r, + struct mail_index_view **view_r, + struct mail_index_transaction **trans_r, + enum mail_index_sync_flags flags); +void index_storage_expunging_deinit(struct mailbox *box); + #endif diff --git a/src/lib-storage/index/index-sync.c b/src/lib-storage/index/index-sync.c index 21baa3fc95..894e4017d4 100644 --- a/src/lib-storage/index/index-sync.c +++ b/src/lib-storage/index/index-sync.c @@ -4,6 +4,7 @@ #include "seq-range-array.h" #include "ioloop.h" #include "array.h" +#include "index-mailbox-size.h" #include "index-sync-private.h" struct index_storage_list_index_record { @@ -334,6 +335,8 @@ int index_mailbox_sync_deinit(struct mailbox_sync_context *_ctx, if (array_is_created(&ctx->all_flag_update_uids)) array_free(&ctx->all_flag_update_uids); + /* update vsize header if wanted */ + index_mailbox_vsize_update_appends(_ctx->box); i_free(ctx); return ret; } diff --git a/src/lib-storage/index/maildir/maildir-sync-index.c b/src/lib-storage/index/maildir/maildir-sync-index.c index f1f34e803a..baacc6c23a 100644 --- a/src/lib-storage/index/maildir/maildir-sync-index.c +++ b/src/lib-storage/index/maildir/maildir-sync-index.c @@ -228,11 +228,9 @@ int maildir_sync_index_begin(struct maildir_mailbox *mbox, if (maildir_sync_ctx == NULL) sync_flags &= ~MAIL_INDEX_SYNC_FLAG_DROP_RECENT; - if (mail_index_sync_begin(_box->index, &sync_ctx, &view, - &trans, sync_flags) < 0) { - mailbox_set_index_error(_box); + if (index_storage_expunged_sync_begin(_box, &sync_ctx, &view, + &trans, sync_flags) < 0) return -1; - } ctx = i_new(struct maildir_index_sync_context, 1); ctx->mbox = mbox; @@ -336,6 +334,7 @@ static int maildir_sync_index_finish(struct maildir_index_sync_context *ctx, mbox->syncing_commit = FALSE; } + index_storage_expunging_deinit(&mbox->box); maildir_keywords_sync_deinit(&ctx->keywords_sync_ctx); index_sync_changes_deinit(&ctx->sync_changes); i_free(ctx); diff --git a/src/lib-storage/index/mbox/mbox-sync.c b/src/lib-storage/index/mbox/mbox-sync.c index 304c179445..6beb1afbc5 100644 --- a/src/lib-storage/index/mbox/mbox-sync.c +++ b/src/lib-storage/index/mbox/mbox-sync.c @@ -1789,6 +1789,7 @@ int mbox_sync_has_changed_full(struct mbox_mailbox *mbox, bool leave_dirty, static void mbox_sync_context_free(struct mbox_sync_context *sync_ctx) { index_sync_changes_deinit(&sync_ctx->sync_changes); + index_storage_expunging_deinit(&sync_ctx->mbox->box); if (sync_ctx->index_sync_ctx != NULL) mail_index_sync_rollback(&sync_ctx->index_sync_ctx); pool_unref(&sync_ctx->mail_keyword_pool); @@ -1877,13 +1878,10 @@ again: if ((flags & MBOX_SYNC_REWRITE) != 0) sync_flags |= MAIL_INDEX_SYNC_FLAG_FLUSH_DIRTY; - ret = mail_index_sync_begin(mbox->box.index, &index_sync_ctx, - &sync_view, &trans, sync_flags); - if (ret <= 0) { - if (ret < 0) - mailbox_set_index_error(&mbox->box); + ret = index_storage_expunged_sync_begin(&mbox->box, &index_sync_ctx, + &sync_view, &trans, sync_flags); + if (ret <= 0) return ret; - } if ((mbox->box.flags & MAILBOX_FLAG_DROP_RECENT) != 0) { /* see if we need to drop recent flags */ @@ -1897,6 +1895,7 @@ again: nothing_to_do: /* index may need to do internal syncing though, so commit instead of rollbacking. */ + index_storage_expunging_deinit(&mbox->box); if (mail_index_sync_commit(&index_sync_ctx) < 0) { mailbox_set_index_error(&mbox->box); return -1; diff --git a/src/plugins/quota/quota-storage.c b/src/plugins/quota/quota-storage.c index a0946a4b00..fae7165080 100644 --- a/src/plugins/quota/quota-storage.c +++ b/src/plugins/quota/quota-storage.c @@ -7,6 +7,7 @@ #include "mail-storage-private.h" #include "mailbox-list-private.h" #include "maildir-storage.h" +#include "index-mailbox-size.h" #include "quota-private.h" #include "quota-plugin.h" @@ -316,6 +317,7 @@ static void quota_mailbox_sync_notify(struct mailbox *box, uint32_t uid, enum mailbox_sync_type sync_type) { struct quota_mailbox *qbox = QUOTA_CONTEXT(box); + struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box); struct quota_user *quser = QUOTA_USER_CONTEXT(box->storage->user); const uint32_t *uids; const uoff_t *sizep; @@ -358,6 +360,11 @@ static void quota_mailbox_sync_notify(struct mailbox *box, uint32_t uid, /* we already know the size */ sizep = array_idx(&qbox->expunge_sizes, i); quota_free_bytes(qbox->expunge_qt, *sizep); + /* FIXME: it's not ideal that we do the vsize update here, but + this is the easiest place for it for now.. maybe the mail + size checking code could be moved to lib-storage */ + if (ibox->vsize_update != NULL && quser->quota->set->vsizes) + index_mailbox_vsize_hdr_expunge(ibox->vsize_update, uid, *sizep); return; } @@ -385,6 +392,8 @@ static void quota_mailbox_sync_notify(struct mailbox *box, uint32_t uid, } } else if (mail_get_virtual_size(qbox->expunge_qt->tmp_mail, &size) == 0) { quota_free_bytes(qbox->expunge_qt, size); + if (ibox->vsize_update != NULL) + index_mailbox_vsize_hdr_expunge(ibox->vsize_update, uid, size); } else { /* there's no way to get the size. recalculate the quota. */ quota_recalculate(qbox->expunge_qt); -- 2.47.3