From: Timo Sirainen Date: Wed, 12 May 2010 11:15:40 +0000 (+0200) Subject: lib-storage: Added support for quickly getting mailbox's virtual size. X-Git-Tag: 2.0.beta6~250 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=35ba99e84f234a96bddb359264b64370757768de;p=thirdparty%2Fdovecot%2Fcore.git lib-storage: Added support for quickly getting mailbox's virtual size. --HG-- branch : HEAD --- diff --git a/src/doveadm/doveadm-dump-index.c b/src/doveadm/doveadm-dump-index.c index 25fad3e6e8..5013e9aa56 100644 --- a/src/doveadm/doveadm-dump-index.c +++ b/src/doveadm/doveadm-dump-index.c @@ -15,6 +15,10 @@ #include #include +struct index_vsize_header { + uint64_t vsize; + uint32_t highest_uid; +}; struct maildir_index_header { uint32_t new_check_time, new_mtime, new_mtime_nsecs; uint32_t cur_check_time, cur_mtime, cur_mtime_nsecs; @@ -105,7 +109,13 @@ static void dump_extension_header(struct mail_index *index, return; data = CONST_PTR_OFFSET(index->map->hdr_base, ext->hdr_offset); - if (strcmp(ext->name, "maildir") == 0) { + if (strcmp(ext->name, "hdr-vsize") == 0) { + const struct index_vsize_header *hdr = data; + + printf("header\n"); + printf(" - highest uid = %u\n", hdr->highest_uid); + printf(" - vsize ..... = %llu\n", (unsigned long long)hdr->vsize); + } else if (strcmp(ext->name, "maildir") == 0) { const struct maildir_index_header *hdr = data; printf("header\n"); diff --git a/src/lib-storage/index/index-status.c b/src/lib-storage/index/index-status.c index b4af44e4a1..74b67c55b4 100644 --- a/src/lib-storage/index/index-status.c +++ b/src/lib-storage/index/index-status.c @@ -3,6 +3,7 @@ #include "lib.h" #include "array.h" #include "mail-cache.h" +#include "mail-search-build.h" #include "index-storage.h" #include "mail-index-modseq.h" @@ -30,6 +31,95 @@ index_storage_get_status_cache_fields(struct mailbox *box, status_r->cache_fields = cache_fields; } +static void +index_storage_virtual_size_add_new(struct mailbox *box, + struct index_vsize_header *vsize_hdr) +{ + struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box); + const struct mail_index_header *hdr; + struct mailbox_transaction_context *trans; + struct mail_search_context *search_ctx; + struct mail_search_args *search_args; + struct mail *mail; + uint32_t seq1, seq2; + uoff_t vsize; + int ret = 0; + + hdr = mail_index_get_header(box->view); + if (!mail_index_lookup_seq_range(box->view, vsize_hdr->highest_uid+1, + hdr->next_uid, &seq1, &seq2)) { + /* the last messages are already expunged, + don't bother updating cache */ + return; + } + + search_args = mail_search_build_init(); + mail_search_build_add_seqset(search_args, seq1, seq2); + + trans = mailbox_transaction_begin(box, 0); + search_ctx = mailbox_search_init(trans, search_args, NULL); + mail = mail_alloc(trans, MAIL_FETCH_VIRTUAL_SIZE, NULL); + while (mailbox_search_next(search_ctx, mail)) { + if (mail_get_virtual_size(mail, &vsize) < 0) { + ret = -1; + break; + } + vsize_hdr->vsize += vsize; + vsize_hdr->highest_uid = mail->uid; + } + mail_free(&mail); + if (mailbox_search_deinit(&search_ctx) < 0) + ret = -1; + + if (ret == 0) { + /* success, cache all */ + vsize_hdr->highest_uid = hdr->next_uid - 1; + } else { + /* search failed, cache only up to highest seen uid */ + } + mail_index_update_header_ext(trans->itrans, ibox->vsize_hdr_ext_id, + 0, vsize_hdr, sizeof(*vsize_hdr)); + (void)mailbox_transaction_commit(&trans); + +} + +static void +index_storage_get_status_virtual_size(struct mailbox *box, + struct mailbox_status *status_r) +{ + struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box); + struct index_vsize_header vsize_hdr; + const void *data; + size_t size; + + mail_index_get_header_ext(box->view, ibox->vsize_hdr_ext_id, + &data, &size); + if (size == sizeof(vsize_hdr)) + memcpy(&vsize_hdr, data, sizeof(vsize_hdr)); + else { + if (size != 0) { + mail_storage_set_critical(box->storage, + "vsize-hdr has invalid size: %"PRIuSIZE_T, + size); + } + memset(&vsize_hdr, 0, sizeof(vsize_hdr)); + } + + if (vsize_hdr.highest_uid + 1 == status_r->uidnext) { + /* up to date */ + status_r->virtual_size = vsize_hdr.vsize; + return; + } + if (vsize_hdr.highest_uid >= status_r->uidnext) { + mail_storage_set_critical(box->storage, + "vsize-hdr has invalid highest-uid (%u >= %u)", + vsize_hdr.highest_uid, status_r->uidnext); + memset(&vsize_hdr, 0, sizeof(vsize_hdr)); + } + index_storage_virtual_size_add_new(box, &vsize_hdr); + status_r->virtual_size = vsize_hdr.vsize; +} + void index_storage_get_status(struct mailbox *box, enum mailbox_status_items items, struct mailbox_status *status_r) @@ -69,4 +159,6 @@ void index_storage_get_status(struct mailbox *box, status_r->keywords = mail_index_get_keywords(box->index); if ((items & STATUS_CACHE_FIELDS) != 0) index_storage_get_status_cache_fields(box, status_r); + if ((items & STATUS_VIRTUAL_SIZE) != 0) + index_storage_get_status_virtual_size(box, status_r); } diff --git a/src/lib-storage/index/index-storage.c b/src/lib-storage/index/index-storage.c index 40cc2ce467..ed276b2563 100644 --- a/src/lib-storage/index/index-storage.c +++ b/src/lib-storage/index/index-storage.c @@ -240,6 +240,10 @@ int index_storage_mailbox_open(struct mailbox *box, bool move_to_memory) index_cache_register_defaults(box); box->view = mail_index_view_open(box->index); ibox->keyword_names = mail_index_get_keywords(box->index); + ibox->vsize_hdr_ext_id = + mail_index_ext_register(box->index, "hdr-vsize", + sizeof(struct index_vsize_header), 0, + sizeof(uint64_t)); box->opened = TRUE; diff --git a/src/lib-storage/index/index-storage.h b/src/lib-storage/index/index-storage.h index 26f1be871d..95a1992fe6 100644 --- a/src/lib-storage/index/index-storage.h +++ b/src/lib-storage/index/index-storage.h @@ -27,6 +27,11 @@ struct index_transaction_context { struct mail_cache_transaction_ctx *cache_trans; }; +struct index_vsize_header { + uint64_t vsize; + uint32_t highest_uid; +}; + struct index_mailbox_context { union mailbox_module_context module_ctx; enum mail_index_open_flags index_flags; @@ -50,6 +55,7 @@ struct index_mailbox_context { ARRAY_TYPE(seq_range) recent_flags; uint32_t recent_flags_prev_uid; uint32_t recent_flags_count; + uint32_t vsize_hdr_ext_id; time_t sync_last_check; }; diff --git a/src/lib-storage/mail-storage.h b/src/lib-storage/mail-storage.h index 589c739cc8..504eeee69e 100644 --- a/src/lib-storage/mail-storage.h +++ b/src/lib-storage/mail-storage.h @@ -62,7 +62,8 @@ enum mailbox_status_items { STATUS_FIRST_UNSEEN_SEQ = 0x20, STATUS_KEYWORDS = 0x40, STATUS_HIGHESTMODSEQ = 0x80, - STATUS_CACHE_FIELDS = 0x100 + STATUS_CACHE_FIELDS = 0x100, + STATUS_VIRTUAL_SIZE = 0x200 }; enum mailbox_search_result_flags { @@ -181,6 +182,8 @@ struct mailbox_status { uint32_t first_unseen_seq; uint64_t highest_modseq; + /* sum of virtual size of all messages in mailbox */ + uint64_t virtual_size; const ARRAY_TYPE(keywords) *keywords; /* Fields that have "temp" or "yes" caching decision. */