lib-storage: Added support for quickly getting mailbox's virtual size.

--HG--
branch : HEAD

diff --git a/src/doveadm/doveadm-dump-index.c b/src/doveadm/doveadm-dump-index.c
index 25fad3e6e8c6704c1714d0809d8fd254e3df302e..5013e9aa56661d607b9e03bbd76841ea8c2cfe7a 100644 (file)
--- a/src/doveadm/doveadm-dump-index.c
+++ b/src/doveadm/doveadm-dump-index.c
@@ -15,6 +15,10 @@
 #include <stdlib.h>
 #include <time.h>
 
+struct index_vsize_header {
+       uint64_t vsize;
+       uint32_t highest_uid;
+};
 struct maildir_index_header {
        uint32_t new_check_time, new_mtime, new_mtime_nsecs;
        uint32_t cur_check_time, cur_mtime, cur_mtime_nsecs;
@@ -105,7 +109,13 @@ static void dump_extension_header(struct mail_index *index,
                return;
 
        data = CONST_PTR_OFFSET(index->map->hdr_base, ext->hdr_offset);
-       if (strcmp(ext->name, "maildir") == 0) {
+       if (strcmp(ext->name, "hdr-vsize") == 0) {
+               const struct index_vsize_header *hdr = data;
+
+               printf("header\n");
+               printf(" - highest uid = %u\n", hdr->highest_uid);
+               printf(" - vsize ..... = %llu\n", (unsigned long long)hdr->vsize);
+       } else if (strcmp(ext->name, "maildir") == 0) {
                const struct maildir_index_header *hdr = data;
 
                printf("header\n");

diff --git a/src/lib-storage/index/index-status.c b/src/lib-storage/index/index-status.c
index b4af44e4a1295b2080ec0fb8be551655260a692a..74b67c55b434bf9386a0dabbcb4c5193dde3cb8d 100644 (file)
--- a/src/lib-storage/index/index-status.c
+++ b/src/lib-storage/index/index-status.c
@@ -3,6 +3,7 @@
 #include "lib.h"
 #include "array.h"
 #include "mail-cache.h"
+#include "mail-search-build.h"
 #include "index-storage.h"
 #include "mail-index-modseq.h"
 
@@ -30,6 +31,95 @@ index_storage_get_status_cache_fields(struct mailbox *box,
        status_r->cache_fields = cache_fields;
 }
 
+static void
+index_storage_virtual_size_add_new(struct mailbox *box,
+                                  struct index_vsize_header *vsize_hdr)
+{
+       struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
+       const struct mail_index_header *hdr;
+       struct mailbox_transaction_context *trans;
+       struct mail_search_context *search_ctx;
+       struct mail_search_args *search_args;
+       struct mail *mail;
+       uint32_t seq1, seq2;
+       uoff_t vsize;
+       int ret = 0;
+
+       hdr = mail_index_get_header(box->view);
+       if (!mail_index_lookup_seq_range(box->view, vsize_hdr->highest_uid+1,
+                                        hdr->next_uid, &seq1, &seq2)) {
+               /* the last messages are already expunged,
+                  don't bother updating cache */
+               return;
+       }
+
+       search_args = mail_search_build_init();
+       mail_search_build_add_seqset(search_args, seq1, seq2);
+
+       trans = mailbox_transaction_begin(box, 0);
+       search_ctx = mailbox_search_init(trans, search_args, NULL);
+       mail = mail_alloc(trans, MAIL_FETCH_VIRTUAL_SIZE, NULL);
+       while (mailbox_search_next(search_ctx, mail)) {
+               if (mail_get_virtual_size(mail, &vsize) < 0) {
+                       ret = -1;
+                       break;
+               }
+               vsize_hdr->vsize += vsize;
+               vsize_hdr->highest_uid = mail->uid;
+       }
+       mail_free(&mail);
+       if (mailbox_search_deinit(&search_ctx) < 0)
+               ret = -1;
+
+       if (ret == 0) {
+               /* success, cache all */
+               vsize_hdr->highest_uid = hdr->next_uid - 1;
+       } else {
+               /* search failed, cache only up to highest seen uid */
+       }
+       mail_index_update_header_ext(trans->itrans, ibox->vsize_hdr_ext_id,
+                                    0, vsize_hdr, sizeof(*vsize_hdr));
+       (void)mailbox_transaction_commit(&trans);
+
+}
+
+static void
+index_storage_get_status_virtual_size(struct mailbox *box,
+                                     struct mailbox_status *status_r)
+{
+       struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
+       struct index_vsize_header vsize_hdr;
+       const void *data;
+       size_t size;
+
+       mail_index_get_header_ext(box->view, ibox->vsize_hdr_ext_id,
+                                 &data, &size);
+       if (size == sizeof(vsize_hdr))
+               memcpy(&vsize_hdr, data, sizeof(vsize_hdr));
+       else {
+               if (size != 0) {
+                       mail_storage_set_critical(box->storage,
+                               "vsize-hdr has invalid size: %"PRIuSIZE_T,
+                               size);
+               }
+               memset(&vsize_hdr, 0, sizeof(vsize_hdr));
+       }
+
+       if (vsize_hdr.highest_uid + 1 == status_r->uidnext) {
+               /* up to date */
+               status_r->virtual_size = vsize_hdr.vsize;
+               return;
+       }
+       if (vsize_hdr.highest_uid >= status_r->uidnext) {
+               mail_storage_set_critical(box->storage,
+                       "vsize-hdr has invalid highest-uid (%u >= %u)",
+                       vsize_hdr.highest_uid, status_r->uidnext);
+               memset(&vsize_hdr, 0, sizeof(vsize_hdr));
+       }
+       index_storage_virtual_size_add_new(box, &vsize_hdr);
+       status_r->virtual_size = vsize_hdr.vsize;
+}
+
 void index_storage_get_status(struct mailbox *box,
                              enum mailbox_status_items items,
                              struct mailbox_status *status_r)
@@ -69,4 +159,6 @@ void index_storage_get_status(struct mailbox *box,
                status_r->keywords = mail_index_get_keywords(box->index);
        if ((items & STATUS_CACHE_FIELDS) != 0)
                index_storage_get_status_cache_fields(box, status_r);
+       if ((items & STATUS_VIRTUAL_SIZE) != 0)
+               index_storage_get_status_virtual_size(box, status_r);
 }

diff --git a/src/lib-storage/index/index-storage.c b/src/lib-storage/index/index-storage.c
index 40cc2ce46744f42b573550df5ddf40a16e622190..ed276b256334a6c7ebdb60badd86676d2c84fc14 100644 (file)
--- a/src/lib-storage/index/index-storage.c
+++ b/src/lib-storage/index/index-storage.c
@@ -240,6 +240,10 @@ int index_storage_mailbox_open(struct mailbox *box, bool move_to_memory)
        index_cache_register_defaults(box);
        box->view = mail_index_view_open(box->index);
        ibox->keyword_names = mail_index_get_keywords(box->index);
+       ibox->vsize_hdr_ext_id =
+               mail_index_ext_register(box->index, "hdr-vsize",
+                                       sizeof(struct index_vsize_header), 0,
+                                       sizeof(uint64_t));
 
        box->opened = TRUE;
 

diff --git a/src/lib-storage/index/index-storage.h b/src/lib-storage/index/index-storage.h
index 26f1be871d97cdcfde57ea56c6085d16731f0bc4..95a1992fe68e029624845a2ff5ac6b30d0e13233 100644 (file)
--- a/src/lib-storage/index/index-storage.h
+++ b/src/lib-storage/index/index-storage.h
@@ -27,6 +27,11 @@ struct index_transaction_context {
        struct mail_cache_transaction_ctx *cache_trans;
 };
 
+struct index_vsize_header {
+       uint64_t vsize;
+       uint32_t highest_uid;
+};
+
 struct index_mailbox_context {
        union mailbox_module_context module_ctx;
        enum mail_index_open_flags index_flags;
@@ -50,6 +55,7 @@ struct index_mailbox_context {
        ARRAY_TYPE(seq_range) recent_flags;
        uint32_t recent_flags_prev_uid;
        uint32_t recent_flags_count;
+       uint32_t vsize_hdr_ext_id;
 
        time_t sync_last_check;
 };

diff --git a/src/lib-storage/mail-storage.h b/src/lib-storage/mail-storage.h
index 589c739cc8f871637f6a4cfe035c99283582d6f9..504eeee69ec40f97cedde8bf2937ad8353fe75a8 100644 (file)
--- a/src/lib-storage/mail-storage.h
+++ b/src/lib-storage/mail-storage.h
@@ -62,7 +62,8 @@ enum mailbox_status_items {
        STATUS_FIRST_UNSEEN_SEQ = 0x20,
        STATUS_KEYWORDS         = 0x40,
        STATUS_HIGHESTMODSEQ    = 0x80,
-       STATUS_CACHE_FIELDS     = 0x100
+       STATUS_CACHE_FIELDS     = 0x100,
+       STATUS_VIRTUAL_SIZE     = 0x200
 };
 
 enum mailbox_search_result_flags {
@@ -181,6 +182,8 @@ struct mailbox_status {
 
        uint32_t first_unseen_seq;
        uint64_t highest_modseq;
+       /* sum of virtual size of all messages in mailbox */
+       uint64_t virtual_size;
 
        const ARRAY_TYPE(keywords) *keywords;
        /* Fields that have "temp" or "yes" caching decision. */