lib-storage: Add mail_vsize_bg_after_count setting.

If folder vsize calculation requires opening more than this many mails from
disk (i.e. mail sizes aren't in cache already), return failure and finish
the calculation via indexer process.

This should be used like:

protocol !indexer-worker {
  mail_vsize_bg_after_count = 10
}

diff --git a/src/lib-storage/index/index-mailbox-size.c b/src/lib-storage/index/index-mailbox-size.c
index 51072268099aa9d690e850ca9cb22292d2313c87..dd0de12096e597d79e47091f05618d453b3f1f5c 100644 (file)
--- a/src/lib-storage/index/index-mailbox-size.c
+++ b/src/lib-storage/index/index-mailbox-size.c
@@ -1,6 +1,10 @@
 /* Copyright (c) 2002-2017 Dovecot authors, see the included COPYING file */
 
 #include "lib.h"
+#include "str.h"
+#include "strescape.h"
+#include "net.h"
+#include "write-full.h"
 #include "file-create-locked.h"
 #include "mail-search-build.h"
 #include "index-storage.h"
@@ -33,6 +37,9 @@
 #define VSIZE_LOCK_SUFFIX ".vsize.lock"
 #define VSIZE_UPDATE_MAX_LOCK_SECS 10
 
+#define INDEXER_SOCKET_NAME "indexer"
+#define INDEXER_HANDSHAKE "VERSION\tindexer\t1\t0\n"
+
 struct mailbox_vsize_update {
        struct mailbox *box;
        struct mail_index_view *view;
@@ -43,6 +50,7 @@ struct mailbox_vsize_update {
        struct file_lock *lock;
        bool rebuild;
        bool written;
+       bool finish_in_background;
 };
 
 static void vsize_header_refresh(struct mailbox_vsize_update *update)
@@ -192,6 +200,36 @@ index_mailbox_vsize_update_write(struct mailbox_vsize_update *update)
        (void)mail_index_transaction_commit(&trans);
 }
 
+static void index_mailbox_vsize_notify_indexer(struct mailbox *box)
+{
+       string_t *str = t_str_new(256);
+       const char *path;
+       int fd;
+
+       path = t_strconcat(box->storage->user->set->base_dir,
+                          "/"INDEXER_SOCKET_NAME, NULL);
+       fd = net_connect_unix(path);
+       if (fd == -1) {
+               mail_storage_set_critical(box->storage,
+                       "Can't start vsize building on background: "
+                       "net_connect_unix(%s) failed: %m", path);
+               return;
+       }
+       str_append(str, INDEXER_HANDSHAKE);
+       str_append(str, "APPEND\t0\t");
+       str_append_tabescaped(str, box->storage->user->username);
+       str_append_c(str, '\t');
+       str_append_tabescaped(str, box->vname);
+       str_append_c(str, '\n');
+
+       if (write_full(fd, str_data(str), str_len(str)) < 0) {
+               mail_storage_set_critical(box->storage,
+                       "Can't start vsize building on background: "
+                       "write(%s) failed: %m", path);
+       }
+       i_close_fd(&fd);
+}
+
 void index_mailbox_vsize_update_deinit(struct mailbox_vsize_update **_update)
 {
        struct mailbox_vsize_update *update = *_update;
@@ -206,6 +244,9 @@ void index_mailbox_vsize_update_deinit(struct mailbox_vsize_update **_update)
                file_lock_free(&update->lock);
                i_close_fd(&update->lock_fd);
        }
+       if (update->finish_in_background)
+               index_mailbox_vsize_notify_indexer(update->box);
+
        mail_index_view_close(&update->view);
        i_free(update->lock_path);
        i_free(update);
@@ -243,6 +284,7 @@ index_mailbox_vsize_hdr_add_missing(struct mailbox_vsize_update *update)
        struct mail_search_args *search_args;
        struct mailbox_status status;
        struct mail *mail;
+       unsigned int mails_left;
        uint32_t seq1, seq2;
        uoff_t vsize;
        int ret = 0;
@@ -272,8 +314,33 @@ index_mailbox_vsize_hdr_add_missing(struct mailbox_vsize_update *update)
        trans = mailbox_transaction_begin(update->box, 0);
        search_ctx = mailbox_search_init(trans, search_args, NULL,
                                         MAIL_FETCH_VIRTUAL_SIZE, NULL);
+       mails_left = update->box->storage->set->mail_vsize_bg_after_count == 0 ?
+               UINT_MAX : update->box->storage->set->mail_vsize_bg_after_count;
        while (mailbox_search_next(search_ctx, &mail)) {
-               if (mail_get_virtual_size(mail, &vsize) < 0) {
+               if (mails_left == UINT_MAX) {
+                       /* we want to build the full vsize here */
+                       ret = mail_get_virtual_size(mail, &vsize);
+               } else {
+                       /* if vsize building wants to open too many mails from
+                          storage, return temporary failure and finish up the
+                          calculation in background. */
+                       mail->lookup_abort = MAIL_LOOKUP_ABORT_NOT_IN_CACHE;
+                       ret = mail_get_virtual_size(mail, &vsize);
+                       mail->lookup_abort = MAIL_LOOKUP_ABORT_NEVER;
+                       if (ret < 0 &&
+                           mailbox_get_last_mail_error(update->box) == MAIL_ERROR_NOTPOSSIBLE) {
+                               /* size isn't in cache. */
+                               if (mails_left == 0) {
+                                       mail_storage_set_error(update->box->storage, MAIL_ERROR_INUSE,
+                                               "Finishing vsize calculation on background");
+                                       update->finish_in_background = TRUE;
+                                       break;
+                               }
+                               mails_left--;
+                               ret = mail_get_virtual_size(mail, &vsize);
+                       }
+               }
+               if (ret < 0) {
                        if (mail->expunged)
                                continue;
                        ret = -1;

diff --git a/src/lib-storage/mail-storage-settings.c b/src/lib-storage/mail-storage-settings.c
index e4aa614023ef53afda6fc4cbef986e6490100024..8f7f034cc1585a9e1ef51b20f86d1ac10b3e7356 100644 (file)
--- a/src/lib-storage/mail-storage-settings.c
+++ b/src/lib-storage/mail-storage-settings.c
@@ -42,6 +42,7 @@ static const struct setting_define mail_storage_setting_defines[] = {
        DEF(SET_UINT, mail_max_keyword_length),
        DEF(SET_TIME, mail_max_lock_timeout),
        DEF(SET_TIME, mail_temp_scan_interval),
+       DEF(SET_UINT, mail_vsize_bg_after_count),
        DEF(SET_BOOL, mail_save_crlf),
        DEF(SET_ENUM, mail_fsync),
        DEF(SET_BOOL, mmap_disable),
@@ -82,6 +83,7 @@ const struct mail_storage_settings mail_storage_default_settings = {
        .mail_max_keyword_length = 50,
        .mail_max_lock_timeout = 0,
        .mail_temp_scan_interval = 7*24*60*60,
+       .mail_vsize_bg_after_count = 0,
        .mail_save_crlf = FALSE,
        .mail_fsync = "optimized:never:always",
        .mmap_disable = FALSE,

diff --git a/src/lib-storage/mail-storage-settings.h b/src/lib-storage/mail-storage-settings.h
index cbed4fd8503e82db29d775099df54f21d4d70eb8..ac5cc264180c178d95a5cf875ff740828b6071e7 100644 (file)
--- a/src/lib-storage/mail-storage-settings.h
+++ b/src/lib-storage/mail-storage-settings.h
@@ -28,6 +28,7 @@ struct mail_storage_settings {
        unsigned int mail_max_keyword_length;
        unsigned int mail_max_lock_timeout;
        unsigned int mail_temp_scan_interval;
+       unsigned int mail_vsize_bg_after_count;
        bool mail_save_crlf;
        const char *mail_fsync;
        bool mmap_disable;

diff --git a/src/plugins/quota/quota-count.c b/src/plugins/quota/quota-count.c
index 64691bb1dbba45d92ad59466e1ff34146efad400..5b7019fa030d4d7c7d3bb20af2e01d9c823cc49e 100644 (file)
--- a/src/plugins/quota/quota-count.c
+++ b/src/plugins/quota/quota-count.c
@@ -55,6 +55,9 @@ quota_count_mailbox(struct quota_root *root, struct mail_namespace *ns,
                        i_error("quota: Couldn't get size of mailbox %s: %s",
                                vname, errstr);
                        ret = -1;
+               } else if (error == MAIL_ERROR_INUSE) {
+                       /* started on background. don't log an error. */
+                       ret = -1;
                } else {
                        /* non-temporary error, e.g. ACLs denied access. */
                        ret = 0;