lib-storage: Add mail_vsize_bg_after_count setting.

If folder vsize calculation requires opening more than this many mails from
disk (i.e. mail sizes aren't in cache already), return failure and finish
the calculation via indexer process.

This should be used like:

protocol !indexer-worker {
  mail_vsize_bg_after_count = 10
}

diff --git a/src/lib-storage/index/index-mailbox-size.c b/src/lib-storage/index/index-mailbox-size.c
index 51072268099aa9d690e850ca9cb22292d2313c87..dd0de12096e597d79e47091f05618d453b3f1f5c 100644 (file)
--- a/src/lib-storage/index/index-mailbox-size.c
+++ b/src/lib-storage/index/index-mailbox-size.c
@@ -1,6 +1,10 @@
 /* Copyright (c) 2002-2017 Dovecot authors, see the included COPYING file */
 
 #include "lib.h"
+#include "str.h"
+#include "strescape.h"
+#include "net.h"
+#include "write-full.h"
 #include "file-create-locked.h"
 #include "mail-search-build.h"
 #include "index-storage.h"
@@ -33,6 +37,9 @@
 #define VSIZE_LOCK_SUFFIX ".vsize.lock"
 #define VSIZE_UPDATE_MAX_LOCK_SECS 10
 
+#define INDEXER_SOCKET_NAME "indexer"
+#define INDEXER_HANDSHAKE "VERSION\tindexer\t1\t0\n"
+
 struct mailbox_vsize_update {
        struct mailbox *box;
        struct mail_index_view *view;
@@ -43,6 +50,7 @@ struct mailbox_vsize_update {
        struct file_lock *lock;
        bool rebuild;
        bool written;
+       bool finish_in_background;
 };
 
 static void vsize_header_refresh(struct mailbox_vsize_update *update)
@@ -192,6 +200,36 @@ index_mailbox_vsize_update_write(struct mailbox_vsize_update *update)
        (void)mail_index_transaction_commit(&trans);
 }
 
+static void index_mailbox_vsize_notify_indexer(struct mailbox *box)
+{
+       string_t *str = t_str_new(256);
+       const char *path;
+       int fd;
+
+       path = t_strconcat(box->storage->user->set->base_dir,
+                          "/"INDEXER_SOCKET_NAME, NULL);
+       fd = net_connect_unix(path);
+       if (fd == -1) {
+               mail_storage_set_critical(box->storage,
+                       "Can't start vsize building on background: "
+                       "net_connect_unix(%s) failed: %m", path);
+               return;
+       }
+       str_append(str, INDEXER_HANDSHAKE);
+       str_append(str, "APPEND\t0\t");
+       str_append_tabescaped(str, box->storage->user->username);
+       str_append_c(str, '\t');
+       str_append_tabescaped(str, box->vname);
+       str_append_c(str, '\n');
+
+       if (write_full(fd, str_data(str), str_len(str)) < 0) {
+               mail_storage_set_critical(box->storage,
+                       "Can't start vsize building on background: "
+                       "write(%s) failed: %m", path);
+       }
+       i_close_fd(&fd);
+}
+
 void index_mailbox_vsize_update_deinit(struct mailbox_vsize_update **_update)
 {
        struct mailbox_vsize_update *update = *_update;
@@ -206,6 +244,9 @@ void index_mailbox_vsize_update_deinit(struct mailbox_vsize_update **_update)
                file_lock_free(&update->lock);
                i_close_fd(&update->lock_fd);
        }
+       if (update->finish_in_background)
+               index_mailbox_vsize_notify_indexer(update->box);
+
        mail_index_view_close(&update->view);
        i_free(update->lock_path);
        i_free(update);
@@ -243,6 +284,7 @@ index_mailbox_vsize_hdr_add_missing(struct mailbox_vsize_update *update)
        struct mail_search_args *search_args;
        struct mailbox_status status;
        struct mail *mail;
+       unsigned int mails_left;
        uint32_t seq1, seq2;
        uoff_t vsize;
        int ret = 0;
@@ -272,8 +314,33 @@ index_mailbox_vsize_hdr_add_missing(struct mailbox_vsize_update *update)
        trans = mailbox_transaction_begin(update->box, 0);
        search_ctx = mailbox_search_init(trans, search_args, NULL,
                                         MAIL_FETCH_VIRTUAL_SIZE, NULL);
+       mails_left = update->box->storage->set->mail_vsize_bg_after_count == 0 ?
+               UINT_MAX : update->box->storage->set->mail_vsize_bg_after_count;
        while (mailbox_search_next(search_ctx, &mail)) {
-               if (mail_get_virtual_size(mail, &vsize) < 0) {
+               if (mails_left == UINT_MAX) {
+                       /* we want to build the full vsize here */
+                       ret = mail_get_virtual_size(mail, &vsize);
+               } else {
+                       /* if vsize building wants to open too many mails from
+                          storage, return temporary failure and finish up the
+                          calculation in background. */
+                       mail->lookup_abort = MAIL_LOOKUP_ABORT_NOT_IN_CACHE;
+                       ret = mail_get_virtual_size(mail, &vsize);
+                       mail->lookup_abort = MAIL_LOOKUP_ABORT_NEVER;
+                       if (ret < 0 &&
+                           mailbox_get_last_mail_error(update->box) == MAIL_ERROR_NOTPOSSIBLE) {
+                               /* size isn't in cache. */
+                               if (mails_left == 0) {
+                                       mail_storage_set_error(update->box->storage, MAIL_ERROR_INUSE,
+                                               "Finishing vsize calculation on background");
+                                       update->finish_in_background = TRUE;
+                                       break;
+                               }
+                               mails_left--;
+                               ret = mail_get_virtual_size(mail, &vsize);
+                       }
+               }
+               if (ret < 0) {
                        if (mail->expunged)
                                continue;
                        ret = -1;

diff --git a/src/lib-storage/mail-storage-settings.c b/src/lib-storage/mail-storage-settings.c
index 754fab0cada3fc9eb1e317dd57fd39ac2497d922..0990bc8f42f31b840dd6036e7e30ac54b16ca91c 100644 (file)
--- a/src/lib-storage/mail-storage-settings.c
+++ b/src/lib-storage/mail-storage-settings.c
@@ -42,6 +42,7 @@ static const struct setting_define mail_storage_setting_defines[] = {
        DEF(SET_UINT, mail_max_keyword_length),
        DEF(SET_TIME, mail_max_lock_timeout),
        DEF(SET_TIME, mail_temp_scan_interval),
+       DEF(SET_UINT, mail_vsize_bg_after_count),
        DEF(SET_BOOL, mail_save_crlf),
        DEF(SET_ENUM, mail_fsync),
        DEF(SET_BOOL, mmap_disable),
@@ -82,6 +83,7 @@ const struct mail_storage_settings mail_storage_default_settings = {
        .mail_max_keyword_length = 50,
        .mail_max_lock_timeout = 0,
        .mail_temp_scan_interval = 7*24*60*60,
+       .mail_vsize_bg_after_count = 0,
        .mail_save_crlf = FALSE,
        .mail_fsync = "optimized:never:always",
        .mmap_disable = FALSE,

diff --git a/src/lib-storage/mail-storage-settings.h b/src/lib-storage/mail-storage-settings.h
index ddf2a86da7b617167bf592403da85c2ac1521404..a292e222d36c9b26a5db120bc93153e3c23c7495 100644 (file)
--- a/src/lib-storage/mail-storage-settings.h
+++ b/src/lib-storage/mail-storage-settings.h
@@ -28,6 +28,7 @@ struct mail_storage_settings {
        unsigned int mail_max_keyword_length;
        unsigned int mail_max_lock_timeout;
        unsigned int mail_temp_scan_interval;
+       unsigned int mail_vsize_bg_after_count;
        bool mail_save_crlf;
        const char *mail_fsync;
        bool mmap_disable;

diff --git a/src/plugins/quota/quota-count.c b/src/plugins/quota/quota-count.c
index 64691bb1dbba45d92ad59466e1ff34146efad400..5b7019fa030d4d7c7d3bb20af2e01d9c823cc49e 100644 (file)
--- a/src/plugins/quota/quota-count.c
+++ b/src/plugins/quota/quota-count.c
@@ -55,6 +55,9 @@ quota_count_mailbox(struct quota_root *root, struct mail_namespace *ns,
                        i_error("quota: Couldn't get size of mailbox %s: %s",
                                vname, errstr);
                        ret = -1;
+               } else if (error == MAIL_ERROR_INUSE) {
+                       /* started on background. don't log an error. */
+                       ret = -1;
                } else {
                        /* non-temporary error, e.g. ACLs denied access. */
                        ret = 0;