From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 7 Feb 2017 14:18:55 +0000 (+0200)
Subject: lib-storage: Add mail_vsize_bg_after_count setting.
X-Git-Tag: 2.3.0.rc1~2080
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=a758be690d736655ed7c1e3dfb46d3aaf4dd5db2;p=thirdparty%2Fdovecot%2Fcore.git

lib-storage: Add mail_vsize_bg_after_count setting.

If folder vsize calculation requires opening more than this many mails from
disk (i.e. mail sizes aren't in cache already), return failure and finish
the calculation via indexer process.

This should be used like:

protocol !indexer-worker {
  mail_vsize_bg_after_count = 10
}
---

diff --git a/src/lib-storage/index/index-mailbox-size.c b/src/lib-storage/index/index-mailbox-size.c
index 5107226809..dd0de12096 100644
--- a/src/lib-storage/index/index-mailbox-size.c
+++ b/src/lib-storage/index/index-mailbox-size.c
@@ -1,6 +1,10 @@
 /* Copyright (c) 2002-2017 Dovecot authors, see the included COPYING file */
 
 #include "lib.h"
+#include "str.h"
+#include "strescape.h"
+#include "net.h"
+#include "write-full.h"
 #include "file-create-locked.h"
 #include "mail-search-build.h"
 #include "index-storage.h"
@@ -33,6 +37,9 @@
 #define VSIZE_LOCK_SUFFIX ".vsize.lock"
 #define VSIZE_UPDATE_MAX_LOCK_SECS 10
 
+#define INDEXER_SOCKET_NAME "indexer"
+#define INDEXER_HANDSHAKE "VERSION\tindexer\t1\t0\n"
+
 struct mailbox_vsize_update {
 	struct mailbox *box;
 	struct mail_index_view *view;
@@ -43,6 +50,7 @@ struct mailbox_vsize_update {
 	struct file_lock *lock;
 	bool rebuild;
 	bool written;
+	bool finish_in_background;
 };
 
 static void vsize_header_refresh(struct mailbox_vsize_update *update)
@@ -192,6 +200,36 @@ index_mailbox_vsize_update_write(struct mailbox_vsize_update *update)
 	(void)mail_index_transaction_commit(&trans);
 }
 
+static void index_mailbox_vsize_notify_indexer(struct mailbox *box)
+{
+	string_t *str = t_str_new(256);
+	const char *path;
+	int fd;
+
+	path = t_strconcat(box->storage->user->set->base_dir,
+			   "/"INDEXER_SOCKET_NAME, NULL);
+	fd = net_connect_unix(path);
+	if (fd == -1) {
+		mail_storage_set_critical(box->storage,
+			"Can't start vsize building on background: "
+			"net_connect_unix(%s) failed: %m", path);
+		return;
+	}
+	str_append(str, INDEXER_HANDSHAKE);
+	str_append(str, "APPEND\t0\t");
+	str_append_tabescaped(str, box->storage->user->username);
+	str_append_c(str, '\t');
+	str_append_tabescaped(str, box->vname);
+	str_append_c(str, '\n');
+
+	if (write_full(fd, str_data(str), str_len(str)) < 0) {
+		mail_storage_set_critical(box->storage,
+			"Can't start vsize building on background: "
+			"write(%s) failed: %m", path);
+	}
+	i_close_fd(&fd);
+}
+
 void index_mailbox_vsize_update_deinit(struct mailbox_vsize_update **_update)
 {
 	struct mailbox_vsize_update *update = *_update;
@@ -206,6 +244,9 @@ void index_mailbox_vsize_update_deinit(struct mailbox_vsize_update **_update)
 		file_lock_free(&update->lock);
 		i_close_fd(&update->lock_fd);
 	}
+	if (update->finish_in_background)
+		index_mailbox_vsize_notify_indexer(update->box);
+
 	mail_index_view_close(&update->view);
 	i_free(update->lock_path);
 	i_free(update);
@@ -243,6 +284,7 @@ index_mailbox_vsize_hdr_add_missing(struct mailbox_vsize_update *update)
 	struct mail_search_args *search_args;
 	struct mailbox_status status;
 	struct mail *mail;
+	unsigned int mails_left;
 	uint32_t seq1, seq2;
 	uoff_t vsize;
 	int ret = 0;
@@ -272,8 +314,33 @@ index_mailbox_vsize_hdr_add_missing(struct mailbox_vsize_update *update)
 	trans = mailbox_transaction_begin(update->box, 0);
 	search_ctx = mailbox_search_init(trans, search_args, NULL,
 					 MAIL_FETCH_VIRTUAL_SIZE, NULL);
+	mails_left = update->box->storage->set->mail_vsize_bg_after_count == 0 ?
+		UINT_MAX : update->box->storage->set->mail_vsize_bg_after_count;
 	while (mailbox_search_next(search_ctx, &mail)) {
-		if (mail_get_virtual_size(mail, &vsize) < 0) {
+		if (mails_left == UINT_MAX) {
+			/* we want to build the full vsize here */
+			ret = mail_get_virtual_size(mail, &vsize);
+		} else {
+			/* if vsize building wants to open too many mails from
+			   storage, return temporary failure and finish up the
+			   calculation in background. */
+			mail->lookup_abort = MAIL_LOOKUP_ABORT_NOT_IN_CACHE;
+			ret = mail_get_virtual_size(mail, &vsize);
+			mail->lookup_abort = MAIL_LOOKUP_ABORT_NEVER;
+			if (ret < 0 &&
+			    mailbox_get_last_mail_error(update->box) == MAIL_ERROR_NOTPOSSIBLE) {
+				/* size isn't in cache. */
+				if (mails_left == 0) {
+					mail_storage_set_error(update->box->storage, MAIL_ERROR_INUSE,
+						"Finishing vsize calculation on background");
+					update->finish_in_background = TRUE;
+					break;
+				}
+				mails_left--;
+				ret = mail_get_virtual_size(mail, &vsize);
+			}
+		}
+		if (ret < 0) {
 			if (mail->expunged)
 				continue;
 			ret = -1;
diff --git a/src/lib-storage/mail-storage-settings.c b/src/lib-storage/mail-storage-settings.c
index 754fab0cad..0990bc8f42 100644
--- a/src/lib-storage/mail-storage-settings.c
+++ b/src/lib-storage/mail-storage-settings.c
@@ -42,6 +42,7 @@ static const struct setting_define mail_storage_setting_defines[] = {
 	DEF(SET_UINT, mail_max_keyword_length),
 	DEF(SET_TIME, mail_max_lock_timeout),
 	DEF(SET_TIME, mail_temp_scan_interval),
+	DEF(SET_UINT, mail_vsize_bg_after_count),
 	DEF(SET_BOOL, mail_save_crlf),
 	DEF(SET_ENUM, mail_fsync),
 	DEF(SET_BOOL, mmap_disable),
@@ -82,6 +83,7 @@ const struct mail_storage_settings mail_storage_default_settings = {
 	.mail_max_keyword_length = 50,
 	.mail_max_lock_timeout = 0,
 	.mail_temp_scan_interval = 7*24*60*60,
+	.mail_vsize_bg_after_count = 0,
 	.mail_save_crlf = FALSE,
 	.mail_fsync = "optimized:never:always",
 	.mmap_disable = FALSE,
diff --git a/src/lib-storage/mail-storage-settings.h b/src/lib-storage/mail-storage-settings.h
index ddf2a86da7..a292e222d3 100644
--- a/src/lib-storage/mail-storage-settings.h
+++ b/src/lib-storage/mail-storage-settings.h
@@ -28,6 +28,7 @@ struct mail_storage_settings {
 	unsigned int mail_max_keyword_length;
 	unsigned int mail_max_lock_timeout;
 	unsigned int mail_temp_scan_interval;
+	unsigned int mail_vsize_bg_after_count;
 	bool mail_save_crlf;
 	const char *mail_fsync;
 	bool mmap_disable;
diff --git a/src/plugins/quota/quota-count.c b/src/plugins/quota/quota-count.c
index 64691bb1db..5b7019fa03 100644
--- a/src/plugins/quota/quota-count.c
+++ b/src/plugins/quota/quota-count.c
@@ -55,6 +55,9 @@ quota_count_mailbox(struct quota_root *root, struct mail_namespace *ns,
 			i_error("quota: Couldn't get size of mailbox %s: %s",
 				vname, errstr);
 			ret = -1;
+		} else if (error == MAIL_ERROR_INUSE) {
+			/* started on background. don't log an error. */
+			ret = -1;
 		} else {
 			/* non-temporary error, e.g. ACLs denied access. */
 			ret = 0;