From: John Fawcett <john@voipsupport.it>
Date: Wed, 24 Apr 2019 07:34:49 +0000 (+0300)
Subject: fts-solr: Add batch_size configuration option
X-Git-Tag: 2.3.6~4
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=46052ee08f7e07dc39d0cd16e168b82aa9ec20b2;p=thirdparty%2Fdovecot%2Fcore.git

fts-solr: Add batch_size configuration option

batch_size: configure the number of emails sent in single requests to solr, default 1000
---

diff --git a/src/plugins/fts-solr/fts-backend-solr.c b/src/plugins/fts-solr/fts-backend-solr.c
index ea698a81af..767289adc9 100644
--- a/src/plugins/fts-solr/fts-backend-solr.c
+++ b/src/plugins/fts-solr/fts-backend-solr.c
@@ -28,8 +28,6 @@
 #define SOLR_HEADER_LINE_MAX_TRUNC_SIZE 1024
 
 #define SOLR_QUERY_MAX_MAILBOX_COUNT 10
-/* How often to flush indexing request to Solr before beginning a new one. */
-#define SOLR_MAIL_FLUSH_INTERVAL 1000
 
 struct solr_fts_backend {
 	struct fts_backend backend;
@@ -391,6 +389,7 @@ fts_backend_solr_update_deinit(struct fts_backend_update_context *_ctx)
 		(struct solr_fts_backend_update_context *)_ctx;
 	struct solr_fts_backend *backend =
 		(struct solr_fts_backend *)_ctx->backend;
+	struct fts_solr_user *fuser = FTS_SOLR_USER_CONTEXT(_ctx->backend->ns->user);
 	struct solr_fts_field *field;
 	const char *str;
 	int ret = _ctx->failed ? -1 : 0;
@@ -493,8 +492,9 @@ fts_backend_solr_uid_changed(struct solr_fts_backend_update_context *ctx,
 {
 	struct solr_fts_backend *backend =
 		(struct solr_fts_backend *)ctx->ctx.backend;
+	struct fts_solr_user *fuser = FTS_SOLR_USER_CONTEXT(ctx->ctx.backend->ns->user);
 
-	if (ctx->mails_since_flush >= SOLR_MAIL_FLUSH_INTERVAL) {
+	if (ctx->mails_since_flush >= fuser->set.batch_size) {
 		if (fts_backed_solr_build_flush(ctx) < 0)
 			ctx->ctx.failed = TRUE;
 	}
diff --git a/src/plugins/fts-solr/fts-solr-plugin.c b/src/plugins/fts-solr/fts-solr-plugin.c
index 1f8f8078c6..89ea9d5f33 100644
--- a/src/plugins/fts-solr/fts-solr-plugin.c
+++ b/src/plugins/fts-solr/fts-solr-plugin.c
@@ -9,6 +9,7 @@
 #include "fts-user.h"
 #include "fts-solr-plugin.h"
 
+#define DEFAULT_SOLR_BATCH_SIZE 1000
 
 const char *fts_solr_plugin_version = DOVECOT_ABI_VERSION;
 struct http_client *solr_http_client = NULL;
@@ -25,6 +26,8 @@ fts_solr_plugin_init_settings(struct mail_user *user,
 	if (str == NULL)
 		str = "";
 
+	set->batch_size = DEFAULT_SOLR_BATCH_SIZE;
+
 	for (tmp = t_strsplit_spaces(str, " "); *tmp != NULL; tmp++) {
 		if (str_begins(*tmp, "url=")) {
 			set->url = p_strdup(user->pool, *tmp + 4);
@@ -39,6 +42,12 @@ fts_solr_plugin_init_settings(struct mail_user *user,
 				p_strdup(user->pool, *tmp + 11);
 		} else if (str_begins(*tmp, "rawlog_dir=")) {
 			set->rawlog_dir = p_strdup(user->pool, *tmp + 11);
+		} else if (str_begins(*tmp, "batch_size=")) {
+			if (str_to_uint(*tmp+11, &set->batch_size) < 0 ||
+			    set->batch_size == 0) {
+				i_error("fts_solr: batch_size must be a positive integer");
+					return -1;
+			}
 		} else {
 			i_error("fts_solr: Invalid setting: %s", *tmp);
 			return -1;
diff --git a/src/plugins/fts-solr/fts-solr-plugin.h b/src/plugins/fts-solr/fts-solr-plugin.h
index 99ed4bbf17..506b2e8947 100644
--- a/src/plugins/fts-solr/fts-solr-plugin.h
+++ b/src/plugins/fts-solr/fts-solr-plugin.h
@@ -12,6 +12,7 @@
 
 struct fts_solr_settings {
 	const char *url, *default_ns_prefix, *rawlog_dir;
+	unsigned int batch_size;
 	bool use_libfts;
 	bool debug;
 };