fts-solr: Add batch_size configuration option

batch_size: configure the number of emails sent in single requests to solr, default 1000

diff --git a/src/plugins/fts-solr/fts-backend-solr.c b/src/plugins/fts-solr/fts-backend-solr.c
index ea698a81af49f565b156ba3b1ec8ad89cb8503ad..767289adc974ee35ea5f8e1f68463c212cce5473 100644 (file)
--- a/src/plugins/fts-solr/fts-backend-solr.c
+++ b/src/plugins/fts-solr/fts-backend-solr.c
@@ -28,8 +28,6 @@
 #define SOLR_HEADER_LINE_MAX_TRUNC_SIZE 1024
 
 #define SOLR_QUERY_MAX_MAILBOX_COUNT 10
-/* How often to flush indexing request to Solr before beginning a new one. */
-#define SOLR_MAIL_FLUSH_INTERVAL 1000
 
 struct solr_fts_backend {
        struct fts_backend backend;
@@ -391,6 +389,7 @@ fts_backend_solr_update_deinit(struct fts_backend_update_context *_ctx)
                (struct solr_fts_backend_update_context *)_ctx;
        struct solr_fts_backend *backend =
                (struct solr_fts_backend *)_ctx->backend;
+       struct fts_solr_user *fuser = FTS_SOLR_USER_CONTEXT(_ctx->backend->ns->user);
        struct solr_fts_field *field;
        const char *str;
        int ret = _ctx->failed ? -1 : 0;
@@ -493,8 +492,9 @@ fts_backend_solr_uid_changed(struct solr_fts_backend_update_context *ctx,
 {
        struct solr_fts_backend *backend =
                (struct solr_fts_backend *)ctx->ctx.backend;
+       struct fts_solr_user *fuser = FTS_SOLR_USER_CONTEXT(ctx->ctx.backend->ns->user);
 
-       if (ctx->mails_since_flush >= SOLR_MAIL_FLUSH_INTERVAL) {
+       if (ctx->mails_since_flush >= fuser->set.batch_size) {
                if (fts_backed_solr_build_flush(ctx) < 0)
                        ctx->ctx.failed = TRUE;
        }

diff --git a/src/plugins/fts-solr/fts-solr-plugin.c b/src/plugins/fts-solr/fts-solr-plugin.c
index 1f8f8078c6136726ac84494328c29c19716310fb..89ea9d5f3348018bd9c045db9bf4e788598f8c37 100644 (file)
--- a/src/plugins/fts-solr/fts-solr-plugin.c
+++ b/src/plugins/fts-solr/fts-solr-plugin.c
@@ -9,6 +9,7 @@
 #include "fts-user.h"
 #include "fts-solr-plugin.h"
 
+#define DEFAULT_SOLR_BATCH_SIZE 1000
 
 const char *fts_solr_plugin_version = DOVECOT_ABI_VERSION;
 struct http_client *solr_http_client = NULL;
@@ -25,6 +26,8 @@ fts_solr_plugin_init_settings(struct mail_user *user,
        if (str == NULL)
                str = "";
 
+       set->batch_size = DEFAULT_SOLR_BATCH_SIZE;
+
        for (tmp = t_strsplit_spaces(str, " "); *tmp != NULL; tmp++) {
                if (str_begins(*tmp, "url=")) {
                        set->url = p_strdup(user->pool, *tmp + 4);
@@ -39,6 +42,12 @@ fts_solr_plugin_init_settings(struct mail_user *user,
                                p_strdup(user->pool, *tmp + 11);
                } else if (str_begins(*tmp, "rawlog_dir=")) {
                        set->rawlog_dir = p_strdup(user->pool, *tmp + 11);
+               } else if (str_begins(*tmp, "batch_size=")) {
+                       if (str_to_uint(*tmp+11, &set->batch_size) < 0 ||
+                           set->batch_size == 0) {
+                               i_error("fts_solr: batch_size must be a positive integer");
+                                       return -1;
+                       }
                } else {
                        i_error("fts_solr: Invalid setting: %s", *tmp);
                        return -1;

diff --git a/src/plugins/fts-solr/fts-solr-plugin.h b/src/plugins/fts-solr/fts-solr-plugin.h
index 99ed4bbf17348174535a3192ce4261b198029d8a..506b2e8947b5d9f8475024a7ef2d20d8a632828c 100644 (file)
--- a/src/plugins/fts-solr/fts-solr-plugin.h
+++ b/src/plugins/fts-solr/fts-solr-plugin.h
@@ -12,6 +12,7 @@
 
 struct fts_solr_settings {
        const char *url, *default_ns_prefix, *rawlog_dir;
+       unsigned int batch_size;
        bool use_libfts;
        bool debug;
 };