* Fix shared usage of statfiles

* Add invalidation of statfiles in case of learning, so now statfiles
  are invalidated in about a minute after learning
* This should fix shared usage of statfile pool by several processes

diff --git a/src/controller.c b/src/controller.c
index bbf50d646a8e2863e29126a53588b0980f73ed80..2f14735404c93a73fe83447b7cd7f731dba4034b 100644 (file)
--- a/src/controller.c
+++ b/src/controller.c
@@ -870,6 +870,7 @@ controller_read_socket (f_str_t * in, void *arg)
                maybe_write_binlog (session->learn_classifier, st, statfile, tokens);
                msg_info ("learn success for message <%s>, for statfile: %s, sum weight: %.2f",
                                task->message_id, session->learn_symbol, sum);
+               statfile_pool_plan_invalidate (session->worker->srv->statfile_pool, DEFAULT_STATFILE_INVALIDATE_TIME, DEFAULT_STATFILE_INVALIDATE_JITTER);
                free_task (task, FALSE);
                i = rspamd_snprintf (out_buf, sizeof (out_buf), "learn ok, sum weight: %.2f" CRLF, sum);
                if (!rspamd_dispatcher_write (session->dispatcher, out_buf, i, FALSE, FALSE)) {

diff --git a/src/filter.c b/src/filter.c
index 7d1de3d208bd18b8472bd1f7544b721ef3dd04b9..f38831c007a99efd776b3fb6ff0b14a8773101a1 100644 (file)
--- a/src/filter.c
+++ b/src/filter.c
@@ -444,6 +444,7 @@ process_autolearn (struct statfile *st, struct worker_task *task, GTree * tokens
 
                        classifier->learn_func (ctx, task->worker->srv->statfile_pool, st->symbol, tokens, TRUE, NULL, 1., NULL);
                        maybe_write_binlog (ctx->cfg, st, statfile, tokens);
+                       statfile_pool_plan_invalidate (task->worker->srv->statfile_pool, DEFAULT_STATFILE_INVALIDATE_TIME, DEFAULT_STATFILE_INVALIDATE_JITTER);
                }
        }
 }

diff --git a/src/statfile.c b/src/statfile.c
index 0460a43c8ebcbdca3393e6e8bd29c5e17a42541f..1660a29f4362743a79c1a1d375ee8ecf24cc4f38 100644 (file)
--- a/src/statfile.c
+++ b/src/statfile.c
@@ -209,11 +209,10 @@ statfile_pool_new (memory_pool_t *pool, size_t max_size)
 {
        statfile_pool_t                *new;
 
-       new = memory_pool_alloc_shared (pool, sizeof (statfile_pool_t));
-       bzero (new, sizeof (statfile_pool_t));
+       new = memory_pool_alloc0 (pool, sizeof (statfile_pool_t));
        new->pool = memory_pool_new (memory_pool_get_size ());
        new->max = max_size;
-       new->files = memory_pool_alloc_shared (new->pool, STATFILES_MAX * sizeof (stat_file_t));
+       new->files = memory_pool_alloc0 (new->pool, STATFILES_MAX * sizeof (stat_file_t));
        new->lock = memory_pool_get_mutex (new->pool);
 
        return new;
@@ -813,3 +812,37 @@ statfile_get_total_blocks (stat_file_t *file)
 
        return header->total_blocks;
 }
+
+static void
+statfile_pool_invalidate_callback (int fd, short what, void *ud)
+{
+       statfile_pool_t                *pool = ud;
+       stat_file_t                    *file;
+       int                             i;
+
+       msg_info ("invalidating %d statfiles", pool->opened);
+
+       for (i = 0; i < pool->opened; i ++) {
+               file = &pool->files[i];
+               msync (file->map, file->len, MS_ASYNC | MS_INVALIDATE);
+       }
+
+}
+
+
+void
+statfile_pool_plan_invalidate (statfile_pool_t *pool, time_t seconds, time_t jitter)
+{
+
+       if (pool->invalidate_event == NULL || ! evtimer_pending (pool->invalidate_event, NULL)) {
+
+               if (pool->invalidate_event == NULL) {
+                       pool->invalidate_event = memory_pool_alloc (pool->pool, sizeof (struct event));
+               }
+               pool->invalidate_tv.tv_sec = seconds + g_random_int_range (0, jitter);
+               pool->invalidate_tv.tv_usec = 0;
+               evtimer_set (pool->invalidate_event, statfile_pool_invalidate_callback, pool);
+               evtimer_add (pool->invalidate_event, &pool->invalidate_tv);
+               msg_info ("invalidate of statfile pool is planned in %d seconds", (int)pool->invalidate_tv.tv_sec);
+       }
+}

diff --git a/src/statfile.h b/src/statfile.h
index 95f800ce6973ff6060f3c62251d4a2b1435ab25e..dfdd16dbe7cf7dac8a6f326e0d400807e23462d0 100644 (file)
--- a/src/statfile.h
+++ b/src/statfile.h
@@ -18,6 +18,9 @@
 #define STATFILE_SECTION_URLS 3
 #define STATFILE_SECTION_REGEXP 4
 
+#define DEFAULT_STATFILE_INVALIDATE_TIME 30
+#define DEFAULT_STATFILE_INVALIDATE_JITTER 30
+
 /**
  * Common statfile header
  */
@@ -89,6 +92,8 @@ typedef struct statfile_pool_s {
        size_t occupied;                                                /**< current size                                               */
        memory_pool_t *pool;                                    /**< memory pool object                                 */
        memory_pool_mutex_t *lock;                              /**< mutex                                                              */
+       struct event  *invalidate_event;        /**< event for pool invalidation        */
+       struct timeval invalidate_tv;
 } statfile_pool_t;
 
 /**
@@ -242,4 +247,10 @@ uint64_t statfile_get_used_blocks (stat_file_t *file);
  */
 uint64_t statfile_get_total_blocks (stat_file_t *file);
 
+
+/**
+ * Plan statfile pool invalidation
+ */
+void statfile_pool_plan_invalidate (statfile_pool_t *pool, time_t seconds, time_t jitter);
+
 #endif

diff --git a/src/tokenizers/tokenizers.c b/src/tokenizers/tokenizers.c
index 2bb478aaa57550dd8ac333d67ec18a2088d1859a..9ca690e4703285eae9b3a4626a8a3b40312da697 100644 (file)
--- a/src/tokenizers/tokenizers.c
+++ b/src/tokenizers/tokenizers.c
@@ -196,15 +196,16 @@ tokenize_headers (memory_pool_t * pool, struct worker_task *task, GTree ** tree)
        const char                     *value;
 
        ls = GMIME_OBJECT (task->message)->headers;
+       iter = g_mime_header_iter_new ();
 
        if (g_mime_header_list_get_iter (ls, iter)) {
                while (g_mime_header_iter_is_valid (iter)) {
                        new = memory_pool_alloc (pool, sizeof (token_node_t));
                        name = g_mime_header_iter_get_name (iter);
                        value = g_mime_header_iter_get_value (iter);
-                       headername.begin = name;
+                       headername.begin = (u_char *)name;
                        headername.len = strlen (name);
-                       headervalue.begin = value;
+                       headervalue.begin = (u_char *)value;
                        headervalue.len = strlen (value);
                        new->h1 = fstrhash (&headername) * primes[0];
                        new->h2 = fstrhash (&headervalue) * primes[1];
@@ -216,6 +217,7 @@ tokenize_headers (memory_pool_t * pool, struct worker_task *task, GTree ** tree)
                        }
                }
        }
+       g_mime_header_iter_free (iter);
 #endif
        return TRUE;
 }