Simplify learn and process functions

diff --git a/src/libstat/backends/backends.h b/src/libstat/backends/backends.h
index 42eba8d61dfdbe21296a27b46a45cb4d3ed7e7dd..800030414bc3f0626a1e67a66df6687b0f2e2421 100644 (file)
--- a/src/libstat/backends/backends.h
+++ b/src/libstat/backends/backends.h
@@ -46,12 +46,14 @@ struct rspamd_stat_backend {
                        struct rspamd_statfile *st);
        gpointer (*runtime)(struct rspamd_task *task,
                        struct rspamd_statfile_config *stcf, gboolean learn, gpointer ctx);
-       gboolean (*process_token)(struct rspamd_task *task, struct token_node_s *tok,
-                       struct rspamd_token_result *res, gpointer ctx);
+       gboolean (*process_tokens)(struct rspamd_task *task, GPtrArray *tokens,
+                       gint id,
+                       gpointer ctx);
        void (*finalize_process)(struct rspamd_task *task,
                        gpointer runtime, gpointer ctx);
-       gboolean (*learn_token)(struct rspamd_task *task, struct token_node_s *tok,
-                       struct rspamd_token_result *res, gpointer ctx);
+       gboolean (*learn_tokens)(struct rspamd_task *task, GPtrArray *tokens,
+                       gint id,
+                       gpointer ctx);
        gulong (*total_learns)(struct rspamd_task *task,
                        gpointer runtime, gpointer ctx);
        void (*finalize_learn)(struct rspamd_task *task,
@@ -73,16 +75,14 @@ struct rspamd_stat_backend {
                gpointer rspamd_##name##_runtime (struct rspamd_task *task, \
                                struct rspamd_statfile_config *stcf, \
                                gboolean learn, gpointer ctx); \
-               gboolean rspamd_##name##_process_token (struct rspamd_task *task, \
-                               struct token_node_s *tok, \
-                               struct rspamd_token_result *res, \
+               gboolean rspamd_##name##_process_tokens (struct rspamd_task *task, \
+                GPtrArray *tokens, gint id, \
                                gpointer ctx); \
                void rspamd_##name##_finalize_process (struct rspamd_task *task, \
                                gpointer runtime, \
                                gpointer ctx); \
-               gboolean rspamd_##name##_learn_token (struct rspamd_task *task, \
-                               struct token_node_s *tok, \
-                               struct rspamd_token_result *res, \
+               gboolean rspamd_##name##_learn_tokens (struct rspamd_task *task, \
+                GPtrArray *tokens, gint id, \
                                gpointer ctx); \
                void rspamd_##name##_finalize_learn (struct rspamd_task *task, \
                                gpointer runtime, \

diff --git a/src/libstat/backends/mmaped_file.c b/src/libstat/backends/mmaped_file.c
index 651e39f38c1225491f000678f19041fd32b353d4..5e0b176ef52b9a5d07b1171994d178d8781fa0af 100644 (file)
--- a/src/libstat/backends/mmaped_file.c
+++ b/src/libstat/backends/mmaped_file.c
@@ -844,45 +844,48 @@ rspamd_mmaped_file_runtime (struct rspamd_task *task,
 }
 
 gboolean
-rspamd_mmaped_file_process_token (struct rspamd_task *task, rspamd_token_t *tok,
-               struct rspamd_token_result *res,
+rspamd_mmaped_file_process_tokens (struct rspamd_task *task, GPtrArray *tokens,
+               gint id,
                gpointer p)
 {
        rspamd_mmaped_file_t *mf = p;
        guint32 h1, h2;
+       rspamd_token_t *tok;
+       guint i;
 
-       g_assert (res != NULL);
+       g_assert (tokens != NULL);
        g_assert (p != NULL);
-       g_assert (tok != NULL);
-       g_assert (tok->datalen >= sizeof (guint32) * 2);
 
-       memcpy (&h1, tok->data, sizeof (h1));
-       memcpy (&h2, tok->data + sizeof (h1), sizeof (h2));
-       res->value = rspamd_mmaped_file_get_block (ctx, mf, h1, h2);
-
-       if (res->value > 0.0) {
-               return TRUE;
+       for (i = 0; i < tokens->len; i++) {
+               tok = g_ptr_array_index (tokens, i);
+               memcpy (&h1, tok->data, sizeof (h1));
+               memcpy (&h2, tok->data + sizeof (h1), sizeof (h2));
+               tok->values[id] = rspamd_mmaped_file_get_block (mf, h1, h2);
        }
 
-       return FALSE;
+       return TRUE;
 }
 
 gboolean
-rspamd_mmaped_file_learn_token (struct rspamd_task *task, rspamd_token_t *tok,
-               struct rspamd_token_result *res,
+rspamd_mmaped_file_learn_tokens (struct rspamd_task *task, GPtrArray *tokens,
+               gint id,
                gpointer p)
 {
        rspamd_mmaped_file_t *mf = p;
        guint32 h1, h2;
+       rspamd_token_t *tok;
+       guint i;
 
-       g_assert (res != NULL);
+       g_assert (tokens != NULL);
        g_assert (p != NULL);
-       g_assert (tok != NULL);
-       g_assert (tok->datalen >= sizeof (guint32) * 2);
 
-       memcpy (&h1, tok->data, sizeof (h1));
-       memcpy (&h2, tok->data + sizeof (h1), sizeof (h2));
-       rspamd_mmaped_file_set_block (task->task_pool, ctx, mf, h1, h2, res->value);
+       for (i = 0; i < tokens->len; i++) {
+               tok = g_ptr_array_index (tokens, i);
+               memcpy (&h1, tok->data, sizeof (h1));
+               memcpy (&h2, tok->data + sizeof (h1), sizeof (h2));
+               rspamd_mmaped_file_set_block (task->task_pool, mf, h1, h2,
+                               tok->values[id]);
+       }
 
        return TRUE;
 }

diff --git a/src/libstat/backends/sqlite3_backend.c b/src/libstat/backends/sqlite3_backend.c
index 591ab7669372b9109357a8d225ae00df9aa1d22f..3e05bee808cd59f973aec448c6705a1797878bf4 100644 (file)
--- a/src/libstat/backends/sqlite3_backend.c
+++ b/src/libstat/backends/sqlite3_backend.c
@@ -691,64 +691,64 @@ rspamd_sqlite3_runtime (struct rspamd_task *task,
 }
 
 gboolean
-rspamd_sqlite3_process_token (struct rspamd_task *task, struct token_node_s *tok,
-               struct rspamd_token_result *res, gpointer p)
+rspamd_sqlite3_process_tokens (struct rspamd_task *task,
+               GPtrArray *tokens,
+               gint id, gpointer p)
 {
        struct rspamd_stat_sqlite3_db *bk;
        struct rspamd_stat_sqlite3_rt *rt = p;
        gint64 iv = 0, idx;
+       guint i;
+       rspamd_token_t *tok;
 
-       g_assert (res != NULL);
        g_assert (p != NULL);
-       g_assert (tok != NULL);
-       g_assert (tok->datalen >= sizeof (guint32) * 2);
+       g_assert (tokens != NULL);
 
        bk = rt->db;
 
-       if (bk == NULL) {
-               /* Statfile is does not exist, so all values are zero */
-               res->value = 0.0;
-               return FALSE;
-       }
-
-       if (!bk->in_transaction) {
-               rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt,
-                               RSPAMD_STAT_BACKEND_TRANSACTION_START_DEF);
-               bk->in_transaction = TRUE;
-       }
+       for (i = 0; i < tokens->len; i ++) {
+               tok = g_ptr_array_index (tokens, i);
 
-       if (rt->user_id == -1) {
-               if (bk->enable_users) {
-                       rt->user_id = rspamd_sqlite3_get_user (bk, task, FALSE);
-               }
-               else {
-                       rt->user_id = 0;
+               if (bk == NULL) {
+                       /* Statfile is does not exist, so all values are zero */
+                       tok->values[id] = 0.0;
+                       continue;
                }
-       }
 
-       if (rt->lang_id == -1) {
-               if (bk->enable_languages) {
-                       rt->lang_id = rspamd_sqlite3_get_language (bk, task, FALSE);
+               if (!bk->in_transaction) {
+                       rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt,
+                                       RSPAMD_STAT_BACKEND_TRANSACTION_START_DEF);
+                       bk->in_transaction = TRUE;
                }
-               else {
-                       rt->lang_id = 0;
+
+               if (rt->user_id == -1) {
+                       if (bk->enable_users) {
+                               rt->user_id = rspamd_sqlite3_get_user (bk, task, FALSE);
+                       }
+                       else {
+                               rt->user_id = 0;
+                       }
                }
-       }
 
-       memcpy (&idx, tok->data, sizeof (idx));
+               if (rt->lang_id == -1) {
+                       if (bk->enable_languages) {
+                               rt->lang_id = rspamd_sqlite3_get_language (bk, task, FALSE);
+                       }
+                       else {
+                               rt->lang_id = 0;
+                       }
+               }
 
-       if (rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt,
-                       RSPAMD_STAT_BACKEND_GET_TOKEN,
-                       idx, rt->user_id, rt->lang_id, &iv) == SQLITE_OK) {
-               res->value = iv;
+               memcpy (&idx, tok->data, sizeof (idx));
 
-               if (iv == 0) {
-                       return FALSE;
+               if (rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt,
+                               RSPAMD_STAT_BACKEND_GET_TOKEN,
+                               idx, rt->user_id, rt->lang_id, &iv) == SQLITE_OK) {
+                       tok->values[id] = iv;
+               }
+               else {
+                       tok->values[id] = 0.0;
                }
-       }
-       else {
-               res->value = 0.0;
-               return FALSE;
        }
 
 
@@ -778,56 +778,63 @@ rspamd_sqlite3_finalize_process (struct rspamd_task *task, gpointer runtime,
 }
 
 gboolean
-rspamd_sqlite3_learn_token (struct rspamd_task *task, struct token_node_s *tok,
-               struct rspamd_token_result *res, gpointer p)
+rspamd_sqlite3_learn_tokens (struct rspamd_task *task, GPtrArray *tokens,
+               gint id, gpointer p)
 {
        struct rspamd_stat_sqlite3_db *bk;
        struct rspamd_stat_sqlite3_rt *rt = p;
        gint64 iv = 0, idx;
+       guint i;
+       rspamd_token_t *tok;
 
-       g_assert (res != NULL);
+       g_assert (tokens != NULL);
        g_assert (p != NULL);
-       g_assert (tok != NULL);
-       g_assert (tok->datalen >= sizeof (guint32) * 2);
 
        bk = rt->db;
 
-       if (bk == NULL) {
-               /* Statfile is does not exist, so all values are zero */
-               return FALSE;
-       }
-
-       if (!bk->in_transaction) {
-               rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt,
-                               RSPAMD_STAT_BACKEND_TRANSACTION_START_IM);
-               bk->in_transaction = TRUE;
-       }
-
-       if (rt->user_id == -1) {
-               if (bk->enable_users) {
-                       rt->user_id = rspamd_sqlite3_get_user (bk, task, TRUE);
+       for (i = 0; i < tokens->len; i++) {
+               tok = g_ptr_array_index (tokens, i);
+               if (bk == NULL) {
+                       /* Statfile is does not exist, so all values are zero */
+                       return FALSE;
                }
-               else {
-                       rt->user_id = 0;
+
+               if (!bk->in_transaction) {
+                       rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt,
+                                       RSPAMD_STAT_BACKEND_TRANSACTION_START_IM);
+                       bk->in_transaction = TRUE;
                }
-       }
 
-       if (rt->lang_id == -1) {
-               if (bk->enable_languages) {
-                       rt->lang_id = rspamd_sqlite3_get_language (bk, task, TRUE);
+               if (rt->user_id == -1) {
+                       if (bk->enable_users) {
+                               rt->user_id = rspamd_sqlite3_get_user (bk, task, TRUE);
+                       }
+                       else {
+                               rt->user_id = 0;
+                       }
                }
-               else {
-                       rt->lang_id = 0;
+
+               if (rt->lang_id == -1) {
+                       if (bk->enable_languages) {
+                               rt->lang_id = rspamd_sqlite3_get_language (bk, task, TRUE);
+                       }
+                       else {
+                               rt->lang_id = 0;
+                       }
                }
-       }
 
-       iv = res->value;
-       memcpy (&idx, tok->data, sizeof (idx));
+               iv = tok->values[id];
+               memcpy (&idx, tok->data, sizeof (idx));
+
+               if (rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt,
+                               RSPAMD_STAT_BACKEND_SET_TOKEN,
+                               idx, rt->user_id, rt->lang_id, iv) != SQLITE_OK) {
+                       rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt,
+                                       RSPAMD_STAT_BACKEND_TRANSACTION_ROLLBACK);
+                       bk->in_transaction = FALSE;
 
-       if (rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt,
-                       RSPAMD_STAT_BACKEND_SET_TOKEN,
-                       idx, rt->user_id, rt->lang_id, iv) != SQLITE_OK) {
-               return FALSE;
+                       return FALSE;
+               }
        }
 
        return TRUE;