[Feature] Allow to get task flags in C expressions

diff --git a/src/libmime/content_type.c b/src/libmime/content_type.c
index 524b6f636558dd144af2bac51a412debe1f93bbc..91c09e4bca1597f913fb2357bd32d689ab0a866d 100644 (file)
--- a/src/libmime/content_type.c
+++ b/src/libmime/content_type.c
@@ -477,7 +477,8 @@ rspamd_content_disposition_add_param (rspamd_mempool_t *pool,
        nparam = rspamd_mempool_alloc (pool, sizeof (*nparam));
        nparam->name.begin = name_start;
        nparam->name.len = name_end - name_start;
-       decoded = rspamd_mime_header_decode (pool, value_start, value_end - value_start);
+       decoded = rspamd_mime_header_decode (pool, value_start,
+                       value_end - value_start, NULL);
        RSPAMD_FTOK_FROM_STR (&nparam->value, decoded);
 
        if (!found) {

diff --git a/src/libmime/email_addr.c b/src/libmime/email_addr.c
index e1d52e1857de8256a0cb5b30f3152cf9467dc7e7..b936af58630a8e2ac504c772001548d9e0c184bb 100644 (file)
--- a/src/libmime/email_addr.c
+++ b/src/libmime/email_addr.c
@@ -137,7 +137,7 @@ rspamd_email_address_add (rspamd_mempool_t *pool,
        }
 
        if (name->len > 0) {
-               elt->name = rspamd_mime_header_decode (pool, name->str, name->len);
+               elt->name = rspamd_mime_header_decode (pool, name->str, name->len, NULL);
        }
 
        g_ptr_array_add (ar, elt);

diff --git a/src/libmime/message.c b/src/libmime/message.c
index 35ea5bb25f705c3ec9f275819f946125419bb9f7..3c29b1170198c62e599e70b410f72ab0045ce0c0 100644 (file)
--- a/src/libmime/message.c
+++ b/src/libmime/message.c
@@ -88,6 +88,12 @@ rspamd_mime_part_extract_words (struct rspamd_task *task,
                                        short_len++;
                                }
                        }
+
+                       if (w->flags & (RSPAMD_STAT_TOKEN_FLAG_BROKEN_UNICODE|
+                                               RSPAMD_STAT_TOKEN_FLAG_NORMALISED|
+                                       RSPAMD_STAT_TOKEN_FLAG_INVISIBLE_SPACES)) {
+                               task->flags |= RSPAMD_TASK_FLAG_BAD_UNICODE;
+                       }
                }
 
                if (part->utf_words && part->utf_words->len) {

diff --git a/src/libmime/mime_expressions.c b/src/libmime/mime_expressions.c
index 535b8a12476bcfe4b8ade43175315fdc0054f446..7bfbf34d4343db42897a0b1da459d7a2ec4f5cc9 100644 (file)
--- a/src/libmime/mime_expressions.c
+++ b/src/libmime/mime_expressions.c
@@ -24,62 +24,65 @@
 #include "utlist.h"
 
 gboolean rspamd_compare_encoding (struct rspamd_task *task,
-       GArray * args,
-       void *unused);
+                                                                 GArray * args,
+                                                                 void *unused);
 gboolean rspamd_header_exists (struct rspamd_task *task,
-       GArray * args,
-       void *unused);
+                                                          GArray * args,
+                                                          void *unused);
 gboolean rspamd_parts_distance (struct rspamd_task *task,
-       GArray * args,
-       void *unused);
+                                                               GArray * args,
+                                                               void *unused);
 gboolean rspamd_recipients_distance (struct rspamd_task *task,
-       GArray * args,
-       void *unused);
+                                                                        GArray * args,
+                                                                        void *unused);
 gboolean rspamd_has_only_html_part (struct rspamd_task *task,
-       GArray * args,
-       void *unused);
+                                                                       GArray * args,
+                                                                       void *unused);
 gboolean rspamd_is_recipients_sorted (struct rspamd_task *task,
-       GArray * args,
-       void *unused);
+                                                                         GArray * args,
+                                                                         void *unused);
 gboolean rspamd_compare_transfer_encoding (struct rspamd_task *task,
-       GArray * args,
-       void *unused);
+                                                                                  GArray * args,
+                                                                                  void *unused);
 gboolean rspamd_is_html_balanced (struct rspamd_task *task,
-       GArray * args,
-       void *unused);
+                                                                 GArray * args,
+                                                                 void *unused);
 gboolean rspamd_has_html_tag (struct rspamd_task *task,
-       GArray * args,
-       void *unused);
+                                                         GArray * args,
+                                                         void *unused);
 gboolean rspamd_has_fake_html (struct rspamd_task *task,
-       GArray * args,
-       void *unused);
+                                                          GArray * args,
+                                                          void *unused);
 static gboolean rspamd_raw_header_exists (struct rspamd_task *task,
-       GArray * args,
-       void *unused);
+                                                                                 GArray * args,
+                                                                                 void *unused);
 static gboolean rspamd_check_smtp_data (struct rspamd_task *task,
-       GArray * args,
-       void *unused);
+                                                                               GArray * args,
+                                                                               void *unused);
 static gboolean rspamd_content_type_is_type (struct rspamd_task * task,
-       GArray * args,
-       void *unused);
+                                                                                        GArray * args,
+                                                                                        void *unused);
 static gboolean rspamd_content_type_is_subtype (struct rspamd_task *task,
-       GArray * args,
-       void *unused);
+                                                                                               GArray * args,
+                                                                                               void *unused);
 static gboolean rspamd_content_type_has_param (struct rspamd_task * task,
-       GArray * args,
-       void *unused);
+                                                                                          GArray * args,
+                                                                                          void *unused);
 static gboolean rspamd_content_type_compare_param (struct rspamd_task * task,
-       GArray * args,
-       void *unused);
+                                                                                                  GArray * args,
+                                                                                                  void *unused);
 static gboolean rspamd_has_content_part (struct rspamd_task *task,
-       GArray * args,
-       void *unused);
+                                                                                GArray * args,
+                                                                                void *unused);
 static gboolean rspamd_has_content_part_len (struct rspamd_task *task,
-       GArray * args,
-       void *unused);
+                                                                                        GArray * args,
+                                                                                        void *unused);
 static gboolean rspamd_is_empty_body (struct rspamd_task *task,
-               GArray * args,
-               void *unused);
+                                                                         GArray * args,
+                                                                         void *unused);
+static gboolean rspamd_has_flag_expr (struct rspamd_task *task,
+                                                                         GArray * args,
+                                                                         void *unused);
 
 static rspamd_expression_atom_t * rspamd_mime_expr_parse (const gchar *line, gsize len,
                rspamd_mempool_t *pool, gpointer ud, GError **err);
@@ -136,25 +139,26 @@ static struct _fl {
        rspamd_internal_func_t func;
        void *user_data;
 } rspamd_functions_list[] = {
-       {"check_smtp_data", rspamd_check_smtp_data, NULL},
-       {"compare_encoding", rspamd_compare_encoding, NULL},
-       {"compare_parts_distance", rspamd_parts_distance, NULL},
-       {"compare_recipients_distance", rspamd_recipients_distance, NULL},
-       {"compare_transfer_encoding", rspamd_compare_transfer_encoding, NULL},
-       {"content_type_compare_param", rspamd_content_type_compare_param, NULL},
-       {"content_type_has_param", rspamd_content_type_has_param, NULL},
-       {"content_type_is_subtype", rspamd_content_type_is_subtype, NULL},
-       {"content_type_is_type", rspamd_content_type_is_type, NULL},
-       {"has_content_part", rspamd_has_content_part, NULL},
-       {"has_content_part_len", rspamd_has_content_part_len, NULL},
-       {"has_fake_html", rspamd_has_fake_html, NULL},
-       {"has_html_tag", rspamd_has_html_tag, NULL},
-       {"has_only_html_part", rspamd_has_only_html_part, NULL},
-       {"header_exists", rspamd_header_exists, NULL},
-       {"is_empty_body", rspamd_is_empty_body, NULL},
-       {"is_html_balanced", rspamd_is_html_balanced, NULL},
-       {"is_recipients_sorted", rspamd_is_recipients_sorted, NULL},
-       {"raw_header_exists", rspamd_raw_header_exists, NULL}
+               {"check_smtp_data", rspamd_check_smtp_data, NULL},
+               {"compare_encoding", rspamd_compare_encoding, NULL},
+               {"compare_parts_distance", rspamd_parts_distance, NULL},
+               {"compare_recipients_distance", rspamd_recipients_distance, NULL},
+               {"compare_transfer_encoding", rspamd_compare_transfer_encoding, NULL},
+               {"content_type_compare_param", rspamd_content_type_compare_param, NULL},
+               {"content_type_has_param", rspamd_content_type_has_param, NULL},
+               {"content_type_is_subtype", rspamd_content_type_is_subtype, NULL},
+               {"content_type_is_type", rspamd_content_type_is_type, NULL},
+               {"has_content_part", rspamd_has_content_part, NULL},
+               {"has_content_part_len", rspamd_has_content_part_len, NULL},
+               {"has_fake_html", rspamd_has_fake_html, NULL},
+               {"has_html_tag", rspamd_has_html_tag, NULL},
+               {"has_only_html_part", rspamd_has_only_html_part, NULL},
+               {"header_exists", rspamd_header_exists, NULL},
+               {"is_empty_body", rspamd_is_empty_body, NULL},
+               {"is_html_balanced", rspamd_is_html_balanced, NULL},
+               {"is_recipients_sorted", rspamd_is_recipients_sorted, NULL},
+               {"raw_header_exists", rspamd_raw_header_exists, NULL},
+               {"has_flag", rspamd_has_flag_expr, NULL},
 };
 
 const struct rspamd_atom_subr mime_expr_subr = {
@@ -2060,4 +2064,63 @@ rspamd_is_empty_body (struct rspamd_task *task,
        }
 
        return TRUE;
+}
+
+#define TASK_FLAG_READ(flag) do { \
+       result = !!(task->flags & (flag)); \
+} while(0)
+
+#define TASK_GET_FLAG(flag, strname, macro) do { \
+       if (!found && strcmp ((flag), strname) == 0) { \
+               TASK_FLAG_READ((macro)); \
+               found = TRUE; \
+       } \
+} while(0)
+
+static gboolean
+rspamd_has_flag_expr (struct rspamd_task *task,
+                                         GArray * args,
+                                         void *unused)
+{
+       gboolean found = FALSE, result = FALSE;
+       struct expression_argument *flag_arg;
+       const gchar *flag_str;
+
+       if (args == NULL) {
+               msg_warn_task ("no parameters to function");
+               return FALSE;
+       }
+
+       flag_arg = &g_array_index (args, struct expression_argument, 0);
+
+       if (flag_arg->type != EXPRESSION_ARGUMENT_NORMAL) {
+               msg_warn_task ("invalid parameter to function");
+               return FALSE;
+       }
+
+       flag_str = (const gchar *)flag_arg->data;
+
+       TASK_GET_FLAG (flag_str, "pass_all", RSPAMD_TASK_FLAG_PASS_ALL);
+       TASK_GET_FLAG (flag_str, "no_log", RSPAMD_TASK_FLAG_NO_LOG);
+       TASK_GET_FLAG (flag_str, "no_stat", RSPAMD_TASK_FLAG_NO_STAT);
+       TASK_GET_FLAG (flag_str, "skip", RSPAMD_TASK_FLAG_SKIP);
+       TASK_GET_FLAG (flag_str, "extended_urls", RSPAMD_TASK_FLAG_EXT_URLS);
+       TASK_GET_FLAG (flag_str, "learn_spam", RSPAMD_TASK_FLAG_LEARN_SPAM);
+       TASK_GET_FLAG (flag_str, "learn_ham", RSPAMD_TASK_FLAG_LEARN_HAM);
+       TASK_GET_FLAG (flag_str, "greylisted", RSPAMD_TASK_FLAG_GREYLISTED);
+       TASK_GET_FLAG (flag_str, "broken_headers",
+                       RSPAMD_TASK_FLAG_BROKEN_HEADERS);
+       TASK_GET_FLAG (flag_str, "skip_process",
+                       RSPAMD_TASK_FLAG_SKIP_PROCESS);
+       TASK_GET_FLAG (flag_str, "milter",
+                       RSPAMD_TASK_FLAG_MILTER);
+       TASK_GET_FLAG (flag_str, "bad_unicode",
+                       RSPAMD_TASK_FLAG_BAD_UNICODE);
+
+       if (!found) {
+               msg_warn_task ("invalid flag name %s", flag_str);
+               return FALSE;
+       }
+
+       return result;
 }
\ No newline at end of file

diff --git a/src/libmime/mime_headers.c b/src/libmime/mime_headers.c
index b782db010986e8225b13fa39c5a5bff0d33ed996..12661f0ea5c503aa7adc2f97fa9aa105b2c91aa1 100644 (file)
--- a/src/libmime/mime_headers.c
+++ b/src/libmime/mime_headers.c
@@ -348,8 +348,15 @@ rspamd_mime_headers_process (struct rspamd_task *task, GHashTable *target,
                        }
 
                        nh->value = tmp;
+
+                       gboolean broken_utf = FALSE;
+
                        nh->decoded = rspamd_mime_header_decode (task->task_pool,
-                                       nh->value, strlen (tmp));
+                                       nh->value, strlen (tmp), &broken_utf);
+
+                       if (broken_utf) {
+                               task->flags |= RSPAMD_TASK_FLAG_BAD_UNICODE;
+                       }
 
                        if (nh->decoded == NULL) {
                                nh->decoded = "";
@@ -531,7 +538,7 @@ rspamd_mime_header_sanity_check (GString *str)
 
 gchar *
 rspamd_mime_header_decode (rspamd_mempool_t *pool, const gchar *in,
-               gsize inlen)
+               gsize inlen, gboolean *invalid_utf)
 {
        GString *out;
        const guchar *c, *p, *end;
@@ -583,6 +590,10 @@ rspamd_mime_header_decode (rspamd_mempool_t *pool, const gchar *in,
                                        off = 0;
                                        U8_APPEND_UNSAFE (out->str + out->len - 3,
                                                        off, 0xfffd);
+
+                                       if (invalid_utf) {
+                                               *invalid_utf = TRUE;
+                                       }
                                }
                                else {
                                        c = p;

diff --git a/src/libmime/mime_headers.h b/src/libmime/mime_headers.h
index 03a7beae96415335a0cdc7f555338aaed04751c0..3c0c23a363eaf75877362091bbd76bfcc4f36a09 100644 (file)
--- a/src/libmime/mime_headers.h
+++ b/src/libmime/mime_headers.h
@@ -76,7 +76,7 @@ void rspamd_mime_headers_process (struct rspamd_task *task, GHashTable *target,
  * @return
  */
 gchar * rspamd_mime_header_decode (rspamd_mempool_t *pool, const gchar *in,
-               gsize inlen);
+               gsize inlen, gboolean *invalid_utf);
 
 /**
  * Encode mime header if needed

diff --git a/src/libserver/task.h b/src/libserver/task.h
index b41b308e4d751545ade99cd8fabc057ee08b0a53..34e160dc028281a971d767a3dc18c22a22668d77 100644 (file)
--- a/src/libserver/task.h
+++ b/src/libserver/task.h
@@ -116,6 +116,7 @@ enum rspamd_task_stage {
 #define RSPAMD_TASK_FLAG_OWN_POOL (1 << 27)
 #define RSPAMD_TASK_FLAG_MILTER (1 << 28)
 #define RSPAMD_TASK_FLAG_SSL (1 << 29)
+#define RSPAMD_TASK_FLAG_BAD_UNICODE (1 << 30)
 
 #define RSPAMD_TASK_IS_SKIPPED(task) (((task)->flags & RSPAMD_TASK_FLAG_SKIP))
 #define RSPAMD_TASK_IS_JSON(task) (((task)->flags & RSPAMD_TASK_FLAG_JSON))

diff --git a/src/lua/lua_task.c b/src/lua/lua_task.c
index 1b5f33cb7d6dcb19bd7283ec9e60cc7ddec8d6ba..05257fcdd3d0ac57c1b012523bd87665868a8e4f 100644 (file)
--- a/src/lua/lua_task.c
+++ b/src/lua/lua_task.c
@@ -3980,6 +3980,8 @@ lua_task_has_flag (lua_State *L)
                                RSPAMD_TASK_FLAG_SKIP_PROCESS);
                LUA_TASK_GET_FLAG (flag, "milter",
                                RSPAMD_TASK_FLAG_MILTER);
+               LUA_TASK_GET_FLAG (flag, "bad_unicode",
+                               RSPAMD_TASK_FLAG_BAD_UNICODE);
 
                if (!found) {
                        msg_warn_task ("unknown flag requested: %s", flag);