* Extract domains from emails to check via surbl

* Fixes to url extracter
* Check urls in redirector even if they are listed in surbl whitelist

diff --git a/src/plugins/surbl.c b/src/plugins/surbl.c
index 2fbeb5e2b3dc0d3006eb90f28c010a6ed9bac656..34280101e5eaa8afd92812b7d0337f6944308c7a 100644 (file)
--- a/src/plugins/surbl.c
+++ b/src/plugins/surbl.c
@@ -307,7 +307,8 @@ surbl_module_reconfig (struct config_file *cfg)
 
 
 static char                    *
-format_surbl_request (memory_pool_t * pool, f_str_t * hostname, struct suffix_item *suffix, gboolean append_suffix, GError ** err)
+format_surbl_request (memory_pool_t * pool, f_str_t * hostname, struct suffix_item *suffix,
+               gboolean append_suffix, GError ** err, gboolean forced)
 {
        GHashTable                     *t;
        char                           *result = NULL, *dots[MAX_LEVELS], num_buf[sizeof("18446744073709551616")], *p;
@@ -409,7 +410,7 @@ format_surbl_request (memory_pool_t * pool, f_str_t * hostname, struct suffix_it
                }
        }
 
-       if (g_hash_table_lookup (surbl_module_ctx->whitelist, result) != NULL) {
+       if (!forced && g_hash_table_lookup (surbl_module_ctx->whitelist, result) != NULL) {
                msg_debug ("url %s is whitelisted", result);
                g_set_error (err, SURBL_ERROR,  /* error domain */
                                                WHITELIST_ERROR,        /* error code */
@@ -429,7 +430,8 @@ format_surbl_request (memory_pool_t * pool, f_str_t * hostname, struct suffix_it
 }
 
 static void
-make_surbl_requests (struct uri *url, struct worker_task *task, GTree * tree, struct suffix_item *suffix)
+make_surbl_requests (struct uri *url, struct worker_task *task, GTree * tree,
+               struct suffix_item *suffix, gboolean forced)
 {
        char                           *surbl_req;
        f_str_t                         f;
@@ -440,7 +442,7 @@ make_surbl_requests (struct uri *url, struct worker_task *task, GTree * tree, st
        f.len = url->hostlen;
 
        if (check_view (task->cfg->views, suffix->symbol, task)) {
-               if ((surbl_req = format_surbl_request (task->task_pool, &f, suffix, TRUE, &err)) != NULL) {
+               if ((surbl_req = format_surbl_request (task->task_pool, &f, suffix, TRUE, &err, forced)) != NULL) {
                        if (g_tree_lookup (tree, surbl_req) == NULL) {
                                g_tree_insert (tree, surbl_req, surbl_req);
                                param = memory_pool_alloc (task->task_pool, sizeof (struct dns_param));
@@ -590,7 +592,7 @@ memcached_callback (memcached_ctx_t * ctx, memc_error_t error, void *data)
                        param->task->save.saved = 1;
                        process_filters (param->task);
                }
-               make_surbl_requests (param->url, param->task, param->tree, param->suffix);
+               make_surbl_requests (param->url, param->task, param->tree, param->suffix, FALSE);
                break;
        default:
                return;
@@ -656,7 +658,7 @@ free_redirector_session (void *ud)
        event_del (&param->ev);
        close (param->sock);
        param->task->save.saved--;
-       make_surbl_requests (param->url, param->task, param->tree, param->suffix);
+       make_surbl_requests (param->url, param->task, param->tree, param->suffix, FALSE);
        if (param->task->save.saved == 0) {
                /* Call other filters */
                param->task->save.saved = 1;
@@ -739,7 +741,7 @@ register_redirector_call (struct uri *url, struct worker_task *task, GTree * url
        if (s == -1) {
                msg_info ("<%s> cannot create tcp socket failed: %s", task->message_id, strerror (errno));
                task->save.saved--;
-               make_surbl_requests (url, task, url_tree, suffix);
+               make_surbl_requests (url, task, url_tree, suffix, FALSE);
                return;
        }
 
@@ -756,6 +758,8 @@ register_redirector_call (struct uri *url, struct worker_task *task, GTree * url
        event_set (&param->ev, s, EV_WRITE, redirector_callback, (void *)param);
        event_add (&param->ev, timeout);
        register_async_event (task->s, free_redirector_session, param, FALSE);
+
+       msg_info ("<%s> registered redirector call for %s", task->message_id, struri (url));
 }
 
 static                          gboolean
@@ -774,14 +778,14 @@ tree_url_callback (gpointer key, gpointer value, void *data)
        if (surbl_module_ctx->use_redirector) {
                f.begin = url->host;
                f.len = url->hostlen;
-               if ((urlstr = format_surbl_request (param->task->task_pool, &f, NULL, FALSE, &err)) != NULL) {
+               if ((urlstr = format_surbl_request (param->task->task_pool, &f, NULL, FALSE, &err, TRUE)) != NULL) {
                        if (g_hash_table_lookup (surbl_module_ctx->redirector_hosts, urlstr) != NULL) {
                                register_redirector_call (url, param->task, param->tree, param->suffix);
                                param->task->save.saved++;
                                return FALSE;
                        }
                }
-               make_surbl_requests (url, param->task, param->tree, param->suffix);
+               make_surbl_requests (url, param->task, param->tree, param->suffix, TRUE);
        }
        else {
                if (param->task->worker->srv->cfg->memcached_servers_num > 0) {
@@ -789,7 +793,7 @@ tree_url_callback (gpointer key, gpointer value, void *data)
                        param->task->save.saved++;
                }
                else {
-                       make_surbl_requests (url, param->task, param->tree, param->suffix);
+                       make_surbl_requests (url, param->task, param->tree, param->suffix, FALSE);
                }
        }
 
@@ -872,7 +876,7 @@ urls_command_handler (struct worker_task *task)
                        g_tree_insert (url_tree, struri (url), url);
                        f.begin = url->host;
                        f.len = url->hostlen;
-                       if ((urlstr = format_surbl_request (task->task_pool, &f, NULL, FALSE, &err)) != NULL) {
+                       if ((urlstr = format_surbl_request (task->task_pool, &f, NULL, FALSE, &err, FALSE)) != NULL) {
                                if (g_list_next (cur) != NULL) {
                                        r += rspamd_snprintf (outbuf + r, buflen - r - 2, "%s <\"%s\">, ", (char *)urlstr, struri (url));
                                }

diff --git a/src/url.c b/src/url.c
index b839d7566801f6641342b1656685a7eb8785a394..5a166567493aef8183be7c3b1bbe317f84edd562 100644 (file)
--- a/src/url.c
+++ b/src/url.c
@@ -105,9 +105,9 @@ static const struct _proto      protocol_backends[] = {
        {"ftp", 21, NULL, 1, 1, 0, 0},
        {"http", 80, NULL, 1, 1, 0, 0},
        {"https", 443, NULL, 1, 1, 0, 1},
-
+       {"mailto", 25, NULL, 1, 1, 0, 0},
        /* Keep these last! */
-       {NULL, 0, NULL, 0, 0, 1, 0},
+       {NULL, 0, NULL, 0, 0, 1, 0}
 };
 
 /* Convert an ASCII hex digit to the corresponding number between 0
@@ -952,6 +952,8 @@ url_web_start (const gchar *begin, const gchar *end, const gchar *pos, url_match
                        return FALSE;
                }
        }
+       match->m_begin = pos;
+
        return TRUE;
 }
 static gboolean
@@ -1109,12 +1111,37 @@ domain:
 static gboolean
 url_email_start (const gchar *begin, const gchar *end, const gchar *pos, url_match_t *match)
 {
+       /* Check what we have found */
+       if (pos > begin && *pos == '@') {
+               if (is_atom (*(pos - 1)) && is_domain (*(pos + 1))) {
+                       match->m_begin = pos + 1;
+                       return TRUE;
+               }
+       }
+       else {
+               while (pos < end && is_atom (*pos)) {
+                       if (*pos == '@') {
+                               match->m_begin = pos + 1;
+                               return TRUE;
+                       }
+                       pos ++;
+               }
+       }
        return FALSE;
 }
+
 static gboolean
 url_email_end (const gchar *begin, const gchar *end, const gchar *pos, url_match_t *match)
 {
-       return FALSE;
+       const gchar                    *p;
+
+       p = pos + strlen (match->pattern);
+
+       while (p < end && (is_domain (*p) || (*p == '.' && p + 1 < end && is_domain (*(p + 1))))) {
+               p ++;
+       }
+       match->m_len = p - match->m_begin;
+       return TRUE;
 }
 
 void
@@ -1149,7 +1176,7 @@ url_parse_text (memory_pool_t * pool, struct worker_task *task, struct mime_text
                                matcher = &matchers[idx];
                                m.pattern = matcher->pattern;
                                m.prefix = matcher->prefix;
-                               if (matcher->start (p, pos, end, &m) && matcher->end (p, pos, end, &m)) {
+                               if (matcher->start (p, end, pos, &m) && matcher->end (p, end, pos, &m)) {
                                        url_str = memory_pool_alloc (task->task_pool, m.m_len + 1);
                                        memcpy (url_str, m.m_begin, m.m_len);
                                        url_str[m.m_len] = '\0';