From: Vsevolod Stakhov Date: Tue, 3 May 2016 17:18:43 +0000 (+0100) Subject: [Feature] Various improvements in greylist module X-Git-Tag: 1.3.0~561 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=36d8785fd0820ab6b5d07aad43d41da427ed9a12;p=thirdparty%2Frspamd.git [Feature] Various improvements in greylist module --- diff --git a/src/libserver/task.c b/src/libserver/task.c index 890e05f207..3bedb7cf14 100644 --- a/src/libserver/task.c +++ b/src/libserver/task.c @@ -430,7 +430,6 @@ rspamd_task_process (struct rspamd_task *task, guint stages) return TRUE; } - if (RSPAMD_TASK_IS_PROCESSED (task)) { return TRUE; } @@ -440,9 +439,9 @@ rspamd_task_process (struct rspamd_task *task, guint stages) task->processed_stages |= RSPAMD_TASK_STAGE_DONE; msg_info_task ("skip filters, as pre-filter returned %s action", rspamd_action_to_str (task->pre_result.action)); + return TRUE; } - task->flags |= RSPAMD_TASK_FLAG_PROCESSING; st = rspamd_task_select_processing_stage (task, stages); diff --git a/src/plugins/lua/greylist.lua b/src/plugins/lua/greylist.lua index 2f058e9a61..1493ad96fd 100644 --- a/src/plugins/lua/greylist.lua +++ b/src/plugins/lua/greylist.lua @@ -23,7 +23,7 @@ local whitelisted_ip local settings = { expire = 86400, -- 1 day by default timeout = 300, -- 5 minutes by default - key_prefix = 'grey.', -- default hash name + key_prefix = 'rg', -- default hash name max_data_len = 10240, -- default data limit to hash message = 'Try again later', -- default greylisted message symbol = 'GREYLIST', @@ -57,7 +57,7 @@ local function data_key(task) local h = hash.create() h:update(body, len) - local b32 = settings['key_prefix'] .. h:base32() + local b32 = settings['key_prefix'] .. 'b' .. h:base32():sub(1, 20) task:get_mempool():set_variable("grey_bodyhash", b32) return b32 end @@ -77,7 +77,6 @@ local function envelope_key(task) end h:update(addr) - local rcpt = task:get_recipients('smtp') if rcpt then table.sort(rcpt, function(r1, r2) @@ -101,36 +100,31 @@ local function envelope_key(task) h:update(s) end - local b32 = settings['key_prefix'] .. h:base32() + local b32 = settings['key_prefix'] .. 'm' .. h:base32():sub(1, 20) task:get_mempool():set_variable("grey_metahash", b32) return b32 end +-- Returns pair of booleans: found,greylisted local function check_time(task, tm, type) local t = tonumber(tm) if not t then rspamd_logger.infox(task, 'not a valid number: %s', tm) - return false + return false,false end local now = rspamd_util.get_time() if now - t < settings['timeout'] then - local end_time = rspamd_util.time_to_string(t + settings['timeout']) - rspamd_logger.infox(task, 'greylisted till "%s" using %s key', - end_time, type) - task:set_pre_result('soft reject', settings['message']) - task:get_mempool():set_variable("grey_greylisted", end_time) - - return true + return true,true else -- We just set variable to pass when in post-filter stage task:get_mempool():set_variable("grey_whitelisted", type) - return true + return true,false end - return false + return false,false end local function greylist_check(task) @@ -140,20 +134,41 @@ local function greylist_check(task) local addr = upstream:get_addr() local function redis_get_cb(task, err, data) - local ret = false + local ret_body = false + local greylisted_body = false + local ret_meta = false + local greylisted_meta = false + local time + local greylist_type + if data then if data[1] and type(data[1]) ~= 'userdata' then - ret = check_time(task, data[1], 'body') + ret_body,greylisted_body = check_time(task, data[1], 'body') + if greylisted_body then + local end_time = rspamd_util.time_to_string(rspamd_util.get_time() + + settings['timeout']) + task:get_mempool():set_variable("grey_greylisted_body", end_time) + end end - if not ret and type(data[2]) ~= 'userdata' then - ret = check_time(task, data[2], 'meta') + if data[2] and type(data[2]) ~= 'userdata' then + if not ret_body or greylisted_body then + ret_meta,greylisted_meta = check_time(task, data[2], 'meta') + + if greylisted_meta then + local end_time = rspamd_util.time_to_string(rspamd_util.get_time() + + settings['timeout']) + task:get_mempool():set_variable("grey_greylisted_meta", end_time) + end + end end upstream:ok() - if not ret then - rspamd_logger.infox(task, 'greylisted till "%s", new record', - rspamd_util.time_to_string(rspamd_util.get_time() + settings['timeout']), - type) + if not ret_body and not ret_meta then + local t = tostring(math.floor(rspamd_util.get_time())) + local end_time = rspamd_util.time_to_string(t + settings['timeout']) + rspamd_logger.infox(task, 'greylisted till "%s", new record', end_time) + task:insert_result(settings['symbol'], 0.0, 'greylisted', end_time, + 'new record') task:set_pre_result('soft reject', settings['message']) -- Create new record if addr then @@ -162,7 +177,6 @@ local function greylist_check(task) host = addr }) - local t = tostring(math.floor(rspamd_util.get_time())) if conn then conn:add_cmd('SETEX', { body_key, tostring(settings['expire']), t @@ -170,10 +184,21 @@ local function greylist_check(task) conn:add_cmd('SETEX', { meta_key, tostring(settings['expire']), t }) + local end_time = rspamd_util.time_to_string(rspamd_util.get_time() + + settings['timeout']) + task:get_mempool():set_variable("grey_greylisted", end_time) else rspamd_logger.infox(task, 'got error while connecting to redis: %1', addr) upstream:fail() end + elseif greylisted_body and greylisted_meta then + local end_time = rspamd_util.time_to_string(time + settings['timeout']) + rspamd_logger.infox(task, 'greylisted till "%s" using %s key', + end_time, type) + task:insert_result(settings['symbol'], 0.0, 'greylisted', end_time, + greylist_type) + task:set_pre_result('soft reject', settings['message']) + task:get_mempool():set_variable("grey_greylisted", end_time) end end elseif err then @@ -183,27 +208,22 @@ local function greylist_check(task) end if addr then - rspamd_redis.make_request(task, addr, redis_get_cb, 'MGET', - {body_key, meta_key}) + if not rspamd_redis.make_request(task, addr, redis_get_cb, 'MGET', + {body_key, meta_key}) then + rspamd_logger.errx(task, 'cannot make redis request to check results') + end end end local function greylist_set(task) local is_whitelisted = task:get_mempool():get_variable("grey_whitelisted") - local is_greylisted = task:get_mempool():get_variable("grey_greylisted") + local action = task:get_metric_action('default') local body_key = data_key(task) local meta_key = envelope_key(task) local upstream = upstreams:get_upstream_by_hash(body_key .. meta_key) local addr = upstream:get_addr() - if is_greylisted then - task:insert_result(settings['symbol'], 0.0, 'delayed', is_greylisted) - rspamd_logger.infox(task, 'greylisting delayed till %s', - is_greylisted) - - return - end if is_whitelisted then if action == 'greylist' then -- We are going to accept message @@ -236,8 +256,23 @@ local function greylist_set(task) end else if action ~= 'no action' then - -- We need to delay message, hence set a temporary result - task:insert_result(settings['symbol'], 0.0, 'greylisted', 'delayed') + local grey_res = task:get_mempool():get_variable("grey_greylisted_body") + + if grey_res then + -- We need to delay message, hence set a temporary result + task:insert_result(settings['symbol'], 0.0, grey_res, 'body') + rspamd_logger.infox(task, 'greylisting delayed till "%s": body', grey_res) + else + grey_res = task:get_mempool():get_variable("grey_greylisted_meta") + + if grey_res then + task:insert_result(settings['symbol'], 0.0, grey_res, 'meta') + rspamd_logger.infox(task, 'greylisting delayed till "%s": meta', grey_res) + else + task:insert_result(settings['symbol'], 0.0, 'unknown') + rspamd_logger.infox(task, 'greylisting delayed: unknown, internal error') + end + end task:set_metric_action('default', 'soft reject') task:set_pre_result('soft reject', settings['message']) else