From: Vsevolod Stakhov Date: Sun, 26 Nov 2017 18:05:23 +0000 (+0000) Subject: [Feature] Implement DKIM reputation adjustments X-Git-Tag: 1.7.0~405 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=e7073a76bd2543beef76428b4662dbc9ad68ee07;p=thirdparty%2Frspamd.git [Feature] Implement DKIM reputation adjustments --- diff --git a/lualib/lua_redis.lua b/lualib/lua_redis.lua index eda8dc37f5..8dafa023ba 100644 --- a/lualib/lua_redis.lua +++ b/lualib/lua_redis.lua @@ -103,9 +103,10 @@ local function rspamd_parse_redis_server(module_name, module_opts, no_fallback) else opts = module_opts end - local ret = false if opts then + local ret + if opts.redis then ret = try_load_redis_servers(opts.redis, result) @@ -127,6 +128,8 @@ local function rspamd_parse_redis_server(module_name, module_opts, no_fallback) opts = rspamd_config:get_all_opt('redis') if opts then + local ret + if opts[module_name] then ret = try_load_redis_servers(opts[module_name], result) if ret then diff --git a/lualib/rspamd_config_transform.lua b/lualib/rspamd_config_transform.lua index af3e8180e8..c7bf56b907 100644 --- a/lualib/rspamd_config_transform.lua +++ b/lualib/rspamd_config_transform.lua @@ -15,7 +15,6 @@ limitations under the License. ]]-- local logger = require "rspamd_logger" -local fun = require "fun" local function override_defaults(def, override) if not override then @@ -83,9 +82,9 @@ local function metric_pairs(t) for k,v in pairs(tbl) do if type(k) ~= 'number' then -- We can also have implicit arrays here - local is_implicit = is_implicit(v) + local sym_implicit = is_implicit(v) - if is_implicit then + if sym_implicit then for _,elt in ipairs(v) do table.insert(keys, {k, elt}) end diff --git a/src/plugins/lua/reputation.lua b/src/plugins/lua/reputation.lua index a85cbe4229..8ff53f09be 100644 --- a/src/plugins/lua/reputation.lua +++ b/src/plugins/lua/reputation.lua @@ -64,8 +64,17 @@ local function gen_dkim_queries(task, rule) local semicolon = lpeg.P(':') local domain = lpeg.C((1 - semicolon)^0) local res = lpeg.S'+-?~' - gr = domain * semicolon * lpeg.C(res) + + local function res_to_label(ch) + if ch == '+' then return 'a' + elseif ch == '-' then return 'r' + else return 'u' + end + end + + gr = domain * semicolon * lpeg.C(res / res_to_label) end + if dkim_trace and dkim_trace.options then for _,opt in ipairs(dkim_trace.options) do local dom,res = lpeg.match(gr, opt) @@ -83,6 +92,8 @@ local function dkim_reputation_filter(task, rule) local requests = gen_dkim_queries(task, rule) local results = {} local nchecked = 0 + local rep_accepted = 0.0 + local rep_rejected = 0.0 local function tokens_cb(err, token, values) nchecked = nchecked + 1 @@ -92,30 +103,33 @@ local function dkim_reputation_filter(task, rule) end if nchecked == #requests then - -- Check the url with maximum hits - local mhits = 0 - for k,_ in pairs(results) do - if requests[k][2] > mhits then - mhits = requests[k][2] + for k,v in pairs(results) do + if requests[k] == 'a' then + rep_accepted = rep_accepted + generic_reputation_calc(v, rule, 1.0) + elseif requests[k] == 'r' then + rep_rejected = rep_rejected + generic_reputation_calc(v, rule, 1.0) end end - if mhits > 0 then - local score = 0 - for k,v in pairs(results) do - score = score + generic_reputation_calc(v, rule, requests[k][2] / mhits) + -- Set local reputation symbol + if rep_accepted > 0 or rep_rejected > 0 then + if rep_accepted > rep_rejected then + task:insert_result(rule.symbol, -(rep_accepted - rep_rejected)) + else + task:insert_result(rule.symbol, (rep_rejected - rep_accepted)) end - if math.abs(score) > 1e-3 then - -- TODO: add description - task:insert_result(rule.symbol, score) - end + -- Store results for future DKIM results adjustments + task:get_mempool():set_variable("dkim_reputation_accept", tostring(rep_accepted)) + task:get_mempool():set_variable("dkim_reputation_reject", tostring(rep_rejected)) end end end for dom,res in pairs(requests) do - rule.backend.get_token(task, rule, dom, tokens_cb) + -- tld + "." + check_result, e.g. example.com.+ - reputation for valid sigs + local query = string.format('%s.%s', dom, res) + rule.backend.get_token(task, rule, query, tokens_cb) end end @@ -138,12 +152,34 @@ local function dkim_reputation_idempotent(task, rule) local requests = gen_dkim_queries(task, rule) - for dom,res in ipairs(requests) do - rule.backend.set_token(task, rule, dom, token) + for dom,res in pairs(requests) do + -- tld + "." + check_result, e.g. example.com.+ - reputation for valid sigs + local query = string.format('%s.%s', dom, res) + rule.backend.set_token(task, rule, query, token) end end end +local function dkim_reputation_postfilter(task, rule) + local sym_accepted = task:get_symbol('R_DKIM_ALLOW') + local accept_adjustment = task:get_mempool():get_variable("dkim_reputation_accept") + + if sym_accepted and accept_adjustment then + local final_adjustment = rule.cfg.max_accept_adjustment * + rspamd_util.tanh(tonumber(accept_adjustment)) + task:adjust_result('R_DKIM_ALLOW', sym_accepted.score * final_adjustment) + end + + local sym_rejected = task:get_symbol('R_DKIM_REJECT') + local reject_adjustment = task:get_mempool():get_variable("dkim_reputation_reject") + + if sym_rejected and reject_adjustment then + local final_adjustment = rule.cfg.max_reject_adjustment * + rspamd_util.tanh(tonumber(reject_adjustment)) + task:adjust_result('R_DKIM_REJECT', sym_rejected.score * final_adjustment) + end +end + local dkim_selector = { config = { -- keys map between actions and hash elements in bucket, @@ -160,12 +196,14 @@ local dkim_selector = { lower_bound = 10, -- minimum number of messages to be scored min_score = nil, max_score = nil, - max_urls = 10, outbound = true, inbound = true, + max_accept_adjustment = 2.0, -- How to adjust accepted DKIM score + max_reject_adjustment = 3.0 -- How to adjust rejected DKIM score }, dependencies = {"DKIM_TRACE"}, filter = dkim_reputation_filter, -- used to get scores + postfilter = dkim_reputation_postfilter, -- used to adjust DKIM scores idempotent = dkim_reputation_idempotent -- used to set scores } @@ -506,6 +544,7 @@ local ip_selector = { local selectors = { ip = ip_selector, url = url_selector, + dkim = dkim_selector } local function reputation_dns_init(rule)