From: Vsevolod Stakhov Date: Wed, 3 Dec 2025 13:16:01 +0000 (+0000) Subject: [Feature] Add --recheck-rua option to dmarc_report for RUA filtering at send time X-Git-Tag: 3.14.2~39^2 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=5f9379aafbbd75b0fce6f3764eb258c5b9e59bfa;p=thirdparty%2Frspamd.git [Feature] Add --recheck-rua option to dmarc_report for RUA filtering at send time - Add -r/--recheck-rua flag to rspamadm dmarc_report to re-check RUA addresses against exclude_rua_addresses map before sending reports - Extend lua_maps to support rspamadm context for external map queries, enabling coroutine-based synchronous HTTP requests - Works with both local maps and external (HTTP) maps - Checks both full email address and domain-only against the map Addresses #5750 Related to #5735 --- diff --git a/lualib/lua_maps.lua b/lualib/lua_maps.lua index b112bbda0f..e0c57b8804 100644 --- a/lualib/lua_maps.lua +++ b/lualib/lua_maps.lua @@ -143,10 +143,14 @@ local function handle_cdb_map(map_config, key, callback, task) return result end -local function query_external_map(map_config, upstreams, key, callback, task) +-- Query external map using HTTP or CDB +-- task_or_ctx can be either a task object or a context table with: +-- { config, ev_base, session, resolver } for rspamadm usage +-- If callback is nil and task_or_ctx is a context table (rspamadm), performs synchronous request +local function query_external_map(map_config, upstreams, key, callback, task_or_ctx) -- Check if this is a CDB map if map_config.cdb then - return handle_cdb_map(map_config, key, callback, task) + return handle_cdb_map(map_config, key, callback, task_or_ctx) end -- Fallback to HTTP local http_method = (map_config.method == 'body' or map_config.method == 'form') and 'POST' or 'GET' @@ -157,6 +161,12 @@ local function query_external_map(map_config, upstreams, key, callback, task) local http_body = nil local url = map_config.backend + -- Determine logging target (task or config) + local log_obj = task_or_ctx + if type(task_or_ctx) == 'table' and task_or_ctx.config then + log_obj = task_or_ctx.config + end + if type(key) == 'string' or type(key) == 'userdata' then if map_config.method == 'body' then http_body = key @@ -178,10 +188,13 @@ local function query_external_map(map_config, upstreams, key, callback, task) http_headers['Content-Type'] = 'application/msgpack' else local caller = debug.getinfo(2) or {} - rspamd_logger.errx(task, + rspamd_logger.errx(log_obj, "requested external map key with a wrong combination body method and missing encode; caller: %s:%s", caller.short_src, caller.currentline) - callback(false, 'invalid map usage', 500, task) + if callback then + callback(false, 'invalid map usage', 500, task_or_ctx) + end + return nil end else -- query/header and no encode @@ -198,29 +211,20 @@ local function query_external_map(map_config, upstreams, key, callback, task) http_headers = key else local caller = debug.getinfo(2) or {} - rspamd_logger.errx(task, + rspamd_logger.errx(log_obj, "requested external map key with a wrong combination of encode and input; caller: %s:%s", caller.short_src, caller.currentline) - callback(false, 'invalid map usage', 500, task) - return + if callback then + callback(false, 'invalid map usage', 500, task_or_ctx) + end + return nil end end end - local function map_callback(err, code, body, _) - if err then - callback(false, err, code, task) - elseif code == 200 then - callback(true, body, 200, task) - else - callback(false, err, code, task) - end - end - - local ret = rspamd_http.request { - task = task, + -- Build HTTP request options - support both task and rspamadm context + local http_opts = { url = url, - callback = map_callback, timeout = map_config.timeout or 1.0, keepalive = true, upstream = upstream, @@ -229,8 +233,49 @@ local function query_external_map(map_config, upstreams, key, callback, task) body = http_body, } + -- Check if task_or_ctx is a context table (rspamadm) or a task (userdata) + -- rspamadm context is a Lua table with ev_base, config, session, resolver fields + -- task is userdata (C object), so type(task) ~= 'table' + local is_rspamadm_ctx = type(task_or_ctx) == 'table' and task_or_ctx.ev_base and task_or_ctx.config + if is_rspamadm_ctx then + -- rspamadm context + http_opts.config = task_or_ctx.config + http_opts.ev_base = task_or_ctx.ev_base + http_opts.session = task_or_ctx.session + http_opts.resolver = task_or_ctx.resolver + elseif task_or_ctx then + -- Regular task (userdata) + http_opts.task = task_or_ctx + end + + -- If no callback and rspamadm context, use coroutine-based synchronous mode + if not callback and is_rspamadm_ctx then + local err, response = rspamd_http.request(http_opts) + if err then + return nil + elseif response and response.code == 200 then + return response.content + else + return nil + end + end + + -- Async mode with callback + local function map_callback(err, code, body, _) + if err then + callback(false, err, code, task_or_ctx) + elseif code == 200 then + callback(true, body, 200, task_or_ctx) + else + callback(false, err, code, task_or_ctx) + end + end + + http_opts.callback = map_callback + local ret = rspamd_http.request(http_opts) + if not ret then - callback(false, 'http request error', 500, task) + callback(false, 'http request error', 500, task_or_ctx) end end @@ -246,24 +291,33 @@ end --]] local function rspamd_map_add_from_ucl(opt, mtype, description, callback) local ret = { - get_key = function(t, k, key_callback, task) + -- get_key supports both task (userdata) and rspamadm context (table with ev_base, config, session, resolver) + -- For external maps with rspamadm context (no callback), uses coroutine-based synchronous request + get_key = function(t, k, key_callback, task_or_ctx) if t.__data then local cb = key_callback or callback if t.__external then - if not cb or not task then + -- Check if this is rspamadm context with no callback - use sync mode + -- rspamadm context is a Lua table; task is userdata (C object) + local is_rspamadm_ctx = type(task_or_ctx) == 'table' and task_or_ctx.ev_base and task_or_ctx.config + if not cb and is_rspamadm_ctx then + -- Coroutine-based synchronous external map query for rspamadm + return query_external_map(t.__data, t.__upstreams, k, nil, task_or_ctx) + elseif not cb or not task_or_ctx then local caller = debug.getinfo(2) or {} - rspamd_logger.errx(rspamd_config, "requested external map key without callback or task; caller: %s:%s", + rspamd_logger.errx(rspamd_config, "requested external map key without callback or task/context; caller: %s:%s", caller.short_src, caller.currentline) return nil + else + query_external_map(t.__data, t.__upstreams, k, cb, task_or_ctx) end - query_external_map(t.__data, t.__upstreams, k, cb, task) else local result = t.__data:get_key(k) if cb then if result then - cb(true, result, 200, task) + cb(true, result, 200, task_or_ctx) else - cb(false, 'not found', 404, task) + cb(false, 'not found', 404, task_or_ctx) end else return result @@ -281,9 +335,9 @@ local function rspamd_map_add_from_ucl(opt, mtype, description, callback) end } local ret_mt = { - __index = function(t, k, key_callback, task) + __index = function(t, k, key_callback, task_or_ctx) if t.__data then - return t.get_key(k, key_callback, task) + return t.get_key(k, key_callback, task_or_ctx) end return nil diff --git a/lualib/rspamadm/dmarc_report.lua b/lualib/rspamadm/dmarc_report.lua index ef4c20d451..f832ca8807 100644 --- a/lualib/rspamadm/dmarc_report.lua +++ b/lualib/rspamadm/dmarc_report.lua @@ -18,6 +18,7 @@ local argparse = require "argparse" local lua_util = require "lua_util" local logger = require "rspamd_logger" local lua_redis = require "lua_redis" +local lua_maps = require "lua_maps" local dmarc_common = require "plugins/dmarc" local rspamd_mempool = require "rspamd_mempool" local rspamd_url = require "rspamd_url" @@ -55,6 +56,9 @@ parser:option "-b --batch-size" :convert(tonumber) :default "10" +parser:flag "-r --recheck-rua" + :description "Re-check RUA addresses against exclude_rua_addresses map before sending" + local report_template = [[From: "{= from_name =}" <{= from_addr =}> To: {= rcpt =} {%+ if is_string(bcc) %}Bcc: {= bcc =}{%- endif %} @@ -105,6 +109,14 @@ local redis_attrs = { local redis_attrs_write = lua_util.shallowcopy(redis_attrs) redis_attrs_write['is_write'] = true local pool +local exclude_rua_map +-- Context for external map queries (used instead of task in rspamadm) +local map_context = { + config = rspamd_config, + ev_base = rspamadm_ev_base, + session = rspamadm_session, + resolver = rspamadm_dns_resolver, +} local function load_config(opts) local _r, err = rspamd_config:load_ucl(opts['config']) @@ -503,6 +515,43 @@ local function prepare_report(opts, start_time, end_time, rep_key) return nil end + -- Re-check RUA addresses against exclude_rua_addresses map if enabled + -- Works with both local maps and external maps (HTTP) using synchronous requests + if exclude_rua_map and dmarc_record.rua then + local filtered_rua = {} + local excluded_count = 0 + for _, rua_elt in ipairs(dmarc_record.rua) do + local rua_email = string.format('%s@%s', rua_elt:get_user(), rua_elt:get_host()) + -- For external maps, pass map_context to enable synchronous HTTP requests + local excluded = exclude_rua_map:get_key(rua_email, nil, map_context) + if not excluded then + -- Also check just the domain part + excluded = exclude_rua_map:get_key(rua_elt:get_host(), nil, map_context) + end + if excluded then + lua_util.debugm(N, 'RUA address %s for domain %s is excluded by map (re-check)', + rua_email, reporting_domain) + excluded_count = excluded_count + 1 + else + table.insert(filtered_rua, rua_elt) + end + end + + if #filtered_rua == 0 then + if not opts.no_opt then + lua_redis.request(redis_params, redis_attrs_write, + { 'DEL', rep_key }) + end + logger.messagex('All RUA addresses for domain %s are excluded by map (re-check), skipping report', + reporting_domain) + return nil + elseif excluded_count > 0 then + logger.messagex('Filtered %s RUA addresses for domain %s, %s remaining', + excluded_count, reporting_domain, #filtered_rua) + dmarc_record.rua = filtered_rua + end + end + -- Get all reports for a domain ret, results = lua_redis.request(redis_params, redis_attrs, { 'ZRANGE', rep_key, '0', '-1', 'WITHSCORES' }) @@ -692,6 +741,25 @@ local function handler(args) os.exit(1) end + -- Load exclude_rua_addresses map if --recheck-rua flag is set + if opts.recheck_rua then + if dmarc_settings.reporting.exclude_rua_addresses then + exclude_rua_map = lua_maps.map_add_from_ucl(dmarc_settings.reporting.exclude_rua_addresses, + 'set', 'DMARC RUA exclusion map for report sending') + if exclude_rua_map then + if exclude_rua_map.__external then + logger.messagex('Loaded exclude_rua_addresses external map for RUA re-checking') + else + logger.messagex('Loaded exclude_rua_addresses map for RUA re-checking') + end + else + logger.warnx('Failed to load exclude_rua_addresses map, RUA re-checking disabled') + end + else + logger.warnx('--recheck-rua specified but no exclude_rua_addresses configured in dmarc settings') + end + end + for _, e in ipairs({ 'email', 'domain', 'org_name' }) do if not dmarc_settings.reporting[e] then logger.errx('Missing required setting: dmarc.reporting.%s', e)