From: Vsevolod Stakhov Date: Thu, 23 Jul 2026 14:15:06 +0000 (+0100) Subject: [Feature] lua_feedback_parsers: enrich sparse ARF reports X-Git-Tag: 4.1.3~25^2 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=refs%2Fpull%2F6148%2Fhead;p=thirdparty%2Frspamd.git [Feature] lua_feedback_parsers: enrich sparse ARF reports Microsoft's JMRP feedback loop ships a well-formed multipart/report;report-type=feedback-report whose message/feedback-report block only carries Feedback-Type/User-Agent/Version. Source-IP, Arrival-Date, Original-Mail-From and Original-Rcpt-To are omitted; that data lives only in the embedded original-message headers. parse_arf now recovers those fields from the original headers, filling only what the report itself left empty and recording provenance in result.derived so callers can tell reported data from inferred data: * source_ip: X-Originating-IP, then client-ip= of Received-SPF / Authentication-Results, then the first public IP while walking the Received chain (is_nonpublic_ip skips RFC1918/CGNAT/ULA hops, which ip:is_local() does not). * arrival_date: topmost Received timestamp, else Date. * original_mail_from: Return-Path, else smtp.mailfrom. * original_rcpt_to: Delivered-To / X-Delivered-To / envelope-to. * reported_domain: falls back to the envelope-from domain. Also accept Received-Date as an Arrival-Date alias in the report block. extract_original_message now returns the raw header maps alongside the parsed subset; DSN parsing is unchanged. Adds unit tests for the Received-SPF path and the public-IP-in-Received-chain fallback. --- diff --git a/lualib/lua_feedback_parsers.lua b/lualib/lua_feedback_parsers.lua index 6e24fc29a9..9c0c0d949f 100644 --- a/lualib/lua_feedback_parsers.lua +++ b/lualib/lua_feedback_parsers.lua @@ -202,6 +202,11 @@ end -- Extract the standard subset of original-message headers we care about from -- the content of a message/rfc822 (or text/rfc822-headers) sub-part. +-- +-- Returns (out, headers, headers_multi) where `out` is the parsed subset and +-- `headers`/`headers_multi` are the raw single/multi field maps. Callers may +-- use the raw maps to enrich a sparse report (see the JMRP handling in +-- parse_arf) without those raw maps leaking into the returned structure. local function extract_original_message(part) local content = part:get_content() if not content then @@ -215,7 +220,7 @@ local function extract_original_message(part) if not lines then return nil end - local headers = parse_field_block(lines, 1) + local headers, headers_multi = parse_field_block(lines, 1) if not headers or next(headers) == nil then return nil end @@ -229,7 +234,203 @@ local function extract_original_message(part) if not (out.message_id or out.from or out.to or out.subject or out.date) then return nil end - return out + return out, headers, headers_multi +end + +-- ---------------------------------------------------------------------------- +-- Enrichment helpers for sparse feedback reports +-- +-- Some providers (most notably Microsoft's JMRP, but also a few consumer +-- FBLs) ship a `message/feedback-report` block that only carries +-- `Feedback-Type`, `User-Agent` and `Version`, omitting the useful metadata +-- (Source-IP, Arrival-Date, Original-Mail-From, Original-Rcpt-To). All of +-- that data still lives in the embedded original-message headers, so we +-- recover it from there when the report itself is silent. +-- ---------------------------------------------------------------------------- + +-- Validate a candidate IP string, tolerating surrounding [], (), an `IPv6:` +-- tag and a trailing `:port` on IPv4. Returns (canonical_string, ip_object) +-- or nil. +local function validate_ip(candidate) + if not candidate then + return nil + end + local rspamd_ip = require 'rspamd_ip' + local s = str_trim(candidate) + s = s:gsub('^[%[%(]', ''):gsub('[%]%)]$', '') + s = s:gsub('^[Ii][Pp][Vv]6:', '') + -- Strip a trailing :port for a bare IPv4 literal (a.b.c.d:NNN) + local v4 = s:match('^(%d+%.%d+%.%d+%.%d+)') + if v4 then + s = v4 + end + local ip = rspamd_ip.from_string(s) + if ip and ip:is_valid() then + return ip:to_string(), ip + end + return nil +end + +-- Classify an IP as non-routable (loopback, RFC1918/ULA private, CGNAT or +-- link-local). `ip:is_local()` only covers loopback and IPv6 link/site-local, +-- so we extend it to the private ranges that show up as internal Received +-- hops. Used to prefer a public sender IP when walking a Received chain. +local function is_nonpublic_ip(ipobj) + if ipobj:is_local() then + return true + end + local ver = ipobj:get_version() + if ver == 4 then + local o1, o2 = ipobj:to_string():match('^(%d+)%.(%d+)%.') + o1, o2 = tonumber(o1), tonumber(o2) + if not o1 then + return false + end + if o1 == 10 or o1 == 127 or o1 == 0 then + return true + end + if o1 == 192 and o2 == 168 then + return true + end + if o1 == 172 and o2 >= 16 and o2 <= 31 then + return true + end + if o1 == 169 and o2 == 254 then + return true + end + if o1 == 100 and o2 >= 64 and o2 <= 127 then + return true + end + return false + elseif ver == 6 then + local s = ipobj:to_string():lower() + -- fc00::/7 (ULA) or fe80::/10 (link-local) + if s:match('^f[cd]') or s:match('^fe[89ab]') then + return true + end + return false + end + return false +end + +-- Derive the sender (source) IP from the original-message headers. +-- Priority: X-Originating-IP, then the client-ip of Received-SPF / +-- Authentication-Results, then the first *public* bracketed IP found while +-- walking the Received chain top-to-bottom (falling back to the first valid +-- IP if every hop is private). Returns (ip_string, provenance) or nil. +local function derive_source_ip(headers, headers_multi) + local xoip = headers['x-originating-ip'] + if xoip then + local ip = validate_ip(xoip) + if ip then + return ip, 'x-originating-ip' + end + end + + local spf = headers['received-spf'] + if spf then + local c = spf:match('[Cc]lient%-[Ii][Pp]=([^;%s]+)') + local ip = validate_ip(c) + if ip then + return ip, 'received-spf' + end + end + + local ar = headers['authentication-results'] + if ar then + local c = ar:match('[Cc]lient%-[Ii][Pp]=([^;%s]+)') or + ar:match('smtp%.remote%-ip=([^;%s]+)') or + ar:match('[Ss]ender IP is ([%x%.:]+)') + local ip = validate_ip(c) + if ip then + return ip, 'authentication-results' + end + end + + local recv = headers_multi and headers_multi['received'] + if recv then + local first_valid + for _, hdr in ipairs(recv) do + for cand in hdr:gmatch('%[([^%]]+)%]') do + local ipstr, ipobj = validate_ip(cand) + if ipstr then + if not is_nonpublic_ip(ipobj) then + return ipstr, 'received' + end + first_valid = first_valid or ipstr + end + end + for cand in hdr:gmatch('%(([^%)]+)%)') do + local ipstr, ipobj = validate_ip(cand) + if ipstr then + if not is_nonpublic_ip(ipobj) then + return ipstr, 'received' + end + first_valid = first_valid or ipstr + end + end + end + if first_valid then + return first_valid, 'received' + end + end + + return nil +end + +-- Derive the arrival (receive) date from the original-message headers: the +-- timestamp of the topmost Received header if present, otherwise the Date +-- header. Returns (date_string, provenance) or nil. +local function derive_arrival_date(headers, headers_multi) + local recv = headers_multi and headers_multi['received'] + if recv and recv[1] then + -- The date is the clause following the last ';' of a Received header. + local d = recv[1]:match('.*;%s*(.+)$') + if d then + d = str_trim(d) + if d ~= '' then + return d, 'received' + end + end + end + if headers['date'] and headers['date'] ~= '' then + return headers['date'], 'date' + end + return nil +end + +-- Derive the envelope sender from the original-message headers: Return-Path +-- is authoritative in stored headers; otherwise fall back to the smtp.mailfrom +-- reported by Authentication-Results. Returns (addr, provenance) or nil. +local function derive_mail_from(headers) + local rp = strip_angles(headers['return-path']) + if rp and rp ~= '' then + return rp, 'return-path' + end + local ar = headers['authentication-results'] + if ar then + local mf = ar:match('smtp%.mailfrom=([^;%s]+)') + if mf and mf ~= '' then + return mf, 'authentication-results' + end + end + return nil +end + +-- Derive the envelope recipient from the original-message headers, trying the +-- usual delivery-agent headers in turn. Returns (addr, provenance) or nil. +local function derive_rcpt_to(headers) + local candidates = { + 'delivered-to', 'x-delivered-to', 'envelope-to', + 'x-envelope-to', 'x-original-to', + } + for _, h in ipairs(candidates) do + local v = strip_angles(headers[h]) + if v and v ~= '' then + return v, h + end + end + return nil end -- ---------------------------------------------------------------------------- @@ -329,6 +530,18 @@ end -- the feedback-report body is unparseable, a table is still returned (with -- mostly-nil fields and `reported_uri = {}`). -- +-- Sparse-report enrichment: some providers (notably Microsoft's JMRP) emit a +-- feedback-report block containing only `Feedback-Type`, `User-Agent` and +-- `Version`, leaving `Source-IP`, `Arrival-Date`, `Original-Mail-From` and +-- `Original-Rcpt-To` empty and shipping the useful data only as the embedded +-- original-message headers. For any of those four fields that the report +-- itself did not provide, this function recovers a value from the original +-- headers (Received-SPF / Received / X-Originating-IP for the IP; the topmost +-- Received or Date for the arrival date; Return-Path / Delivered-To for the +-- envelope addresses). When one or more fields are recovered this way, a +-- `derived` sub-table maps each recovered field name to the header it came +-- from, so callers can distinguish reported values from inferred ones. +-- -- @param {rspamd_task} task message to inspect -- @return {table|nil} parsed ARF, see module doc for the shape --]] @@ -373,7 +586,7 @@ function exports.parse_arf(task) result.user_agent = f['user-agent'] result.original_mail_from = strip_angles(f['original-mail-from']) result.original_rcpt_to = strip_angles(f['original-rcpt-to']) - result.arrival_date = f['arrival-date'] + result.arrival_date = f['arrival-date'] or f['received-date'] result.source_ip = f['source-ip'] result.reported_domain = f['reported-domain'] result.authentication_results = f['authentication-results'] @@ -398,7 +611,7 @@ function exports.parse_arf(task) local orig_part = find_original_message_part(task) if orig_part then - local om = extract_original_message(orig_part) + local om, oheaders, oheaders_multi = extract_original_message(orig_part) if om then -- RFC 5965 consumers typically only care about Message-ID and From. result.original_message = { @@ -406,6 +619,60 @@ function exports.parse_arf(task) from = om.from, } end + + -- Enrich sparse reports (e.g. Microsoft JMRP) from the embedded original + -- headers. We only fill fields the report itself left empty, and we record + -- where each value came from in `result.derived` so consumers can tell + -- reported data from inferred data. + if oheaders then + local derived = {} + + if not result.source_ip then + local ip, from = derive_source_ip(oheaders, oheaders_multi) + if ip then + result.source_ip = ip + derived.source_ip = from + end + end + + if not result.arrival_date then + local d, from = derive_arrival_date(oheaders, oheaders_multi) + if d then + result.arrival_date = d + derived.arrival_date = from + end + end + + if not result.original_mail_from then + local mf, from = derive_mail_from(oheaders) + if mf then + result.original_mail_from = mf + derived.original_mail_from = from + end + end + + if not result.original_rcpt_to then + local rt, from = derive_rcpt_to(oheaders) + if rt then + result.original_rcpt_to = rt + derived.original_rcpt_to = from + end + end + + -- If the reported domain is missing, fall back to the domain of the + -- (possibly derived) envelope sender. + if not result.reported_domain and result.original_mail_from then + local dom = result.original_mail_from:match('@([^@%s>]+)%s*$') + if dom then + result.reported_domain = dom:lower() + derived.reported_domain = 'original_mail_from' + end + end + + if next(derived) then + result.derived = derived + end + end end return result diff --git a/test/lua/unit/lua_feedback_parsers.lua b/test/lua/unit/lua_feedback_parsers.lua index 88f441e17b..2f90bb63c5 100644 --- a/test/lua/unit/lua_feedback_parsers.lua +++ b/test/lua/unit/lua_feedback_parsers.lua @@ -243,6 +243,112 @@ context("Lua feedback parsers - DSN/ARF on synthetic tasks", function() task:destroy() end) + test("parse_arf enriches a sparse Microsoft JMRP report from original headers", function() + -- JMRP ships a feedback-report block with only Feedback-Type/User-Agent/ + -- Version and puts the useful data in a text/rfc822-headers part. + local message = "Return-Path: \r\n" .. + "From: staff@hotmail.com\r\n" .. + "To: fbl@example.org\r\n" .. + "Subject: complaint\r\n" .. + "Date: Thu, 23 Jul 2026 10:00:00 +0000\r\n" .. + "Message-ID: \r\n" .. + "MIME-Version: 1.0\r\n" .. + "Content-Type: multipart/report; report-type=feedback-report; boundary=\"bnd2\"\r\n" .. + "\r\n" .. + "--bnd2\r\n" .. + "Content-Type: text/plain; charset=us-ascii\r\n" .. + "\r\n" .. + "This is an abuse report from JMRP.\r\n" .. + "\r\n" .. + "--bnd2\r\n" .. + "Content-Type: message/feedback-report\r\n" .. + "\r\n" .. + "Feedback-Type: abuse\r\n" .. + "User-Agent: JMRP/1.0\r\n" .. + "Version: 1\r\n" .. + "\r\n" .. + "--bnd2\r\n" .. + "Content-Type: text/rfc822-headers\r\n" .. + "\r\n" .. + "Return-Path: \r\n" .. + "Delivered-To: victim@hotmail.com\r\n" .. + "Received: from mail.sender.example (mail.sender.example [203.0.113.9])\r\n" .. + " by mx.hotmail.com with ESMTPS id abc123;\r\n" .. + " Thu, 23 Jul 2026 09:58:12 +0000\r\n" .. + "Received-SPF: Pass (protection.outlook.com: domain of sender.example\r\n" .. + " designates 203.0.113.9 as permitted sender) client-ip=203.0.113.9;\r\n" .. + " helo=mail.sender.example;\r\n" .. + "From: newsletter@sender.example\r\n" .. + "To: victim@hotmail.com\r\n" .. + "Subject: Special offer\r\n" .. + "Date: Thu, 23 Jul 2026 09:58:00 +0000\r\n" .. + "Message-ID: \r\n" .. + "\r\n" .. + "--bnd2--\r\n" + + local task = load_task(message) + assert_not_nil(task, "failed to load JMRP message") + local arf = lua_feedback_parsers.parse_arf(task) + assert_not_nil(arf) + assert_equal("abuse", arf.feedback_type) + assert_equal("JMRP/1.0", arf.user_agent) + -- Source-IP recovered from Received-SPF client-ip + assert_equal("203.0.113.9", arf.source_ip) + assert_equal("received-spf", arf.derived.source_ip) + -- Arrival-Date recovered from the topmost Received header timestamp + assert_equal("Thu, 23 Jul 2026 09:58:12 +0000", arf.arrival_date) + assert_equal("received", arf.derived.arrival_date) + -- Envelope addresses recovered from Return-Path / Delivered-To + assert_equal("bounce@sender.example", arf.original_mail_from) + assert_equal("return-path", arf.derived.original_mail_from) + assert_equal("victim@hotmail.com", arf.original_rcpt_to) + assert_equal("delivered-to", arf.derived.original_rcpt_to) + -- Reported domain falls back to the envelope-from domain + assert_equal("sender.example", arf.reported_domain) + assert_not_nil(arf.original_message) + assert_equal("orig-jmrp-001@sender.example", arf.original_message.message_id) + task:destroy() + end) + + test("parse_arf source-ip fallback prefers a public IP in the Received chain", function() + -- No Received-SPF or X-Originating-IP: the parser must walk the Received + -- chain and skip the internal hop in favour of the public sender IP. + local message = "From: staff@hotmail.com\r\n" .. + "To: fbl@example.org\r\n" .. + "Subject: complaint\r\n" .. + "MIME-Version: 1.0\r\n" .. + "Content-Type: multipart/report; report-type=feedback-report; boundary=\"bnd3\"\r\n" .. + "\r\n" .. + "--bnd3\r\n" .. + "Content-Type: message/feedback-report\r\n" .. + "\r\n" .. + "Feedback-Type: abuse\r\n" .. + "User-Agent: JMRP/1.0\r\n" .. + "Version: 1\r\n" .. + "\r\n" .. + "--bnd3\r\n" .. + "Content-Type: text/rfc822-headers\r\n" .. + "\r\n" .. + "Received: from internal.hotmail.com (internal.hotmail.com [10.0.0.5])\r\n" .. + " by store.hotmail.com; Thu, 23 Jul 2026 10:01:00 +0000\r\n" .. + "Received: from mail.sender.example (mail.sender.example [198.51.100.7])\r\n" .. + " by mx.hotmail.com; Thu, 23 Jul 2026 10:00:30 +0000\r\n" .. + "From: newsletter@sender.example\r\n" .. + "Message-ID: \r\n" .. + "\r\n" .. + "--bnd3--\r\n" + + local task = load_task(message) + assert_not_nil(task, "failed to load JMRP message") + local arf = lua_feedback_parsers.parse_arf(task) + assert_not_nil(arf) + assert_equal("198.51.100.7", arf.source_ip) + assert_equal("received", arf.derived.source_ip) + -- Arrival date is the topmost (most recent) Received hop + assert_equal("Thu, 23 Jul 2026 10:01:00 +0000", arf.arrival_date) + task:destroy() + end) + test("parse_arf returns nil when report-type is not feedback-report", function() local message = "From: a@example.com\r\n" .. "To: b@example.com\r\n" ..