[Rework] Multimap should use only distinct text parts for content matching

Issue: #5248

diff --git a/lualib/lua_mime.lua b/lualib/lua_mime.lua
index 1135f2b63a9f677796a4eb102b58902c27328e6f..f68758ec9ea44f679ee50d84d1458f617074ac01 100644 (file)
--- a/lualib/lua_mime.lua
+++ b/lualib/lua_mime.lua
@@ -956,6 +956,49 @@ exports.get_displayed_text_part = function(task)
   return html_part or text_part
 end
 
+--[[[
+-- @function lua_mime.get_distinct_text_parts(task)
+-- Returns the list of parts that are visible or have a distinct content
+-- @param {task} task Rspamd task object
+-- @return array of {text_part} a selected part
+--]]
+exports.get_distinct_text_parts = function(task)
+  local text_parts = task:get_text_parts()
+  if not text_parts then
+    return {}
+  end
+
+  local text_part_idx
+
+  local distance = task:get_mempool():get_variable('parts_distance', 'double')
+  if not distance then
+    return text_parts
+  end
+  distance = tonumber(distance)
+
+  if distance > 0.5 then
+    -- Parts are distinct
+    return text_parts
+  end
+
+  -- First pass: categorize parts
+  for i, part in ipairs(text_parts) do
+    local mp = part:get_mimepart()
+    if not mp:is_attachment() then
+      if not part:is_html() then
+        -- Found text part that is similar to html part
+        text_part_idx = i
+      end
+    end
+  end
+
+  if text_part_idx then
+    table.remove(text_parts, text_part_idx)
+  end
+
+  return text_parts
+end
+
 --[[[
 -- @function lua_mime.anonymize_message(task, settings)
 -- Anonymizes message content by replacing sensitive data

diff --git a/src/plugins/lua/multimap.lua b/src/plugins/lua/multimap.lua
index e852ce15e0f372beb39d7a7f2e8a35c9b3a7d171..a61da606b8616dbb038f8545758f9dfc9ad14d0d 100644 (file)
--- a/src/plugins/lua/multimap.lua
+++ b/src/plugins/lua/multimap.lua
@@ -29,6 +29,7 @@ local rspamd_ip = require "rspamd_ip"
 local lua_util = require "lua_util"
 local lua_selectors = require "lua_selectors"
 local lua_maps = require "lua_maps"
+local lua_mime = require "lua_mime"
 local redis_params
 local fun = require "fun"
 local N = 'multimap'
@@ -453,19 +454,19 @@ local function apply_content_filter(task, filter)
     return { task:get_raw_headers() }
   elseif filter == 'text' then
     local ret = {}
-    for _, p in ipairs(task:get_text_parts()) do
+    for _, p in ipairs(lua_mime.get_distinct_text_parts(task)) do
       table.insert(ret, p:get_content())
     end
     return ret
   elseif filter == 'rawtext' then
     local ret = {}
-    for _, p in ipairs(task:get_text_parts()) do
+    for _, p in ipairs(lua_mime.get_distinct_text_parts(task)) do
       table.insert(ret, p:get_content('raw_parsed'))
     end
     return ret
   elseif filter == 'oneline' then
     local ret = {}
-    for _, p in ipairs(task:get_text_parts()) do
+    for _, p in ipairs(lua_mime.get_distinct_text_parts(task)) do
       table.insert(ret, p:get_content_oneline())
     end
     return ret