From: Vsevolod Stakhov <vsevolod@rspamd.com>
Date: Mon, 9 Dec 2024 15:25:26 +0000 (+0000)
Subject: [Minor] Make urls and emails unique
X-Git-Tag: 3.11.0~15^2~1
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=717cfba84f7a80e0d164a03a2bf45b06b2f07075;p=thirdparty%2Frspamd.git

[Minor] Make urls and emails unique
---

diff --git a/lualib/lua_mime.lua b/lualib/lua_mime.lua
index d6a8a70bf6..24d02c304c 100644
--- a/lualib/lua_mime.lua
+++ b/lualib/lua_mime.lua
@@ -967,6 +967,7 @@ end
 --]]
 exports.anonymize_message = function(task, settings)
   local rspamd_re = require "rspamd_regexp"
+  local lua_util = require "lua_util"
   -- We exclude words with digits, currency symbols and so on
   local exclude_words_re = rspamd_re.create_cached([[/^(?:\d+|\d+\D{1,3}|\p{Sc}.*|(\+?\d{1,3}[\s\-]?)?)$/u]])
   local newline_s = newline(task)
@@ -1106,7 +1107,7 @@ exports.anonymize_message = function(task, settings)
   end
 
   for _, url in ipairs(task:get_urls(true)) do
-    table.insert(urls, process_url(url))
+    urls[process_url(url)] = true
   end
 
   -- Process emails
@@ -1115,14 +1116,14 @@ exports.anonymize_message = function(task, settings)
   end
 
   for _, email in ipairs(task:get_emails()) do
-    table.insert(emails, process_email(email))
+    emails[process_email(email)] = true
   end
 
   -- Construct new message
   table.insert(text_content, '\nurls: ')
-  table.insert(text_content, table.concat(urls, ', '))
+  table.insert(text_content, table.concat(lua_util.keys(urls), ', '))
   table.insert(text_content, '\nemails: ')
-  table.insert(text_content, table.concat(emails, ', '))
+  table.insert(text_content, table.concat(lua_util.keys(emails), ', '))
   local new_text = table.concat(text_content, ' ')
 
   -- Create new message structure