From: Vsevolod Stakhov Date: Thu, 2 Oct 2025 13:53:25 +0000 (+0100) Subject: [Feature] Improve LLM prompt and add sender frequency tracking X-Git-Tag: 3.13.2~11^2 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=refs%2Fpull%2F5647%2Fhead;p=thirdparty%2Frspamd.git [Feature] Improve LLM prompt and add sender frequency tracking * Update default prompt to reduce false positives on legitimate emails - Explicitly recognize verification emails as legitimate - Require MULTIPLE red flags for phishing classification - Add guidance on known/frequent senders * Add sender frequency detection in context - Classify senders as: new, occasional, known, frequent - Based on sender_counts from user context - Passed to LLM via context snippet * Prompt instructs LLM to reduce phishing score for known senders * Helps avoid false positives on transactional/verification emails --- diff --git a/lualib/llm_context.lua b/lualib/llm_context.lua index c01850b1a5..7973e9e1d2 100644 --- a/lualib/llm_context.lua +++ b/lualib/llm_context.lua @@ -336,12 +336,28 @@ local function join_list(arr) return table.concat(arr, ', ') end -local function format_context_prompt(ctx) +local function format_context_prompt(ctx, task) local bullets = to_bullets_recent(ctx.recent_messages or {}, 5) local top_senders = join_list(ctx.top_senders or {}) local flagged = join_list(ctx.flagged_phrases or {}) local spam_types = join_list(ctx.last_spam_labels or {}) + -- Check if current sender is known + local sender_frequency = 'new' + if task then + local from = ((task:get_from('smtp') or EMPTY)[1] or EMPTY)['addr'] + if from and ctx.sender_counts and ctx.sender_counts[from] then + local count = ctx.sender_counts[from] + if count >= 10 then + sender_frequency = 'frequent' + elseif count >= 3 then + sender_frequency = 'known' + else + sender_frequency = 'occasional' + end + end + end + local parts = {} table.insert(parts, 'User recent correspondence summary:') if bullets ~= '' then @@ -356,6 +372,7 @@ local function format_context_prompt(ctx) if spam_types ~= '' then table.insert(parts, string.format('Last detected spam types: %s', spam_types)) end + table.insert(parts, string.format('Current sender: %s', sender_frequency)) return table.concat(parts, '\n') end @@ -409,7 +426,7 @@ function M.fetch(task, redis_params, opts, callback, debug_module) lua_util.debugm(N, task, 'context warm-up OK: %s messages, generating snippet', tostring(msg_count)) - local prompt_snippet = format_context_prompt(ctx) + local prompt_snippet = format_context_prompt(ctx, task) callback(nil, ctx, prompt_snippet) end diff --git a/src/plugins/lua/gpt.lua b/src/plugins/lua/gpt.lua index 10526361f2..0a5f1fee8b 100644 --- a/src/plugins/lua/gpt.lua +++ b/src/plugins/lua/gpt.lua @@ -1239,19 +1239,43 @@ if opts then if not settings.prompt then if settings.extra_symbols then - settings.prompt = "Analyze this email strictly as a spam detector given the email message, subject, " .. - "FROM and url domains. Evaluate spam probability (0-1). " .. + settings.prompt = "Analyze this email as a spam detector. Evaluate spam probability (0-1).\n\n" .. + "LEGITIMATE patterns to recognize:\n" .. + "- Verification emails with time-limited codes are NORMAL and legitimate\n" .. + "- Transactional emails (receipts, confirmations, password resets) from services\n" .. + "- 'Verify email' or 'confirmation code' is NOT automatically phishing\n" .. + "- Emails from frequent/known senders (see context) are more trustworthy\n\n" .. + "Flag as SPAM/PHISHING only with MULTIPLE red flags:\n" .. + "- Urgent threats or fear tactics (account closure, legal action)\n" .. + "- Domain impersonation or suspicious lookalikes\n" .. + "- Requests for passwords, SSN, credit card numbers\n" .. + "- Mismatched URLs pointing to different domains than sender\n" .. + "- Poor grammar/spelling in supposedly professional emails\n\n" .. + "IMPORTANT: If sender is 'frequent' or 'known', reduce phishing probability " .. + "unless there are strong contradictory signals.\n\n" .. "Output ONLY 3 lines:\n" .. "1. Numeric score (0.00-1.00)\n" .. - "2. One-sentence reason citing whether it is spam, the strongest red flag, or why it is ham\n" .. - "3. Empty line or mention ONLY the primary concern category if found from the list: " .. + "2. One-sentence reason citing the strongest indicator\n" .. + "3. Primary category if applicable: " .. table.concat(lua_util.keys(categories_map), ', ') else - settings.prompt = "Analyze this email strictly as a spam detector given the email message, subject, " .. - "FROM and url domains. Evaluate spam probability (0-1). " .. + settings.prompt = "Analyze this email as a spam detector. Evaluate spam probability (0-1).\n\n" .. + "LEGITIMATE patterns to recognize:\n" .. + "- Verification emails with time-limited codes are NORMAL and legitimate\n" .. + "- Transactional emails (receipts, confirmations, password resets) from services\n" .. + "- 'Verify email' or 'confirmation code' is NOT automatically phishing\n" .. + "- Emails from frequent/known senders (see context) are more trustworthy\n\n" .. + "Flag as SPAM/PHISHING only with MULTIPLE red flags:\n" .. + "- Urgent threats or fear tactics (account closure, legal action)\n" .. + "- Domain impersonation or suspicious lookalikes\n" .. + "- Requests for passwords, SSN, credit card numbers\n" .. + "- Mismatched URLs pointing to different domains than sender\n" .. + "- Poor grammar/spelling in supposedly professional emails\n\n" .. + "IMPORTANT: If sender is 'frequent' or 'known', reduce phishing probability " .. + "unless there are strong contradictory signals.\n\n" .. "Output ONLY 2 lines:\n" .. "1. Numeric score (0.00-1.00)\n" .. - "2. One-sentence reason citing whether it is spam, the strongest red flag, or why it is ham\n" + "2. One-sentence reason citing the strongest indicator" end end