From c45ab97a4ec2f3c83e20058ede9bf3b589a95f44 Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Fri, 24 Jul 2026 16:34:34 +0100 Subject: [PATCH] [Feature] fuzzy_check: anchor probability weight curve at the match threshold The score multiplier for non-exact matches was sqrt(prob): a concave curve anchored at zero. Since the storage never returns prob below the shingle match threshold (0.5), the whole reachable range collapsed into (~0.73 .. 1.0], so a marginal 17/32 shingle overlap scored almost as high as an exact match; combined with high-weight deny lists this turned weak matches into instant rejects. Replace it with ((prob - prob_bias) / (1 - prob_bias)) ^ prob_power, anchored at the match threshold (prob_bias 0.5). prob_power defaults to 1.0: the multiplier is the fraction of the way from the threshold to an exact match (19/32 -> 0.19, 24/32 -> 0.50, 28/32 -> 0.75); raise it per rule for harsher discounting of weak matches. The old zero-anchored sqrt curve is not preserved: it was simply broken. Applies to text and HTML hashes; small images keep their existing normalized curve. --- src/plugins/fuzzy_check.c | 74 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 71 insertions(+), 3 deletions(-) diff --git a/src/plugins/fuzzy_check.c b/src/plugins/fuzzy_check.c index 9175623b4a..0501a6d309 100644 --- a/src/plugins/fuzzy_check.c +++ b/src/plugins/fuzzy_check.c @@ -110,6 +110,8 @@ struct fuzzy_rule { struct rspamd_cryptobox_pubkey *write_peer_key; double hits_limit; double weight_threshold; + double prob_bias; /* Anchor of the probability weight curve (default 0.5, the shingle match threshold) */ + double prob_power; /* Exponent of the curve (default 1.0) */ double html_weight; /* Weight multiplier for HTML hashes (default 1.0) */ enum fuzzy_rule_mode mode; gboolean skip_unknown; @@ -532,6 +534,27 @@ fuzzy_normalize(int32_t in, double weight) #endif } +/* + * Convert the reply probability into a weight multiplier. + * + * The server never replies with prob below the shingle match threshold + * (0.5), so a curve anchored at zero (like the old sqrt(prob)) collapses + * the whole reachable range into (~0.73 .. 1.0] and barely depends on the + * match quality: a marginal 17/32 shingle overlap yields almost the same + * score as an exact match. Anchor at prob_bias instead: + * ((prob - bias) / (1 - bias)) ^ prob_power, which spreads match quality + * over the full 0..1 range. + */ +static double +fuzzy_weight_from_prob(struct fuzzy_rule *rule, double prob) +{ + double norm = (prob - rule->prob_bias) / (1.0 - rule->prob_bias); + + norm = MIN(1.0, MAX(0.0, norm)); + + return pow(norm, rule->prob_power); +} + static struct fuzzy_rule * fuzzy_rule_new(const char *default_symbol, rspamd_mempool_t *pool) { @@ -546,6 +569,8 @@ fuzzy_rule_new(const char *default_symbol, rspamd_mempool_t *pool) rule->mappings); rule->mode = fuzzy_rule_read_write; rule->weight_threshold = NAN; + rule->prob_bias = 0.5; + rule->prob_power = 1.0; rule->html_weight = 1.0; rule->html_shingles = FALSE; rule->text_hashes = TRUE; @@ -2233,6 +2258,28 @@ fuzzy_parse_rule(struct rspamd_config *cfg, const ucl_object_t *obj, rule->weight_threshold = ucl_object_todouble(value); } + if ((value = ucl_object_lookup(obj, "prob_power")) != NULL) { + rule->prob_power = ucl_object_todouble(value); + + if (!(rule->prob_power > 0)) { + msg_warn_config("prob_power must be positive in rule %s, " + "using the default 1.0", + rule->name); + rule->prob_power = 1.0; + } + } + + if ((value = ucl_object_lookup(obj, "prob_bias")) != NULL) { + rule->prob_bias = ucl_object_todouble(value); + + if (rule->prob_bias < 0 || rule->prob_bias >= 1.0) { + msg_warn_config("prob_bias must be in [0, 1) in rule %s, " + "using the default 0.5", + rule->name); + rule->prob_bias = 0.5; + } + } + if ((value = ucl_object_lookup(obj, "text_hashes")) != NULL) { rule->text_hashes = ucl_obj_toboolean(value); } @@ -2504,6 +2551,27 @@ int fuzzy_check_module_init(struct rspamd_config *cfg, struct module_ctx **ctx) 0, NULL, 0); + rspamd_rcl_add_doc_by_path(cfg, + "fuzzy_check.rule", + "Exponent of the probability weight curve: score multiplier is " + "((prob - prob_bias) / (1 - prob_bias)) ^ prob_power, so weak shingle " + "matches score much lower than exact ones (default: 1.0)", + "prob_power", + UCL_FLOAT, + NULL, + 0, + NULL, + 0); + rspamd_rcl_add_doc_by_path(cfg, + "fuzzy_check.rule", + "Anchor of the probability weight curve, normally the shingle match " + "threshold (default: 0.5)", + "prob_bias", + UCL_FLOAT, + NULL, + 0, + NULL, + 0); rspamd_rcl_add_doc_by_path(cfg, "fuzzy_check.rule", "List of servers to check (or learn)", @@ -4525,7 +4593,7 @@ fuzzy_insert_result(struct fuzzy_client_session *session, } else if ((io->flags & FUZZY_CMD_FLAG_HTML_DOMAINS)) { /* HTML domain-sensitive hash (structure + domains) */ - nval *= sqrtf(rep->v1.prob); + nval *= fuzzy_weight_from_prob(session->rule, rep->v1.prob); nval *= session->rule->html_weight; type = "htmld"; @@ -4533,7 +4601,7 @@ fuzzy_insert_result(struct fuzzy_client_session *session, } else if ((io->flags & FUZZY_CMD_FLAG_HTML)) { /* HTML structural hash (template mode, domains ignored) */ - nval *= sqrtf(rep->v1.prob); + nval *= fuzzy_weight_from_prob(session->rule, rep->v1.prob); /* Apply HTML weight multiplier from rule config */ nval *= session->rule->html_weight; @@ -4542,7 +4610,7 @@ fuzzy_insert_result(struct fuzzy_client_session *session, } else { /* Calc real probability */ - nval *= sqrtf(rep->v1.prob); + nval *= fuzzy_weight_from_prob(session->rule, rep->v1.prob); if (cmd->shingles_count > 0) { type = "txt"; -- 2.47.3