[Feature] fuzzy_check: anchor probability weight curve at the match threshold

The score multiplier for non-exact matches was sqrt(prob): a concave
curve anchored at zero. Since the storage never returns prob below the
shingle match threshold (0.5), the whole reachable range collapsed into
(~0.73 .. 1.0], so a marginal 17/32 shingle overlap scored almost as
high as an exact match; combined with high-weight deny lists this
turned weak matches into instant rejects.

Replace it with ((prob - prob_bias) / (1 - prob_bias)) ^ prob_power,
anchored at the match threshold (prob_bias 0.5). prob_power defaults to
1.0: the multiplier is the fraction of the way from the threshold to an
exact match (19/32 -> 0.19, 24/32 -> 0.50, 28/32 -> 0.75); raise it per
rule for harsher discounting of weak matches. The old zero-anchored
sqrt curve is not preserved: it was simply broken. Applies to text and
HTML hashes; small images keep their existing normalized curve.

diff --git a/src/plugins/fuzzy_check.c b/src/plugins/fuzzy_check.c
index 9175623b4a6c3dba253d21787ca83e4e8b0f4d33..0501a6d3099f8e9c1ae4b297a45e414ca37cacc4 100644 (file)
--- a/src/plugins/fuzzy_check.c
+++ b/src/plugins/fuzzy_check.c
@@ -110,6 +110,8 @@ struct fuzzy_rule {
        struct rspamd_cryptobox_pubkey *write_peer_key;
        double hits_limit;
        double weight_threshold;
+       double prob_bias;   /* Anchor of the probability weight curve (default 0.5, the shingle match threshold) */
+       double prob_power;  /* Exponent of the curve (default 1.0) */
        double html_weight; /* Weight multiplier for HTML hashes (default 1.0) */
        enum fuzzy_rule_mode mode;
        gboolean skip_unknown;
@@ -532,6 +534,27 @@ fuzzy_normalize(int32_t in, double weight)
 #endif
 }
 
+/*
+ * Convert the reply probability into a weight multiplier.
+ *
+ * The server never replies with prob below the shingle match threshold
+ * (0.5), so a curve anchored at zero (like the old sqrt(prob)) collapses
+ * the whole reachable range into (~0.73 .. 1.0] and barely depends on the
+ * match quality: a marginal 17/32 shingle overlap yields almost the same
+ * score as an exact match. Anchor at prob_bias instead:
+ * ((prob - bias) / (1 - bias)) ^ prob_power, which spreads match quality
+ * over the full 0..1 range.
+ */
+static double
+fuzzy_weight_from_prob(struct fuzzy_rule *rule, double prob)
+{
+       double norm = (prob - rule->prob_bias) / (1.0 - rule->prob_bias);
+
+       norm = MIN(1.0, MAX(0.0, norm));
+
+       return pow(norm, rule->prob_power);
+}
+
 static struct fuzzy_rule *
 fuzzy_rule_new(const char *default_symbol, rspamd_mempool_t *pool)
 {
@@ -546,6 +569,8 @@ fuzzy_rule_new(const char *default_symbol, rspamd_mempool_t *pool)
                                                                  rule->mappings);
        rule->mode = fuzzy_rule_read_write;
        rule->weight_threshold = NAN;
+       rule->prob_bias = 0.5;
+       rule->prob_power = 1.0;
        rule->html_weight = 1.0;
        rule->html_shingles = FALSE;
        rule->text_hashes = TRUE;
@@ -2233,6 +2258,28 @@ fuzzy_parse_rule(struct rspamd_config *cfg, const ucl_object_t *obj,
                rule->weight_threshold = ucl_object_todouble(value);
        }
 
+       if ((value = ucl_object_lookup(obj, "prob_power")) != NULL) {
+               rule->prob_power = ucl_object_todouble(value);
+
+               if (!(rule->prob_power > 0)) {
+                       msg_warn_config("prob_power must be positive in rule %s, "
+                                                       "using the default 1.0",
+                                                       rule->name);
+                       rule->prob_power = 1.0;
+               }
+       }
+
+       if ((value = ucl_object_lookup(obj, "prob_bias")) != NULL) {
+               rule->prob_bias = ucl_object_todouble(value);
+
+               if (rule->prob_bias < 0 || rule->prob_bias >= 1.0) {
+                       msg_warn_config("prob_bias must be in [0, 1) in rule %s, "
+                                                       "using the default 0.5",
+                                                       rule->name);
+                       rule->prob_bias = 0.5;
+               }
+       }
+
        if ((value = ucl_object_lookup(obj, "text_hashes")) != NULL) {
                rule->text_hashes = ucl_obj_toboolean(value);
        }
@@ -2504,6 +2551,27 @@ int fuzzy_check_module_init(struct rspamd_config *cfg, struct module_ctx **ctx)
                                                           0,
                                                           NULL,
                                                           0);
+       rspamd_rcl_add_doc_by_path(cfg,
+                                                          "fuzzy_check.rule",
+                                                          "Exponent of the probability weight curve: score multiplier is "
+                                                          "((prob - prob_bias) / (1 - prob_bias)) ^ prob_power, so weak shingle "
+                                                          "matches score much lower than exact ones (default: 1.0)",
+                                                          "prob_power",
+                                                          UCL_FLOAT,
+                                                          NULL,
+                                                          0,
+                                                          NULL,
+                                                          0);
+       rspamd_rcl_add_doc_by_path(cfg,
+                                                          "fuzzy_check.rule",
+                                                          "Anchor of the probability weight curve, normally the shingle match "
+                                                          "threshold (default: 0.5)",
+                                                          "prob_bias",
+                                                          UCL_FLOAT,
+                                                          NULL,
+                                                          0,
+                                                          NULL,
+                                                          0);
        rspamd_rcl_add_doc_by_path(cfg,
                                                           "fuzzy_check.rule",
                                                           "List of servers to check (or learn)",
@@ -4525,7 +4593,7 @@ fuzzy_insert_result(struct fuzzy_client_session *session,
                }
                else if ((io->flags & FUZZY_CMD_FLAG_HTML_DOMAINS)) {
                        /* HTML domain-sensitive hash (structure + domains) */
-                       nval *= sqrtf(rep->v1.prob);
+                       nval *= fuzzy_weight_from_prob(session->rule, rep->v1.prob);
                        nval *= session->rule->html_weight;
 
                        type = "htmld";
@@ -4533,7 +4601,7 @@ fuzzy_insert_result(struct fuzzy_client_session *session,
                }
                else if ((io->flags & FUZZY_CMD_FLAG_HTML)) {
                        /* HTML structural hash (template mode, domains ignored) */
-                       nval *= sqrtf(rep->v1.prob);
+                       nval *= fuzzy_weight_from_prob(session->rule, rep->v1.prob);
                        /* Apply HTML weight multiplier from rule config */
                        nval *= session->rule->html_weight;
 
@@ -4542,7 +4610,7 @@ fuzzy_insert_result(struct fuzzy_client_session *session,
                }
                else {
                        /* Calc real probability */
-                       nval *= sqrtf(rep->v1.prob);
+                       nval *= fuzzy_weight_from_prob(session->rule, rep->v1.prob);
 
                        if (cmd->shingles_count > 0) {
                                type = "txt";