sources: Store false-positives

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/src/database.sql b/src/database.sql
index 05e79ce94592070464ac7a4c2f816af38f927910..f167e9420bb08095cf0aec4c06f6fa54e74e35bc 100644 (file)
--- a/src/database.sql
+++ b/src/database.sql
@@ -2,7 +2,7 @@
 -- PostgreSQL database dump
 --
 
-\restrict jGGNPqbE1Kefwc4K0XaP8dH0RVN0MDuO5uFb1iLUfv4FFw5hpjZgrK0wYsIH7r2
+\restrict ojEgXNAwBkGyL3RsBRmmfNIe4rzZCESF05HETamfMJWgxDjOkXoCxupLP3Ug6qU
 
 -- Dumped from database version 17.6 (Debian 17.6-0+deb13u1)
 -- Dumped by pg_dump version 17.6 (Debian 17.6-0+deb13u1)
@@ -239,7 +239,8 @@ CREATE TABLE public.source_stats (
     source_id integer NOT NULL,
     ts timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL,
     total_domains integer NOT NULL,
-    dead_domains integer NOT NULL
+    dead_domains integer NOT NULL,
+    false_positives integer DEFAULT 0 NOT NULL
 );
 
 
@@ -281,7 +282,8 @@ CREATE TABLE public.sources (
     etag text,
     updated_at timestamp with time zone,
     total_domains integer,
-    dead_domains integer
+    dead_domains integer,
+    false_positives integer DEFAULT 0 NOT NULL
 );
 
 
@@ -585,5 +587,5 @@ ALTER TABLE ONLY public.sources
 -- PostgreSQL database dump complete
 --
 
-\unrestrict jGGNPqbE1Kefwc4K0XaP8dH0RVN0MDuO5uFb1iLUfv4FFw5hpjZgrK0wYsIH7r2
+\unrestrict ojEgXNAwBkGyL3RsBRmmfNIe4rzZCESF05HETamfMJWgxDjOkXoCxupLP3Ug6qU
 

diff --git a/src/dnsbl/sources.py b/src/dnsbl/sources.py
index 27bdf7d4aecae2ae0841604ee0cde4b04e3e02ed..ea4024de94e4047d25def186c9cebb5f635b78f6 100644 (file)
--- a/src/dnsbl/sources.py
+++ b/src/dnsbl/sources.py
@@ -601,6 +601,8 @@ class Source(sqlmodel.SQLModel, database.BackendMixin, table=True):
 
        dead_domains : int | None
 
+       false_positives: int = 0
+
        def update_stats(self):
                """
                        Updates the stats of this source
@@ -646,12 +648,66 @@ class Source(sqlmodel.SQLModel, database.BackendMixin, table=True):
                # Store the total number of dead domains
                self.dead_domains = self.backend.db.fetch_one(stmt)
 
+               whitelisted_domains = (
+                       sqlmodel
+                       .select(
+                               domains.Domain,
+                       )
+                       .where(
+                               # Select only domains from this list
+                               domains.Domain.list == self.list,
+
+                               # Only select domains that should not be blocked
+                               domains.Domain.block == False,
+
+                               # Ignore domains that have been removed
+                               domains.Domain.removed_at == None,
+                       )
+                       .cte("whitelisted_domains")
+               )
+
+               stmt = (
+                       sqlmodel
+                       .select(
+                               sqlmodel.func.count(),
+                       )
+                       .select_from(
+                               domains.Domain,
+                       )
+                       .where(
+                               # Domains must be from this source
+                               domains.Domain.source == self,
+
+                               # Domains must try to block something
+                               domains.Domain.block == True,
+
+                               # Ignore domains that have been removed
+                               domains.Domain.removed_at == None,
+
+                               # Find anything that is whitelisted
+                               sqlmodel.exists(
+                                       sqlmodel
+                                       .select(
+                                               whitelisted_domains.c.name,
+                                       )
+                                       .where(
+                                               (domains.Domain.name == whitelisted_domains.c.name) |
+                                               (domains.Domain.name.like("%." + whitelisted_domains.c.name))
+                                       )
+                               ),
+                       )
+               )
+
+               # Store the number of false positives
+               self.false_positives = self.backend.db.fetch_one(stmt)
+
                # Store the stats history
                self.backend.db.insert(
                        SourceStats,
-                       source        = self,
-                       total_domains = self.total_domains,
-                       dead_domains  = self.dead_domains,
+                       source          = self,
+                       total_domains   = self.total_domains,
+                       dead_domains    = self.dead_domains,
+                       false_positives = self.false_positives,
                )
 
 
@@ -677,3 +733,6 @@ class SourceStats(sqlmodel.SQLModel, table=True):
 
        # Dead Domains
        dead_domains: int
+
+       # False Positives
+       false_positives: int