lists: Support whitelisting domains

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/src/database.sql b/src/database.sql
index 6df255e6c6d495c2eab8305d88d03ffe122f4f50..eee4c161d47cd919e106f259feb33c9978221a49 100644 (file)
--- a/src/database.sql
+++ b/src/database.sql
@@ -2,7 +2,7 @@
 -- PostgreSQL database dump
 --
 
-\restrict Ve94ffDejRZQh0k9F68Ox20B8hRjYq5z5defUdh578csjd9BHIC5XKZB0q1nuan
+\restrict EeYKZnslrxGmNaeqO4o6gPUM6JSs7wWdVd6NvcQqHzBRusgbfSbKjYZDw8t1hjq
 
 -- Dumped from database version 17.6 (Debian 17.6-0+deb13u1)
 -- Dumped by pg_dump version 17.6 (Debian 17.6-0+deb13u1)
@@ -83,7 +83,8 @@ CREATE TABLE public.domains (
     removed_at timestamp with time zone,
     removed_by text,
     updated_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL,
-    report_id uuid
+    report_id uuid,
+    block boolean DEFAULT true
 );
 
 
@@ -429,5 +430,5 @@ ALTER TABLE ONLY public.sources
 -- PostgreSQL database dump complete
 --
 
-\unrestrict Ve94ffDejRZQh0k9F68Ox20B8hRjYq5z5defUdh578csjd9BHIC5XKZB0q1nuan
+\unrestrict EeYKZnslrxGmNaeqO4o6gPUM6JSs7wWdVd6NvcQqHzBRusgbfSbKjYZDw8t1hjq
 

diff --git a/src/dnsbl/domains.py b/src/dnsbl/domains.py
index f9697a1a33a23cea778fa39e5a3b50c07770a85f..50e7d2ddf713662c6d5d93f2022d951a58e75701 100644 (file)
--- a/src/dnsbl/domains.py
+++ b/src/dnsbl/domains.py
@@ -68,9 +68,10 @@ class Domain(sqlmodel.SQLModel, database.BackendMixin, table=True):
        )
 
        # Report ID
-
        report_id: uuid.UUID | None = sqlmodel.Field(foreign_key="reports.id", default=None)
 
        # Report
-
        report: "Report" = sqlmodel.Relationship()
+
+       # Block?
+       block: bool = True

diff --git a/src/dnsbl/lists.py b/src/dnsbl/lists.py
index 523fdb8eb44e6cdbc8dc2c63d930539cec10ec14..6edcb2bcbc86b17b9d8fe5596198df47481ce41b 100644 (file)
--- a/src/dnsbl/lists.py
+++ b/src/dnsbl/lists.py
@@ -198,13 +198,11 @@ class List(sqlmodel.SQLModel, database.BackendMixin, table=True):
                """
                        A CTE to access all (active) domains on this list
                """
-               cte = (
+               # Fetch all domains that should be blocked
+               blocked_domains = (
                        sqlmodel
                        .select(
-                               domains.Domain.name.label("name"),
-                       )
-                       .distinct(
-                               domains.Domain.name,
+                               domains.Domain,
                        )
                        .join(
                                checker.CheckerDomain,
@@ -215,6 +213,9 @@ class List(sqlmodel.SQLModel, database.BackendMixin, table=True):
                                # Select only domains from this list
                                domains.Domain.list == self,
 
+                               # Only select domains that should be blocked
+                               domains.Domain.block == True,
+
                                # Ignore domains that have been removed
                                domains.Domain.removed_at == None,
 
@@ -225,13 +226,54 @@ class List(sqlmodel.SQLModel, database.BackendMixin, table=True):
                                        checker.CheckerDomain.status == True,
                                ),
                        )
-                       .order_by(
-                               domains.Domain.name,
+                       .cte("blocked_domains")
+               )
+
+               # Fetch all whitelisted domains
+               whitelisted_domains = (
+                       sqlmodel
+                       .select(
+                               domains.Domain,
+                       )
+                       .where(
+                               # Select only domains from this list
+                               domains.Domain.list == self,
+
+                               # Only select domains that should not be blocked
+                               domains.Domain.block == False,
+
+                               # Ignore domains that have been removed
+                               domains.Domain.removed_at == None,
+                       )
+                       .cte("whitelisted_domains")
+               )
+
+               # Remove any whitelisted and subdomains of any whitelisted domains
+               # from the list of blocked domains
+               listed_domains = (
+                       sqlmodel
+                       .select(
+                               blocked_domains.c.name,
+                       )
+                       .distinct(
+                               blocked_domains.c.name,
+                       )
+                       .where(
+                               ~sqlmodel.exists(
+                                       sqlmodel
+                                       .select(
+                                               whitelisted_domains.c.name,
+                                       )
+                                       .where(
+                                               (blocked_domains.c.name == whitelisted_domains.c.name) |
+                                               (blocked_domains.c.name.like("%." + whitelisted_domains.c.name))
+                                       )
+                               )
                        )
-                       .cte("domains")
+                       .cte("listed_domains")
                )
 
-               return cte
+               return listed_domains
 
        @property
        def domains(self):