checker: Move check results into the main domains table

This will save us some time when composing the final lists as we are
getting rid of a join. As we are storing one row for each domain in each
table it makes sense to store this all together.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/src/database.sql b/src/database.sql
index 28bd738c0048d67d8357912de5f1d092c98737e7..82b5501c58557e7e9b482fec8003fcfcc83b6b70 100644 (file)
--- a/src/database.sql
+++ b/src/database.sql
@@ -2,7 +2,7 @@
 -- PostgreSQL database dump
 --
 
-\restrict HnyHE9wdHUeukhgNR1DDlycTFTTbv9pygL10rs3e3gZDrAeYEOJZ2ID4FIwBH1T
+\restrict 8o0t5OTJIfaTvsY8tJmP42PzP48zYSIzdnWkVDzdbzece2fzwS3EaVGiSKAGHZF
 
 -- Dumped from database version 17.6 (Debian 17.6-0+deb13u1)
 -- Dumped by pg_dump version 17.6 (Debian 17.6-0+deb13u1)
@@ -58,17 +58,6 @@ CREATE SEQUENCE public.api_keys_id_seq
 ALTER SEQUENCE public.api_keys_id_seq OWNED BY public.api_keys.id;
 
 
---
--- Name: checker_domains; Type: TABLE; Schema: public; Owner: -
---
-
-CREATE TABLE public.checker_domains (
-    name text NOT NULL,
-    checked_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL,
-    status boolean NOT NULL
-);
-
-
 --
 -- Name: domains; Type: TABLE; Schema: public; Owner: -
 --
@@ -85,7 +74,9 @@ CREATE TABLE public.domains (
     updated_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL,
     block boolean DEFAULT true,
     report_add_id uuid,
-    report_remove_id uuid
+    report_remove_id uuid,
+    checked_at timestamp with time zone,
+    dead boolean DEFAULT false
 );
 
 
@@ -425,14 +416,6 @@ ALTER TABLE ONLY public.api_keys
     ADD CONSTRAINT api_keys_pkey PRIMARY KEY (id);
 
 
---
--- Name: checker_domains checker_domains_pkey; Type: CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.checker_domains
-    ADD CONSTRAINT checker_domains_pkey PRIMARY KEY (name);
-
-
 --
 -- Name: domains domains_pkey; Type: CONSTRAINT; Schema: public; Owner: -
 --
@@ -655,5 +638,5 @@ ALTER TABLE ONLY public.sources
 -- PostgreSQL database dump complete
 --
 
-\unrestrict HnyHE9wdHUeukhgNR1DDlycTFTTbv9pygL10rs3e3gZDrAeYEOJZ2ID4FIwBH1T
+\unrestrict 8o0t5OTJIfaTvsY8tJmP42PzP48zYSIzdnWkVDzdbzece2fzwS3EaVGiSKAGHZF
 

diff --git a/src/dnsbl/__init__.py b/src/dnsbl/__init__.py
index 161b474b9783b2b919c7a850234915ddf4988f32..bed4806bd8d36b69d32fd4f70f252bf74c974749 100644 (file)
--- a/src/dnsbl/__init__.py
+++ b/src/dnsbl/__init__.py
@@ -159,11 +159,11 @@ class Backend(object):
                stmt = (
                        sqlmodel
                        .select(
-                               checker.CheckerDomain.status,
+                               domains.Domain.dead,
                        )
                        .where(
-                               checker.CheckerDomain.name == domain,
+                               domains.Domain.name == domain,
                        )
                )
 
-               return self.db.fetch_one(stmt)
+               return not self.db.fetch_one(stmt)

diff --git a/src/dnsbl/checker.py b/src/dnsbl/checker.py
index 927796dfd22930e31d6052261eb1bda5f8203902..1e6baee271905f159374cc5bdd4d09e1c0bbecec 100644 (file)
--- a/src/dnsbl/checker.py
+++ b/src/dnsbl/checker.py
@@ -103,23 +103,18 @@ class Checker(object):
                        .select(
                                domains.Domain.name,
                        )
-                       .join(
-                               CheckerDomain,
-                               domains.Domain.name == CheckerDomain.name,
-                               isouter=True,
-                       )
                        .where(
                                domains.Domain.removed_at == None,
 
                                # Only return domains that have not been checked or where the last check
                                # was at least 4 weeks ago
                                sqlmodel.or_(
-                                       CheckerDomain.checked_at == None,
-                                       CheckerDomain.checked_at <= cutoff,
+                                       domains.Domain.checked_at == None,
+                                       domains.Domain.checked_at <= cutoff,
                                ),
                        )
                        .order_by(
-                               sqlmodel.nullsfirst(CheckerDomain.checked_at),
+                               sqlmodel.nullsfirst(domains.Domain.checked_at),
                                domains.Domain.name,
                        )
                )
@@ -166,7 +161,7 @@ class Checker(object):
                        Called after we have received a result for the queried domain
                """
                # Fetch the domain name
-               domain = self.results.pop(result)
+               name = self.results.pop(result)
 
                # Fetch the result or raise any exceptions
                try:
@@ -176,15 +171,15 @@ class Checker(object):
                except dns.resolver.NoAnswer as e:
                        # If we have received no response, that does mean that we might not
                        # have found the root of the domain, but something at least responded.
-                       status = True
+                       dead = False
 
                # NXDOMAIN
                except dns.resolver.NXDOMAIN as e:
-                       status = False
+                       dead = True
 
                # SERVFAIL
                except dns.resolver.NoNameservers as e:
-                       status = False
+                       dead = True
 
                # Raise any other exception
                except Exception as e:
@@ -192,43 +187,27 @@ class Checker(object):
 
                # There has been no exception, the query returned some data
                else:
-                       status = True
+                       dead = False
 
-               log.debug("Storing result for %s..." % domain)
+               log.debug("Storing result for %s..." % name)
 
                stmt = (
-                       sqlalchemy.dialects.postgresql
-                       .insert(
-                               CheckerDomain,
+                       sqlmodel
+                       .update(
+                               domains.Domain,
+                       )
+                       .where(
+                               # The name must match
+                               domains.Domain.name == name,
+
+                               # Don't mess with removed domains
+                               domains.Domain.removed_at == None,
                        )
                        .values({
-                               "name"   : domain,
-                               "status" : status,
+                               "checked_at" : sqlmodel.func.current_timestamp(),
+                               "dead"       : dead,
                        })
-                       .on_conflict_do_update(
-                               index_elements = [
-                                       CheckerDomain.name,
-                               ],
-                               set_ = {
-                                       "checked_at" : sqlmodel.func.current_timestamp(),
-                               }
-                       )
                )
 
                # Store the result
                self.backend.db.execute(stmt)
-
-
-class CheckerDomain(sqlmodel.SQLModel, database.BackendMixin, table=True):
-       __tablename__ = "checker_domains"
-
-       # Name
-       name: str = sqlmodel.Field(primary_key=True)
-
-       # Checked At
-       checked_at : datetime.datetime = sqlmodel.Field(
-               sa_column_kwargs = {"server_default" : sqlmodel.text("CURRENT_TIMESTAMP")}
-       )
-
-       # Status
-       status : bool

diff --git a/src/dnsbl/domains.py b/src/dnsbl/domains.py
index c1a0799415dfb81bfe94ceef43997832124211e8..085b6790f0b558374cd51d006721d76f028904a5 100644 (file)
--- a/src/dnsbl/domains.py
+++ b/src/dnsbl/domains.py
@@ -134,6 +134,12 @@ class Domain(sqlmodel.SQLModel, database.BackendMixin, table=True):
                # Log action
                log.info("%s (block = %s) has been removed from %s" % (self.name, self.block, self.list))
 
+       # Checked At
+       checked_at: datetime.datetime | None = None
+
+       # Dead?
+       dead: bool = False
+
 
 class DomainEvent(sqlmodel.SQLModel, table=True):
        """

diff --git a/src/dnsbl/lists.py b/src/dnsbl/lists.py
index 18502fd7294cb371177872a522b5e90708373e62..06c9d80afd2a4163291846d75300b02f204681b8 100644 (file)
--- a/src/dnsbl/lists.py
+++ b/src/dnsbl/lists.py
@@ -254,11 +254,6 @@ class List(sqlmodel.SQLModel, database.BackendMixin, table=True):
                        .select(
                                domains.Domain,
                        )
-                       .join(
-                               checker.CheckerDomain,
-                               checker.CheckerDomain.name == domains.Domain.name,
-                               isouter=True,
-                       )
                        .where(
                                # Select only domains from this list
                                domains.Domain.list == self,
@@ -269,11 +264,11 @@ class List(sqlmodel.SQLModel, database.BackendMixin, table=True):
                                # Ignore domains that have been removed
                                domains.Domain.removed_at == None,
 
-                               # Only select domains that have been checked positive
+                               # Only select domains that are not dead
                                # or have not been checked, yet.
                                sqlmodel.or_(
-                                       checker.CheckerDomain.status == None,
-                                       checker.CheckerDomain.status == True,
+                                       domains.Domain.dead == None,
+                                       domains.Domain.dead == False,
                                ),
                        )
                        .cte("blocked_domains")

diff --git a/src/dnsbl/sources.py b/src/dnsbl/sources.py
index d981f0e892225f018ecabe32f2c7099108fd4849..72101bb72e679ff383f57b7c1b1a383762fef23d 100644 (file)
--- a/src/dnsbl/sources.py
+++ b/src/dnsbl/sources.py
@@ -632,16 +632,12 @@ class Source(sqlmodel.SQLModel, database.BackendMixin, table=True):
                        .select_from(
                                domains.Domain,
                        )
-                       .join(
-                               checker.CheckerDomain,
-                               checker.CheckerDomain.name == domains.Domain.name,
-                       )
                        .where(
                                domains.Domain.source == self,
                                domains.Domain.removed_at == None,
 
                                # Only check dead domains
-                               checker.CheckerDomain.status == False,
+                               domains.Domain.dead == True,
                        )
                )