From: Michael Tremer Date: Thu, 8 Jan 2026 11:28:29 +0000 (+0000) Subject: checker: Move check results into the main domains table X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=398382542bfe3ba76f69efa380aa0e99845fc59b;p=dbl.git checker: Move check results into the main domains table This will save us some time when composing the final lists as we are getting rid of a join. As we are storing one row for each domain in each table it makes sense to store this all together. Signed-off-by: Michael Tremer --- diff --git a/src/database.sql b/src/database.sql index 28bd738..82b5501 100644 --- a/src/database.sql +++ b/src/database.sql @@ -2,7 +2,7 @@ -- PostgreSQL database dump -- -\restrict HnyHE9wdHUeukhgNR1DDlycTFTTbv9pygL10rs3e3gZDrAeYEOJZ2ID4FIwBH1T +\restrict 8o0t5OTJIfaTvsY8tJmP42PzP48zYSIzdnWkVDzdbzece2fzwS3EaVGiSKAGHZF -- Dumped from database version 17.6 (Debian 17.6-0+deb13u1) -- Dumped by pg_dump version 17.6 (Debian 17.6-0+deb13u1) @@ -58,17 +58,6 @@ CREATE SEQUENCE public.api_keys_id_seq ALTER SEQUENCE public.api_keys_id_seq OWNED BY public.api_keys.id; --- --- Name: checker_domains; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.checker_domains ( - name text NOT NULL, - checked_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, - status boolean NOT NULL -); - - -- -- Name: domains; Type: TABLE; Schema: public; Owner: - -- @@ -85,7 +74,9 @@ CREATE TABLE public.domains ( updated_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, block boolean DEFAULT true, report_add_id uuid, - report_remove_id uuid + report_remove_id uuid, + checked_at timestamp with time zone, + dead boolean DEFAULT false ); @@ -425,14 +416,6 @@ ALTER TABLE ONLY public.api_keys ADD CONSTRAINT api_keys_pkey PRIMARY KEY (id); --- --- Name: checker_domains checker_domains_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.checker_domains - ADD CONSTRAINT checker_domains_pkey PRIMARY KEY (name); - - -- -- Name: domains domains_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -655,5 +638,5 @@ ALTER TABLE ONLY public.sources -- PostgreSQL database dump complete -- -\unrestrict HnyHE9wdHUeukhgNR1DDlycTFTTbv9pygL10rs3e3gZDrAeYEOJZ2ID4FIwBH1T +\unrestrict 8o0t5OTJIfaTvsY8tJmP42PzP48zYSIzdnWkVDzdbzece2fzwS3EaVGiSKAGHZF diff --git a/src/dnsbl/__init__.py b/src/dnsbl/__init__.py index 161b474..bed4806 100644 --- a/src/dnsbl/__init__.py +++ b/src/dnsbl/__init__.py @@ -159,11 +159,11 @@ class Backend(object): stmt = ( sqlmodel .select( - checker.CheckerDomain.status, + domains.Domain.dead, ) .where( - checker.CheckerDomain.name == domain, + domains.Domain.name == domain, ) ) - return self.db.fetch_one(stmt) + return not self.db.fetch_one(stmt) diff --git a/src/dnsbl/checker.py b/src/dnsbl/checker.py index 927796d..1e6baee 100644 --- a/src/dnsbl/checker.py +++ b/src/dnsbl/checker.py @@ -103,23 +103,18 @@ class Checker(object): .select( domains.Domain.name, ) - .join( - CheckerDomain, - domains.Domain.name == CheckerDomain.name, - isouter=True, - ) .where( domains.Domain.removed_at == None, # Only return domains that have not been checked or where the last check # was at least 4 weeks ago sqlmodel.or_( - CheckerDomain.checked_at == None, - CheckerDomain.checked_at <= cutoff, + domains.Domain.checked_at == None, + domains.Domain.checked_at <= cutoff, ), ) .order_by( - sqlmodel.nullsfirst(CheckerDomain.checked_at), + sqlmodel.nullsfirst(domains.Domain.checked_at), domains.Domain.name, ) ) @@ -166,7 +161,7 @@ class Checker(object): Called after we have received a result for the queried domain """ # Fetch the domain name - domain = self.results.pop(result) + name = self.results.pop(result) # Fetch the result or raise any exceptions try: @@ -176,15 +171,15 @@ class Checker(object): except dns.resolver.NoAnswer as e: # If we have received no response, that does mean that we might not # have found the root of the domain, but something at least responded. - status = True + dead = False # NXDOMAIN except dns.resolver.NXDOMAIN as e: - status = False + dead = True # SERVFAIL except dns.resolver.NoNameservers as e: - status = False + dead = True # Raise any other exception except Exception as e: @@ -192,43 +187,27 @@ class Checker(object): # There has been no exception, the query returned some data else: - status = True + dead = False - log.debug("Storing result for %s..." % domain) + log.debug("Storing result for %s..." % name) stmt = ( - sqlalchemy.dialects.postgresql - .insert( - CheckerDomain, + sqlmodel + .update( + domains.Domain, + ) + .where( + # The name must match + domains.Domain.name == name, + + # Don't mess with removed domains + domains.Domain.removed_at == None, ) .values({ - "name" : domain, - "status" : status, + "checked_at" : sqlmodel.func.current_timestamp(), + "dead" : dead, }) - .on_conflict_do_update( - index_elements = [ - CheckerDomain.name, - ], - set_ = { - "checked_at" : sqlmodel.func.current_timestamp(), - } - ) ) # Store the result self.backend.db.execute(stmt) - - -class CheckerDomain(sqlmodel.SQLModel, database.BackendMixin, table=True): - __tablename__ = "checker_domains" - - # Name - name: str = sqlmodel.Field(primary_key=True) - - # Checked At - checked_at : datetime.datetime = sqlmodel.Field( - sa_column_kwargs = {"server_default" : sqlmodel.text("CURRENT_TIMESTAMP")} - ) - - # Status - status : bool diff --git a/src/dnsbl/domains.py b/src/dnsbl/domains.py index c1a0799..085b679 100644 --- a/src/dnsbl/domains.py +++ b/src/dnsbl/domains.py @@ -134,6 +134,12 @@ class Domain(sqlmodel.SQLModel, database.BackendMixin, table=True): # Log action log.info("%s (block = %s) has been removed from %s" % (self.name, self.block, self.list)) + # Checked At + checked_at: datetime.datetime | None = None + + # Dead? + dead: bool = False + class DomainEvent(sqlmodel.SQLModel, table=True): """ diff --git a/src/dnsbl/lists.py b/src/dnsbl/lists.py index 18502fd..06c9d80 100644 --- a/src/dnsbl/lists.py +++ b/src/dnsbl/lists.py @@ -254,11 +254,6 @@ class List(sqlmodel.SQLModel, database.BackendMixin, table=True): .select( domains.Domain, ) - .join( - checker.CheckerDomain, - checker.CheckerDomain.name == domains.Domain.name, - isouter=True, - ) .where( # Select only domains from this list domains.Domain.list == self, @@ -269,11 +264,11 @@ class List(sqlmodel.SQLModel, database.BackendMixin, table=True): # Ignore domains that have been removed domains.Domain.removed_at == None, - # Only select domains that have been checked positive + # Only select domains that are not dead # or have not been checked, yet. sqlmodel.or_( - checker.CheckerDomain.status == None, - checker.CheckerDomain.status == True, + domains.Domain.dead == None, + domains.Domain.dead == False, ), ) .cte("blocked_domains") diff --git a/src/dnsbl/sources.py b/src/dnsbl/sources.py index d981f0e..72101bb 100644 --- a/src/dnsbl/sources.py +++ b/src/dnsbl/sources.py @@ -632,16 +632,12 @@ class Source(sqlmodel.SQLModel, database.BackendMixin, table=True): .select_from( domains.Domain, ) - .join( - checker.CheckerDomain, - checker.CheckerDomain.name == domains.Domain.name, - ) .where( domains.Domain.source == self, domains.Domain.removed_at == None, # Only check dead domains - checker.CheckerDomain.status == False, + domains.Domain.dead == True, ) )