From: Michael Tremer Date: Fri, 5 Dec 2025 17:56:14 +0000 (+0000) Subject: dnsbl: Add domains to sources and add export command X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=b43d285be0b6d02c09263a8781e94f9de198415c;p=dbl.git dnsbl: Add domains to sources and add export command Signed-off-by: Michael Tremer --- diff --git a/src/database.sql b/src/database.sql index 1c80838..5ac2336 100644 --- a/src/database.sql +++ b/src/database.sql @@ -2,7 +2,7 @@ -- PostgreSQL database dump -- -\restrict MdBTHRTrPo5KwL7rBdfmjwUQ78IadFTz2sk6xzAeY0VTkdkQ2ifc9gAhyzISo3V +\restrict Ygem6wvGn2DEdbwftbB0nUDuErUteV8NLEaojWTpdRkf4XgJXIO8rDTezxoNLXd -- Dumped from database version 17.6 (Debian 17.6-0+deb13u1) -- Dumped by pg_dump version 17.6 (Debian 17.6-0+deb13u1) @@ -60,6 +60,39 @@ CREATE SEQUENCE public.lists_id_seq ALTER SEQUENCE public.lists_id_seq OWNED BY public.lists.id; +-- +-- Name: source_domains; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.source_domains ( + id integer NOT NULL, + source_id integer NOT NULL, + name text NOT NULL, + added_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, + removed_at timestamp with time zone +); + + +-- +-- Name: source_domains_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.source_domains_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: source_domains_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.source_domains_id_seq OWNED BY public.source_domains.id; + + -- -- Name: sources; Type: TABLE; Schema: public; Owner: - -- @@ -104,6 +137,13 @@ ALTER SEQUENCE public.sources_id_seq OWNED BY public.sources.id; ALTER TABLE ONLY public.lists ALTER COLUMN id SET DEFAULT nextval('public.lists_id_seq'::regclass); +-- +-- Name: source_domains id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.source_domains ALTER COLUMN id SET DEFAULT nextval('public.source_domains_id_seq'::regclass); + + -- -- Name: sources id; Type: DEFAULT; Schema: public; Owner: - -- @@ -119,6 +159,14 @@ ALTER TABLE ONLY public.lists ADD CONSTRAINT lists_pkey PRIMARY KEY (id); +-- +-- Name: source_domains source_domains_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.source_domains + ADD CONSTRAINT source_domains_pkey PRIMARY KEY (id); + + -- -- Name: sources sources_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -134,6 +182,13 @@ ALTER TABLE ONLY public.sources CREATE UNIQUE INDEX lists_unique ON public.lists USING btree (slug) WHERE (deleted_at IS NULL); +-- +-- Name: source_domains_unique; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX source_domains_unique ON public.source_domains USING btree (source_id, name) WHERE (removed_at IS NULL); + + -- -- Name: sources_unique; Type: INDEX; Schema: public; Owner: - -- @@ -141,6 +196,14 @@ CREATE UNIQUE INDEX lists_unique ON public.lists USING btree (slug) WHERE (delet CREATE UNIQUE INDEX sources_unique ON public.sources USING btree (list_id, url) WHERE (deleted_at IS NULL); +-- +-- Name: source_domains source_domains_source_id; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.source_domains + ADD CONSTRAINT source_domains_source_id FOREIGN KEY (source_id) REFERENCES public.sources(id); + + -- -- Name: sources sources_list_id; Type: FK CONSTRAINT; Schema: public; Owner: - -- @@ -153,5 +216,5 @@ ALTER TABLE ONLY public.sources -- PostgreSQL database dump complete -- -\unrestrict MdBTHRTrPo5KwL7rBdfmjwUQ78IadFTz2sk6xzAeY0VTkdkQ2ifc9gAhyzISo3V +\unrestrict Ygem6wvGn2DEdbwftbB0nUDuErUteV8NLEaojWTpdRkf4XgJXIO8rDTezxoNLXd diff --git a/src/dnsbl/lists.py b/src/dnsbl/lists.py index 38cab1e..9fa0ef5 100644 --- a/src/dnsbl/lists.py +++ b/src/dnsbl/lists.py @@ -131,6 +131,43 @@ class List(sqlmodel.SQLModel, database.BackendMixin, table=True): # Updated At updated_at : datetime.datetime | None + @property + def domains(self): + """ + Returns all domains that are on this list + """ + stmt = ( + sqlmodel + .select( + sources.SourceDomain.name, + ) + .distinct( + sources.SourceDomain.name, + ) + .select_from( + sources.Source, + ) + .join( + sources.SourceDomain, + sources.SourceDomain.source_id == sources.Source.id, + ) + .where( + # Only select sources that belong to this list + sources.Source.list_id == self.id, + + # Ignore deleted sources + sources.Source.deleted_at == None, + + # Ignore domains that have been removed + sources.SourceDomain.removed_at == None, + ) + .order_by( + sources.SourceDomain.name, + ) + ) + + return self.backend.db.fetch(stmt) + # Update! def update(self): @@ -144,3 +181,12 @@ class List(sqlmodel.SQLModel, database.BackendMixin, table=True): # Update all sources for source in self.sources: source.update() + + # Export! + + def export(self, f): + """ + Exports the list + """ + for domain in self.domains: + f.write("%s\n" % domain) diff --git a/src/dnsbl/sources.py b/src/dnsbl/sources.py index f586ddc..8bc4b7d 100644 --- a/src/dnsbl/sources.py +++ b/src/dnsbl/sources.py @@ -96,8 +96,38 @@ class Source(sqlmodel.SQLModel, database.BackendMixin, table=True): # List list : "List" = sqlmodel.Relationship(back_populates="sources") + # Domains + domains : "SourceDomain" = sqlmodel.Relationship(back_populates="source") + def update(self): """ Updates this source. """ log.debug("%s: Updating source %s" % (self.list, self)) + + +class SourceDomain(sqlmodel.SQLModel, database.BackendMixin, table=True): + __tablename__ = "source_domains" + + def __str__(self): + return self.name + + # ID + id : int = sqlmodel.Field(primary_key=True) + + # Source ID + source_id : int = sqlmodel.Field(foreign_key="sources.id") + + # Source + source : "Source" = sqlmodel.Relationship(back_populates="domains") + + # Name + name : str + + # Added At + added_at : datetime.datetime = sqlmodel.Field( + sa_column_kwargs = {"server_default" : sqlmodel.text("CURRENT_TIMESTAMP")} + ) + + # Removed At + removed_at : datetime.datetime | None diff --git a/src/scripts/dnsbl.in b/src/scripts/dnsbl.in index b779b84..e7406c9 100644 --- a/src/scripts/dnsbl.in +++ b/src/scripts/dnsbl.in @@ -76,6 +76,11 @@ class CLI(object): list_update.add_argument("list", help=_("The name of the list")) list_update.set_defaults(func=self.__list_update) + # list-export + list_export = subparsers.add_parser("list-export", help=_("Exports a list")) + list_export.add_argument("list", help=_("The name of the list")) + list_export.set_defaults(func=self.__list_export) + # list-add-source list_add_source = subparsers.add_parser("list-add-source", help=_("Creates a new source to a list")) @@ -209,6 +214,16 @@ class CLI(object): # Update! list.update() + def __list_export(self, backend, args): + """ + Exports a list + """ + # Fetch the list + list = backend.lists.get_by_slug(args.list) + + # Export! + list.export(sys.stdout) + def __list_add_source(self, backend, args): """ Adds a new source to a list