dnsbl: Add domains to sources and add export command

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/src/database.sql b/src/database.sql
index 1c8083882c919b20a7e7c3d9b1252a23781f7723..5ac2336ce6a53e096318e9f346dfc8009741fc1f 100644 (file)
--- a/src/database.sql
+++ b/src/database.sql
@@ -2,7 +2,7 @@
 -- PostgreSQL database dump
 --
 
-\restrict MdBTHRTrPo5KwL7rBdfmjwUQ78IadFTz2sk6xzAeY0VTkdkQ2ifc9gAhyzISo3V
+\restrict Ygem6wvGn2DEdbwftbB0nUDuErUteV8NLEaojWTpdRkf4XgJXIO8rDTezxoNLXd
 
 -- Dumped from database version 17.6 (Debian 17.6-0+deb13u1)
 -- Dumped by pg_dump version 17.6 (Debian 17.6-0+deb13u1)
@@ -60,6 +60,39 @@ CREATE SEQUENCE public.lists_id_seq
 ALTER SEQUENCE public.lists_id_seq OWNED BY public.lists.id;
 
 
+--
+-- Name: source_domains; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.source_domains (
+    id integer NOT NULL,
+    source_id integer NOT NULL,
+    name text NOT NULL,
+    added_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL,
+    removed_at timestamp with time zone
+);
+
+
+--
+-- Name: source_domains_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public.source_domains_id_seq
+    AS integer
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: source_domains_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE public.source_domains_id_seq OWNED BY public.source_domains.id;
+
+
 --
 -- Name: sources; Type: TABLE; Schema: public; Owner: -
 --
@@ -104,6 +137,13 @@ ALTER SEQUENCE public.sources_id_seq OWNED BY public.sources.id;
 ALTER TABLE ONLY public.lists ALTER COLUMN id SET DEFAULT nextval('public.lists_id_seq'::regclass);
 
 
+--
+-- Name: source_domains id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.source_domains ALTER COLUMN id SET DEFAULT nextval('public.source_domains_id_seq'::regclass);
+
+
 --
 -- Name: sources id; Type: DEFAULT; Schema: public; Owner: -
 --
@@ -119,6 +159,14 @@ ALTER TABLE ONLY public.lists
     ADD CONSTRAINT lists_pkey PRIMARY KEY (id);
 
 
+--
+-- Name: source_domains source_domains_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.source_domains
+    ADD CONSTRAINT source_domains_pkey PRIMARY KEY (id);
+
+
 --
 -- Name: sources sources_pkey; Type: CONSTRAINT; Schema: public; Owner: -
 --
@@ -134,6 +182,13 @@ ALTER TABLE ONLY public.sources
 CREATE UNIQUE INDEX lists_unique ON public.lists USING btree (slug) WHERE (deleted_at IS NULL);
 
 
+--
+-- Name: source_domains_unique; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE UNIQUE INDEX source_domains_unique ON public.source_domains USING btree (source_id, name) WHERE (removed_at IS NULL);
+
+
 --
 -- Name: sources_unique; Type: INDEX; Schema: public; Owner: -
 --
@@ -141,6 +196,14 @@ CREATE UNIQUE INDEX lists_unique ON public.lists USING btree (slug) WHERE (delet
 CREATE UNIQUE INDEX sources_unique ON public.sources USING btree (list_id, url) WHERE (deleted_at IS NULL);
 
 
+--
+-- Name: source_domains source_domains_source_id; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.source_domains
+    ADD CONSTRAINT source_domains_source_id FOREIGN KEY (source_id) REFERENCES public.sources(id);
+
+
 --
 -- Name: sources sources_list_id; Type: FK CONSTRAINT; Schema: public; Owner: -
 --
@@ -153,5 +216,5 @@ ALTER TABLE ONLY public.sources
 -- PostgreSQL database dump complete
 --
 
-\unrestrict MdBTHRTrPo5KwL7rBdfmjwUQ78IadFTz2sk6xzAeY0VTkdkQ2ifc9gAhyzISo3V
+\unrestrict Ygem6wvGn2DEdbwftbB0nUDuErUteV8NLEaojWTpdRkf4XgJXIO8rDTezxoNLXd
 

diff --git a/src/dnsbl/lists.py b/src/dnsbl/lists.py
index 38cab1ee40165424047f027bf003b6d5d3bbefd3..9fa0ef5b4d2be7f89709154b1fb9b1dde679ca55 100644 (file)
--- a/src/dnsbl/lists.py
+++ b/src/dnsbl/lists.py
@@ -131,6 +131,43 @@ class List(sqlmodel.SQLModel, database.BackendMixin, table=True):
        # Updated At
        updated_at : datetime.datetime | None
 
+       @property
+       def domains(self):
+               """
+                       Returns all domains that are on this list
+               """
+               stmt = (
+                       sqlmodel
+                       .select(
+                               sources.SourceDomain.name,
+                       )
+                       .distinct(
+                               sources.SourceDomain.name,
+                       )
+                       .select_from(
+                               sources.Source,
+                       )
+                       .join(
+                               sources.SourceDomain,
+                               sources.SourceDomain.source_id == sources.Source.id,
+                       )
+                       .where(
+                               # Only select sources that belong to this list
+                               sources.Source.list_id == self.id,
+
+                               # Ignore deleted sources
+                               sources.Source.deleted_at == None,
+
+                               # Ignore domains that have been removed
+                               sources.SourceDomain.removed_at == None,
+                       )
+                       .order_by(
+                               sources.SourceDomain.name,
+                       )
+               )
+
+               return self.backend.db.fetch(stmt)
+
        # Update!
 
        def update(self):
@@ -144,3 +181,12 @@ class List(sqlmodel.SQLModel, database.BackendMixin, table=True):
                        # Update all sources
                        for source in self.sources:
                                source.update()
+
+       # Export!
+
+       def export(self, f):
+               """
+                       Exports the list
+               """
+               for domain in self.domains:
+                       f.write("%s\n" % domain)

diff --git a/src/dnsbl/sources.py b/src/dnsbl/sources.py
index f586ddc911678566dc3875c07d0df72bff39f902..8bc4b7d273d17412432633ad02f37638225557a7 100644 (file)
--- a/src/dnsbl/sources.py
+++ b/src/dnsbl/sources.py
@@ -96,8 +96,38 @@ class Source(sqlmodel.SQLModel, database.BackendMixin, table=True):
        # List
        list : "List" = sqlmodel.Relationship(back_populates="sources")
 
+       # Domains
+       domains : "SourceDomain" = sqlmodel.Relationship(back_populates="source")
+
        def update(self):
                """
                        Updates this source.
                """
                log.debug("%s: Updating source %s" % (self.list, self))
+
+
+class SourceDomain(sqlmodel.SQLModel, database.BackendMixin, table=True):
+       __tablename__ = "source_domains"
+
+       def __str__(self):
+               return self.name
+
+       # ID
+       id : int = sqlmodel.Field(primary_key=True)
+
+       # Source ID
+       source_id : int = sqlmodel.Field(foreign_key="sources.id")
+
+       # Source
+       source : "Source" = sqlmodel.Relationship(back_populates="domains")
+
+       # Name
+       name : str
+
+       # Added At
+       added_at : datetime.datetime = sqlmodel.Field(
+               sa_column_kwargs = {"server_default" : sqlmodel.text("CURRENT_TIMESTAMP")}
+       )
+
+       # Removed At
+       removed_at : datetime.datetime | None

diff --git a/src/scripts/dnsbl.in b/src/scripts/dnsbl.in
index b779b8418c69afd9a7d2d7017c792df6518b5521..e7406c99ddec980c89cbab02cc2fcb2af1b5856a 100644 (file)
--- a/src/scripts/dnsbl.in
+++ b/src/scripts/dnsbl.in
@@ -76,6 +76,11 @@ class CLI(object):
                list_update.add_argument("list", help=_("The name of the list"))
                list_update.set_defaults(func=self.__list_update)
 
+               # list-export
+               list_export = subparsers.add_parser("list-export", help=_("Exports a list"))
+               list_export.add_argument("list", help=_("The name of the list"))
+               list_export.set_defaults(func=self.__list_export)
+
                # list-add-source
                list_add_source = subparsers.add_parser("list-add-source",
                                help=_("Creates a new source to a list"))
@@ -209,6 +214,16 @@ class CLI(object):
                # Update!
                list.update()
 
+       def __list_export(self, backend, args):
+               """
+                       Exports a list
+               """
+               # Fetch the list
+               list = backend.lists.get_by_slug(args.list)
+
+               # Export!
+               list.export(sys.stdout)
+
        def __list_add_source(self, backend, args):
                """
                        Adds a new source to a list