sources: Store a stats history

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/src/database.sql b/src/database.sql
index 43998da6e4503e416f084f00ded9461597984372..94146f1060ac702b6aa869c9403dbe9b0c0f1281 100644 (file)
--- a/src/database.sql
+++ b/src/database.sql
@@ -2,7 +2,7 @@
 -- PostgreSQL database dump
 --
 
-\restrict CceG0LqeHWTFqDb6DQrSpNSpqyV7f4lvN0FfZf1Vzji48NH0WT6mIEhc9x3L4V9
+\restrict pXUPUyLkgUWquVqcqOb0Ffu3EcVPJGROwmyXhgPEICcDG5af5bsnhpnMAO9PMhG
 
 -- Dumped from database version 17.6 (Debian 17.6-0+deb13u1)
 -- Dumped by pg_dump version 17.6 (Debian 17.6-0+deb13u1)
@@ -198,6 +198,39 @@ CREATE TABLE public.reports (
 );
 
 
+--
+-- Name: source_stats; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.source_stats (
+    id integer NOT NULL,
+    source_id integer NOT NULL,
+    ts timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL,
+    total_domains integer NOT NULL,
+    dead_domains integer NOT NULL
+);
+
+
+--
+-- Name: source_stats_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public.source_stats_id_seq
+    AS integer
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: source_stats_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE public.source_stats_id_seq OWNED BY public.source_stats.id;
+
+
 --
 -- Name: sources; Type: TABLE; Schema: public; Owner: -
 --
@@ -268,6 +301,13 @@ ALTER TABLE ONLY public.lists ALTER COLUMN id SET DEFAULT nextval('public.lists_
 ALTER TABLE ONLY public.nameservers ALTER COLUMN id SET DEFAULT nextval('public.nameservers_id_seq'::regclass);
 
 
+--
+-- Name: source_stats id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.source_stats ALTER COLUMN id SET DEFAULT nextval('public.source_stats_id_seq'::regclass);
+
+
 --
 -- Name: sources id; Type: DEFAULT; Schema: public; Owner: -
 --
@@ -323,6 +363,14 @@ ALTER TABLE ONLY public.reports
     ADD CONSTRAINT reports_pkey PRIMARY KEY (id);
 
 
+--
+-- Name: source_stats source_stats_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.source_stats
+    ADD CONSTRAINT source_stats_pkey PRIMARY KEY (id);
+
+
 --
 -- Name: sources sources_pkey; Type: CONSTRAINT; Schema: public; Owner: -
 --
@@ -345,6 +393,20 @@ CREATE INDEX api_keys_prefix ON public.api_keys USING btree (prefix) WHERE (dele
 CREATE INDEX domains_list_id ON public.domains USING btree (list_id) WHERE (removed_at IS NULL);
 
 
+--
+-- Name: domains_list_id_added_at; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX domains_list_id_added_at ON public.domains USING btree (added_at DESC);
+
+
+--
+-- Name: domains_list_id_removed_at; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX domains_list_id_removed_at ON public.domains USING btree (removed_at DESC) WHERE (removed_at IS NOT NULL);
+
+
 --
 -- Name: domains_search; Type: INDEX; Schema: public; Owner: -
 --
@@ -387,6 +449,13 @@ CREATE UNIQUE INDEX lists_unique ON public.lists USING btree (slug) WHERE (delet
 CREATE INDEX reports_open ON public.reports USING btree (name) WHERE (closed_at IS NULL);
 
 
+--
+-- Name: source_stats_unique; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE UNIQUE INDEX source_stats_unique ON public.source_stats USING btree (source_id, ts DESC);
+
+
 --
 -- Name: sources_unique; Type: INDEX; Schema: public; Owner: -
 --
@@ -434,6 +503,14 @@ ALTER TABLE ONLY public.reports
     ADD CONSTRAINT reports_list_id FOREIGN KEY (list_id) REFERENCES public.lists(id);
 
 
+--
+-- Name: source_stats source_stats_source_id; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.source_stats
+    ADD CONSTRAINT source_stats_source_id FOREIGN KEY (source_id) REFERENCES public.sources(id);
+
+
 --
 -- Name: sources sources_list_id; Type: FK CONSTRAINT; Schema: public; Owner: -
 --
@@ -446,5 +523,5 @@ ALTER TABLE ONLY public.sources
 -- PostgreSQL database dump complete
 --
 
-\unrestrict CceG0LqeHWTFqDb6DQrSpNSpqyV7f4lvN0FfZf1Vzji48NH0WT6mIEhc9x3L4V9
+\unrestrict pXUPUyLkgUWquVqcqOb0Ffu3EcVPJGROwmyXhgPEICcDG5af5bsnhpnMAO9PMhG
 

diff --git a/src/dnsbl/sources.py b/src/dnsbl/sources.py
index c3fcd99a554dd987ce11fac47f1b89b086e0e02a..a6617e2183f14f229aeb24b13cb7d4eb37020a37 100644 (file)
--- a/src/dnsbl/sources.py
+++ b/src/dnsbl/sources.py
@@ -642,3 +642,35 @@ class Source(sqlmodel.SQLModel, database.BackendMixin, table=True):
 
                # Store the total number of dead domains
                self.dead_domains = self.backend.db.fetch_one(stmt)
+
+               # Store the stats history
+               self.backend.db.insert(
+                       SourceStats,
+                       source        = self,
+                       total_domains = self.total_domains,
+                       dead_domains  = self.dead_domains,
+               )
+
+
+class SourceStats(sqlmodel.SQLModel, table=True):
+       __tablename__ = "source_stats"
+
+       # ID
+       id: int = sqlmodel.Field(primary_key=True, exclude=True)
+
+       # Source ID
+       source_id: int = sqlmodel.Field(foreign_key="sources.id", exclude=True)
+
+       # Source
+       source: "Source" = sqlmodel.Relationship()
+
+       # Timestamp
+       ts: datetime.datetime = sqlmodel.Field(
+               sa_column_kwargs = { "server_default" : sqlmodel.text("CURRENT_TIMESTAMP") },
+       )
+
+       # Total Domains
+       total_domains: int
+
+       # Dead Domains
+       dead_domains: int