From: Michael Tremer Date: Mon, 10 Feb 2025 11:20:20 +0000 (+0000) Subject: cache: Add a simple key/value cache with expiry time X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=e294d6ad5a92febc9e22512063ab827020d26347;p=pbs.git cache: Add a simple key/value cache with expiry time Signed-off-by: Michael Tremer --- diff --git a/Makefile.am b/Makefile.am index e1b118c0..3d9d3b63 100644 --- a/Makefile.am +++ b/Makefile.am @@ -85,6 +85,7 @@ pkgpython_PYTHON = \ src/buildservice/bugtracker.py \ src/buildservice/builders.py \ src/buildservice/builds.py \ + src/buildservice/cache.py \ src/buildservice/config.py \ src/buildservice/constants.py \ src/buildservice/database.py \ diff --git a/src/buildservice/__init__.py b/src/buildservice/__init__.py index b08e1fcc..074f7b08 100644 --- a/src/buildservice/__init__.py +++ b/src/buildservice/__init__.py @@ -17,6 +17,7 @@ from . import aws from . import bugtracker from . import builders from . import builds +from . import cache from . import config from . import database from . import distros @@ -62,6 +63,7 @@ class Backend(object): # Initialize the HTTP Client self.httpclient = httpclient.HTTPClient(self) + self.cache = cache.Cache(self) self.aws = aws.AWS(self) self.builds = builds.Builds(self) self.jobs = jobs.Jobs(self) @@ -503,6 +505,9 @@ class Backend(object): """ Called regularly to cleanup any left-over resources """ + # Cache + await self.cache.cleanup() + # Messages await self.messages.queue.cleanup() diff --git a/src/buildservice/cache.py b/src/buildservice/cache.py new file mode 100644 index 00000000..8f75d417 --- /dev/null +++ b/src/buildservice/cache.py @@ -0,0 +1,121 @@ +############################################################################### +# # +# Pakfire - The IPFire package management system # +# Copyright (C) 2025 Pakfire development team # +# # +# This program is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# This program is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with this program. If not, see . # +# # +############################################################################### + +from . import base +from . import database + +import sqlalchemy +from sqlalchemy import Column, DateTime, PickleType, Text + +cache = sqlalchemy.Table( + "cache", database.Base.metadata, + + # Key + Column("key", Text, nullable=False), + + # Value + Column("value", PickleType, nullable=False), + + # Timestamp + Column("created_at", DateTime(timezone=False), nullable=False, + server_default=sqlalchemy.func.current_timestamp()), + + # Expires At + Column("expires_at", DateTime(timezone=False), nullable=False), +) + +class Cache(base.Object): + async def get(self, key): + """ + Fetches an item from the cache by its key + """ + stmt = ( + sqlalchemy + .select( + cache.c.value + ) + .where( + cache.c.key == key, + + # The entry must have no expiry time or not be expired, yet + sqlalchemy.or_( + cache.c.expires_at == None, + cache.c.expires_at > sqlalchemy.func.current_timestamp(), + ) + ) + ) + + # Fetch the item + return await self.db.select_one(stmt, "value") + + async def set(self, key, value, expires_at=None): + """ + Stores an item in the cache + """ + if expires_at: + if not isinstance(expires_at, datetime.timedelta): + expires_at = datetime.timedelta(seconds=expires_at) + + # Make it an absolute timestamp + expires_at = sqlalchemy.func.current_timestamp() + expires_at + + # Create a new entry to the database + insert_stmt = ( + sqlalchemy.dialects.postgresql + .insert( + cache, + ) + .values({ + "key" : key, + "value" : value, + "expires_at" : expires_at, + }) + ) + + # If the entry exist already, we just update the value and expiry time + upsert_stmt = insert_stmt.on_conflict_do_update( + index_elements = [ + "key", + ], + set_ = { + "value" : cache.c.value, + "expires_at" : cache.c.expires_at, + }, + ) + + # Run the query + await self.db.execute(upsert_stmt) + + async def cleanup(self): + """ + Called to cleanup the cache from expired entries + """ + # Delete everything that has expired in the past + stmt = ( + cache + .delete() + .where( + cache.c.expires_at <= sqlalchemy.func.current_timestamp(), + ) + ) + + # Run the query + async with await self.db.transaction(): + await self.db.execute(stmt) diff --git a/src/database.sql b/src/database.sql index a028b888..cf7c7803 100644 --- a/src/database.sql +++ b/src/database.sql @@ -300,6 +300,18 @@ CREATE SEQUENCE public.builds_id_seq ALTER SEQUENCE public.builds_id_seq OWNED BY public.builds.id; +-- +-- Name: cache; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.cache ( + key text NOT NULL, + value bytea NOT NULL, + created_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, + expires_at timestamp without time zone +); + + -- -- Name: distributions; Type: TABLE; Schema: public; Owner: - -- @@ -1412,6 +1424,14 @@ ALTER TABLE ONLY public.builds ADD CONSTRAINT builds_pkey PRIMARY KEY (id); +-- +-- Name: cache cache_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.cache + ADD CONSTRAINT cache_pkey PRIMARY KEY (key); + + -- -- Name: distributions distributions_pkey; Type: CONSTRAINT; Schema: public; Owner: - --