From: Michael Tremer Date: Fri, 15 Jul 2022 10:30:34 +0000 (+0000) Subject: backend: Refactor and unify the cleanup job X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=5590b6459dbc548d7af773fa54f662d33752af24;p=pbs.git backend: Refactor and unify the cleanup job Signed-off-by: Michael Tremer --- diff --git a/src/buildservice/__init__.py b/src/buildservice/__init__.py index 496f1b0b..f689a7d5 100644 --- a/src/buildservice/__init__.py +++ b/src/buildservice/__init__.py @@ -271,46 +271,12 @@ class Backend(object): # Open the archive return p.open(path) - def delete_file(self, path, not_before=None): - self.db.execute("INSERT INTO queue_delete(path, not_before) \ - VALUES(%s, %s)", path, not_before) - - def cleanup_files(self): - query = self.db.query("SELECT * FROM queue_delete \ - WHERE (not_before IS NULL OR not_before <= NOW())") - - for row in query: - if not row.path: - continue - - path = row.path - - if not path or not path.startswith("%s/" % PAKFIRE_DIR): - log.warning("Cannot delete file outside of the tree") - continue - - try: - logging.debug("Removing %s..." % path) - os.unlink(path) - except OSError as e: - logging.error("Could not remove %s: %s" % (path, e)) - - while True: - path = os.path.dirname(path) - - # Stop if we are running outside of the tree. - if not path.startswith(PAKFIRE_DIR): - break - - # If the directory is not empty, we cannot remove it. - if os.path.exists(path) and os.listdir(path): - break - - try: - logging.debug("Removing %s..." % path) - os.rmdir(path) - except OSError as e: - logging.error("Could not remove %s: %s" % (path, e)) - break + async def cleanup(self): + """ + Called regularly to cleanup any left-over resources + """ + # Sessions + await self.sessions.cleanup() - self.db.execute("DELETE FROM queue_delete WHERE id = %s", row.id) + # Uploads + await self.uploads.cleanup() diff --git a/src/buildservice/sessions.py b/src/buildservice/sessions.py index b3a1a136..0f4ec1be 100644 --- a/src/buildservice/sessions.py +++ b/src/buildservice/sessions.py @@ -43,9 +43,12 @@ class Sessions(base.Object): # Alias function get = get_by_session_id - def cleanup(self): - # Delete all sessions that are not valid any more. - self.db.execute("DELETE FROM sessions WHERE valid_until < NOW()") + async def cleanup(self): + """ + Deletes all sessions that are not valid any more + """ + with self.db.transaction(): + self.db.execute("DELETE FROM sessions WHERE valid_until < CURRENT_TIMESTAMP") class Session(base.DataObject): diff --git a/src/crontab/pakfire-build-service b/src/crontab/pakfire-build-service index 9fd92507..8453651a 100644 --- a/src/crontab/pakfire-build-service +++ b/src/crontab/pakfire-build-service @@ -4,6 +4,9 @@ # Synchronize repositories once every five minutes */5 * * * * pakfire pakfire-build-service repo:sync +# Cleanup +*/5 * * * * pakfire pakfire-build-service cleanup + # Check build dependencies #* * * * * root pakfire-build-service check-build-dependencies &>/dev/null @@ -16,14 +19,5 @@ # Send updates to Bugzilla #*/5 * * * * pakfire pakfire-build-service send-bug-updates &>/dev/null -# Cleanup files -#*/5 * * * * pakfire pakfire-build-service cleanup-files &>/dev/null - -# Cleanup uploads once an hour -#0 * * * * pakfire pakfire-build-service uploads:cleanup - -# Cleanup expired sessions -#0 0 * * * pakfire pakfire-build-service cleanup-sessions &>/dev/null - # Run mirror check #*/30 * * * * pakfire pakfire-build-service check-mirrors &>/dev/null diff --git a/src/database.sql b/src/database.sql index d1cb7b51..8d2ec1e0 100644 --- a/src/database.sql +++ b/src/database.sql @@ -1001,40 +1001,6 @@ ALTER TABLE public.packages_properties_id_seq OWNER TO pakfire; ALTER SEQUENCE public.packages_properties_id_seq OWNED BY public.packages_properties.id; --- --- Name: queue_delete; Type: TABLE; Schema: public; Owner: pakfire --- - -CREATE TABLE public.queue_delete ( - id integer NOT NULL, - path text NOT NULL, - not_before timestamp without time zone -); - - -ALTER TABLE public.queue_delete OWNER TO pakfire; - --- --- Name: queue_delete_id_seq; Type: SEQUENCE; Schema: public; Owner: pakfire --- - -CREATE SEQUENCE public.queue_delete_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - - -ALTER TABLE public.queue_delete_id_seq OWNER TO pakfire; - --- --- Name: queue_delete_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: pakfire --- - -ALTER SEQUENCE public.queue_delete_id_seq OWNED BY public.queue_delete.id; - - -- -- Name: relation_sizes; Type: VIEW; Schema: public; Owner: pakfire -- @@ -1618,13 +1584,6 @@ ALTER TABLE ONLY public.packages ALTER COLUMN id SET DEFAULT nextval('public.pac ALTER TABLE ONLY public.packages_properties ALTER COLUMN id SET DEFAULT nextval('public.packages_properties_id_seq'::regclass); --- --- Name: queue_delete id; Type: DEFAULT; Schema: public; Owner: pakfire --- - -ALTER TABLE ONLY public.queue_delete ALTER COLUMN id SET DEFAULT nextval('public.queue_delete_id_seq'::regclass); - - -- -- Name: repositories id; Type: DEFAULT; Schema: public; Owner: pakfire -- @@ -1816,14 +1775,6 @@ ALTER TABLE ONLY public.packages_properties ADD CONSTRAINT idx_2198147_primary PRIMARY KEY (id); --- --- Name: queue_delete idx_2198155_primary; Type: CONSTRAINT; Schema: public; Owner: pakfire --- - -ALTER TABLE ONLY public.queue_delete - ADD CONSTRAINT idx_2198155_primary PRIMARY KEY (id); - - -- -- Name: repositories_builds idx_2198189_primary; Type: CONSTRAINT; Schema: public; Owner: pakfire -- diff --git a/src/scripts/pakfire-build-service b/src/scripts/pakfire-build-service index 83b9841b..822969fc 100644 --- a/src/scripts/pakfire-build-service +++ b/src/scripts/pakfire-build-service @@ -18,6 +18,9 @@ class Cli(object): # Bugzilla "bugzilla:version" : self.backend.bugzilla.version, + # Cleanup + "cleanup" : self.backend.cleanup, + # Repositories "repos:sync" : self.backend.repos.sync, "repos:write" : self.backend.repos.write, @@ -28,12 +31,6 @@ class Cli(object): # Run mirror check #"check-mirrors" : self.backend.mirrors.check, - # Cleanup files - #"cleanup-files" : self.backend.cleanup_files, - - # Cleanup sessions - #"cleanup-sessions" : self.backend.sessions.cleanup, - # Dist #"dist" : self.backend.sources.dist, @@ -48,9 +45,6 @@ class Cli(object): # Send bug updates to Bugzilla #"send-bug-updates" : self.backend.bugzilla.send_all, - - # Cleanup uploads - #"uploads:cleanup" : self.backend.uploads.cleanup, } async def __call__(self, *args):