backend: Refactor and unify the cleanup job

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/src/buildservice/__init__.py b/src/buildservice/__init__.py
index 496f1b0b1e4ff1907b24a332fb2e4e91da2404ed..f689a7d5bf2650571b2934937801804fc348ab67 100644 (file)
--- a/src/buildservice/__init__.py
+++ b/src/buildservice/__init__.py
@@ -271,46 +271,12 @@ class Backend(object):
                # Open the archive
                return p.open(path)
 
-       def delete_file(self, path, not_before=None):
-               self.db.execute("INSERT INTO queue_delete(path, not_before) \
-                       VALUES(%s, %s)", path, not_before)
-
-       def cleanup_files(self):
-               query = self.db.query("SELECT * FROM queue_delete \
-                       WHERE (not_before IS NULL OR not_before <= NOW())")
-
-               for row in query:
-                       if not row.path:
-                               continue
-
-                       path = row.path
-
-                       if not path or not path.startswith("%s/" % PAKFIRE_DIR):
-                               log.warning("Cannot delete file outside of the tree")
-                               continue
-
-                       try:
-                               logging.debug("Removing %s..." % path)
-                               os.unlink(path)
-                       except OSError as e:
-                               logging.error("Could not remove %s: %s" % (path, e))
-
-                       while True:                     
-                               path = os.path.dirname(path)
-
-                               # Stop if we are running outside of the tree.
-                               if not path.startswith(PAKFIRE_DIR):
-                                       break
-
-                               # If the directory is not empty, we cannot remove it.
-                               if os.path.exists(path) and os.listdir(path):
-                                       break
-
-                               try:
-                                       logging.debug("Removing %s..." % path)
-                                       os.rmdir(path)
-                               except OSError as e:
-                                       logging.error("Could not remove %s: %s" % (path, e))
-                                       break
+       async def cleanup(self):
+               """
+                       Called regularly to cleanup any left-over resources
+               """
+               # Sessions
+               await self.sessions.cleanup()
 
-                       self.db.execute("DELETE FROM queue_delete WHERE id = %s", row.id)
+               # Uploads
+               await self.uploads.cleanup()

diff --git a/src/buildservice/sessions.py b/src/buildservice/sessions.py
index b3a1a136b26f8120372d8a271cc1032908b91aa8..0f4ec1be2d28ce3b37feec6f019a308a49c982e5 100644 (file)
--- a/src/buildservice/sessions.py
+++ b/src/buildservice/sessions.py
@@ -43,9 +43,12 @@ class Sessions(base.Object):
        # Alias function
        get = get_by_session_id
 
-       def cleanup(self):
-               # Delete all sessions that are not valid any more.
-               self.db.execute("DELETE FROM sessions WHERE valid_until < NOW()")
+       async def cleanup(self):
+               """
+                       Deletes all sessions that are not valid any more
+               """
+               with self.db.transaction():
+                       self.db.execute("DELETE FROM sessions WHERE valid_until < CURRENT_TIMESTAMP")
 
 
 class Session(base.DataObject):

diff --git a/src/crontab/pakfire-build-service b/src/crontab/pakfire-build-service
index 9fd92507b198303d134a6c09c3a7fa86bcbc6fcb..8453651af8b0797ed94e78694723f1f576875ecb 100644 (file)
--- a/src/crontab/pakfire-build-service
+++ b/src/crontab/pakfire-build-service
@@ -4,6 +4,9 @@
 # Synchronize repositories once every five minutes
 */5 * * * *    pakfire pakfire-build-service repo:sync
 
+# Cleanup
+*/5 * * * *    pakfire pakfire-build-service cleanup
+
 # Check build dependencies
 #* * * * *     root    pakfire-build-service check-build-dependencies &>/dev/null
 
@@ -16,14 +19,5 @@
 # Send updates to Bugzilla
 #*/5 * * * *   pakfire pakfire-build-service send-bug-updates &>/dev/null
 
-# Cleanup files
-#*/5 * * * *   pakfire pakfire-build-service cleanup-files &>/dev/null
-
-# Cleanup uploads once an hour
-#0 * * * *     pakfire pakfire-build-service uploads:cleanup
-
-# Cleanup expired sessions
-#0 0 * * *     pakfire pakfire-build-service cleanup-sessions &>/dev/null
-
 # Run mirror check
 #*/30 * * * *  pakfire pakfire-build-service check-mirrors &>/dev/null

diff --git a/src/database.sql b/src/database.sql
index d1cb7b515c30ea6c9da4592233fa6f52db35f565..8d2ec1e01c4f8eb5443670f66da44ae094bcb1eb 100644 (file)
--- a/src/database.sql
+++ b/src/database.sql
@@ -1001,40 +1001,6 @@ ALTER TABLE public.packages_properties_id_seq OWNER TO pakfire;
 ALTER SEQUENCE public.packages_properties_id_seq OWNED BY public.packages_properties.id;
 
 
---
--- Name: queue_delete; Type: TABLE; Schema: public; Owner: pakfire
---
-
-CREATE TABLE public.queue_delete (
-    id integer NOT NULL,
-    path text NOT NULL,
-    not_before timestamp without time zone
-);
-
-
-ALTER TABLE public.queue_delete OWNER TO pakfire;
-
---
--- Name: queue_delete_id_seq; Type: SEQUENCE; Schema: public; Owner: pakfire
---
-
-CREATE SEQUENCE public.queue_delete_id_seq
-    START WITH 1
-    INCREMENT BY 1
-    NO MINVALUE
-    NO MAXVALUE
-    CACHE 1;
-
-
-ALTER TABLE public.queue_delete_id_seq OWNER TO pakfire;
-
---
--- Name: queue_delete_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: pakfire
---
-
-ALTER SEQUENCE public.queue_delete_id_seq OWNED BY public.queue_delete.id;
-
-
 --
 -- Name: relation_sizes; Type: VIEW; Schema: public; Owner: pakfire
 --
@@ -1618,13 +1584,6 @@ ALTER TABLE ONLY public.packages ALTER COLUMN id SET DEFAULT nextval('public.pac
 ALTER TABLE ONLY public.packages_properties ALTER COLUMN id SET DEFAULT nextval('public.packages_properties_id_seq'::regclass);
 
 
---
--- Name: queue_delete id; Type: DEFAULT; Schema: public; Owner: pakfire
---
-
-ALTER TABLE ONLY public.queue_delete ALTER COLUMN id SET DEFAULT nextval('public.queue_delete_id_seq'::regclass);
-
-
 --
 -- Name: repositories id; Type: DEFAULT; Schema: public; Owner: pakfire
 --
@@ -1816,14 +1775,6 @@ ALTER TABLE ONLY public.packages_properties
     ADD CONSTRAINT idx_2198147_primary PRIMARY KEY (id);
 
 
---
--- Name: queue_delete idx_2198155_primary; Type: CONSTRAINT; Schema: public; Owner: pakfire
---
-
-ALTER TABLE ONLY public.queue_delete
-    ADD CONSTRAINT idx_2198155_primary PRIMARY KEY (id);
-
-
 --
 -- Name: repositories_builds idx_2198189_primary; Type: CONSTRAINT; Schema: public; Owner: pakfire
 --

diff --git a/src/scripts/pakfire-build-service b/src/scripts/pakfire-build-service
index 83b9841b9b93eab65f5e55d77e9a64f5a194c8b0..822969fc44647f07a9aac3e7e5f57089cf8c7210 100644 (file)
--- a/src/scripts/pakfire-build-service
+++ b/src/scripts/pakfire-build-service
@@ -18,6 +18,9 @@ class Cli(object):
                        # Bugzilla
                        "bugzilla:version" : self.backend.bugzilla.version,
 
+                       # Cleanup
+                       "cleanup"          : self.backend.cleanup,
+
                        # Repositories
                        "repos:sync"       : self.backend.repos.sync,
                        "repos:write"      : self.backend.repos.write,
@@ -28,12 +31,6 @@ class Cli(object):
                        # Run mirror check
                        #"check-mirrors" : self.backend.mirrors.check,
 
-                       # Cleanup files
-                       #"cleanup-files" : self.backend.cleanup_files,
-
-                       # Cleanup sessions
-                       #"cleanup-sessions" : self.backend.sessions.cleanup,
-
                        # Dist
                        #"dist" : self.backend.sources.dist,
 
@@ -48,9 +45,6 @@ class Cli(object):
 
                        # Send bug updates to Bugzilla
                        #"send-bug-updates" : self.backend.bugzilla.send_all,
-
-                       # Cleanup uploads
-                       #"uploads:cleanup" : self.backend.uploads.cleanup,
                }
 
        async def __call__(self, *args):