jobs: Implement importing logs

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/src/buildservice/__init__.py b/src/buildservice/__init__.py
index 35cd4d239abcd05412f6cd2ef36530ff49fa2301..8c256f6b9749c8d41f52d7ef0bb03525ad3f4ae8 100644 (file)
--- a/src/buildservice/__init__.py
+++ b/src/buildservice/__init__.py
@@ -202,6 +202,23 @@ class Backend(object):
                        "kinit", "-k", "-t", keytab, principal,
                )
 
+       async def copy(self, src, dst):
+               """
+                       Copies a file from src to dst
+               """
+               log.debug("Copying %s to %s" % (src, dst))
+
+               path = os.path.dirname(dst)
+
+               # Create destination path (if it does not exist)
+               try:
+                       await asyncio.to_thread(os.makedirs, path)
+               except FileExistsError:
+                       pass
+
+               # Copy data without any metadata
+               await asyncio.to_thread(shutil.copyfile, src, dst)
+
        def _write_tempfile(self, content):
                """
                        Writes the content to a temporary file and returns its path

diff --git a/src/buildservice/constants.py.in b/src/buildservice/constants.py.in
index 4c2d96be4d635b983d37a59769a3d42fc25c1ba3..acc45de4424e947cd3f8050618668cc394739f1e 100644 (file)
--- a/src/buildservice/constants.py.in
+++ b/src/buildservice/constants.py.in
@@ -14,6 +14,7 @@ TEMPLATESDIR = "@templatesdir@"
 STATICDIR    = "@staticdir@"
 
 PAKFIRE_DIR  = "/pub/pakfire"
+LOGS_DIR     = os.path.join(PAKFIRE_DIR, "logs")
 PACKAGES_DIR = os.path.join(PAKFIRE_DIR, "packages")
 REPOS_DIR    = os.path.join(PAKFIRE_DIR, "repos")
 SOURCES_DIR  = os.path.join(PAKFIRE_DIR, "sources")

diff --git a/src/buildservice/jobs.py b/src/buildservice/jobs.py
index 0d3f646476e223eb0e7ce8100d306dacd3ebee06..75e5388f42710476d4333b91cd87c5552dc47ea5 100644 (file)
--- a/src/buildservice/jobs.py
+++ b/src/buildservice/jobs.py
@@ -224,13 +224,17 @@ class Job(base.DataObject):
                # Store the time
                self._set_attribute_now("started_at")
 
-       def finished(self, success, message=None):
+       async def finished(self, success, message=None, log=None):
                """
                        Called when this job has finished
                """
                # Store the time
                self._set_attribute_now("finished_at")
 
+               # Import log
+               if log:
+                       await self._import_log(log)
+
                # Did this build fail?
                if not success:
                        # Clone the build
@@ -311,6 +315,27 @@ class Job(base.DataObject):
 
                return job
 
+       async def _import_log(self, upload):
+               # Create some destination path
+               path = os.path.join(
+                       LOGS_DIR,
+                       "jobs",
+                       self.uuid[0:2],
+                       self.uuid[2:4],
+                       "%s.log" % self.uuid[4:],
+               )
+
+               # Copy file to its destination
+               await self.backend.copy(upload.path, path)
+
+               # Compute a digest for integrity
+               digest = await upload.digest("blake2s")
+
+               # Store everything in the database
+               self._set_attribute("log_path", path)
+               self._set_attribute("log_size", upload.size)
+               self._set_attribute("log_digest_blake2s", digest)
+
        # Builder
 
        @lazy_property