repositories: Master repositories as async as possible

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/src/buildservice/__init__.py b/src/buildservice/__init__.py
index 4517a7806860ac90be5ee1413c9c539838eecc3c..c525084d088cdb9a35a2cd6a2621856d6793ea38 100644 (file)
--- a/src/buildservice/__init__.py
+++ b/src/buildservice/__init__.py
@@ -322,6 +322,12 @@ class Backend(object):
                        "kinit", "-k", "-t", keytab, principal, **kwargs,
                )
 
+       async def makedirs(self, path, **kwargs):
+               """
+                       Creates a new directory
+               """
+               return await asyncio.to_thread(os.makedirs, path, **kwargs)
+
        async def copy(self, src, dst, mode=None):
                """
                        Copies a file from src to dst
@@ -338,17 +344,25 @@ class Backend(object):
                if mode:
                        await asyncio.to_thread(os.chmod, dst, mode)
 
+       async def move(self, src, dst, **kwargs):
+               """
+                       Moves something from src to dst
+               """
+               log.debug("Moving %s to %s" % (src, dst))
+
+               # Create parent directory
+               await self.make_parent_directory(dst)
+
+               # Move!
+               await asyncio.to_thread(shutil.move, src, dst, **kwargs)
+
        async def make_parent_directory(self, path):
                """
                        Creates the parent directory of path
                """
                path = os.path.dirname(path)
 
-               # Create destination path (if it does not exist)
-               try:
-                       await asyncio.to_thread(os.makedirs, path)
-               except FileExistsError:
-                       pass
+               return await self.makedirs(path, exist_ok=True)
 
        async def unlink(self, path):
                """
@@ -409,6 +423,9 @@ class Backend(object):
                except FileNotFoundError:
                        pass
 
+       async def chmod(self, *args, **kwargs):
+               return await asyncio.to_thread(os.chmod, *args, **kwargs)
+
        def tempfile(self, mode="w+b", delete=True):
                """
                        Returns an open file handle to a new temporary file
@@ -417,6 +434,15 @@ class Backend(object):
 
                return tempfile.NamedTemporaryFile(mode=mode, dir=path, delete=delete)
 
+       def tempdir(self, **kwargs):
+               """
+                       Asynchronously creates a new temporary directory
+               """
+               # Default to our own tmp directory
+               path = self.path("tmp")
+
+               return tempfile.TemporaryDirectory(dir=path, **kwargs)
+
        def _write_tempfile(self, content):
                """
                        Writes the content to a temporary file and returns its path

diff --git a/src/buildservice/repository.py b/src/buildservice/repository.py
index 82f48a4139a5422651ffdf2ad47f10180368b42a..e1b0d2376c66b7fd2ef77b466beb93bf5f6b8699 100644 (file)
--- a/src/buildservice/repository.py
+++ b/src/buildservice/repository.py
@@ -924,11 +924,8 @@ class Repository(base.DataObject):
                log.info("Writing repository %s..." % self)
                key = None
 
-               # Write any new repositories to the temporary space first
-               tmp = self.backend.path("tmp")
-
                # Create a new pakfire instance
-               with tempfile.TemporaryDirectory(dir=tmp, prefix="pakfire-repo-") as t:
+               with self.backend.tempdir(prefix="pakfire-repo-") as t:
                        with self.pakfire() as p:
                                # Fetch the key
                                if self.key:
@@ -948,26 +945,29 @@ class Repository(base.DataObject):
                                        await asyncio.to_thread(p.repo_compose, path=path, key=key, files=files)
 
                        # Mark the repository as read-only
-                       os.chmod(t, 0o755)
+                       await self.backend.chmod(t, 0o755)
 
                        # Perform an atomic replace of the repository directory
-                       with tempfile.TemporaryDirectory(dir=tmp, prefix="pakfire-repo-") as bin:
+                       with self.backend.tempdir(prefix="pakfire-repo-") as bin:
                                # Local path
                                path = self.local_path()
 
-                               # Ensure the parent directory exists
-                               os.makedirs(os.path.dirname(path), exist_ok=True)
-
                                # Move the old data to the temporary space
                                try:
-                                       shutil.move(path, bin)
+                                       await self.backend.move(path, bin)
 
                                # Ignore if this repository is being created for the first time
                                except FileNotFoundError:
                                        pass
 
                                # Move the new repository data to its destination
-                               shutil.move(t, path)
+                               await self.backend.move(t, path)
+
+                               # Remove the temporary directory
+                               # The block would normally automatically cleanup after itself, but
+                               # would block. Therefore we are removing the vast majority of the
+                               # data here so that the blocking operation would be shorter.
+                               await self.backend.rmtree(bin)
 
        # Delete