]> git.ipfire.org Git - pbs.git/commitdiff
backend: Use aiofiles for asyncio handling of temporary files
authorMichael Tremer <michael.tremer@ipfire.org>
Tue, 1 Aug 2023 16:47:29 +0000 (16:47 +0000)
committerMichael Tremer <michael.tremer@ipfire.org>
Tue, 1 Aug 2023 16:49:41 +0000 (16:49 +0000)
Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
src/buildservice/__init__.py
src/buildservice/config.py
src/buildservice/releasemonitoring.py
src/buildservice/repository.py
src/buildservice/sources.py
src/buildservice/uploads.py
src/web/uploads.py
tests/upload.py

index c525084d088cdb9a35a2cd6a2621856d6793ea38..8260860d18794308f7a51f351425d87a7f91d99a 100644 (file)
@@ -1,5 +1,6 @@
 #!/usr/bin/python
 
+import aiofiles
 import asyncio
 import configparser
 import inspect
@@ -210,7 +211,7 @@ class Backend(object):
                """
                        Runs this shell command
                """
-               with tempfile.TemporaryDirectory() as tmp:
+               async with self.tempdir() as tmp:
                        # Create a minimal environment
                        env = {
                                "HOME"       : os.environ.get("HOME", tmp),
@@ -432,7 +433,7 @@ class Backend(object):
                """
                path = self.path("tmp")
 
-               return tempfile.NamedTemporaryFile(mode=mode, dir=path, delete=delete)
+               return aiofiles.tempfile.NamedTemporaryFile(mode=mode, dir=path, delete=delete)
 
        def tempdir(self, **kwargs):
                """
@@ -441,7 +442,7 @@ class Backend(object):
                # Default to our own tmp directory
                path = self.path("tmp")
 
-               return tempfile.TemporaryDirectory(dir=path, **kwargs)
+               return aiofiles.tempfile.TemporaryDirectory(dir=path, **kwargs)
 
        def _write_tempfile(self, content):
                """
index 794b4b88eedc828f0918f608313a3212d83dc0fc..cdb3b75f9feb828f77014406da032ea9705befb5 100644 (file)
@@ -4,6 +4,7 @@ import configparser
 import io
 import logging
 import pakfire
+import tempfile
 
 from . import base
 
@@ -80,7 +81,7 @@ class PakfireConfig(base.Object):
 
                log.debug("Launching Pakfire with configuration:\n%s", self._to_string(config))
 
-               with self.backend.tempfile(mode="w+") as t:
+               with tempfile.NamedTemporaryFile(mode="w+") as t:
                        # Write the configuration to disk
                        config.write(t)
                        t.flush()
index 5f213916a99268c0b463f7ac43fde95a53bf6251..5036b0d57802e0fa83a44545763edbc452f99238 100644 (file)
@@ -25,7 +25,6 @@ import logging
 import os
 import re
 import shutil
-import tempfile
 import urllib.parse
 
 from . import base
@@ -731,7 +730,7 @@ class Release(base.DataObject):
                                self.monitoring.distro, "Test Build for %s" % self, owner=owner)
 
                        # Create a new temporary space for the
-                       with tempfile.TemporaryDirectory() as target:
+                       async with self.backend.tempdir() as target:
                                # Create a new source package
                                file = await self._update_source_package(build.pkg, target)
 
@@ -768,7 +767,7 @@ class Release(base.DataObject):
                        raise RuntimeError("%s is not a source package" % package)
 
                # Create temporary directory to extract the package to
-               with tempfile.TemporaryDirectory() as tmp:
+               async with self.backend.tempdir() as tmp:
                        # Path to downloaded files
                        files = os.path.join(tmp, "files")
 
index e8829d6d089ce9b7fd1c9a0e17b2aa8f5e4a465e..2ea4bfa54feaa34904fbbfbb989edcd26f37bf71 100644 (file)
@@ -929,7 +929,7 @@ class Repository(base.DataObject):
                key = None
 
                # Create a new pakfire instance
-               with self.backend.tempdir(prefix="pakfire-repo-") as t:
+               async with self.backend.tempdir(prefix="pakfire-repo-") as t:
                        with self.pakfire() as p:
                                # Fetch the key
                                if self.key:
@@ -952,7 +952,7 @@ class Repository(base.DataObject):
                        await self.backend.chmod(t, 0o755)
 
                        # Perform an atomic replace of the repository directory
-                       with self.backend.tempdir(prefix="pakfire-repo-") as bin:
+                       async with self.backend.tempdir(prefix="pakfire-repo-") as bin:
                                # Local path
                                path = self.local_path()
 
@@ -967,12 +967,6 @@ class Repository(base.DataObject):
                                # Move the new repository data to its destination
                                await self.backend.move(t, path)
 
-                               # Remove the temporary directory
-                               # The block would normally automatically cleanup after itself, but
-                               # would block. Therefore we are removing the vast majority of the
-                               # data here so that the blocking operation would be shorter.
-                               await self.backend.rmtree(bin)
-
        # Delete
 
        async def delete(self, user=None):
index 966517ae18bd6657ec5bcda9b6307986963e88dd..7d1e3331e6f69ed3358f6ac8c941cc056a7562e8 100644 (file)
@@ -804,7 +804,7 @@ class Job(base.DataObject):
 
                try:
                        # Create a new temporary directory and check out the requested revision
-                       with tempfile.TemporaryDirectory() as path:
+                       async with self.backend.tempdir() as path:
                                await self.source.git.checkout(self.commit.revision, path)
 
                                # Create a path for the source packages
index 113762fde2b6f89a1a6e3e866bc2597c2d159cc4..733baef464304f963f49597bf3b42e8ea638aa62 100644 (file)
@@ -50,7 +50,7 @@ class Uploads(base.Object):
                        """, uuid,
                )
 
-       def create(self, filename, size, owner=None):
+       async def create(self, filename, size, owner=None):
                builder = None
                user = None
 
@@ -66,33 +66,32 @@ class Uploads(base.Object):
                        user.check_quota(size)
 
                # Allocate a new temporary file
-               f = self.backend.tempfile(delete=False)
-
-               upload = self._get_upload("""
-                       INSERT INTO
-                               uploads
-                       (
+               async with self.backend.tempfile(delete=False) as f:
+                       upload = self._get_upload("""
+                               INSERT INTO
+                                       uploads
+                               (
+                                       filename,
+                                       path,
+                                       size,
+                                       builder_id,
+                                       user_id
+                               )
+                               VALUES
+                               (
+                                       %s,
+                                       %s,
+                                       %s,
+                                       %s,
+                                       %s
+                               )
+                               RETURNING *""",
                                filename,
-                               path,
+                               f.name,
                                size,
-                               builder_id,
-                               user_id
+                               builder,
+                               user,
                        )
-                       VALUES
-                       (
-                               %s,
-                               %s,
-                               %s,
-                               %s,
-                               %s
-                       )
-                       RETURNING *""",
-                       filename,
-                       f.name,
-                       size,
-                       builder,
-                       user,
-               )
 
                # Return the newly created upload object
                return upload
@@ -109,7 +108,7 @@ class Uploads(base.Object):
                size = os.path.getsize(path)
 
                # Create the new upload object
-               upload = self.create(filename=filename, size=size, **kwargs)
+               upload = await self.create(filename=filename, size=size, **kwargs)
 
                # Import the data
                with open(path, "rb") as f:
index 70e3b4c0fa5a3af514d5b103a53dd7d0408d5b03..9d5426556bb9e8a2ec01e747e62c47c2d830f6f7 100644 (file)
@@ -82,7 +82,7 @@ class APIv1IndexHandler(base.APIMixin, tornado.web.RequestHandler):
                # Create a new upload
                with self.db.transaction():
                        try:
-                               upload = self.backend.uploads.create(
+                               upload = await self.backend.uploads.create(
                                        filename,
                                        size=size,
                                        owner=self.current_user,
index cd772f049746f84042bfff1c137c38aa427cac09..5cca730687e0a93c8ff5759e91b8334035cf563d 100755 (executable)
@@ -39,7 +39,7 @@ class UploadTestCase(test.TestCase):
                payload = io.BytesIO(b"012345678901234567890123456789")
 
                with self.db.transaction():
-                       upload = self.backend.uploads.create("test.blob", size=20, owner=self.user)
+                       upload = await self.backend.uploads.create("test.blob", size=20, owner=self.user)
 
                # Try to write more than 20 bytes
                with self.assertRaises(OverflowError):
@@ -52,7 +52,7 @@ class UploadTestCase(test.TestCase):
                payload = io.BytesIO(b"01234567890123456789")
 
                with self.db.transaction():
-                       upload = self.backend.uploads.create("test.blob", size=20, owner=self.user)
+                       upload = await self.backend.uploads.create("test.blob", size=20, owner=self.user)
 
                # Write some payload
                await upload.copyfrom(payload)
@@ -70,7 +70,7 @@ class UploadTestCase(test.TestCase):
                # Create an upload that is 200 MiB large
                with self.db.transaction():
                        with self.assertRaises(users.QuotaExceededError):
-                               self.backend.uploads.create(
+                               await self.backend.uploads.create(
                                        "test.blob",
                                        size=209715200,
                                        owner=self.user,