From: Michael Tremer <michael.tremer@ipfire.org>
Date: Tue, 1 Aug 2023 16:47:29 +0000 (+0000)
Subject: backend: Use aiofiles for asyncio handling of temporary files
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=739864145fa05cf8b3ddc8eb73dfe8fe083711a7;p=pbs.git

backend: Use aiofiles for asyncio handling of temporary files

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---

diff --git a/src/buildservice/__init__.py b/src/buildservice/__init__.py
index c525084d..8260860d 100644
--- a/src/buildservice/__init__.py
+++ b/src/buildservice/__init__.py
@@ -1,5 +1,6 @@
 #!/usr/bin/python
 
+import aiofiles
 import asyncio
 import configparser
 import inspect
@@ -210,7 +211,7 @@ class Backend(object):
 		"""
 			Runs this shell command
 		"""
-		with tempfile.TemporaryDirectory() as tmp:
+		async with self.tempdir() as tmp:
 			# Create a minimal environment
 			env = {
 				"HOME"       : os.environ.get("HOME", tmp),
@@ -432,7 +433,7 @@ class Backend(object):
 		"""
 		path = self.path("tmp")
 
-		return tempfile.NamedTemporaryFile(mode=mode, dir=path, delete=delete)
+		return aiofiles.tempfile.NamedTemporaryFile(mode=mode, dir=path, delete=delete)
 
 	def tempdir(self, **kwargs):
 		"""
@@ -441,7 +442,7 @@ class Backend(object):
 		# Default to our own tmp directory
 		path = self.path("tmp")
 
-		return tempfile.TemporaryDirectory(dir=path, **kwargs)
+		return aiofiles.tempfile.TemporaryDirectory(dir=path, **kwargs)
 
 	def _write_tempfile(self, content):
 		"""
diff --git a/src/buildservice/config.py b/src/buildservice/config.py
index 794b4b88..cdb3b75f 100644
--- a/src/buildservice/config.py
+++ b/src/buildservice/config.py
@@ -4,6 +4,7 @@ import configparser
 import io
 import logging
 import pakfire
+import tempfile
 
 from . import base
 
@@ -80,7 +81,7 @@ class PakfireConfig(base.Object):
 
 		log.debug("Launching Pakfire with configuration:\n%s", self._to_string(config))
 
-		with self.backend.tempfile(mode="w+") as t:
+		with tempfile.NamedTemporaryFile(mode="w+") as t:
 			# Write the configuration to disk
 			config.write(t)
 			t.flush()
diff --git a/src/buildservice/releasemonitoring.py b/src/buildservice/releasemonitoring.py
index 5f213916..5036b0d5 100644
--- a/src/buildservice/releasemonitoring.py
+++ b/src/buildservice/releasemonitoring.py
@@ -25,7 +25,6 @@ import logging
 import os
 import re
 import shutil
-import tempfile
 import urllib.parse
 
 from . import base
@@ -731,7 +730,7 @@ class Release(base.DataObject):
 				self.monitoring.distro, "Test Build for %s" % self, owner=owner)
 
 			# Create a new temporary space for the
-			with tempfile.TemporaryDirectory() as target:
+			async with self.backend.tempdir() as target:
 				# Create a new source package
 				file = await self._update_source_package(build.pkg, target)
 
@@ -768,7 +767,7 @@ class Release(base.DataObject):
 			raise RuntimeError("%s is not a source package" % package)
 
 		# Create temporary directory to extract the package to
-		with tempfile.TemporaryDirectory() as tmp:
+		async with self.backend.tempdir() as tmp:
 			# Path to downloaded files
 			files = os.path.join(tmp, "files")
 
diff --git a/src/buildservice/repository.py b/src/buildservice/repository.py
index e8829d6d..2ea4bfa5 100644
--- a/src/buildservice/repository.py
+++ b/src/buildservice/repository.py
@@ -929,7 +929,7 @@ class Repository(base.DataObject):
 		key = None
 
 		# Create a new pakfire instance
-		with self.backend.tempdir(prefix="pakfire-repo-") as t:
+		async with self.backend.tempdir(prefix="pakfire-repo-") as t:
 			with self.pakfire() as p:
 				# Fetch the key
 				if self.key:
@@ -952,7 +952,7 @@ class Repository(base.DataObject):
 			await self.backend.chmod(t, 0o755)
 
 			# Perform an atomic replace of the repository directory
-			with self.backend.tempdir(prefix="pakfire-repo-") as bin:
+			async with self.backend.tempdir(prefix="pakfire-repo-") as bin:
 				# Local path
 				path = self.local_path()
 
@@ -967,12 +967,6 @@ class Repository(base.DataObject):
 				# Move the new repository data to its destination
 				await self.backend.move(t, path)
 
-				# Remove the temporary directory
-				# The block would normally automatically cleanup after itself, but
-				# would block. Therefore we are removing the vast majority of the
-				# data here so that the blocking operation would be shorter.
-				await self.backend.rmtree(bin)
-
 	# Delete
 
 	async def delete(self, user=None):
diff --git a/src/buildservice/sources.py b/src/buildservice/sources.py
index 966517ae..7d1e3331 100644
--- a/src/buildservice/sources.py
+++ b/src/buildservice/sources.py
@@ -804,7 +804,7 @@ class Job(base.DataObject):
 
 		try:
 			# Create a new temporary directory and check out the requested revision
-			with tempfile.TemporaryDirectory() as path:
+			async with self.backend.tempdir() as path:
 				await self.source.git.checkout(self.commit.revision, path)
 
 				# Create a path for the source packages
diff --git a/src/buildservice/uploads.py b/src/buildservice/uploads.py
index 113762fd..733baef4 100644
--- a/src/buildservice/uploads.py
+++ b/src/buildservice/uploads.py
@@ -50,7 +50,7 @@ class Uploads(base.Object):
 			""", uuid,
 		)
 
-	def create(self, filename, size, owner=None):
+	async def create(self, filename, size, owner=None):
 		builder = None
 		user = None
 
@@ -66,33 +66,32 @@ class Uploads(base.Object):
 			user.check_quota(size)
 
 		# Allocate a new temporary file
-		f = self.backend.tempfile(delete=False)
-
-		upload = self._get_upload("""
-			INSERT INTO
-				uploads
-			(
+		async with self.backend.tempfile(delete=False) as f:
+			upload = self._get_upload("""
+				INSERT INTO
+					uploads
+				(
+					filename,
+					path,
+					size,
+					builder_id,
+					user_id
+				)
+				VALUES
+				(
+					%s,
+					%s,
+					%s,
+					%s,
+					%s
+				)
+				RETURNING *""",
 				filename,
-				path,
+				f.name,
 				size,
-				builder_id,
-				user_id
+				builder,
+				user,
 			)
-			VALUES
-			(
-				%s,
-				%s,
-				%s,
-				%s,
-				%s
-			)
-			RETURNING *""",
-			filename,
-			f.name,
-			size,
-			builder,
-			user,
-		)
 
 		# Return the newly created upload object
 		return upload
@@ -109,7 +108,7 @@ class Uploads(base.Object):
 		size = os.path.getsize(path)
 
 		# Create the new upload object
-		upload = self.create(filename=filename, size=size, **kwargs)
+		upload = await self.create(filename=filename, size=size, **kwargs)
 
 		# Import the data
 		with open(path, "rb") as f:
diff --git a/src/web/uploads.py b/src/web/uploads.py
index 70e3b4c0..9d542655 100644
--- a/src/web/uploads.py
+++ b/src/web/uploads.py
@@ -82,7 +82,7 @@ class APIv1IndexHandler(base.APIMixin, tornado.web.RequestHandler):
 		# Create a new upload
 		with self.db.transaction():
 			try:
-				upload = self.backend.uploads.create(
+				upload = await self.backend.uploads.create(
 					filename,
 					size=size,
 					owner=self.current_user,
diff --git a/tests/upload.py b/tests/upload.py
index cd772f04..5cca7306 100755
--- a/tests/upload.py
+++ b/tests/upload.py
@@ -39,7 +39,7 @@ class UploadTestCase(test.TestCase):
 		payload = io.BytesIO(b"012345678901234567890123456789")
 
 		with self.db.transaction():
-			upload = self.backend.uploads.create("test.blob", size=20, owner=self.user)
+			upload = await self.backend.uploads.create("test.blob", size=20, owner=self.user)
 
 		# Try to write more than 20 bytes
 		with self.assertRaises(OverflowError):
@@ -52,7 +52,7 @@ class UploadTestCase(test.TestCase):
 		payload = io.BytesIO(b"01234567890123456789")
 
 		with self.db.transaction():
-			upload = self.backend.uploads.create("test.blob", size=20, owner=self.user)
+			upload = await self.backend.uploads.create("test.blob", size=20, owner=self.user)
 
 		# Write some payload
 		await upload.copyfrom(payload)
@@ -70,7 +70,7 @@ class UploadTestCase(test.TestCase):
 		# Create an upload that is 200 MiB large
 		with self.db.transaction():
 			with self.assertRaises(users.QuotaExceededError):
-				self.backend.uploads.create(
+				await self.backend.uploads.create(
 					"test.blob",
 					size=209715200,
 					owner=self.user,