From: Michael Tremer <michael.tremer@ipfire.org>
Date: Fri, 21 Jul 2023 17:42:12 +0000 (+0000)
Subject: repositories: Master repositories as async as possible
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=2a0cad296bd9d81ba9c6144bcee8785bfb3d70df;p=pbs.git

repositories: Master repositories as async as possible

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---

diff --git a/src/buildservice/__init__.py b/src/buildservice/__init__.py
index 4517a780..c525084d 100644
--- a/src/buildservice/__init__.py
+++ b/src/buildservice/__init__.py
@@ -322,6 +322,12 @@ class Backend(object):
 			"kinit", "-k", "-t", keytab, principal, **kwargs,
 		)
 
+	async def makedirs(self, path, **kwargs):
+		"""
+			Creates a new directory
+		"""
+		return await asyncio.to_thread(os.makedirs, path, **kwargs)
+
 	async def copy(self, src, dst, mode=None):
 		"""
 			Copies a file from src to dst
@@ -338,17 +344,25 @@ class Backend(object):
 		if mode:
 			await asyncio.to_thread(os.chmod, dst, mode)
 
+	async def move(self, src, dst, **kwargs):
+		"""
+			Moves something from src to dst
+		"""
+		log.debug("Moving %s to %s" % (src, dst))
+
+		# Create parent directory
+		await self.make_parent_directory(dst)
+
+		# Move!
+		await asyncio.to_thread(shutil.move, src, dst, **kwargs)
+
 	async def make_parent_directory(self, path):
 		"""
 			Creates the parent directory of path
 		"""
 		path = os.path.dirname(path)
 
-		# Create destination path (if it does not exist)
-		try:
-			await asyncio.to_thread(os.makedirs, path)
-		except FileExistsError:
-			pass
+		return await self.makedirs(path, exist_ok=True)
 
 	async def unlink(self, path):
 		"""
@@ -409,6 +423,9 @@ class Backend(object):
 		except FileNotFoundError:
 			pass
 
+	async def chmod(self, *args, **kwargs):
+		return await asyncio.to_thread(os.chmod, *args, **kwargs)
+
 	def tempfile(self, mode="w+b", delete=True):
 		"""
 			Returns an open file handle to a new temporary file
@@ -417,6 +434,15 @@ class Backend(object):
 
 		return tempfile.NamedTemporaryFile(mode=mode, dir=path, delete=delete)
 
+	def tempdir(self, **kwargs):
+		"""
+			Asynchronously creates a new temporary directory
+		"""
+		# Default to our own tmp directory
+		path = self.path("tmp")
+
+		return tempfile.TemporaryDirectory(dir=path, **kwargs)
+
 	def _write_tempfile(self, content):
 		"""
 			Writes the content to a temporary file and returns its path
diff --git a/src/buildservice/repository.py b/src/buildservice/repository.py
index 82f48a41..e1b0d237 100644
--- a/src/buildservice/repository.py
+++ b/src/buildservice/repository.py
@@ -924,11 +924,8 @@ class Repository(base.DataObject):
 		log.info("Writing repository %s..." % self)
 		key = None
 
-		# Write any new repositories to the temporary space first
-		tmp = self.backend.path("tmp")
-
 		# Create a new pakfire instance
-		with tempfile.TemporaryDirectory(dir=tmp, prefix="pakfire-repo-") as t:
+		with self.backend.tempdir(prefix="pakfire-repo-") as t:
 			with self.pakfire() as p:
 				# Fetch the key
 				if self.key:
@@ -948,26 +945,29 @@ class Repository(base.DataObject):
 					await asyncio.to_thread(p.repo_compose, path=path, key=key, files=files)
 
 			# Mark the repository as read-only
-			os.chmod(t, 0o755)
+			await self.backend.chmod(t, 0o755)
 
 			# Perform an atomic replace of the repository directory
-			with tempfile.TemporaryDirectory(dir=tmp, prefix="pakfire-repo-") as bin:
+			with self.backend.tempdir(prefix="pakfire-repo-") as bin:
 				# Local path
 				path = self.local_path()
 
-				# Ensure the parent directory exists
-				os.makedirs(os.path.dirname(path), exist_ok=True)
-
 				# Move the old data to the temporary space
 				try:
-					shutil.move(path, bin)
+					await self.backend.move(path, bin)
 
 				# Ignore if this repository is being created for the first time
 				except FileNotFoundError:
 					pass
 
 				# Move the new repository data to its destination
-				shutil.move(t, path)
+				await self.backend.move(t, path)
+
+				# Remove the temporary directory
+				# The block would normally automatically cleanup after itself, but
+				# would block. Therefore we are removing the vast majority of the
+				# data here so that the blocking operation would be shorter.
+				await self.backend.rmtree(bin)
 
 	# Delete