From: Michael Tremer <michael.tremer@ipfire.org>
Date: Wed, 24 May 2023 15:07:38 +0000 (+0000)
Subject: repositories: Make updates smarter
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=a88c6c158cab2e8b0acbdaf2ba1c9c48fd23bcb0;p=pbs.git

repositories: Make updates smarter

Repositories won't be (re-)written more than once at a time. Instead we
will mark that another update was requested and execute it as soon as
the first one finished.

That way, multiple updates will only cause the repository to be
rewritten twice.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---

diff --git a/src/buildservice/builds.py b/src/buildservice/builds.py
index faae08cc..25752b85 100644
--- a/src/buildservice/builds.py
+++ b/src/buildservice/builds.py
@@ -776,7 +776,7 @@ class Build(base.DataObject):
 			This method should be called if the repositories should be updated
 		"""
 		for repo in self.repos:
-			await repo.has_changed()
+			await repo.changed()
 
 	## Bugs
 
diff --git a/src/buildservice/repository.py b/src/buildservice/repository.py
index 347da7da..1d60956c 100644
--- a/src/buildservice/repository.py
+++ b/src/buildservice/repository.py
@@ -443,7 +443,7 @@ class Repository(base.DataObject):
 		# XXX TODO
 
 		# Update the repository (in the background)
-		await self.has_changed()
+		await self.changed()
 
 	async def remove_build(self, build, user=None):
 		"""
@@ -469,7 +469,7 @@ class Repository(base.DataObject):
 			pass
 
 		# Update the repository (in the background)
-		await self.has_changed()
+		await self.changed()
 
 	# Sources
 
@@ -825,7 +825,23 @@ class Repository(base.DataObject):
 
 		return self.backend.pakfire(distro=self.distro, repos=[self], **kwargs)
 
-	async def has_changed(self):
+	# Has Changed?
+
+	__has_changed = {}
+
+	@property
+	def has_changed(self):
+		"""
+			This is set when the repository has been changed and needs an update
+		"""
+		try:
+			event = self.__has_changed[self.id]
+		except KeyError:
+			event = self.__has_changed[self.id] = asyncio.Event()
+
+		return event
+
+	async def changed(self):
 		"""
 			Convenience function that should be called when a package has been
 			added/removed from this repository.
@@ -833,12 +849,45 @@ class Repository(base.DataObject):
 		# Run the update process at the next convenient moment
 		self.backend.run_task(self.update)
 
+	__locks = {}
+
+	@property
+	def lock(self):
+		"""
+			Locks the repository when it is being rewritten
+		"""
+		try:
+			lock = self.__locks[self.id]
+		except KeyError:
+			lock = self.__locks[self.id] = asyncio.Lock()
+
+		return lock
+
 	async def update(self):
 		"""
 			Called to perform an update of this repository
 		"""
-		# Write/rewrite the repository
-		await self.write()
+		# If the repository is currently being rewritten,
+		# we will mark it as changed and return.
+		if self.lock.locked():
+			log.debug("%s is currently being updated" % self)
+
+			# Set flag
+			self.has_changed.set()
+			return
+
+		# Otherwise, try to acquire the lock
+		async with self.lock:
+			# Clear the "has changed" flag
+			self.has_changed.clear()
+
+			# Write/rewrite the repository
+			await self.write()
+
+		# Check if the repository has been changed in the meantime
+		# and if so, call ourselves again
+		if self.has_changed.is_set():
+			await self.update()
 
 		# Relaunch any pending jobs
 		await self.relaunch_pending_jobs()
@@ -862,23 +911,11 @@ class Repository(base.DataObject):
 
 	# Write repository
 
-	__locks = {}
-
-	@property
-	def __lock(self):
-		try:
-			lock = self.__locks[self.id]
-		except KeyError:
-			lock = self.__locks[self.id] = asyncio.Lock()
-
-		return lock
-
 	async def write(self):
 		"""
 			Called to write/rewrite/update the repository metadata
 		"""
-		with self.__lock:
-			return await asyncio.to_thread(self._write)
+		return await asyncio.to_thread(self._write)
 
 	def _write(self):
 		log.info("Writing repository %s..." % self)