From: Michael Tremer <michael.tremer@ipfire.org>
Date: Sat, 18 Jun 2022 12:52:14 +0000 (+0000)
Subject: builders: Implement automatic shutdown of builders
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=b76b5d0a1eecb1f94b4aafffe42cac1dfb459e33;p=pbs.git

builders: Implement automatic shutdown of builders

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---

diff --git a/src/buildservice/builders.py b/src/buildservice/builders.py
index da8cf16c..b5557d87 100644
--- a/src/buildservice/builders.py
+++ b/src/buildservice/builders.py
@@ -125,6 +125,47 @@ class Builders(base.Object):
 		# Sync all builders
 		await asyncio.gather(*(builder.sync() for builder in self))
 
+	async def autoscale(self, wait=False):
+		"""
+			This method performs two tasks:
+
+			* It will launch any new builders if more are required
+
+			* It will shutdown any builders which are no longer required
+		"""
+		log.debug("Running autoscaling schedulder")
+
+		builders_to_be_shut_down = []
+
+		# Check if there are any builders which can be shut down
+		for builder in self:
+			# Don't consider builders which are not running
+			if not await builder.is_running():
+				log.debug("Won't consider %s which is not running" % builder)
+				continue
+
+			# Check if there are any jobs
+			if len(builder.jobs) > 0:
+				log.debug("Won't shut down %s which has active jobs" % builder)
+				continue
+
+			# Check if there any jobs in the queue that are suitable for this builder
+			job = self.backend.jobqueue.pop(builder)
+			if job:
+				log.debug("Won't shut down %s which has pending jobs" % builder)
+				continue
+
+			# Shutting down this builder
+			builders_to_be_shut_down.append(builder)
+
+		# Perform shutdowns
+		if builders_to_be_shut_down:
+			await asyncio.gather(
+				*(builder.stop(wait=wait) for builder in builders_to_be_shut_down),
+			)
+
+		# XXX implement starting builders if needed
+
 
 class Builder(base.DataObject):
 	table = "builders"
@@ -466,6 +507,37 @@ class Builder(base.DataObject):
 		# Launch in a separate thread
 		await asyncio.to_thread(callback)
 
+	async def is_running(self):
+		"""
+			Returns True if this builder is currently running
+		"""
+		state = await asyncio.to_thread(self._fetch_state)
+
+		return state in ("pending", "running")
+
+	async def is_shutdown(self):
+		"""
+			Returns True if this builder is shut down
+		"""
+		state = await asyncio.to_thread(self._fetch_state)
+
+		return state == "stopped"
+
+	async def is_shutting_down(self):
+		"""
+			Returns True if this builder is currently shutting down
+		"""
+		state = await asyncio.to_thread(self._fetch_state)
+
+		return state in ("shutting-down", "stopping")
+
+	def _fetch_state(self):
+		"""
+			Returns the current state of this instance
+		"""
+		if self.instance:
+			return self.instance.state.get("Name")
+
 	async def start(self):
 		"""
 			Starts the instance on AWS
@@ -514,18 +586,22 @@ class Builder(base.DataObject):
 		except botocore.exceptions.ClientError as e:
 			log.warning("Could not change instance type of %s: %s" % (self, e))
 
-	async def stop(self):
+	async def stop(self, wait=True):
 		"""
 			Stops this instance on AWS
 		"""
-		await asyncio.to_thread(self._stop)
+		await asyncio.to_thread(self._stop, wait=wait)
 
-	def _stop(self):
+	def _stop(self, wait):
 		log.info("Stopping %s" % self)
 
 		# Send the stop signal
 		self.instance.stop()
 
+		# End here if we don't want to wait
+		if not wait:
+			return
+
 		log.debug("Waiting until %s has stopped" % self)
 
 		# And wait until the instance has been stopped
diff --git a/src/hub/queue.py b/src/hub/queue.py
index f2d42ce9..1a8054e2 100644
--- a/src/hub/queue.py
+++ b/src/hub/queue.py
@@ -34,6 +34,9 @@ async def dispatch_jobs(backend):
 		Will be called regularly and will dispatch any pending jobs to any
 		available builders
 	"""
+	# Perform any autoscaling action
+	await backend.builders.autoscale()
+
 	log.debug("Dispatching jobs...")
 
 	# Exit if there are no builders connected