From: Michael Tremer <michael.tremer@ipfire.org>
Date: Mon, 20 Jun 2022 16:57:40 +0000 (+0000)
Subject: builders: Implement starting new builders as needed
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=ea2d036d9e07cc0f20b0426595862045f640b5d5;p=pbs.git

builders: Implement starting new builders as needed

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---

diff --git a/src/buildservice/builders.py b/src/buildservice/builders.py
index b5557d87..d4ef32ce 100644
--- a/src/buildservice/builders.py
+++ b/src/buildservice/builders.py
@@ -166,6 +166,56 @@ class Builders(base.Object):
 
 		# XXX implement starting builders if needed
 
+		await self._autoscale_start()
+
+	async def _autoscale_start(self, wait=False):
+		# XXX max queue length
+		threshold = datetime.timedelta(minutes=5)
+
+		# Fetch all enabled builders and whether they are running or not
+		builders = {
+			builder : await builder.is_running() for builder in self if builder.enabled
+		}
+
+		# Sort all running builders to the beginning of the list
+		builders = sorted(builders, key=lambda b: (builders[b], b))
+
+		# Store the length of the queue for each builder
+		queue = {
+			builder : datetime.timedelta(0) for builder in builders
+		}
+
+		# Run through all build jobs and allocate them to a builder.
+		# If a builder is full (i.e. reaches the threshold of its build time),
+		# we move on to the next builder until that is full and so on.
+		for job in self.backend.jobqueue:
+			log.debug("Processing job %s..." % job)
+
+			for builder in builders:
+				# Skip disabled builders
+				if not builder.enabled:
+					continue
+
+				# Check if this builder is already at capacity
+				if queue[builder] / builder.max_jobs >= threshold:
+					log.debug("Builder %s is already full" % builder)
+					continue
+
+				# Skip if this builder cannot build this job
+				if not builder.can_build(job):
+					continue
+
+				log.debug("Builder %s can build %s" % (builder, job))
+
+				# Add the job to the total build time
+				queue[builder] += job.estimated_build_time
+				break
+
+		# Start all builders that have been allocated at least one job
+		await asyncio.gather(
+			*(builder.start(wait=wait) for builder in builders if queue[builder]),
+		)
+
 
 class Builder(base.DataObject):
 	table = "builders"
@@ -176,6 +226,9 @@ class Builder(base.DataObject):
 
 		return NotImplemented
 
+	def __repr__(self):
+		return "<%s %s>" % (self.__class__.__name__, self.hostname)
+
 	def __str__(self):
 		return self.hostname
 
@@ -374,6 +427,9 @@ class Builder(base.DataObject):
 
 		return sorted(arches)
 
+	def can_build(self, job):
+		return job.arch in self.supported_arches
+
 	def set_testmode(self, testmode):
 		self._set_attribute("testmode", testmode)
 
@@ -538,13 +594,13 @@ class Builder(base.DataObject):
 		if self.instance:
 			return self.instance.state.get("Name")
 
-	async def start(self):
+	async def start(self, wait=True):
 		"""
 			Starts the instance on AWS
 		"""
-		await asyncio.to_thread(self._start)
+		await asyncio.to_thread(self._start, wait=wait)
 
-	def _start(self):
+	def _start(self, wait):
 		log.info("Starting %s" % self)
 
 		# Set correct instance type
@@ -553,6 +609,10 @@ class Builder(base.DataObject):
 		# Send the start signal
 		self.instance.start()
 
+		# End here if we don't want to wait
+		if not wait:
+			return
+
 		log.debug("Waiting until %s has started" % self)
 
 		# And wait until the instance is running
diff --git a/src/buildservice/jobs.py b/src/buildservice/jobs.py
index a6785463..6b8cb813 100644
--- a/src/buildservice/jobs.py
+++ b/src/buildservice/jobs.py
@@ -197,6 +197,26 @@ class Job(base.DataObject):
 		if res:
 			return res.rank
 
+	@lazy_property
+	def estimated_build_time(self):
+		"""
+			Returns the time we expect this job to run for
+		"""
+		res = self.db.get("""
+			SELECT
+				build_time
+			FROM
+				package_estimated_build_times
+			WHERE
+				name = %s
+			AND
+				arch = %s""",
+			self.pkg.name, self.arch,
+		)
+
+		if res:
+			return res.build_time
+
 	@property
 	def distro(self):
 		return self.build.distro
diff --git a/src/database.sql b/src/database.sql
index 09ede674..600b521d 100644
--- a/src/database.sql
+++ b/src/database.sql
@@ -991,6 +991,30 @@ CREATE TABLE public.packages (
 
 ALTER TABLE public.packages OWNER TO pakfire;
 
+--
+-- Name: package_estimated_build_times; Type: VIEW; Schema: public; Owner: pakfire
+--
+
+CREATE VIEW public.package_estimated_build_times AS
+ SELECT packages.name,
+    jobs.arch,
+    avg((jobs.time_finished - jobs.time_started)) AS build_time
+   FROM ((public.jobs
+     LEFT JOIN public.builds ON ((jobs.build_id = builds.id)))
+     LEFT JOIN public.packages ON ((builds.pkg_id = packages.id)))
+  WHERE ((jobs.state = 'finished'::text) AND (jobs.test IS FALSE) AND (jobs.time_started IS NOT NULL) AND (jobs.time_finished IS NOT NULL))
+  GROUP BY packages.name, jobs.arch;
+
+
+ALTER TABLE public.package_estimated_build_times OWNER TO pakfire;
+
+--
+-- Name: VIEW package_estimated_build_times; Type: COMMENT; Schema: public; Owner: pakfire
+--
+
+COMMENT ON VIEW public.package_estimated_build_times IS 'Should add this later: AND jobs.time_finished >= (CURRENT_TIMESTAMP - ''180 days''::interval)';
+
+
 --
 -- Name: packages_deps; Type: TABLE; Schema: public; Owner: pakfire
 --