builders: Implement starting new builders as needed

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/src/buildservice/builders.py b/src/buildservice/builders.py
index b5557d87118c76c3d52c0efdf73744a8802e5192..d4ef32ceb56e78a0c3276afb1c906cb52a0b15c9 100644 (file)
--- a/src/buildservice/builders.py
+++ b/src/buildservice/builders.py
@@ -166,6 +166,56 @@ class Builders(base.Object):
 
                # XXX implement starting builders if needed
 
+               await self._autoscale_start()
+
+       async def _autoscale_start(self, wait=False):
+               # XXX max queue length
+               threshold = datetime.timedelta(minutes=5)
+
+               # Fetch all enabled builders and whether they are running or not
+               builders = {
+                       builder : await builder.is_running() for builder in self if builder.enabled
+               }
+
+               # Sort all running builders to the beginning of the list
+               builders = sorted(builders, key=lambda b: (builders[b], b))
+
+               # Store the length of the queue for each builder
+               queue = {
+                       builder : datetime.timedelta(0) for builder in builders
+               }
+
+               # Run through all build jobs and allocate them to a builder.
+               # If a builder is full (i.e. reaches the threshold of its build time),
+               # we move on to the next builder until that is full and so on.
+               for job in self.backend.jobqueue:
+                       log.debug("Processing job %s..." % job)
+
+                       for builder in builders:
+                               # Skip disabled builders
+                               if not builder.enabled:
+                                       continue
+
+                               # Check if this builder is already at capacity
+                               if queue[builder] / builder.max_jobs >= threshold:
+                                       log.debug("Builder %s is already full" % builder)
+                                       continue
+
+                               # Skip if this builder cannot build this job
+                               if not builder.can_build(job):
+                                       continue
+
+                               log.debug("Builder %s can build %s" % (builder, job))
+
+                               # Add the job to the total build time
+                               queue[builder] += job.estimated_build_time
+                               break
+
+               # Start all builders that have been allocated at least one job
+               await asyncio.gather(
+                       *(builder.start(wait=wait) for builder in builders if queue[builder]),
+               )
+
 
 class Builder(base.DataObject):
        table = "builders"
@@ -176,6 +226,9 @@ class Builder(base.DataObject):
 
                return NotImplemented
 
+       def __repr__(self):
+               return "<%s %s>" % (self.__class__.__name__, self.hostname)
+
        def __str__(self):
                return self.hostname
 
@@ -374,6 +427,9 @@ class Builder(base.DataObject):
 
                return sorted(arches)
 
+       def can_build(self, job):
+               return job.arch in self.supported_arches
+
        def set_testmode(self, testmode):
                self._set_attribute("testmode", testmode)
 
@@ -538,13 +594,13 @@ class Builder(base.DataObject):
                if self.instance:
                        return self.instance.state.get("Name")
 
-       async def start(self):
+       async def start(self, wait=True):
                """
                        Starts the instance on AWS
                """
-               await asyncio.to_thread(self._start)
+               await asyncio.to_thread(self._start, wait=wait)
 
-       def _start(self):
+       def _start(self, wait):
                log.info("Starting %s" % self)
 
                # Set correct instance type
@@ -553,6 +609,10 @@ class Builder(base.DataObject):
                # Send the start signal
                self.instance.start()
 
+               # End here if we don't want to wait
+               if not wait:
+                       return
+
                log.debug("Waiting until %s has started" % self)
 
                # And wait until the instance is running

diff --git a/src/buildservice/jobs.py b/src/buildservice/jobs.py
index a678546330caea0a6941605eca4826dbbc808267..6b8cb813c65efc0ce6b3605309f0970a05e6d978 100644 (file)
--- a/src/buildservice/jobs.py
+++ b/src/buildservice/jobs.py
@@ -197,6 +197,26 @@ class Job(base.DataObject):
                if res:
                        return res.rank
 
+       @lazy_property
+       def estimated_build_time(self):
+               """
+                       Returns the time we expect this job to run for
+               """
+               res = self.db.get("""
+                       SELECT
+                               build_time
+                       FROM
+                               package_estimated_build_times
+                       WHERE
+                               name = %s
+                       AND
+                               arch = %s""",
+                       self.pkg.name, self.arch,
+               )
+
+               if res:
+                       return res.build_time
+
        @property
        def distro(self):
                return self.build.distro

diff --git a/src/database.sql b/src/database.sql
index 09ede6746daf806842ca5916f268795037705153..600b521da0c69d48abcf7b3f7c8eca50a685a7b6 100644 (file)
--- a/src/database.sql
+++ b/src/database.sql
@@ -991,6 +991,30 @@ CREATE TABLE public.packages (
 
 ALTER TABLE public.packages OWNER TO pakfire;
 
+--
+-- Name: package_estimated_build_times; Type: VIEW; Schema: public; Owner: pakfire
+--
+
+CREATE VIEW public.package_estimated_build_times AS
+ SELECT packages.name,
+    jobs.arch,
+    avg((jobs.time_finished - jobs.time_started)) AS build_time
+   FROM ((public.jobs
+     LEFT JOIN public.builds ON ((jobs.build_id = builds.id)))
+     LEFT JOIN public.packages ON ((builds.pkg_id = packages.id)))
+  WHERE ((jobs.state = 'finished'::text) AND (jobs.test IS FALSE) AND (jobs.time_started IS NOT NULL) AND (jobs.time_finished IS NOT NULL))
+  GROUP BY packages.name, jobs.arch;
+
+
+ALTER TABLE public.package_estimated_build_times OWNER TO pakfire;
+
+--
+-- Name: VIEW package_estimated_build_times; Type: COMMENT; Schema: public; Owner: pakfire
+--
+
+COMMENT ON VIEW public.package_estimated_build_times IS 'Should add this later: AND jobs.time_finished >= (CURRENT_TIMESTAMP - ''180 days''::interval)';
+
+
 --
 -- Name: packages_deps; Type: TABLE; Schema: public; Owner: pakfire
 --