builders: Implement automatic shutdown of builders

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/src/buildservice/builders.py b/src/buildservice/builders.py
index da8cf16c505cad2a55c5a177290a0a3f2288b2ea..b5557d87118c76c3d52c0efdf73744a8802e5192 100644 (file)
--- a/src/buildservice/builders.py
+++ b/src/buildservice/builders.py
@@ -125,6 +125,47 @@ class Builders(base.Object):
                # Sync all builders
                await asyncio.gather(*(builder.sync() for builder in self))
 
+       async def autoscale(self, wait=False):
+               """
+                       This method performs two tasks:
+
+                       * It will launch any new builders if more are required
+
+                       * It will shutdown any builders which are no longer required
+               """
+               log.debug("Running autoscaling schedulder")
+
+               builders_to_be_shut_down = []
+
+               # Check if there are any builders which can be shut down
+               for builder in self:
+                       # Don't consider builders which are not running
+                       if not await builder.is_running():
+                               log.debug("Won't consider %s which is not running" % builder)
+                               continue
+
+                       # Check if there are any jobs
+                       if len(builder.jobs) > 0:
+                               log.debug("Won't shut down %s which has active jobs" % builder)
+                               continue
+
+                       # Check if there any jobs in the queue that are suitable for this builder
+                       job = self.backend.jobqueue.pop(builder)
+                       if job:
+                               log.debug("Won't shut down %s which has pending jobs" % builder)
+                               continue
+
+                       # Shutting down this builder
+                       builders_to_be_shut_down.append(builder)
+
+               # Perform shutdowns
+               if builders_to_be_shut_down:
+                       await asyncio.gather(
+                               *(builder.stop(wait=wait) for builder in builders_to_be_shut_down),
+                       )
+
+               # XXX implement starting builders if needed
+
 
 class Builder(base.DataObject):
        table = "builders"
@@ -466,6 +507,37 @@ class Builder(base.DataObject):
                # Launch in a separate thread
                await asyncio.to_thread(callback)
 
+       async def is_running(self):
+               """
+                       Returns True if this builder is currently running
+               """
+               state = await asyncio.to_thread(self._fetch_state)
+
+               return state in ("pending", "running")
+
+       async def is_shutdown(self):
+               """
+                       Returns True if this builder is shut down
+               """
+               state = await asyncio.to_thread(self._fetch_state)
+
+               return state == "stopped"
+
+       async def is_shutting_down(self):
+               """
+                       Returns True if this builder is currently shutting down
+               """
+               state = await asyncio.to_thread(self._fetch_state)
+
+               return state in ("shutting-down", "stopping")
+
+       def _fetch_state(self):
+               """
+                       Returns the current state of this instance
+               """
+               if self.instance:
+                       return self.instance.state.get("Name")
+
        async def start(self):
                """
                        Starts the instance on AWS
@@ -514,18 +586,22 @@ class Builder(base.DataObject):
                except botocore.exceptions.ClientError as e:
                        log.warning("Could not change instance type of %s: %s" % (self, e))
 
-       async def stop(self):
+       async def stop(self, wait=True):
                """
                        Stops this instance on AWS
                """
-               await asyncio.to_thread(self._stop)
+               await asyncio.to_thread(self._stop, wait=wait)
 
-       def _stop(self):
+       def _stop(self, wait):
                log.info("Stopping %s" % self)
 
                # Send the stop signal
                self.instance.stop()
 
+               # End here if we don't want to wait
+               if not wait:
+                       return
+
                log.debug("Waiting until %s has stopped" % self)
 
                # And wait until the instance has been stopped

diff --git a/src/hub/queue.py b/src/hub/queue.py
index f2d42ce90c4504e1d8aa068b5abd68b317454cc0..1a8054e2a806de6f0db73e276725caaf029564df 100644 (file)
--- a/src/hub/queue.py
+++ b/src/hub/queue.py
@@ -34,6 +34,9 @@ async def dispatch_jobs(backend):
                Will be called regularly and will dispatch any pending jobs to any
                available builders
        """
+       # Perform any autoscaling action
+       await backend.builders.autoscale()
+
        log.debug("Dispatching jobs...")
 
        # Exit if there are no builders connected