builders: Add some basic integration with AWS

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/Makefile.am b/Makefile.am
index 2be1ef5dc67b16485ff4a813aead5c41cec9aae8..740f61f2c6bddcd2a67d86bd2b2e76f1a1e92ee2 100644 (file)
--- a/Makefile.am
+++ b/Makefile.am
@@ -76,6 +76,7 @@ buildservice_PYTHON = \
        src/buildservice/__init__.py \
        src/buildservice/__version__.py \
        src/buildservice/arches.py \
+       src/buildservice/aws.py \
        src/buildservice/base.py \
        src/buildservice/bugtracker.py \
        src/buildservice/builders.py \

diff --git a/src/buildservice/__init__.py b/src/buildservice/__init__.py
index 7212c217812baeedb26f0d421cc8ff16f5033d1b..7abb5f2d4daa46daf6a2f572172f054d73ba41b6 100644 (file)
--- a/src/buildservice/__init__.py
+++ b/src/buildservice/__init__.py
@@ -8,6 +8,7 @@ import os
 import pakfire
 
 from . import arches
+from . import aws
 from . import bugtracker
 from . import builders
 from . import builds
@@ -49,6 +50,7 @@ class Backend(object):
                self.settings = settings.Settings(self)
 
                self.arches      = arches.Arches(self)
+               self.aws         = aws.AWS(self)
                self.builds      = builds.Builds(self)
                self.cache       = cache.Cache(self)
                self.jobs        = jobs.Jobs(self)

diff --git a/src/buildservice/aws.py b/src/buildservice/aws.py
new file mode 100644 (file)
index 0000000..039d330
--- /dev/null
+++ b/src/buildservice/aws.py
@@ -0,0 +1,39 @@
+#!/usr/bin/python3
+###############################################################################
+#                                                                             #
+# Pakfire - The IPFire package management system                              #
+# Copyright (C) 2022 Pakfire development team                                 #
+#                                                                             #
+# This program is free software: you can redistribute it and/or modify        #
+# it under the terms of the GNU General Public License as published by        #
+# the Free Software Foundation, either version 3 of the License, or           #
+# (at your option) any later version.                                         #
+#                                                                             #
+# This program is distributed in the hope that it will be useful,             #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of              #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
+# GNU General Public License for more details.                                #
+#                                                                             #
+# You should have received a copy of the GNU General Public License           #
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
+#                                                                             #
+###############################################################################
+
+import boto3
+import botocore.config
+
+from . import base
+from .decorators import *
+
+class AWS(base.Object):
+       @property
+       def session(self):
+               return boto3.session.Session(
+                       region_name=self.settings.get("aws-region"),
+                       aws_access_key_id=self.settings.get("aws-access-key"),
+                       aws_secret_access_key=self.settings.get("aws-access-secret"),
+               )
+
+       @property
+       def ec2(self):
+               return self.session.resource("ec2")

diff --git a/src/buildservice/builders.py b/src/buildservice/builders.py
index 8cf5286942fab3cac06dbbb7af7c6e288de3d050..0fb0a8d4d666fb7aedf1d35dc24d0adddcd87765 100644 (file)
--- a/src/buildservice/builders.py
+++ b/src/buildservice/builders.py
@@ -1,5 +1,6 @@
 #!/usr/bin/python
 
+import asyncio
 import datetime
 import hashlib
 import logging
@@ -13,6 +14,8 @@ from . import misc
 
 from .decorators import *
 
+log = logging.getLogger("pakfire.builders")
+
 ACTIVE_STATES = [
        "dispatching",
        "running",
@@ -112,6 +115,15 @@ class Builders(base.Object):
 
                return entries
 
+       async def sync(self, *args, **kwargs):
+               """
+                       Synchronize any state with AWS
+               """
+               log.info("Syncing state with AWS")
+
+               # Sync all builders
+               await asyncio.gather(*(builder.sync() for builder in self))
+
 
 class Builder(base.DataObject):
        table = "builders"
@@ -426,6 +438,32 @@ class Builder(base.DataObject):
                # Looks like we are ready
                return True
 
+       # AWS
+
+       @property
+       def instance_id(self):
+               return self.data.instance_id
+
+       @lazy_property
+       def instance(self):
+               if self.instance_id:
+                       return self.backend.aws.ec2.Instance(self.instance_id)
+
+       async def sync(self):
+               log.info("Syncing AWS state for %s" % self)
+
+               if not self.instance:
+                       log.debug("%s does not have an instance ID" % self)
+                       return
+
+               # This callback is being executed in a separate thread
+               # because boto3 is not thread-safe
+               def callback():
+                       log.debug("%s is currently in state: %s" % (self, self.instance.state))
+
+               # Launch in a separate thread
+               return await asyncio.to_thread(callback)
+
 
 def generate_password_hash(password, salt=None, algo="sha512"):
        """

diff --git a/src/database.sql b/src/database.sql
index 5183df06c1ae7982c10b219e8211fca05e77dfa4..d079b476fa488ffc057f718b3e1db1230eaf95b3 100644 (file)
--- a/src/database.sql
+++ b/src/database.sql
@@ -129,7 +129,8 @@ CREATE TABLE public.builders (
     updated_at timestamp without time zone,
     time_keepalive timestamp without time zone,
     online_until timestamp without time zone,
-    cpu_arch text
+    cpu_arch text,
+    instance_id text
 );
 
 

diff --git a/src/hub/__init__.py b/src/hub/__init__.py
index 95ed5f0320aecbb87ce888a0bcac428ce02231d2..e99d91de329bff4fee37837d950aab12fa1e6ee0 100644 (file)
--- a/src/hub/__init__.py
+++ b/src/hub/__init__.py
@@ -62,15 +62,21 @@ class Application(tornado.web.Application):
 
                logging.info("Successfully initialied application")
 
-       def _run_task(self, callback, t):
+               # Perform some initial tasks
+               self._run_task(self.backend.builders.sync)
+
+       def _run_task(self, callback, t=None):
                """
-                       Runs the callback every t seconds in the background
+                       Runs the callback every t seconds in the background or once if t is None
                """
                # Pass backend to the function
                callback = functools.partial(callback, self.backend)
 
                # Create a periodic callback object
-               task = tornado.ioloop.PeriodicCallback(callback, t * 1000)
+               if t:
+                       task = tornado.ioloop.PeriodicCallback(callback, t * 1000)
+                       task.start()
 
-               # Start the task
-               task.start()
+               else:
+                       ioloop = tornado.ioloop.IOLoop.current()
+                       ioloop.add_callback(callback)