From: Michael Tremer Date: Fri, 17 Jun 2022 15:28:17 +0000 (+0000) Subject: builders: Add some basic integration with AWS X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=316268ce839a0eac69c48f949ca773bc9916fd89;p=pbs.git builders: Add some basic integration with AWS Signed-off-by: Michael Tremer --- diff --git a/Makefile.am b/Makefile.am index 2be1ef5d..740f61f2 100644 --- a/Makefile.am +++ b/Makefile.am @@ -76,6 +76,7 @@ buildservice_PYTHON = \ src/buildservice/__init__.py \ src/buildservice/__version__.py \ src/buildservice/arches.py \ + src/buildservice/aws.py \ src/buildservice/base.py \ src/buildservice/bugtracker.py \ src/buildservice/builders.py \ diff --git a/src/buildservice/__init__.py b/src/buildservice/__init__.py index 7212c217..7abb5f2d 100644 --- a/src/buildservice/__init__.py +++ b/src/buildservice/__init__.py @@ -8,6 +8,7 @@ import os import pakfire from . import arches +from . import aws from . import bugtracker from . import builders from . import builds @@ -49,6 +50,7 @@ class Backend(object): self.settings = settings.Settings(self) self.arches = arches.Arches(self) + self.aws = aws.AWS(self) self.builds = builds.Builds(self) self.cache = cache.Cache(self) self.jobs = jobs.Jobs(self) diff --git a/src/buildservice/aws.py b/src/buildservice/aws.py new file mode 100644 index 00000000..039d3304 --- /dev/null +++ b/src/buildservice/aws.py @@ -0,0 +1,39 @@ +#!/usr/bin/python3 +############################################################################### +# # +# Pakfire - The IPFire package management system # +# Copyright (C) 2022 Pakfire development team # +# # +# This program is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# This program is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with this program. If not, see . # +# # +############################################################################### + +import boto3 +import botocore.config + +from . import base +from .decorators import * + +class AWS(base.Object): + @property + def session(self): + return boto3.session.Session( + region_name=self.settings.get("aws-region"), + aws_access_key_id=self.settings.get("aws-access-key"), + aws_secret_access_key=self.settings.get("aws-access-secret"), + ) + + @property + def ec2(self): + return self.session.resource("ec2") diff --git a/src/buildservice/builders.py b/src/buildservice/builders.py index 8cf52869..0fb0a8d4 100644 --- a/src/buildservice/builders.py +++ b/src/buildservice/builders.py @@ -1,5 +1,6 @@ #!/usr/bin/python +import asyncio import datetime import hashlib import logging @@ -13,6 +14,8 @@ from . import misc from .decorators import * +log = logging.getLogger("pakfire.builders") + ACTIVE_STATES = [ "dispatching", "running", @@ -112,6 +115,15 @@ class Builders(base.Object): return entries + async def sync(self, *args, **kwargs): + """ + Synchronize any state with AWS + """ + log.info("Syncing state with AWS") + + # Sync all builders + await asyncio.gather(*(builder.sync() for builder in self)) + class Builder(base.DataObject): table = "builders" @@ -426,6 +438,32 @@ class Builder(base.DataObject): # Looks like we are ready return True + # AWS + + @property + def instance_id(self): + return self.data.instance_id + + @lazy_property + def instance(self): + if self.instance_id: + return self.backend.aws.ec2.Instance(self.instance_id) + + async def sync(self): + log.info("Syncing AWS state for %s" % self) + + if not self.instance: + log.debug("%s does not have an instance ID" % self) + return + + # This callback is being executed in a separate thread + # because boto3 is not thread-safe + def callback(): + log.debug("%s is currently in state: %s" % (self, self.instance.state)) + + # Launch in a separate thread + return await asyncio.to_thread(callback) + def generate_password_hash(password, salt=None, algo="sha512"): """ diff --git a/src/database.sql b/src/database.sql index 5183df06..d079b476 100644 --- a/src/database.sql +++ b/src/database.sql @@ -129,7 +129,8 @@ CREATE TABLE public.builders ( updated_at timestamp without time zone, time_keepalive timestamp without time zone, online_until timestamp without time zone, - cpu_arch text + cpu_arch text, + instance_id text ); diff --git a/src/hub/__init__.py b/src/hub/__init__.py index 95ed5f03..e99d91de 100644 --- a/src/hub/__init__.py +++ b/src/hub/__init__.py @@ -62,15 +62,21 @@ class Application(tornado.web.Application): logging.info("Successfully initialied application") - def _run_task(self, callback, t): + # Perform some initial tasks + self._run_task(self.backend.builders.sync) + + def _run_task(self, callback, t=None): """ - Runs the callback every t seconds in the background + Runs the callback every t seconds in the background or once if t is None """ # Pass backend to the function callback = functools.partial(callback, self.backend) # Create a periodic callback object - task = tornado.ioloop.PeriodicCallback(callback, t * 1000) + if t: + task = tornado.ioloop.PeriodicCallback(callback, t * 1000) + task.start() - # Start the task - task.start() + else: + ioloop = tornado.ioloop.IOLoop.current() + ioloop.add_callback(callback)