database: Move all sorts of database operations into the main thread

We no longer support any database operations from any other thread than
the main thread (because otherwise we cannot figure out which Task we
are running in).

For that reason, we have to split a couple of tasks off at different
times and send blocking operations to an executor thread later.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/src/buildservice/aws.py b/src/buildservice/aws.py
index 039d3304845663edfd4d57183ceebb60990c0a76..77a0a9c351a8b4439c1efa6289d5480be61a6abc 100644 (file)
--- a/src/buildservice/aws.py
+++ b/src/buildservice/aws.py
@@ -26,13 +26,16 @@ from . import base
 from .decorators import *
 
 class AWS(base.Object):
+       def init(self):
+               self._session_args = {
+                       "region_name"           : self.settings.get("aws-region"),
+                       "aws_access_key_id"     : self.settings.get("aws-access-key"),
+                       "aws_secret_access_key" : self.settings.get("aws-access-secret"),
+               }
+
        @property
        def session(self):
-               return boto3.session.Session(
-                       region_name=self.settings.get("aws-region"),
-                       aws_access_key_id=self.settings.get("aws-access-key"),
-                       aws_secret_access_key=self.settings.get("aws-access-secret"),
-               )
+               return boto3.session.Session(**self._session_args)
 
        @property
        def ec2(self):

diff --git a/src/buildservice/jobs.py b/src/buildservice/jobs.py
index db0c1ebb2d6cf7f628cdc10079fba6221c492484..9db4d90cb368711df1d2b7e3159880a3ca00f3dd 100644 (file)
--- a/src/buildservice/jobs.py
+++ b/src/buildservice/jobs.py
@@ -918,7 +918,7 @@ class Job(base.DataObject):
                """
                await self.build.build_repo.installcheck([self])
 
-       def _installcheck(self, p):
+       async def _installcheck(self, p):
                """
                        Implementation that takes an active Pakfire instance and
                        performs a check on whether the source package can be installed
@@ -929,14 +929,14 @@ class Job(base.DataObject):
                repo = p.get_repo("@commandline")
 
                # Open the archive
-               archive = p.open(self.pkg.path)
+               archive = await asyncio.to_thread(p.open, self.pkg.path)
 
                # Fetch the package
                package = archive.get_package(repo)
 
                # Perform the installcheck
                try:
-                       package.installcheck()
+                       await asyncio.to_thread(package.installcheck)
 
                # Store any dependency errors
                except DependencyError as e:

diff --git a/src/buildservice/keys.py b/src/buildservice/keys.py
index 62dbe1f343a1fd7a21d8483be65a7d0b0e46ec82..c780b2958ddda8b502a60669be2e8f3a90193c17 100644 (file)
--- a/src/buildservice/keys.py
+++ b/src/buildservice/keys.py
@@ -34,13 +34,10 @@ class Keys(base.Object):
                                created_at
                """)
 
-       async def create(self, *args, **kwargs):
+       async def create(self, comment, user=None):
                """
                        Creates a new key
                """
-               return await asyncio.to_thread(self._create, *args, **kwargs)
-
-       def _create(self, comment, user=None):
                # Launch a new Pakfire instance
                with self.backend.pakfire() as p:
                        # Generate the new key

diff --git a/src/buildservice/releasemonitoring.py b/src/buildservice/releasemonitoring.py
index 5bb756650e73dae3ce71d189c94bc486c774e79b..5f213916a99268c0b463f7ac43fde95a53bf6251 100644 (file)
--- a/src/buildservice/releasemonitoring.py
+++ b/src/buildservice/releasemonitoring.py
@@ -733,8 +733,7 @@ class Release(base.DataObject):
                        # Create a new temporary space for the
                        with tempfile.TemporaryDirectory() as target:
                                # Create a new source package
-                               file = await asyncio.to_thread(
-                                       self._update_source_package, build.pkg, target)
+                               file = await self._update_source_package(build.pkg, target)
 
                                # Upload the file
                                upload = await self.backend.uploads.create_from_local(file)
@@ -761,7 +760,7 @@ class Release(base.DataObject):
                        self.build = build
                        self.repo  = repo
 
-       def _update_source_package(self, package, target):
+       async def _update_source_package(self, package, target):
                """
                        Takes a package and recreates it with this release
                """
@@ -779,19 +778,19 @@ class Release(base.DataObject):
                        # Create a Pakfire instance from this distribution
                        with self.monitoring.distro.pakfire() as p:
                                # Open the archive
-                               archive = p.open(package.path)
+                               archive = await asyncio.to_thread(p.open, package.path)
 
                                # Extract the archive into the temporary space
-                               archive.extract(path=tmp)
+                               await asyncio.to_thread(archive.extract, path=tmp)
 
                                # XXX directories are being created with the wrong permissions
                                os.system("chmod a+x -R %s" % tmp)
 
                                # Remove any downloaded files
-                               shutil.rmtree(files)
+                               await asyncio.to_thread(shutil.rmtree, files)
 
                                # Update the makefile
-                               diff = self._update_makefile(makefile)
+                               diff = await self._update_makefile(makefile)
 
                                # Log the diff
                                log.info("Generated diff:\n%s" % diff)
@@ -800,9 +799,9 @@ class Release(base.DataObject):
                                self._set_attribute("diff", diff)
 
                                # Generate a new source package
-                               return p.dist(makefile, target)
+                               return await asyncio.to_thread(p.dist, makefile, target)
 
-       def _update_makefile(self, path):
+       async def _update_makefile(self, path):
                """
                        Reads the makefile in path and updates it with the newer version
                        returning a diff between the two.

diff --git a/src/buildservice/repository.py b/src/buildservice/repository.py
index da686b9dff3302251bed2a448fe964fbc770ee25..ea36609e101aaa92662b071f6a54a37298a8b653 100644 (file)
--- a/src/buildservice/repository.py
+++ b/src/buildservice/repository.py
@@ -890,12 +890,6 @@ class Repository(base.DataObject):
                """
                        Performs an installation check for all given jobs
                """
-               return await asyncio.to_thread(self._installcheck, jobs)
-
-       def _installcheck(self, jobs):
-               """
-                       Implementation of installcheck which is performed on all given jobs.
-               """
                # We create a new context which allows us to enter many (but unknown) numbers of
                # Pakfire instances at a time which leads to Pakfire only being initialized once
                # per architecture. We are then able to run all dependency checks one after the
@@ -916,7 +910,7 @@ class Repository(base.DataObject):
 
                                # Perform the check for the job
                                with self.db.transaction():
-                                       if job._installcheck(p):
+                                       if await job._installcheck(p):
                                                success = True
 
                        return success