From: Michael Tremer Date: Fri, 21 Jul 2023 13:31:31 +0000 (+0000) Subject: database: Move all sorts of database operations into the main thread X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=a1a0576dfb3c8b5e8d6cb339a2b4555695b455b5;p=pbs.git database: Move all sorts of database operations into the main thread We no longer support any database operations from any other thread than the main thread (because otherwise we cannot figure out which Task we are running in). For that reason, we have to split a couple of tasks off at different times and send blocking operations to an executor thread later. Signed-off-by: Michael Tremer --- diff --git a/src/buildservice/aws.py b/src/buildservice/aws.py index 039d3304..77a0a9c3 100644 --- a/src/buildservice/aws.py +++ b/src/buildservice/aws.py @@ -26,13 +26,16 @@ from . import base from .decorators import * class AWS(base.Object): + def init(self): + self._session_args = { + "region_name" : self.settings.get("aws-region"), + "aws_access_key_id" : self.settings.get("aws-access-key"), + "aws_secret_access_key" : self.settings.get("aws-access-secret"), + } + @property def session(self): - return boto3.session.Session( - region_name=self.settings.get("aws-region"), - aws_access_key_id=self.settings.get("aws-access-key"), - aws_secret_access_key=self.settings.get("aws-access-secret"), - ) + return boto3.session.Session(**self._session_args) @property def ec2(self): diff --git a/src/buildservice/jobs.py b/src/buildservice/jobs.py index db0c1ebb..9db4d90c 100644 --- a/src/buildservice/jobs.py +++ b/src/buildservice/jobs.py @@ -918,7 +918,7 @@ class Job(base.DataObject): """ await self.build.build_repo.installcheck([self]) - def _installcheck(self, p): + async def _installcheck(self, p): """ Implementation that takes an active Pakfire instance and performs a check on whether the source package can be installed @@ -929,14 +929,14 @@ class Job(base.DataObject): repo = p.get_repo("@commandline") # Open the archive - archive = p.open(self.pkg.path) + archive = await asyncio.to_thread(p.open, self.pkg.path) # Fetch the package package = archive.get_package(repo) # Perform the installcheck try: - package.installcheck() + await asyncio.to_thread(package.installcheck) # Store any dependency errors except DependencyError as e: diff --git a/src/buildservice/keys.py b/src/buildservice/keys.py index 62dbe1f3..c780b295 100644 --- a/src/buildservice/keys.py +++ b/src/buildservice/keys.py @@ -34,13 +34,10 @@ class Keys(base.Object): created_at """) - async def create(self, *args, **kwargs): + async def create(self, comment, user=None): """ Creates a new key """ - return await asyncio.to_thread(self._create, *args, **kwargs) - - def _create(self, comment, user=None): # Launch a new Pakfire instance with self.backend.pakfire() as p: # Generate the new key diff --git a/src/buildservice/releasemonitoring.py b/src/buildservice/releasemonitoring.py index 5bb75665..5f213916 100644 --- a/src/buildservice/releasemonitoring.py +++ b/src/buildservice/releasemonitoring.py @@ -733,8 +733,7 @@ class Release(base.DataObject): # Create a new temporary space for the with tempfile.TemporaryDirectory() as target: # Create a new source package - file = await asyncio.to_thread( - self._update_source_package, build.pkg, target) + file = await self._update_source_package(build.pkg, target) # Upload the file upload = await self.backend.uploads.create_from_local(file) @@ -761,7 +760,7 @@ class Release(base.DataObject): self.build = build self.repo = repo - def _update_source_package(self, package, target): + async def _update_source_package(self, package, target): """ Takes a package and recreates it with this release """ @@ -779,19 +778,19 @@ class Release(base.DataObject): # Create a Pakfire instance from this distribution with self.monitoring.distro.pakfire() as p: # Open the archive - archive = p.open(package.path) + archive = await asyncio.to_thread(p.open, package.path) # Extract the archive into the temporary space - archive.extract(path=tmp) + await asyncio.to_thread(archive.extract, path=tmp) # XXX directories are being created with the wrong permissions os.system("chmod a+x -R %s" % tmp) # Remove any downloaded files - shutil.rmtree(files) + await asyncio.to_thread(shutil.rmtree, files) # Update the makefile - diff = self._update_makefile(makefile) + diff = await self._update_makefile(makefile) # Log the diff log.info("Generated diff:\n%s" % diff) @@ -800,9 +799,9 @@ class Release(base.DataObject): self._set_attribute("diff", diff) # Generate a new source package - return p.dist(makefile, target) + return await asyncio.to_thread(p.dist, makefile, target) - def _update_makefile(self, path): + async def _update_makefile(self, path): """ Reads the makefile in path and updates it with the newer version returning a diff between the two. diff --git a/src/buildservice/repository.py b/src/buildservice/repository.py index da686b9d..ea36609e 100644 --- a/src/buildservice/repository.py +++ b/src/buildservice/repository.py @@ -890,12 +890,6 @@ class Repository(base.DataObject): """ Performs an installation check for all given jobs """ - return await asyncio.to_thread(self._installcheck, jobs) - - def _installcheck(self, jobs): - """ - Implementation of installcheck which is performed on all given jobs. - """ # We create a new context which allows us to enter many (but unknown) numbers of # Pakfire instances at a time which leads to Pakfire only being initialized once # per architecture. We are then able to run all dependency checks one after the @@ -916,7 +910,7 @@ class Repository(base.DataObject): # Perform the check for the job with self.db.transaction(): - if job._installcheck(p): + if await job._installcheck(p): success = True return success