From: Michael Tremer Date: Fri, 15 Jul 2022 12:00:19 +0000 (+0000) Subject: jobs: Refactor dependency check X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=6a98844023654fce6ae2956ad27e1ca10107de1d;p=pbs.git jobs: Refactor dependency check Signed-off-by: Michael Tremer --- diff --git a/src/buildservice/jobs.py b/src/buildservice/jobs.py index 54f1e74b..76faa03a 100644 --- a/src/buildservice/jobs.py +++ b/src/buildservice/jobs.py @@ -1,5 +1,6 @@ #!/usr/bin/python +import asyncio import datetime import hashlib import logging @@ -85,6 +86,12 @@ class Jobs(base.Object): return list(jobs) + async def depcheck(self, jobs=None): + """ + Performs a dependency check on all given jobs concurrently + """ + await asyncio.gather(*(job.depcheck() for job in jobs)) + class Job(base.DataObject): table = "jobs" @@ -468,47 +475,55 @@ class Job(base.DataObject): self.backend.messages.send_template_to_many(self.message_recipients, "messages/jobs/failed", job=self) + def pakfire(self, **kwargs): + """ + Returns a local Pakfire instance with the configuration for this job + """ + config = self.get_config(local=True) + + return self.backend.pakfire(config=config, **kwargs) + def get_config(self, local=False): """ Get configuration file that is sent to the builder. """ return self.distro.get_config(local=local, repo=self.build.build_repo) - def set_dependency_check_succeeded(self, value): - self._set_attribute("dependency_check_succeeded", value) - self._set_attribute("dependency_check_at", datetime.datetime.utcnow()) + async def depcheck(self): + """ + Perform dependency check + """ + log.info("Performing dependency check for %s" % self) - # Reset the message - if value is True: - self.message = None + with self.db.transaction(): + return await asyncio.to_thread(self._depcheck) - dependency_check_succeeded = property( - lambda s: s.data.dependency_check_succeeded, - set_dependency_check_succeeded) + def _depcheck(self): + # Create a Pakfire instance + p = self.pakfire() - def resolvdep(self): - log.info("Processing dependencies for %s..." % self) + # Try to install the source package + try: + p.install([self.pkg.path], dryrun=True) - config = pakfire.config.Config(files=["general.conf"]) - config.parse(self.get_config(local=True)) + # XXX Pakfire should throw a better exception + except Exception as e: + self._set_attribute("depcheck_succeeded", False) - # The filename of the source file. - filename = os.path.join(PACKAGES_DIR, self.build.pkg.path) - assert os.path.exists(filename), filename + # Store the message + self._set_attribute("message", "%s" % e) - # Create a new pakfire instance with the configuration for - # this build. - p = pakfire.PakfireServer(config=config, arch=self.arch) + # Everything OK + else: + self._set_attribute("depcheck_succeeded", True) - # Try to solve the build dependencies. - try: - solver = p.resolvdep(filename) + # Store the timestamp + self._set_attribute_now("depcheck_performed_at") - # Catch dependency errors and log the problem string. - except DependencyError as e: - self.dependency_check_succeeded = False - self.message = e + @property + def depcheck_succeeded(self): + return self.data.depcheck_succeeded - # The dependency check has succeeded - else: - self.dependency_check_succeeded = True + @property + def depcheck_performed_at(self): + return self.data.depcheck_performed_at diff --git a/src/database.sql b/src/database.sql index 2ba7c9fd..6d5d2a44 100644 --- a/src/database.sql +++ b/src/database.sql @@ -481,8 +481,8 @@ CREATE TABLE public.jobs ( message text, test boolean DEFAULT true NOT NULL, superseeded_by integer, - dependency_check_succeeded boolean, - dependency_check_at timestamp without time zone, + depcheck_succeeded boolean, + depcheck_performed_at timestamp without time zone, deleted boolean DEFAULT false NOT NULL, failed boolean DEFAULT false NOT NULL, log_path text, @@ -503,7 +503,7 @@ CREATE VIEW public.job_queue AS jobs.arch FROM (public.jobs LEFT JOIN public.builds ON ((jobs.build_id = builds.id))) - WHERE ((jobs.deleted IS FALSE) AND (jobs.started_at IS NULL) AND (jobs.finished_at IS NULL) AND (jobs.dependency_check_succeeded IS TRUE)); + WHERE ((jobs.deleted IS FALSE) AND (jobs.started_at IS NULL) AND (jobs.finished_at IS NULL) AND (jobs.depcheck_succeeded IS TRUE)); ALTER TABLE public.job_queue OWNER TO pakfire; @@ -2121,7 +2121,7 @@ CREATE INDEX jobs_finished_at ON public.jobs USING btree (finished_at DESC) WHER -- Name: jobs_pending; Type: INDEX; Schema: public; Owner: pakfire -- -CREATE INDEX jobs_pending ON public.jobs USING btree (id) WHERE ((deleted IS FALSE) AND (started_at IS NULL) AND (finished_at IS NULL) AND (dependency_check_succeeded IS TRUE)); +CREATE INDEX jobs_pending ON public.jobs USING btree (id) WHERE ((deleted IS FALSE) AND (started_at IS NULL) AND (finished_at IS NULL) AND (depcheck_succeeded IS TRUE)); -- diff --git a/src/scripts/pakfire-build-service b/src/scripts/pakfire-build-service index 822969fc..b7ec2863 100644 --- a/src/scripts/pakfire-build-service +++ b/src/scripts/pakfire-build-service @@ -21,6 +21,9 @@ class Cli(object): # Cleanup "cleanup" : self.backend.cleanup, + # Jobs + "jobs:depcheck" : self._jobs_depcheck, + # Repositories "repos:sync" : self.backend.repos.sync, "repos:write" : self.backend.repos.write, @@ -71,6 +74,24 @@ class Cli(object): # Exit with error code sys.exit(r or 0) + async def _jobs_depcheck(self, *uuids): + """ + Test command to trigger a dependency check + """ + jobs = [] + + # Collect jobs by UUID + for uuid in uuids: + job = self.backend.jobs.get_by_uuid(uuid) + if not job: + log.error("Could not find job '%s'" % uuid) + continue + + jobs.append(job) + + # Run the checks + await self.backend.jobs.depcheck(jobs or None) + def _list_repository(self, distro_name, repo_name, arch): # Get distribution distro = self.backend.distros.get_by_slug(distro_name)