jobs: Refactor dependency check

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/src/buildservice/jobs.py b/src/buildservice/jobs.py
index 54f1e74b01adfdf56f888f50e1b06fe2a2f4e499..76faa03ac74fb34cb3847bc0fbff5ee0e06311a0 100644 (file)
--- a/src/buildservice/jobs.py
+++ b/src/buildservice/jobs.py
@@ -1,5 +1,6 @@
 #!/usr/bin/python
 
+import asyncio
 import datetime
 import hashlib
 import logging
@@ -85,6 +86,12 @@ class Jobs(base.Object):
 
                return list(jobs)
 
+       async def depcheck(self, jobs=None):
+               """
+                       Performs a dependency check on all given jobs concurrently
+               """
+               await asyncio.gather(*(job.depcheck() for job in jobs))
+
 
 class Job(base.DataObject):
        table = "jobs"
@@ -468,47 +475,55 @@ class Job(base.DataObject):
                self.backend.messages.send_template_to_many(self.message_recipients,
                        "messages/jobs/failed", job=self)
 
+       def pakfire(self, **kwargs):
+               """
+                       Returns a local Pakfire instance with the configuration for this job
+               """
+               config = self.get_config(local=True)
+
+               return self.backend.pakfire(config=config, **kwargs)
+
        def get_config(self, local=False):
                """
                        Get configuration file that is sent to the builder.
                """
                return self.distro.get_config(local=local, repo=self.build.build_repo)
 
-       def set_dependency_check_succeeded(self, value):
-               self._set_attribute("dependency_check_succeeded", value)
-               self._set_attribute("dependency_check_at", datetime.datetime.utcnow())
+       async def depcheck(self):
+               """
+                       Perform dependency check
+               """
+               log.info("Performing dependency check for %s" % self)
 
-               # Reset the message
-               if value is True:
-                       self.message = None
+               with self.db.transaction():
+                       return await asyncio.to_thread(self._depcheck)
 
-       dependency_check_succeeded = property(
-               lambda s: s.data.dependency_check_succeeded,
-               set_dependency_check_succeeded)
+       def _depcheck(self):
+               # Create a Pakfire instance
+               p = self.pakfire()
 
-       def resolvdep(self):
-               log.info("Processing dependencies for %s..." % self)
+               # Try to install the source package
+               try:
+                       p.install([self.pkg.path], dryrun=True)
 
-               config = pakfire.config.Config(files=["general.conf"])
-               config.parse(self.get_config(local=True))
+               # XXX Pakfire should throw a better exception
+               except Exception as e:
+                       self._set_attribute("depcheck_succeeded", False)
 
-               # The filename of the source file.
-               filename = os.path.join(PACKAGES_DIR, self.build.pkg.path)
-               assert os.path.exists(filename), filename
+                       # Store the message
+                       self._set_attribute("message", "%s" % e)
 
-               # Create a new pakfire instance with the configuration for
-               # this build.
-               p = pakfire.PakfireServer(config=config, arch=self.arch)
+               # Everything OK
+               else:
+                       self._set_attribute("depcheck_succeeded", True)
 
-               # Try to solve the build dependencies.
-               try:
-                       solver = p.resolvdep(filename)
+               # Store the timestamp
+               self._set_attribute_now("depcheck_performed_at")
 
-               # Catch dependency errors and log the problem string.
-               except DependencyError as e:
-                       self.dependency_check_succeeded = False
-                       self.message = e
+       @property
+       def depcheck_succeeded(self):
+               return self.data.depcheck_succeeded
 
-               # The dependency check has succeeded
-               else:
-                       self.dependency_check_succeeded = True
+       @property
+       def depcheck_performed_at(self):
+               return self.data.depcheck_performed_at

diff --git a/src/database.sql b/src/database.sql
index 2ba7c9fd9dc6b38f340a46ac243d7923b2ed2bca..6d5d2a4419423bd5e3c9350270e05cc23dceddac 100644 (file)
--- a/src/database.sql
+++ b/src/database.sql
@@ -481,8 +481,8 @@ CREATE TABLE public.jobs (
     message text,
     test boolean DEFAULT true NOT NULL,
     superseeded_by integer,
-    dependency_check_succeeded boolean,
-    dependency_check_at timestamp without time zone,
+    depcheck_succeeded boolean,
+    depcheck_performed_at timestamp without time zone,
     deleted boolean DEFAULT false NOT NULL,
     failed boolean DEFAULT false NOT NULL,
     log_path text,
@@ -503,7 +503,7 @@ CREATE VIEW public.job_queue AS
     jobs.arch
    FROM (public.jobs
      LEFT JOIN public.builds ON ((jobs.build_id = builds.id)))
-  WHERE ((jobs.deleted IS FALSE) AND (jobs.started_at IS NULL) AND (jobs.finished_at IS NULL) AND (jobs.dependency_check_succeeded IS TRUE));
+  WHERE ((jobs.deleted IS FALSE) AND (jobs.started_at IS NULL) AND (jobs.finished_at IS NULL) AND (jobs.depcheck_succeeded IS TRUE));
 
 
 ALTER TABLE public.job_queue OWNER TO pakfire;
@@ -2121,7 +2121,7 @@ CREATE INDEX jobs_finished_at ON public.jobs USING btree (finished_at DESC) WHER
 -- Name: jobs_pending; Type: INDEX; Schema: public; Owner: pakfire
 --
 
-CREATE INDEX jobs_pending ON public.jobs USING btree (id) WHERE ((deleted IS FALSE) AND (started_at IS NULL) AND (finished_at IS NULL) AND (dependency_check_succeeded IS TRUE));
+CREATE INDEX jobs_pending ON public.jobs USING btree (id) WHERE ((deleted IS FALSE) AND (started_at IS NULL) AND (finished_at IS NULL) AND (depcheck_succeeded IS TRUE));
 
 
 --

diff --git a/src/scripts/pakfire-build-service b/src/scripts/pakfire-build-service
index 822969fc44647f07a9aac3e7e5f57089cf8c7210..b7ec2863cb8186459ce80e7ca9d1ed1179700b06 100644 (file)
--- a/src/scripts/pakfire-build-service
+++ b/src/scripts/pakfire-build-service
@@ -21,6 +21,9 @@ class Cli(object):
                        # Cleanup
                        "cleanup"          : self.backend.cleanup,
 
+                       # Jobs
+                       "jobs:depcheck"    : self._jobs_depcheck,
+
                        # Repositories
                        "repos:sync"       : self.backend.repos.sync,
                        "repos:write"      : self.backend.repos.write,
@@ -71,6 +74,24 @@ class Cli(object):
                # Exit with error code
                sys.exit(r or 0)
 
+       async def _jobs_depcheck(self, *uuids):
+               """
+                       Test command to trigger a dependency check
+               """
+               jobs = []
+
+               # Collect jobs by UUID
+               for uuid in uuids:
+                       job = self.backend.jobs.get_by_uuid(uuid)
+                       if not job:
+                               log.error("Could not find job '%s'" % uuid)
+                               continue
+
+                       jobs.append(job)
+
+               # Run the checks
+               await self.backend.jobs.depcheck(jobs or None)
+
        def _list_repository(self, distro_name, repo_name, arch):
                # Get distribution
                distro = self.backend.distros.get_by_slug(distro_name)