From ac10fd4363467b4b408ad89c5eeeecacd59d93d3 Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Tue, 10 Oct 2017 17:58:39 +0100 Subject: [PATCH] Add command to create test jobs Signed-off-by: Michael Tremer --- src/buildservice/builds.py | 30 ---------------------- src/buildservice/jobqueue.py | 40 ++++++++++++++++++++++++++++- src/crontab/pakfire-build-service | 3 +++ src/manager/__init__.py | 2 +- src/manager/builds.py | 42 ------------------------------- src/scripts/pakfire-build-service | 3 +++ src/scripts/pakfire-manager | 1 - 7 files changed, 46 insertions(+), 75 deletions(-) diff --git a/src/buildservice/builds.py b/src/buildservice/builds.py index 03bc3fc..fae1032 100644 --- a/src/buildservice/builds.py +++ b/src/buildservice/builds.py @@ -225,36 +225,6 @@ class Builds(base.Object): if builds: return builds.count - def needs_test(self, threshold, arch, limit=None, randomize=False): - query = "SELECT id FROM builds \ - WHERE NOT EXISTS \ - (SELECT * FROM jobs WHERE \ - jobs.build_id = builds.id AND \ - jobs.arch = %s AND \ - (jobs.state != 'finished' OR \ - jobs.time_finished >= %s) \ - ) \ - AND EXISTS \ - (SELECT * FROM jobs WHERE \ - jobs.build_id = builds.id AND \ - jobs.arch = %s AND \ - jobs.type = 'build' AND \ - jobs.state = 'finished' AND \ - jobs.time_finished < %s \ - ) \ - AND builds.type = 'release' \ - AND (builds.state = 'stable' OR builds.state = 'testing')" - args = [arch, threshold, arch, threshold] - - if randomize: - query += " ORDER BY RAND()" - - if limit: - query += " LIMIT %s" - args.append(limit) - - return [Build(self.pakfire, b.id) for b in self.db.query(query, *args)] - def get_obsolete(self, repo=None): """ Get all obsoleted builds. diff --git a/src/buildservice/jobqueue.py b/src/buildservice/jobqueue.py index 23f6d4f..f803271 100644 --- a/src/buildservice/jobqueue.py +++ b/src/buildservice/jobqueue.py @@ -1,7 +1,13 @@ #!/usr/bin/python +import datetime +import logging + from . import base +log = logging.getLogger("jobqueue") +log.propagate = 1 + PENDING_STATE = "pending" class JobQueue(base.Object): @@ -38,4 +44,36 @@ class JobQueue(base.Object): res = self.db.get("SELECT AVG(NOW() - COALESCE(jobs.start_not_before, jobs.time_created)) AS avg \ FROM jobs_queue queue LEFT JOIN jobs ON queue.job_id = jobs.id") - return res.avg \ No newline at end of file + return res.avg + + def create_test_jobs(self): + max_queue_length = self.backend.settings.get_int("test_queue_limit", 25) + + threshold_days = self.backend.settings.get_int("test_threshold_days", 14) + threshold = datetime.datetime.utcnow() - datetime.timedelta(days=threshold_days) + + for arch in self.backend.arches: + # Skip adding new jobs if there are more too many jobs in the queue. + limit = max_queue_length - self.backend.jobqueue.get_length_for_arch(arch) + if limit <= 0: + log.debug("Already too many jobs in queue of %s to create tests." % arch) + continue + + # Get a list of builds, with potentially need a test build. + # Randomize the output and do not return more jobs than we are + # allowed to put into the build queue. + builds = self.backend.builds._get_builds("SELECT builds.* FROM builds \ + LEFT JOIN jobs ON builds.id = jobs.build_id \ + WHERE builds.type = %s AND builds.state = ANY(%s) AND jobs.state = %s \ + AND NOT EXISTS (SELECT 1 FROM jobs test_jobs \ + WHERE test_jobs.build_id = builds.id AND jobs.type = %s \ + AND (test_jobs.state <> %s OR test_jobs.state = %s AND test_jobs.time_finished >= %s)) LIMIT %s", + "release", ["stable", "testing"], "finished", "test", "finished", "finished", threshold, limit) + + # Search for the job with the right architecture in each + # build and schedule a test job. + for build in builds: + for job in build: + if job.arch == arch: + job.schedule("test") + break diff --git a/src/crontab/pakfire-build-service b/src/crontab/pakfire-build-service index a4dc947..d29b852 100644 --- a/src/crontab/pakfire-build-service +++ b/src/crontab/pakfire-build-service @@ -10,6 +10,9 @@ # Send updates to Bugzilla */5 * * * * nobody pakfire-build-service send-bug-updates &>/dev/null +# Create test jobs +*/15 * * * * nobody pakfire-build-service create-test-jobs &>/dev/null + # Cleanup files */5 * * * * nobody pakfire-build-service cleanup-files &>/dev/null diff --git a/src/manager/__init__.py b/src/manager/__init__.py index fe248fe..4a9b735 100644 --- a/src/manager/__init__.py +++ b/src/manager/__init__.py @@ -1,4 +1,4 @@ #!/usr/bin/python from .builds import BuildsFailedRestartEvent, CheckBuildDependenciesEvent -from .builds import CreateTestBuildsEvent, DistEvent +from .builds import DistEvent diff --git a/src/manager/builds.py b/src/manager/builds.py index a1a0604..9ff85d2 100644 --- a/src/manager/builds.py +++ b/src/manager/builds.py @@ -92,48 +92,6 @@ class CheckBuildDependencyEvent(base.Event): job.resolvdep() -class CreateTestBuildsEvent(base.Event): - # Run this every five minutes. - interval = 300 - - # Run when the build service is idle. - priority = 10 - - @property - def test_threshold(self): - threshold_days = self.pakfire.settings.get_int("test_threshold_days", 14) - - return datetime.datetime.utcnow() - datetime.timedelta(days=threshold_days) - - def run(self): - max_queue_length = self.pakfire.settings.get_int("test_queue_limit", 10) - - for arch in self.backend.arches: - # Skip adding new jobs if there are more too many jobs in the queue. - limit = max_queue_length - self.backend.jobqueue.get_length_for_arch(arch) - if limit <= 0: - logging.debug("Already too many jobs in queue of %s to create tests." % arch) - continue - - # Get a list of builds, with potentially need a test build. - # Randomize the output and do not return more jobs than we are - # allowed to put into the build queue. - builds = self.pakfire.builds.needs_test(self.test_threshold, - arch=arch, limit=limit) - - if not builds: - logging.debug("No builds needs a test for %s." % arch.name) - continue - - # Search for the job with the right architecture in each - # build and schedule a test job. - for build in builds: - for job in build.jobs: - if job.arch == arch: - job.schedule("test") - break - - class DistEvent(base.Event): interval = 60 diff --git a/src/scripts/pakfire-build-service b/src/scripts/pakfire-build-service index 5ee79e7..470af17 100644 --- a/src/scripts/pakfire-build-service +++ b/src/scripts/pakfire-build-service @@ -26,6 +26,9 @@ class Cli(object): # Cleanup uploads "cleanup-uploads" : self.backend.uploads.cleanup, + # Create test jobs + "create-test-jobs" : self.backend.jobqueue.create_test_jobs, + # List repository "list-repository" : self._list_repository, diff --git a/src/scripts/pakfire-manager b/src/scripts/pakfire-manager index 81baed3..9e9dac8 100644 --- a/src/scripts/pakfire-manager +++ b/src/scripts/pakfire-manager @@ -18,7 +18,6 @@ p = pakfire.buildservice.Pakfire() events = ( pakfire.buildservice.manager.BuildsFailedRestartEvent, pakfire.buildservice.manager.CheckBuildDependenciesEvent, - pakfire.buildservice.manager.CreateTestBuildsEvent, pakfire.buildservice.manager.DistEvent, pakfire.buildservice.manager.RepositoriesUpdateEvent, ) -- 2.47.3