src/buildservice/jobqueue.py

   1 #!/usr/bin/python
   2
   3 import datetime
   4 import logging
   5
   6 from . import base
   7
   8 log = logging.getLogger("jobqueue")
   9 log.propagate = 1
  10
  11 PENDING_STATE = "pending"
  12
  13 class JobQueue(base.Object):
  14         def __iter__(self):
  15                 jobs = self.backend.jobs._get_jobs("SELECT jobs.* FROM jobs_queue queue \
  16                         LEFT JOIN jobs ON queue.job_id = jobs.id")
  17
  18                 return iter(jobs)
  19
  20         def __len__(self):
  21                 res = self.db.get("SELECT COUNT(*) AS len FROM jobs_queue")
  22
  23                 return res.len
  24
  25         def for_arches(self, arches, limit=None):
  26                 jobs = self.backend.jobs._get_jobs("SELECT jobs.* FROM jobs_queue queue \
  27                         LEFT JOIN jobs ON queue.job_id = jobs.id \
  28                                 WHERE jobs.arch = ANY(%s) LIMIT %s", arches, limit)
  29
  30                 return jobs
  31
  32         def get_length_for_arch(self, arch):
  33                 res = self.db.get("SELECT COUNT(*) AS len FROM jobs_queue queue \
  34                         LEFT JOIN jobs on queue.job_id = jobs.id \
  35                                 WHERE jobs.arch = %s", arch)
  36
  37                 return res.len
  38
  39         @property
  40         def average_waiting_time(self):
  41                 """
  42                         Returns how long the jobs in the queue have been waiting on average
  43                 """
  44                 res = self.db.get("SELECT AVG(NOW() - COALESCE(jobs.start_not_before, jobs.time_created)) AS avg \
  45                         FROM jobs_queue queue LEFT JOIN jobs ON queue.job_id = jobs.id")
  46
  47                 return res.avg
  48
  49         def create_test_jobs(self):
  50                 max_queue_length = self.backend.settings.get_int("test_queue_limit", 25)
  51
  52                 threshold_days = self.backend.settings.get_int("test_threshold_days", 14)
  53                 threshold = datetime.datetime.utcnow() - datetime.timedelta(days=threshold_days)
  54
  55                 for arch in self.backend.arches:
  56                         # Skip adding new jobs if there are more too many jobs in the queue.
  57                         limit = max_queue_length - self.backend.jobqueue.get_length_for_arch(arch)
  58                         if limit <= 0:
  59                                 log.debug("Already too many jobs in queue of %s to create tests." % arch)
  60                                 continue
  61
  62                         # Get a list of builds, with potentially need a test build.
  63                         # Randomize the output and do not return more jobs than we are
  64                         # allowed to put into the build queue.
  65                         builds = self.backend.builds._get_builds("SELECT builds.* FROM builds \
  66                                 LEFT JOIN jobs ON builds.id = jobs.build_id \
  67                                 WHERE builds.type = %s AND builds.state = ANY(%s) AND jobs.state = %s \
  68                                         AND NOT EXISTS (SELECT 1 FROM jobs test_jobs \
  69                                                 WHERE test_jobs.build_id = builds.id AND jobs.type = %s \
  70                                                         AND (test_jobs.state <> %s OR test_jobs.state = %s AND test_jobs.time_finished >= %s)) LIMIT %s",
  71                                 "release", ["stable", "testing"], "finished", "test", "finished", "finished", threshold, limit)
  72
  73                         # Search for the job with the right architecture in each
  74                         # build and schedule a test job.
  75                         for build in builds:
  76                                 for job in build:
  77                                         if job.arch == arch:
  78                                                 job.schedule("test")
  79                                                 break