src/buildservice/distribution.py \
src/buildservice/geoip.py \
src/buildservice/git.py \
+ src/buildservice/jobqueue.py \
src/buildservice/keys.py \
src/buildservice/logs.py \
src/buildservice/messages.py \
src/hub/handlers.py
src/hub/__init__.py
src/manager/base.py
-src/manager/bugs.py
src/manager/builds.py
src/manager/__init__.py
src/manager/repositories.py
-src/manager/sources.py
src/web/handlers_api.py
src/web/handlers_auth.py
src/web/handlers_base.py
from . import database
from . import distribution
from . import geoip
+from . import jobqueue
from . import keys
from . import logs
from . import messages
self.jobs = builds.Jobs(self)
self.builders = builders.Builders(self)
self.distros = distribution.Distributions(self)
+ self.jobqueue = jobqueue.JobQueue(self)
self.keys = keys.Keys(self)
self.messages = messages.Messages(self)
self.mirrors = mirrors.Mirrors(self)
return sorted(arches)
- def get_build_release(self):
- return self.data.build_release == "Y"
+ def set_testmode(self, testmode):
+ self._set_attribute("testmode", testmode)
- def set_build_release(self, value):
- if value:
- value = "Y"
- else:
- value = "N"
-
- self.db.execute("UPDATE builders SET build_release = %s WHERE id = %s",
- value, self.id)
-
- # Update the cache.
- if self._data:
- self._data["build_release"] = value
-
- build_release = property(get_build_release, set_build_release)
-
- def get_build_scratch(self):
- return self.data.build_scratch == "Y"
-
- def set_build_scratch(self, value):
- if value:
- value = "Y"
- else:
- value = "N"
-
- self.db.execute("UPDATE builders SET build_scratch = %s WHERE id = %s",
- value, self.id)
-
- # Update the cache.
- if self._data:
- self._data["build_scratch"] = value
-
- build_scratch = property(get_build_scratch, set_build_scratch)
-
- def get_build_test(self):
- return self.data.build_test == "Y"
-
- def set_build_test(self, value):
- if value:
- value = "Y"
- else:
- value = "N"
-
- self.db.execute("UPDATE builders SET build_test = %s WHERE id = %s",
- value, self.id)
-
- # Update the cache.
- if self._data:
- self._data["build_test"] = value
-
- build_test = property(get_build_test, set_build_test)
-
- @property
- def build_types(self):
- ret = []
-
- if self.build_release:
- ret.append("release")
-
- if self.build_scratch:
- ret.append("scratch")
-
- if self.build_test:
- ret.append("test")
-
- return ret
+ testmode = property(lambda s: s.data.testmode, set_testmode)
def set_max_jobs(self, value):
self._set_attribute("max_jobs", value)
"""
return len(self.active_jobs) >= self.max_jobs
- def get_next_jobs(self, limit=None):
- """
- Returns a list of jobs that can be built on this host.
- """
- return self.pakfire.jobs.get_next(arches=self.buildable_arches, limit=limit)
+ @lazy_property
+ def jobqueue(self):
+ return self.backend.jobqueue.for_arches(self.supported_arches)
def get_next_job(self):
"""
Returns the next job in line for this builder.
"""
- # Get the first item of all jobs in the list.
- jobs = self.pakfire.jobs.get_next(builder=self, state="pending", limit=1)
+ # Don't return anything if the builder has already too many jobs running
+ if self.too_many_jobs:
+ return
+
+ for job in self.jobqueue:
+ # Only allow building test jobs in test mode
+ if self.testmode and not job.type == "test":
+ continue
- if jobs:
- return jobs[0]
+ return job
def get_history(self, *args, **kwargs):
kwargs["builder"] = self
class Jobs(base.Object):
+ def _get_jobs(self, query, *args):
+ res = self.db.query(query, *args)
+
+ for row in res:
+ yield Job(self.backend, row.id, data=row)
+
def get_by_id(self, id, data=None):
return Job(self.pakfire, id, data)
return [Job(self.pakfire, j.id, j) for j in self.db.query(query, *args)]
- def get_next_iter(self, *args, **kwargs):
- return iter(self.get_next(*args, **kwargs))
-
- def get_next(self, arches=None, builder=None, limit=None, offset=None, type=None,
- state=None, states=None, max_tries=None):
-
- if state and states is None:
- states = [state,]
-
- query = "SELECT * FROM jobs \
- INNER JOIN jobs_queue ON jobs.id = jobs_queue.id"
- args = []
-
- if arches:
- query += " AND jobs_queue.arch IN (%s)" % ", ".join(["%s"] * len(arches))
- args.extend(arches)
-
- if builder:
- query += " AND jobs_queue.designated_builder_id = %s"
- args.append(builder.id)
-
- if max_tries:
- query += " AND jobs.max_tries <= %s"
- args.append(max_tries)
-
- if states:
- query += " AND jobs.state IN (%s)" % ", ".join(["%s"] * len(states))
- args.extend(states)
-
- if type:
- query += " AND jobs.type = %s"
- args.append(type)
-
- if limit:
- query += " LIMIT %s"
- args.append(limit)
-
- jobs = []
- for row in self.db.query(query, *args):
- job = self.pakfire.jobs.get_by_id(row.id, row)
- jobs.append(job)
-
- # Reverse the order of the builds.
- jobs.reverse()
-
- return jobs
-
def get_latest(self, arch=None, builder=None, limit=None, age=None, date=None):
query = "SELECT * FROM jobs"
args = []
if jobs:
return jobs.count
- def get_queue_length(self, state=None):
- if state:
- res = self.db.get("SELECT COUNT(*) AS count FROM jobs_queue \
- LEFT JOIN jobs ON jobs_queue.id = jobs.id WHERE state = %s", state)
- else:
- res = self.db.get("SELECT COUNT(*) AS count FROM jobs_queue")
-
- if res:
- return res.count
-
- return 0
-
- def get_avg_wait_time(self):
- res = self.db.get("SELECT AVG(time_waiting) AS time_waiting FROM jobs_waiting")
-
- if res and res.time_waiting:
- try:
- return int(res.time_waiting)
- except ValueError:
- return 0
-
- return 0
-
def get_state_stats(self):
res = self.db.query("SELECT state, COUNT(*) AS count FROM jobs GROUP BY state")
--- /dev/null
+#!/usr/bin/python
+
+from . import base
+
+PENDING_STATE = "pending"
+
+class JobQueue(base.Object):
+ def __iter__(self):
+ jobs = self.backend.jobs._get_jobs("SELECT jobs.* FROM jobs_queue queue \
+ LEFT JOIN jobs ON queue.job_id = jobs.id")
+
+ return iter(jobs)
+
+ def __len__(self):
+ res = self.db.get("SELECT COUNT(*) AS len FROM jobs_queue")
+
+ return res.len
+
+ def for_arches(self, arches, limit=None):
+ jobs = self.backend.jobs._get_jobs("SELECT jobs.* FROM jobs_queue queue \
+ LEFT JOIN jobs ON queue.job_id = jobs.id \
+ WHERE jobs.arch = ANY(%s) LIMIT %s", arches, limit)
+
+ return jobs
+
+ def get_length_for_arch(self, arch):
+ res = self.db.get("SELECT COUNT(*) AS len FROM jobs_queue queue \
+ LEFT JOIN jobs on queue.job_id = jobs.id \
+ WHERE jobs.arch = %s", arch)
+
+ return res.len
+
+ @property
+ def average_waiting_time(self):
+ """
+ Returns how long the jobs in the queue have been waiting on average
+ """
+ res = self.db.get("SELECT AVG(NOW() - COALESCE(jobs.start_not_before, jobs.time_created)) AS avg \
+ FROM jobs_queue queue LEFT JOIN jobs ON queue.job_id = jobs.id")
+
+ return res.avg
\ No newline at end of file
ALTER TABLE arches_compat OWNER TO pakfire;
+--
+-- Name: arches_id_seq; Type: SEQUENCE; Schema: public; Owner: pakfire
+--
+
+CREATE SEQUENCE arches_id_seq
+ START WITH 1
+ INCREMENT BY 1
+ NO MINVALUE
+ NO MAXVALUE
+ CACHE 1;
+
+
+ALTER TABLE arches_id_seq OWNER TO pakfire;
+
+--
+-- Name: arches_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: pakfire
+--
+
+ALTER SEQUENCE arches_id_seq OWNED BY arches.id;
+
+
--
-- Name: builders; Type: TABLE; Schema: public; Owner: pakfire; Tablespace:
--
deleted boolean DEFAULT false NOT NULL,
loadavg text DEFAULT '0'::character varying NOT NULL,
arches text,
- build_release builders_build_release DEFAULT 'N'::builders_build_release NOT NULL,
- build_scratch builders_build_scratch DEFAULT 'N'::builders_build_scratch NOT NULL,
- build_test builders_build_test DEFAULT 'N'::builders_build_test NOT NULL,
+ testmode boolean DEFAULT true NOT NULL,
max_jobs bigint DEFAULT 1::bigint NOT NULL,
pakfire_version text,
os_name text,
ALTER TABLE builders OWNER TO pakfire;
---
--- Name: arches_builders; Type: VIEW; Schema: public; Owner: pakfire
---
-
-CREATE VIEW arches_builders AS
- SELECT arches_compat.build_arch AS arch,
- builders.id AS builder_id
- FROM (arches_compat
- LEFT JOIN builders ON ((arches_compat.native_arch = builders.cpu_arch)));
-
-
-ALTER TABLE arches_builders OWNER TO pakfire;
-
---
--- Name: arches_id_seq; Type: SEQUENCE; Schema: public; Owner: pakfire
---
-
-CREATE SEQUENCE arches_id_seq
- START WITH 1
- INCREMENT BY 1
- NO MINVALUE
- NO MAXVALUE
- CACHE 1;
-
-
-ALTER TABLE arches_id_seq OWNER TO pakfire;
-
---
--- Name: arches_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: pakfire
---
-
-ALTER SEQUENCE arches_id_seq OWNED BY arches.id;
-
-
--
-- Name: builders_history; Type: TABLE; Schema: public; Owner: pakfire; Tablespace:
--
ALTER SEQUENCE builders_id_seq OWNED BY builders.id;
---
--- Name: jobs; Type: TABLE; Schema: public; Owner: pakfire; Tablespace:
---
-
-CREATE TABLE jobs (
- id integer NOT NULL,
- uuid text NOT NULL,
- type jobs_type DEFAULT 'build'::jobs_type NOT NULL,
- build_id integer NOT NULL,
- state jobs_state DEFAULT 'new'::jobs_state NOT NULL,
- arch text NOT NULL,
- time_created timestamp without time zone NOT NULL,
- time_started timestamp without time zone,
- time_finished timestamp without time zone,
- start_not_before timestamp without time zone,
- builder_id integer,
- tries integer DEFAULT 0 NOT NULL,
- aborted_state integer DEFAULT 0 NOT NULL,
- message text
-);
-
-
-ALTER TABLE jobs OWNER TO pakfire;
-
---
--- Name: jobs_active; Type: VIEW; Schema: public; Owner: pakfire
---
-
-CREATE VIEW jobs_active AS
- SELECT jobs.id,
- jobs.uuid,
- jobs.type,
- jobs.build_id,
- jobs.state,
- jobs.arch,
- jobs.time_created,
- jobs.time_started,
- jobs.time_finished,
- jobs.start_not_before,
- jobs.builder_id,
- jobs.tries,
- jobs.aborted_state,
- jobs.message
- FROM jobs
- WHERE (jobs.state = ANY (ARRAY['dispatching'::jobs_state, 'running'::jobs_state, 'uploading'::jobs_state]))
- ORDER BY jobs.time_started;
-
-
-ALTER TABLE jobs_active OWNER TO pakfire;
-
---
--- Name: builders_ready; Type: VIEW; Schema: public; Owner: pakfire
---
-
-CREATE VIEW builders_ready AS
- SELECT builders.id AS builder_id,
- builders.cpu_arch AS builder_arch,
- builders.build_release,
- builders.build_scratch,
- builders.build_test
- FROM builders
- WHERE ((((builders.deleted IS FALSE) AND (builders.enabled IS TRUE)) AND (builders.time_keepalive >= (now() - '00:05:00'::interval))) AND (builders.max_jobs > ( SELECT count(*) AS count
- FROM jobs_active
- WHERE (jobs_active.builder_id = builders.id))))
- ORDER BY ( SELECT count(*) AS count
- FROM jobs_active
- WHERE (jobs_active.builder_id = builders.id)), builders.cpu_bogomips DESC;
-
-
-ALTER TABLE builders_ready OWNER TO pakfire;
-
--
-- Name: builds; Type: TABLE; Schema: public; Owner: pakfire; Tablespace:
--
resolution text,
comment text,
"time" timestamp without time zone NOT NULL,
- error builds_bugs_updates_error DEFAULT 'N'::builds_bugs_updates_error NOT NULL,
+ error boolean DEFAULT false NOT NULL,
error_msg text
);
ALTER TABLE builds_latest OWNER TO pakfire;
+--
+-- Name: jobs; Type: TABLE; Schema: public; Owner: pakfire; Tablespace:
+--
+
+CREATE TABLE jobs (
+ id integer NOT NULL,
+ uuid text NOT NULL,
+ type jobs_type DEFAULT 'build'::jobs_type NOT NULL,
+ build_id integer NOT NULL,
+ state jobs_state DEFAULT 'new'::jobs_state NOT NULL,
+ arch text NOT NULL,
+ time_created timestamp without time zone NOT NULL,
+ time_started timestamp without time zone,
+ time_finished timestamp without time zone,
+ start_not_before timestamp without time zone,
+ builder_id integer,
+ tries integer DEFAULT 0 NOT NULL,
+ aborted_state integer DEFAULT 0 NOT NULL,
+ message text
+);
+
+
+ALTER TABLE jobs OWNER TO pakfire;
+
--
-- Name: builds_times; Type: VIEW; Schema: public; Owner: pakfire
--
ALTER SEQUENCE images_types_id_seq OWNED BY images_types.id;
+--
+-- Name: jobs_active; Type: VIEW; Schema: public; Owner: pakfire
+--
+
+CREATE VIEW jobs_active AS
+ SELECT jobs.id,
+ jobs.uuid,
+ jobs.type,
+ jobs.build_id,
+ jobs.state,
+ jobs.arch,
+ jobs.time_created,
+ jobs.time_started,
+ jobs.time_finished,
+ jobs.start_not_before,
+ jobs.builder_id,
+ jobs.tries,
+ jobs.aborted_state,
+ jobs.message
+ FROM jobs
+ WHERE (jobs.state = ANY (ARRAY['dispatching'::jobs_state, 'running'::jobs_state, 'uploading'::jobs_state]))
+ ORDER BY jobs.time_started;
+
+
+ALTER TABLE jobs_active OWNER TO pakfire;
+
--
-- Name: jobs_buildroots; Type: TABLE; Schema: public; Owner: pakfire; Tablespace:
--
--
CREATE VIEW jobs_queue AS
- SELECT jobs.id,
- arches.name AS arch,
- ( SELECT builders_ready.builder_id
- FROM builders_ready
- WHERE (builders_ready.builder_id IN ( SELECT arches_builders.builder_id
- FROM arches_builders
- WHERE ((arches_builders.arch = arches.name) AND
- CASE
- WHEN ((builds.type = 'release'::builds_type) AND (jobs.type = 'build'::jobs_type)) THEN (builders_ready.build_release = 'Y'::builders_build_release)
- WHEN ((builds.type = 'scratch'::builds_type) AND (jobs.type = 'build'::jobs_type)) THEN (builders_ready.build_scratch = 'Y'::builders_build_scratch)
- WHEN (jobs.type = 'test'::jobs_type) THEN (builders_ready.build_test = 'Y'::builders_build_test)
- ELSE NULL::boolean
- END)))
- LIMIT 1) AS designated_builder_id
- FROM ((jobs
- LEFT JOIN arches ON ((jobs.arch = arches.name)))
- LEFT JOIN builds ON ((jobs.build_id = builds.id)))
- WHERE ((jobs.state = ANY (ARRAY['pending'::jobs_state, 'new'::jobs_state])) AND ((jobs.start_not_before IS NULL) OR (jobs.start_not_before <= now())))
- ORDER BY
- CASE
- WHEN (jobs.type = 'build'::jobs_type) THEN 0
- WHEN (jobs.type = 'test'::jobs_type) THEN 1
- ELSE NULL::integer
- END, builds.priority DESC, jobs.tries, jobs.time_created;
+ WITH queue AS (
+ SELECT jobs.id,
+ rank() OVER (ORDER BY jobs.type, builds.priority DESC, jobs.tries, jobs.time_created) AS rank
+ FROM (jobs
+ LEFT JOIN builds ON ((jobs.build_id = builds.id)))
+ WHERE (jobs.state = 'pending'::jobs_state)
+ )
+ SELECT queue.id AS job_id,
+ queue.rank
+ FROM queue;
ALTER TABLE jobs_queue OWNER TO pakfire;
ALTER TABLE jobs_repos OWNER TO pakfire;
---
--- Name: jobs_waiting; Type: VIEW; Schema: public; Owner: pakfire
---
-
-CREATE VIEW jobs_waiting AS
- SELECT jobs_queue.id,
- (now() - (jobs.time_created)::timestamp with time zone) AS time_waiting
- FROM (jobs_queue
- LEFT JOIN jobs ON ((jobs_queue.id = jobs.id)))
- WHERE (jobs.start_not_before IS NULL)
-UNION
- SELECT jobs_queue.id,
- (now() - (jobs.start_not_before)::timestamp with time zone) AS time_waiting
- FROM (jobs_queue
- LEFT JOIN jobs ON ((jobs_queue.id = jobs.id)))
- WHERE (jobs.start_not_before IS NOT NULL);
-
-
-ALTER TABLE jobs_waiting OWNER TO pakfire;
-
--
-- Name: keys; Type: TABLE; Schema: public; Owner: pakfire; Tablespace:
--
ret = {}
# Queue length(s).
- ret["job_queue_length"] = self.backend.jobs.get_queue_length()
- for state in ("new", "pending"):
- ret["job_queue_length_%s" % state] = self.backend.jobs.get_queue_length(state)
+ ret["job_queue_length"] = len(self.backend.jobqueue)
# Average waiting time.
- ret["job_queue_avg_wait_time"] = self.backend.jobs.get_avg_wait_time()
+ ret["job_queue_avg_wait_time"] = self.backend.jobqueue.average_waiting_time
self.write(ret)
limit = self.get_argument_int("limit", 5)
# Get the job queue.
- jobs = self.backend.jobs.get_next(limit=limit)
+ jobs = []
+ for job in self.backend.jobqueue:
+ jobs.append(job)
+
+ limit -= 1
+ if not limit: break
args = {
"jobs" : [self.job2json(j) for j in jobs],
arches.append(noarch)
for arch in arches:
- # Get the job queue for each architecture.
- queue = self.pakfire.jobs.get_next(arches=[arch,])
-
# Skip adding new jobs if there are more too many jobs in the queue.
- limit = max_queue_length - len(queue)
+ limit = max_queue_length - self.backend.jobqueue.get_length_for_arch(arch.name)
if limit <= 0:
logging.debug("Already too many jobs in queue of %s to create tests." % arch.name)
continue
<h2>{{ _("Builder") }}: {{ builder.name }}</h2>
</div>
+ {% if builder.testmode %}
+ <div class="alert alert-block alert-warning">
+ {{ _("This builder is in test mode!") }}
+ </div>
+ {% end %}
+
{% if builder.overload %}
<div class="alert alert-block alert-warning">
<h4 class="alert-heading">{{ _("Warning") }}!</h4>
<td>{{ _("Parallel builds") }}</td>
<td>{{ _("One job only.", "Up to %(num)s jobs.", builder.max_jobs) % { "num" : builder.max_jobs } }}</td>
</tr>
- <tr>
- <td>{{ _("This host builds") }}</td>
- <td>
- <ul>
- {% for type in builder.build_types %}
- <li>
- {% if type == "release" %}
- {{ _("Release builds") }}
- {% elif type == "scratch" %}
- {{ _("Scratch builds") }}
- {% elif type == "test" %}
- {{ _("Test builds") }}
- {% end %}
- </li>
- {% end %}
- </ul>
- </td>
- </tr>
</tbody>
</table>
<div class="control-group">
<div class="controls">
<label class="checkbox">
- <input type="checkbox" id="build_release" name="build_release" {%if builder.build_release %}checked="checked"{% end %}>
- {{ _("Authorized to build release builds.") }}
- </label>
- </div>
- </div>
-
- <div class="control-group">
- <div class="controls">
- <label class="checkbox">
- <input type="checkbox" id="build_scratch" name="build_scratch" {%if builder.build_scratch %}checked="checked"{% end %}>
- {{ _("Authorized to build scratch builds.") }}
- </label>
- </div>
- </div>
-
- <div class="control-group">
- <div class="controls">
- <label class="checkbox">
- <input type="checkbox" id="build_test" name="build_test" {%if builder.build_test %}checked="checked"{% end %}>
- {{ _("Authorized to build test builds.") }}
+ <input type="checkbox" name="testmode" {%if builder.testmode %}checked{% end %}>
+ {{ _("Test Mode") }}
</label>
</div>
</div>
class IndexHandler(BaseHandler):
def get(self):
jobs = self.pakfire.jobs.get_active()
- jobs += self.pakfire.jobs.get_next()
+ jobs += self.backend.jobqueue
jobs += self.pakfire.jobs.get_latest(age="24 hours", limit=5)
# Updates
# Get running and pending jobs.
jobs = self.pakfire.jobs.get_active(builder=builder)
- jobs += self.pakfire.jobs.get_next(builder=builder)
+ jobs += builder.jobqueue
# Get log.
log = builder.get_history(limit=5)
if not self.current_user.has_perm("maintain_builders"):
raise tornado.web.HTTPError(403)
- builder.enabled = self.get_argument("enabled", False)
- builder.build_release = self.get_argument("build_release", False)
- builder.build_scratch = self.get_argument("build_scratch", False)
- builder.build_test = self.get_argument("build_test", False)
-
- # Save max_jobs.
- max_jobs = self.get_argument("max_jobs", builder.max_jobs)
- try:
- max_jobs = int(max_jobs)
- except TypeError:
- max_jobs = 1
-
- if not max_jobs in range(1, 100):
- max_jobs = 1
- builder.max_jobs = max_jobs
+ with self.db.transaction():
+ builder.enabled = self.get_argument("enabled", False)
+ builder.testmode = self.get_argument("testmode", True)
+
+ # Save max_jobs.
+ max_jobs = self.get_argument("max_jobs", builder.max_jobs)
+ try:
+ max_jobs = int(max_jobs)
+ except TypeError:
+ max_jobs = 1
+
+ if not max_jobs in range(1, 100):
+ max_jobs = 1
+ builder.max_jobs = max_jobs
self.redirect("/builder/%s" % builder.hostname)
class BuildQueueHandler(BaseHandler):
def get(self):
- jobs = self.pakfire.jobs.get_next(limit=1000, states=["pending",])
-
- self.render("build-queue.html", jobs=jobs)
+ self.render("build-queue.html", jobs=self.backend.jobqueue)
class BuildDetailCommentHandler(BaseHandler):
projects / pbs.git / commitdiff
