From: Michael Tremer Date: Tue, 21 Jun 2022 12:45:31 +0000 (+0000) Subject: jobs: Simplify states and rename lots of fields X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=02de33ff0458a9f374bb8d48470fe4d416d05c12;p=pbs.git jobs: Simplify states and rename lots of fields Signed-off-by: Michael Tremer --- diff --git a/src/buildservice/base.py b/src/buildservice/base.py index 224d3e88..cd979995 100644 --- a/src/buildservice/base.py +++ b/src/buildservice/base.py @@ -78,3 +78,14 @@ class DataObject(Object): # Update the cached attribute self.data[key] = val + + def _set_attribute_now(self, key): + """ + Sets the given key to CURRENT_TIMESTAMP + """ + res = self.db.get("UPDATE %s SET %s = CURRENT_TIMESTAMP \ + WHERE id = %%s RETURNING %s" % (self.table, key, key), self.id) + + # Update cached attribute + if res: + self.data[key] = res[key] diff --git a/src/buildservice/builders.py b/src/buildservice/builders.py index 15f36ab1..8863e7e6 100644 --- a/src/buildservice/builders.py +++ b/src/buildservice/builders.py @@ -421,13 +421,13 @@ class Builder(base.DataObject): FROM jobs WHERE - time_started IS NOT NULL + started_at IS NOT NULL AND - time_finished IS NULL + finished_at IS NULL AND builder_id = %s ORDER BY - time_started""", + started_at DESC""", self.id, ) diff --git a/src/buildservice/jobs.py b/src/buildservice/jobs.py index e0af5288..74ab5dc4 100644 --- a/src/buildservice/jobs.py +++ b/src/buildservice/jobs.py @@ -53,12 +53,22 @@ class Jobs(base.Object): def get_by_uuid(self, uuid): return self._get_job("SELECT * FROM jobs WHERE uuid = %s", uuid) - def get_active(self, limit=None): - jobs = self._get_jobs("SELECT jobs.* FROM jobs \ - WHERE time_started IS NOT NULL AND time_finished IS NULL \ - ORDER BY time_started LIMIT %s", limit) + @property + def running(self): + jobs = self._get_jobs(""" + SELECT + jobs.* + FROM + jobs + WHERE + started_at IS NOT NULL + AND + finished_at IS NULL + ORDER BY + finished_at + """) - return jobs + return list(jobs) def get_recently_ended(self, limit=None): jobs = self._get_jobs("SELECT jobs.* FROM jobs \ @@ -172,55 +182,82 @@ class Job(base.DataObject): superseeded_by = lazy_property(get_superseeded_by, set_superseeded_by) - def start(self, builder): + @property + def created_at(self): """ - Starts this job on builder + Returns when this job was created """ - self.builder = builder + return self.data.created_at - # Start to dispatch the build job - self.state = "dispatching" + @property + def started_at(self): + """ + Returns when this job was started + """ + return self.data.started_at - def running(self): - self.state = "running" + @property + def finished_at(self): + """ + Returns when this job finished + """ + return self.data.finished_at - # Set start time - self.time_started = datetime.datetime.utcnow() - self.time_finished = None + def start(self, builder): + """ + Starts this job on builder + """ + log.info("Starting job %s on %s" % (self, builder)) - def finished(self): - self.state = "finished" + # Store the assigned builder + self._set_attribute("builder_id", builder) - # Log end time - self.time_finished = datetime.datetime.utcnow() + # Store the time + self._set_attribute_now("started_at") - # Notify users - self.send_finished_message() + def finished(self, success, message=None): + """ + Called when this job has finished + """ + # Store the time + self._set_attribute_now("finished_at") - def failed(self, message): - self.state = "failed" - self.message = message + # XXX handle success status - # Log end time - self.time_finished = datetime.datetime.utcnow() + # Store message + self._set_attribute("message", message) # Notify users - self.send_failed_message() + if success: + self.send_finished_message() + else: + self._set_attribute("failed", True) + self.send_failed_message() + + # XXX propagate any changes to the build - def is_active(self): + def is_running(self): """ - Returns True if this job is active + Returns True if this job is running """ - return self.time_started and not self.time_finished + return self.started_at and not self.finished_at def has_finished(self): - if self.time_finished: + if self.finished_at: return True return False - def has_failed(self): - return self.state == "failed" + @property + def failed(self): + """ + Indicates whether this job has failed + """ + return self.data.failed + + @property + def message(self): + return self.data.message def delete(self): """ @@ -296,91 +333,26 @@ class Job(base.DataObject): return entries - def is_running(self): - """ - Returns True if job is in a running state. - """ - return self.state in ("pending", "dispatching", "running", "uploading") - - def get_state(self): - return self.data.state - - def set_state(self, state): - self._set_attribute("state", state) + # Builder - # Automatically update the state of the build (not on test builds) - if not self.test: - self.build.auto_update_state() - - state = property(get_state, set_state) - - def set_message(self, message): - if message: - message = "%s" % message - - self._set_attribute("message", message) - - message = property(lambda s: s.data.message, set_message) - - def get_builder(self): + @lazy_property + def builder(self): if self.data.builder_id: return self.backend.builders.get_by_id(self.data.builder_id) - def set_builder(self, builder, user=None): - log.info("Builder %s has been assigned to %s" % (builder.name, self.name)) - - self._set_attribute("builder_id", builder.id) - - # Log the event. - if user: - self.log("builder_assigned", builder=builder, user=user) - - builder = lazy_property(get_builder, set_builder) - @property def arch(self): return self.data.arch @property def duration(self): - if not self.time_started: - return 0 - - if self.time_finished: - delta = self.time_finished - self.time_started + """ + Returns the total build duration or elapsed time + """ + if self.has_finished(): + return self.finished_at - self.started_at else: - delta = datetime.datetime.utcnow() - self.time_started - - return delta.total_seconds() - - @property - def time_created(self): - return self.data.time_created - - def set_time_started(self, time_started): - self._set_attribute("time_started", time_started) - - time_started = property(lambda s: s.data.time_started, set_time_started) - - def set_time_finished(self, time_finished): - self._set_attribute("time_finished", time_finished) - - time_finished = property(lambda s: s.data.time_finished, set_time_finished) - - def set_start_not_before(self, start_not_before): - self._set_attribute("start_not_before", start_not_before) - - start_not_before = property(lambda s: s.data.start_not_before, set_start_not_before) - - def get_pkg_by_uuid(self, uuid): - pkg = self.backend.packages._get_package("SELECT packages.id FROM packages \ - JOIN jobs_packages ON jobs_packages.pkg_id = packages.id \ - WHERE jobs_packages.job_id = %s AND packages.uuid = %s", - self.id, uuid) - - if pkg: - pkg.job = self - return pkg + return datetime.datetime.utcnow() - self.started_at @lazy_property def logfiles(self): @@ -466,14 +438,6 @@ class Job(base.DataObject): self.db.execute("INSERT INTO jobs_packages(job_id, pkg_id) VALUES(%s, %s)", self.id, pkg.id) - def get_aborted_state(self): - return self.data.aborted_state - - def set_aborted_state(self, state): - self._set_attribute("aborted_state", state) - - aborted_state = property(get_aborted_state, set_aborted_state) - @property def message_recipients(self): l = [] diff --git a/src/database.sql b/src/database.sql index 51a10afd..232dc930 100644 --- a/src/database.sql +++ b/src/database.sql @@ -388,33 +388,6 @@ ALTER TABLE public.builds_id_seq OWNER TO pakfire; ALTER SEQUENCE public.builds_id_seq OWNED BY public.builds.id; --- --- Name: jobs; Type: TABLE; Schema: public; Owner: pakfire --- - -CREATE TABLE public.jobs ( - id integer NOT NULL, - uuid uuid DEFAULT gen_random_uuid() NOT NULL, - build_id integer NOT NULL, - state text DEFAULT 'pending'::text NOT NULL, - arch text NOT NULL, - time_created timestamp without time zone DEFAULT now() NOT NULL, - time_started timestamp without time zone, - time_finished timestamp without time zone, - start_not_before timestamp without time zone, - builder_id integer, - aborted_state integer DEFAULT 0 NOT NULL, - message text, - test boolean DEFAULT true NOT NULL, - superseeded_by integer, - dependency_check_succeeded boolean, - dependency_check_at timestamp without time zone, - CONSTRAINT jobs_states CHECK ((state = ANY (ARRAY['pending'::text, 'dispatching'::text, 'running'::text, 'uploading'::text, 'finished'::text, 'aborted'::text, 'download_error'::text, 'failed'::text]))) -); - - -ALTER TABLE public.jobs OWNER TO pakfire; - -- -- Name: builds_watchers; Type: TABLE; Schema: public; Owner: pakfire -- @@ -577,17 +550,42 @@ ALTER TABLE public.images_types_id_seq OWNER TO pakfire; ALTER SEQUENCE public.images_types_id_seq OWNED BY public.images_types.id; +-- +-- Name: jobs; Type: TABLE; Schema: public; Owner: pakfire +-- + +CREATE TABLE public.jobs ( + id integer NOT NULL, + uuid uuid DEFAULT gen_random_uuid() NOT NULL, + build_id integer NOT NULL, + arch text NOT NULL, + created_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, + started_at timestamp without time zone, + finished_at timestamp without time zone, + builder_id integer, + message text, + test boolean DEFAULT true NOT NULL, + superseeded_by integer, + dependency_check_succeeded boolean, + dependency_check_at timestamp without time zone, + deleted boolean DEFAULT false NOT NULL, + failed boolean DEFAULT false NOT NULL +); + + +ALTER TABLE public.jobs OWNER TO pakfire; + -- -- Name: job_queue; Type: VIEW; Schema: public; Owner: pakfire -- CREATE VIEW public.job_queue AS SELECT jobs.id AS job_id, - rank() OVER (ORDER BY (NOT jobs.test), builds.priority DESC, jobs.time_created) AS rank, + rank() OVER (ORDER BY (NOT jobs.test), builds.priority DESC, jobs.created_at) AS rank, jobs.arch FROM (public.jobs LEFT JOIN public.builds ON ((jobs.build_id = builds.id))) - WHERE ((jobs.state = 'pending'::text) AND (jobs.dependency_check_succeeded IS TRUE)); + WHERE ((jobs.deleted IS FALSE) AND (jobs.started_at IS NULL) AND (jobs.finished_at IS NULL) AND (jobs.dependency_check_succeeded IS TRUE)); ALTER TABLE public.job_queue OWNER TO pakfire; @@ -984,11 +982,11 @@ ALTER TABLE public.packages OWNER TO pakfire; CREATE VIEW public.package_estimated_build_times AS SELECT packages.name, jobs.arch, - avg((jobs.time_finished - jobs.time_started)) AS build_time + avg((jobs.finished_at - jobs.started_at)) AS build_time FROM ((public.jobs LEFT JOIN public.builds ON ((jobs.build_id = builds.id))) LEFT JOIN public.packages ON ((builds.pkg_id = packages.id))) - WHERE ((jobs.state = 'finished'::text) AND (jobs.test IS FALSE) AND (jobs.time_started IS NOT NULL) AND (jobs.time_finished IS NOT NULL)) + WHERE ((jobs.deleted IS FALSE) AND (jobs.started_at IS NOT NULL) AND (jobs.finished_at IS NOT NULL) AND (jobs.failed IS FALSE) AND (jobs.test IS FALSE)) GROUP BY packages.name, jobs.arch; @@ -1889,14 +1887,6 @@ ALTER TABLE ONLY public.images_types ADD CONSTRAINT idx_2198057_primary PRIMARY KEY (id); --- --- Name: jobs idx_2198063_primary; Type: CONSTRAINT; Schema: public; Owner: pakfire --- - -ALTER TABLE ONLY public.jobs - ADD CONSTRAINT idx_2198063_primary PRIMARY KEY (id); - - -- -- Name: jobs_packages idx_2198085_primary; Type: CONSTRAINT; Schema: public; Owner: pakfire -- @@ -2049,6 +2039,14 @@ ALTER TABLE ONLY public.jobs_packages ADD CONSTRAINT jobs_packages_unique UNIQUE (job_id, pkg_id); +-- +-- Name: jobs jobs_pkey; Type: CONSTRAINT; Schema: public; Owner: pakfire +-- + +ALTER TABLE ONLY public.jobs + ADD CONSTRAINT jobs_pkey PRIMARY KEY (id); + + -- -- Name: mirrors_checks mirrors_checks_pkey; Type: CONSTRAINT; Schema: public; Owner: pakfire -- @@ -2197,27 +2195,6 @@ CREATE INDEX idx_2198018_build_id ON public.builds_comments USING btree (build_i CREATE INDEX idx_2198018_user_id ON public.builds_comments USING btree (user_id); --- --- Name: idx_2198063_build_id; Type: INDEX; Schema: public; Owner: pakfire --- - -CREATE INDEX idx_2198063_build_id ON public.jobs USING btree (build_id); - - --- --- Name: idx_2198063_state; Type: INDEX; Schema: public; Owner: pakfire --- - -CREATE INDEX idx_2198063_state ON public.jobs USING btree (state); - - --- --- Name: idx_2198063_uuid; Type: INDEX; Schema: public; Owner: pakfire --- - -CREATE UNIQUE INDEX idx_2198063_uuid ON public.jobs USING btree (uuid); - - -- -- Name: idx_2198080_job_id; Type: INDEX; Schema: public; Owner: pakfire -- @@ -2338,10 +2315,10 @@ CREATE INDEX jobs_arch ON public.jobs USING btree (arch); -- --- Name: jobs_builders_active_jobs; Type: INDEX; Schema: public; Owner: pakfire +-- Name: jobs_build_id; Type: INDEX; Schema: public; Owner: pakfire -- -CREATE INDEX jobs_builders_active_jobs ON public.jobs USING btree (builder_id) WHERE (state = ANY (ARRAY['dispatching'::text, 'running'::text])); +CREATE INDEX jobs_build_id ON public.jobs USING btree (build_id) WHERE (deleted IS FALSE); -- @@ -2361,24 +2338,31 @@ CREATE INDEX jobs_buildroots_pkg_uuid ON public.jobs_buildroots USING btree (pkg -- --- Name: jobs_queue_ready; Type: INDEX; Schema: public; Owner: pakfire +-- Name: jobs_finished_at; Type: INDEX; Schema: public; Owner: pakfire +-- + +CREATE INDEX jobs_finished_at ON public.jobs USING btree (finished_at DESC) WHERE (finished_at IS NOT NULL); + + +-- +-- Name: jobs_pending; Type: INDEX; Schema: public; Owner: pakfire -- -CREATE INDEX jobs_queue_ready ON public.jobs USING btree (id) WHERE ((state = 'new'::text) AND (dependency_check_succeeded IS TRUE)); +CREATE INDEX jobs_pending ON public.jobs USING btree (id) WHERE ((deleted IS FALSE) AND (started_at IS NULL) AND (finished_at IS NULL) AND (dependency_check_succeeded IS TRUE)); -- --- Name: jobs_time_finished; Type: INDEX; Schema: public; Owner: pakfire +-- Name: jobs_started_at; Type: INDEX; Schema: public; Owner: pakfire -- -CREATE INDEX jobs_time_finished ON public.jobs USING btree (time_finished DESC) WHERE (time_finished IS NOT NULL); +CREATE INDEX jobs_started_at ON public.jobs USING btree (started_at) WHERE ((started_at IS NOT NULL) AND (finished_at IS NULL)); -- --- Name: jobs_time_started; Type: INDEX; Schema: public; Owner: pakfire +-- Name: jobs_uuid; Type: INDEX; Schema: public; Owner: pakfire -- -CREATE INDEX jobs_time_started ON public.jobs USING btree (time_started) WHERE ((time_started IS NOT NULL) AND (time_finished IS NULL)); +CREATE UNIQUE INDEX jobs_uuid ON public.jobs USING btree (uuid) WHERE (deleted IS FALSE); -- diff --git a/src/templates/modules/jobs/list.html b/src/templates/modules/jobs/list.html index 8d2c23c0..9633b01c 100644 --- a/src/templates/modules/jobs/list.html +++ b/src/templates/modules/jobs/list.html @@ -1,5 +1,5 @@ {% for job in jobs %} -
+
@@ -15,7 +15,7 @@ {# Show all packages that have been built #} {% if show_packages and job.packages %}