From: Michael Tremer Date: Thu, 9 Mar 2023 16:16:58 +0000 (+0000) Subject: repositories: Refactor builds in repositories X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=ee20e731e3003c9ae9ad46b412687ca59926ac20;p=pbs.git repositories: Refactor builds in repositories Signed-off-by: Michael Tremer --- diff --git a/src/buildservice/builds.py b/src/buildservice/builds.py index 41494fba..e51cbf50 100644 --- a/src/buildservice/builds.py +++ b/src/buildservice/builds.py @@ -170,6 +170,9 @@ class Builds(base.Object): if group: build.group = group + # Add the build into its repository + repo.add_build(build, user=owner) + # Create all jobs build._create_jobs() @@ -232,10 +235,6 @@ class Build(base.DataObject): Deletes this build including all jobs, packages and the source package. """ - # If the build is in a repository, we need to remove it. - if self.repo: - self.repo.rem_build(self) - # Delete all jobs await asyncio.gather( *(job.delete(user=user) for job in self.jobs + self.test_jobs), @@ -604,21 +603,28 @@ class Build(base.DataObject): # Send an email to the user user.send_email(*args, build=self, **kwargs) - @lazy_property - def repo(self): - res = self.db.get("SELECT repo_id FROM repositories_builds \ - WHERE build_id = %s", self.id) - - if res: - return self.backend.repos.get_by_id(res.repo_id) + # Repositories - @property - def repo_time(self): - repo = self.db.get("SELECT time_added FROM repositories_builds \ - WHERE build_id = %s", self.id) + @lazy_property + def repos(self): + """ + Return a list of all repositories this package is in + """ + repos = self.backend.repos._get_repositories(""" + SELECT + repositories.* + FROM + repository_builds + LEFT JOIN + repositories ON repository_builds.repo_id = repositories.id + WHERE + repository_builds.build_id = %s + AND + repository_builds.removed_at IS NULL + """, self.id, + ) - if repo: - return repo.time_added + return list(repos) ## Bugs diff --git a/src/buildservice/jobs.py b/src/buildservice/jobs.py index a4091130..56a2dedc 100644 --- a/src/buildservice/jobs.py +++ b/src/buildservice/jobs.py @@ -312,6 +312,12 @@ class Job(base.DataObject): # Propagate any changes to the build await self.build._job_finished(job=self) + # On success, update all repositories + if success: + await asyncio.gather( + *(repo.has_changed() for repo in self.build.repos), + ) + def is_running(self): """ Returns True if this job is running diff --git a/src/buildservice/repository.py b/src/buildservice/repository.py index a60f1ee2..55a28fe9 100644 --- a/src/buildservice/repository.py +++ b/src/buildservice/repository.py @@ -439,25 +439,52 @@ class Repository(base.DataObject): # XXX TODO return [] - def add_build(self, build, user=None, log=True): - self.db.execute("INSERT INTO repositories_builds(repo_id, build_id, time_added)" - " VALUES(%s, %s, NOW())", self.id, build.id) + # Add/Remove Builds - # Update bug status. - build._update_bugs_helper(self) + def add_build(self, build, user=None): + """ + Adds a build to this repository + """ + self.db.execute(""" + INSERT INTO + repository_builds( + repo_id, + build_id, + added_by + ) + VALUES( + %s, %s, %s + )""", self.id, build, user, + ) - if log: - self._log_build("added", build, to_repo=self, user=user) + # Update the cache + build.repos.append(self) - def rem_build(self, build, user=None, log=True): - self.db.execute("DELETE FROM repositories_builds \ - WHERE repo_id = %s AND build_id = %s", self.id, build.id) + # Update bug status. + #build._update_bugs_helper(self) - # Force regenerating the index - self.update_forced = True + def remove_build(self, build, user=None): + """ + Removes a build from this repository + """ + self.db.execute(""" + UPDATE + repository_builds + SET + removed_at = CURRENT_TIMESTAMP, + removed_by = %s + WHERE + repo_id = %s + AND + build_id = %s + """, self.id, build, + ) - if log: - self._log_build("removed", build, from_repo=self, user=user) + # Update the cache + try: + build.repos.remove(self) + except IndexError: + pass # Sources @@ -500,15 +527,15 @@ class Repository(base.DataObject): SELECT builds.* FROM - repositories_builds + repository_builds LEFT JOIN - builds ON repositories_builds.build_id = builds.id + builds ON repository_builds.build_id = builds.id LEFT JOIN packages ON builds.pkg_id = packages.id WHERE - builds.deleted IS FALSE + builds.deleted_at IS NULL AND - repositories_builds.repo_id = %s + repository_builds.repo_id = %s ORDER BY packages.name, packages.evr""", self.id, @@ -524,13 +551,13 @@ class Repository(base.DataObject): SELECT builds.* FROM - repositories_builds + repository_builds LEFT JOIN - builds ON repositories_builds.build_id = builds.id + builds ON repository_builds.build_id = builds.id LEFT JOIN packages ON builds.pkg_id = packages.id WHERE - repositories_builds.repo_id = %s + repository_builds.repo_id = %s AND packages.name = %s ORDER BY @@ -546,9 +573,9 @@ class Repository(base.DataObject): SELECT packages.* FROM - repositories_builds + repository_builds LEFT JOIN - builds ON repositories_builds.build_id = builds.id + builds ON repository_builds.build_id = builds.id LEFT JOIN packages ON builds.pkg_id = packages.id WHERE @@ -556,18 +583,19 @@ class Repository(base.DataObject): AND packages.deleted_at IS NULL AND - repositories_builds.repo_id = %s + repository_builds.repo_id = %s + AND + repository_builds.removed_at IS NULL """, self.id, ) - else: packages = self.backend.packages._get_packages(""" SELECT packages.* FROM - repositories_builds + repository_builds LEFT JOIN - builds ON repositories_builds.build_id = builds.id + builds ON repository_builds.build_id = builds.id LEFT JOIN jobs ON builds.id = jobs.build_id LEFT JOIN @@ -581,7 +609,9 @@ class Repository(base.DataObject): AND packages.deleted_at IS NULL AND - repositories_builds.repo_id = %s + repository_builds.repo_id = %s + AND + repository_builds.removed_at IS NULL AND packages.arch = ANY(%s) """, @@ -599,9 +629,9 @@ class Repository(base.DataObject): packages.arch AS arch, SUM(packages.filesize) AS size FROM - repositories_builds + repository_builds LEFT JOIN - builds ON repositories_builds.build_id = builds.id + builds ON repository_builds.build_id = builds.id LEFT JOIN jobs ON builds.id = jobs.build_id LEFT JOIN @@ -615,7 +645,9 @@ class Repository(base.DataObject): AND packages.deleted_at IS NULL AND - repositories_builds.repo_id = %s + repository_builds.repo_id = %s + AND + repository_builds.removed_at IS NULL GROUP BY packages.arch """, self.id, @@ -638,6 +670,23 @@ class Repository(base.DataObject): return self.backend.pakfire(distro=self.distro, repos=[self], **kwargs) + async def has_changed(self): + """ + Convenience function that should be called when a package has been + added/removed from this repository. + """ + # Run the update process at the next convenient moment + self.backend.run_task(self.update) + + async def update(self): + """ + Called to perform an update of this repository + """ + # Write/rewrite the repository + await self.write() + + # XXX Perform dependency checks on all pending packages + # Write repository async def write(self, *args, **kwargs): diff --git a/src/database.sql b/src/database.sql index 5904c615..e7711479 100644 --- a/src/database.sql +++ b/src/database.sql @@ -815,22 +815,10 @@ CREATE TABLE public.repositories ( -- --- Name: repositories_builds; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.repositories_builds ( - id integer NOT NULL, - repo_id integer NOT NULL, - build_id bigint NOT NULL, - time_added timestamp without time zone NOT NULL -); - - --- --- Name: repositories_builds_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- Name: repositories_id_seq; Type: SEQUENCE; Schema: public; Owner: - -- -CREATE SEQUENCE public.repositories_builds_id_seq +CREATE SEQUENCE public.repositories_id_seq START WITH 1 INCREMENT BY 1 NO MINVALUE @@ -839,17 +827,32 @@ CREATE SEQUENCE public.repositories_builds_id_seq -- --- Name: repositories_builds_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- Name: repositories_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - -- -ALTER SEQUENCE public.repositories_builds_id_seq OWNED BY public.repositories_builds.id; +ALTER SEQUENCE public.repositories_id_seq OWNED BY public.repositories.id; -- --- Name: repositories_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- Name: repository_builds; Type: TABLE; Schema: public; Owner: - -- -CREATE SEQUENCE public.repositories_id_seq +CREATE TABLE public.repository_builds ( + id integer NOT NULL, + repo_id integer NOT NULL, + build_id bigint NOT NULL, + added_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, + added_by integer, + removed_at timestamp without time zone, + removed_by integer +); + + +-- +-- Name: repository_builds_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.repository_builds_id_seq START WITH 1 INCREMENT BY 1 NO MINVALUE @@ -858,10 +861,10 @@ CREATE SEQUENCE public.repositories_id_seq -- --- Name: repositories_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- Name: repository_builds_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - -- -ALTER SEQUENCE public.repositories_id_seq OWNED BY public.repositories.id; +ALTER SEQUENCE public.repository_builds_id_seq OWNED BY public.repository_builds.id; -- @@ -1198,10 +1201,10 @@ ALTER TABLE ONLY public.repositories ALTER COLUMN id SET DEFAULT nextval('public -- --- Name: repositories_builds id; Type: DEFAULT; Schema: public; Owner: - +-- Name: repository_builds id; Type: DEFAULT; Schema: public; Owner: - -- -ALTER TABLE ONLY public.repositories_builds ALTER COLUMN id SET DEFAULT nextval('public.repositories_builds_id_seq'::regclass); +ALTER TABLE ONLY public.repository_builds ALTER COLUMN id SET DEFAULT nextval('public.repository_builds_id_seq'::regclass); -- @@ -1319,14 +1322,6 @@ ALTER TABLE ONLY public.mirrors ADD CONSTRAINT idx_2198115_primary PRIMARY KEY (id); --- --- Name: repositories_builds idx_2198189_primary; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.repositories_builds - ADD CONSTRAINT idx_2198189_primary PRIMARY KEY (id); - - -- -- Name: users idx_2198244_primary; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -1384,19 +1379,19 @@ ALTER TABLE ONLY public.packages -- --- Name: repositories_builds repositories_builds_unique; Type: CONSTRAINT; Schema: public; Owner: - +-- Name: repositories repositories_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- -ALTER TABLE ONLY public.repositories_builds - ADD CONSTRAINT repositories_builds_unique UNIQUE (repo_id, build_id); +ALTER TABLE ONLY public.repositories + ADD CONSTRAINT repositories_pkey PRIMARY KEY (id); -- --- Name: repositories repositories_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- Name: repository_builds repository_builds_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- -ALTER TABLE ONLY public.repositories - ADD CONSTRAINT repositories_pkey PRIMARY KEY (id); +ALTER TABLE ONLY public.repository_builds + ADD CONSTRAINT repository_builds_pkey PRIMARY KEY (id); -- @@ -1537,13 +1532,6 @@ CREATE UNIQUE INDEX builds_uuid ON public.builds USING btree (uuid) WHERE (delet CREATE UNIQUE INDEX distributions_unique ON public.distributions USING btree (distro_id, version_id) WHERE (deleted IS FALSE); --- --- Name: idx_2198189_build_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE UNIQUE INDEX idx_2198189_build_id ON public.repositories_builds USING btree (build_id); - - -- -- Name: idx_2198199_k; Type: INDEX; Schema: public; Owner: - -- @@ -1689,17 +1677,31 @@ CREATE UNIQUE INDEX repo_builds_unique ON public.repo_builds USING btree (repo_i -- --- Name: repositories_builds_repo_id; Type: INDEX; Schema: public; Owner: - +-- Name: repositories_unique; Type: INDEX; Schema: public; Owner: - -- -CREATE INDEX repositories_builds_repo_id ON public.repositories_builds USING btree (repo_id); +CREATE UNIQUE INDEX repositories_unique ON public.repositories USING btree (owner_id, distro_id, slug) WHERE (deleted IS FALSE); -- --- Name: repositories_unique; Type: INDEX; Schema: public; Owner: - +-- Name: repository_builds_build_id; Type: INDEX; Schema: public; Owner: - -- -CREATE UNIQUE INDEX repositories_unique ON public.repositories USING btree (owner_id, distro_id, slug) WHERE (deleted IS FALSE); +CREATE INDEX repository_builds_build_id ON public.repository_builds USING btree (build_id); + + +-- +-- Name: repository_builds_repo_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX repository_builds_repo_id ON public.repository_builds USING btree (repo_id) WHERE (removed_at IS NULL); + + +-- +-- Name: repository_builds_unique; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX repository_builds_unique ON public.repository_builds USING btree (repo_id, build_id) WHERE (removed_at IS NULL); -- @@ -2009,22 +2011,6 @@ ALTER TABLE ONLY public.repo_builds ADD CONSTRAINT repo_builds_repo_id FOREIGN KEY (repo_id) REFERENCES public.repositories(id); --- --- Name: repositories_builds repositories_builds_build_id; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.repositories_builds - ADD CONSTRAINT repositories_builds_build_id FOREIGN KEY (build_id) REFERENCES public.builds(id); - - --- --- Name: repositories_builds repositories_builds_repo_id; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.repositories_builds - ADD CONSTRAINT repositories_builds_repo_id FOREIGN KEY (repo_id) REFERENCES public.repositories(id); - - -- -- Name: repositories repositories_distro_id; Type: FK CONSTRAINT; Schema: public; Owner: - -- @@ -2057,6 +2043,38 @@ ALTER TABLE ONLY public.repositories ADD CONSTRAINT repositories_parent_id FOREIGN KEY (parent_id) REFERENCES public.repositories(id); +-- +-- Name: repository_builds repository_builds_added_by; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.repository_builds + ADD CONSTRAINT repository_builds_added_by FOREIGN KEY (added_by) REFERENCES public.users(id); + + +-- +-- Name: repository_builds repository_builds_build_id; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.repository_builds + ADD CONSTRAINT repository_builds_build_id FOREIGN KEY (build_id) REFERENCES public.builds(id); + + +-- +-- Name: repository_builds repository_builds_removed_by; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.repository_builds + ADD CONSTRAINT repository_builds_removed_by FOREIGN KEY (removed_by) REFERENCES public.users(id); + + +-- +-- Name: repository_builds repository_builds_repo_id; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.repository_builds + ADD CONSTRAINT repository_builds_repo_id FOREIGN KEY (repo_id) REFERENCES public.repositories(id); + + -- -- Name: sessions sessions_user_id; Type: FK CONSTRAINT; Schema: public; Owner: - -- diff --git a/src/web/builds.py b/src/web/builds.py index 29c1c2ba..ca005c21 100644 --- a/src/web/builds.py +++ b/src/web/builds.py @@ -32,7 +32,7 @@ class ShowHandler(base.BaseHandler): bugs = await build.get_bugs() self.render("builds/show.html", build=build, pkg=build.pkg, - distro=build.distro, bugs=bugs, repo=build.repo) + distro=build.distro, bugs=bugs) class DeleteHandler(base.BaseHandler):