From: Michael Tremer Date: Mon, 24 Oct 2022 17:10:53 +0000 (+0000) Subject: sources: Refactor vast parts of this X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=c701b87e68c3af458d205b142effe848d19357e3;p=pbs.git sources: Refactor vast parts of this Signed-off-by: Michael Tremer --- diff --git a/src/buildservice/repository.py b/src/buildservice/repository.py index e00eeee6..44773bb0 100644 --- a/src/buildservice/repository.py +++ b/src/buildservice/repository.py @@ -444,6 +444,28 @@ class Repository(base.DataObject): if log: self._log_build("removed", build, from_repo=self, user=user) + # Sources + + @lazy_property + def sources(self): + sources = self.backend.sources._get_sources(""" + SELECT + * + FROM + sources + WHERE + deleted_at IS NULL + AND + repo_id = %s + ORDER BY + name + """, self.id, + ) + + return list(sources) + + # Builds + @lazy_property def builds(self): """ diff --git a/src/buildservice/sources.py b/src/buildservice/sources.py index c4694ca6..fb56a248 100644 --- a/src/buildservice/sources.py +++ b/src/buildservice/sources.py @@ -1,6 +1,5 @@ #!/usr/bin/python -import datetime import logging import os import pakfire @@ -12,6 +11,7 @@ import tempfile from . import base from . import git +from . import misc from .constants import * from .decorators import * @@ -28,17 +28,75 @@ VALID_TAGS = ( ) class Sources(base.Object): + def _get_sources(self, query, *args): + res = self.db.query(query, *args) + + for row in res: + yield Source(self.backend, row.id, data=row) + def _get_source(self, query, *args): res = self.db.get(query, *args) if res: return Source(self.backend, res.id, data=res) - def _get_sources(self, query, *args): - res = self.db.query(query, *args) + def get_by_id(self, id): + return self._get_source(""" + SELECT + * + FROM + sources + WHERE + id = %s + """, id, + ) + + def get_by_slug(self, slug): + return self._get_source(""" + SELECT + * + FROM + sources + WHERE + deleted_at IS NULL + AND + slug = %s + """, slug, + ) + + def create(self, repo, name, url, user): + # Make slug + slug = self._make_slug(name) + + # Insert into the database + return self._get_source(""" + INSERT INTO + sources( + name, + url, + created_by, + repo_id + ) + VALUES( + %s, %s, %s, %s + ) + RETURNING + * + """, name, url, user, repo, + ) + + def _make_slug(self, name): + for i in misc.infinity(): + slug = misc.normalize(name, iteration=i) + + # Check if the source already exists + source = self.get_by_slug(slug) + if source: + continue - for row in res: - yield Source(self.backend, row.id, data=row) + return slug + + # Commits def _get_commit(self, query, *args): res = self.db.get(query, *args) @@ -52,22 +110,6 @@ class Sources(base.Object): for row in res: yield Commit(self.backend, row.id, data=row) - def __iter__(self): - return self._get_sources("SELECT * FROM sources") - - def get_by_id(self, id): - return self._get_source("SELECT * FROM sources \ - WHERE id = %s", id) - - def get_by_distro(self, distro): - return self._get_sources("SELECT * FROM sources \ - WHERE distro_id = %s", distro.id) - - def update_revision(self, source_id, revision): - query = "UPDATE sources SET revision = %s WHERE id = %s" - - return self.db.execute(query, revision, source_id) - def get_commit_by_id(self, commit_id): commit = self.db.get("SELECT id FROM sources_commits WHERE id = %s", commit_id) @@ -157,8 +199,108 @@ class Sources(base.Object): commit.state = "finished" +class Source(base.DataObject): + table = "sources" + + # Name + + def get_name(self): + return self.data.name + + def set_name(self, name): + self._set_attribute("name", name) + + name = property(get_name, set_name) + + # Slug + + @property + def slug(self): + return self.data.slug + + # URL + + def get_url(self): + return self.data.url + + def set_url(self, url): + self._set_attribute("url", url) + + url = property(get_url, set_url) + + # Gitweb + + def get_gitweb(self): + return self.data.gitweb + + def set_gitweb(self, url): + self._set_attribute("gitweb", url) + + gitweb = property(get_gitweb, set_gitweb) + + # Revision + + @property + def revision(self): + return self.data.revision + + # Branch + + def get_branch(self): + return self.data.branch + + def set_branch(self, branch): + self._set_attribute("branch", branch) + + # Commits + + def create_commit(self, revision, author, committer, subject, body, date): + commit = self.backend.sources._get_commit("INSERT INTO source_commits(source_id, \ + revision, author, committer, subject, body, date) VALUES(%s, %s, %s, %s, %s, %s, %s) \ + RETURNING *", self.id, revision, author, committer, subject, body, date) + + # Commit + commit.source = self + + return commit + + @property + def commits(self): + # XXX using the ID is an incorrect way to sort them + return self.backend.sources._get_commits(""" + SELECT + * + FROM + source_commits + WHERE + source_id = %s + ORDER BY + id DESC + """, self.id, + ) + + def get_commit(self, revision): + commit = self.backend.sources._get_commit(""" + SELECT + * + FROM + sources_commits + WHERE + source_id = %s + AND + revision = %s + """, self.id, revision, + ) + + # Populate cache + if commit: + commit.source = self + + return commit + + class Commit(base.DataObject): - table = "sources_commits" + table = "source_commits" @property def revision(self): @@ -168,14 +310,6 @@ class Commit(base.DataObject): def source(self): return self.backend.sources.get_by_id(self.data.source_id) - @property - def distro(self): - """ - A shortcut to the distribution this commit - belongs to. - """ - return self.source.distro - def set_state(self, state): self._set_attribute("state", state) @@ -310,115 +444,3 @@ class Commit(base.DataObject): def packages(self): return self.backend.packages._get_packages("SELECT * FROM packages \ WHERE commit_id = %s", self.id) - - def reset(self): - """ - Removes all packages that have been created by this commit and - resets the state so it will be processed again. - """ - # Remove all packages and corresponding builds. - for pkg in self.packages: - # Check if there is a build associated with the package. - # If so, the whole build will be deleted. - if pkg.build: - pkg.build.delete() - - else: - # Delete the package. - pkg.delete() - - # Clear the cache. - del self.packages - - # Reset the state to 'pending'. - self.state = "pending" - - -class Source(base.DataObject): - table = "sources" - - def __len__(self): - ret = self.db.get("SELECT COUNT(*) AS len FROM sources_commits \ - WHERE source_id = %s", self.id) - - return ret.len - - def create_commit(self, revision, author, committer, subject, body, date): - commit = self.backend.sources._get_commit("INSERT INTO sources_commits(source_id, \ - revision, author, committer, subject, body, date) VALUES(%s, %s, %s, %s, %s, %s, %s) \ - RETURNING *", self.id, revision, author, committer, subject, body, date) - - # Commit - commit.source = self - - return commit - - @property - def info(self): - return { - "id" : self.id, - "name" : self.name, - "url" : self.url, - "path" : self.path, - "targetpath" : self.targetpath, - "revision" : self.revision, - "branch" : self.branch, - } - - @property - def name(self): - return self.data.name - - @property - def identifier(self): - return self.data.identifier - - @property - def url(self): - return self.data.url - - @property - def gitweb(self): - return self.data.gitweb - - @property - def revision(self): - return self.data.revision - - @property - def branch(self): - return self.data.branch - - @property - def builds(self): - return self.pakfire.builds.get_by_source(self.id) - - @lazy_property - def distro(self): - return self.pakfire.distros.get_by_id(self.data.distro_id) - - @property - def start_revision(self): - return self.data.revision - - @lazy_property - def head_revision(self): - return self.backend.sources._get_commit("SELECT * FROM sources_commits \ - WHERE source_id = %s ORDER BY id DESC LIMIT 1", self.id) - - def get_commits(self, limit=None, offset=None): - return self.backend.sources._get_commits("SELECT * FROM sources_commits \ - WHERE source_id = %s ORDER BY id DESC LIMIT %s OFFSET %s", self.id, limit, offset) - - def get_commit(self, revision): - commit = self.backend.sources._get_commit("SELECT * FROM sources_commits \ - WHERE source_id = %s AND revision = %s", self.id, revision) - - if commit: - commit.source = self - return commit - - @property - def pending_commits(self): - return self.backend.sources._get_commits("SELECT * FROM sources_commits \ - WHERE state = %s ORDER BY imported_at", "pending") diff --git a/src/database.sql b/src/database.sql index 0786ab8c..e986abd6 100644 --- a/src/database.sql +++ b/src/database.sql @@ -874,27 +874,10 @@ CREATE TABLE public.settings ( -- --- Name: sources; Type: TABLE; Schema: public; Owner: - +-- Name: source_commits; Type: TABLE; Schema: public; Owner: - -- -CREATE TABLE public.sources ( - id integer NOT NULL, - name text NOT NULL, - identifier text NOT NULL, - url text NOT NULL, - gitweb text, - revision text NOT NULL, - branch text NOT NULL, - updated timestamp without time zone, - distro_id integer NOT NULL -); - - --- --- Name: sources_commits; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.sources_commits ( +CREATE TABLE public.source_commits ( id integer NOT NULL, source_id integer NOT NULL, revision text NOT NULL, @@ -908,6 +891,27 @@ CREATE TABLE public.sources_commits ( ); +-- +-- Name: sources; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.sources ( + id integer NOT NULL, + name text NOT NULL, + slug text NOT NULL, + url text NOT NULL, + gitweb text, + revision text NOT NULL, + branch text NOT NULL, + last_updated_at timestamp without time zone, + repo_id integer NOT NULL, + created_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, + created_by integer NOT NULL, + deleted_at timestamp without time zone, + deleted_by integer +); + + -- -- Name: sources_commits_id_seq; Type: SEQUENCE; Schema: public; Owner: - -- @@ -924,7 +928,7 @@ CREATE SEQUENCE public.sources_commits_id_seq -- Name: sources_commits_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - -- -ALTER SEQUENCE public.sources_commits_id_seq OWNED BY public.sources_commits.id; +ALTER SEQUENCE public.sources_commits_id_seq OWNED BY public.source_commits.id; -- @@ -1166,17 +1170,17 @@ ALTER TABLE ONLY public.sessions ALTER COLUMN id SET DEFAULT nextval('public.ses -- --- Name: sources id; Type: DEFAULT; Schema: public; Owner: - +-- Name: source_commits id; Type: DEFAULT; Schema: public; Owner: - -- -ALTER TABLE ONLY public.sources ALTER COLUMN id SET DEFAULT nextval('public.sources_id_seq'::regclass); +ALTER TABLE ONLY public.source_commits ALTER COLUMN id SET DEFAULT nextval('public.sources_commits_id_seq'::regclass); -- --- Name: sources_commits id; Type: DEFAULT; Schema: public; Owner: - +-- Name: sources id; Type: DEFAULT; Schema: public; Owner: - -- -ALTER TABLE ONLY public.sources_commits ALTER COLUMN id SET DEFAULT nextval('public.sources_commits_id_seq'::regclass); +ALTER TABLE ONLY public.sources ALTER COLUMN id SET DEFAULT nextval('public.sources_id_seq'::regclass); -- @@ -1273,22 +1277,6 @@ ALTER TABLE ONLY public.repositories_builds ADD CONSTRAINT idx_2198189_primary PRIMARY KEY (id); --- --- Name: sources idx_2198213_primary; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.sources - ADD CONSTRAINT idx_2198213_primary PRIMARY KEY (id); - - --- --- Name: sources_commits idx_2198222_primary; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.sources_commits - ADD CONSTRAINT idx_2198222_primary PRIMARY KEY (id); - - -- -- Name: users idx_2198244_primary; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -1377,6 +1365,22 @@ ALTER TABLE ONLY public.sessions ADD CONSTRAINT sessions_session_id_key UNIQUE (session_id); +-- +-- Name: source_commits source_commits_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.source_commits + ADD CONSTRAINT source_commits_pkey PRIMARY KEY (id); + + +-- +-- Name: sources sources_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.sources + ADD CONSTRAINT sources_pkey PRIMARY KEY (id); + + -- -- Name: uploads uploads_id; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -1476,20 +1480,6 @@ CREATE UNIQUE INDEX idx_2198189_build_id ON public.repositories_builds USING btr CREATE UNIQUE INDEX idx_2198199_k ON public.settings USING btree (k); --- --- Name: idx_2198213_identifier; Type: INDEX; Schema: public; Owner: - --- - -CREATE UNIQUE INDEX idx_2198213_identifier ON public.sources USING btree (identifier); - - --- --- Name: idx_2198222_revision; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_2198222_revision ON public.sources_commits USING btree (revision); - - -- -- Name: idx_2198244_name; Type: INDEX; Schema: public; Owner: - -- @@ -1641,6 +1631,27 @@ CREATE INDEX repositories_builds_repo_id ON public.repositories_builds USING btr CREATE UNIQUE INDEX repositories_unique ON public.repositories USING btree (owner_id, distro_id, slug) WHERE (deleted IS FALSE); +-- +-- Name: source_commits_unique; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX source_commits_unique ON public.source_commits USING btree (source_id, revision); + + +-- +-- Name: sources_repo_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX sources_repo_id ON public.sources USING btree (repo_id) WHERE (deleted_at IS NULL); + + +-- +-- Name: sources_slug; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX sources_slug ON public.sources USING btree (slug) WHERE (deleted_at IS NULL); + + -- -- Name: uploads_uuid; Type: INDEX; Schema: public; Owner: - -- @@ -1852,7 +1863,7 @@ ALTER TABLE ONLY public.package_files -- ALTER TABLE ONLY public.packages - ADD CONSTRAINT packages_commit_id FOREIGN KEY (commit_id) REFERENCES public.sources_commits(id); + ADD CONSTRAINT packages_commit_id FOREIGN KEY (commit_id) REFERENCES public.source_commits(id); -- @@ -1960,19 +1971,35 @@ ALTER TABLE ONLY public.sessions -- --- Name: sources_commits sources_commits_source_id; Type: FK CONSTRAINT; Schema: public; Owner: - +-- Name: source_commits sources_commits_source_id; Type: FK CONSTRAINT; Schema: public; Owner: - -- -ALTER TABLE ONLY public.sources_commits +ALTER TABLE ONLY public.source_commits ADD CONSTRAINT sources_commits_source_id FOREIGN KEY (source_id) REFERENCES public.sources(id); -- --- Name: sources sources_distro_id; Type: FK CONSTRAINT; Schema: public; Owner: - +-- Name: sources sources_created_by; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.sources + ADD CONSTRAINT sources_created_by FOREIGN KEY (created_by) REFERENCES public.users(id); + + +-- +-- Name: sources sources_deleted_by; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.sources + ADD CONSTRAINT sources_deleted_by FOREIGN KEY (deleted_by) REFERENCES public.users(id); + + +-- +-- Name: sources sources_repo_id; Type: FK CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY public.sources - ADD CONSTRAINT sources_distro_id FOREIGN KEY (distro_id) REFERENCES public.distributions(id); + ADD CONSTRAINT sources_repo_id FOREIGN KEY (repo_id) REFERENCES public.repositories(id); --