From 3d7f0b67fbe02eaaddd9f7f9b2d77d7598d435cb Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Mon, 15 May 2023 10:25:15 +0000 Subject: [PATCH] packages: Toy around with the release-monitoring.org API Signed-off-by: Michael Tremer --- Makefile.am | 1 + src/buildservice/__init__.py | 38 +-- src/buildservice/packages.py | 20 ++ src/buildservice/releasemonitoring.py | 338 +++++++++++++++++++++++ src/database.sql | 170 ++++++++++++ src/scripts/pakfire-build-service | 24 ++ src/templates/packages/modules/info.html | 12 + 7 files changed, 585 insertions(+), 18 deletions(-) create mode 100644 src/buildservice/releasemonitoring.py diff --git a/Makefile.am b/Makefile.am index 0a43f456..184926af 100644 --- a/Makefile.am +++ b/Makefile.am @@ -102,6 +102,7 @@ buildservice_PYTHON = \ src/buildservice/mirrors.py \ src/buildservice/misc.py \ src/buildservice/packages.py \ + src/buildservice/releasemonitoring.py \ src/buildservice/repository.py \ src/buildservice/sessions.py \ src/buildservice/settings.py \ diff --git a/src/buildservice/__init__.py b/src/buildservice/__init__.py index 9646135f..98c06621 100644 --- a/src/buildservice/__init__.py +++ b/src/buildservice/__init__.py @@ -27,6 +27,7 @@ from . import logstreams from . import messages from . import mirrors from . import packages +from . import releasemonitoring from . import repository from . import settings from . import sessions @@ -61,26 +62,27 @@ class Backend(object): # Global pakfire settings (from database). self.settings = settings.Settings(self) - self.aws = aws.AWS(self) - self.builds = builds.Builds(self) - self.cache = cache.Cache(self) - self.jobs = jobs.Jobs(self) - self.builders = builders.Builders(self) - self.distros = distribution.Distributions(self) - self.events = events.Events(self) - self.keys = keys.Keys(self) - self.logstreams = logstreams.LogStreams(self) - self.messages = messages.Messages(self) - self.mirrors = mirrors.Mirrors(self) - self.packages = packages.Packages(self) - self.repos = repository.Repositories(self) - self.sessions = sessions.Sessions(self) - self.sources = sources.Sources(self) - self.uploads = uploads.Uploads(self) - self.users = users.Users(self) + self.aws = aws.AWS(self) + self.builds = builds.Builds(self) + self.cache = cache.Cache(self) + self.jobs = jobs.Jobs(self) + self.builders = builders.Builders(self) + self.distros = distribution.Distributions(self) + self.events = events.Events(self) + self.keys = keys.Keys(self) + self.logstreams = logstreams.LogStreams(self) + self.messages = messages.Messages(self) + self.mirrors = mirrors.Mirrors(self) + self.packages = packages.Packages(self) + self.releasemonitoring = releasemonitoring.ReleaseMonitoring(self) + self.repos = repository.Repositories(self) + self.sessions = sessions.Sessions(self) + self.sources = sources.Sources(self) + self.uploads = uploads.Uploads(self) + self.users = users.Users(self) # Open a connection to bugzilla. - self.bugzilla = bugtracker.Bugzilla(self) + self.bugzilla = bugtracker.Bugzilla(self) # Create a temporary directory self._create_tmp_path() diff --git a/src/buildservice/packages.py b/src/buildservice/packages.py index 05dc51e0..1ab79ec4 100644 --- a/src/buildservice/packages.py +++ b/src/buildservice/packages.py @@ -553,6 +553,26 @@ class Package(base.DataObject): """ return await self.backend.open(self.path) + # Release Monitoring + + @lazy_property + def release_monitoring(self): + """ + Returns the release monitoring package if one exists + """ + if not self.is_source(): + raise AttributeError("Release Monitoring is only available for source packages") + + return self.backend.releasemonitoring.get_package(self.distro.name, self.name) + + @property + def latest_release(self): + """ + The latest known release of this package (if known) + """ + if self.release_monitoring: + return self.release_monitoring.latest_release + class File(base.Object): def init(self, package, data): diff --git a/src/buildservice/releasemonitoring.py b/src/buildservice/releasemonitoring.py new file mode 100644 index 00000000..9d13af34 --- /dev/null +++ b/src/buildservice/releasemonitoring.py @@ -0,0 +1,338 @@ +############################################################################### +# # +# Pakfire - The IPFire package management system # +# Copyright (C) 2022 Pakfire development team # +# # +# This program is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# This program is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with this program. If not, see . # +# # +############################################################################### + +import json +import logging +import tornado.httpclient +import urllib.parse + +from . import base +from . import database +from .decorators import * + +# Setup logging +log = logging.getLogger("pbs.releasemonitoring") + +class ReleaseMonitoring(base.Object): + baseurl = "https://release-monitoring.org" + + def init(self): + self.client = tornado.httpclient.AsyncHTTPClient() + + @property + def api_key(self): + return self.settings.get("release-monitoring-api-key") + + async def _request(self, method, url, data=None): + # Authenticate to the API + headers = { + "Authorization" : "Token %s" % self.api_key, + } + + # Compose the url + url = urllib.parse.urljoin(self.baseurl, url) + + if method == "GET": + url = "%s?%s" % (url, urllib.parse.urlencode(data)) + + # Reset data + data = None + + # For POST requests, encode the payload in JSON + elif method == "POST": + data = urllib.parse.urlencode(data) + + # Create a new request + req = tornado.httpclient.HTTPRequest( + method=method, url=url, headers=headers, body=data, + ) + + # Send the request and wait for a response + try: + res = await self.client.fetch(req) + + except tornado.httpclient.HTTPError as e: + print(e.response.body) + + raise e + + log.debug("Response received in %.2fms" % (res.request_time * 1000)) + + # Decode JSON response + if res.body: + return json.loads(res.body) + + # Return an empty response + return {} + + # Packages + + def _get_package(self, query, *args): + res = self.db.get(query, *args) + + if res: + return Package(self.backend, id=res.id, data=res) + + def get_package(self, distro, name): + return self._get_package(""" + SELECT + * + FROM + release_monitoring_packages + WHERE + deleted_at IS NULL + AND + distro = %s + AND + name = %s + """, distro, name, + ) + + def create_package(self, distro, name): + pass + + async def search(self, name): + """ + Returns a bunch of packages that match the given name + """ + # Send the request + response = await self._request("GET", "/api/v2/projects", + { + "name" : name, + "items_per_page" : 250, + }, + ) + + # Return all packages + return [database.Row(item) for item in response.get("items")] + + +class Package(base.DataObject): + table = "release_monitoring_packages" + + @property + def distro(self): + return self.data.distro + + @property + def name(self): + return self.data.name + + # Releases + + def _get_releases(self, query, *args): + res = self.db.query(query, *args) + + for row in res: + yield Release(self.backend, id=row.id, data=row) + + def _get_release(self, query, *args): + res = self.db.get(query, *args) + + if res: + return Release(self.backend, id=res.id, data=res) + + @property + def latest_release(self): + """ + Returns the latest release of this package + """ + return self._get_release(""" + SELECT + * + FROM + release_monitoring_releases + WHERE + deleted_at IS NULL + AND + package_id = %s + ORDER BY + created_at DESC + LIMIT 1 + """, self.id, + ) + + @property + def releases(self): + releases = self._get_releases(""" + SELECT + * + FROM + release_monitoring_releases + WHERE + deleted_at IS NULL + AND + package_id = %s + ORDER BY + created_at DESC + """, self.id, + ) + + return list(releases) + + def _create_release(self, version): + """ + Creates a new release for this package + """ + # XXX Do we need to check whether we are going backwards? + + release = self._get_release(""" + INSERT INTO + release_monitoring_releases + ( + package_id, + version + ) + VALUES + ( + %s, %s + ) + ON CONFLICT + (package_id, version) + WHERE + deleted_at IS NULL + DO + NOTHING + RETURNING + * + """, self.id, version, + ) + + # Return the release + return release + + # Update + + async def _fetch(self): + """ + Fetches all sorts of information about a package + """ + response = await self.backend.releasemonitoring._request( + "GET", "/api/v2/packages/", { + "distribution" : self.distro, + "name" : self.name, + }, + ) + + # Fetch all received items + items = response.get("items") + + # Is the package known? + if not items: + # Automatically try to create the package + if await self._auto_create_package(): + return await self._fetch() + + raise PackageNotFoundError(self.name) + + # Did we receive more than one item? + elif len(items) > 1: + raise RuntimeError("More than one item received for %s" % name) + + for item in items: + return database.Row({ + "version" : item.get("version"), + "stable_version" : item.get("stable_version"), + }) + + async def _auto_create_package(self): + """ + Tries to automatically create a package + """ + packages = await self.backend.releasemonitoring.search(self.name) + + # Use the first exact match + for package in packages: + if not package.name == self.name: + continue + + print(package) + + # Automatically create a mapping + await self.update_mapping( + project_name=package.name, + project_ecosystem=package.ecosystem, + ) + + return True + + # We could not create a mapping + return False + + async def update_mapping(self, project_name=None, project_ecosystem=None): + if project_name is None: + project_name = self.name + + # Send request + await self.backend.releasemonitoring._request("POST", "/api/v2/packages/", + { + "distribution" : self.distro, + "package_name" : self.name, + "project_name" : project_name, + "project_ecosystem" : project_ecosystem, + }, + ) + + async def update(self): + """ + Update the information we have about this package + """ + # Pull information from the API + package = await self._fetch() + + # Try to create a new release + release = self._create_release(package.stable_version) + + # Return the new release + return release + + +class Release(base.DataObject): + table = "release_monitoring_releases" + + def __str__(self): + return self.version + + # Version + + @property + def version(self): + return self.data.version + + # Builds + + @lazy_property + def builds(self): + builds = self.backend.builds._get_builds(""" + SELECT + builds.* + FROM + release_monitoring_release_builds + LEFT JOIN + builds ON release_monitoring_release_builds.build_id = builds.id + WHERE + release_monitoring_release_builds.release_id = %s + AND + builds.deleted_at IS NULL + """, self.id, + ) + + # Return builds by distribution + return { build.distro : build for build in builds } diff --git a/src/database.sql b/src/database.sql index 5ba1e8ff..52657065 100644 --- a/src/database.sql +++ b/src/database.sql @@ -758,6 +758,85 @@ CREATE VIEW public.relation_sizes AS ORDER BY (pg_relation_size((c.oid)::regclass)) DESC; +-- +-- Name: release_monitoring_packages; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.release_monitoring_packages ( + id integer NOT NULL, + distro text NOT NULL, + name text NOT NULL, + created_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, + deleted_at timestamp without time zone, + auto_update boolean DEFAULT true NOT NULL +); + + +-- +-- Name: release_monitoring_packages_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.release_monitoring_packages_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: release_monitoring_packages_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.release_monitoring_packages_id_seq OWNED BY public.release_monitoring_packages.id; + + +-- +-- Name: release_monitoring_release_builds; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.release_monitoring_release_builds ( + id integer NOT NULL, + release_id integer NOT NULL, + distro_id integer NOT NULL, + build_id integer NOT NULL +); + + +-- +-- Name: release_monitoring_releases; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.release_monitoring_releases ( + id integer NOT NULL, + package_id integer NOT NULL, + version text NOT NULL, + created_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, + deleted_at timestamp without time zone +); + + +-- +-- Name: release_monitoring_releases_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.release_monitoring_releases_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: release_monitoring_releases_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.release_monitoring_releases_id_seq OWNED BY public.release_monitoring_releases.id; + + -- -- Name: repo_builds; Type: TABLE; Schema: public; Owner: - -- @@ -1158,6 +1237,20 @@ ALTER TABLE ONLY public.mirrors_checks ALTER COLUMN id SET DEFAULT nextval('publ ALTER TABLE ONLY public.packages ALTER COLUMN id SET DEFAULT nextval('public.packages_id_seq'::regclass); +-- +-- Name: release_monitoring_packages id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.release_monitoring_packages ALTER COLUMN id SET DEFAULT nextval('public.release_monitoring_packages_id_seq'::regclass); + + +-- +-- Name: release_monitoring_releases id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.release_monitoring_releases ALTER COLUMN id SET DEFAULT nextval('public.release_monitoring_releases_id_seq'::regclass); + + -- -- Name: repositories id; Type: DEFAULT; Schema: public; Owner: - -- @@ -1319,6 +1412,30 @@ ALTER TABLE ONLY public.packages ADD CONSTRAINT packages_pkey PRIMARY KEY (id); +-- +-- Name: release_monitoring_packages release_monitoring_packages_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.release_monitoring_packages + ADD CONSTRAINT release_monitoring_packages_pkey PRIMARY KEY (id); + + +-- +-- Name: release_monitoring_release_builds release_monitoring_release_builds_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.release_monitoring_release_builds + ADD CONSTRAINT release_monitoring_release_builds_pkey PRIMARY KEY (id); + + +-- +-- Name: release_monitoring_releases release_monitoring_releases_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.release_monitoring_releases + ADD CONSTRAINT release_monitoring_releases_pkey PRIMARY KEY (id); + + -- -- Name: repositories repositories_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -1601,6 +1718,27 @@ CREATE UNIQUE INDEX package_search_index_unique ON public.package_search_index U CREATE INDEX packages_name ON public.packages USING btree (name); +-- +-- Name: release_monitoring_packages_unique; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX release_monitoring_packages_unique ON public.release_monitoring_packages USING btree (distro, name) WHERE (deleted_at IS NULL); + + +-- +-- Name: release_monitoring_release_builds_release_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX release_monitoring_release_builds_release_id ON public.release_monitoring_release_builds USING btree (release_id); + + +-- +-- Name: release_monitoring_releases_unique; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX release_monitoring_releases_unique ON public.release_monitoring_releases USING btree (package_id, version) WHERE (deleted_at IS NULL); + + -- -- Name: repo_builds_build_id; Type: INDEX; Schema: public; Owner: - -- @@ -1957,6 +2095,38 @@ ALTER TABLE ONLY public.packages ADD CONSTRAINT packages_distro_id FOREIGN KEY (distro_id) REFERENCES public.distributions(id); +-- +-- Name: release_monitoring_release_builds release_monitoring_release_builds_build_id; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.release_monitoring_release_builds + ADD CONSTRAINT release_monitoring_release_builds_build_id FOREIGN KEY (build_id) REFERENCES public.builds(id); + + +-- +-- Name: release_monitoring_release_builds release_monitoring_release_builds_distro_id; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.release_monitoring_release_builds + ADD CONSTRAINT release_monitoring_release_builds_distro_id FOREIGN KEY (distro_id) REFERENCES public.distributions(id); + + +-- +-- Name: release_monitoring_release_builds release_monitoring_release_builds_release_id; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.release_monitoring_release_builds + ADD CONSTRAINT release_monitoring_release_builds_release_id FOREIGN KEY (release_id) REFERENCES public.release_monitoring_releases(id); + + +-- +-- Name: release_monitoring_releases release_monitoring_releases_package_id; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.release_monitoring_releases + ADD CONSTRAINT release_monitoring_releases_package_id FOREIGN KEY (package_id) REFERENCES public.release_monitoring_packages(id); + + -- -- Name: repo_builds repo_builds_added_by; Type: FK CONSTRAINT; Schema: public; Owner: - -- diff --git a/src/scripts/pakfire-build-service b/src/scripts/pakfire-build-service index 5ac6451d..ac5256e2 100644 --- a/src/scripts/pakfire-build-service +++ b/src/scripts/pakfire-build-service @@ -37,6 +37,9 @@ class Cli(object): # Messages "messages:queue:send" : self.backend.messages.queue.send, + # Release Monitoring + "releasemonitoring:update" : self._release_monitoring_update, + # Repositories "repos:fetch-sources" : self.backend.repos.fetch_sources, "repos:relaunch-pending-jobs" : self._repos_relaunch_pending_jobs, @@ -167,6 +170,27 @@ class Cli(object): for repo in sorted(self.backend.repos): await repo.relaunch_pending_jobs() + async def _release_monitoring_update(self, *names): + """ + Performs an update for the given package + """ + for name in names: + build = self.backend.builds.get_latest_by_name(name) + if not build: + log.error("Could not find package %s" % name) + continue + + with self.backend.db.transaction(): + # Show what we are working on + print(build.pkg) + + # Perform the update + release = await build.pkg.release_monitoring.update() + + # Show any new releases + if release: + print(" Found new release: %s" % release) + async def main(): cli = Cli() diff --git a/src/templates/packages/modules/info.html b/src/templates/packages/modules/info.html index 7bbd4217..e8771640 100644 --- a/src/templates/packages/modules/info.html +++ b/src/templates/packages/modules/info.html @@ -19,6 +19,18 @@