src/buildservice/mirrors.py \
src/buildservice/misc.py \
src/buildservice/packages.py \
+ src/buildservice/releasemonitoring.py \
src/buildservice/repository.py \
src/buildservice/sessions.py \
src/buildservice/settings.py \
from . import messages
from . import mirrors
from . import packages
+from . import releasemonitoring
from . import repository
from . import settings
from . import sessions
# Global pakfire settings (from database).
self.settings = settings.Settings(self)
- self.aws = aws.AWS(self)
- self.builds = builds.Builds(self)
- self.cache = cache.Cache(self)
- self.jobs = jobs.Jobs(self)
- self.builders = builders.Builders(self)
- self.distros = distribution.Distributions(self)
- self.events = events.Events(self)
- self.keys = keys.Keys(self)
- self.logstreams = logstreams.LogStreams(self)
- self.messages = messages.Messages(self)
- self.mirrors = mirrors.Mirrors(self)
- self.packages = packages.Packages(self)
- self.repos = repository.Repositories(self)
- self.sessions = sessions.Sessions(self)
- self.sources = sources.Sources(self)
- self.uploads = uploads.Uploads(self)
- self.users = users.Users(self)
+ self.aws = aws.AWS(self)
+ self.builds = builds.Builds(self)
+ self.cache = cache.Cache(self)
+ self.jobs = jobs.Jobs(self)
+ self.builders = builders.Builders(self)
+ self.distros = distribution.Distributions(self)
+ self.events = events.Events(self)
+ self.keys = keys.Keys(self)
+ self.logstreams = logstreams.LogStreams(self)
+ self.messages = messages.Messages(self)
+ self.mirrors = mirrors.Mirrors(self)
+ self.packages = packages.Packages(self)
+ self.releasemonitoring = releasemonitoring.ReleaseMonitoring(self)
+ self.repos = repository.Repositories(self)
+ self.sessions = sessions.Sessions(self)
+ self.sources = sources.Sources(self)
+ self.uploads = uploads.Uploads(self)
+ self.users = users.Users(self)
# Open a connection to bugzilla.
- self.bugzilla = bugtracker.Bugzilla(self)
+ self.bugzilla = bugtracker.Bugzilla(self)
# Create a temporary directory
self._create_tmp_path()
"""
return await self.backend.open(self.path)
+ # Release Monitoring
+
+ @lazy_property
+ def release_monitoring(self):
+ """
+ Returns the release monitoring package if one exists
+ """
+ if not self.is_source():
+ raise AttributeError("Release Monitoring is only available for source packages")
+
+ return self.backend.releasemonitoring.get_package(self.distro.name, self.name)
+
+ @property
+ def latest_release(self):
+ """
+ The latest known release of this package (if known)
+ """
+ if self.release_monitoring:
+ return self.release_monitoring.latest_release
+
class File(base.Object):
def init(self, package, data):
--- /dev/null
+###############################################################################
+# #
+# Pakfire - The IPFire package management system #
+# Copyright (C) 2022 Pakfire development team #
+# #
+# This program is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# This program is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with this program. If not, see <http://www.gnu.org/licenses/>. #
+# #
+###############################################################################
+
+import json
+import logging
+import tornado.httpclient
+import urllib.parse
+
+from . import base
+from . import database
+from .decorators import *
+
+# Setup logging
+log = logging.getLogger("pbs.releasemonitoring")
+
+class ReleaseMonitoring(base.Object):
+ baseurl = "https://release-monitoring.org"
+
+ def init(self):
+ self.client = tornado.httpclient.AsyncHTTPClient()
+
+ @property
+ def api_key(self):
+ return self.settings.get("release-monitoring-api-key")
+
+ async def _request(self, method, url, data=None):
+ # Authenticate to the API
+ headers = {
+ "Authorization" : "Token %s" % self.api_key,
+ }
+
+ # Compose the url
+ url = urllib.parse.urljoin(self.baseurl, url)
+
+ if method == "GET":
+ url = "%s?%s" % (url, urllib.parse.urlencode(data))
+
+ # Reset data
+ data = None
+
+ # For POST requests, encode the payload in JSON
+ elif method == "POST":
+ data = urllib.parse.urlencode(data)
+
+ # Create a new request
+ req = tornado.httpclient.HTTPRequest(
+ method=method, url=url, headers=headers, body=data,
+ )
+
+ # Send the request and wait for a response
+ try:
+ res = await self.client.fetch(req)
+
+ except tornado.httpclient.HTTPError as e:
+ print(e.response.body)
+
+ raise e
+
+ log.debug("Response received in %.2fms" % (res.request_time * 1000))
+
+ # Decode JSON response
+ if res.body:
+ return json.loads(res.body)
+
+ # Return an empty response
+ return {}
+
+ # Packages
+
+ def _get_package(self, query, *args):
+ res = self.db.get(query, *args)
+
+ if res:
+ return Package(self.backend, id=res.id, data=res)
+
+ def get_package(self, distro, name):
+ return self._get_package("""
+ SELECT
+ *
+ FROM
+ release_monitoring_packages
+ WHERE
+ deleted_at IS NULL
+ AND
+ distro = %s
+ AND
+ name = %s
+ """, distro, name,
+ )
+
+ def create_package(self, distro, name):
+ pass
+
+ async def search(self, name):
+ """
+ Returns a bunch of packages that match the given name
+ """
+ # Send the request
+ response = await self._request("GET", "/api/v2/projects",
+ {
+ "name" : name,
+ "items_per_page" : 250,
+ },
+ )
+
+ # Return all packages
+ return [database.Row(item) for item in response.get("items")]
+
+
+class Package(base.DataObject):
+ table = "release_monitoring_packages"
+
+ @property
+ def distro(self):
+ return self.data.distro
+
+ @property
+ def name(self):
+ return self.data.name
+
+ # Releases
+
+ def _get_releases(self, query, *args):
+ res = self.db.query(query, *args)
+
+ for row in res:
+ yield Release(self.backend, id=row.id, data=row)
+
+ def _get_release(self, query, *args):
+ res = self.db.get(query, *args)
+
+ if res:
+ return Release(self.backend, id=res.id, data=res)
+
+ @property
+ def latest_release(self):
+ """
+ Returns the latest release of this package
+ """
+ return self._get_release("""
+ SELECT
+ *
+ FROM
+ release_monitoring_releases
+ WHERE
+ deleted_at IS NULL
+ AND
+ package_id = %s
+ ORDER BY
+ created_at DESC
+ LIMIT 1
+ """, self.id,
+ )
+
+ @property
+ def releases(self):
+ releases = self._get_releases("""
+ SELECT
+ *
+ FROM
+ release_monitoring_releases
+ WHERE
+ deleted_at IS NULL
+ AND
+ package_id = %s
+ ORDER BY
+ created_at DESC
+ """, self.id,
+ )
+
+ return list(releases)
+
+ def _create_release(self, version):
+ """
+ Creates a new release for this package
+ """
+ # XXX Do we need to check whether we are going backwards?
+
+ release = self._get_release("""
+ INSERT INTO
+ release_monitoring_releases
+ (
+ package_id,
+ version
+ )
+ VALUES
+ (
+ %s, %s
+ )
+ ON CONFLICT
+ (package_id, version)
+ WHERE
+ deleted_at IS NULL
+ DO
+ NOTHING
+ RETURNING
+ *
+ """, self.id, version,
+ )
+
+ # Return the release
+ return release
+
+ # Update
+
+ async def _fetch(self):
+ """
+ Fetches all sorts of information about a package
+ """
+ response = await self.backend.releasemonitoring._request(
+ "GET", "/api/v2/packages/", {
+ "distribution" : self.distro,
+ "name" : self.name,
+ },
+ )
+
+ # Fetch all received items
+ items = response.get("items")
+
+ # Is the package known?
+ if not items:
+ # Automatically try to create the package
+ if await self._auto_create_package():
+ return await self._fetch()
+
+ raise PackageNotFoundError(self.name)
+
+ # Did we receive more than one item?
+ elif len(items) > 1:
+ raise RuntimeError("More than one item received for %s" % name)
+
+ for item in items:
+ return database.Row({
+ "version" : item.get("version"),
+ "stable_version" : item.get("stable_version"),
+ })
+
+ async def _auto_create_package(self):
+ """
+ Tries to automatically create a package
+ """
+ packages = await self.backend.releasemonitoring.search(self.name)
+
+ # Use the first exact match
+ for package in packages:
+ if not package.name == self.name:
+ continue
+
+ print(package)
+
+ # Automatically create a mapping
+ await self.update_mapping(
+ project_name=package.name,
+ project_ecosystem=package.ecosystem,
+ )
+
+ return True
+
+ # We could not create a mapping
+ return False
+
+ async def update_mapping(self, project_name=None, project_ecosystem=None):
+ if project_name is None:
+ project_name = self.name
+
+ # Send request
+ await self.backend.releasemonitoring._request("POST", "/api/v2/packages/",
+ {
+ "distribution" : self.distro,
+ "package_name" : self.name,
+ "project_name" : project_name,
+ "project_ecosystem" : project_ecosystem,
+ },
+ )
+
+ async def update(self):
+ """
+ Update the information we have about this package
+ """
+ # Pull information from the API
+ package = await self._fetch()
+
+ # Try to create a new release
+ release = self._create_release(package.stable_version)
+
+ # Return the new release
+ return release
+
+
+class Release(base.DataObject):
+ table = "release_monitoring_releases"
+
+ def __str__(self):
+ return self.version
+
+ # Version
+
+ @property
+ def version(self):
+ return self.data.version
+
+ # Builds
+
+ @lazy_property
+ def builds(self):
+ builds = self.backend.builds._get_builds("""
+ SELECT
+ builds.*
+ FROM
+ release_monitoring_release_builds
+ LEFT JOIN
+ builds ON release_monitoring_release_builds.build_id = builds.id
+ WHERE
+ release_monitoring_release_builds.release_id = %s
+ AND
+ builds.deleted_at IS NULL
+ """, self.id,
+ )
+
+ # Return builds by distribution
+ return { build.distro : build for build in builds }
ORDER BY (pg_relation_size((c.oid)::regclass)) DESC;
+--
+-- Name: release_monitoring_packages; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.release_monitoring_packages (
+ id integer NOT NULL,
+ distro text NOT NULL,
+ name text NOT NULL,
+ created_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL,
+ deleted_at timestamp without time zone,
+ auto_update boolean DEFAULT true NOT NULL
+);
+
+
+--
+-- Name: release_monitoring_packages_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public.release_monitoring_packages_id_seq
+ AS integer
+ START WITH 1
+ INCREMENT BY 1
+ NO MINVALUE
+ NO MAXVALUE
+ CACHE 1;
+
+
+--
+-- Name: release_monitoring_packages_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE public.release_monitoring_packages_id_seq OWNED BY public.release_monitoring_packages.id;
+
+
+--
+-- Name: release_monitoring_release_builds; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.release_monitoring_release_builds (
+ id integer NOT NULL,
+ release_id integer NOT NULL,
+ distro_id integer NOT NULL,
+ build_id integer NOT NULL
+);
+
+
+--
+-- Name: release_monitoring_releases; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.release_monitoring_releases (
+ id integer NOT NULL,
+ package_id integer NOT NULL,
+ version text NOT NULL,
+ created_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL,
+ deleted_at timestamp without time zone
+);
+
+
+--
+-- Name: release_monitoring_releases_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public.release_monitoring_releases_id_seq
+ AS integer
+ START WITH 1
+ INCREMENT BY 1
+ NO MINVALUE
+ NO MAXVALUE
+ CACHE 1;
+
+
+--
+-- Name: release_monitoring_releases_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE public.release_monitoring_releases_id_seq OWNED BY public.release_monitoring_releases.id;
+
+
--
-- Name: repo_builds; Type: TABLE; Schema: public; Owner: -
--
ALTER TABLE ONLY public.packages ALTER COLUMN id SET DEFAULT nextval('public.packages_id_seq'::regclass);
+--
+-- Name: release_monitoring_packages id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.release_monitoring_packages ALTER COLUMN id SET DEFAULT nextval('public.release_monitoring_packages_id_seq'::regclass);
+
+
+--
+-- Name: release_monitoring_releases id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.release_monitoring_releases ALTER COLUMN id SET DEFAULT nextval('public.release_monitoring_releases_id_seq'::regclass);
+
+
--
-- Name: repositories id; Type: DEFAULT; Schema: public; Owner: -
--
ADD CONSTRAINT packages_pkey PRIMARY KEY (id);
+--
+-- Name: release_monitoring_packages release_monitoring_packages_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.release_monitoring_packages
+ ADD CONSTRAINT release_monitoring_packages_pkey PRIMARY KEY (id);
+
+
+--
+-- Name: release_monitoring_release_builds release_monitoring_release_builds_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.release_monitoring_release_builds
+ ADD CONSTRAINT release_monitoring_release_builds_pkey PRIMARY KEY (id);
+
+
+--
+-- Name: release_monitoring_releases release_monitoring_releases_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.release_monitoring_releases
+ ADD CONSTRAINT release_monitoring_releases_pkey PRIMARY KEY (id);
+
+
--
-- Name: repositories repositories_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
CREATE INDEX packages_name ON public.packages USING btree (name);
+--
+-- Name: release_monitoring_packages_unique; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE UNIQUE INDEX release_monitoring_packages_unique ON public.release_monitoring_packages USING btree (distro, name) WHERE (deleted_at IS NULL);
+
+
+--
+-- Name: release_monitoring_release_builds_release_id; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX release_monitoring_release_builds_release_id ON public.release_monitoring_release_builds USING btree (release_id);
+
+
+--
+-- Name: release_monitoring_releases_unique; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE UNIQUE INDEX release_monitoring_releases_unique ON public.release_monitoring_releases USING btree (package_id, version) WHERE (deleted_at IS NULL);
+
+
--
-- Name: repo_builds_build_id; Type: INDEX; Schema: public; Owner: -
--
ADD CONSTRAINT packages_distro_id FOREIGN KEY (distro_id) REFERENCES public.distributions(id);
+--
+-- Name: release_monitoring_release_builds release_monitoring_release_builds_build_id; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.release_monitoring_release_builds
+ ADD CONSTRAINT release_monitoring_release_builds_build_id FOREIGN KEY (build_id) REFERENCES public.builds(id);
+
+
+--
+-- Name: release_monitoring_release_builds release_monitoring_release_builds_distro_id; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.release_monitoring_release_builds
+ ADD CONSTRAINT release_monitoring_release_builds_distro_id FOREIGN KEY (distro_id) REFERENCES public.distributions(id);
+
+
+--
+-- Name: release_monitoring_release_builds release_monitoring_release_builds_release_id; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.release_monitoring_release_builds
+ ADD CONSTRAINT release_monitoring_release_builds_release_id FOREIGN KEY (release_id) REFERENCES public.release_monitoring_releases(id);
+
+
+--
+-- Name: release_monitoring_releases release_monitoring_releases_package_id; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.release_monitoring_releases
+ ADD CONSTRAINT release_monitoring_releases_package_id FOREIGN KEY (package_id) REFERENCES public.release_monitoring_packages(id);
+
+
--
-- Name: repo_builds repo_builds_added_by; Type: FK CONSTRAINT; Schema: public; Owner: -
--
# Messages
"messages:queue:send" : self.backend.messages.queue.send,
+ # Release Monitoring
+ "releasemonitoring:update" : self._release_monitoring_update,
+
# Repositories
"repos:fetch-sources" : self.backend.repos.fetch_sources,
"repos:relaunch-pending-jobs" : self._repos_relaunch_pending_jobs,
for repo in sorted(self.backend.repos):
await repo.relaunch_pending_jobs()
+ async def _release_monitoring_update(self, *names):
+ """
+ Performs an update for the given package
+ """
+ for name in names:
+ build = self.backend.builds.get_latest_by_name(name)
+ if not build:
+ log.error("Could not find package %s" % name)
+ continue
+
+ with self.backend.db.transaction():
+ # Show what we are working on
+ print(build.pkg)
+
+ # Perform the update
+ release = await build.pkg.release_monitoring.update()
+
+ # Show any new releases
+ if release:
+ print(" Found new release: %s" % release)
+
async def main():
cli = Cli()
<div class="block">
<nav class="level">
+ {# Release Monitoring #}
+ {% if package.is_source() %}
+ {% if package.latest_release %}
+ <div class="level-item has-text-centered">
+ <div>
+ <p class="heading">{{ _("Latest Release") }}</p>
+ <p>{{ package.latest_release }}</p>
+ </div>
+ </div>
+ {% end %}
+ {% end %}
+
{# Size #}
{% if show_size %}
<div class="level-item has-text-centered">
projects / pbs.git / commitdiff
