From: Michael Tremer Date: Thu, 23 Jun 2022 11:28:08 +0000 (+0000) Subject: bugzilla: Replace old API with new async REST API implementation X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=b2ab0de7498baac6161ff718b50d3f1cf7260d70;p=pbs.git bugzilla: Replace old API with new async REST API implementation Signed-off-by: Michael Tremer --- diff --git a/src/buildservice/bugtracker.py b/src/buildservice/bugtracker.py index 9eca7ced..0ced1904 100644 --- a/src/buildservice/bugtracker.py +++ b/src/buildservice/bugtracker.py @@ -1,218 +1,220 @@ -#!/usr/bin/python - -import xmlrpc.client +#!/usr/bin/python3 +############################################################################### +# # +# Pakfire - The IPFire package management system # +# Copyright (C) 2022 Pakfire development team # +# # +# This program is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# This program is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with this program. If not, see . # +# # +############################################################################### + +import asyncio +import datetime +import json +import logging +import tornado.httpclient +import urllib.parse from . import base - from .decorators import * -class BugzillaBug(base.Object): - def __init__(self, bugzilla, bug_id): - base.Object.__init__(self, bugzilla.pakfire) - self.bugzilla = bugzilla +log = logging.getLogger("pakfire.bugzilla") - self.bug_id = bug_id - self._data = None +TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" - def __cmp__(self, other): - return cmp(self.bug_id, other.bug_id) - - def call(self, *args, **kwargs): - args = (("Bug",) + args) +class Bugzilla(base.Object): + def init(self): + # Set up HTTP Client + self.client = tornado.httpclient.AsyncHTTPClient() - return self.bugzilla.call(*args, **kwargs) + @property + def url(self): + """ + Returns the base URL of a Bugzilla instance + """ + return self.settings.get("bugzilla-url") @property - def id(self): - return self.bug_id + def product(self): + return self.settings.get("bugzilla-product") + + def make_url(self, *args, **kwargs): + """ + Composes a URL based on the base URL + """ + url = urllib.parse.urljoin(self.url, *args) + + # Append any query arguments + if kwargs: + url = "%s?%s" % (url, urllib.parse.urlencode(kwargs)) + + return url + + async def _request(self, method, url, **kwargs): + # Headers + headers = { + # Authenticate all requests + "X-BUGZILLA-API-KEY" : self.settings.get("bugzilla-api-key"), + } - @lazy_property - def data(self): - # Fetch bug information from cache - data = self.backend.cache.get(self._cache_key) + # Make the URL + url = self.make_url(url) - # Hit - if data: - return data + # Encode body + body = urllib.parse.urlencode(kwargs) - # Fetch bug information from Bugzilla - for data in self.call("get", ids=[self.id,])["bugs"]: - # Put it into the cache - self.backend.cache.set(self._cache_key, data, self.backend.bugzilla.cache_lifetime) + # For GET requests, append query arguments + if method == "GET": + if body: + url = "%s?%s" % (url, body) - return data + body = None - @property - def _cache_key(self): - return "bug-%s" % self.bug_id + # XXX proxy settings - @property - def url(self): - return self.bugzilla.bug_url(self.id) + # Create a new request + req = tornado.httpclient.HTTPRequest( + method=method, url=url, headers=headers, body=body, + ) - @property - def summary(self): - return self.data.get("summary") + # Send the request and wait for a response + res = await self.client.fetch(req) - @property - def assignee(self): - return self.data.get("assigned_to") + log.debug("Response received in %.2fms" % (res.request_time * 1000)) - @property - def status(self): - return self.data.get("status") + # Decode JSON response + if res.body: + return json.loads(res.body) - @property - def resolution(self): - return self.data.get("resolution") + # Return an empty response + return {} - @property - def is_closed(self): - return not self.data["is_open"] + def enter_url(self, package): + """ + Creates an URL to create a new bug for package + """ + return self.make_url("/enter_bug.cgi", product=self.product, component=package) - def set_status(self, status, resolution=None, comment=None): - kwargs = { "status" : status } - if resolution: - kwargs["resolution"] = resolution - if comment: - kwargs["comment"] = { "body" : comment } + def list_url(self, package): + """ + Creates an URL to list all bugs for package + """ + return self.make_url("/buglist.cgi", product=self.product, component=package) - self.call("update", ids=[self.id,], **kwargs) + async def version(self): + """ + Returns the version number of Bugzilla + """ + response = await self._request("GET", "/rest/version") - # Invalidate cache - self.backend.cache.delete(self.cache_key) + return response.get("version") + async def search(self, **kwargs): + # Send request + response = await self._request("GET", "/rest/bug", **kwargs) -class Bugzilla(base.Object): - def __init__(self, pakfire): - base.Object.__init__(self, pakfire) + # Parse the response + bugs = [Bug(self.backend, data) for data in response.get("bugs")] - # Open the connection to the server. - self.server = xmlrpc.client.ServerProxy(self.url, use_datetime=True) + # Sort and return in reverse order + return sorted(bugs, reverse=True) - # Cache the credentials. - self.__credentials = { - "Bugzilla_login" : self.user, - "Bugzilla_password" : self.password, - } + async def get_bugs(self, bugs): + """ + Fetches multiple bugs concurrently + """ + return asyncio.gather( + *(self.get_bug(bug) for bug in bugs), + ) - def call(self, *args, **kwargs): - # Add authentication information. - kwargs.update(self.__credentials) + async def get_bug(self, bug): + """ + Fetches one bug + """ + log.debug("Fetching bug %s" % bug) - method = self.server - for arg in args: - method = getattr(method, arg) + response = await self._request("GET", "/rest/bug/%s" % bug) - return method(kwargs) + for data in response.get("bugs"): + return Bug(self.backend, data) - def bug_url(self, bug_id): - url = self.settings.get("bugzilla_url", None) - return url % { "bug_id" : bug_id } +class Bug(base.Object): + def init(self, data): + self.data = data - def enter_url(self, component): - args = { - "product" : self.settings.get("bugzilla_product", ""), - "component" : component, - } + def __eq__(self, other): + if isinstance(other, self.__class__): + return self.id == other.id - url = self.settings.get("bugzilla_url_new") - - return url % args - - def buglist_url(self, component): - args = { - "product" : self.settings.get("bugzilla_product", ""), - "component" : component, - } + raise NotImplemented - url = self.settings.get("bugzilla_url_buglist") + def __lt__(self, other): + if isinstance(other, self.__class__): + return self.created_at < other.created_at - return url % args + raise NotImplemented @property - def url(self): - return self.settings.get("bugzilla_url_xmlrpc", None) + def id(self): + return self.data.get("id") @property - def user(self): - return self.settings.get("bugzilla_xmlrpc_user", "") + def url(self): + return self.backend.bugzilla.make_url("/show_bug.cgi", id=self.id) @property - def password(self): - return self.settings.get("bugzilla_xmlrpc_password") - - @lazy_property - def cache_lifetime(self): - return self.settings.get("bugzilla_cache_lifetime", 3600) - - def get_bug(self, bug_id): - try: - bug = BugzillaBug(self, bug_id) - - except xmlrpc.client.Fault: - return None - - return bug - - def find_users(self, pattern): - users = self.call("User", "get", match=[pattern,]) - if users: - return users["users"] + def created_at(self): + t = self.data.get("creation_time") - def find_user(self, pattern): - users = self.find_users(pattern) + return datetime.datetime.strptime(t, TIME_FORMAT) - if not users: - return - - elif len(users) > 1: - raise Exception("Got more than one result.") - - return users[0] - - def get_bugs_from_component(self, component, closed=False): - kwargs = { - "product" : self.settings.get("bugzilla_product", ""), - "component" : component, - } - - query = self.call("Bug", "search", include_fields=["id"], **kwargs) + @property + def summary(self): + return self.data.get("summary") - bugs = [] - for bug in query["bugs"]: - bug = self.get_bug(bug["id"]) + @property + def component(self): + return self.data.get("component") - if not bug.is_closed == closed: - continue + @property + def creator(self): + creator = self.data.get("creator") - bugs.append(bug) + return self.backend.users.get_by_email(creator) - return bugs + @property + def assignee(self): + assignee = self.data.get("assigned_to") - def send_all(self, limit=100): - # Get up to ten updates. - query = self.db.query("SELECT * FROM builds_bugs_updates \ - WHERE error IS FALSE ORDER BY time LIMIT %s", limit) + return self.backend.users.get_by_email(assignee) - # XXX CHECK IF BZ IS ACTUALLY REACHABLE AND WORKING + @property + def status(self): + return self.data.get("status") - for update in query: - try: - bug = self.backend.bugzilla.get_bug(update.bug_id) - if not bug: - logging.error("Bug #%s does not exist." % update.bug_id) - continue + @property + def resolution(self): + return self.data.get("resolution") - # Set the changes. - bug.set_status(update.status, update.resolution, update.comment) + @property + def severity(self): + return self.data.get("severity") - except Exception as e: - # If there was an error, we save that and go on. - self.db.execute("UPDATE builds_bugs_updates SET error = 'Y', error_msg = %s \ - WHERE id = %s", "%s" % e, update.id) + def is_closed(self): + return not self.data["is_open"] - else: - # Remove the update when it has been done successfully. - self.db.execute("DELETE FROM builds_bugs_updates WHERE id = %s", update.id) + @property + def keywords(self): + return self.data.get("keywords") diff --git a/src/templates/package-detail-list.html b/src/templates/package-detail-list.html index 6a7a64d0..40b3e8db 100644 --- a/src/templates/package-detail-list.html +++ b/src/templates/package-detail-list.html @@ -91,7 +91,7 @@ {{ _("File new bug") }} - + {{ _("Show all bugs") }} diff --git a/src/web/packages.py b/src/web/packages.py index a953cb20..0d2109a3 100644 --- a/src/web/packages.py +++ b/src/web/packages.py @@ -35,13 +35,13 @@ class PackageIDDetailHandler(base.BaseHandler): class PackageNameHandler(base.BaseHandler): - def get(self, name): + async def get(self, name): build = self.backend.builds.get_latest_by_name(name) if not build: raise tornado.web.HTTPError(404, "Package '%s' was not found" % name) - # Get the latest bugs from bugzilla. - bugs = self.backend.bugzilla.get_bugs_from_component(name) + # Get the latest bugs from Bugzilla + bugs = await self.backend.bugzilla.search(component=name, resolution="") self.render("package-detail-list.html", package=build.pkg, bugs=bugs)