From c162004bf5edeadb8ed6a5f54359db3b9e0ee27c Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Mon, 15 May 2023 14:44:50 +0000 Subject: [PATCH] http: Build a custom HTTP client based on cURL This will help us to debug any API communication better and we won't have to copy too much code for multiple services that use an API. Signed-off-by: Michael Tremer --- Makefile.am | 1 + src/buildservice/__init__.py | 4 + src/buildservice/bugtracker.py | 39 +++------ src/buildservice/httpclient.py | 152 +++++++++++++++++++++++++++++++++ 4 files changed, 167 insertions(+), 29 deletions(-) create mode 100644 src/buildservice/httpclient.py diff --git a/Makefile.am b/Makefile.am index 0570d17e..ec44dc19 100644 --- a/Makefile.am +++ b/Makefile.am @@ -95,6 +95,7 @@ buildservice_PYTHON = \ src/buildservice/errors.py \ src/buildservice/events.py \ src/buildservice/git.py \ + src/buildservice/httpclient.py \ src/buildservice/jobs.py \ src/buildservice/keys.py \ src/buildservice/logstreams.py \ diff --git a/src/buildservice/__init__.py b/src/buildservice/__init__.py index 3aa06f27..9a5766ce 100644 --- a/src/buildservice/__init__.py +++ b/src/buildservice/__init__.py @@ -21,6 +21,7 @@ from . import config from . import database from . import distribution from . import events +from . import httpclient from . import jobs from . import keys from . import logstreams @@ -62,6 +63,9 @@ class Backend(object): # Global pakfire settings (from database). self.settings = settings.Settings(self) + # Initialize the HTTP Client + self.httpclient = httpclient.HTTPClient(self) + self.aws = aws.AWS(self) self.builds = builds.Builds(self) self.cache = cache.Cache(self) diff --git a/src/buildservice/bugtracker.py b/src/buildservice/bugtracker.py index 27897915..a32dc72a 100644 --- a/src/buildservice/bugtracker.py +++ b/src/buildservice/bugtracker.py @@ -26,10 +26,10 @@ import io import json import logging import mimetypes -import tornado.httpclient import urllib.parse from . import base +from . import httpclient from .decorators import * # Setup logging @@ -48,9 +48,6 @@ class Bugzilla(base.Object): # Store the API key self.api_key = api_key - # Set up HTTP Client - self.client = tornado.httpclient.AsyncHTTPClient() - @property def url(self): """ @@ -109,6 +106,9 @@ class Bugzilla(base.Object): return url async def _request(self, method, url, data=None): + if data is None: + data = {} + # Headers headers = { # Authenticate all requests @@ -118,9 +118,6 @@ class Bugzilla(base.Object): # Make the URL url = self.make_url(url) - if data is None: - data = {} - # Fallback authentication because some API endpoints # do not accept the API key in the header data |= { "api_key" : self.api_key } @@ -135,32 +132,16 @@ class Bugzilla(base.Object): body = None - # XXX proxy settings - # Create a new request - req = tornado.httpclient.HTTPRequest( - method=method, url=url, headers=headers, body=body, + req = httpclient.HTTPRequest( + method=method, + url=url, + headers=headers, + body=body, ) # Send the request and wait for a response - try: - res = await self.client.fetch(req) - - # Catch any HTTP Errors - except tornado.httpclient.HTTPClientError as e: - try: - error = json.loads(e.response.body) - except json.DecodeError: - error = None - - # Catch bad requests - if e.code == 400: - raise BadRequestError(error) from e - - # Raise any other exceptions - raise e - - log.debug("Response received in %.2fms" % (res.request_time * 1000)) + res = await self.backend.httpclient.fetch(req) # Decode JSON response if res.body: diff --git a/src/buildservice/httpclient.py b/src/buildservice/httpclient.py new file mode 100644 index 00000000..698e696d --- /dev/null +++ b/src/buildservice/httpclient.py @@ -0,0 +1,152 @@ +############################################################################### +# # +# Pakfire - The IPFire package management system # +# Copyright (C) 2023 Pakfire development team # +# # +# This program is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# This program is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with this program. If not, see . # +# # +############################################################################### + +import json +import logging +import tornado.curl_httpclient +import tornado.httpclient +import urllib.parse + +from . import base +from .decorators import * + +# Import version +from .__version__ import VERSION as __version__ + +# Setup logging +log = logging.getLogger("pbs.httpclient") + +HTTPRequest = tornado.httpclient.HTTPRequest + +class HTTPClient(tornado.curl_httpclient.CurlAsyncHTTPClient): + """ + This is a wrapper over Tornado's HTTP client that performs some extra + logging and makes it easier to compose a request. + """ + async def fetch(self, request): + """ + Sends a request + """ + # Set User-Agent + request.user_agent = "PakfireBuildService/%s" % __version__ + + # Log what we are sending + log.debug("Sending %s request to %s:" % (request.method, request.url)) + + # Log headers + if request.headers: + log_headers(request.headers) + + # Log the body + if request.body: + log_body(request.body) + + # Send the request + try: + response = await super().fetch(request) + + # Log any errors + except torando.httpclient.HTTPError as e: + log.error("Received status %s:" % e.code) + + # Log the response body + if e.response: + log_body(e.response.body, level=logging.ERROR) + + # Raise the error + raise e + + # Log successful responses + else: + log.debug("Received response: %s (%s - %s) in %.2fms:" % ( + response.effective_url, response.code, response.reason, + response.request_time * 1000.0, + )) + + # Log headers + if response.headers: + log_headers(response.headers) + + # Log body + if response.body: + log_body(response.body) + + # Return the response + return response + + +def log_headers(headers): + """ + Helper function to log request/response headers + """ + log.debug(" Headers:") + + for header in sorted(headers): + log.debug(" %-32s: %s" % (header, headers[header])) + +def decode_body(body): + # Try parsing this as JSON and reformat it + try: + body = json.loads(body) + body = json.dumps(body, indent=4, sort_keys=True) + except (json.JSONDecodeError, UnicodeDecodeError): + pass + else: + return body + + # Try parsing this as query arguments + try: + query = urllib.parse.parse_qs(body, strict_parsing=True) + if query: + body = format_query(query) + except ValueError: + pass + else: + return body + + # Decode as string + if isinstance(body, bytes): + try: + body = body.decode() + except UnicodeError: + pass + else: + return body + +def format_query(query): + lines = [] + + for key in sorted(query): + for val in query[key]: + lines.append(b" %-32s : %s" % (key, val)) + + return b"\n".join(lines) + +def log_body(body, level=logging.DEBUG): + """ + Helper function to log the request/response body + """ + body = decode_body(body) + + if body: + log.log(level, " Body:") + + for line in body.splitlines(): + log.log(level, " %s" % line) -- 2.47.2