From: Michael Tremer Date: Tue, 29 Nov 2016 16:17:37 +0000 (+0100) Subject: Add a basic HTTP client X-Git-Tag: 0.9.28~1285^2~1462 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=7eec9830ad5f9728740b02b99411585182d476d2;p=pakfire.git Add a basic HTTP client In Python3, urlgrabber is no longer available. This patch adds a component which will replace urlgrabber by an own custom implementation that only relies on Python3-internal modules (i.e. urllib). Signed-off-by: Michael Tremer --- diff --git a/Makefile.am b/Makefile.am index 07249794a..5c2d6c45d 100644 --- a/Makefile.am +++ b/Makefile.am @@ -107,6 +107,7 @@ pakfire_PYTHON = \ src/pakfire/downloader.py \ src/pakfire/errors.py \ src/pakfire/filelist.py \ + src/pakfire/http.py \ src/pakfire/i18n.py \ src/pakfire/keyring.py \ src/pakfire/logger.py \ diff --git a/src/pakfire/http.py b/src/pakfire/http.py new file mode 100644 index 000000000..3e4bd2bb9 --- /dev/null +++ b/src/pakfire/http.py @@ -0,0 +1,292 @@ +#!/usr/bin/python3 +############################################################################### +# # +# Pakfire - The IPFire package management system # +# Copyright (C) 2011 Pakfire development team # +# # +# This program is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# This program is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with this program. If not, see . # +# # +############################################################################### + +import logging +import ssl +import urllib.parse +import urllib.request + +from .ui import progressbar + +from .constants import * +from . import errors + +log = logging.getLogger("pakfire.http") +log.propagate = 1 + +class Client(object): + """ + Implements a basic HTTP client which is used to download + repository data, packages and communicate with the Pakfire Hub. + """ + def __init__(self, baseurl=None): + self.baseurl = None + + # Stores any proxy configuration + self.proxies = {} + + # Create an SSL context to HTTPS connections + self.ssl_context = ssl.create_default_context() + + def set_proxy(self, protocol, host): + """ + Sets a proxy that will be used to send this request + """ + self.proxies[protocol] = host + + def disable_certificate_validation(self): + # Disable checking hostname + self.ssl_context.check_hostname = False + + # Disable any certificate validation + self.ssl_context.verify_mode = ssl.CERT_NONE + + def _make_request(self, url, method="GET", data=None): + # Add the baseurl + if self.baseurl: + url = urllib.parse.urljoin(self.baseurl, url) + + req = urllib.request.Request(url) + + # Add our user agent + req.add_header("User-Agent", "pakfire/%s" % PAKFIRE_VERSION) + + # Configure proxies + for protocol, host in self.proxies.items(): + req.set_proxy(host, protocol) + + # Check if method is correct + assert method == req.get_method() + + return req + + def _send_request(self, req): + log.debug("HTTP Request to %s" % req.host) + log.debug(" URL: %s" % req.full_url) + log.debug(" Headers:") + for k, v in req.header_items(): + log.debug(" %s: %s" % (k, v)) + + try: + res = urllib.request.urlopen(req, context=self.ssl_context) + + # Catch any HTTP errors + except urllib.error.HTTPError as e: + if e.code == 403: + raise ForbiddenError() + elif e.code == 404: + raise NotFoundError() + elif e.code == 500: + raise InternalServerError() + elif e.code in (502, 503): + raise BadGatewayError() + elif e.code == 504: + raise ConnectionTimeoutError() + + # Raise any unhandled exception + raise + + log.debug("HTTP Response: %s" % res.code) + log.debug(" Headers:") + for k, v in res.getheaders(): + log.debug(" %s: %s" % (k, v)) + + return res + + def _one_request(self, url, **kwargs): + r = self._make_request(url, **kwargs) + + # Send request and return the entire response at once + with self._send_request(r) as f: + return f.read() + + def get(self, url, **kwargs): + """ + Shortcut to GET content and have it returned + """ + return self._one_request(url, method="GET", **kwargs) + + def request(self, url, tries=None, **kwargs): + # tries = None implies wait infinitely + + while tries is None or tries > 0: + if tries: + tries -= 1 + + try: + return self._one_request(url, **kwargs) + + # 500 - Internal Server Error, 502 + 503 Bad Gateway Error + except (InternalServerError, BadGatewayError) as e: + log.exception("%s" % e.__class__.__name__) + + # Wait a minute before trying again. + time.sleep(60) + + # Retry on connection problems. + except ConnectionError as e: + log.exception("%s" % e.__class__.__name__) + + # Wait for 10 seconds. + time.sleep(10) + + except (KeyboardInterrupt, SystemExit): + break + + raise MaxTriesExceededError + + def retrieve(self, url, filename, show_progress=True, message=None, **kwargs): + p = None + + if message is None: + message = os.path.basename(url) + + buffer_size = 100 * 1024 # 100k + + # Make a progressbar + if show_progress: + p = self._make_progressbar(message) + + # Prepare HTTP request + r = self._make_request(url, **kwargs) + + # Send the request + with self._send_request(r) as f: + # Try setting progress bar to correct maximum value + # XXX this might need a function in ProgressBar + l = self._get_content_length(f) + if p: + p.value_max = l + + buf = f.read(buffer_size) + while buf: + l = len(buf) + if p: + p.update_increment(l) + + buf = f.read(buffer_size) + + if p: + p.finish() + + def _get_content_length(self, response): + s = response.getheader("Content-Length") + + try: + return int(s) + except TypeError: + pass + + def _make_progressbar(self, message=None): + p = progressbar.ProgressBar() + + # Show message (e.g. filename) + if message: + p.add(message) + + # Show percentage + w = progressbar.WidgetPercentage(clear_when_finished=True) + p.add(w) + + # Add a bar + w = progressbar.WidgetBar() + p.add(w) + + # Show transfer speed + # XXX just shows the average speed which is probably + # not what we want here. Might want an average over the + # last x (maybe ten?) seconds + w = progressbar.WidgetFileTransferSpeed() + p.add(w) + + # Spacer + p.add("|") + + # Show downloaded bytes + w = progressbar.WidgetBytesReceived() + p.add(w) + + # ETA + w = progressbar.WidgetETA() + p.add(w) + + return p.start() + + +class HTTPError(errors.Error): + pass + + +class ForbiddenError(HTTPError): + """ + HTTP Error 403 - Forbidden + """ + pass + + +class NotFoundError(HTTPError): + """ + HTTP Error 404 - Not Found + """ + pass + + +class InternalServerError(HTTPError): + """ + HTTP Error 500 - Internal Server Error + """ + pass + + +class BadGatewayError(HTTPError): + """ + HTTP Error 502+503 - Bad Gateway + """ + pass + + +class ConnectionTimeoutError(HTTPError): + """ + HTTP Error 504 - Connection Timeout + """ + pass + + +class ConnectionError(Exception): + """ + Raised when there is problems with the connection + (on an IP sort of level). + """ + pass + + +class SSLError(ConnectionError): + """ + Raised when there are any SSL problems. + """ + pass + + +class MaxTriedExceededError(errors.Error): + """ + Raised when the maximum number of tries has been exceeded + """ + pass