client: Refactor hub communication based on tornado HTTP client

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/src/pakfire/client.py b/src/pakfire/client.py
index fcfb575871cf998cd1955845ce23a224f714b897..39daed23d4aa99c3869819aded183f8c19218ec0 100644 (file)
--- a/src/pakfire/client.py
+++ b/src/pakfire/client.py
@@ -74,8 +74,8 @@ class Client(object):
 
        # Uploads
 
-       def upload_file(self, path):
-               return self.hub.upload_file(path)
+       def upload(self, *args, **kwargs):
+               return self.hub.upload(*args, **kwargs)
 
 
 class _ClientObject(object):

diff --git a/src/pakfire/hub.py b/src/pakfire/hub.py
index b302f9681639b0f14ac48e573406797ebdd7cfec..0b5c689e100bbe0d6b6fb77a4de35b4ac5692347 100644 (file)
--- a/src/pakfire/hub.py
+++ b/src/pakfire/hub.py
@@ -19,50 +19,67 @@
 #                                                                             #
 ###############################################################################
 
-import base64
+import functools
 import hashlib
+import json
 import logging
-import math
 import os.path
-import time
-
-from . import http
+import tornado.httpclient
+import urllib.parse
 
 from .constants import *
 
+# Setup logging
 log = logging.getLogger("pakfire.hub")
-log.propagate = 1
+
+# Configure some useful defaults for all requests
+tornado.httpclient.AsyncHTTPClient.configure(
+       None, defaults = {
+               "user_agent" : "pakfire/%s" % PAKFIRE_VERSION,
+       },
+)
 
 class Hub(object):
-       def __init__(self, huburl, username, password):
+       def __init__(self, url, username, password):
+               self.url = url
                self.username = username
                self.password = password
 
                # Initialise the HTTP client
-               self.http = http.Client(baseurl=huburl)
+               self.client = tornado.httpclient.AsyncHTTPClient()
 
-       @property
-       def _request_args(self):
-               """
-                       Arguments sent with each request
-               """
-               return {
-                       "auth" : (self.username, self.password),
-               }
+               # XXX support proxies
 
-       def _request(self, *args, **kwargs):
-               """
-                       Wrapper function around the HTTP Client request()
-                       function that adds authentication, etc.
-               """
-               kwargs.update(self._request_args)
+       async def _request(self, method, path, body_producer=None, **kwargs):
+               # Make absolute URL
+               url = urllib.parse.urljoin(self.url, path)
 
-               return self.http.request(*args, **kwargs)
+               # Add query arguments
+               if method in ("GET", "PUT"):
+                       url = "%s?%s" % (url, urllib.parse.urlencode(kwargs))
 
-       def _upload(self, *args, **kwargs):
-               kwargs.update(self._request_args)
+               # Make the request
+               req = tornado.httpclient.HTTPRequest(
+                       method=method, url=url,
 
-               return self.http.upload(*args, **kwargs)
+                       # Authentication
+                       auth_username=self.username, auth_password=self.password,
+
+                       # All all the rest
+                       body_producer=body_producer,
+               )
+
+               # Send the request and wait for a response
+               res = await self.client.fetch(req)
+
+               # XXX Do we have to catch any errors here?
+
+               # Decode JSON response
+               if res.body:
+                       return json.loads(res.body)
+
+               # Empty response
+               return {}
 
        # Test functions
 
@@ -114,19 +131,54 @@ class Hub(object):
 
        # File uploads
 
-       def upload_file(self, path):
-               # Send some basic information to the hub
-               # and request an upload ID
-               data = {
-                       "filename" : os.path.basename(path),
-                       "filesize" : os.path.getsize(path),
-               }
-               upload_id = self._request("/uploads/create", method="GET",
-                       decode="ascii", data=data)
+       async def upload(self, path, filename=None):
+               """
+                       Uploads the file to the hub returning the upload ID
+               """
+               log.debug("Uploading %s..." % path)
+
+               # Use the basename of the file if no name was given
+               if filename is None:
+                       filename = os.path.basename(path)
+
+               # Determine the filesize
+               size = os.path.getsize(path)
+
+               # Compute a digest
+               digest = self._compute_digest("blake2b", path)
+
+               # Prepare the file for streaming
+               body_producer = functools.partial(self._stream_file, path)
+
+               # Perform upload
+               response = await self._request("PUT", "/upload",
+                       body_producer=body_producer,
+                       filename=filename, size=size, digest=digest
+               )
+
+               # Return the upload ID
+               return response.get("id")
+
+       @staticmethod
+       def _stream_file(path, write):
+               with open(path, "rb") as f:
+                       while True:
+                               buf = f.read(64 * 1024)
+                               if not buf:
+                                       break
+
+                               write(buf)
+
+       @staticmethod
+       def _compute_digest(algo, path):
+               h = hashlib.new(algo)
 
-               log.debug("Upload ID: %s" % upload_id)
+               with open(path, "rb") as f:
+                       while True:
+                               buf = f.read(64 * 1024)
+                               if not buf:
+                                       break
 
-               # Upload the data
-               self._upload("/uploads/stream?id=%s" % upload_id, path)
+                               h.update(buf)
 
-               return upload_id
\ No newline at end of file
+               return "%s:%s" % (algo, h.hexdigest())

diff --git a/src/scripts/pakfire-client.in b/src/scripts/pakfire-client.in
index 13f1fef57c8f3ca24b8e19a0060f1e50be9622b3..9bec6154e591239aa72eb0a455f64d5623625c4b 100644 (file)
--- a/src/scripts/pakfire-client.in
+++ b/src/scripts/pakfire-client.in
@@ -52,7 +52,7 @@ class Cli(object):
                # upload
                upload = subparsers.add_parser("upload",
                        help=_("Upload a file to the build service"))
-               upload.add_argument("file",
+               upload.add_argument("file", nargs="+",
                        help=_("Filename"))
                upload.set_defaults(func=self._upload)