client/hub: Use new streaming upload method

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/src/pakfire/cli.py b/src/pakfire/cli.py
index 611e6b7d98098feec0b221ae87cfeeb283f56b34..79f59af9ae13301da6f92475edd80d1b38476d9a 100644 (file)
--- a/src/pakfire/cli.py
+++ b/src/pakfire/cli.py
@@ -642,6 +642,11 @@ class CliClient(Cli):
                        help=_("Check the connection to the hub"))
                check_connection.set_defaults(func=self.handle_check_connection)
 
+               # upload
+               upload = subparsers.add_parser("upload", help=_("Upload a file to the build service"))
+               upload.add_argument("file", nargs=1, help=_("Filename"))
+               upload.set_defaults(func=self.handle_upload)
+
                # watch-build
                watch_build = subparsers.add_parser("watch-build", help=_("Watch the status of a build"))
                watch_build.add_argument("id", nargs=1, help=_("Build ID"))
@@ -703,6 +708,10 @@ class CliClient(Cli):
                if success:
                        print("%s: %s" % (_("Connection OK"), success))
 
+       def handle_upload(self, ns):
+               for path in ns.file:
+                       self.client.upload_file(path)
+
        def handle_watch_build(self, ns):
                build = self.client.get_build(ns.id[0])
 

diff --git a/src/pakfire/client.py b/src/pakfire/client.py
index b2a06e7739c5567f77f4df4d280b85c1e330421e..8ba56c05424c2afc52cf6cfd24b1028b2758a059 100644 (file)
--- a/src/pakfire/client.py
+++ b/src/pakfire/client.py
@@ -74,6 +74,11 @@ class Client(object):
        def get_job(self, job_id):
                return Job(self, job_id)
 
+       # Uploads
+
+       def upload_file(self, path):
+               return self.hub.upload_file(path)
+
 
 class _ClientObject(object):
        def __init__(self, client, id):

diff --git a/src/pakfire/hub.py b/src/pakfire/hub.py
index 84c74cb4fe019693857a776317aec1c9a67d6331..b302f9681639b0f14ac48e573406797ebdd7cfec 100644 (file)
--- a/src/pakfire/hub.py
+++ b/src/pakfire/hub.py
@@ -41,17 +41,29 @@ class Hub(object):
                # Initialise the HTTP client
                self.http = http.Client(baseurl=huburl)
 
+       @property
+       def _request_args(self):
+               """
+                       Arguments sent with each request
+               """
+               return {
+                       "auth" : (self.username, self.password),
+               }
+
        def _request(self, *args, **kwargs):
                """
                        Wrapper function around the HTTP Client request()
                        function that adds authentication, etc.
                """
-               kwargs.update({
-                       "auth" : (self.username, self.password),
-               })
+               kwargs.update(self._request_args)
 
                return self.http.request(*args, **kwargs)
 
+       def _upload(self, *args, **kwargs):
+               kwargs.update(self._request_args)
+
+               return self.http.upload(*args, **kwargs)
+
        # Test functions
 
        def test(self):
@@ -103,128 +115,18 @@ class Hub(object):
        # File uploads
 
        def upload_file(self, path):
-               uploader = FileUploader(self, path)
-
-               return uploader.upload()
-
-
-class FileUploader(object):
-       """
-               Handles file uploads to the Pakfire Hub
-       """
-       def __init__(self, hub, path):
-               self.hub = hub
-               self.path = path
-
-       @property
-       def filename(self):
-               """
-                       Returns the basename of the uploaded file
-               """
-               return os.path.basename(self.path)
-
-       @property
-       def filesize(self):
-               """
-                       The filesize of the uploaded file
-               """
-               return os.path.getsize(self.path)
-
-       @staticmethod
-       def _make_checksum(algo, path):
-               h = hashlib.new(algo)
-
-               with open(path, "rb") as f:
-                       while True:
-                               buf = f.read(CHUNK_SIZE)
-                               if not buf:
-                                       break
-
-                               h.update(buf)
-
-               return h.hexdigest()
-
-       def _get_upload_id(self):
-               """
-                       Sends some basic information to the hub
-                       and requests an upload id.
-               """
+               # Send some basic information to the hub
+               # and request an upload ID
                data = {
-                       "filename" : self.filename,
-                       "filesize" : self.filesize,
-                       "hash"     : self._make_checksum("sha1", self.path),
+                       "filename" : os.path.basename(path),
+                       "filesize" : os.path.getsize(path),
                }
+               upload_id = self._request("/uploads/create", method="GET",
+                       decode="ascii", data=data)
 
-               return self.hub._request("/uploads/create", method="GET", decode="ascii", data=data)
-
-       def _send_chunk(self, upload_id, chunk):
-               """
-                       Sends a chunk at a time
-               """
-               # Compute the SHA512 checksum of this chunk
-               h = hashlib.new("sha512")
-               h.update(chunk)
-
-               # Encode data in base64
-               data = base64.b64encode(chunk)
-
-               # Send chunk to the server
-               self.hub._request("/uploads/%s/sendchunk" % upload_id, method="POST",
-                       data={ "chksum" : h.hexdigest(), "data" : data })
-
-               return len(chunk)
-
-       def upload(self):
-               """
-                       Main function which runs the upload
-               """
-               # Borrow progressbar from downloader
-               p = self.hub.http._make_progressbar(message=self.filename, value_max=self.filesize)
-
-               with p:
-                       # Request an upload ID
-                       upload_id = self._get_upload_id()
-                       assert upload_id
-
-                       log.debug("Starting upload with id %s" % upload_id)
-
-                       # Initial chunk size
-                       chunk_size = CHUNK_SIZE
-
-                       try:
-                               with open(self.path, "rb") as f:
-                                       while True:
-                                               chunk = f.read(chunk_size)
-                                               if not chunk:
-                                                       break
-
-                                               # Save the time when we started to send this bit
-                                               time_started = time.time()
-
-                                               # Send the chunk to the server
-                                               self._send_chunk(upload_id, chunk)
-
-                                               # Determine the size of the next chunk
-                                               duration = time_started - time.time()
-                                               chunk_size = math.ceil(chunk_size / duration)
-
-                                               # Never let chunk_size drop under CHUNK_SIZE
-                                               if chunk_size < CHUNK_SIZE:
-                                                       chunk_size = CHUNK_SIZE
-
-                                               # Update progressbar
-                                               p.increment(len(chunk))
-
-                       # Catch any unhandled exception here, tell the hub to delete the
-                       # file and raise the original exception
-                       except Exception as e:
-                               self.hub._request("/uploads/%s/destroy" % upload_id)
-
-                               raise
+               log.debug("Upload ID: %s" % upload_id)
 
-                       # If all went well, we finish the upload
-                       else:
-                               self.hub._request("/uploads/%s/finished" % upload_id)
+               # Upload the data
+               self._upload("/uploads/stream?id=%s" % upload_id, path)
 
-                               # Return the upload ID if the upload was successful
-                               return upload_id
+               return upload_id
\ No newline at end of file