From: Michael Tremer Date: Fri, 3 Nov 2017 16:30:32 +0000 (+0100) Subject: client/hub: Use new streaming upload method X-Git-Tag: 0.9.28~1285^2~1303 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=ec5917f30b21baa63d95b52a7b3e264023d6983a;p=pakfire.git client/hub: Use new streaming upload method Signed-off-by: Michael Tremer --- diff --git a/src/pakfire/cli.py b/src/pakfire/cli.py index 611e6b7d9..79f59af9a 100644 --- a/src/pakfire/cli.py +++ b/src/pakfire/cli.py @@ -642,6 +642,11 @@ class CliClient(Cli): help=_("Check the connection to the hub")) check_connection.set_defaults(func=self.handle_check_connection) + # upload + upload = subparsers.add_parser("upload", help=_("Upload a file to the build service")) + upload.add_argument("file", nargs=1, help=_("Filename")) + upload.set_defaults(func=self.handle_upload) + # watch-build watch_build = subparsers.add_parser("watch-build", help=_("Watch the status of a build")) watch_build.add_argument("id", nargs=1, help=_("Build ID")) @@ -703,6 +708,10 @@ class CliClient(Cli): if success: print("%s: %s" % (_("Connection OK"), success)) + def handle_upload(self, ns): + for path in ns.file: + self.client.upload_file(path) + def handle_watch_build(self, ns): build = self.client.get_build(ns.id[0]) diff --git a/src/pakfire/client.py b/src/pakfire/client.py index b2a06e773..8ba56c054 100644 --- a/src/pakfire/client.py +++ b/src/pakfire/client.py @@ -74,6 +74,11 @@ class Client(object): def get_job(self, job_id): return Job(self, job_id) + # Uploads + + def upload_file(self, path): + return self.hub.upload_file(path) + class _ClientObject(object): def __init__(self, client, id): diff --git a/src/pakfire/hub.py b/src/pakfire/hub.py index 84c74cb4f..b302f9681 100644 --- a/src/pakfire/hub.py +++ b/src/pakfire/hub.py @@ -41,17 +41,29 @@ class Hub(object): # Initialise the HTTP client self.http = http.Client(baseurl=huburl) + @property + def _request_args(self): + """ + Arguments sent with each request + """ + return { + "auth" : (self.username, self.password), + } + def _request(self, *args, **kwargs): """ Wrapper function around the HTTP Client request() function that adds authentication, etc. """ - kwargs.update({ - "auth" : (self.username, self.password), - }) + kwargs.update(self._request_args) return self.http.request(*args, **kwargs) + def _upload(self, *args, **kwargs): + kwargs.update(self._request_args) + + return self.http.upload(*args, **kwargs) + # Test functions def test(self): @@ -103,128 +115,18 @@ class Hub(object): # File uploads def upload_file(self, path): - uploader = FileUploader(self, path) - - return uploader.upload() - - -class FileUploader(object): - """ - Handles file uploads to the Pakfire Hub - """ - def __init__(self, hub, path): - self.hub = hub - self.path = path - - @property - def filename(self): - """ - Returns the basename of the uploaded file - """ - return os.path.basename(self.path) - - @property - def filesize(self): - """ - The filesize of the uploaded file - """ - return os.path.getsize(self.path) - - @staticmethod - def _make_checksum(algo, path): - h = hashlib.new(algo) - - with open(path, "rb") as f: - while True: - buf = f.read(CHUNK_SIZE) - if not buf: - break - - h.update(buf) - - return h.hexdigest() - - def _get_upload_id(self): - """ - Sends some basic information to the hub - and requests an upload id. - """ + # Send some basic information to the hub + # and request an upload ID data = { - "filename" : self.filename, - "filesize" : self.filesize, - "hash" : self._make_checksum("sha1", self.path), + "filename" : os.path.basename(path), + "filesize" : os.path.getsize(path), } + upload_id = self._request("/uploads/create", method="GET", + decode="ascii", data=data) - return self.hub._request("/uploads/create", method="GET", decode="ascii", data=data) - - def _send_chunk(self, upload_id, chunk): - """ - Sends a chunk at a time - """ - # Compute the SHA512 checksum of this chunk - h = hashlib.new("sha512") - h.update(chunk) - - # Encode data in base64 - data = base64.b64encode(chunk) - - # Send chunk to the server - self.hub._request("/uploads/%s/sendchunk" % upload_id, method="POST", - data={ "chksum" : h.hexdigest(), "data" : data }) - - return len(chunk) - - def upload(self): - """ - Main function which runs the upload - """ - # Borrow progressbar from downloader - p = self.hub.http._make_progressbar(message=self.filename, value_max=self.filesize) - - with p: - # Request an upload ID - upload_id = self._get_upload_id() - assert upload_id - - log.debug("Starting upload with id %s" % upload_id) - - # Initial chunk size - chunk_size = CHUNK_SIZE - - try: - with open(self.path, "rb") as f: - while True: - chunk = f.read(chunk_size) - if not chunk: - break - - # Save the time when we started to send this bit - time_started = time.time() - - # Send the chunk to the server - self._send_chunk(upload_id, chunk) - - # Determine the size of the next chunk - duration = time_started - time.time() - chunk_size = math.ceil(chunk_size / duration) - - # Never let chunk_size drop under CHUNK_SIZE - if chunk_size < CHUNK_SIZE: - chunk_size = CHUNK_SIZE - - # Update progressbar - p.increment(len(chunk)) - - # Catch any unhandled exception here, tell the hub to delete the - # file and raise the original exception - except Exception as e: - self.hub._request("/uploads/%s/destroy" % upload_id) - - raise + log.debug("Upload ID: %s" % upload_id) - # If all went well, we finish the upload - else: - self.hub._request("/uploads/%s/finished" % upload_id) + # Upload the data + self._upload("/uploads/stream?id=%s" % upload_id, path) - # Return the upload ID if the upload was successful - return upload_id + return upload_id \ No newline at end of file