Upload all packages in chunks.

diff --git a/pakfire/server/base.py b/pakfire/server/base.py
index 95879490447a21f43abab987d7fc1f12342e85a1..2df0e72b14890a85b16e5ac93cf6f034216204a9 100644 (file)
--- a/pakfire/server/base.py
+++ b/pakfire/server/base.py
@@ -1,5 +1,6 @@
 #!/usr/bin/python
 
+import hashlib
 import logging
 import os
 import socket
@@ -7,6 +8,8 @@ import xmlrpclib
 
 import pakfire.packages
 
+CHUNK_SIZE = 2097152 # 2M
+
 class MasterSlave(object):
        @property
        def hostname(self):
@@ -15,14 +18,40 @@ class MasterSlave(object):
                """
                return socket.gethostname()
 
-       def upload_package_file(self, source_id, pkg_id, pkg):
-               logging.info("Adding package file: %s" % pkg.filename)
+       def _chunked_upload(self, filename):
+               # Update the amount of chunks that there will be to be uploaded.
+               chunks = (os.path.getsize(filename) / CHUNK_SIZE) + 1
+
+               # Open the file for read.
+               f = open(filename, "rb")
+
+               chunk, id = 0, ""
+               while True:
+                       # Read a chunk and break if we reached the end of the file.
+                       data = f.read(CHUNK_SIZE)
+                       if not data:
+                               break
+
+                       chunk += 1
+                       logging.info("Uploading chunk %s/%s of %s." % (chunk, chunks, filename))
+
+                       # Calc the hash of the chunk.
+                       hash = hashlib.sha1(data)
+
+                       # Actually do the upload and make sure we got an ID.
+                       id = self.conn.chunk_upload(id, hash.hexdigest(), xmlrpclib.Binary(data))
+                       assert id
 
-               # Read-in the package payload.
-               f = open(pkg.filename, "rb")
-               payload = xmlrpclib.Binary(f.read())
                f.close()
 
+               return id
+
+       def upload_package_file(self, source_id, pkg_id, pkg):
+               logging.info("Uploading package file: %s" % pkg.filename)
+
+               # Upload the file at first to the server.
+               file_id = self._chunked_upload(pkg.filename)
+
                info = {
                        "filename"    : os.path.basename(pkg.filename),
                        "source_id"   : source_id,
@@ -43,7 +72,6 @@ class MasterSlave(object):
                        "build_id"    : pkg.build_id,
                        "build_time"  : pkg.build_time,
                        "uuid"        : pkg.uuid,
-                       "payload"     : payload,
                }
 
                if isinstance(pkg, pakfire.packages.BinaryPackage):
@@ -53,7 +81,7 @@ class MasterSlave(object):
                                "conflicts"   : " ".join(pkg.conflicts),
                        })
 
-               return self.conn.upload_package_file(pkg_id, info)
+               return self.conn.package_add_file(pkg_id, file_id, info)
 
        def upload_log_file(self):
                pass # XXX TO BE DONE