From: Michael Tremer <michael.tremer@ipfire.org>
Date: Thu, 14 Apr 2011 19:24:22 +0000 (+0200)
Subject: Upload all packages in chunks.
X-Git-Tag: 0.9.3~47^2~1
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=434863197152127330aef0fae2a81c0648fda13f;p=pakfire.git

Upload all packages in chunks.
---

diff --git a/pakfire/server/base.py b/pakfire/server/base.py
index 958794904..2df0e72b1 100644
--- a/pakfire/server/base.py
+++ b/pakfire/server/base.py
@@ -1,5 +1,6 @@
 #!/usr/bin/python
 
+import hashlib
 import logging
 import os
 import socket
@@ -7,6 +8,8 @@ import xmlrpclib
 
 import pakfire.packages
 
+CHUNK_SIZE = 2097152 # 2M
+
 class MasterSlave(object):
 	@property
 	def hostname(self):
@@ -15,14 +18,40 @@ class MasterSlave(object):
 		"""
 		return socket.gethostname()
 
-	def upload_package_file(self, source_id, pkg_id, pkg):
-		logging.info("Adding package file: %s" % pkg.filename)
+	def _chunked_upload(self, filename):
+		# Update the amount of chunks that there will be to be uploaded.
+		chunks = (os.path.getsize(filename) / CHUNK_SIZE) + 1
+
+		# Open the file for read.
+		f = open(filename, "rb")
+
+		chunk, id = 0, ""
+		while True:
+			# Read a chunk and break if we reached the end of the file.
+			data = f.read(CHUNK_SIZE)
+			if not data:
+				break
+
+			chunk += 1
+			logging.info("Uploading chunk %s/%s of %s." % (chunk, chunks, filename))
+
+			# Calc the hash of the chunk.
+			hash = hashlib.sha1(data)
+
+			# Actually do the upload and make sure we got an ID.
+			id = self.conn.chunk_upload(id, hash.hexdigest(), xmlrpclib.Binary(data))
+			assert id
 
-		# Read-in the package payload.
-		f = open(pkg.filename, "rb")
-		payload = xmlrpclib.Binary(f.read())
 		f.close()
 
+		return id
+
+	def upload_package_file(self, source_id, pkg_id, pkg):
+		logging.info("Uploading package file: %s" % pkg.filename)
+
+		# Upload the file at first to the server.
+		file_id = self._chunked_upload(pkg.filename)
+
 		info = {
 			"filename"    : os.path.basename(pkg.filename),
 			"source_id"   : source_id,
@@ -43,7 +72,6 @@ class MasterSlave(object):
 			"build_id"    : pkg.build_id,
 			"build_time"  : pkg.build_time,
 			"uuid"        : pkg.uuid,
-			"payload"     : payload,
 		}
 
 		if isinstance(pkg, pakfire.packages.BinaryPackage):
@@ -53,7 +81,7 @@ class MasterSlave(object):
 				"conflicts"   : " ".join(pkg.conflicts),
 			})
 
-		return self.conn.upload_package_file(pkg_id, info)
+		return self.conn.package_add_file(pkg_id, file_id, info)
 
 	def upload_log_file(self):
 		pass # XXX TO BE DONE