From: Michael Tremer <michael.tremer@ipfire.org>
Date: Fri, 25 Feb 2011 23:14:16 +0000 (+0100)
Subject: Add support for compression of database.
X-Git-Tag: 0.9.3~124
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=93135ecb788c3cd1622bf43548612536776234e8;p=pakfire.git

Add support for compression of database.
---

diff --git a/pakfire/index.py b/pakfire/index.py
index 631e94da1..4082fc669 100644
--- a/pakfire/index.py
+++ b/pakfire/index.py
@@ -2,9 +2,11 @@
 
 import json
 import logging
+import lzma
 import os
 import random
 import shutil
+import zlib
 
 import database
 import downloader
@@ -272,7 +274,34 @@ class DatabaseIndex(InstalledIndex):
 			with cache.open(filename, "w") as o:
 				o.write(data)
 
-			# XXX possibly, the database needs to be decompressed
+			# decompress the database
+			if self.metadata.database_compression:
+				# Open input file and remove the file immediately.
+				# The fileobj is still open and the data will be removed
+				# when it is closed.
+				i = cache.open(filename)
+				cache.remove(filename)
+
+				# Open output file.
+				o = cache.open(filename, "w")
+
+				# Choose a decompessor.
+				if self.metadata.database_compression == "xz":
+					comp = lzma.LZMADecompressor()
+
+				elif self.metadata.database_compression == "zlib":
+					comp = zlib.decompressobj()
+
+				buf = i.read(BUFFER_SIZE)
+				while buf:
+					o.write(comp.decompress(buf))
+
+					buf = i.read(BUFFER_SIZE)
+
+				o.write(decompressor.flush())
+
+				i.close()
+				o.close()
 
 		# (Re-)open the database.
 		self.db = database.RemotePackageDatabase(self.pakfire,
@@ -296,9 +325,8 @@ class DatabaseIndex(InstalledIndex):
 		# XXX this code needs lots of work:
 		# XXX   * make checks for downloads (hashsums)
 		# XXX   * check the metadata content
-		# XXX   * use compression
 
-	def save(self, path=None):
+	def save(self, path=None, compress="xz"):
 		"""
 			This function saves the database and metadata to path so it can
 			be exported to a remote repository.
@@ -314,9 +342,33 @@ class DatabaseIndex(InstalledIndex):
 		if not os.path.exists(metapath):
 			os.makedirs(metapath)
 
-		# Save the database to path and get the filename
+		# Save the database to path and get the filename.
 		self.db.save(db_path)
 
+		# Compress the database.
+		if compress:
+			i = open(db_path)
+			os.unlink(db_path)
+
+			o = open(db_path, "w")
+
+			# Choose a compressor.
+			if compress == "xz":
+				comp = lzma.LZMACompressor()
+			elif compress == "zlib":
+				comp = zlib.compressobj(9)
+
+			buf = i.read(BUFFER_SIZE)
+			while buf:
+				o.write(comp.compress(buf))
+
+				buf = i.read(BUFFER_SIZE)
+
+			o.write(comp.flush())
+
+			i.close()
+			o.close()
+
 		# Make a reference to the database file that it will get a unique name
 		# so we won't get into any trouble with caching proxies.
 		db_hash = util.calc_hash1(db_path)
@@ -333,6 +385,7 @@ class DatabaseIndex(InstalledIndex):
 		# Save name of the hashed database to the metadata.
 		md.database = os.path.basename(db_path2)
 		md.database_hash1 = db_hash
+		md.database_compression = compress
 
 		# Save metdata to repository.
 		md.save(md_path)
diff --git a/pakfire/metadata.py b/pakfire/metadata.py
index a6350b4bb..6a1168a18 100644
--- a/pakfire/metadata.py
+++ b/pakfire/metadata.py
@@ -94,3 +94,12 @@ class Metadata(object):
 		self._data["database_hash1"] = val
 
 	database_hash1 = property(get_database_hash1, set_database_hash1)
+
+	def get_database_compression(self):
+		return self._data.get("database_compression", None)
+
+	def set_database_compression(self, val):
+		self._data["database_compression"] = val
+
+	database_compression = property(get_database_compression,
+		set_database_compression)