From: Michael Tremer <michael.tremer@ipfire.org>
Date: Sun, 13 Feb 2011 11:04:24 +0000 (+0100)
Subject: Bunch of misc. changes.
X-Git-Tag: 0.9.3~182
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=fa6d335b1cb53b887ecde9c2a7ad836d20a58bc2;p=pakfire.git

Bunch of misc. changes.

Some code to export repository metadata.
---

diff --git a/pakfire/__init__.py b/pakfire/__init__.py
index 25329169c..ea7a59775 100644
--- a/pakfire/__init__.py
+++ b/pakfire/__init__.py
@@ -159,3 +159,29 @@ class Pakfire(object):
 
 		ts.run()
 
+	def provides(self, patterns):
+		pkgs = []
+
+		for pattern in patterns:
+			pkgs += self.repos.get_by_provides(pattern)
+
+		pkgs = packages.PackageListing(pkgs)
+		#pkgs.unique()
+
+		return pkgs
+
+	def repo_create(self, path):
+		if not os.path.exists(path) or not os.path.isdir(path):
+			raise PakfireError, "Given path is not existant or not a directory: %s" % path
+
+		repo = repository.RemoteRepository(
+			self,
+			name="new",
+			description="New repository.",
+			url="file://%s" % path,
+			gpgkey="XXX",
+			enabled=True,
+		)
+
+		repo.save_index()
+
diff --git a/pakfire/builder.py b/pakfire/builder.py
index dee646674..97cfe302d 100644
--- a/pakfire/builder.py
+++ b/pakfire/builder.py
@@ -12,6 +12,7 @@ import time
 
 import depsolve
 import packages
+import repository
 import transaction
 import util
 
@@ -29,7 +30,7 @@ class Builder(object):
 
 		self.settings = {
 			"enable_loop_devices" : True,
-			"enable_icecream" : False,
+			"enable_icecream" : True,
 		}
 		self.settings.update(settings)
 
@@ -452,7 +453,7 @@ class Builder(object):
 
 		self._packages = []
 		for pkg in pkgs:
-			pkg = packages.VirtualPackage(self.pakfire, repo, pkg)
+			pkg = packages.VirtualPackage(self.pakfire, pkg) # XXX had to remove repo here?!
 			self._packages.append(pkg)
 
 		return self._packages
diff --git a/pakfire/cli.py b/pakfire/cli.py
index e554b174f..a7e09f844 100644
--- a/pakfire/cli.py
+++ b/pakfire/cli.py
@@ -4,6 +4,7 @@ import argparse
 import sys
 
 import packages
+import repository
 
 from pakfire import Pakfire
 
@@ -50,6 +51,7 @@ class Cli(object):
 		self.parse_command_info()
 		self.parse_command_search()
 		self.parse_command_update()
+		self.parse_command_provides()
 
 		# Finally parse all arguments from the command line and save them.
 		self.args = self.parser.parse_args()
@@ -67,6 +69,7 @@ class Cli(object):
 			"update"       : self.handle_update,
 			"info"         : self.handle_info,
 			"search"       : self.handle_search,
+			"provides"     : self.handle_provides,
 		}
 
 	def parse_common_arguments(self):
@@ -119,6 +122,14 @@ class Cli(object):
 			help=_("A pattern to search for."))
 		sub_search.add_argument("action", action="store_const", const="search")
 
+	def parse_command_provides(self):
+		# Implement the "provides" command
+		sub_provides = self.sub_commands.add_parser("provides",
+			help=_("Get a list of packages that provide a given file or feature."))
+		sub_provides.add_argument("pattern", nargs="+",
+			help=_("File or feature to search for."))
+		sub_provides.add_argument("action", action="store_const", const="provides")
+
 	def run(self):
 		action = self.args.action
 
@@ -168,6 +179,12 @@ class Cli(object):
 	def handle_localinstall(self):
 		return self.handle_install(local=True)
 
+	def handle_provides(self):
+		pkgs = self.pakfire.provides(self.args.pattern)
+
+		for pkg in pkgs:
+			print pkg.dump()
+
 
 class CliBuilder(Cli):
 	def __init__(self):
@@ -186,6 +203,7 @@ class CliBuilder(Cli):
 		self.parse_command_search()
 		self.parse_command_shell()
 		self.parse_command_update()
+		self.parse_command_repo()
 
 		# Finally parse all arguments from the command line and save them.
 		self.args = self.parser.parse_args()
@@ -197,12 +215,13 @@ class CliBuilder(Cli):
 		)
 
 		self.action2func = {
-			"build"   : self.handle_build,
-			"dist"    : self.handle_dist,
-			"update"  : self.handle_update,
-			"info"    : self.handle_info,
-			"search"  : self.handle_search,
-			"shell"   : self.handle_shell,
+			"build"       : self.handle_build,
+			"dist"        : self.handle_dist,
+			"update"      : self.handle_update,
+			"info"        : self.handle_info,
+			"search"      : self.handle_search,
+			"shell"       : self.handle_shell,
+			"repo_create" : self.handle_repo_create,
 		}
 
 	def parse_command_update(self):
@@ -246,6 +265,20 @@ class CliBuilder(Cli):
 		sub_dist.add_argument("--resultdir", nargs="?",
 			help=_("Path were the output files should be copied to."))
 
+	def parse_command_repo(self):
+		sub_repo = self.sub_commands.add_parser("repo",
+			help=_("Repository management commands."))
+
+		sub_repo_commands = sub_repo.add_subparsers()
+
+		self.parse_command_repo_create(sub_repo_commands)
+
+	def parse_command_repo_create(self, sub_commands):
+		sub_create = sub_commands.add_parser("create",
+			help=_("Create a new repository index."))
+		sub_create.add_argument("path", nargs=1, help=_("Path to the packages."))
+		sub_create.add_argument("action", action="store_const", const="repo_create")
+
 	def handle_build(self):
 		print self.args
 		# Get the package descriptor from the command line options
@@ -308,3 +341,8 @@ class CliBuilder(Cli):
 
 		self.pakfire.dist(pkg, self.args.resultdir)
 
+	def handle_repo_create(self):
+		path = self.args.path[0]
+
+		self.pakfire.repo_create(path)
+
diff --git a/pakfire/constants.py b/pakfire/constants.py
index a91137567..326777b57 100644
--- a/pakfire/constants.py
+++ b/pakfire/constants.py
@@ -10,6 +10,7 @@ CONFIG_FILE = os.path.join(SYSCONFDIR, "pakfire.conf")
 CACHE_DIR = "/var/cache/pakfire"
 
 PACKAGES_DB = "var/lib/pakfire/packages.db"
+REPOSITORY_DB = "index.db"
 
 BUFFER_SIZE = 1024**2
 
diff --git a/pakfire/database.py b/pakfire/database.py
index 69724dcf1..4c1a1c610 100644
--- a/pakfire/database.py
+++ b/pakfire/database.py
@@ -16,7 +16,7 @@ class Database(object):
 
 	def __del__(self):
 		if self._db:
-			self._db.commit()
+			#self._db.commit()
 			self._db.close()
 
 	def create(self):
@@ -50,7 +50,7 @@ class Database(object):
 		return self._db.cursor()
 
 
-class LocalPackageDatabase(Database):
+class PackageDatabase(Database):
 	def create(self):
 		c = self.cursor()
 
@@ -71,6 +71,7 @@ class LocalPackageDatabase(Database):
 				epoch		INTEGER,
 				version		TEXT,
 				release		TEXT,
+				filename	TEXT,
 				installed	INTEGER,
 				reason		TEXT,
 				repository	TEXT,
@@ -101,6 +102,66 @@ class LocalPackageDatabase(Database):
 
 		c.close()
 
+	def package_exists(self, pkg):
+		return not self.get_id_by_pkg(pkg) is None
+
+	def get_id_by_pkg(self, pkg):
+		c = self.cursor()
+
+		c.execute("SELECT id FROM packages WHERE name = ? AND version = ? AND \
+			release = ? AND epoch = ? LIMIT 1", (pkg.name, pkg.version, pkg.release, pkg.epoch))
+
+		ret = None
+		for i in c:
+			ret = i["id"]
+			break
+		
+		c.close()
+
+		return ret
+
+	def add_package(self, pkg):
+		if self.package_exists(pkg):
+			logging.debug("Skipping package which already exists in database: %s" % pkg.friendly_name)
+			return
+
+		logging.debug("Adding package to database: %s" % pkg.friendly_name)
+
+		c = self.cursor()
+		c.execute("""
+			INSERT INTO packages(
+				name,
+				epoch,
+				version,
+				release,
+				filename,
+				provides,
+				requires
+			) VALUES(?, ?, ?, ?, ?, ?, ?)""",
+			(
+				pkg.name,
+				pkg.epoch,
+				pkg.version,
+				pkg.release,
+				pkg.filename,
+				" ".join(pkg.provides),
+				" ".join(pkg.requires),
+			)
+		)
+		c.close()
+		self.commit()
+
+		pkg_id = self.get_id_by_pkg(pkg)
+
+		c = self.cursor()
+		for file in pkg.filelist:
+			c.execute("INSERT INTO files(name, pkg) VALUES(?, ?)", (file, pkg_id))
+
+		c.close()
+		self.commit()
+
+
+class LocalPackageDatabase(PackageDatabase):
 	def add_package(self, pkg, installed=True):
 		c = self.cursor()
 
diff --git a/pakfire/index.py b/pakfire/index.py
index bb6b43c7a..757d7a47c 100644
--- a/pakfire/index.py
+++ b/pakfire/index.py
@@ -3,6 +3,7 @@
 import logging
 import os
 
+import database
 import packages
 
 from constants import *
@@ -61,9 +62,20 @@ class DirectoryIndex(Index):
 
 		Index.__init__(self, pakfire, repo)
 
+		# Always update this because it will otherwise contain no data
+		self.update(force=True)
+
 	def update(self, force=False):
 		logging.debug("Updating repository index '%s' (force=%s)" % (self.path, force))
 
+		# Do nothing if the update is not forced but populate the database
+		# if no packages are present.
+		if not force and self._packages:
+			return
+
+		# If we update the cache, we clear it first.
+		self._packages = []
+
 		for dir, subdirs, files in os.walk(self.path):
 			for file in files:
 				# Skip files that do not have the right extension
@@ -81,19 +93,38 @@ class DirectoryIndex(Index):
 
 				self._packages.append(package)
 
+	def save(self, path=None):
+		if not path:
+			path = self.path
+
+		path = os.path.join(path, "index.db")
+
+		db = database.PackageDatabase(self.pakfire, path)
 
-class InstalledIndex(Index):
+		for pkg in self.packages:
+			db.add_package(pkg)
+
+		db.close()
+
+
+class DatabaseIndex(Index):
 	def __init__(self, pakfire, repo, db):
 		self.db = db
 
 		Index.__init__(self, pakfire, repo)
 
+	def update(self, force=False):
+		"""
+			Nothing to do here.
+		"""
+		pass
+
 	def get_all_by_name(self, name):
 		c = self.db.cursor()
 		c.execute("SELECT * FROM packages WHERE name = ?", name)
 
 		for pkg in c:
-			yield package.InstalledPackage(self.pakfire, self.db, pkg)
+			yield package.DatabasePackage(self.pakfire, self.db, pkg)
 
 		c.close()
 
@@ -113,7 +144,12 @@ class InstalledIndex(Index):
 		c.execute("SELECT * FROM packages")
 
 		for pkg in c:
-			yield packages.InstalledPackage(self.pakfire, self.db, pkg)
+			yield packages.DatabasePackage(self.pakfire, self.db, pkg)
 
 		c.close()
 
+
+# XXX maybe this can be removed later?
+class InstalledIndex(DatabaseIndex):
+	pass
+
diff --git a/pakfire/packages/__init__.py b/pakfire/packages/__init__.py
index bb302e905..af5711871 100644
--- a/pakfire/packages/__init__.py
+++ b/pakfire/packages/__init__.py
@@ -1,7 +1,7 @@
 #!/usr/bin/python
 
 from binary import BinaryPackage
-from installed import InstalledPackage
+from installed import DatabasePackage, InstalledPackage
 from source import SourcePackage
 from virtual import VirtualPackage
 
diff --git a/pakfire/packages/installed.py b/pakfire/packages/installed.py
index 5ddb2d0e7..05c43a811 100644
--- a/pakfire/packages/installed.py
+++ b/pakfire/packages/installed.py
@@ -7,11 +7,8 @@ import util
 
 from base import Package
 
-
-# XXX maybe this gets renamed to "DatabasePackage" or something similar.
-
-class InstalledPackage(Package):
-	type = "installed"
+class DatabasePackage(Package):
+	type = "db"
 
 	def __init__(self, pakfire, db, data):
 		Package.__init__(self, pakfire, pakfire.repos.local)
@@ -153,3 +150,8 @@ class InstalledPackage(Package):
 			(filename, self.id, size, type, hash1, time.time()))
 		c.close()
 
+
+# XXX maybe we can remove this later?
+class InstalledPackage(DatabasePackage):
+	type = "installed"
+
diff --git a/pakfire/packages/packager.py b/pakfire/packages/packager.py
index 6d82d63d1..8c4229f7e 100644
--- a/pakfire/packages/packager.py
+++ b/pakfire/packages/packager.py
@@ -23,7 +23,10 @@ class Extractor(object):
 		self.archive = None
 		self._tempfile = None
 
-		self._uncompress_data()
+		if pkg.payload_compression == "XXX":
+			self.archive = tarfile.open(fileobj=self.data)
+		else:
+			self._uncompress_data()
 
 	def cleanup(self):
 		# XXX not called by anything
@@ -87,16 +90,16 @@ class Extractor(object):
 		if member.isdir() and os.path.exists(target):
 			return
 
-		if self.pakfire.config.get("debug"):
-			msg = "Creating file (%s:%03d:%03d) " % \
-				(tarfile.filemode(member.mode), member.uid, member.gid)
-			if member.issym():
-				msg += "/%s -> %s" % (member.name, member.linkname)
-			elif member.islnk():
-				msg += "/%s link to /%s" % (member.name, member.linkname)
-			else:
-				msg += "/%s" % member.name
-			logging.debug(msg)
+		#if self.pakfire.config.get("debug"):
+		#	msg = "Creating file (%s:%03d:%03d) " % \
+		#		(tarfile.filemode(member.mode), member.uid, member.gid)
+		#	if member.issym():
+		#		msg += "/%s -> %s" % (member.name, member.linkname)
+		#	elif member.islnk():
+		#		msg += "/%s link to /%s" % (member.name, member.linkname)
+		#	else:
+		#		msg += "/%s" % member.name
+		#	logging.debug(msg)
 
 		# Remove file if it has been existant
 		if not member.isdir() and os.path.exists(target):
diff --git a/pakfire/repository.py b/pakfire/repository.py
index 276c64f0e..c0a2b66e4 100644
--- a/pakfire/repository.py
+++ b/pakfire/repository.py
@@ -300,6 +300,9 @@ class RemoteRepository(RepositoryFactory):
 		if self.index:
 			self.index.update(force=force)
 
+	def save_index(self, path=None):
+		self.index.save(path)
+
 	#def get_all(self, requires):
 	#	for pkg in self.index.get_all():
 	#		if pkg.does_provide(requires):
diff --git a/pakfire/transaction.py b/pakfire/transaction.py
index 283b789e7..f5f42c142 100644
--- a/pakfire/transaction.py
+++ b/pakfire/transaction.py
@@ -268,3 +268,4 @@ class TransactionSet(object):
 		s.append("")
 
 		print "\n".join(s)
+