From: Michael Tremer Date: Sat, 26 Feb 2011 17:37:14 +0000 (+0100) Subject: Make package loading (esp. from database) faster. X-Git-Tag: 0.9.3~112 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=b6da0663223818c8bc7c2d9a8a13e3856ca321c2;p=pakfire.git Make package loading (esp. from database) faster. --- diff --git a/pakfire/database.py b/pakfire/database.py index 50416d6a0..54812ec2c 100644 --- a/pakfire/database.py +++ b/pakfire/database.py @@ -10,6 +10,14 @@ import packages from constants import * +class Cursor(sqlite3.Cursor): + def execute(self, *args, **kwargs): + # For debugging of SQL queries. + #print args, kwargs + + return sqlite3.Cursor.execute(self, *args, **kwargs) + + class Database(object): def __init__(self, pakfire, filename): self.pakfire = pakfire @@ -52,7 +60,7 @@ class Database(object): self._db.commit() def cursor(self): - return self._db.cursor() + return self._db.cursor(Cursor) def save(self, path): """ diff --git a/pakfire/depsolve.py b/pakfire/depsolve.py index 5653cf670..32a4ba52d 100644 --- a/pakfire/depsolve.py +++ b/pakfire/depsolve.py @@ -164,7 +164,13 @@ class DependencySet(object): # Fetch all candidates from the repositories and save the # best one - candidates = packages.PackageListing(self.repos.get_by_provides(requires)) + if requires.type == "file": + candidates = self.repos.get_by_file(requires.requires) + else: + candidates = self.repos.get_by_provides(requires) + + # Turn the candidates into a package listing. + candidates = packages.PackageListing(candidates) if not candidates: logging.debug(" Got no candidates for that") diff --git a/pakfire/index.py b/pakfire/index.py index f560c63f1..86f4d2252 100644 --- a/pakfire/index.py +++ b/pakfire/index.py @@ -48,20 +48,24 @@ class Index(object): # Return the last one. return p[-1] + def get_by_file(self, filename): + for pkg in self.packages: + if filename in pkg.filelist: + yield pkg + + def get_by_id(self, id): + raise NotImplementedError + + def get_by_uuid(self, uuid): + for pkg in self.packages: + if pkg.uuid == uuid: + return pkg + @property def packages(self): for pkg in self._packages: yield pkg - @property - def package_names(self): - names = [] - for name in [p.name for p in self.packages]: - if not name in names: - names.append(name) - - return sorted(names) - def update(self, force=False): raise NotImplementedError @@ -132,7 +136,7 @@ class InstalledIndex(Index): # Open the database. self.db = database.LocalPackageDatabase(self.pakfire) - def __get_from_cache(self, pkg): + def _get_from_cache(self, pkg): """ Check if package is already in cache and return an instance of BinaryPackage instead. @@ -147,44 +151,31 @@ class InstalledIndex(Index): return pkg - def get_all_by_name(self, name): - c = self.db.cursor() - c.execute("SELECT * FROM packages WHERE name = ?", name) - - for pkg in c: - pkg = package.DatabasePackage(self.pakfire, self.repo, self.db, pkg) - - # Try to get package from cache. - yield self.__get_from_cache(pkg) - - c.close() + def add_package(self, pkg, reason=None): + return self.db.add_package(pkg, reason) - @property - def package_names(self): + def get_by_id(self, id): c = self.db.cursor() - c.execute("SELECT DISTINCT name FROM packages ORDER BY name") + c.execute("SELECT uuid FROM packages WHERE id = ?", (id,)) for pkg in c: - yield pkg["name"] + break c.close() - @property - def packages(self): - c = self.db.cursor() - c.execute("SELECT * FROM packages") + return self.get_by_uuid(pkg["uuid"]) - for pkg in c: - pkg = packages.DatabasePackage(self.pakfire, self.repo, self.db, pkg) + def get_by_file(self, filename): + c = self.db.cursor() + c.execute("SELECT pkg FROM files WHERE name = ?", (filename,)) - # Try to get package from cache. - yield self.__get_from_cache(pkg) + for file in c: + pkg = self.get_by_id(file["pkg"]) + if pkg: + yield pkg c.close() - def add_package(self, pkg, reason=None): - return self.db.add_package(pkg, reason) - class DatabaseIndex(InstalledIndex): def __init__(self, pakfire, repo): @@ -204,6 +195,23 @@ class DatabaseIndex(InstalledIndex): os.unlink(self.db.filename) + def load_database(self): + """ + Read all packages into RAM. + """ + self._packages = [] + + c = self.db.cursor() + c.execute("SELECT * FROM packages") + + for pkg in c: + pkg = packages.DatabasePackage(self.pakfire, self.repo, self.db, pkg) + + # Try to get package from cache. + self._packages.append(self._get_from_cache(pkg)) + + c.close() + def _update_metadata(self, force): # Shortcut to repository cache. cache = self.repo.cache @@ -314,6 +322,7 @@ class DatabaseIndex(InstalledIndex): # (Re-)open the database. self.db = database.RemotePackageDatabase(self.pakfire, cache.abspath(filename)) + self.load_database() def update(self, force=False): """ diff --git a/pakfire/packages/base.py b/pakfire/packages/base.py index d38b89f7c..a1bae3518 100644 --- a/pakfire/packages/base.py +++ b/pakfire/packages/base.py @@ -256,11 +256,6 @@ class Package(object): if not isinstance(requires, pakfire.depsolve.Requires): requires = pakfire.depsolve.Requires(self, requires) - # If the provides string equals the name of the package, we - # return true. - if self.name == requires.requires: - return True - # Get all provide strings from the package data # and return true if requires is matched. if requires.requires in self.provides: diff --git a/pakfire/packages/binary.py b/pakfire/packages/binary.py index c58b2df32..b42778f56 100644 --- a/pakfire/packages/binary.py +++ b/pakfire/packages/binary.py @@ -26,14 +26,18 @@ class BinaryPackage(FilePackage): @property def provides(self): - provides = self.metadata.get("PKG_PROVIDES").split() + if not hasattr(self, "__provides"): + # Get automatic provides + provides = self._provides - # Add autoprovides - for prov in self._provides: - if not prov in provides: - provides.append(prov) + # Add other provides + for prov in self.metadata.get("PKG_PROVIDES").split(): + if not prov in provides: + provides.append(prov) - return provides + self.__provides = provides + + return self.__provides @property def conflicts(self): diff --git a/pakfire/packages/installed.py b/pakfire/packages/installed.py index e55cba2e4..944de61c0 100644 --- a/pakfire/packages/installed.py +++ b/pakfire/packages/installed.py @@ -101,14 +101,18 @@ class DatabasePackage(Package): @property def provides(self): - provides = self.metadata.get("provides", "").split() + if not hasattr(self, "__provides"): + # Get automatic provides + provides = self._provides - # Add autoprovides - for prov in self._provides: - if not prov in provides: - provides.append(prov) + # Add other provides + for prov in self.metadata.get("provides", "").split(): + if not prov in provides: + provides.append(prov) - return provides + self.__provides = provides + + return self.__provides @property def requires(self): @@ -142,17 +146,17 @@ class DatabasePackage(Package): @property def filelist(self): - c = self.db.cursor() - c.execute("SELECT name FROM files WHERE pkg = '%s'" % self.id) # XXX? + if not hasattr(self, "__filelist"): + c = self.db.cursor() + c.execute("SELECT name FROM files WHERE pkg = ?", (self.id,)) - for f in c: - filename = f["name"] - if not filename.startswith("/"): - filename = "/%s" % filename + self.__filelist = [] + for f in c: + self.__filelist.append(f["name"]) - yield filename + c.close() - c.close() + return self.__filelist def _does_provide_file(self, requires): """ diff --git a/pakfire/repository.py b/pakfire/repository.py index 531c57b76..a892718af 100644 --- a/pakfire/repository.py +++ b/pakfire/repository.py @@ -39,7 +39,8 @@ class Repositories(object): self.add_repo(self.local) # If we running in build mode, we include our local build repository. - if self.pakfire.builder: + #if self.pakfire.builder: + if True: self.local_build = LocalBuildRepository(self.pakfire) self.add_repo(self.local_build) @@ -117,6 +118,11 @@ class Repositories(object): for pkg in repo.get_by_provides(requires): yield pkg + def get_by_file(self, filename): + for repo in self.enabled: + for pkg in repo.get_by_file(filename): + yield pkg + def search(self, pattern): pkg_names = [] @@ -198,6 +204,11 @@ class RepositoryFactory(object): if pkg.does_provide(requires): yield pkg + def get_by_file(self, filename): + for pkg in self.packages: + if filename in pkg.filelist: + yield pkg + def search(self, pattern): """ Returns a list of packages, that match the given pattern, @@ -513,3 +524,6 @@ class RemoteRepository(RepositoryFactory): # if pkg.does_provide(requires): # yield pkg + def get_by_file(self, filename): + return self.index.get_by_file(filename) +