]> git.ipfire.org Git - pakfire.git/commitdiff
Make package loading (esp. from database) faster.
authorMichael Tremer <michael.tremer@ipfire.org>
Sat, 26 Feb 2011 17:37:14 +0000 (18:37 +0100)
committerMichael Tremer <michael.tremer@ipfire.org>
Sat, 26 Feb 2011 17:37:14 +0000 (18:37 +0100)
pakfire/database.py
pakfire/depsolve.py
pakfire/index.py
pakfire/packages/base.py
pakfire/packages/binary.py
pakfire/packages/installed.py
pakfire/repository.py

index 50416d6a001215e70142a3a226e3f64ac84cea72..54812ec2c584e1f0385b95a4ca200912c3f1798d 100644 (file)
@@ -10,6 +10,14 @@ import packages
 
 from constants import *
 
+class Cursor(sqlite3.Cursor):
+       def execute(self, *args, **kwargs):
+               # For debugging of SQL queries.
+               #print args, kwargs
+
+               return sqlite3.Cursor.execute(self, *args, **kwargs)
+
+
 class Database(object):
        def __init__(self, pakfire, filename):
                self.pakfire = pakfire
@@ -52,7 +60,7 @@ class Database(object):
                self._db.commit()
 
        def cursor(self):
-               return self._db.cursor()
+               return self._db.cursor(Cursor)
 
        def save(self, path):
                """
index 5653cf670f3735975768f6ced229ffd720095fc0..32a4ba52dc506bea8470892d16a0c0f67a820ca2 100644 (file)
@@ -164,7 +164,13 @@ class DependencySet(object):
 
                        # Fetch all candidates from the repositories and save the
                        # best one
-                       candidates = packages.PackageListing(self.repos.get_by_provides(requires))
+                       if requires.type == "file":
+                               candidates = self.repos.get_by_file(requires.requires)
+                       else:
+                               candidates = self.repos.get_by_provides(requires)
+
+                       # Turn the candidates into a package listing.
+                       candidates = packages.PackageListing(candidates)
 
                        if not candidates:
                                logging.debug("  Got no candidates for that")
index f560c63f1b849d051ab3c1e80cd1beaa79a88706..86f4d22523c2b0761b815e5e92f91d233a05ef1a 100644 (file)
@@ -48,20 +48,24 @@ class Index(object):
                # Return the last one.
                return p[-1]
 
+       def get_by_file(self, filename):
+               for pkg in self.packages:
+                       if filename in pkg.filelist:
+                               yield pkg
+
+       def get_by_id(self, id):
+               raise NotImplementedError
+
+       def get_by_uuid(self, uuid):
+               for pkg in self.packages:
+                       if pkg.uuid == uuid:
+                               return pkg
+
        @property
        def packages(self):
                for pkg in self._packages:
                        yield pkg
 
-       @property
-       def package_names(self):
-               names = []
-               for name in [p.name for p in self.packages]:
-                       if not name in names:
-                               names.append(name)
-
-               return sorted(names)
-
        def update(self, force=False):
                raise NotImplementedError
 
@@ -132,7 +136,7 @@ class InstalledIndex(Index):
                # Open the database.
                self.db = database.LocalPackageDatabase(self.pakfire)
 
-       def __get_from_cache(self, pkg):
+       def _get_from_cache(self, pkg):
                """
                        Check if package is already in cache and return an instance of
                        BinaryPackage instead.
@@ -147,44 +151,31 @@ class InstalledIndex(Index):
 
                return pkg
 
-       def get_all_by_name(self, name):
-               c = self.db.cursor()
-               c.execute("SELECT * FROM packages WHERE name = ?", name)
-
-               for pkg in c:
-                       pkg = package.DatabasePackage(self.pakfire, self.repo, self.db, pkg)
-
-                       # Try to get package from cache.
-                       yield self.__get_from_cache(pkg)
-
-               c.close()
+       def add_package(self, pkg, reason=None):
+               return self.db.add_package(pkg, reason)
 
-       @property
-       def package_names(self):
+       def get_by_id(self, id):
                c = self.db.cursor()
-               c.execute("SELECT DISTINCT name FROM packages ORDER BY name")
+               c.execute("SELECT uuid FROM packages WHERE id = ?", (id,))
 
                for pkg in c:
-                       yield pkg["name"]
+                       break
 
                c.close()
 
-       @property
-       def packages(self):
-               c = self.db.cursor()
-               c.execute("SELECT * FROM packages")
+               return self.get_by_uuid(pkg["uuid"])
 
-               for pkg in c:
-                       pkg = packages.DatabasePackage(self.pakfire, self.repo, self.db, pkg)
+       def get_by_file(self, filename):
+               c = self.db.cursor()
+               c.execute("SELECT pkg FROM files WHERE name = ?", (filename,))
 
-                       # Try to get package from cache.
-                       yield self.__get_from_cache(pkg)
+               for file in c:
+                       pkg = self.get_by_id(file["pkg"])
+                       if pkg:
+                               yield pkg
 
                c.close()
 
-       def add_package(self, pkg, reason=None):
-               return self.db.add_package(pkg, reason)
-
 
 class DatabaseIndex(InstalledIndex):
        def __init__(self, pakfire, repo):
@@ -204,6 +195,23 @@ class DatabaseIndex(InstalledIndex):
 
                        os.unlink(self.db.filename)
 
+       def load_database(self):
+               """
+                       Read all packages into RAM.
+               """
+               self._packages = []
+
+               c = self.db.cursor()
+               c.execute("SELECT * FROM packages")
+
+               for pkg in c:
+                       pkg = packages.DatabasePackage(self.pakfire, self.repo, self.db, pkg)
+
+                       # Try to get package from cache.
+                       self._packages.append(self._get_from_cache(pkg))
+
+               c.close()
+
        def _update_metadata(self, force):
                # Shortcut to repository cache.
                cache = self.repo.cache
@@ -314,6 +322,7 @@ class DatabaseIndex(InstalledIndex):
                # (Re-)open the database.
                self.db = database.RemotePackageDatabase(self.pakfire,
                        cache.abspath(filename))
+               self.load_database()
 
        def update(self, force=False):
                """
index d38b89f7c156dcbfcab79cee62cbea42bdd9ea74..a1bae35181b39ee0ca0302251fe38c8a7a853523 100644 (file)
@@ -256,11 +256,6 @@ class Package(object):
                if not isinstance(requires, pakfire.depsolve.Requires):
                        requires = pakfire.depsolve.Requires(self, requires)
 
-               # If the provides string equals the name of the package, we
-               # return true.
-               if self.name == requires.requires:
-                       return True
-
                # Get all provide strings from the package data
                # and return true if requires is matched.
                if requires.requires in self.provides:
index c58b2df3280f0c7889a56956d7dd243b1c7c97b7..b42778f56191ae04ffb8a264c8d77f3f72d08c15 100644 (file)
@@ -26,14 +26,18 @@ class BinaryPackage(FilePackage):
 
        @property
        def provides(self):
-               provides = self.metadata.get("PKG_PROVIDES").split()
+               if not hasattr(self, "__provides"):
+                       # Get automatic provides
+                       provides = self._provides
 
-               # Add autoprovides
-               for prov in self._provides:
-                       if not prov in provides:
-                               provides.append(prov)
+                       # Add other provides
+                       for prov in self.metadata.get("PKG_PROVIDES").split():
+                               if not prov in provides:
+                                       provides.append(prov)
 
-               return provides
+                       self.__provides = provides
+
+               return self.__provides
 
        @property
        def conflicts(self):
index e55cba2e41083132b1c7972061a21119c6e0f759..944de61c07d620a7046065706b5a12cbba0a4bfb 100644 (file)
@@ -101,14 +101,18 @@ class DatabasePackage(Package):
 
        @property
        def provides(self):
-               provides = self.metadata.get("provides", "").split()
+               if not hasattr(self, "__provides"):
+                       # Get automatic provides
+                       provides = self._provides
 
-               # Add autoprovides
-               for prov in self._provides:
-                       if not prov in provides:
-                               provides.append(prov)
+                       # Add other provides
+                       for prov in self.metadata.get("provides", "").split():
+                               if not prov in provides:
+                                       provides.append(prov)
 
-               return provides
+                       self.__provides = provides
+
+               return self.__provides
 
        @property
        def requires(self):
@@ -142,17 +146,17 @@ class DatabasePackage(Package):
 
        @property
        def filelist(self):
-               c = self.db.cursor()
-               c.execute("SELECT name FROM files WHERE pkg = '%s'" % self.id) # XXX?
+               if not hasattr(self, "__filelist"):
+                       c = self.db.cursor()
+                       c.execute("SELECT name FROM files WHERE pkg = ?", (self.id,))
 
-               for f in c:
-                       filename = f["name"]
-                       if not filename.startswith("/"):
-                               filename = "/%s" % filename
+                       self.__filelist = []
+                       for f in c:
+                               self.__filelist.append(f["name"])
 
-                       yield filename
+                       c.close()
 
-               c.close()
+               return self.__filelist
 
        def _does_provide_file(self, requires):
                """
index 531c57b7689a1ea189907574fd2eab6e6cbb4ac8..a892718afe6944abd4b953d29f4189b583de5064 100644 (file)
@@ -39,7 +39,8 @@ class Repositories(object):
                self.add_repo(self.local)
 
                # If we running in build mode, we include our local build repository.
-               if self.pakfire.builder:
+               #if self.pakfire.builder:
+               if True:
                        self.local_build = LocalBuildRepository(self.pakfire)
                        self.add_repo(self.local_build)
 
@@ -117,6 +118,11 @@ class Repositories(object):
                        for pkg in repo.get_by_provides(requires):
                                yield pkg
 
+       def get_by_file(self, filename):
+               for repo in self.enabled:
+                       for pkg in repo.get_by_file(filename):
+                               yield pkg
+
        def search(self, pattern):
                pkg_names = []
 
@@ -198,6 +204,11 @@ class RepositoryFactory(object):
                        if pkg.does_provide(requires):
                                yield pkg
 
+       def get_by_file(self, filename):
+               for pkg in self.packages:
+                       if filename in pkg.filelist:
+                               yield pkg
+
        def search(self, pattern):
                """
                        Returns a list of packages, that match the given pattern,
@@ -513,3 +524,6 @@ class RemoteRepository(RepositoryFactory):
        #               if pkg.does_provide(requires):
        #                       yield pkg
 
+       def get_by_file(self, filename):
+               return self.index.get_by_file(filename)
+