From 9729c5b3033369cb1ec4eeb0e870d0fd7c71a1a7 Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Tue, 10 Oct 2017 15:57:31 +0100 Subject: [PATCH] Move remastering repositories Signed-off-by: Michael Tremer --- Makefile.am | 3 +- src/buildservice/repository.py | 136 ++++++++++++++++++++++++------ src/manager/__init__.py | 1 - src/manager/repositories.py | 128 ---------------------------- src/scripts/pakfire-build-service | 3 + 5 files changed, 114 insertions(+), 157 deletions(-) delete mode 100644 src/manager/repositories.py diff --git a/Makefile.am b/Makefile.am index 4ba4a80..1364572 100644 --- a/Makefile.am +++ b/Makefile.am @@ -124,8 +124,7 @@ hubdir = $(buildservicedir)/hub manager_PYTHON = \ src/manager/__init__.py \ src/manager/base.py \ - src/manager/builds.py \ - src/manager/repositories.py + src/manager/builds.py managerdir = $(buildservicedir)/manager diff --git a/src/buildservice/repository.py b/src/buildservice/repository.py index a1205da..e842d69 100644 --- a/src/buildservice/repository.py +++ b/src/buildservice/repository.py @@ -1,10 +1,15 @@ #!/usr/bin/python +import logging import os.path +log = logging.getLogger("repositories") +log.propagate = 1 + from . import base from . import logs +from .constants import * from .decorators import * class Repositories(base.Object): @@ -34,18 +39,6 @@ class Repositories(base.Object): return self._get_repository("SELECT * FROM repositories \ WHERE id = %s", repo_id) - def get_needs_update(self, limit=None): - query = "SELECT id FROM repositories WHERE needs_update = 'Y'" - query += " ORDER BY last_update ASC" - - # Append limit if any - if limit: - query += " LIMIT %d" % limit - - repos = self.db.query(query) - - return [Repository(self.pakfire, r.id) for r in repos] - def get_history(self, limit=None, offset=None, build=None, repo=None, user=None): query = "SELECT * FROM repositories_history" args = [] @@ -67,6 +60,19 @@ class Repositories(base.Object): return entries + def remaster(self): + """ + Remasters all repositories + """ + for repo in self: + # Skip all repositories that don't need an update + if not repo.needs_update: + log.debug("Repository %s does not need an update" % repo) + continue + + with self.db.transaction(): + repo.remaster() + class Repository(base.DataObject): table = "repositories" @@ -201,7 +207,7 @@ class Repository(base.DataObject): @property def arches(self): - return self.distro.arches + return self.distro.arches + ["src"] @property def mirrored(self): @@ -326,24 +332,20 @@ class Repository(base.DataObject): def packages(self): return self.get_packages() - def get_unpushed_builds(self): - query = self.db.query("SELECT build_id FROM repositories_builds \ - WHERE repo_id = %s AND \ - time_added > (SELECT last_update FROM repositories WHERE id = %s)", - self.id, self.id) - - ret = [] - for row in query: - b = self.pakfire.builds.get_by_id(row.build_id) - ret.append(b) - - return ret + @property + def unpushed_builds(self): + return self.backend.builds._get_builds("SELECT builds.* FROM repositories \ + LEFT JOIN repositories_builds ON repositories.id = repositories_builds.repo_id \ + LEFT JOIN builds ON repositories_builds.build_id = builds.id \ + WHERE repositories.id = %s \ + AND repositories_builds.time_added >= repositories.last_update", self.id) def get_obsolete_builds(self): return self.pakfire.builds.get_obsolete(self) + @property def needs_update(self): - if self.get_unpushed_builds: + if self.unpushed_builds: return True return False @@ -352,6 +354,88 @@ class Repository(base.DataObject): self.db.execute("UPDATE repositories SET last_update = NOW() \ WHERE id = %s", self.id) + def remaster(self): + log.info("Going to update repository %s..." % self.name) + + # Update the timestamp when we started at last. + self.updated() + + for arch in self.arches: + changed = False + + # Get all package paths that are to be included in this repository. + paths = self.get_paths(arch) + + repo_path = os.path.join( + REPOS_DIR, + self.distro.identifier, + self.identifier, + arch + ) + + if not os.path.exists(repo_path): + os.makedirs(repo_path) + + source_files = [] + remove_files = [] + + for filename in os.listdir(repo_path): + path = os.path.join(repo_path, filename) + + if not os.path.isfile(path): + continue + + remove_files.append(path) + + for path in paths: + filename = os.path.basename(path) + + source_file = os.path.join(PACKAGES_DIR, path) + target_file = os.path.join(repo_path, filename) + + # Do not add duplicate files twice. + if source_file in source_files: + continue + + source_files.append(source_file) + + try: + remove_files.remove(target_file) + except ValueError: + changed = True + + if remove_files: + changed = True + + # If nothing in the repository data has changed, there + # is nothing to do. + if changed: + log.info("The repository has updates...") + else: + log.info("Nothing to update.") + continue + + # Find the key to sign the package. + key_id = None + if repo.key: + key_id = self.key.fingerprint + + # Create package index. + p = pakfire.PakfireServer(arch=arch) + + p.repo_create(repo_path, source_files, + name="%s - %s.%s" % (self.distro.name, self.name, arch), + key_id=key_id) + + # Remove files afterwards. + for file in remove_files: + file = os.path.join(repo_path, file) + + try: + os.remove(file) + except OSError: + log.warning("Could not remove %s." % file) + def get_history(self, **kwargs): kwargs.update({ "repo" : self, diff --git a/src/manager/__init__.py b/src/manager/__init__.py index 27d0046..fe248fe 100644 --- a/src/manager/__init__.py +++ b/src/manager/__init__.py @@ -2,4 +2,3 @@ from .builds import BuildsFailedRestartEvent, CheckBuildDependenciesEvent from .builds import CreateTestBuildsEvent, DistEvent -from .repositories import RepositoriesUpdateEvent diff --git a/src/manager/repositories.py b/src/manager/repositories.py deleted file mode 100644 index 0d8c07c..0000000 --- a/src/manager/repositories.py +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/python - -import logging -import os -import pakfire - -from . import base - -from ..constants import * - -class RepositoriesUpdateEvent(base.Event): - priority = 6 - - @property - def interval(self): - return self.pakfire.settings.get_int("repository_update_interval", 600) - - def run(self): - for distro in self.pakfire.distros.get_all(): - for repo in distro.repositories: - # Skip repostories that do not need an update at all. - if not repo.needs_update(): - logging.info("Repository %s - %s is already up to date." % (distro.name, repo.name)) - continue - - e = RepositoryUpdateEvent(self.pakfire, repo.id) - self.scheduler.add_event(e) - - -class RepositoryUpdateEvent(base.Event): - # This is an important task, but it may take a while to process. - priority = 5 - - def run(self, repo_id): - # Run this in a new process. - self.run_subprocess_background(self.update_repo, repo_id) - - @staticmethod - def update_repo(_pakfire, repo_id): - repo = _pakfire.repos.get_by_id(repo_id) - assert repo - - logging.info("Going to update repository %s..." % repo.name) - - # Update the timestamp when we started at last. - repo.updated() - - # Find out for which arches this repository is created. - arches = repo.arches - - # Add the source repository. - arches.append("src") - - for arch in arches: - changed = False - - # Get all package paths that are to be included in this repository. - paths = repo.get_paths(arch) - - repo_path = os.path.join( - REPOS_DIR, - repo.distro.identifier, - repo.identifier, - arch.name - ) - - if not os.path.exists(repo_path): - os.makedirs(repo_path) - - source_files = [] - remove_files = [] - - for filename in os.listdir(repo_path): - path = os.path.join(repo_path, filename) - - if not os.path.isfile(path): - continue - - remove_files.append(path) - - for path in paths: - filename = os.path.basename(path) - - source_file = os.path.join(PACKAGES_DIR, path) - target_file = os.path.join(repo_path, filename) - - # Do not add duplicate files twice. - if source_file in source_files: - continue - - source_files.append(source_file) - - try: - remove_files.remove(target_file) - except ValueError: - changed = True - - if remove_files: - changed = True - - # If nothing in the repository data has changed, there - # is nothing to do. - if changed: - logging.info("The repository has updates...") - else: - logging.info("Nothing to update.") - continue - - # Find the key to sign the package. - key_id = None - if repo.key: - key_id = repo.key.fingerprint - - # Create package index. - p = pakfire.PakfireServer(arch=arch.name) - - p.repo_create(repo_path, source_files, - name="%s - %s.%s" % (repo.distro.name, repo.name, arch.name), - key_id=key_id) - - # Remove files afterwards. - for file in remove_files: - file = os.path.join(repo_path, file) - - try: - os.remove(file) - except OSError: - logging.warning("Could not remove %s." % file) diff --git a/src/scripts/pakfire-build-service b/src/scripts/pakfire-build-service index 17883b2..f471a3a 100644 --- a/src/scripts/pakfire-build-service +++ b/src/scripts/pakfire-build-service @@ -35,6 +35,9 @@ class Cli(object): # Pull sources "pull-sources" : self.backend.sources.pull, + # Remaster Repositories + "remaster-repositories" : self.backend.repositories.remaster, + # Send bug updates to Bugzilla "send-bug-updates" : self.backend.bugzilla.send_all, } -- 2.39.2