From: Michael Tremer Date: Tue, 17 Oct 2017 17:46:27 +0000 (+0100) Subject: Use with context to deal with git repositories X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=29e226fcf0a2c5912f9accf3d68c68894b6e39b2;p=people%2Fjschlag%2Fpbs.git Use with context to deal with git repositories Signed-off-by: Michael Tremer --- diff --git a/src/buildservice/git.py b/src/buildservice/git.py index bfbf28a..c47c462 100644 --- a/src/buildservice/git.py +++ b/src/buildservice/git.py @@ -6,30 +6,26 @@ import os import subprocess from . import base +from .decorators import * class Repo(base.Object): - def __init__(self, pakfire, source, mode="normal"): - base.Object.__init__(self, pakfire) - + def init(self, source, mode="normal"): assert mode in ("normal", "bare", "mirror") - # Get the source object. self.source = source self.mode = mode - @property - def path(self): - return os.path.join("/var/cache/pakfire/git-repos", self.source.identifier, self.mode) - - def git(self, cmd, path=None): - if not path: - path = self.path + def __enter__(self): + return RepoContext(self.backend, self) - cmd = "cd %s && git %s" % (path, cmd) + def __exit__(self, type, value, traceback): + pass - logging.debug("Running command: %s" % cmd) + @lazy_property + def path(self): + path = os.path.join("~/.pakfire/cache/git-repos", self.source.identifier, self.mode) - return subprocess.check_output(["/bin/sh", "-c", cmd]) + return os.path.expanduser(path) @property def cloned(self): @@ -38,50 +34,75 @@ class Repo(base.Object): """ return os.path.exists(self.path) + +class RepoContext(base.Object): + def init(self, repo): + self.repo = repo + + # Clone repository if not cloned, yet + if not self.repo.cloned: + self.clone() + + self._lock() + + def __del__(self): + self._release() + + def _lock(self): + pass # XXX needs to be implemented + + def _release(self): + pass + + def git(self, cmd, path=None): + if not path: + path = self.repo.path + + cmd = "cd %s && git %s" % (path, cmd) + + logging.debug("Running command: %s" % cmd) + + return subprocess.check_output(["/bin/sh", "-c", cmd]) + def clone(self): - if self.cloned: + if self.repo.cloned: return - path = os.path.dirname(self.path) - repo = os.path.basename(self.path) + path, repo = os.path.dirname(self.repo.path), os.path.basename(self.repo.path) # Create the repository home directory if not exists. if not os.path.exists(path): os.makedirs(path) command = ["clone"] - if self.mode == "bare": + if self.repo.mode == "bare": command.append("--bare") - elif self.mode == "mirror": + elif self.repo.mode == "mirror": command.append("--mirror") - command.append(self.source.url) + command.append(self.repo.source.url) command.append(repo) # Clone the repository. try: self.git(" ".join(command), path=path) except Exception: - shutil.rmtree(self.path) + shutil.rmtree(self.repo.path) raise def fetch(self): - # Make sure, the repository was already cloned. - if not self.cloned: - self.clone() - self.git("fetch") def rev_list(self, revision=None): if not revision: - if self.source.head_revision: - revision = self.source.head_revision.revision + if self.repo.source.head_revision: + revision = self.repo.source.head_revision.revision else: - revision = self.source.start_revision + revision = self.repo.source.start_revision - command = "rev-list %s..%s" % (revision, self.source.branch) + command = "rev-list %s..%s" % (revision, self.repo.source.branch) - # Get all merge commits. + # Get all merge commits merges = self.git("%s --merges" % command).splitlines() revisions = [] @@ -112,8 +133,8 @@ class Repo(base.Object): rev_date = datetime.datetime.utcfromtimestamp(float(rev_date)) # Create a new commit object in the database - return self.source.create_commit(revision, rev_author, rev_committer, - rev_subject, rev_body, rev_date) + return self.repo.source.create_commit(revision, + rev_author, rev_committer, rev_subject, rev_body, rev_date) def checkout(self, revision, update=False): for update in (0, 1): @@ -132,9 +153,9 @@ class Repo(base.Object): def changed_files(self, revision): files = self.git("diff --name-only %s^ %s" % (revision, revision)) - return [os.path.join(self.path, f) for f in files.splitlines()] + return [os.path.join(self.repo.path, f) for f in files.splitlines()] def get_all_files(self): files = self.git("ls-files") - return [os.path.join(self.path, f) for f in files.splitlines()] + return [os.path.join(self.repo.path, f) for f in files.splitlines()] \ No newline at end of file diff --git a/src/buildservice/sources.py b/src/buildservice/sources.py index fdadcff..d05883f 100644 --- a/src/buildservice/sources.py +++ b/src/buildservice/sources.py @@ -10,10 +10,10 @@ import subprocess import tempfile from . import base -from . import builds from . import database from . import git +from .constants import * from .decorators import * class Sources(base.Object): @@ -82,110 +82,83 @@ class Sources(base.Object): def pull(self): for source in self: - repo = git.Repo(self.backend, source, mode="mirror") - - # If the repository is not yet cloned, we need to make a local - # clone to work with. - if not repo.cloned: - repo.clone() - - # Otherwise we just fetch updates. - else: + with git.Repo(self.backend, source, mode="mirror") as repo: + # Fetch the latest updates repo.fetch() - # Import all new revisions. - repo.import_revisions() + # Import all new revisions + repo.import_revisions() def dist(self): - self._init_repos() - for commit in self.get_pending_commits(): commit.state = "running" logging.debug("Processing commit %s: %s" % (commit.revision, commit.subject)) - # Get the repository of this commit. - repo = git.Repo(self.pakfire, commit.source) - - # Make sure, it is checked out. - if not repo.cloned: - repo.clone() - - # Navigate to the right revision. - repo.checkout(commit.revision) - - # Get all changed makefiles. - deleted_files = [] - updated_files = [] - - for file in repo.changed_files(commit.revision): - # Don't care about files that are not a makefile. - if not file.endswith(".%s" % MAKEFILE_EXTENSION): - continue - - if os.path.exists(file): - updated_files.append(file) - else: - deleted_files.append(file) - - if updated_files: - # Create a temporary directory where to put all the files - # that are generated here. - pkg_dir = tempfile.mkdtemp() - - try: - config = pakfire.config.Config(["general.conf",]) - config.parse(source.distro.get_config()) - - p = pakfire.PakfireServer(config=config) - - pkgs = [] - for file in updated_files: - try: - pkg_file = p.dist(file, pkg_dir) - pkgs.append(pkg_file) - except: - raise - - # Import all packages in one swoop. - for pkg in pkgs: - # Import the package file and create a build out of it. - builds.import_from_package(_pakfire, pkg, - distro=source.distro, commit=commit, type="release") - - except: - if commit: - commit.state = "failed" - - raise - - finally: - if os.path.exists(pkg_dir): - shutil.rmtree(pkg_dir) - - for file in deleted_files: - # Determine the name of the package. - name = os.path.basename(file) - name = name[:len(MAKEFILE_EXTENSION) + 1] - - source.distro.delete_package(name) - - if commit: - commit.state = "finished" - - def _init_repos(self): - """ - Initialize all repositories. - """ - for source in self.get_all(): - # Skip those which already have a revision. - if source.revision: - continue - - # Initialize the repository or and clone it if necessary. - repo = git.Repo(self.pakfire, source) - if not repo.cloned: - repo.clone() + # Get the repository of this commit + with git.Repo(self.pakfire, commit.source) as repo: + # Navigate to the right revision. + repo.checkout(commit.revision) + + # Get all changed makefiles. + deleted_files = [] + updated_files = [] + + for file in repo.changed_files(commit.revision): + # Don't care about files that are not a makefile. + if not file.endswith(".%s" % MAKEFILE_EXTENSION): + continue + + if os.path.exists(file): + updated_files.append(file) + else: + deleted_files.append(file) + + if updated_files: + # Create a temporary directory where to put all the files + # that are generated here. + pkg_dir = tempfile.mkdtemp() + + try: + config = pakfire.config.Config(["general.conf",]) + config.parse(commit.source.distro.get_config()) + + p = pakfire.PakfireServer(config=config) + + pkgs = [] + for file in updated_files: + try: + pkg_file = p.dist(file, pkg_dir) + pkgs.append(pkg_file) + except: + raise + + # Import all packages in one swoop. + for pkg in pkgs: + # Import the package file and create a build out of it. + from . import builds + builds.import_from_package(_pakfire, pkg, + distro=commit.source.distro, commit=commit, type="release") + + except: + if commit: + commit.state = "failed" + + raise + + finally: + if os.path.exists(pkg_dir): + shutil.rmtree(pkg_dir) + + for file in deleted_files: + # Determine the name of the package. + name = os.path.basename(file) + name = name[:len(MAKEFILE_EXTENSION) + 1] + + commit.source.distro.delete_package(name) + + if commit: + commit.state = "finished" class Commit(base.DataObject):