From: Michael Tremer Date: Tue, 25 Oct 2022 09:59:50 +0000 (+0000) Subject: sources: Refactor importing commits and packages X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=663f3048060a54e6956e616661ffb397b2854fe8;p=pbs.git sources: Refactor importing commits and packages Signed-off-by: Michael Tremer --- diff --git a/src/buildservice/sources.py b/src/buildservice/sources.py index 89481069..8f77bef4 100644 --- a/src/buildservice/sources.py +++ b/src/buildservice/sources.py @@ -1,5 +1,8 @@ #!/usr/bin/python +import asyncio +import datetime +import fnmatch import logging import os import pakfire @@ -10,7 +13,6 @@ import subprocess import tempfile from . import base -from . import git from . import misc from .constants import * @@ -122,88 +124,6 @@ class Sources(base.Object): if commit: return Commit(self.pakfire, commit.id) - def pull(self): - for source in self: - with git.Repo(self.backend, source, mode="mirror") as repo: - # Fetch the latest updates - repo.fetch() - - # Import all new revisions - repo.import_revisions() - - def dist(self): - # Walk through all source repositories - for source in self: - # Get access to the git repo - with git.Repo(self.pakfire, source) as repo: - # Walk through all pending commits - for commit in source.pending_commits: - commit.state = "running" - - logging.debug("Processing commit %s: %s" % (commit.revision, commit.subject)) - - # Navigate to the right revision. - repo.checkout(commit.revision) - - # Get all changed makefiles. - deleted_files = [] - updated_files = [] - - for file in repo.changed_files(commit.revision): - # Don't care about files that are not a makefile. - if not file.endswith(".%s" % MAKEFILE_EXTENSION): - continue - - if os.path.exists(file): - updated_files.append(file) - else: - deleted_files.append(file) - - if updated_files: - # Create a temporary directory where to put all the files - # that are generated here. - pkg_dir = tempfile.mkdtemp() - - try: - config = pakfire.config.Config(["general.conf",]) - config.parse(source.distro.get_config()) - - p = pakfire.PakfireServer(config=config) - - pkgs = [] - for file in updated_files: - try: - pkg_file = p.dist(file, pkg_dir) - pkgs.append(pkg_file) - except: - raise - - # Import all packages in one swoop. - for pkg in pkgs: - with self.db.transaction(): - self.backend.builds.create_from_source_package(pkg, - source.distro, commit=commit, type="release") - - except: - if commit: - commit.state = "failed" - - raise - - finally: - if os.path.exists(pkg_dir): - shutil.rmtree(pkg_dir) - - for file in deleted_files: - # Determine the name of the package. - name = os.path.basename(file) - name = name[:len(MAKEFILE_EXTENSION) + 1] - - source.distro.delete_package(name) - - if commit: - commit.state = "finished" - class Source(base.DataObject): table = "sources" @@ -287,12 +207,48 @@ class Source(base.DataObject): # Commits - def create_commit(self, revision, author, committer, subject, body, date): - commit = self.backend.sources._get_commit("INSERT INTO source_commits(source_id, \ - revision, author, committer, subject, body, date) VALUES(%s, %s, %s, %s, %s, %s, %s) \ - RETURNING *", self.id, revision, author, committer, subject, body, date) + async def _import_commit(self, git, revision): + """ + Imports the commit with the given revision + """ + # Fetch the author's name and email address + author = await git.show_attribute(revision, r"%an <%ae>") + + # Fetch the committer's name and email address + committer = await git.show_attribute(revision, r"%cn <%ce>") - # Commit + # Subject + subject = await git.show_attribute(revision, r"%s") + + # Body + body = await git.show_attribute(revision, r"%b") + + # Date + date = datetime.datetime.fromisoformat( + await git.show_attribute(revision, r"%aI"), + ) + + # Insert into the database + commit = self.backend.sources._get_commit(""" + INSERT INTO + source_commits( + source_id, + revision, + author, + committer, + subject, + body, + date + ) + VALUES( + %s, %s, %s, %s, %s, %s, %s + ) + RETURNING + * + """, self.id, revision, author, committer, subject, body, date, + ) + + # Populate the cache commit.source = self return commit @@ -348,6 +304,98 @@ class Source(base.DataObject): else: await git.clone() + # Determine which commits there are to process + revisions = [ + await git.show_ref(self.branch), + ] + + # Process all revisions one after the other + for revision in revisions: + with self.db.transaction(): + await self._process_revision(git, revision) + + async def _process_revision(self, git, revision): + """ + Processes a certain revision + """ + # Create the commit metadata + commit = await self._import_commit(git, revision) + + # Find changed files + changed_files = await git.changed_files(revision, filter="*/*.nm") + + # Exit if there have not been any changes + if not changed_files: + log.debug("%s has no changes") + return + + deleted_packages = [] + dist_files = [] + + # Run through all changed files + for status, filename in changed_files: + # Collect the names of all deleted packages + if status == "D": + deleted_packages.append( + self._filename_to_package(filename), + ) + + # For any other changed makefiles, we collect their paths + else: + dist_files.append(filename) + + # Obsolete any deleted packages + if deleted_packages: + pass # XXX TODO + + # Create builds from all other changed files + if dist_files: + await self._dist_files(git, revision, dist_files) + + def _filename_to_package(self, filename): + """ + Maps a filename to a package name + """ + name = os.path.dirname(filename) + + # Check that the file part matches + if not filename.endswith("/%s.nm" % name): + raise ValueError("Invalid package name") + + return name + + async def _dist_files(self, git, revision, files): + """ + Runs "pakfire dist" on all given files at the given revision + """ + with tempfile.TemporaryDirectory() as path: + # Checkout the revision into a new temporary directory + await git.checkout(revision, path) + + uploads = [] + + # Launch a Pakfire instance with the repository configuration + # XXX vendor needs to be set from the repository metadata + try: + with self.backend.pakfire(distro=self.repo.distro) as p: + # Walk through all files one by one + for file in files: + # Run dist() + file = p.dist(file) + + # Upload the file + uploads.append( + await upload.create_from_file(file), + ) + + # Remove the source file + os.unlink(file) + + finally: + # Delete any uploads + await asyncio.gather( + *(upload.delete() for upload in uploads) + ) class Commit(base.DataObject): @@ -511,7 +559,7 @@ class Git(object): def __repr__(self): return "<%s %s>" % (self.__class__.__name__, self.path) - async def command(self, *args): + async def command(self, *args, **kwargs): """ Executes a Git command """ @@ -521,7 +569,7 @@ class Git(object): if self.is_cloned(): cwd = self.path - return await self.backend.command("git", *args, cwd=cwd) + return await self.backend.command("git", *args, cwd=cwd, **kwargs) def is_cloned(self): """ @@ -564,3 +612,71 @@ class Git(object): Fetches any changes """ await self.command("fetch", self.url, self.branch) + + async def show_attribute(self, revision, format): + return await self.command( + "show", + "--no-patch", + "--format=format:%s" % format, + revision, + return_output=True, + ) + + async def show_ref(self, branch): + """ + Resolves a ref into a hash + """ + return await self.command("show-ref", "--hash", branch, return_output=True) + + async def changed_files(self, revision, filter=None): + """ + Returns a list of files that has been changed + """ + changed_files = [] + + output = await self.command( + "show", + + # Don't show anything else + "--format=format:", + + # Show the status of each changed file + "--name-status", + + # Revision + revision, + + # Ask for the output to be returned + return_output=True, + ) + + for line in output.splitlines(): + m = re.match(r"^([A-Z])\s+(.*)$", line) + if not m: + raise ValueError("Could not parse line: %s" % line) + + # Extract values + status, filename = m.groups() + + # Filter out anything unwanted + if filter and not fnmatch.fnmatch(filename, filter): + continue + + # Append the file to the list + changed_files.append((status, filename)) + + return changed_files + + async def checkout(self, revision, path): + """ + Creates a working directory at the revision in path + """ + log.debug("Checking out %s into %s" % (revision, path)) + + await self.command( + # Set the destination path as work tree + "--work-tree", path, + + # Perform a checkout of the specified revision + "checkout", revision, ".", + )