From: Michael Tremer Date: Mon, 29 May 2023 13:05:52 +0000 (+0000) Subject: monitoring: Implement creating builds X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=1648678e16ff901e38e63ac2b4b6c259b861040f;p=pbs.git monitoring: Implement creating builds Signed-off-by: Michael Tremer --- diff --git a/src/buildservice/distribution.py b/src/buildservice/distribution.py index e32aba9c..e98c7c71 100644 --- a/src/buildservice/distribution.py +++ b/src/buildservice/distribution.py @@ -211,6 +211,15 @@ class Distribution(base.DataObject): def tag(self): return "%s%s" % (self.distro_id, self.version_id) + # Pakfire + + def pakfire(self, **kwargs): + """ + Returns a Pakfire configuration for this distribution + without any repositories (useful for dist) + """ + return self.backend.pakfire(distro=self, **kwargs) + # Custom Configuration def get_custom_config(self): diff --git a/src/buildservice/releasemonitoring.py b/src/buildservice/releasemonitoring.py index 05e9fedc..0bba4471 100644 --- a/src/buildservice/releasemonitoring.py +++ b/src/buildservice/releasemonitoring.py @@ -19,8 +19,13 @@ ############################################################################### import asyncio +import difflib import json import logging +import os +import re +import shutil +import tempfile import urllib.parse from . import base @@ -280,6 +285,8 @@ class Monitoring(base.DataObject): # Check async def check(self): + release = None + # Wait until we are allowed to send an API request async with ratelimiter: log.info("Checking for new releases for %s" % self) @@ -293,7 +300,7 @@ class Monitoring(base.DataObject): try: if self.follow == "mainline": - await self._follow_mainline(versions) + release = await self._follow_mainline(versions) else: raise ValueError("Cannot handle follow: %s" % self.follow) @@ -301,6 +308,10 @@ class Monitoring(base.DataObject): except ReleaseExistsError as e: log.debug("Release %s already exists" % e) + # Launch the build + if release and release.build: + await self.backend.builds.launch([release.build]) + async def _fetch_versions(self): """ Fetches all versions for this project @@ -408,7 +419,10 @@ class Monitoring(base.DataObject): # Create a build if self.data.create_builds: - await release._create_build() + await release._create_build( + build=self.latest_build, + owner=self.backend.users.pakfire, + ) # Return the release return release @@ -468,6 +482,10 @@ class Release(base.DataObject): Deletes this release """ async with asyncio.TaskGroup() as tasks: + # Delete the repository + if self.repo: + await self.repo.delete() + # Delete the build if self.build: tasks.create_task(self.build.delete(user=user)) @@ -546,6 +564,20 @@ class Release(base.DataObject): if self.bug_id: return await self.backend.bugzilla.get_bug(self.bug_id) + # Repo + + def get_repo(self): + if self.data.repo_id: + return self.backend.repos.get_by_id(self.data.repo_id) + + def set_repo(self, repo): + if self.repo: + raise AttributeError("Cannot reset repo") + + self._set_attribute("repo_id", repo) + + repo = lazy_property(get_repo, set_repo) + # Build def get_build(self): @@ -560,17 +592,131 @@ class Release(base.DataObject): build = lazy_property(get_build, set_build) - async def _create_build(self): + async def _create_build(self, build, owner): """ Creates a build """ if self.build: raise RuntimeError("Build already exists") - # XXX since we cannot yet update any builds, we will simply clone the latest one - # to test the tooling - #print(self.monitoring.latest_build) + log.info("Creating build for %s from %s" % (self, build)) + + try: + # Create a new repository + repo = await self.backend.repos.create( + self.monitoring.distro, "Test Build for %s" % self, owner=owner) + + # Create a new temporary space for the + with tempfile.TemporaryDirectory() as target: + # Create a new source package + file = await asyncio.to_thread( + self._update_source_package, build.pkg, target) + + # Upload the file + upload = await self.backend.uploads.create_from_local(file) + + try: + # Create a package + package = await self.backend.packages.create(upload) + + # Create the build + build = await self.backend.builds.create(repo, package, owner=owner) + + finally: + await upload.delete() + + # If anything went wrong, then remove the repository + except Exception as e: + await repo.delete() + + raise e + + else: + # Store the objects + self.build = build + self.repo = repo + + def _update_source_package(self, package, target): + """ + Takes a package and recreates it with this release + """ + if not package.is_source(): + raise RuntimeError("%s is not a source package" % package) + + # Create temporary directory to extract the package to + with tempfile.TemporaryDirectory() as tmp: + # Path to downloaded files + files = os.path.join(tmp, "files") + + # Path to the makefile + makefile = os.path.join(tmp, "%s.nm" % package.name) + + # Create a Pakfire instance from this distribution + with self.monitoring.distro.pakfire() as p: + # Open the archive + archive = p.open(package.path) + + # Extract the archive into the temporary space + archive.extract(path=tmp) + + # XXX directories are being created with the wrong permissions + os.system("chmod a+x -R %s" % tmp) + + # Remove any downloaded files + shutil.rmtree(files) + + # Update the makefile + diff = self._update_makefile(makefile) + + # Log the diff + log.info("Generated diff:\n%s" % diff) + + # Store the diff + self._set_attribute("diff", diff) + + # Generate a new source package + return p.dist(makefile, target) + + def _update_makefile(self, path): + """ + Reads the makefile in path and updates it with the newer version + returning a diff between the two. + """ + filename = os.path.basename(path) + + # Read the makefile + with open(path, "r") as f: + orig = f.readlines() + + # Replace the version & release + updated = self._update_makefile_version(orig) + + # Write the new file + with open(path, "w") as f: + f.writelines(updated) + + # Generate a diff + return "".join( + difflib.unified_diff(orig, updated, fromfile=filename, tofile=filename), + ) + + def _update_makefile_version(self, lines, release=1): + result = [] + + # Walk through the file line by line and replace everything that + # starts with version or release. + for line in lines: + if line and not line.startswith("#"): + # Replace version + m = re.match(r"^(version\s*=)\s*(.*)$", line) + if m: + line = "%s %s\n" % (m.group(1), self.version) + + # Replace release + m = re.match(r"^(release\s*=)\s*(.*)$", line) + if m: + line = "%s %s\n" % (m.group(1), release) + + result.append(line) - # XXX does this need an owner? - #self.build = await self.monitoring.latest_build.create( - # repo=None, package=self.monitoring.latest_build.package) + return result diff --git a/src/buildservice/users.py b/src/buildservice/users.py index 847f363b..f2964f55 100644 --- a/src/buildservice/users.py +++ b/src/buildservice/users.py @@ -313,6 +313,19 @@ class Users(base.Object): return sorted(users) + # Pakfire + + @property + def pakfire(self): + """ + This is a c + """ + user = self.get_by_email("pakfire@ipfire.org") + if not user: + raise RuntimeError("Missing Pakfire user") + + return user + @property def top(self): """ diff --git a/src/database.sql b/src/database.sql index 54b66adb..431d4159 100644 --- a/src/database.sql +++ b/src/database.sql @@ -765,7 +765,9 @@ CREATE TABLE public.release_monitoring_releases ( version text NOT NULL, created_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, bug_id integer, - build_id integer + build_id integer, + diff text, + repo_id integer ); @@ -2219,6 +2221,14 @@ ALTER TABLE ONLY public.release_monitoring_releases ADD CONSTRAINT release_monitoring_releases_monitoring_id FOREIGN KEY (monitoring_id) REFERENCES public.release_monitorings(id); +-- +-- Name: release_monitoring_releases release_monitoring_releases_repo_id; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.release_monitoring_releases + ADD CONSTRAINT release_monitoring_releases_repo_id FOREIGN KEY (repo_id) REFERENCES public.repositories(id); + + -- -- Name: release_monitorings release_monitorings_created_by; Type: FK CONSTRAINT; Schema: public; Owner: - --