From 695cec6642a06730ef8ef434709fc13aee8c6910 Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Wed, 3 May 2023 09:22:07 +0000 Subject: [PATCH] builds: Automatically create test builds when a build finishes Signed-off-by: Michael Tremer --- src/buildservice/builds.py | 126 +++++++++++++++++++++++++++--- src/buildservice/jobs.py | 29 ++++++- src/database.sql | 11 ++- src/scripts/pakfire-build-service | 34 ++++++++ 4 files changed, 189 insertions(+), 11 deletions(-) diff --git a/src/buildservice/builds.py b/src/buildservice/builds.py index a61cd6d3..908ab1b0 100644 --- a/src/buildservice/builds.py +++ b/src/buildservice/builds.py @@ -1,6 +1,7 @@ #!/usr/bin/python import asyncio +import itertools import logging import os import re @@ -162,7 +163,43 @@ class Builds(base.Object): return list(builds) - async def create(self, repo, package, owner=None, group=None): + def get_by_package_uuids(self, uuids): + """ + Returns a list of builds that contain the given packages + """ + builds = self._get_builds(""" + SELECT + DISTINCT builds.* + FROM + builds + LEFT JOIN + packages source_packages ON builds.pkg_id = source_packages.id + LEFT JOIN + jobs ON builds.id = jobs.build_id + LEFT JOIN + jobs_packages ON jobs.id = jobs_packages.job_id + LEFT JOIN + packages ON jobs_packages.pkg_id = packages.id + WHERE + builds.deleted_at IS NULL + AND + jobs.deleted_at IS NULL + AND + source_packages.deleted_at IS NULL + AND + packages.deleted_at IS NULL + AND + ( + packages.uuid = ANY(%s::uuid[]) + OR + source_packages.uuid = ANY(%s::uuid[]) + ) + """, uuids, uuids, + ) + + return list(builds) + + async def create(self, repo, package, owner=None, group=None, test_for_build=None): """ Creates a new build based on the given distribution and package """ @@ -177,34 +214,38 @@ class Builds(base.Object): build_repo_id, pkg_id, owner_id, - build_group_id + build_group_id, + test_for_build_id ) VALUES ( - %s, %s, %s, %s + %s, %s, %s, %s, %s ) RETURNING *""", repo, package, owner, group, + test_for_build, ) # Populate cache if group: build.group = group + group.builds.append(build) # Create all jobs build._create_jobs() - # Deprecate previous builds - build._deprecate_others() + if not build.is_test(): + # Deprecate previous builds + build._deprecate_others() - # Add watchers - build._add_watchers() + # Add watchers + build._add_watchers() - # Add the build into its repository - await repo.add_build(build, user=owner) + # Add the build into its repository + await repo.add_build(build, user=owner) return build @@ -622,6 +663,10 @@ class Build(base.DataObject): else: self._send_email("builds/messages/failed.txt") + # Create any test builds + if not self.test: + await self.create_test_builds() + def has_finished(self): """ Returns True if this build has finished @@ -844,6 +889,69 @@ class Build(base.DataObject): return list(builds) + # Reverse Requires + + async def reverse_requires(self): + """ + Returns a list of builds that reverse-depend on this build + """ + log.debug("Calculating reverse requires for %s..." % self) + + # Collect the builds from all jobs + reverse_requires = await asyncio.gather( + *(job._reverse_requires() for job in self.jobs), + ) + + # Join all builds together + return itertools.chain(*reverse_requires) + + # Tests Builds + + def is_test(self): + if self.data.test_for_build_id: + return True + + return False + + @lazy_property + def test_for_build(self): + if self.data.test_for_build_id: + return self.backend.builds.get_by_id(self.data.test_for_build_id) + + async def create_test_builds(self): + """ + This creates test builds for this build + """ + builds = {} + tests = [] + + # Map all builds by their name + for build in await self.reverse_requires(): + try: + builds[build.pkg.name].append(build) + except KeyError: + builds[build.pkg.name] = [build] + + # Create a build group for all tests + group = self.backend.builds.groups.create() + + # Create a test build only for the latest version of each package + for name in builds: + build = max(builds[name]) + + # Launch test build + t = await self.backend.builds.create( + # Use the same build repository + repo=self.build_repo, + package=build.pkg, + owner=self.owner, + group=group, + test_for_build=self, + ) + tests.append(t) + + return tests + class Groups(base.Object): """ diff --git a/src/buildservice/jobs.py b/src/buildservice/jobs.py index fac47cfe..c48fcbac 100644 --- a/src/buildservice/jobs.py +++ b/src/buildservice/jobs.py @@ -104,7 +104,9 @@ class Jobs(base.Object): return # Perform dependency checks for all jobs - results = await asyncio.gather(*(job.depcheck() for job in jobs)) + results = await asyncio.gather( + *(job.depcheck() for job in jobs), + ) # Try to dispatch any jobs afterwards if any(results): @@ -848,3 +850,28 @@ class Job(base.DataObject): @property def depcheck_performed_at(self): return self.data.depcheck_performed_at + + # Reverse Requires + + async def _reverse_requires(self): + packages = [] + + with self.pakfire(include_source=True) as p: + for package in self.packages: + for pkg in p.whatprovides("%s = %s" % (package.name, package.evr)): + # XXX we should search straight away for the UUID here, + # but that doesn't seem to work right now + if not package.uuid == pkg.uuid: + continue + + # Find all packages that depend on the current package + for r in pkg.reverse_requires: + # Skip packages that are provided by ourselves + if r.name in (p.name for p in self.packages): + continue + + # Rebuild this package! + packages.append(r.uuid) + + # Return any builds that generated those packages + return self.backend.builds.get_by_package_uuids(packages) diff --git a/src/database.sql b/src/database.sql index af57b3b2..366226b0 100644 --- a/src/database.sql +++ b/src/database.sql @@ -215,7 +215,8 @@ CREATE TABLE public.builds ( build_group_id integer, deprecating_build_id integer, deprecated_at timestamp without time zone, - deprecated_by integer + deprecated_by integer, + test_for_build_id integer ); @@ -1867,6 +1868,14 @@ ALTER TABLE ONLY public.builds ADD CONSTRAINT builds_pkg_id FOREIGN KEY (pkg_id) REFERENCES public.packages(id); +-- +-- Name: builds builds_test_for_build_id; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.builds + ADD CONSTRAINT builds_test_for_build_id FOREIGN KEY (test_for_build_id) REFERENCES public.builds(id); + + -- -- Name: jobs jobs_aborted_by; Type: FK CONSTRAINT; Schema: public; Owner: - -- diff --git a/src/scripts/pakfire-build-service b/src/scripts/pakfire-build-service index e922fa60..d3b89ff5 100644 --- a/src/scripts/pakfire-build-service +++ b/src/scripts/pakfire-build-service @@ -18,6 +18,10 @@ class Cli(object): # Bugzilla "bugzilla:version" : self.backend.bugzilla.version, + # Builds + "builds:create-test-builds" : self._builds_create_test_builds, + "builds:reverse-requires" : self._builds_reverse_requires, + # Certificates "load-certificate" : self.backend.load_certificate, @@ -126,6 +130,36 @@ class Cli(object): for pkg in job: print(pkg) + async def _builds_reverse_requires(self, *uuids): + for uuid in uuids: + build = self.backend.builds.get_by_uuid(uuid) + if not build: + log.error("Could not find build %s" % uuid) + continue + + reverse_requires = await build.reverse_requires() + + # Print the build name + print(build) + + # Print all reverse requirements + for build in reverse_requires: + print(" Requires: %s (UUID %s)" % (build, build.uuid)) + + # Print an empty line + print() + + async def _builds_create_test_builds(self, *uuids): + for uuid in uuids: + build = self.backend.builds.get_by_uuid(uuid) + if not build: + log.error("Could not find build %s" % uuid) + continue + + with self.backend.db.transaction(): + await build.create_test_builds() + + async def main(): cli = Cli() await cli(*sys.argv) -- 2.47.2