ci: Add tools trees to integration test matrix

Let's make sure our tools trees actually work as intended by
introducing a second axis to our test matrix.

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 19b27f7ccd97cbacbca54ddc28ddeda88e9b6e81..baedf946106b018eb4c82981458c249f477cd2e8 100644 (file)
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -87,7 +87,7 @@ jobs:
     runs-on: ubuntu-22.04
     needs: unit-test
     concurrency:
-      group: ${{ github.workflow }}-${{ matrix.distro }}-${{ github.ref }}
+      group: ${{ github.workflow }}-${{ matrix.distro }}-${{ matrix.tools }}-${{ github.ref }}
       cancel-in-progress: true
     strategy:
       fail-fast: false
@@ -96,9 +96,58 @@ jobs:
           - arch
           - centos
           - debian
+          - fedora
+          - opensuse
           - ubuntu
+        tools:
+          - ""
+          - arch
+          - debian
           - fedora
           - opensuse
+          # TODO: Add Ubuntu and CentOS once they have systemd v254 or newer.
+        exclude:
+          # pacman and archlinux-keyring are not packaged in OpenSUSE.
+          - distro: arch
+            tools: opensuse
+          # apt, debian-keyring and ubuntu-keyring are not packaged in OpenSUSE.
+          - distro: debian
+            tools: opensuse
+          - distro: ubuntu
+            tools: opensuse
+          # Installing dnf in OpenSUSE images is broken until https://build.opensuse.org/request/show/1133852 shows up
+          # in the Tumbleweed repositories.
+          - distro: centos
+            tools: opensuse
+          - distro: fedora
+            tools: opensuse
+          - distro: opensuse
+            tools: opensuse
+          # Debian test_boot[cpio] is super flaky on these combinations until the next v255 stable release with
+          # https://github.com/systemd/systemd/pull/30511 is available in Debian unstable.
+          - distro: debian
+            tools: arch
+          - distro: debian
+            tools: fedora
+          - distro: debian
+            tools: centos
+          - distro: debian
+            tools: debian
+          # Fedora and CentOS don't package ubuntu-keyring.
+          - distro: ubuntu
+            tools: centos
+          - distro: ubuntu
+            tools: fedora
+          # rpm in Debian is currently missing
+          # https://github.com/rpm-software-management/rpm/commit/ea3187cfcf9cac87e5bc5e7db79b0338da9e355e
+          - distro: fedora
+            tools: debian
+          - distro: centos
+            tools: debian
+          # This combination results in rpm failing because of SIGPIPE.
+          # TODO: Try again once Arch gets a new rpm release.
+          - distro: centos
+            tools: arch
 
     steps:
     - uses: actions/checkout@v3
@@ -130,4 +179,12 @@ jobs:
         EOF
 
     - name: Run integration tests
-      run: sudo --preserve-env timeout -k 30 1h python3 -m pytest --tb=no -sv -m integration -D ${{ matrix.distro }} tests/
+      run: |
+        sudo timeout -k 30 1h python3 -m pytest \
+            --tb=no \
+            --capture=no \
+            --verbose \
+            -m integration \
+            --distribution ${{ matrix.distro }} \
+            $([[ -n "${{ matrix.tools }}" ]] && echo --tools-tree-distribution=${{ matrix.tools }}) \
+            tests/

diff --git a/tests/__init__.py b/tests/__init__.py
index c89f780d912846355e0415943be5a8fa4bdebc20..561d11ffe53eb46ca1160bd12e3e6483599ff609 100644 (file)
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -20,6 +20,7 @@ class Image:
     class Config(NamedTuple):
         distribution: Distribution
         release: str
+        tools_tree_distribution: Optional[Distribution]
 
     def __init__(self, config: Config, options: Sequence[PathString] = []) -> None:
         self.options = options
@@ -65,6 +66,12 @@ class Image:
             "python3", "-m", "mkosi",
             "--distribution", str(self.config.distribution),
             "--release", self.config.release,
+            *(["--tools-tree=default"] if self.config.tools_tree_distribution else []),
+            *(
+                ["--tools-tree-distribution", str(self.config.tools_tree_distribution)]
+                if self.config.tools_tree_distribution
+                else []
+            ),
             *self.options,
             *options,
             "--output-dir", self.output_dir.name,

diff --git a/tests/conftest.py b/tests/conftest.py
index eaec5be32111fe68df09a0810b7ba5b50d2bbcbc..f93787843ea875d92549edeacb8e4c04645d8e9e 100644 (file)
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -25,6 +25,14 @@ def pytest_addoption(parser: Any) -> None:
         metavar="RELEASE",
         help="Run the integration tests for the given release.",
     )
+    parser.addoption(
+        "-T",
+        "--tools-tree-distribution",
+        metavar="DISTRIBUTION",
+        help="Use the given tools tree distribution to build the integration test images",
+        type=Distribution,
+        choices=[Distribution(d) for d in Distribution.values()],
+    )
 
 
 @pytest.fixture(scope="session")
@@ -34,4 +42,5 @@ def config(request: Any) -> Image.Config:
     return Image.Config(
         distribution=distribution,
         release=release,
+        tools_tree_distribution=cast(Distribution, request.config.getoption("--tools-tree-distribution")),
     )