Move integration tests into python

Instead of vendor locking ourselves to Github Actions, let's move
the integration tests into python so we can run them locally and
on other CI systems.

We opt to use unittest style test cases so that we can have a
configurable base class that can be used for various integration
tests. Unfortunately, I haven't found a nice and type safe way to
make pytest fixtures configurable so we opt for unittest instead.

Note that while we use the subTest() feature of unittest, pytest
still considers test_boot() a single test because it doesn't support
this particular feature of unittest. Ideally we switch our test runner
to something else in the future which does support the subTest() feature.

We always run steps that can run unprivileged without privileges even
if we're running as root so that tests can be run locally with root
privileges without ending up with a bunch of files owned by root
afterwards.

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 6edc3e8457a627a817bef6174f421e56c5d3093c..8e37244deb5a5b5ab553ddb1cc5e5c7514873364 100644 (file)
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -20,9 +20,9 @@ jobs:
     - name: Install
       run: |
         sudo apt-get update
-        sudo apt-get install pandoc
+        sudo apt-get install pandoc python3-pytest
         python3 -m pip install --upgrade setuptools wheel pip
-        python3 -m pip install pytest mypy types-cryptography isort pyflakes
+        python3 -m pip install mypy isort pyflakes
         npm install -g pyright
 
     - name: Check that imports are sorted
@@ -41,7 +41,7 @@ jobs:
       run: pyright mkosi/ tests/
 
     - name: Unit Tests
-      run: python3 -m pytest -sv tests
+      run: python3 -m pytest -sv tests/
 
     - name: Test execution from current working directory
       run: python3 -m mkosi -h
@@ -128,66 +128,27 @@ jobs:
           - rocky
           - alma
           - opensuse
-        format:
-          - directory
-          - tar
-          - cpio
-          - disk
-          - uki
-          - esp
-        exclude:
-          - distro: rhel-ubi
-            format: uki
-          - distro: rhel-ubi
-            format: esp
 
     steps:
     - uses: actions/checkout@v3
     - uses: ./
 
-    # Make sure the latest changes from the pull request are used.
     - name: Install
-      run: sudo ln -svf $PWD/bin/mkosi /usr/bin/mkosi
+      run: |
+        sudo apt-get update
+        sudo apt-get install python3-pytest
+        # Make sure the latest changes from the pull request are used.
+        sudo ln -svf $PWD/bin/mkosi /usr/bin/mkosi
       working-directory: ./
 
-    - name: Configure ${{ matrix.distro }}/${{ matrix.format }}
+    - name: Configure
       run: |
-        tee mkosi.conf <<- EOF
-        [Distribution]
-        Distribution=${{ matrix.distro }}
-
-        [Output]
-        Format=${{ matrix.format }}
-
-        [Content]
-        KernelCommandLine=console=ttyS0
-                          systemd.unit=mkosi-check-and-shutdown.service
-                          systemd.log_target=console
-                          systemd.default_standard_output=journal+console
-
+        tee mkosi.local.conf <<- EOF
         [Host]
-        QemuVsock=yes
-        QemuMem=4G
+        QemuKvm=no
         EOF
 
-    - name: Build ${{ matrix.distro }}/${{ matrix.format }}
-      run: mkosi --debug build
-
-    # systemd-resolved is enabled by default in Arch/Debian/Ubuntu (systemd default preset) but fails to
-    # start in a systemd-nspawn container with --private-users so we mask it out here to avoid CI failures.
-    # FIXME: Remove when Arch/Debian/Ubuntu ship systemd v253
-    - name: Mask systemd-resolved
-      if: matrix.format == 'directory'
-      run: sudo systemctl --root mkosi.output/image mask systemd-resolved
-
-    - name: Boot ${{ matrix.distro }}/${{ matrix.format }} systemd-nspawn
-      if: matrix.format == 'disk' || matrix.format == 'directory'
-      run: sudo mkosi --debug boot
-
-    - name: Boot ${{ matrix.distro }}/${{ matrix.format }} QEMU
-      if: matrix.distro != 'rhel-ubi' && matrix.format != 'directory' && matrix.format != 'tar'
-      run: timeout -k 30 10m mkosi --debug qemu
-
-    - name: Boot ${{ matrix.distro }}/${{ matrix.format }} BIOS
-      if: matrix.distro != 'rhel-ubi' && matrix.format == 'disk'
-      run: timeout -k 30 10m mkosi --debug --qemu-firmware bios qemu
+    - name: Run integration tests
+      run: sudo timeout -k 30 10m python3 -m pytest --tb=no -sv -m integration tests/
+      env:
+        MKOSI_TEST_DISTRIBUTION: ${{ matrix.distro }}

diff --git a/pytest.ini b/pytest.ini
new file mode 100644 (file)
index 0000000..c24ad46
--- /dev/null
+++ b/pytest.ini
@@ -0,0 +1,4 @@
+[pytest]
+markers =
+    integration: mark a test as an integration test.
+addopts = -m "not integration"

diff --git a/tests/__init__.py b/tests/__init__.py
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..8281c09a8be9f39c6b8bae76562442b6e8e028ca 100644 (file)
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -0,0 +1,74 @@
+# SPDX-License-Identifier: LGPL-2.1+
+
+import os
+import tempfile
+from collections.abc import Sequence
+from typing import Optional
+from types import TracebackType
+
+from mkosi.distributions import Distribution, detect_distribution
+from mkosi.log import die
+from mkosi.run import run
+from mkosi.types import CompletedProcess
+from mkosi.util import INVOKING_USER
+
+
+class Image:
+    def __init__(self, options: Sequence[str] = []) -> None:
+        self.options = options
+
+        if d := os.getenv("MKOSI_TEST_DISTRIBUTION"):
+            self.distribution = Distribution[d]
+        elif detected := detect_distribution()[0]:
+            self.distribution = detected
+        else:
+            die("Cannot detect host distribution, please set $MKOSI_TEST_DISTRIBUTION to be able to run the tests")
+
+        if r := os.getenv("MKOSI_TEST_RELEASE"):
+            self.release = r
+        else:
+            self.release = self.distribution.default_release()
+
+    def __enter__(self) -> "Image":
+        self.output_dir = tempfile.TemporaryDirectory(dir="/var/tmp")
+        os.chown(self.output_dir.name, INVOKING_USER.uid, INVOKING_USER.gid)
+
+        return self
+
+    def __exit__(
+        self,
+        type: Optional[type[BaseException]],
+        value: Optional[BaseException],
+        traceback: Optional[TracebackType],
+    ) -> None:
+        self.mkosi("clean", user=INVOKING_USER.uid, group=INVOKING_USER.gid)
+
+    def mkosi(
+        self,
+        verb: str,
+        options: Sequence[str] = (),
+        args: Sequence[str] = (),
+        user: Optional[int] = None,
+        group: Optional[int] = None,
+    ) -> CompletedProcess:
+        return run([
+            "python3", "-m", "mkosi",
+            *self.options,
+            *options,
+            "--output-dir", self.output_dir.name,
+            "--cache-dir", "mkosi.cache",
+            "--debug",
+            "--distribution", str(self.distribution),
+            "--release", self.release,
+            verb,
+            *args,
+        ], user=user, group=group)
+
+    def build(self, options: Sequence[str] = (), args: Sequence[str] = ()) -> CompletedProcess:
+        return self.mkosi("build", [*options, "--force"], args, user=INVOKING_USER.uid, group=INVOKING_USER.gid)
+
+    def boot(self, options: Sequence[str] = (), args: Sequence[str] = ()) -> CompletedProcess:
+        return self.mkosi("boot", options, args)
+
+    def qemu(self, options: Sequence[str] = (), args: Sequence[str] = ()) -> CompletedProcess:
+        return self.mkosi("qemu", options, args, user=INVOKING_USER.uid, group=INVOKING_USER.gid)

diff --git a/tests/test_boot.py b/tests/test_boot.py
new file mode 100644 (file)
index 0000000..0487dc7
--- /dev/null
+++ b/tests/test_boot.py
@@ -0,0 +1,53 @@
+# SPDX-License-Identifier: LGPL-2.1+
+
+import os
+import pytest
+
+from mkosi.config import OutputFormat
+from mkosi.distributions import Distribution
+
+from . import Image
+
+
+@pytest.mark.integration
+@pytest.mark.parametrize("format", [f for f in OutputFormat if f != OutputFormat.none])
+def test_boot(format: OutputFormat) -> None:
+    with Image(
+        options=[
+            "--kernel-command-line=console=ttyS0",
+            "--kernel-command-line=systemd.unit=mkosi-check-and-shutdown.service",
+            "--kernel-command-line=systemd.log_target=console",
+            "--kernel-command-line=systemd.default_standard_output=journal+console",
+            "--qemu-vsock=yes",
+            "--qemu-mem=4G",
+            "--incremental",
+            "--ephemeral",
+        ],
+    ) as image:
+        if image.distribution == Distribution.rhel_ubi and format in (OutputFormat.esp, OutputFormat.uki):
+            pytest.skip("Cannot build RHEL-UBI images with format 'esp' or 'uki'")
+
+        options = ["--format", str(format)]
+
+        image.build(options=options)
+
+        if format in (OutputFormat.disk, OutputFormat.directory) and os.getuid() == 0:
+            # systemd-resolved is enabled by default in Arch/Debian/Ubuntu (systemd default preset) but fails
+            # to start in a systemd-nspawn container with --private-users so we mask it out here to avoid CI
+            # failures.
+            # FIXME: Remove when Arch/Debian/Ubuntu ship systemd v253
+            args = ["systemd.mask=systemd-resolved.service"] if format == OutputFormat.directory else []
+            image.boot(options=options, args=args)
+
+        if image.distribution == Distribution.rhel_ubi:
+            return
+
+        if format in (OutputFormat.directory, OutputFormat.tar):
+            return
+
+        image.qemu(options=options)
+
+        if format != OutputFormat.disk:
+            return
+
+        image.qemu(options=options + ["--qemu-firmware=bios"])