From b5b9963298900453446ca407d45cdc0085ef1649 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Fri, 1 Dec 2023 11:22:21 +0100 Subject: [PATCH] Move integration tests into python Instead of vendor locking ourselves to Github Actions, let's move the integration tests into python so we can run them locally and on other CI systems. We opt to use unittest style test cases so that we can have a configurable base class that can be used for various integration tests. Unfortunately, I haven't found a nice and type safe way to make pytest fixtures configurable so we opt for unittest instead. Note that while we use the subTest() feature of unittest, pytest still considers test_boot() a single test because it doesn't support this particular feature of unittest. Ideally we switch our test runner to something else in the future which does support the subTest() feature. We always run steps that can run unprivileged without privileges even if we're running as root so that tests can be run locally with root privileges without ending up with a bunch of files owned by root afterwards. --- .github/workflows/ci.yml | 69 ++++++++----------------------------- pytest.ini | 4 +++ tests/__init__.py | 74 ++++++++++++++++++++++++++++++++++++++++ tests/test_boot.py | 53 ++++++++++++++++++++++++++++ 4 files changed, 146 insertions(+), 54 deletions(-) create mode 100644 pytest.ini create mode 100644 tests/test_boot.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6edc3e845..8e37244de 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,9 +20,9 @@ jobs: - name: Install run: | sudo apt-get update - sudo apt-get install pandoc + sudo apt-get install pandoc python3-pytest python3 -m pip install --upgrade setuptools wheel pip - python3 -m pip install pytest mypy types-cryptography isort pyflakes + python3 -m pip install mypy isort pyflakes npm install -g pyright - name: Check that imports are sorted @@ -41,7 +41,7 @@ jobs: run: pyright mkosi/ tests/ - name: Unit Tests - run: python3 -m pytest -sv tests + run: python3 -m pytest -sv tests/ - name: Test execution from current working directory run: python3 -m mkosi -h @@ -128,66 +128,27 @@ jobs: - rocky - alma - opensuse - format: - - directory - - tar - - cpio - - disk - - uki - - esp - exclude: - - distro: rhel-ubi - format: uki - - distro: rhel-ubi - format: esp steps: - uses: actions/checkout@v3 - uses: ./ - # Make sure the latest changes from the pull request are used. - name: Install - run: sudo ln -svf $PWD/bin/mkosi /usr/bin/mkosi + run: | + sudo apt-get update + sudo apt-get install python3-pytest + # Make sure the latest changes from the pull request are used. + sudo ln -svf $PWD/bin/mkosi /usr/bin/mkosi working-directory: ./ - - name: Configure ${{ matrix.distro }}/${{ matrix.format }} + - name: Configure run: | - tee mkosi.conf <<- EOF - [Distribution] - Distribution=${{ matrix.distro }} - - [Output] - Format=${{ matrix.format }} - - [Content] - KernelCommandLine=console=ttyS0 - systemd.unit=mkosi-check-and-shutdown.service - systemd.log_target=console - systemd.default_standard_output=journal+console - + tee mkosi.local.conf <<- EOF [Host] - QemuVsock=yes - QemuMem=4G + QemuKvm=no EOF - - name: Build ${{ matrix.distro }}/${{ matrix.format }} - run: mkosi --debug build - - # systemd-resolved is enabled by default in Arch/Debian/Ubuntu (systemd default preset) but fails to - # start in a systemd-nspawn container with --private-users so we mask it out here to avoid CI failures. - # FIXME: Remove when Arch/Debian/Ubuntu ship systemd v253 - - name: Mask systemd-resolved - if: matrix.format == 'directory' - run: sudo systemctl --root mkosi.output/image mask systemd-resolved - - - name: Boot ${{ matrix.distro }}/${{ matrix.format }} systemd-nspawn - if: matrix.format == 'disk' || matrix.format == 'directory' - run: sudo mkosi --debug boot - - - name: Boot ${{ matrix.distro }}/${{ matrix.format }} QEMU - if: matrix.distro != 'rhel-ubi' && matrix.format != 'directory' && matrix.format != 'tar' - run: timeout -k 30 10m mkosi --debug qemu - - - name: Boot ${{ matrix.distro }}/${{ matrix.format }} BIOS - if: matrix.distro != 'rhel-ubi' && matrix.format == 'disk' - run: timeout -k 30 10m mkosi --debug --qemu-firmware bios qemu + - name: Run integration tests + run: sudo timeout -k 30 10m python3 -m pytest --tb=no -sv -m integration tests/ + env: + MKOSI_TEST_DISTRIBUTION: ${{ matrix.distro }} diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 000000000..c24ad4606 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,4 @@ +[pytest] +markers = + integration: mark a test as an integration test. +addopts = -m "not integration" diff --git a/tests/__init__.py b/tests/__init__.py index e69de29bb..8281c09a8 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -0,0 +1,74 @@ +# SPDX-License-Identifier: LGPL-2.1+ + +import os +import tempfile +from collections.abc import Sequence +from typing import Optional +from types import TracebackType + +from mkosi.distributions import Distribution, detect_distribution +from mkosi.log import die +from mkosi.run import run +from mkosi.types import CompletedProcess +from mkosi.util import INVOKING_USER + + +class Image: + def __init__(self, options: Sequence[str] = []) -> None: + self.options = options + + if d := os.getenv("MKOSI_TEST_DISTRIBUTION"): + self.distribution = Distribution[d] + elif detected := detect_distribution()[0]: + self.distribution = detected + else: + die("Cannot detect host distribution, please set $MKOSI_TEST_DISTRIBUTION to be able to run the tests") + + if r := os.getenv("MKOSI_TEST_RELEASE"): + self.release = r + else: + self.release = self.distribution.default_release() + + def __enter__(self) -> "Image": + self.output_dir = tempfile.TemporaryDirectory(dir="/var/tmp") + os.chown(self.output_dir.name, INVOKING_USER.uid, INVOKING_USER.gid) + + return self + + def __exit__( + self, + type: Optional[type[BaseException]], + value: Optional[BaseException], + traceback: Optional[TracebackType], + ) -> None: + self.mkosi("clean", user=INVOKING_USER.uid, group=INVOKING_USER.gid) + + def mkosi( + self, + verb: str, + options: Sequence[str] = (), + args: Sequence[str] = (), + user: Optional[int] = None, + group: Optional[int] = None, + ) -> CompletedProcess: + return run([ + "python3", "-m", "mkosi", + *self.options, + *options, + "--output-dir", self.output_dir.name, + "--cache-dir", "mkosi.cache", + "--debug", + "--distribution", str(self.distribution), + "--release", self.release, + verb, + *args, + ], user=user, group=group) + + def build(self, options: Sequence[str] = (), args: Sequence[str] = ()) -> CompletedProcess: + return self.mkosi("build", [*options, "--force"], args, user=INVOKING_USER.uid, group=INVOKING_USER.gid) + + def boot(self, options: Sequence[str] = (), args: Sequence[str] = ()) -> CompletedProcess: + return self.mkosi("boot", options, args) + + def qemu(self, options: Sequence[str] = (), args: Sequence[str] = ()) -> CompletedProcess: + return self.mkosi("qemu", options, args, user=INVOKING_USER.uid, group=INVOKING_USER.gid) diff --git a/tests/test_boot.py b/tests/test_boot.py new file mode 100644 index 000000000..0487dc77c --- /dev/null +++ b/tests/test_boot.py @@ -0,0 +1,53 @@ +# SPDX-License-Identifier: LGPL-2.1+ + +import os +import pytest + +from mkosi.config import OutputFormat +from mkosi.distributions import Distribution + +from . import Image + + +@pytest.mark.integration +@pytest.mark.parametrize("format", [f for f in OutputFormat if f != OutputFormat.none]) +def test_boot(format: OutputFormat) -> None: + with Image( + options=[ + "--kernel-command-line=console=ttyS0", + "--kernel-command-line=systemd.unit=mkosi-check-and-shutdown.service", + "--kernel-command-line=systemd.log_target=console", + "--kernel-command-line=systemd.default_standard_output=journal+console", + "--qemu-vsock=yes", + "--qemu-mem=4G", + "--incremental", + "--ephemeral", + ], + ) as image: + if image.distribution == Distribution.rhel_ubi and format in (OutputFormat.esp, OutputFormat.uki): + pytest.skip("Cannot build RHEL-UBI images with format 'esp' or 'uki'") + + options = ["--format", str(format)] + + image.build(options=options) + + if format in (OutputFormat.disk, OutputFormat.directory) and os.getuid() == 0: + # systemd-resolved is enabled by default in Arch/Debian/Ubuntu (systemd default preset) but fails + # to start in a systemd-nspawn container with --private-users so we mask it out here to avoid CI + # failures. + # FIXME: Remove when Arch/Debian/Ubuntu ship systemd v253 + args = ["systemd.mask=systemd-resolved.service"] if format == OutputFormat.directory else [] + image.boot(options=options, args=args) + + if image.distribution == Distribution.rhel_ubi: + return + + if format in (OutputFormat.directory, OutputFormat.tar): + return + + image.qemu(options=options) + + if format != OutputFormat.disk: + return + + image.qemu(options=options + ["--qemu-firmware=bios"]) -- 2.47.2