From 419dec7ae99cd69a6a9adb193a074dcdaa247e63 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Fri, 13 Sep 2024 20:05:42 +0200 Subject: [PATCH] Enable history for the default image The integration tests are also rewritten to take advantage of the functionality provided by enabling History=. --- mkosi.conf | 1 + tests/__init__.py | 100 ++++++++++++++++++++++++------------------- tests/test_boot.py | 27 ++++-------- tests/test_initrd.py | 97 ++++++++++++++++++++--------------------- tests/test_sysext.py | 21 +++------ 5 files changed, 119 insertions(+), 127 deletions(-) diff --git a/mkosi.conf b/mkosi.conf index 13add506f..a65bfaec1 100644 --- a/mkosi.conf +++ b/mkosi.conf @@ -1,6 +1,7 @@ # SPDX-License-Identifier: LGPL-2.1-or-later [Build] CacheDirectory=mkosi.cache +History=yes [Output] # These images are (among other things) used for running mkosi which means we need some disk space available so diff --git a/tests/__init__.py b/tests/__init__.py index 1e9eae4c3..fb31afc5a 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -14,7 +14,9 @@ from typing import Any, Optional import pytest from mkosi.distributions import Distribution -from mkosi.run import run +from mkosi.run import fork_and_wait, run +from mkosi.sandbox import acquire_privileges +from mkosi.tree import rmtree from mkosi.types import _FILE, CompletedProcess, PathString @@ -28,8 +30,7 @@ class ImageConfig: class Image: - def __init__(self, config: ImageConfig, options: Sequence[PathString] = []) -> None: - self.options = options + def __init__(self, config: ImageConfig) -> None: self.config = config st = Path.cwd().stat() self.uid = st.st_uid @@ -46,7 +47,11 @@ class Image: value: Optional[BaseException], traceback: Optional[TracebackType], ) -> None: - self.mkosi("clean", user=self.uid, group=self.gid) + def clean() -> None: + acquire_privileges() + rmtree(self.output_dir) + + fork_and_wait(clean) def mkosi( self, @@ -58,6 +63,23 @@ class Image: group: Optional[int] = None, check: bool = True, ) -> CompletedProcess: + return run( + [ + "python3", "-m", "mkosi", + "--debug", + *options, + verb, + *args, + ], + check=check, + stdin=stdin, + stdout=sys.stdout, + user=user, + group=group, + env=os.environ, + ) + + def build(self, options: Sequence[PathString] = (), args: Sequence[str] = ()) -> CompletedProcess: kcl = [ "loglevel=6", "systemd.log_level=debug", @@ -69,8 +91,7 @@ class Image: "systemd.unit=mkosi-check-and-shutdown.service", ] - return run([ - "python3", "-m", "mkosi", + opt: list[PathString] = [ "--distribution", str(self.config.distribution), "--release", self.config.release, *(["--tools-tree=default"] if self.config.tools_tree_distribution else []), @@ -80,26 +101,19 @@ class Image: else [] ), *(["--tools-tree-release", self.config.tools_tree_release] if self.config.tools_tree_release else []), + *(f"--kernel-command-line={i}" for i in kcl), + "--force", "--incremental", - "--ephemeral", - "--runtime-build-sources=no", - *self.options, - *options, "--output-dir", self.output_dir, - *(f"--kernel-command-line={i}" for i in kcl), - "--qemu-vsock=yes", - # TODO: Drop once both Hyper-V bugs are fixed in Github Actions. - "--qemu-args=-cpu max,pcid=off", - "--qemu-mem=2G", - verb, - *args, - ], check=check, stdin=stdin, stdout=sys.stdout, user=user, group=group, env=os.environ) + *(["--debug-shell"] if self.config.debug_shell else []), + *options, + ] + + self.mkosi("summary", options, user=self.uid, group=self.uid) - def build(self, options: Sequence[str] = (), args: Sequence[str] = ()) -> CompletedProcess: return self.mkosi( "build", - [*options, "--debug", "--force", *(["--debug-shell"] if self.config.debug_shell else [])], - args, + opt, stdin=sys.stdin if sys.stdin.isatty() else None, user=self.uid, group=self.gid, @@ -108,8 +122,13 @@ class Image: def boot(self, options: Sequence[str] = (), args: Sequence[str] = ()) -> CompletedProcess: result = self.mkosi( "boot", - [*options, "--debug"], - args, stdin=sys.stdin if sys.stdin.isatty() else None, + [ + "--runtime-build-sources=no", + "--ephemeral", + *options, + ], + args, + stdin=sys.stdin if sys.stdin.isatty() else None, check=False, ) @@ -118,10 +137,18 @@ class Image: return result - def qemu(self, options: Sequence[str] = (), args: Sequence[str] = ()) -> CompletedProcess: + def vm(self, verb: str, options: Sequence[str] = (), args: Sequence[str] = ()) -> CompletedProcess: result = self.mkosi( - "qemu", - [*options, "--debug"], + verb, + [ + "--runtime-build-sources=no", + "--qemu-vsock=yes", + # TODO: Drop once both Hyper-V bugs are fixed in Github Actions. + "--qemu-args=-cpu max,pcid=off", + "--qemu-mem=2G", + "--ephemeral", + *options, + ], args, stdin=sys.stdin if sys.stdin.isatty() else None, user=self.uid, @@ -136,24 +163,11 @@ class Image: return result - def vmspawn(self, options: Sequence[str] = (), args: Sequence[str] = ()) -> CompletedProcess: - result = self.mkosi( - "vmspawn", - [*options, "--debug"], - args, - stdin=sys.stdin if sys.stdin.isatty() else None, - check=False, - ) - - rc = 0 if self.config.distribution.is_centos_variant() else 123 - - if result.returncode != rc: - raise subprocess.CalledProcessError(result.returncode, result.args, result.stdout, result.stderr) - - return result + def qemu(self, options: Sequence[str] = (), args: Sequence[str] = ()) -> CompletedProcess: + return self.vm("qemu", options, args) - def summary(self, options: Sequence[str] = ()) -> CompletedProcess: - return self.mkosi("summary", options, user=self.uid, group=self.gid) + def vmspawn(self, options: Sequence[str] = (), args: Sequence[str] = ()) -> CompletedProcess: + return self.vm("vmspawn", options, args) def genkey(self) -> CompletedProcess: return self.mkosi("genkey", ["--force"], user=self.uid, group=self.gid) diff --git a/tests/test_boot.py b/tests/test_boot.py index fa245725a..3ee3af7bf 100644 --- a/tests/test_boot.py +++ b/tests/test_boot.py @@ -30,11 +30,8 @@ def test_format(config: ImageConfig, format: OutputFormat) -> None: if image.config.distribution == Distribution.rhel_ubi and format in (OutputFormat.esp, OutputFormat.uki): pytest.skip("Cannot build RHEL-UBI images with format 'esp' or 'uki'") - options = ["--format", str(format)] - - image.summary(options) image.genkey() - image.build(options=options) + image.build(options=["--format", str(format)]) if format in (OutputFormat.disk, OutputFormat.directory) and os.getuid() == 0: # systemd-resolved is enabled by default in Arch/Debian/Ubuntu (systemd default preset) but fails @@ -42,7 +39,7 @@ def test_format(config: ImageConfig, format: OutputFormat) -> None: # failures. # FIXME: Remove when Arch/Debian/Ubuntu ship systemd v253 args = ["systemd.mask=systemd-resolved.service"] if format == OutputFormat.directory else [] - image.boot(options=options, args=args) + image.boot(args=args) if format in (OutputFormat.cpio, OutputFormat.uki, OutputFormat.esp): pytest.skip("Default image is too large to be able to boot in CPIO/UKI/ESP format") @@ -56,17 +53,17 @@ def test_format(config: ImageConfig, format: OutputFormat) -> None: if format == OutputFormat.directory and not find_virtiofsd(): return - image.qemu(options=options) + image.qemu() if have_vmspawn() and format in (OutputFormat.disk, OutputFormat.directory): - image.vmspawn(options=options) + image.vmspawn() # TODO: Remove the opensuse check again when https://bugzilla.opensuse.org/show_bug.cgi?id=1227464 is resolved # and we install the grub tools in the openSUSE tools tree again. if format != OutputFormat.disk or config.tools_tree_distribution == Distribution.opensuse: return - image.qemu(options=options + ["--qemu-firmware=bios"]) + image.qemu(["--qemu-firmware=bios"]) @pytest.mark.parametrize("bootloader", Bootloader) @@ -81,15 +78,7 @@ def test_bootloader(config: ImageConfig, bootloader: Bootloader) -> None: firmware = QemuFirmware.linux if bootloader == Bootloader.none else QemuFirmware.auto - with Image( - config, - options=[ - "--format=disk", - "--bootloader", str(bootloader), - "--qemu-firmware", str(firmware) - ], - ) as image: - image.summary() + with Image(config) as image: image.genkey() - image.build() - image.qemu() + image.build(["--format=disk", "--bootloader", str(bootloader)]) + image.qemu(["--qemu-firmware", str(firmware)]) diff --git a/tests/test_initrd.py b/tests/test_initrd.py index ae91a6fd7..0e39b19ce 100644 --- a/tests/test_initrd.py +++ b/tests/test_initrd.py @@ -51,29 +51,21 @@ def passphrase() -> Iterator[Path]: def test_initrd(config: ImageConfig) -> None: - with Image(config, options=["--format=disk"]) as image: - image.build() + with Image(config) as image: + image.build(options=["--format=disk"]) image.qemu() @pytest.mark.skipif(os.getuid() != 0, reason="mkosi-initrd LVM test can only be executed as root") def test_initrd_lvm(config: ImageConfig) -> None: - with Image( - config, - options=[ - # LVM confuses systemd-repart so we mask it for this test. - "--kernel-command-line=systemd.mask=systemd-repart.service", - "--kernel-command-line=root=LABEL=root", - "--qemu-firmware=linux", - ] - ) as image, contextlib.ExitStack() as stack: - image.build(["--format", "directory"]) + with Image(config) as image, contextlib.ExitStack() as stack: + image.build(["--format=disk"]) - drive = Path(image.output_dir) / "image.raw" - drive.touch() - os.truncate(drive, 5000 * 1024**2) + lvm = Path(image.output_dir) / "lvm.raw" + lvm.touch() + os.truncate(lvm, 5000 * 1024**2) - lodev = run(["losetup", "--show", "--find", "--partscan", drive], stdout=subprocess.PIPE).stdout.strip() + lodev = run(["losetup", "--show", "--find", "--partscan", lvm], stdout=subprocess.PIPE).stdout.strip() stack.callback(lambda: run(["losetup", "--detach", lodev])) run(["sfdisk", "--label", "gpt", lodev], input="type=E6D6D379-F507-44C2-A23C-238F2A3DF928 bootable") run(["lvm", "pvcreate", f"{lodev}p1"]) @@ -87,14 +79,25 @@ def test_initrd_lvm(config: ImageConfig) -> None: run(["udevadm", "wait", "--timeout=30", "/dev/vg_mkosi/lv0"]) run([f"mkfs.{image.config.distribution.filesystem()}", "-L", "root", "/dev/vg_mkosi/lv0"]) - with tempfile.TemporaryDirectory() as mnt, mount(Path("/dev/vg_mkosi/lv0"), Path(mnt)): - # The image might have been built unprivileged so we need to fix the file ownership. Making all the - # files owned by root isn't completely correct but good enough for the purposes of the test. - copy_tree(Path(image.output_dir) / "image", Path(mnt), preserve=False) + src = Path(stack.enter_context(tempfile.TemporaryDirectory())) + run(["systemd-dissect", "--mount", "--mkdir", Path(image.output_dir) / "image.raw", src]) + stack.callback(lambda: run(["systemd-dissect", "--umount", "--rmdir", src])) + + dst = Path(stack.enter_context(tempfile.TemporaryDirectory())) + stack.enter_context(mount(Path("/dev/vg_mkosi/lv0"), dst)) + + copy_tree(src, dst) stack.close() - image.qemu(["--format=disk"]) + lvm.rename(Path(image.output_dir) / "image.raw") + + image.qemu([ + "--qemu-firmware=linux", + # LVM confuses systemd-repart so we mask it for this test. + "--kernel-command-line-extra=systemd.mask=systemd-repart.service", + "--kernel-command-line-extra=root=LABEL=root", + ]) def test_initrd_luks(config: ImageConfig, passphrase: Path) -> None: @@ -142,36 +145,21 @@ def test_initrd_luks(config: ImageConfig, passphrase: Path) -> None: ) ) - with Image( - config, - options=[ - "--repart-dir", repartd, - "--passphrase", passphrase, - "--credential=cryptsetup.passphrase=mkosi", - "--format=disk", - ] - ) as image: - image.build() - image.qemu() + with Image(config) as image: + image.build(["--repart-dir", repartd, "--passphrase", passphrase, "--format=disk"]) + image.qemu(["--credential=cryptsetup.passphrase=mkosi"]) @pytest.mark.skipif(os.getuid() != 0, reason="mkosi-initrd LUKS+LVM test can only be executed as root") def test_initrd_luks_lvm(config: ImageConfig, passphrase: Path) -> None: - with Image( - config, - options=[ - "--kernel-command-line=root=LABEL=root", - "--credential=cryptsetup.passphrase=mkosi", - "--qemu-firmware=linux", - ] - ) as image, contextlib.ExitStack() as stack: - image.build(["--format", "directory"]) + with Image(config) as image, contextlib.ExitStack() as stack: + image.build(["--format=disk"]) - drive = Path(image.output_dir) / "image.raw" - drive.touch() - os.truncate(drive, 5000 * 1024**2) + lvm = Path(image.output_dir) / "lvm.raw" + lvm.touch() + os.truncate(lvm, 5000 * 1024**2) - lodev = run(["losetup", "--show", "--find", "--partscan", drive], stdout=subprocess.PIPE).stdout.strip() + lodev = run(["losetup", "--show", "--find", "--partscan", lvm], stdout=subprocess.PIPE).stdout.strip() stack.callback(lambda: run(["losetup", "--detach", lodev])) run(["sfdisk", "--label", "gpt", lodev], input="type=E6D6D379-F507-44C2-A23C-238F2A3DF928 bootable") run( @@ -199,16 +187,25 @@ def test_initrd_luks_lvm(config: ImageConfig, passphrase: Path) -> None: run(["udevadm", "wait", "--timeout=30", "/dev/vg_mkosi/lv0"]) run([f"mkfs.{image.config.distribution.filesystem()}", "-L", "root", "/dev/vg_mkosi/lv0"]) - with tempfile.TemporaryDirectory() as mnt, mount(Path("/dev/vg_mkosi/lv0"), Path(mnt)): - # The image might have been built unprivileged so we need to fix the file ownership. Making all the - # files owned by root isn't completely correct but good enough for the purposes of the test. - copy_tree(Path(image.output_dir) / "image", Path(mnt), preserve=False) + src = Path(stack.enter_context(tempfile.TemporaryDirectory())) + run(["systemd-dissect", "--mount", "--mkdir", Path(image.output_dir) / "image.raw", src]) + stack.callback(lambda: run(["systemd-dissect", "--umount", "--rmdir", src])) + + dst = Path(stack.enter_context(tempfile.TemporaryDirectory())) + stack.enter_context(mount(Path("/dev/vg_mkosi/lv0"), dst)) + + copy_tree(src, dst) stack.close() + lvm.rename(Path(image.output_dir) / "image.raw") + image.qemu([ "--format=disk", - f"--kernel-command-line=rd.luks.uuid={luks_uuid}", + "--credential=cryptsetup.passphrase=mkosi", + "--qemu-firmware=linux", + "--kernel-command-line-extra=root=LABEL=root", + f"--kernel-command-line-extra=rd.luks.uuid={luks_uuid}", ]) diff --git a/tests/test_sysext.py b/tests/test_sysext.py index 6b5cf244e..6650aa50a 100644 --- a/tests/test_sysext.py +++ b/tests/test_sysext.py @@ -10,25 +10,16 @@ pytestmark = pytest.mark.integration def test_sysext(config: ImageConfig) -> None: - with Image( - config, - options=[ - "--clean-package-metadata=no", - "--format=directory", - ], - ) as image: - image.build() - - with Image( - image.config, - options=[ + with Image(config) as image: + image.build(["--clean-package-metadata=no", "--format=directory"]) + + with Image(image.config) as sysext: + sysext.build([ "--directory", "", "--incremental=no", "--base-tree", Path(image.output_dir) / "image", "--overlay", "--package=dnsmasq", "--format=disk", - ], - ) as sysext: - sysext.build() + ]) -- 2.47.2