From 35397affed57c656967367b15bff723a441fec2a Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Sun, 3 Sep 2023 13:55:00 +0200 Subject: [PATCH] Allow using a default tools tree Instead of requiring users to always set up a tools tree preset, let's allow specifying "default" as the tools tree to have mkosi build a default tools tree itself. This default tools tree includes all the software that might be necessary to build an image (excluding software that might be required by various user scripts). For distributions that do not have a rolling release variant, we use the closest matching distribution. e.g. for CentOS, we use Fedora and for Ubuntu we use Debian. --- mkosi/__init__.py | 45 +++++++++++++++++++++++++++++ mkosi/config.py | 20 ++++++++++++- mkosi/distributions/__init__.py | 14 +++++++++ mkosi/distributions/arch.py | 41 +++++++++++++++++++++++++- mkosi/distributions/centos.py | 36 ++++++++++++++++++++++- mkosi/distributions/debian.py | 44 +++++++++++++++++++++++++++- mkosi/distributions/fedora.py | 44 +++++++++++++++++++++++++++- mkosi/distributions/gentoo.py | 6 +++- mkosi/distributions/mageia.py | 6 +++- mkosi/distributions/openmandriva.py | 6 +++- mkosi/distributions/opensuse.py | 40 ++++++++++++++++++++++++- mkosi/resources/mkosi.md | 16 ++++++++++ 12 files changed, 309 insertions(+), 9 deletions(-) diff --git a/mkosi/__init__.py b/mkosi/__init__.py index 2f451e657..508f8e374 100644 --- a/mkosi/__init__.py +++ b/mkosi/__init__.py @@ -2145,6 +2145,49 @@ def prepend_to_environ_path(config: MkosiConfig) -> Iterator[None]: os.environ["PATH"] = ":".join(olds) +def finalize_tools(args: MkosiArgs, presets: Sequence[MkosiConfig]) -> Sequence[MkosiConfig]: + new = [] + + for p in presets: + if not p.tools_tree or p.tools_tree.name != "default": + new.append(p) + continue + + distribution = p.tools_tree_distribution or p.distribution.default_tools_tree_distribution() + release = p.tools_tree_release or distribution.default_release() + + cmdline = [ + "--directory", "", + "--distribution", str(distribution), + "--release", release, + "--repository-key-check", str(p.repository_key_check), + "--cache-only", str(p.cache_only), + "--output-dir", str(p.output_dir), + "--workspace-dir", str(p.workspace_dir), + *(["--cache-dir", str(p.cache_dir.parent)] if p.cache_dir else []), + "--incremental", str(p.incremental), + "--acl", str(p.acl), + "--format", "directory", + *flatten(["--package", package] for package in distribution.tools_tree_packages()), + "--output", f"{distribution}-tools", + "--bootable", "no", + "--manifest-format", "", + *(["--source-date-epoch", str(p.source_date_epoch)] if p.source_date_epoch is not None else []), + *(["-f"] * args.force), + "build", + ] + + _, [config] = parse_config(cmdline) + config = dataclasses.replace(config, preset=f"{distribution}-tools") + + if config not in new: + new.append(config) + + new.append(dataclasses.replace(p, tools_tree=config.output_dir / config.output)) + + return new + + def run_verb(args: MkosiArgs, presets: Sequence[MkosiConfig]) -> None: if args.verb.needs_root() and os.getuid() != 0: die(f"Must be root to run the {args.verb} command") @@ -2158,6 +2201,8 @@ def run_verb(args: MkosiArgs, presets: Sequence[MkosiConfig]) -> None: if args.verb == Verb.bump: return bump_image_version() + presets = finalize_tools(args, presets) + if args.verb == Verb.summary: text = "" diff --git a/mkosi/config.py b/mkosi/config.py index bd3df9a02..bbfe26f90 100644 --- a/mkosi/config.py +++ b/mkosi/config.py @@ -704,6 +704,8 @@ class MkosiConfig: kernel_command_line_extra: list[str] acl: bool tools_tree: Optional[Path] + tools_tree_distribution: Optional[Distribution] + tools_tree_release: Optional[str] # QEMU-specific options qemu_gui: bool @@ -1617,10 +1619,24 @@ SETTINGS = ( dest="tools_tree", metavar="PATH", section="Host", - parse=config_make_path_parser(required=False, resolve=False), + parse=config_make_path_parser(required=False), paths=("mkosi.tools",), help="Look up programs to execute inside the given tree", ), + MkosiConfigSetting( + dest="tools_tree_distribution", + metavar="DISTRIBUTION", + section="Host", + parse=config_make_enum_parser(Distribution), + help="Set the distribution to use for the default tools tree", + ), + MkosiConfigSetting( + dest="tools_tree_release", + metavar="RELEASE", + section="Host", + parse=config_parse_string, + help="Set the release to use for the default tools tree", + ), ) MATCHES = ( @@ -2373,6 +2389,8 @@ Clean Package Manager Metadata: {yes_no_auto(config.clean_package_metadata)} Extra Kernel Command Line: {line_join_list(config.kernel_command_line_extra)} Use ACLs: {config.acl} Tools Tree: {config.tools_tree} + Tools Tree Distribution: {none_to_none(config.tools_tree_distribution)} + Tools Tree Release: {none_to_none(config.tools_tree_release)} QEMU GUI: {yes_no(config.qemu_gui)} QEMU CPU Cores: {config.qemu_smp} diff --git a/mkosi/distributions/__init__.py b/mkosi/distributions/__init__.py index 7e2d837b9..10fb97ce9 100644 --- a/mkosi/distributions/__init__.py +++ b/mkosi/distributions/__init__.py @@ -54,6 +54,14 @@ class DistributionInstaller: def default_release(cls) -> str: raise NotImplementedError() + @classmethod + def default_tools_tree_distribution(cls) -> "Distribution": + raise NotImplementedError() + + @classmethod + def tools_tree_packages(cls) -> list[str]: + raise NotImplementedError() + class Distribution(StrEnum): fedora = enum.auto() @@ -108,6 +116,12 @@ class Distribution(StrEnum): def default_release(self) -> str: return self.installer().default_release() + def default_tools_tree_distribution(self) -> "Distribution": + return self.installer().default_tools_tree_distribution() + + def tools_tree_packages(self) -> list[str]: + return self.installer().tools_tree_packages() + def installer(self) -> Type[DistributionInstaller]: try: mod = importlib.import_module(f"mkosi.distributions.{self}") diff --git a/mkosi/distributions/arch.py b/mkosi/distributions/arch.py index 9585a2336..02139b008 100644 --- a/mkosi/distributions/arch.py +++ b/mkosi/distributions/arch.py @@ -3,7 +3,7 @@ from collections.abc import Sequence from mkosi.architecture import Architecture -from mkosi.distributions import DistributionInstaller, PackageType +from mkosi.distributions import Distribution, DistributionInstaller, PackageType from mkosi.installer.pacman import invoke_pacman, setup_pacman from mkosi.log import die from mkosi.state import MkosiState @@ -22,6 +22,45 @@ class ArchInstaller(DistributionInstaller): def default_release(cls) -> str: return "rolling" + @classmethod + def default_tools_tree_distribution(cls) -> Distribution: + return Distribution.arch + + @classmethod + def tools_tree_packages(cls) -> list[str]: + return [ + "archlinux-keyring", + "base", + "bash", + "btrfs-progs", + "bubblewrap", + "coreutils", + "cpio", + "debian-archive-keyring", + "dnf", + "dosfstools", + "e2fsprogs", + "edk2-ovmf", + "erofs-utils", + "mtools", + "openssh", + "openssl", + "pacman", + "python-cryptography", + "qemu-base", + "sbsigntools", + "socat", + "squashfs-tools", + "strace", + "swtpm", + "systemd-ukify", + "systemd", + "tar", + "xfsprogs", + "xz", + "zstd", + ] + @classmethod def setup(cls, state: MkosiState) -> None: setup_pacman(state) diff --git a/mkosi/distributions/centos.py b/mkosi/distributions/centos.py index 5fe944743..cbca31fcc 100644 --- a/mkosi/distributions/centos.py +++ b/mkosi/distributions/centos.py @@ -8,7 +8,7 @@ from pathlib import Path from mkosi.architecture import Architecture from mkosi.config import MkosiConfig -from mkosi.distributions import DistributionInstaller, PackageType +from mkosi.distributions import Distribution, DistributionInstaller, PackageType from mkosi.installer.dnf import Repo, invoke_dnf, setup_dnf from mkosi.log import complete_step, die from mkosi.state import MkosiState @@ -42,6 +42,40 @@ class CentosInstaller(DistributionInstaller): def default_release(cls) -> str: return "9" + @classmethod + def default_tools_tree_distribution(cls) -> Distribution: + return Distribution.fedora + + @classmethod + def tools_tree_packages(cls) -> list[str]: + return [ + "bash", + "bubblewrap", + "coreutils", + "cpio", + "dnf", + "dosfstools", + "e2fsprogs", + "edk2-ovmf", + "mtools", + "openssh-clients", + "openssl", + "python3-cryptography", + "qemu-kvm-core", + "pesign", + "socat", + "squashfs-tools", + "strace", + "swtpm", + "systemd-container", + "systemd-udev", + "systemd", + "tar", + "xfsprogs", + "xz", + "zstd", + ] + @classmethod def setup(cls, state: MkosiState) -> None: release = int(state.config.release) diff --git a/mkosi/distributions/debian.py b/mkosi/distributions/debian.py index 803d0f253..dffb2b98f 100644 --- a/mkosi/distributions/debian.py +++ b/mkosi/distributions/debian.py @@ -7,7 +7,7 @@ from pathlib import Path from mkosi.architecture import Architecture from mkosi.archive import extract_tar -from mkosi.distributions import DistributionInstaller, PackageType +from mkosi.distributions import Distribution, DistributionInstaller, PackageType from mkosi.installer.apt import invoke_apt, setup_apt from mkosi.log import die from mkosi.run import run @@ -28,6 +28,48 @@ class DebianInstaller(DistributionInstaller): def default_release(cls) -> str: return "testing" + @classmethod + def default_tools_tree_distribution(cls) -> Distribution: + return Distribution.debian + + @classmethod + def tools_tree_packages(cls) -> list[str]: + return [ + "apt", + "bash", + "btrfs-progs", + "bubblewrap", + "coreutils", + "cpio", + "debian-archive-keyring", + "dnf", + "dosfstools", + "e2fsprogs", + "erofs-utils", + "libtss2-dev", + "mtools", + "openssh-client", + "openssl", + "ovmf", + "pacman-package-manager", + "python3-cryptography", + "python3-pefile", + "qemu-system", + "sbsigntool", + "socat", + "squashfs-tools", + "strace", + "swtpm", + "systemd-boot", + "systemd-container", + "systemd", + "tar", + "xfsprogs", + "xz-utils", + "zstd", + "zypper", + ] + @staticmethod def repositories(state: MkosiState, local: bool = True) -> list[str]: assert state.config.mirror diff --git a/mkosi/distributions/fedora.py b/mkosi/distributions/fedora.py index 799dacc13..3f6c4b199 100644 --- a/mkosi/distributions/fedora.py +++ b/mkosi/distributions/fedora.py @@ -4,7 +4,7 @@ import urllib.parse from collections.abc import Sequence from mkosi.architecture import Architecture -from mkosi.distributions import DistributionInstaller, PackageType +from mkosi.distributions import Distribution, DistributionInstaller, PackageType from mkosi.installer.dnf import Repo, invoke_dnf, setup_dnf from mkosi.log import die from mkosi.state import MkosiState @@ -23,6 +23,48 @@ class FedoraInstaller(DistributionInstaller): def default_release(cls) -> str: return "39" + @classmethod + def default_tools_tree_distribution(cls) -> Distribution: + return Distribution.fedora + + @classmethod + def tools_tree_packages(cls) -> list[str]: + return [ + "apt", + "archlinux-keyring", + "bash", + "btrfs-progs", + "bubblewrap", + "coreutils", + "cpio", + "debian-keyring", + "dnf5", + "dosfstools", + "e2fsprogs", + "edk2-ovmf", + "erofs-utils", + "mtools", + "openssh-clients", + "openssl", + "pacman", + "python3-cryptography", + "qemu-kvm-core", + "sbsigntools", + "socat", + "squashfs-tools", + "strace", + "swtpm", + "systemd-container", + "systemd-udev", + "systemd-ukify", + "systemd", + "tar", + "xfsprogs", + "xz", + "zstd", + "zypper", + ] + @classmethod def setup(cls, state: MkosiState) -> None: # See: https://fedoraproject.org/security/ diff --git a/mkosi/distributions/gentoo.py b/mkosi/distributions/gentoo.py index f354a9dca..16cfe9cc0 100644 --- a/mkosi/distributions/gentoo.py +++ b/mkosi/distributions/gentoo.py @@ -9,7 +9,7 @@ from pathlib import Path from mkosi.architecture import Architecture from mkosi.archive import extract_tar -from mkosi.distributions import DistributionInstaller, PackageType +from mkosi.distributions import Distribution, DistributionInstaller, PackageType from mkosi.log import ARG_DEBUG, complete_step, die from mkosi.run import apivfs_cmd, bwrap, chroot_cmd, run from mkosi.state import MkosiState @@ -70,6 +70,10 @@ class GentooInstaller(DistributionInstaller): def default_release(cls) -> str: return "17.1" + @classmethod + def default_tools_tree_distribution(cls) -> Distribution: + return Distribution.gentoo + @classmethod def setup(cls, state: MkosiState) -> None: pass diff --git a/mkosi/distributions/mageia.py b/mkosi/distributions/mageia.py index f8db15c5f..969af9b34 100644 --- a/mkosi/distributions/mageia.py +++ b/mkosi/distributions/mageia.py @@ -3,7 +3,7 @@ from collections.abc import Sequence from mkosi.architecture import Architecture -from mkosi.distributions import DistributionInstaller, PackageType +from mkosi.distributions import Distribution, DistributionInstaller, PackageType from mkosi.installer.dnf import Repo, invoke_dnf, setup_dnf from mkosi.log import die from mkosi.state import MkosiState @@ -22,6 +22,10 @@ class MageiaInstaller(DistributionInstaller): def default_release(cls) -> str: return "cauldron" + @classmethod + def default_tools_tree_distribution(cls) -> Distribution: + return Distribution.mageia + @classmethod def setup(cls, state: MkosiState) -> None: release = state.config.release.strip("'") diff --git a/mkosi/distributions/openmandriva.py b/mkosi/distributions/openmandriva.py index a036948b8..d6c39397d 100644 --- a/mkosi/distributions/openmandriva.py +++ b/mkosi/distributions/openmandriva.py @@ -3,7 +3,7 @@ from collections.abc import Sequence from mkosi.architecture import Architecture -from mkosi.distributions import DistributionInstaller, PackageType +from mkosi.distributions import Distribution, DistributionInstaller, PackageType from mkosi.installer.dnf import Repo, invoke_dnf, setup_dnf from mkosi.log import die from mkosi.state import MkosiState @@ -22,6 +22,10 @@ class OpenmandrivaInstaller(DistributionInstaller): def default_release(cls) -> str: return "cooker" + @classmethod + def default_tools_tree_distribution(cls) -> Distribution: + return Distribution.openmandriva + @classmethod def setup(cls, state: MkosiState) -> None: release = state.config.release.strip("'") diff --git a/mkosi/distributions/opensuse.py b/mkosi/distributions/opensuse.py index 6d267023f..a80c4fd5d 100644 --- a/mkosi/distributions/opensuse.py +++ b/mkosi/distributions/opensuse.py @@ -6,7 +6,7 @@ import xml.etree.ElementTree as ElementTree from collections.abc import Sequence from mkosi.architecture import Architecture -from mkosi.distributions import DistributionInstaller, PackageType +from mkosi.distributions import Distribution, DistributionInstaller, PackageType from mkosi.installer.dnf import Repo, invoke_dnf, setup_dnf from mkosi.installer.zypper import invoke_zypper, setup_zypper from mkosi.log import die @@ -26,6 +26,44 @@ class OpensuseInstaller(DistributionInstaller): def default_release(cls) -> str: return "tumbleweed" + @classmethod + def default_tools_tree_distribution(cls) -> Distribution: + return Distribution.opensuse + + @classmethod + def tools_tree_packages(cls) -> list[str]: + return [ + "bash", + "btrfs-progs", + "bubblewrap", + "coreutils", + "cpio", + "dnf", + "dosfstools", + "e2fsprogs", + "erofs-utils", + "grep", + "mtools", + "openssh-clients", + "openssl", + "ovmf", + "qemu-headless", + "sbsigntools", + "socat", + "squashfs", + "strace", + "swtpm", + "systemd-boot", + "systemd-container", + "systemd-experimental", + "systemd", + "tar", + "xfsprogs", + "xz", + "zstd", + "zypper", + ] + @classmethod def setup(cls, state: MkosiState) -> None: release = state.config.release diff --git a/mkosi/resources/mkosi.md b/mkosi/resources/mkosi.md index 9f7af025b..439f6ad87 100644 --- a/mkosi/resources/mkosi.md +++ b/mkosi/resources/mkosi.md @@ -1114,6 +1114,22 @@ boolean argument: either `1`, `yes`, or `true` to enable, or `0`, `no`, `--extra-search-path=` are ignored when looking up binaries in the given tools tree. +: If set to `default`, mkosi will automatically add an extra tools tree + preset and use it as the tools tree. + +`ToolsTreeDistribution=`, `--tools-tree-distribution=` + +: Set the distribution to use for the default tools tree. By default, + the same distribution as the image that's being built is used, except + for CentOS and Ubuntu images, in which case Fedora and Debian are used + respectively. + +`ToolsTreeRelease=`, `--tools-tree-release=` + +: Set the distribution release to use for the default tools tree. By + default, the hardcoded default release in mkosi for the distribution + is used. + ## Supported distributions Images may be created containing installations of the following -- 2.47.2