From: Daan De Meyer Date: Tue, 25 Apr 2023 13:04:19 +0000 (+0200) Subject: Add preset support X-Git-Tag: v15~188^2 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=refs%2Fpull%2F1506%2Fhead;p=thirdparty%2Fmkosi.git Add preset support Presets can be defined in mkosi.presets/. A preset is just like a regular config file/directory, except that mkosi can build multiple presets sequentially. If mkosi.presets/ exists, for each preset mkosi will read the global configuration, followed by the individual preset configuration. It will then build each of the presets in alpha-numerical order. Later presets can use outputs of earlier presets, specifically using the BaseTrees= and Initrds= options. While this has many use cases, one promising use case is to allow building an initrd and a final image that uses that initrd within a single invocation of mkosi. --- diff --git a/mkosi/__init__.py b/mkosi/__init__.py index bf82a5aba..415c2dfea 100644 --- a/mkosi/__init__.py +++ b/mkosi/__init__.py @@ -715,7 +715,7 @@ def install_unified_kernel(state: MkosiState, roothash: Optional[str]) -> None: # Default values are assigned via the parser so we go via the argument parser to construct # the config for the initrd. with complete_step("Building initrd"): - args, config = MkosiConfigParser().parse([ + args, presets = MkosiConfigParser().parse([ "--directory", "", "--distribution", str(state.config.distribution), "--release", state.config.release, @@ -744,9 +744,9 @@ def install_unified_kernel(state: MkosiState, roothash: Optional[str]) -> None: "build", ]) - build_stuff(state.uid, state.gid, args, config) + build_stuff(state.uid, state.gid, args, presets[0]) - initrds = [config.output_compressed] + initrds = [presets[0].output_compressed] for kver, kimg in gen_kernel_images(state): with complete_step(f"Generating unified kernel image for {kimg}"): @@ -1083,6 +1083,13 @@ def check_inputs(config: MkosiConfig) -> None: config.postinst_script, config.finalize_script): check_script_input(path) + + for p in config.initrds: + if not p.exists(): + die(f"Initrd {p} not found") + if not p.is_file(): + die(f"Initrd {p} is not a file") + except OSError as e: die(f'{e.filename}: {e.strerror}') @@ -1140,7 +1147,7 @@ def line_join_list( return "none" items = (str(path_or_none(cast(Path, item), checker=checker)) for item in array) - return "\n ".join(items) + return "\n ".join(items) def line_join_source_target_list(array: Sequence[tuple[Path, Optional[Path]]]) -> str: @@ -1148,7 +1155,7 @@ def line_join_source_target_list(array: Sequence[tuple[Path, Optional[Path]]]) - return "none" items = [f"{source}:{target}" if target else f"{source}" for source, target in array] - return "\n ".join(items) + return "\n ".join(items) def print_summary(args: MkosiArgs, config: MkosiConfig) -> None: @@ -1160,75 +1167,77 @@ def print_summary(args: MkosiArgs, config: MkosiConfig) -> None: env = [f"{k}={v}" for k, v in config.environment.items()] summary = f"""\ -{bold("COMMANDS")}: - verb: {bold(args.verb)} - cmdline: {bold(" ".join(args.cmdline))} - -{bold("DISTRIBUTION")} - Distribution: {bold(config.distribution.name)} - Release: {bold(none_to_na(config.release))} - Architecture: {config.architecture} - Mirror: {none_to_default(config.mirror)} - Local Mirror (build): {none_to_none(config.local_mirror)} - Repo Signature/Key check: {yes_no(config.repository_key_check)} - Repositories: {",".join(config.repositories)} - Initrds: {",".join(os.fspath(p) for p in config.initrds)} - -{bold("OUTPUT")}: - Image ID: {config.image_id} - Image Version: {config.image_version} - Output Format: {config.output_format.name} - Manifest Formats: {maniformats} - Output Directory: {none_to_default(config.output_dir)} - Workspace Directory: {none_to_default(config.workspace_dir)} - Output: {bold(config.output_compressed)} - Output Checksum: {none_to_na(config.output_checksum if config.checksum else None)} - Output Signature: {none_to_na(config.output_signature if config.sign else None)} - Output nspawn Settings: {none_to_na(config.output_nspawn_settings if config.nspawn_settings is not None else None)} - Incremental: {yes_no(config.incremental)} - Compression: {config.compress_output} - Bootable: {config.bootable} - Kernel Command Line: {" ".join(config.kernel_command_line)} - UEFI SecureBoot: {yes_no(config.secure_boot)} - SecureBoot Sign Key: {none_to_none(config.secure_boot_key)} - SecureBoot Certificate: {none_to_none(config.secure_boot_certificate)} - -{bold("CONTENT")}: - Packages: {line_join_list(config.packages)} - With Documentation: {yes_no(config.with_docs)} - Package Cache: {none_to_none(config.cache_dir)} - Skeleton Trees: {line_join_source_target_list(config.skeleton_trees)} - Extra Trees: {line_join_source_target_list(config.extra_trees)} - Clean Package Metadata: {yes_no_auto(config.clean_package_metadata)} - Remove Files: {line_join_list(config.remove_files)} - Remove Packages: {line_join_list(config.remove_packages)} - Build Sources: {config.build_sources} - Build Directory: {none_to_none(config.build_dir)} - Install Directory: {none_to_none(config.install_dir)} - Build Packages: {line_join_list(config.build_packages)} - Build Script: {path_or_none(config.build_script, check_script_input)} - Run Tests in Build Script: {yes_no(config.with_tests)} - Postinstall Script: {path_or_none(config.postinst_script, check_script_input)} - Prepare Script: {path_or_none(config.prepare_script, check_script_input)} - Finalize Script: {path_or_none(config.finalize_script, check_script_input)} - Script Environment: {line_join_list(env)} - Scripts with network: {yes_no(config.with_network)} - nspawn Settings: {none_to_none(config.nspawn_settings)} - Password: {("(default)" if config.password is None else "(set)")} - Autologin: {yes_no(config.autologin)} - -{bold("HOST CONFIGURATION")}: - Extra search paths: {line_join_list(config.extra_search_paths)} - QEMU Extra Arguments: {line_join_list(config.qemu_args)} - """ +{bold(f"PRESET: {config.preset or 'default'}")} + + {bold("COMMANDS")}: + verb: {bold(args.verb)} + cmdline: {bold(" ".join(args.cmdline))} + + {bold("DISTRIBUTION")}: + Distribution: {bold(config.distribution.name)} + Release: {bold(none_to_na(config.release))} + Architecture: {config.architecture} + Mirror: {none_to_default(config.mirror)} + Local Mirror (build): {none_to_none(config.local_mirror)} + Repo Signature/Key check: {yes_no(config.repository_key_check)} + Repositories: {",".join(config.repositories)} + Initrds: {",".join(os.fspath(p) for p in config.initrds)} + + {bold("OUTPUT")}: + Image ID: {config.image_id} + Image Version: {config.image_version} + Output Format: {config.output_format.name} + Manifest Formats: {maniformats} + Output Directory: {none_to_default(config.output_dir)} + Workspace Directory: {none_to_default(config.workspace_dir)} + Output: {bold(config.output_compressed)} + Output Checksum: {none_to_na(config.output_checksum if config.checksum else None)} + Output Signature: {none_to_na(config.output_signature if config.sign else None)} + Output nspawn Settings: {none_to_na(config.output_nspawn_settings if config.nspawn_settings is not None else None)} + Incremental: {yes_no(config.incremental)} + Compression: {config.compress_output.name} + Bootable: {yes_no_auto(config.bootable)} + Kernel Command Line: {" ".join(config.kernel_command_line)} + UEFI SecureBoot: {yes_no(config.secure_boot)} + SecureBoot Sign Key: {none_to_none(config.secure_boot_key)} + SecureBoot Certificate: {none_to_none(config.secure_boot_certificate)} + + {bold("CONTENT")}: + Packages: {line_join_list(config.packages)} + With Documentation: {yes_no(config.with_docs)} + Package Cache: {none_to_none(config.cache_dir)} + Skeleton Trees: {line_join_source_target_list(config.skeleton_trees)} + Extra Trees: {line_join_source_target_list(config.extra_trees)} + Clean Package Metadata: {yes_no_auto(config.clean_package_metadata)} + Remove Files: {line_join_list(config.remove_files)} + Remove Packages: {line_join_list(config.remove_packages)} + Build Sources: {config.build_sources} + Build Directory: {none_to_none(config.build_dir)} + Install Directory: {none_to_none(config.install_dir)} + Build Packages: {line_join_list(config.build_packages)} + Build Script: {path_or_none(config.build_script, check_script_input)} + Run Tests in Build Script: {yes_no(config.with_tests)} + Postinstall Script: {path_or_none(config.postinst_script, check_script_input)} + Prepare Script: {path_or_none(config.prepare_script, check_script_input)} + Finalize Script: {path_or_none(config.finalize_script, check_script_input)} + Script Environment: {line_join_list(env)} + Scripts with network: {yes_no(config.with_network)} + nspawn Settings: {none_to_none(config.nspawn_settings)} + Password: {("(default)" if config.password is None else "(set)")} + Autologin: {yes_no(config.autologin)} + + {bold("HOST CONFIGURATION")}: + Extra search paths: {line_join_list(config.extra_search_paths)} + QEMU Extra Arguments: {line_join_list(config.qemu_args)} + """ if config.output_format == OutputFormat.disk: summary += f"""\ -{bold("VALIDATION")}: - Checksum: {yes_no(config.checksum)} - Sign: {yes_no(config.sign)} - GPG Key: ({"default" if config.key is None else config.key}) + {bold("VALIDATION")}: + Checksum: {yes_no(config.checksum)} + Sign: {yes_no(config.sign)} + GPG Key: ({"default" if config.key is None else config.key}) """ page(summary, args.pager) @@ -2096,56 +2105,89 @@ def needs_build(args: MkosiArgs, config: MkosiConfig) -> bool: return args.verb == Verb.build or (args.verb in MKOSI_COMMANDS_NEED_BUILD and (not config.output_compressed.exists() or args.force > 0)) -def run_verb(args: MkosiArgs, config: MkosiConfig) -> None: - with prepend_to_environ_path(config.extra_search_paths): - if args.verb == Verb.genkey: - return generate_secure_boot_key(args) +def run_verb(args: MkosiArgs, presets: Sequence[MkosiConfig]) -> None: + if args.verb in MKOSI_COMMANDS_SUDO: + check_root() - if args.verb == Verb.bump: - return bump_image_version() + if args.verb == Verb.genkey: + return generate_secure_boot_key(args) - if args.verb == Verb.summary: - return print_summary(args, config) + if args.verb == Verb.bump: + return bump_image_version() - if args.verb in MKOSI_COMMANDS_SUDO: - check_root() + if args.verb == Verb.summary: + for config in presets: + print_summary(args, config) - if args.verb == Verb.build: - check_inputs(config) + return - if not args.force: - check_outputs(config) + last = presets[-1] - if needs_build(args, config) or args.verb == Verb.clean: - def target() -> None: - become_root() - unlink_output(args, config) + if args.verb == Verb.qemu and last.output_format in ( + OutputFormat.directory, + OutputFormat.subvolume, + OutputFormat.tar, + ): + die(f"{last.output_format} images cannot be booted in qemu.") + + if args.verb in (Verb.shell, Verb.boot): + opname = "acquire shell in" if args.verb == Verb.shell else "boot" + if last.output_format in (OutputFormat.tar, OutputFormat.cpio): + die(f"Sorry, can't {opname} a {last.output_format} archive.") + if last.compress_output: + die(f"Sorry, can't {opname} a compressed image.") + + # First, process all directory removals because otherwise if different presets share directories a later + # preset could end up output generated by an earlier preset. - fork_and_wait(target) + for config in presets: + if not needs_build(args, config) and args.verb != Verb.clean: + continue + + def target() -> None: + become_root() + unlink_output(args, config) + + fork_and_wait(target) - if needs_build(args, config): + build = False + + for config in presets: + if not needs_build(args, config): + continue + + check_inputs(config) + + if not args.force: + check_outputs(config) + + with prepend_to_environ_path(config.extra_search_paths): def target() -> None: # Get the user UID/GID either on the host or in the user namespace running the build uid, gid = become_root() init_mount_namespace() build_stuff(uid, gid, args, config) - # We only want to run the build in a user namespace but not the following steps. Since we can't - # rejoin the parent user namespace after unsharing from it, let's run the build in a fork so that - # the main process does not leave its user namespace. - fork_and_wait(target) + # We only want to run the build in a user namespace but not the following steps. Since we + # can't rejoin the parent user namespace after unsharing from it, let's run the build in a + # fork so that the main process does not leave its user namespace. + with complete_step(f"Building {config.preset or 'default'} image"): + fork_and_wait(target) + + build = True - if args.auto_bump: - bump_image_version() + if build and args.auto_bump: + bump_image_version() + with prepend_to_environ_path(last.extra_search_paths): if args.verb in (Verb.shell, Verb.boot): - run_shell(args, config) + run_shell(args, last) if args.verb == Verb.qemu: - run_qemu(args, config) + run_qemu(args, last) if args.verb == Verb.ssh: - run_ssh(args, config) + run_ssh(args, last) if args.verb == Verb.serve: - run_serve(config) + run_serve(last) diff --git a/mkosi/__main__.py b/mkosi/__main__.py index 31383dfb3..7cce4b6d3 100644 --- a/mkosi/__main__.py +++ b/mkosi/__main__.py @@ -41,12 +41,12 @@ def propagate_failed_return() -> Iterator[None]: @propagate_failed_return() def main() -> None: log_setup() - args, config = MkosiConfigParser().parse() + args, presets = MkosiConfigParser().parse() if ARG_DEBUG.get(): logging.getLogger().setLevel(logging.DEBUG) - run_verb(args, config) + run_verb(args, presets) if __name__ == "__main__": diff --git a/mkosi/config.py b/mkosi/config.py index 261705982..ccbd2c3da 100644 --- a/mkosi/config.py +++ b/mkosi/config.py @@ -1,5 +1,6 @@ import argparse import configparser +import copy import dataclasses import enum import fnmatch @@ -11,6 +12,7 @@ import os.path import platform import shlex import shutil +import string import subprocess import sys import textwrap @@ -496,6 +498,7 @@ class MkosiArgs: secure_boot_valid_days: str secure_boot_common_name: str auto_bump: bool + presets: list[str] @classmethod def from_namespace(cls, ns: argparse.Namespace) -> "MkosiArgs": @@ -587,6 +590,8 @@ class MkosiConfig: passphrase: Optional[Path] + preset: Optional[str] + @classmethod def from_namespace(cls, ns: argparse.Namespace) -> "MkosiConfig": return cls(**{ @@ -870,7 +875,7 @@ class MkosiConfigParser: MkosiConfigSetting( dest="base_trees", section="Content", - parse=config_make_list_parser(delimiter=",", parse=make_path_parser()), + parse=config_make_list_parser(delimiter=",", parse=make_path_parser(required=False)), ), MkosiConfigSetting( dest="extra_trees", @@ -1213,6 +1218,13 @@ class MkosiConfigParser: action="store_true", default=False, ) + parser.add_argument( + "--preset", + action="append", + dest="presets", + default=[], + help="Build the specified preset", + ) group = parser.add_argument_group("Distribution options") group.add_argument( @@ -1665,7 +1677,8 @@ class MkosiConfigParser: return parser - def parse(self, argv: Optional[Sequence[str]] = None) -> tuple[MkosiArgs, MkosiConfig]: + def parse(self, argv: Optional[Sequence[str]] = None) -> tuple[MkosiArgs, tuple[MkosiConfig, ...]]: + presets = [] namespace = argparse.Namespace() if argv is None: @@ -1704,21 +1717,45 @@ class MkosiConfigParser: if args.directory != "": self.parse_config(Path("."), namespace) - for s in self.SETTINGS: - if s.dest in namespace: - continue + if Path("mkosi.presets").exists(): + for p in sorted(Path("mkosi.presets").iterdir()): + name = p.name.lstrip(string.digits + "-").removesuffix(".conf") + if not name: + die(f"{p} is not a valid preset name") + if args.presets and name not in args.presets: + continue - if s.default_factory: - default = s.default_factory(namespace) - elif s.default is None: - default = s.parse(s.dest, None, namespace) - else: - default = s.default + cp = copy.deepcopy(namespace) + + with chdir(p if p.is_dir() else Path.cwd()): + self.parse_config(p if p.is_file() else Path("."), cp) + + setattr(cp, "preset", name) - setattr(namespace, s.dest, default) + presets += [cp] - return args, load_config(namespace) + if not presets: + setattr(namespace, "preset", None) + presets = [namespace] + if not presets: + die("No presets defined in mkosi.presets/") + + for ns in presets: + for s in self.SETTINGS: + if s.dest in ns: + continue + + if s.default_factory: + default = s.default_factory(ns) + elif s.default is None: + default = s.parse(s.dest, None, ns) + else: + default = s.default + + setattr(ns, s.dest, default) + + return args, tuple(load_config(ns) for ns in presets) class GenericVersion: def __init__(self, version: str): @@ -1899,13 +1936,6 @@ def load_config(args: argparse.Namespace) -> MkosiConfig: if args.cmdline and args.verb not in MKOSI_COMMANDS_CMDLINE: die(f"Parameters after verb are only accepted for {' '.join(verb.name for verb in MKOSI_COMMANDS_CMDLINE)}.") - if args.verb == Verb.qemu and args.output_format in ( - OutputFormat.directory, - OutputFormat.subvolume, - OutputFormat.tar, - ): - die("Directory, subvolume, tar, cpio, and plain squashfs images cannot be booted in qemu.") - if shutil.which("bsdtar") and args.distribution == Distribution.openmandriva and args.tar_strip_selinux_context: die("Sorry, bsdtar on OpenMandriva is incompatible with --tar-strip-selinux-context") @@ -1921,7 +1951,7 @@ def load_config(args: argparse.Namespace) -> MkosiConfig: args.compress_output = Compression.zst if args.output_format == OutputFormat.cpio else Compression.none if args.output is None: - iid = args.image_id if args.image_id is not None else "image" + iid = args.image_id or args.preset or "image" prefix = f"{iid}_{args.image_version}" if args.image_version is not None else iid if args.output_format == OutputFormat.disk: @@ -1973,13 +2003,6 @@ def load_config(args: argparse.Namespace) -> MkosiConfig: # Resolve passwords late so we can accurately determine whether a build is needed find_password(args) - if args.verb in (Verb.shell, Verb.boot): - opname = "acquire shell" if args.verb == Verb.shell else "boot" - if args.output_format in (OutputFormat.tar, OutputFormat.cpio): - die(f"Sorry, can't {opname} with a {args.output_format} archive.") - if args.compress_output: - die(f"Sorry, can't {opname} with a compressed image.") - if args.repo_dirs and not ( is_dnf_distribution(args.distribution) or is_apt_distribution(args.distribution) @@ -1998,11 +2021,6 @@ def load_config(args: argparse.Namespace) -> MkosiConfig: if args.initrds: args.initrds = [p.absolute() for p in args.initrds] - for p in args.initrds: - if not p.exists(): - die(f"Initrd {p} not found") - if not p.is_file(): - die(f"Initrd {p} is not a file") if args.overlay and not args.base_trees: die("--overlay can only be used with --base-tree") diff --git a/tests/test_parse_load_args.py b/tests/test_parse_load_args.py index b983b39e7..a532073d9 100644 --- a/tests/test_parse_load_args.py +++ b/tests/test_parse_load_args.py @@ -28,7 +28,7 @@ def cd_temp_dir() -> Iterator[None]: chdir(old_dir) -def parse(argv: Optional[List[str]] = None) -> tuple[MkosiArgs, MkosiConfig]: +def parse(argv: Optional[List[str]] = None) -> tuple[MkosiArgs, tuple[MkosiConfig, ...]]: return MkosiConfigParser().parse(argv) @@ -52,7 +52,7 @@ def test_parse_load_verb() -> None: def test_os_distribution() -> None: with cd_temp_dir(): for dist in Distribution: - assert parse(["-d", dist.name])[1].distribution == dist + assert parse(["-d", dist.name])[1][0].distribution == dist with pytest.raises(tuple((argparse.ArgumentError, SystemExit))): parse(["-d", "invalidDistro"]) @@ -62,7 +62,7 @@ def test_os_distribution() -> None: for dist in Distribution: config = Path("mkosi.conf") config.write_text(f"[Distribution]\nDistribution={dist}") - assert parse([])[1].distribution == dist + assert parse([])[1][0].distribution == dist def test_parse_config_files_filter() -> None: @@ -73,24 +73,12 @@ def test_parse_config_files_filter() -> None: (confd / "10-file.conf").write_text("[Content]\nPackages=yes") (confd / "20-file.noconf").write_text("[Content]\nPackages=nope") - assert parse([])[1].packages == ["yes"] - - -def test_shell_boot() -> None: - with cd_temp_dir(): - with pytest.raises(SystemExit): - parse(["--format", "tar", "boot"]) - - with pytest.raises(SystemExit): - parse(["--format", "cpio", "boot"]) - - with pytest.raises(SystemExit): - parse(["--format", "disk", "--compress-output=yes", "boot"]) + assert parse([])[1][0].packages == ["yes"] def test_compression() -> None: with cd_temp_dir(): - assert parse(["--format", "disk", "--compress-output", "False"])[1].compress_output == Compression.none + assert parse(["--format", "disk", "--compress-output", "False"])[1][0].compress_output == Compression.none @pytest.mark.parametrize("dist1,dist2", itertools.combinations_with_replacement(Distribution, 2)) @@ -145,7 +133,7 @@ def test_match_distribution(dist1: Distribution, dist2: Distribution) -> None: ) ) - conf = parse([])[1] + conf = parse([])[1][0] assert "testpkg1" in conf.packages if dist1 == dist2: assert "testpkg2" in conf.packages @@ -207,7 +195,7 @@ def test_match_release(release1: int, release2: int) -> None: ) ) - conf = parse([])[1] + conf = parse([])[1][0] assert "testpkg1" in conf.packages if release1 == release2: assert "testpkg2" in conf.packages @@ -283,7 +271,7 @@ def test_match_imageid(image1: str, image2: str) -> None: ) ) - conf = parse([])[1] + conf = parse([])[1][0] assert "testpkg1" in conf.packages if image1 == image2: assert "testpkg2" in conf.packages @@ -357,7 +345,7 @@ def test_match_imageversion(op: str, version: str) -> None: ) ) - conf = parse([])[1] + conf = parse([])[1][0] assert ("testpkg1" in conf.packages) == opfunc(123, version) assert ("testpkg2" in conf.packages) == opfunc(123, version) assert "testpkg3" not in conf.packages