From: Daan De Meyer Date: Sat, 27 Sep 2025 20:58:04 +0000 (+0200) Subject: Treat default initrd as a regular subimage X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=refs%2Fpull%2F3922%2Fhead;p=thirdparty%2Fmkosi.git Treat default initrd as a regular subimage Currently, because we build the default initrd as a substep of building a regular image, we have lots of special cased logic for it and we still propagate settings manually from the regular image to its default initrd. Let's streamline this by treating the default initrd as a regular image. The only complication about making this change is that we used to build the default initrd on demand only if a kernel was actually installed into the image. Because we have to make the decision of whether to build the default initrd or not way earlier now, we can't check if kernels were installed into the image or not. Instead, we check if any known kernel packages are listed to be installed which should be a decent enough heuristic. Another regression is that the default initrd won't have access to any packages built as part of the main image build anymore. We used to rely on this in systemd but now we build the systemd packages in a separate build subimage and those will still be available to the default initrd image build. We have to stop using Bootable=yes in a few tests as using it now means the resources folder has to be available and we don't propagate it during tests. --- diff --git a/mkosi/__init__.py b/mkosi/__init__.py index ee61f7c30..3eb22c224 100644 --- a/mkosi/__init__.py +++ b/mkosi/__init__.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: LGPL-2.1-or-later import contextlib -import dataclasses import datetime import functools import getpass @@ -82,6 +81,7 @@ from mkosi.config import ( resolve_deps, summary, systemd_tool_version, + want_kernel, want_selinux_relabel, yes_no, ) @@ -154,6 +154,7 @@ from mkosi.tree import copy_tree, make_tree, move_tree, rmtree from mkosi.user import INVOKING_USER, become_root_cmd from mkosi.util import ( PathString, + chdir, flatten, flock_or_die, format_rlimit, @@ -1347,143 +1348,6 @@ def want_initrd(context: Context) -> bool: return True -def finalize_default_initrd( - config: Config, - *, - resources: Path, - tools: bool = True, - output_dir: Optional[Path] = None, -) -> Config: - if config.root_password: - password, hashed = config.root_password - rootpwopt = f"hashed:{password}" if hashed else password - else: - rootpwopt = None - - relabel = ( - ConfigFeature.auto if config.selinux_relabel == ConfigFeature.enabled else config.selinux_relabel - ) - - # Default values are assigned via the parser so we go via the argument parser to construct - # the config for the initrd. - cmdline = [ - "--directory=", - f"--distribution={config.distribution}", - f"--release={config.release}", - f"--architecture={config.architecture}", - *([f"--mirror={config.mirror}"] if config.mirror else []), - *([f"--snapshot={config.snapshot}"] if config.snapshot else []), - f"--repository-key-check={config.repository_key_check}", - f"--repository-key-fetch={config.repository_key_fetch}", - *([f"--repositories={repository}" for repository in config.repositories]), - *([f"--sandbox-tree={tree}" for tree in config.sandbox_trees]), - # Note that when compress_output == Compression.none == 0 we don't pass --compress-output - # which means the default compression will get picked. This is exactly what we want so that - # initrds are always compressed. - *([f"--compress-output={c}"] if (c := config.compress_output) else []), - f"--compress-level={config.compress_level}", - f"--with-network={config.with_network}", - f"--cache-only={config.cacheonly}", - *([f"--output-directory={os.fspath(output_dir)}"] if output_dir else []), - *([f"--workspace-directory={os.fspath(config.workspace_dir)}"] if config.workspace_dir else []), - *([f"--cache-directory={os.fspath(config.cache_dir)}"] if config.cache_dir else []), - f"--cache-key={config.cache_key or '-'}", - *([f"--package-cache-directory={os.fspath(p)}"] if (p := config.package_cache_dir) else []), - *([f"--local-mirror={config.local_mirror}"] if config.local_mirror else []), - f"--incremental={config.incremental}", - *(f"--profile={profile}" for profile in config.initrd_profiles), - *(f"--package={package}" for package in config.initrd_packages), - *(f"--volatile-package={package}" for package in config.initrd_volatile_packages), - *(f"--package-directory={d}" for d in config.package_directories), - *(f"--volatile-package-directory={d}" for d in config.volatile_package_directories), - "--output=initrd", - *([f"--image-id={config.image_id}"] if config.image_id else []), - *([f"--image-version={config.image_version}"] if config.image_version else []), - *( - [f"--source-date-epoch={config.source_date_epoch}"] - if config.source_date_epoch is not None else - [] - ), - *([f"--locale={config.locale}"] if config.locale else []), - *([f"--locale-messages={config.locale_messages}"] if config.locale_messages else []), - *([f"--keymap={config.keymap}"] if config.keymap else []), - *([f"--timezone={config.timezone}"] if config.timezone else []), - *([f"--hostname={config.hostname}"] if config.hostname else []), - *([f"--root-password={rootpwopt}"] if rootpwopt else []), - *([f"--environment={k}='{v}'" for k, v in config.environment.items()]), - *([f"--tools-tree={os.fspath(config.tools_tree)}"] if config.tools_tree and tools else []), - f"--tools-tree-certificates={config.tools_tree_certificates}", - *([f"--extra-search-path={os.fspath(p)}" for p in config.extra_search_paths]), - *([f"--proxy-url={config.proxy_url}"] if config.proxy_url else []), - *([f"--proxy-exclude={host}" for host in config.proxy_exclude]), - *([f"--proxy-peer-certificate={os.fspath(p)}"] if (p := config.proxy_peer_certificate) else []), - *([f"--proxy-client-certificate={os.fspath(p)}"] if (p := config.proxy_client_certificate) else []), - *([f"--proxy-client-key={os.fspath(p)}"] if (p := config.proxy_client_key) else []), - f"--selinux-relabel={relabel}", - "--include=mkosi-initrd", - ] # fmt: skip - - _, _, [initrd] = parse_config(cmdline + ["build"], resources=resources) - - run_configure_scripts(initrd) - - initrd = dataclasses.replace(initrd, image="default-initrd") - - if initrd.is_incremental() and initrd.expand_key_specifiers( - initrd.cache_key - ) == config.expand_key_specifiers(config.cache_key): - die( - f"Image '{config.image}' and its default initrd image have the same cache key '{config.expand_key_specifiers(config.cache_key)}'", # noqa: E501 - hint="Add the &I specifier to the cache key to avoid this issue", - ) - - return initrd - - -def build_default_initrd(context: Context) -> Path: - if context.config.distribution == Distribution.custom: - die("Building a default initrd is not supported for custom distributions") - - config = finalize_default_initrd( - context.config, - resources=context.resources, - output_dir=context.workspace, - ) - - assert config.output_dir - - if config.is_incremental() and config.incremental == Incremental.strict and not have_cache(config): - die( - f"Strict incremental mode is enabled and cache for image {config.image} is out-of-date", - hint="Build once with -i yes to update the image cache", - ) - - config.output_dir.mkdir(exist_ok=True) - - if (config.output_dir / config.output).exists(): - return config.output_dir / config.output - - with ( - complete_step("Building default initrd"), - setup_workspace(context.args, config) as workspace, - ): - build_image( - Context( - context.args, - config, - workspace=workspace, - resources=context.resources, - # Reuse the keyring, repository metadata and local package repository from the main image for - # the default initrd. - keyring_dir=context.keyring_dir, - metadata_dir=context.metadata_dir, - package_dir=context.package_dir, - ) - ) - - return config.output_dir / config.output - - def identify_cpu(root: Path) -> tuple[Optional[Path], Optional[Path]]: for entry in Path("/proc/cpuinfo").read_text().split("\n\n"): vendor_id = family = model = stepping = None @@ -1574,7 +1438,8 @@ def build_microcode_initrd(context: Context) -> list[Path]: def finalize_kernel_modules_include(context: Context, *, include: Sequence[str], host: bool) -> set[str]: final = {i for i in include if i not in ("default", "host")} if "default" in include: - initrd = finalize_default_initrd(context.config, resources=context.resources) + with chdir(context.resources / "mkosi-initrd"): + _, _, [initrd] = parse_config([], resources=context.resources) final.update(initrd.kernel_modules_include) if host or "host" in include: final.update(loaded_modules()) @@ -2009,11 +1874,7 @@ def finalize_microcode(context: Context) -> list[Path]: def finalize_initrds(context: Context) -> list[Path]: - if context.config.initrds: - return context.config.initrds - elif any((context.artifacts / "io.mkosi.initrd").glob("*")): - return sorted((context.artifacts / "io.mkosi.initrd").iterdir()) - return [build_default_initrd(context)] + return context.config.initrds + sorted((context.artifacts / "io.mkosi.initrd").glob("*")) def install_type1( @@ -2294,17 +2155,7 @@ def install_kernel(context: Context, partitions: Sequence[Partition]) -> None: # single-file images have the benefit that they can be signed like normal EFI binaries, and can # encode everything necessary to boot a specific root device, including the root hash. - if context.config.output_format in (OutputFormat.uki, OutputFormat.esp): - return - - if context.config.bootable == ConfigFeature.disabled: - return - - if context.config.bootable == ConfigFeature.auto and ( - context.config.output_format == OutputFormat.cpio - or context.config.output_format.is_extension_or_portable_image() - or context.config.overlay - ): + if not want_kernel(context.config): return stub = systemd_stub_binary(context) @@ -2731,7 +2582,7 @@ def check_inputs(config: Config) -> None: ): die(f"Must run as root to use disk images in {name} trees") - if config.output_format != OutputFormat.none and config.bootable != ConfigFeature.disabled: + if want_kernel(config): for p in config.initrds: if not p.exists(): die(f"Initrd {p} not found") @@ -5038,9 +4889,6 @@ def run_verb(args: Args, tools: Optional[Config], images: Sequence[Config], *, r for config in images: run_clean(args, config) - if args.force > 0 and last.distribution != Distribution.custom: - remove_cache_entries(finalize_default_initrd(last, tools=False, resources=resources)) - rmtree(Path(".mkosi-private")) return @@ -5173,12 +5021,6 @@ def run_verb(args: Args, tools: Optional[Config], images: Sequence[Config], *, r for config in images: run_clean(args, config) - if last.distribution != Distribution.custom: - initrd = finalize_default_initrd(last, resources=resources) - - if args.force > 1 or not have_cache(initrd): - remove_cache_entries(initrd) - for i, config in enumerate(images): if args.verb != Verb.build: check_tools(config, args.verb) diff --git a/mkosi/config.py b/mkosi/config.py index 9f8b73ded..0c0cbd2ac 100644 --- a/mkosi/config.py +++ b/mkosi/config.py @@ -10,6 +10,7 @@ import functools import getpass import graphlib import io +import itertools import json import logging import math @@ -1652,6 +1653,19 @@ class SettingScope(StrEnum): main = enum.auto() # Only passed down to tools tree, can only be configured in main image. tools = enum.auto() + # Only passed down to initrd, can only be configured in main image. + initrd = enum.auto() + + def is_main_setting(self) -> bool: + return self in (SettingScope.main, SettingScope.tools, SettingScope.initrd, SettingScope.multiversal) + + def removeprefix(self, setting: str) -> str: + if self == SettingScope.tools: + return setting.removeprefix("tools_tree_") + elif self == SettingScope.initrd: + return setting.removeprefix("initrd_") + else: + return setting @dataclasses.dataclass(frozen=True) @@ -3165,6 +3179,7 @@ SETTINGS: list[ConfigSetting[Any]] = [ choices=InitrdProfile.values(), default=[], help="Which profiles to enable for the default initrd", + scope=SettingScope.initrd, ), ConfigSetting( dest="initrd_packages", @@ -3173,6 +3188,7 @@ SETTINGS: list[ConfigSetting[Any]] = [ section="Content", parse=config_make_list_parser(delimiter=","), help="Add additional packages to the default initrd", + scope=SettingScope.initrd, ), ConfigSetting( dest="initrd_volatile_packages", @@ -3181,6 +3197,7 @@ SETTINGS: list[ConfigSetting[Any]] = [ section="Content", parse=config_make_list_parser(delimiter=","), help="Packages to install in the initrd that are not cached", + scope=SettingScope.initrd, ), ConfigSetting( dest="devicetrees", @@ -4617,10 +4634,7 @@ class ParseContext: return ( (not setting.tools and image == "tools") - or ( - setting.scope in (SettingScope.main, SettingScope.tools, SettingScope.multiversal) - and image != "main" - ) + or (setting.scope.is_main_setting() and image != "main") or (setting.scope == SettingScope.universal and image not in ("main", "tools")) ) @@ -4758,10 +4772,7 @@ class ParseContext: # If the type is a collection or optional and the setting was set explicitly, don't use the default # value. - fieldname = ( - setting.dest.removeprefix("tools_tree_") if setting.scope == SettingScope.tools else setting.dest - ) - field = Config.fields().get(fieldname) + field = Config.fields().get(setting.scope.removeprefix(setting.dest)) origin = typing.get_origin(field.type) if field else None args = typing.get_args(field.type) if field else [] if ( @@ -5062,7 +5073,7 @@ def finalize_default_tools( elif s.scope == SettingScope.tools: # If the setting was specified on the CLI for the main config, we treat it as specified on the # CLI for the tools tree as well. Idem for config and defaults. - dest = s.dest.removeprefix("tools_tree_") + dest = s.scope.removeprefix(s.dest) if s.dest in main.cli: ns = context.cli @@ -5098,6 +5109,50 @@ def finalize_default_tools( return Config.from_dict(context.finalize()) +def finalize_default_initrd( + main: ParseContext, + finalized: dict[str, Any], + *, + resources: Path, +) -> Config: + context = ParseContext(resources) + + for s in SETTINGS: + if s.scope in (SettingScope.universal, SettingScope.multiversal): + context.cli[s.dest] = copy.deepcopy(finalized[s.dest]) + elif s.scope == SettingScope.initrd: + # If the setting was specified on the CLI for the main config, we treat it as specified on the + # CLI for the default initrd as well. Idem for config and defaults. + dest = s.scope.removeprefix(s.dest) + + if s.dest in main.cli: + ns = context.cli + if f"{s.dest}_was_none" in main.cli: + ns[f"{dest}_was_none"] = main.cli[f"{s.dest}_was_none"] + elif s.dest in main.config: + ns = context.config + else: + ns = context.defaults + + ns[dest] = copy.deepcopy(finalized[s.dest]) + + context.config |= { + "image": "default-initrd", + "directory": finalized["directory"], + "files": [], + } + + context.config["environment"] = { + name: finalized["environment"][name] + for name in finalized.get("environment", {}).keys() & finalized.get("pass_environment", []) + } + + with chdir(resources / "mkosi-initrd"): + context.parse_config_one(resources / "mkosi-initrd", parse_profiles=True) + + return Config.from_dict(context.finalize()) + + def finalize_configdir(directory: Optional[Path]) -> Optional[Path]: """Allow locating all mkosi configuration in a mkosi/ subdirectory instead of in the top-level directory of a git repository. @@ -5143,6 +5198,36 @@ def bump_image_version(configdir: Path) -> str: return new_version +def want_kernel(config: Config) -> bool: + if config.output_format in (OutputFormat.uki, OutputFormat.esp): + return False + + if config.bootable == ConfigFeature.disabled: + return False + + if config.bootable == ConfigFeature.auto and ( + config.output_format == OutputFormat.cpio + or config.output_format.is_extension_or_portable_image() + or config.overlay + ): + return False + + return True + + +def want_default_initrd(config: Config) -> bool: + if not want_kernel(config): + return False + + if config.bootable == ConfigFeature.auto and not any( + config.distribution.is_kernel_package(p) + for p in itertools.chain(config.packages, config.volatile_packages) + ): + return False + + return True + + def parse_config( argv: Sequence[str] = (), *, @@ -5325,8 +5410,29 @@ def parse_config( config["dependencies"] = dependencies main = Config.from_dict(config) - subimages = [Config.from_dict(ns) for ns in images] + + if any(want_default_initrd(image) for image in subimages + [main]): + initrd = finalize_default_initrd(context, config, resources=resources) + + if want_default_initrd(main): + main = dataclasses.replace( + main, + initrds=[*main.initrds, initrd.output_dir_or_cwd() / initrd.output], + dependencies=main.dependencies + [initrd.image], + ) + + subimages = [ + dataclasses.replace( + image, + initrds=[*image.initrds, initrd.output_dir_or_cwd() / initrd.output], + dependencies=image.dependencies + [initrd.image], + ) + for image in subimages + ] + + subimages += [initrd] + subimages = resolve_deps(subimages, main.dependencies) return args, tools, tuple(subimages + [main]) diff --git a/mkosi/distributions/__init__.py b/mkosi/distributions/__init__.py index 13f48e0e9..aae64480a 100644 --- a/mkosi/distributions/__init__.py +++ b/mkosi/distributions/__init__.py @@ -72,6 +72,10 @@ class DistributionInstaller: def latest_snapshot(cls, config: "Config") -> str: die(f"{cls.pretty_name()} does not support snapshots") + @classmethod + def is_kernel_package(cls, package: str) -> bool: + return False + class Distribution(StrEnum): # Please consult docs/distribution-policy.md and contact one @@ -158,6 +162,9 @@ class Distribution(StrEnum): def latest_snapshot(self, config: "Config") -> str: return self.installer().latest_snapshot(config) + def is_kernel_package(self, package: str) -> bool: + return self.installer().is_kernel_package(package) + def installer(self) -> type[DistributionInstaller]: modname = str(self).replace("-", "_") mod = importlib.import_module(f"mkosi.distributions.{modname}") diff --git a/mkosi/distributions/arch.py b/mkosi/distributions/arch.py index 6ce0ccf40..e809fdafc 100644 --- a/mkosi/distributions/arch.py +++ b/mkosi/distributions/arch.py @@ -130,3 +130,7 @@ class Installer(DistributionInstaller): return datetime.datetime.fromtimestamp(int(curl(config, url)), datetime.timezone.utc).strftime( "%Y/%m/%d" ) + + @classmethod + def is_kernel_package(cls, package: str) -> bool: + return package in ("kernel", "linux-lts", "linux-zen", "linux-hardened", "linux-rt", "linux-rt-lts") diff --git a/mkosi/distributions/centos.py b/mkosi/distributions/centos.py index 3c6d1623b..40b0ffc1a 100644 --- a/mkosi/distributions/centos.py +++ b/mkosi/distributions/centos.py @@ -471,3 +471,7 @@ class Installer(DistributionInstaller): return snapshot die("composeinfo is missing compose ID field") + + @classmethod + def is_kernel_package(cls, package: str) -> bool: + return package in ("kernel", "kernel-core") diff --git a/mkosi/distributions/debian.py b/mkosi/distributions/debian.py index 5873c5493..4eef0a23a 100644 --- a/mkosi/distributions/debian.py +++ b/mkosi/distributions/debian.py @@ -248,6 +248,10 @@ class Installer(DistributionInstaller): url = join_mirror(config.mirror or "https://snapshot.debian.org", "mr/timestamp") return cast(str, json.loads(curl(config, url))["result"]["debian"][-1]) + @classmethod + def is_kernel_package(cls, package: str) -> bool: + return package.startswith("linux-image-") + def install_apt_sources(context: Context, repos: Iterable[AptRepository]) -> None: sources = context.root / f"etc/apt/sources.list.d/{context.config.release}.sources" diff --git a/mkosi/distributions/fedora.py b/mkosi/distributions/fedora.py index c01b54fd2..ee690d4a6 100644 --- a/mkosi/distributions/fedora.py +++ b/mkosi/distributions/fedora.py @@ -267,3 +267,7 @@ class Installer(DistributionInstaller): ) return curl(config, url).removeprefix(f"Fedora-{config.release.capitalize()}-").strip() + + @classmethod + def is_kernel_package(cls, package: str) -> bool: + return package in ("kernel", "kernel-core") diff --git a/mkosi/distributions/opensuse.py b/mkosi/distributions/opensuse.py index 84971e2c1..dfeb5ad26 100644 --- a/mkosi/distributions/opensuse.py +++ b/mkosi/distributions/opensuse.py @@ -258,6 +258,10 @@ class Installer(DistributionInstaller): url = join_mirror(config.mirror or "https://download.opensuse.org", "history/latest") return curl(config, url).strip() + @classmethod + def is_kernel_package(cls, package: str) -> bool: + return package in ("kernel-default", "kernel-kvmsmall") + def fetch_gpgurls(context: Context, repourl: str) -> tuple[str, ...]: gpgurls = [f"{repourl}/repodata/repomd.xml.key"] diff --git a/mkosi/distributions/postmarketos.py b/mkosi/distributions/postmarketos.py index a5fa17ddd..c62d17a7e 100644 --- a/mkosi/distributions/postmarketos.py +++ b/mkosi/distributions/postmarketos.py @@ -109,3 +109,8 @@ class Installer(DistributionInstaller): die(f"Architecture {a} is not supported by postmarketOS") return a + + @classmethod + def is_kernel_package(cls, package: str) -> bool: + # TODO: Cover all of postmarketos's kernel packages. + return package == "linux-virt" diff --git a/mkosi/distributions/ubuntu.py b/mkosi/distributions/ubuntu.py index 889705614..560ba00a7 100644 --- a/mkosi/distributions/ubuntu.py +++ b/mkosi/distributions/ubuntu.py @@ -106,3 +106,7 @@ class Installer(debian.Installer): locale.setlocale(locale.LC_TIME, lc) die("Release file is missing Date field") + + @classmethod + def is_kernel_package(cls, package: str) -> bool: + return package.startswith("linux-") diff --git a/tests/test_config.py b/tests/test_config.py index 409d4e050..2385b2445 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -223,8 +223,10 @@ def test_parse_config(tmp_path: Path) -> None: (d / "abc/mkosi.conf").write_text( """\ [Content] - Bootable=yes BuildPackages=abc + + [Runtime] + CXL=yes """ ) (d / "abc/mkosi.conf.d").mkdir() @@ -237,19 +239,19 @@ def test_parse_config(tmp_path: Path) -> None: with chdir(d): _, _, [config] = parse_config() - assert config.bootable == ConfigFeature.auto + assert not config.cxl assert config.split_artifacts == ArtifactOutput.compat_no() # Passing the directory should include both the main config file and the dropin. _, _, [config] = parse_config(["--include", os.fspath(d / "abc")] * 2) - assert config.bootable == ConfigFeature.enabled + assert config.cxl assert config.split_artifacts == ArtifactOutput.compat_yes() # The same extra config should not be parsed more than once. assert config.build_packages == ["abc"] # Passing the main config file should not include the dropin. _, _, [config] = parse_config(["--include", os.fspath(d / "abc/mkosi.conf")]) - assert config.bootable == ConfigFeature.enabled + assert config.cxl assert config.split_artifacts == ArtifactOutput.compat_no() (d / "mkosi.images").mkdir() @@ -318,7 +320,6 @@ def test_parse_includes_once(tmp_path: Path) -> None: (d / "mkosi.conf").write_text( """\ [Content] - Bootable=yes BuildPackages=abc """ )