From: Daan De Meyer Date: Mon, 6 Mar 2023 10:09:31 +0000 (+0100) Subject: Use an overlay for the build image instead of a full image X-Git-Tag: v15~304 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=77fc1a0ab4e9741d5465e349adf6d3df4c9b62f0;p=thirdparty%2Fmkosi.git Use an overlay for the build image instead of a full image Instead of building a second image for the build image, let's just make it an overlay for the final image since the only difference between the two is the list of installed packages. This speeds up image builds and allows us to simplify the internal logic as well. --- diff --git a/NEWS.md b/NEWS.md index af3724cc8..2b4abfb0c 100644 --- a/NEWS.md +++ b/NEWS.md @@ -46,6 +46,9 @@ - Dropped `--include-dir` option. Usage can be replaced by using `--incremental` and reading includes from the cached build image tree. - Removed `--machine-id` in favor of shipping images without a machine ID at all. +- Removed `--skip-final-phase` as we only have a single phase now. +- The post install script is only called for the final image now and not for the build image anymore. Use the + prepare script instead. ## v14 diff --git a/mkosi.md b/mkosi.md index b6004a93e..3d54fc677 100644 --- a/mkosi.md +++ b/mkosi.md @@ -127,119 +127,29 @@ The following command line verbs are known: : This verb is equivalent to the `--help` switch documented below: it shows a brief usage explanation. -## Execution flow +## Execution Flow -Execution flow for `mkosi build`. Columns represent the execution context. -Default values/calls are shown in parentheses. +Execution flow for `mkosi build`. Default values/calls are shown in parentheses. When building with `--incremental` mkosi creates a cache of the distribution -installation for both images if not already existing and replaces the -distribution installation in consecutive runs with data from the cached one. - -``` - HOST . BUILD . FINAL - . IMAGE . IMAGE - . . - start . . - | . . - v . . -build script? -------exists-----> copy . - | . skeleton trees . - | . (mkosi.skeleton/) . - none . | . - | . v . - v . install . - skip . distribution, . - build image . packages and . - | . build packages, . - | . run . - | . prepare script . - | . (mkosi.prepare build) . - | . or if incremental . - | . use cached build image . - | . | . - | . v . - | . copy . - | . build sources . - | . (./) . - | . | . - | . v . - | . copy . - | . extra trees . - | . (mkosi.extra/) . - | . | . - | . v . - | . run . - | . postinstall script . - | . (mkosi.postinst build) . - | . | . - | .-------------------------' . - | | . . - | v . . - | run . . - | finalize script . . - |(mkosi.finalize build). . - | | . . - | '-------------------------. . - | . | . - | . v . - | . run . - | . build script . - | . (mkosi.build) . - | . | . - '-----------------------------------+------------------------. - . . | - . . v - . . copy - . . skeleton trees - . . (mkosi.skeleton/) - . . | - . . v - . . install - . . distribution - . . and packages, - . . run - . . prepare script - . . (mkosi.prepare final) - . . or if incremental - . . use cached final image - . . | - . . v - . . copy - . . build results - . . | - . . v - . . copy - . . extra trees - . . (mkosi.extra/) - . . | - . . v - . . run - . . postinstall script - . . (mkosi.postinst final) - . . | - . . v - . . | - . . perform cleanup - . . (remove files, packages, - . . package metadata) - . . | - .--------------------------------------------------' - | . . - v . . - run . . - finalize script . . - (mkosi.finalize final) . . - | . . - .---------' . . - | . . - v . . - end . . - . . - HOST . BUILD . FINAL - . IMAGE . IMAGE - . . -``` - +installation if not already existing and replaces the distribution installation +in consecutive runs with data from the cached one. + +* Copy skeleton trees (`mkosi.skeleton`) into image +* Install distribution and packages into image or use cache tree if available +* Install build packages in overlay if a build script is configured +* Run prepare script on image and on image + build overlay if a build script is configured (`mkosi.prepare`) +* Run build script on image + build overlay if a build script is configured (`mkosi.build`) +* Copy the build script outputs into the image +* Copy the extra trees into the image (`mkosi.extra`) +* Run `kernel-install` +* Install systemd-boot +* Run post-install script (`mkosi.postinst`) +* Run `systemctl preset-all` +* Remove packages and files (`RemovePackages=`, `RemoveFiles=`) +* Run finalize script (`mkosi.finalize`) +* Run SELinux relabel is a SELinux policy is installed +* Generate unified kernel image +* Generate final output format ## Supported output formats The following output formats are supported: @@ -752,23 +662,14 @@ a boolean argument: either "1", "yes", or "true" to enable, or "0", `/dev/tty1` (QEMU) and `/dev/ttyS0` (QEMU with `QemuHeadless=yes`) by patching `/etc/pam.d/login`. -`SkipFinalPhase=`, `--skip-final-phase=` - -: Causes the (second) final image build stage to be skipped. This is - useful in combination with a build script, for when you care about - the artifacts that were created locally in `$BUILDDIR`, but - ultimately plan to discard the final image. - `BuildScript=`, `--build-script=` : Takes a path to an executable that is used as build script for this - image. If this option is used the build process will be two-phased - instead of single-phased. The specified script is copied onto the - development image and executed inside a namespaced chroot environment. - If this option is not used, but the `mkosi.build` file found in the - local directory it is automatically used for this purpose (also see - the "Files" section below). Specify an empty value to disable - automatic detection. + image. The specified script is copied onto the development image and + executed inside a namespaced chroot environment. If this option is not + used, but the `mkosi.build` file found in the local directory it is + automatically used for this purpose (also see the "Files" section below). + Specify an empty value to disable automatic detection. `PrepareScript=`, `--prepare-script=` diff --git a/mkosi/__init__.py b/mkosi/__init__.py index 17ea9bdde..4b0d307e9 100644 --- a/mkosi/__init__.py +++ b/mkosi/__init__.py @@ -26,7 +26,17 @@ import uuid from collections.abc import Iterable, Iterator, Sequence from pathlib import Path from textwrap import dedent, wrap -from typing import Any, Callable, NoReturn, Optional, TextIO, TypeVar, Union, cast +from typing import ( + Any, + Callable, + ContextManager, + NoReturn, + Optional, + TextIO, + TypeVar, + Union, + cast, +) from mkosi.backend import ( Distribution, @@ -42,7 +52,6 @@ from mkosi.backend import ( is_centos_variant, is_dnf_distribution, patch_file, - path_relative_to_cwd, set_umask, should_compress_output, tmp_dir, @@ -161,9 +170,7 @@ def mount_image(state: MkosiState, cached: bool) -> Iterator[None]: else: base = stack.enter_context(dissect_and_mount(state.config.base_image, state.workspace / "base")) - workdir = state.workspace / "workdir" - workdir.mkdir() - stack.enter_context(mount_overlay(base, state.root, workdir, state.root)) + stack.enter_context(mount_overlay(base, state.root, state.workdir, state.root)) yield @@ -198,7 +205,7 @@ def configure_hostname(state: MkosiState, cached: bool) -> None: def configure_dracut(state: MkosiState, cached: bool) -> None: - if not state.config.bootable or state.do_run_build_script or cached: + if not state.config.bootable or cached: return dracut_dir = state.root / "etc/dracut.conf.d" @@ -334,7 +341,7 @@ def clean_package_manager_metadata(state: MkosiState) -> None: """ assert state.config.clean_package_metadata in (False, True, 'auto') - if state.config.clean_package_metadata is False or state.do_run_build_script or state.for_cache: + if state.config.clean_package_metadata is False or state.for_cache: return # we try then all: metadata will only be touched if any of them are in the @@ -353,7 +360,7 @@ def clean_package_manager_metadata(state: MkosiState) -> None: def remove_files(state: MkosiState) -> None: """Remove files based on user-specified patterns""" - if not state.config.remove_files or state.do_run_build_script or state.for_cache: + if not state.config.remove_files or state.for_cache: return with complete_step("Removing files…"): @@ -369,10 +376,18 @@ def install_distribution(state: MkosiState, cached: bool) -> None: state.installer.install(state) +def install_build_packages(state: MkosiState, cached: bool) -> None: + if state.config.build_script is None or cached: + return + + with mount_build_overlay(state): + state.installer.install_packages(state, state.config.build_packages) + + def remove_packages(state: MkosiState) -> None: """Remove packages listed in config.remove_packages""" - if not state.config.remove_packages or state.do_run_build_script or state.for_cache: + if not state.config.remove_packages or state.for_cache: return with complete_step(f"Removing {len(state.config.packages)} packages…"): @@ -390,8 +405,6 @@ def reset_machine_id(state: MkosiState) -> None: each boot (if the image is read-only). """ - if state.do_run_build_script: - return if state.for_cache: return @@ -414,8 +427,6 @@ def reset_random_seed(root: Path) -> None: def configure_root_password(state: MkosiState, cached: bool) -> None: "Set the root account password, or just delete it so it's easy to log in" - if state.do_run_build_script: - return if cached: return @@ -457,7 +468,7 @@ def pam_add_autologin(root: Path, ttys: list[str]) -> None: def configure_autologin(state: MkosiState, cached: bool) -> None: - if state.do_run_build_script or cached or not state.config.autologin: + if cached or not state.config.autologin: return with complete_step("Setting up autologin…"): @@ -484,7 +495,7 @@ def configure_autologin(state: MkosiState, cached: bool) -> None: def configure_serial_terminal(state: MkosiState, cached: bool) -> None: """Override TERM for the serial console with the terminal type from the host.""" - if state.do_run_build_script or cached or not state.config.qemu_headless: + if cached or not state.config.qemu_headless: return with complete_step("Configuring serial tty (/dev/ttyS0)…"): @@ -504,31 +515,53 @@ def cache_params(state: MkosiState, root: Path) -> list[PathString]: return flatten(("--bind", state.cache, root / p) for p in state.installer.cache_path()) -def run_prepare_script(state: MkosiState, cached: bool) -> None: +def mount_build_overlay(state: MkosiState) -> ContextManager[Path]: + return mount_overlay(state.root, state.build_overlay, state.workdir, state.root) + + +def run_prepare_script(state: MkosiState, cached: bool, build: bool) -> None: if state.config.prepare_script is None: return if cached: return + if build and state.config.build_script is None: + return - verb = "build" if state.do_run_build_script else "final" - - with complete_step("Running prepare script…"): - bwrap: list[PathString] = [ - "--bind", state.config.build_sources, "/root/src", - "--bind", state.config.prepare_script, "/root/prepare", - *cache_params(state, Path("/")), - "--chdir", "/root/src", - ] - - run_workspace_command(state, ["/root/prepare", verb], network=True, bwrap_params=bwrap, - env=dict(SRCDIR="/root/src")) + bwrap: list[PathString] = [ + "--bind", state.config.build_sources, "/root/src", + "--bind", state.config.prepare_script, "/root/prepare", + *cache_params(state, Path("/")), + "--chdir", "/root/src", + ] + def clean() -> None: srcdir = state.root / "root/src" if srcdir.exists(): srcdir.rmdir() state.root.joinpath("root/prepare").unlink() + if build: + with complete_step("Running prepare script in build overlay…"), mount_build_overlay(state): + run_workspace_command( + state, + ["/root/prepare", "build"], + network=True, + bwrap_params=bwrap, + env=dict(SRCDIR="/root/src"), + ) + clean() + else: + with complete_step("Running prepare script…"): + run_workspace_command( + state, + ["/root/prepare", "final"], + network=True, + bwrap_params=bwrap, + env=dict(SRCDIR="/root/src"), + ) + clean() + def run_postinst_script(state: MkosiState) -> None: if state.config.postinst_script is None: @@ -536,15 +569,13 @@ def run_postinst_script(state: MkosiState) -> None: if state.for_cache: return - verb = "build" if state.do_run_build_script else "final" - with complete_step("Running postinstall script…"): bwrap: list[PathString] = [ "--bind", state.config.postinst_script, "/root/postinst", *cache_params(state, Path("/")), ] - run_workspace_command(state, ["/root/postinst", verb], bwrap_params=bwrap, + run_workspace_command(state, ["/root/postinst", "final"], bwrap_params=bwrap, network=state.config.with_network is True) state.root.joinpath("root/postinst").unlink() @@ -556,15 +587,13 @@ def run_finalize_script(state: MkosiState) -> None: if state.for_cache: return - verb = "build" if state.do_run_build_script else "final" - with complete_step("Running finalize script…"): - run([state.config.finalize_script, verb], + run([state.config.finalize_script], env={**state.environment, "BUILDROOT": str(state.root), "OUTPUTDIR": str(state.config.output_dir or Path.cwd())}) def install_boot_loader(state: MkosiState) -> None: - if not state.config.bootable or state.do_run_build_script or state.for_cache: + if not state.config.bootable or state.for_cache: return if state.config.secure_boot: @@ -656,8 +685,6 @@ def install_extra_trees(state: MkosiState) -> None: def install_build_dest(state: MkosiState) -> None: - if state.do_run_build_script: - return if state.for_cache: return @@ -696,8 +723,6 @@ def tar_binary() -> str: def make_tar(state: MkosiState) -> None: - if state.do_run_build_script: - return if state.config.output_format != OutputFormat.tar: return if state.for_cache: @@ -720,8 +745,6 @@ def find_files(dir: Path, root: Path) -> Iterator[Path]: def make_initrd(state: MkosiState) -> None: - if state.do_run_build_script: - return if state.config.output_format != OutputFormat.cpio: return if state.for_cache: @@ -747,7 +770,7 @@ def make_cpio(root: Path, files: Iterator[Path], output: Path) -> None: def make_directory(state: MkosiState) -> None: - if state.do_run_build_script or state.config.output_format != OutputFormat.directory or state.for_cache: + if state.config.output_format != OutputFormat.directory or state.for_cache: return os.rename(state.root, state.staging / state.config.output.name) @@ -781,15 +804,6 @@ def install_unified_kernel(state: MkosiState, roothash: Optional[str]) -> None: if state.for_cache: return - # Don't bother running dracut if this is a development build. Strictly speaking it would probably be a - # good idea to run it, so that the development environment differs as little as possible from the final - # build, but then again the initrd should not be relevant for building, and dracut is simply very slow, - # hence let's avoid it invoking it needlessly, given that we never actually invoke the boot loader on the - # development image. - if state.do_run_build_script: - return - - with complete_step("Generating combined kernel + initrd boot file…"): for kver, kimg in gen_kernel_images(state): image_id = state.config.image_id or f"mkosi-{state.config.distribution}" @@ -988,12 +1002,17 @@ def acl_toggle_remove(root: Path, uid: int, *, allow: bool) -> None: def save_cache(state: MkosiState) -> None: - cache = cache_tree_path(state.config, is_final_image=False) if state.do_run_build_script else cache_tree_path(state.config, is_final_image=True) + final, build = cache_tree_paths(state.config) + + with complete_step("Installing cache copies"): + unlink_try_hard(final) + shutil.move(state.root, final) + acl_toggle_remove(final, state.uid, allow=True) - with complete_step("Installing cache copy…", f"Installed cache copy {path_relative_to_cwd(cache)}"): - unlink_try_hard(cache) - shutil.move(state.root, cache) - acl_toggle_remove(cache, state.uid, allow=True) + if state.config.build_script: + unlink_try_hard(build) + shutil.move(state.build_overlay, build) + acl_toggle_remove(build, state.uid, allow=True) def dir_size(path: PathString) -> int: @@ -1760,13 +1779,6 @@ def create_parser() -> ArgumentParserMkosi: help="Additional packages needed for build script", metavar="PACKAGE", ) - group.add_argument( - "--skip-final-phase", - metavar="BOOL", - action=BooleanAction, - help="Skip the (second) final image building phase.", - default=False, - ) group.add_argument( "--build-script", help="Build script to run inside image", @@ -2109,35 +2121,34 @@ def empty_directory(path: Path) -> None: def unlink_output(config: MkosiConfig) -> None: - if not config.skip_final_phase: - with complete_step("Removing output files…"): - if config.output.parent.exists(): - for p in config.output.parent.iterdir(): - if p.name.startswith(config.output.name) and "cache" not in p.name: - unlink_try_hard(p) - unlink_try_hard(Path(f"{config.output}.manifest")) - unlink_try_hard(Path(f"{config.output}.changelog")) + with complete_step("Removing output files…"): + if config.output.parent.exists(): + for p in config.output.parent.iterdir(): + if p.name.startswith(config.output.name) and "cache" not in p.name: + unlink_try_hard(p) + unlink_try_hard(Path(f"{config.output}.manifest")) + unlink_try_hard(Path(f"{config.output}.changelog")) - if config.checksum: - unlink_try_hard(config.output_checksum) + if config.checksum: + unlink_try_hard(config.output_checksum) - if config.sign: - unlink_try_hard(config.output_signature) + if config.sign: + unlink_try_hard(config.output_signature) - if config.bmap: - unlink_try_hard(config.output_bmap) + if config.bmap: + unlink_try_hard(config.output_bmap) - if config.output_split_kernel.parent.exists(): - for p in config.output_split_kernel.parent.iterdir(): - if p.name.startswith(config.output_split_kernel.name): - unlink_try_hard(p) - unlink_try_hard(config.output_split_kernel) + if config.output_split_kernel.parent.exists(): + for p in config.output_split_kernel.parent.iterdir(): + if p.name.startswith(config.output_split_kernel.name): + unlink_try_hard(p) + unlink_try_hard(config.output_split_kernel) - if config.nspawn_settings is not None: - unlink_try_hard(config.output_nspawn_settings) + if config.nspawn_settings is not None: + unlink_try_hard(config.output_nspawn_settings) - if config.ssh and config.output_sshkey is not None: - unlink_try_hard(config.output_sshkey) + if config.ssh and config.output_sshkey is not None: + unlink_try_hard(config.output_sshkey) # We remove any cached images if either the user used --force # twice, or he/she called "clean" with it passed once. Let's also @@ -2153,8 +2164,8 @@ def unlink_output(config: MkosiConfig) -> None: if remove_build_cache: with complete_step("Removing incremental cache files…"): - unlink_try_hard(cache_tree_path(config, is_final_image=False)) - unlink_try_hard(cache_tree_path(config, is_final_image=True)) + for p in cache_tree_paths(config): + unlink_try_hard(p) if config.build_dir is not None: with complete_step("Clearing out build directory…"): @@ -2537,9 +2548,6 @@ def load_args(args: argparse.Namespace) -> MkosiConfig: if needs_build(args) and args.verb == Verb.qemu and not args.bootable: die("Images built without the --bootable option cannot be booted using qemu", MkosiNotSupportedException) - if args.skip_final_phase and args.verb != Verb.build: - die("--skip-final-phase can only be used when building an image using 'mkosi build'", MkosiNotSupportedException) - if args.ssh_timeout < 0: die("--ssh-timeout must be >= 0") @@ -2569,18 +2577,19 @@ def load_args(args: argparse.Namespace) -> MkosiConfig: return MkosiConfig(**vars(args)) -def cache_tree_path(config: MkosiConfig, is_final_image: bool) -> Path: - suffix = "final-cache" if is_final_image else "build-cache" +def cache_tree_paths(config: MkosiConfig) -> tuple[Path, Path]: # If the image ID is specified, use cache file names that are independent of the image versions, so that # rebuilding and bumping versions is cheap and reuses previous versions if cached. if config.image_id is not None and config.output_dir: - return config.output_dir / f"{config.image_id}.{suffix}" + prefix = config.output_dir / config.image_id elif config.image_id: - return Path(f"{config.image_id}.{suffix}") + prefix = Path(config.image_id) # Otherwise, derive the cache file names directly from the output file names. else: - return Path(f"{config.output}.{suffix}") + prefix = config.output + + return (Path(f"{prefix}.cache"), Path(f"{prefix}.build.cache")) def check_tree_input(path: Optional[Path]) -> None: @@ -2624,9 +2633,6 @@ def check_inputs(config: MkosiConfig) -> None: def check_outputs(config: MkosiConfig) -> None: - if config.skip_final_phase: - return - for f in ( config.output, config.output_checksum if config.checksum else None, @@ -2776,7 +2782,6 @@ def print_summary(config: MkosiConfig) -> None: print(" Build Directory:", none_to_none(config.build_dir)) print(" Install Directory:", none_to_none(config.install_dir)) print(" Build Packages:", line_join_list(config.build_packages)) - print(" Skip final phase:", yes_no(config.skip_final_phase)) print(" Build Script:", path_or_none(config.build_script, check_script_input)) @@ -2846,7 +2851,7 @@ def make_install_dir(state: MkosiState) -> None: def configure_ssh(state: MkosiState) -> None: - if state.do_run_build_script or state.for_cache or not state.config.ssh: + if state.for_cache or not state.config.ssh: return if state.config.distribution in (Distribution.debian, Distribution.ubuntu): @@ -2899,7 +2904,7 @@ def configure_ssh(state: MkosiState) -> None: def configure_netdev(state: MkosiState, cached: bool) -> None: - if state.do_run_build_script or cached or not state.config.netdev: + if cached or not state.config.netdev: return with complete_step("Setting up netdev…"): @@ -2932,7 +2937,7 @@ def configure_netdev(state: MkosiState, cached: bool) -> None: def run_kernel_install(state: MkosiState, cached: bool) -> None: - if not state.config.bootable or state.do_run_build_script: + if not state.config.bootable: return if not state.config.cache_initrd and state.for_cache: @@ -2966,7 +2971,7 @@ def run_kernel_install(state: MkosiState, cached: bool) -> None: def run_preset_all(state: MkosiState) -> None: - if state.for_cache or state.do_run_build_script: + if state.for_cache: return with complete_step("Applying presets…"): @@ -2974,7 +2979,7 @@ def run_preset_all(state: MkosiState) -> None: def run_selinux_relabel(state: MkosiState) -> None: - if state.for_cache or state.do_run_build_script: + if state.for_cache: return selinux = state.root / "etc/selinux/config" @@ -2999,21 +3004,24 @@ def reuse_cache_tree(state: MkosiState) -> bool: if not state.config.incremental: return False - cache = cache_tree_path(state.config, is_final_image=not state.do_run_build_script) - if not cache.exists(): + final, build = cache_tree_paths(state.config) + if not final.exists() or (state.config.build_script and not build.exists()): return False - if state.for_cache and cache.exists(): + if state.for_cache and final.exists() and (not state.config.build_script or build.exists()): return True - with complete_step(f"Basing off cached tree {cache}", "Copied cached tree"): - copy_path(cache, state.root) + with complete_step("Copying cached trees"): + copy_path(final, state.root) acl_toggle_remove(state.root, state.uid, allow=False) + if state.config.build_script: + copy_path(build, state.build_overlay) + acl_toggle_remove(state.build_overlay, state.uid, allow=False) return True def invoke_repart(state: MkosiState, skip: Sequence[str] = [], split: bool = False) -> Optional[str]: - if not state.config.output_format == OutputFormat.disk or state.for_cache or state.do_run_build_script: + if not state.config.output_format == OutputFormat.disk or state.for_cache: return None cmdline: list[PathString] = [ @@ -3094,11 +3102,6 @@ def invoke_repart(state: MkosiState, skip: Sequence[str] = [], split: bool = Fal def build_image(state: MkosiState, *, manifest: Optional[Manifest] = None) -> None: - # If there's no build script set, there's no point in executing - # the build script iteration. Let's quit early. - if state.config.build_script is None and state.do_run_build_script: - return - cached = reuse_cache_tree(state) if state.for_cache and cached: return @@ -3113,7 +3116,10 @@ def build_image(state: MkosiState, *, manifest: Optional[Manifest] = None) -> No configure_autologin(state, cached) configure_dracut(state, cached) configure_netdev(state, cached) - run_prepare_script(state, cached) + run_prepare_script(state, cached, build=False) + install_build_packages(state, cached) + run_prepare_script(state, cached, build=True) + run_build_script(state) install_build_dest(state) install_extra_trees(state) run_kernel_install(state, cached) @@ -3152,10 +3158,10 @@ def install_dir(state: MkosiState) -> Path: def run_build_script(state: MkosiState) -> None: - if state.config.build_script is None: + if state.config.build_script is None or state.for_cache: return - with complete_step("Running build script…"): + with complete_step("Running build script…"), mount_build_overlay(state): # Bubblewrap creates bind mount point parent directories with restrictive permissions so we create # the work directory outselves here. state.root.joinpath("work").mkdir(mode=0o755) @@ -3203,27 +3209,16 @@ def run_build_script(state: MkosiState) -> None: state.root.joinpath("work").rmdir() -def need_cache_trees(state: MkosiState) -> bool: +def need_cache_tree(state: MkosiState) -> bool: if not state.config.incremental: return False if state.config.force > 1: return True - return not cache_tree_path(state.config, is_final_image=True).exists() or state.config.build_script is not None and not cache_tree_path(state.config, is_final_image=False).exists() + final, build = cache_tree_paths(state.config) - -def remove_artifacts(state: MkosiState, for_cache: bool = False) -> None: - if for_cache: - what = "cache build" - elif state.do_run_build_script: - what = "development build" - else: - return - - with complete_step(f"Removing artifacts from {what}…"): - unlink_try_hard(state.root) - unlink_try_hard(state.var_tmp) + return not final.exists() or (state.config.build_script is not None and not build.exists()) def build_stuff(uid: int, gid: int, config: MkosiConfig) -> None: @@ -3237,7 +3232,6 @@ def build_stuff(uid: int, gid: int, config: MkosiConfig) -> None: config=config, workspace=workspace_dir, cache=cache, - do_run_build_script=False, for_cache=False, ) @@ -3252,39 +3246,15 @@ def build_stuff(uid: int, gid: int, config: MkosiConfig) -> None: # while we are working on it. with flock(workspace_dir), workspace: # If caching is requested, then make sure we have cache trees around we can make use of - if need_cache_trees(state): - - # There is no point generating a pre-dev cache image if no build script is provided - if config.build_script: - with complete_step("Running first (development) stage to generate cached copy…"): - # Generate the cache version of the build image, and store it as "cache-pre-dev" - state = dataclasses.replace(state, do_run_build_script=True, for_cache=True) - build_image(state) - save_cache(state) - remove_artifacts(state) - - with complete_step("Running second (final) stage to generate cached copy…"): - # Generate the cache version of the build image, and store it as "cache-pre-inst" - state = dataclasses.replace(state, do_run_build_script=False, for_cache=True) + if need_cache_tree(state): + with complete_step("Building cache image"): + state = dataclasses.replace(state, for_cache=True) build_image(state) save_cache(state) - remove_artifacts(state) - if config.build_script: - with complete_step("Running first (development) stage…"): - # Run the image builder for the first (development) stage in preparation for the build script - state = dataclasses.replace(state, do_run_build_script=True, for_cache=False) - build_image(state) - run_build_script(state) - remove_artifacts(state) - - # Run the image builder for the second (final) stage - if not config.skip_final_phase: - with complete_step("Running second (final) stage…"): - state = dataclasses.replace(state, do_run_build_script=False, for_cache=False) - build_image(state, manifest=manifest) - else: - MkosiPrinter.print_step("Skipping (second) final image build phase.") + with complete_step("Building image"): + state = dataclasses.replace(state, for_cache=False) + build_image(state, manifest=manifest) qcow2_output(state) calculate_bmap(state) diff --git a/mkosi/backend.py b/mkosi/backend.py index da836666e..b42c17d44 100644 --- a/mkosi/backend.py +++ b/mkosi/backend.py @@ -280,7 +280,6 @@ class MkosiConfig: build_dir: Optional[Path] install_dir: Optional[Path] build_packages: list[str] - skip_final_phase: bool build_script: Optional[Path] prepare_script: Optional[Path] postinst_script: Optional[Path] @@ -383,7 +382,6 @@ class MkosiState: config: MkosiConfig workspace: Path cache: Path - do_run_build_script: bool for_cache: bool environment: dict[str, str] = dataclasses.field(init=False) installer: DistributionInstaller = dataclasses.field(init=False) @@ -406,6 +404,8 @@ class MkosiState: self.installer = instance self.root.mkdir(exist_ok=True, mode=0o755) + self.build_overlay.mkdir(exist_ok=True, mode=0o755) + self.workdir.mkdir(exist_ok=True) self.var_tmp.mkdir(exist_ok=True) self.staging.mkdir(exist_ok=True) @@ -413,6 +413,14 @@ class MkosiState: def root(self) -> Path: return self.workspace / "root" + @property + def build_overlay(self) -> Path: + return self.workspace / "build-overlay" + + @property + def workdir(self) -> Path: + return self.workspace / "workdir" + @property def var_tmp(self) -> Path: return self.workspace / "var-tmp" @@ -461,14 +469,6 @@ def patch_file(filepath: Path, line_rewriter: Callable[[str], str]) -> None: shutil.move(temp_new_filepath, filepath) -def path_relative_to_cwd(path: Path) -> Path: - "Return path as relative to $PWD if underneath, absolute path otherwise" - try: - return path.relative_to(os.getcwd()) - except ValueError: - return path - - def safe_tar_extract(tar: tarfile.TarFile, path: Path=Path("."), *, numeric_owner: bool=False) -> None: """Extract a tar without CVE-2007-4559. diff --git a/mkosi/distributions/arch.py b/mkosi/distributions/arch.py index ec87648bd..1e00e031f 100644 --- a/mkosi/distributions/arch.py +++ b/mkosi/distributions/arch.py @@ -93,7 +93,7 @@ def install_arch(state: MkosiState) -> None: packages = state.config.packages.copy() add_packages(state.config, packages, "base") - if not state.do_run_build_script and state.config.bootable: + if state.config.bootable: add_packages(state.config, packages, "dracut") official_kernel_packages = { @@ -104,14 +104,11 @@ def install_arch(state: MkosiState) -> None: } has_kernel_package = official_kernel_packages.intersection(state.config.packages) - if not state.do_run_build_script and state.config.bootable and not has_kernel_package: + if state.config.bootable and not has_kernel_package: # No user-specified kernel add_packages(state.config, packages, "linux") - if state.do_run_build_script: - packages += state.config.build_packages - - if not state.do_run_build_script and state.config.ssh: + if state.config.ssh: add_packages(state.config, packages, "openssh") invoke_pacman(state, packages) diff --git a/mkosi/distributions/centos.py b/mkosi/distributions/centos.py index 4115d1147..99952afbc 100644 --- a/mkosi/distributions/centos.py +++ b/mkosi/distributions/centos.py @@ -71,23 +71,18 @@ class CentosInstaller(DistributionInstaller): packages = state.config.packages.copy() add_packages(state.config, packages, "systemd", "rpm") - if not state.do_run_build_script: - if state.config.bootable: - add_packages(state.config, packages, "kernel", "dracut", "dracut-config-generic") - add_packages(state.config, packages, "systemd-udev", conditional="systemd") - if state.config.ssh: - add_packages(state.config, packages, "openssh-server") - - if state.do_run_build_script: - packages += state.config.build_packages + if state.config.bootable: + add_packages(state.config, packages, "kernel", "dracut", "dracut-config-generic") + add_packages(state.config, packages, "systemd-udev", conditional="systemd") + if state.config.ssh: + add_packages(state.config, packages, "openssh-server") if "epel" in state.config.repositories: add_packages(state.config, packages, "epel-release") - if not state.do_run_build_script: - if state.config.netdev: - add_packages(state.config, packages, "systemd-networkd", conditional="systemd") - if state.config.distribution != Distribution.centos and release >= 9: - add_packages(state.config, packages, "systemd-boot", conditional="systemd") + if state.config.netdev: + add_packages(state.config, packages, "systemd-networkd", conditional="systemd") + if state.config.distribution != Distribution.centos and release >= 9: + add_packages(state.config, packages, "systemd-boot", conditional="systemd") # Make sure we only install the minimal language files by default on CentOS Stream 8 which still # defaults to all langpacks. diff --git a/mkosi/distributions/debian.py b/mkosi/distributions/debian.py index 357d61b56..4e52eca5c 100644 --- a/mkosi/distributions/debian.py +++ b/mkosi/distributions/debian.py @@ -96,14 +96,11 @@ class DebianInstaller(DistributionInstaller): packages = state.config.packages.copy() add_packages(state.config, packages, "systemd", "systemd-sysv", "dbus", "libpam-systemd") - if state.do_run_build_script: - packages += state.config.build_packages - - if not state.do_run_build_script and state.config.bootable: + if state.config.bootable: add_packages(state.config, packages, "dracut", "dracut-config-generic") cls._add_default_kernel_package(state, packages) - if not state.do_run_build_script and state.config.ssh: + if state.config.ssh: add_packages(state.config, packages, "openssh-server") # Debian policy is to start daemons by default. The policy-rc.d script can be used choose which ones to @@ -136,7 +133,7 @@ class DebianInstaller(DistributionInstaller): with dpkg_nodoc_conf.open("w") as f: f.writelines(f"path-exclude {d}/*\n" for d in doc_paths) - if not state.do_run_build_script and state.config.bootable and state.config.base_image is None: + if state.config.bootable and state.config.base_image is None: # systemd-boot won't boot unified kernel images generated without a BUILD_ID or VERSION_ID in # /etc/os-release. Build one with the mtime of os-release if we don't find them. with state.root.joinpath("etc/os-release").open("r+") as f: @@ -154,9 +151,11 @@ class DebianInstaller(DistributionInstaller): invoke_apt(state, "get", "update", ["--assume-yes"]) - if state.config.bootable and not state.do_run_build_script: + if state.config.bootable: # Ensure /efi exists so that the ESP is mounted there, and we never run dpkg -i on vfat state.root.joinpath("efi").mkdir(mode=0o755) + + if state.config.bootable: add_apt_package_if_exists(state, packages, "systemd-boot") # systemd-resolved was split into a separate package @@ -192,10 +191,9 @@ class DebianInstaller(DistributionInstaller): state.root.joinpath("etc/default/locale").symlink_to("../locale.conf") # Don't enable any services by default. - if not state.do_run_build_script: - presetdir = state.root / "etc/systemd/system-preset" - presetdir.mkdir(exist_ok=True, mode=0o755) - presetdir.joinpath("99-mkosi-disable.preset").write_text("disable *") + presetdir = state.root / "etc/systemd/system-preset" + presetdir.mkdir(exist_ok=True, mode=0o755) + presetdir.joinpath("99-mkosi-disable.preset").write_text("disable *") @classmethod def install_packages(cls, state: MkosiState, packages: Sequence[str]) -> None: diff --git a/mkosi/distributions/fedora.py b/mkosi/distributions/fedora.py index dfbad2fe6..7310d53ea 100644 --- a/mkosi/distributions/fedora.py +++ b/mkosi/distributions/fedora.py @@ -102,12 +102,10 @@ def install_fedora(state: MkosiState) -> None: packages = state.config.packages.copy() add_packages(state.config, packages, "systemd", "util-linux", "rpm") - if not state.do_run_build_script and state.config.bootable: + if state.config.bootable: add_packages(state.config, packages, "kernel-core", "kernel-modules", "dracut", "dracut-config-generic") add_packages(state.config, packages, "systemd-udev", conditional="systemd") - if state.do_run_build_script: - packages += state.config.build_packages - if not state.do_run_build_script and state.config.netdev: + if state.config.netdev: add_packages(state.config, packages, "systemd-networkd", conditional="systemd") if state.config.ssh: add_packages(state.config, packages, "openssh-server") diff --git a/mkosi/distributions/gentoo.py b/mkosi/distributions/gentoo.py index 0b49252c0..0ef891eae 100644 --- a/mkosi/distributions/gentoo.py +++ b/mkosi/distributions/gentoo.py @@ -324,8 +324,6 @@ class Gentoo: self.invoke_emerge(actions=["--depclean"]) def merge_user_pkgs(self) -> None: - if self.state.do_run_build_script: - self.invoke_emerge(pkgs=self.state.config.build_packages) if self.state.config.packages: self.invoke_emerge(pkgs=self.state.config.packages) diff --git a/mkosi/distributions/mageia.py b/mkosi/distributions/mageia.py index 284022d9b..b4b51294f 100644 --- a/mkosi/distributions/mageia.py +++ b/mkosi/distributions/mageia.py @@ -63,7 +63,7 @@ def install_mageia(state: MkosiState) -> None: packages = state.config.packages.copy() add_packages(state.config, packages, "basesystem-minimal", "dnf") - if not state.do_run_build_script and state.config.bootable: + if state.config.bootable: add_packages(state.config, packages, "kernel-server-latest", "dracut") # Mageia ships /etc/50-mageia.conf that omits systemd from the initramfs and disables hostonly. # We override that again so our defaults get applied correctly on Mageia as well. @@ -74,9 +74,6 @@ def install_mageia(state: MkosiState) -> None: if state.config.ssh: add_packages(state.config, packages, "openssh-server") - if state.do_run_build_script: - packages += state.config.build_packages - invoke_dnf(state, "install", packages) disable_pam_securetty(state.root) diff --git a/mkosi/distributions/openmandriva.py b/mkosi/distributions/openmandriva.py index 1c90feecb..a0854514d 100644 --- a/mkosi/distributions/openmandriva.py +++ b/mkosi/distributions/openmandriva.py @@ -65,7 +65,7 @@ def install_openmandriva(state: MkosiState) -> None: packages = state.config.packages.copy() # well we may use basesystem here, but that pulls lot of stuff add_packages(state.config, packages, "basesystem-minimal", "systemd", "dnf") - if not state.do_run_build_script and state.config.bootable: + if state.config.bootable: add_packages(state.config, packages, "systemd-boot", "systemd-cryptsetup", conditional="systemd") add_packages(state.config, packages, "kernel-release-server", "dracut", "timezone") if state.config.netdev: @@ -73,7 +73,4 @@ def install_openmandriva(state: MkosiState) -> None: if state.config.ssh: add_packages(state.config, packages, "openssh-server") - if state.do_run_build_script: - packages += state.config.build_packages - invoke_dnf(state, "install", packages) diff --git a/mkosi/distributions/opensuse.py b/mkosi/distributions/opensuse.py index c4d895c61..6d2bf6049 100644 --- a/mkosi/distributions/opensuse.py +++ b/mkosi/distributions/opensuse.py @@ -155,16 +155,13 @@ def install_opensuse(state: MkosiState) -> None: else: add_packages(state.config, packages, "patterns-base-minimal_base") - if not state.do_run_build_script and state.config.bootable: + if state.config.bootable: add_packages(state.config, packages, "kernel-default", "dracut") if state.config.netdev: add_packages(state.config, packages, "systemd-network") - if state.do_run_build_script: - packages += state.config.build_packages - - if not state.do_run_build_script and state.config.ssh: + if state.config.ssh: add_packages(state.config, packages, "openssh-server") zypper_install(state, packages) @@ -201,7 +198,7 @@ def install_opensuse(state: MkosiState) -> None: shutil.copy2(state.root / f"usr/{prefix}/pam.d/login", state.root / "etc/pam.d/login") break - if state.config.bootable and not state.do_run_build_script: + if state.config.bootable: dracut_dir = state.root / "etc/dracut.conf.d" dracut_dir.mkdir(mode=0o755, exist_ok=True) diff --git a/mkosi/mounts.py b/mkosi/mounts.py index 5fecc91d1..0a1b0cb38 100644 --- a/mkosi/mounts.py +++ b/mkosi/mounts.py @@ -89,7 +89,7 @@ def mount_overlay( workdir: Path, where: Path ) -> Iterator[Path]: - options = [f'lowerdir={lower}', f'upperdir={upper}', f'workdir={workdir}'] + options = [f"lowerdir={lower}", f"upperdir={upper}", f"workdir={workdir}", "userxattr"] try: with mount("overlay", where, options=options, type="overlay"): diff --git a/mkosi/run.py b/mkosi/run.py index ca7e38713..b13586e28 100644 --- a/mkosi/run.py +++ b/mkosi/run.py @@ -336,4 +336,5 @@ def run_workspace_command( die(f"\"{shlex.join(str(s) for s in cmd)}\" returned non-zero exit code {e.returncode}.") finally: if state.workspace.joinpath("resolv.conf").is_symlink(): + resolve.unlink(missing_ok=True) shutil.move(state.workspace.joinpath("resolv.conf"), resolve)