From 8320c48616ef491eee5c081f441d8b60aaf34f9d Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Sun, 2 Jul 2023 23:24:13 +0200 Subject: [PATCH] Add --tools-tree= option Currently, mkosi image builds can differ depending on the host they were built from. This can happen because we execute all kinds of binaries to build the image and depending on the host these binaries can differ. Usually, it's different versions of tools causing issues, but it can also be due to different build configurations, such as rpm writing its database in a different format depending on whether it's executed from CentOS, Fedora, or Opensuse. To allow for more reproducibility in image builds regardless of the host system, this commit adds a new option --tools-tree= that allows specifying a tree in which we look up most of the programs that we execute during an image build. Of course, that still leaves the question of what tree should be passed to --tools-tree=. To solve that problem, --tools-tree= can be used together with presets, so that as the first preset, a "bootstrap" image can be built which can then be used with --tools-tree= in later presets. Note that we only use /usr from the given tree. If tools end up using config files from /etc or such, we expect those tools to expose a knob to specify a different configuration file (instead of us overmounting /etc). Note that in a few cases, we don't yet execute tools in the given tree: - systemd-analyze in GenericVersion() can't be executed in the tree because it could be executed during config parsing when we don't know the tree to use yet. - newuidmap/newgidmap have to be executed before we can run bubblewrap so we can't run them in bubblewrap itself - Figuring out the credentials is inherently tied to the host system so we execute all scripts and tools to figure out credentials on the host system as well - mount because bubblewrap does not propagate mounts to the real root so any mounts we do within bubblewrap don't survive the bubblewrap process - systemd-dissect for the same reason --- mkosi.md | 8 + mkosi/__init__.py | 273 +++++++++++++++++++------------- mkosi/btrfs.py | 25 +-- mkosi/config.py | 12 +- mkosi/distributions/arch.py | 5 +- mkosi/distributions/debian.py | 11 +- mkosi/distributions/fedora.py | 6 +- mkosi/distributions/gentoo.py | 25 ++- mkosi/distributions/opensuse.py | 7 +- mkosi/install.py | 7 +- mkosi/manifest.py | 51 +++--- mkosi/mounts.py | 3 + mkosi/qemu.py | 22 +-- mkosi/run.py | 48 ++++-- 14 files changed, 301 insertions(+), 202 deletions(-) diff --git a/mkosi.md b/mkosi.md index a097ab2fc..e1a4c10ca 100644 --- a/mkosi.md +++ b/mkosi.md @@ -1032,6 +1032,14 @@ they should be specified with a boolean argument: either "1", "yes", or "true" t : If specified, ACLs will be set on any generated root filesystem directories that allow the user running mkosi to remove them without needing privileges. +`ToolsTree=`, `--tools-tree=` + +: If specified, programs executed by mkosi are looked up inside the given tree instead of in the host system + (aside from a few exceptions). Use this option to make image builds more reproducible by always using the + same versions of programs to build the final image instead of whatever version is installed on the host + system. If this option is not used, but the `mkosi.tools/` directory is found in the local directory it is + automatically used for this purpose with the root directory as target. + ### Commandline-only Options Those settings cannot be configured in the configuration files. diff --git a/mkosi/__init__.py b/mkosi/__init__.py index 8ec89c995..44903e59f 100644 --- a/mkosi/__init__.py +++ b/mkosi/__init__.py @@ -37,7 +37,15 @@ from mkosi.mounts import mount_overlay, scandir_recursive from mkosi.pager import page from mkosi.qemu import copy_ephemeral, machine_cid, run_qemu from mkosi.remove import unlink_try_hard -from mkosi.run import become_root, fork_and_wait, run, run_workspace_command, spawn +from mkosi.run import ( + become_root, + bwrap, + bwrap_cmd, + fork_and_wait, + run, + run_workspace_command, + spawn, +) from mkosi.state import MkosiState from mkosi.types import PathString from mkosi.util import ( @@ -76,6 +84,7 @@ def mount_image(state: MkosiState) -> Iterator[None]: shutil.unpack_archive(path, d) bases += [d] elif path.suffix == ".raw": + # We want to use bwrap() here but it doesn't propagate mounts so we use run() instead. run(["systemd-dissect", "-M", path, d]) stack.callback(lambda: run(["systemd-dissect", "-U", d])) bases += [d] @@ -282,7 +291,7 @@ def mount_build_overlay(state: MkosiState, read_only: bool = False) -> ContextMa d = state.workspace / "build-overlay" if not d.is_symlink(): d.mkdir(mode=0o755, exist_ok=True) - return mount_overlay([state.root], state.workspace.joinpath("build-overlay"), state.root, read_only) + return mount_overlay([state.root], state.workspace / "build-overlay", state.root, read_only) def run_prepare_script(state: MkosiState, build: bool) -> None: @@ -346,19 +355,20 @@ def run_finalize_script(state: MkosiState) -> None: return with complete_step("Running finalize script…"): - run([state.config.finalize_script], - env={**state.environment, "BUILDROOT": str(state.root), "OUTPUTDIR": str(state.staging)}) + bwrap([state.config.finalize_script], + root=state.config.tools_tree, + env={**state.environment, "BUILDROOT": str(state.root), "OUTPUTDIR": str(state.staging)}) -def certificate_common_name(certificate: Path) -> str: - output = run([ +def certificate_common_name(state: MkosiState, certificate: Path) -> str: + output = bwrap([ "openssl", "x509", "-noout", "-subject", "-nameopt", "multiline", "-in", certificate, - ], text=True, stdout=subprocess.PIPE).stdout + ], root=state.config.tools_tree, stdout=subprocess.PIPE).stdout for line in output.splitlines(): if not line.strip().startswith("commonName"): @@ -385,23 +395,25 @@ def pesign_prepare(state: MkosiState) -> None: # pesign takes a certificate directory and a certificate common name as input arguments, so we have # to transform our input key and cert into that format. Adapted from # https://www.mankier.com/1/pesign#Examples-Signing_with_the_certificate_and_private_key_in_individual_files - run(["openssl", - "pkcs12", - "-export", - # Arcane incantation to create a pkcs12 certificate without a password. - "-keypbe", "NONE", - "-certpbe", "NONE", - "-nomaciter", - "-passout", "pass:", - "-out", state.workspace / "secure-boot.p12", - "-inkey", state.config.secure_boot_key, - "-in", state.config.secure_boot_certificate]) - - run(["pk12util", - "-K", "", - "-W", "", - "-i", state.workspace / "secure-boot.p12", - "-d", state.workspace / "pesign"]) + bwrap(["openssl", + "pkcs12", + "-export", + # Arcane incantation to create a pkcs12 certificate without a password. + "-keypbe", "NONE", + "-certpbe", "NONE", + "-nomaciter", + "-passout", "pass:", + "-out", state.workspace / "secure-boot.p12", + "-inkey", state.config.secure_boot_key, + "-in", state.config.secure_boot_certificate], + root=state.config.tools_tree) + + bwrap(["pk12util", + "-K", "", + "-W", "", + "-i", state.workspace / "secure-boot.p12", + "-d", state.workspace / "pesign"], + root=state.config.tools_tree) def install_boot_loader(state: MkosiState) -> None: @@ -437,27 +449,30 @@ def install_boot_loader(state: MkosiState) -> None: if (state.config.secure_boot_sign_tool == SecureBootSignTool.sbsign or state.config.secure_boot_sign_tool == SecureBootSignTool.auto and shutil.which("sbsign") is not None): - run(["sbsign", - "--key", state.config.secure_boot_key, - "--cert", state.config.secure_boot_certificate, - "--output", output, - input]) + bwrap(["sbsign", + "--key", state.config.secure_boot_key, + "--cert", state.config.secure_boot_certificate, + "--output", output, + input], + root=state.config.tools_tree) elif (state.config.secure_boot_sign_tool == SecureBootSignTool.pesign or state.config.secure_boot_sign_tool == SecureBootSignTool.auto and shutil.which("pesign") is not None): pesign_prepare(state) - run(["pesign", - "--certdir", state.workspace / "pesign", - "--certificate", certificate_common_name(state.config.secure_boot_certificate), - "--sign", - "--force", - "--in", input, - "--out", output]) + bwrap(["pesign", + "--certdir", state.workspace / "pesign", + "--certificate", certificate_common_name(state, state.config.secure_boot_certificate), + "--sign", + "--force", + "--in", input, + "--out", output], + root=state.config.tools_tree) else: die("One of sbsign or pesign is required to use SecureBoot=") with complete_step("Installing boot loader…"): - run(["bootctl", "install", "--root", state.root, "--all-architectures"], env={"SYSTEMD_ESP_PATH": "/efi"}) + bwrap(["bootctl", "install", "--root", state.root, "--all-architectures"], + env={"SYSTEMD_ESP_PATH": "/efi"}, root=state.config.tools_tree) if state.config.secure_boot: assert state.config.secure_boot_key @@ -468,27 +483,30 @@ def install_boot_loader(state: MkosiState) -> None: keys.mkdir(parents=True, exist_ok=True) # sbsiglist expects a DER certificate. - run(["openssl", - "x509", - "-outform", "DER", - "-in", state.config.secure_boot_certificate, - "-out", state.workspace / "mkosi.der"]) - run(["sbsiglist", - "--owner", str(uuid.uuid4()), - "--type", "x509", - "--output", state.workspace / "mkosi.esl", - state.workspace / "mkosi.der"]) + bwrap(["openssl", + "x509", + "-outform", "DER", + "-in", state.config.secure_boot_certificate, + "-out", state.workspace / "mkosi.der"], + root=state.config.tools_tree) + bwrap(["sbsiglist", + "--owner", str(uuid.uuid4()), + "--type", "x509", + "--output", state.workspace / "mkosi.esl", + state.workspace / "mkosi.der"], + root=state.config.tools_tree) # We reuse the key for all secure boot databases to keep things simple. for db in ["PK", "KEK", "db"]: - run(["sbvarsign", - "--attr", - "NON_VOLATILE,BOOTSERVICE_ACCESS,RUNTIME_ACCESS,TIME_BASED_AUTHENTICATED_WRITE_ACCESS", - "--key", state.config.secure_boot_key, - "--cert", state.config.secure_boot_certificate, - "--output", keys / f"{db}.auth", - db, - state.workspace / "mkosi.esl"]) + bwrap(["sbvarsign", + "--attr", + "NON_VOLATILE,BOOTSERVICE_ACCESS,RUNTIME_ACCESS,TIME_BASED_AUTHENTICATED_WRITE_ACCESS", + "--key", state.config.secure_boot_key, + "--cert", state.config.secure_boot_certificate, + "--output", keys / f"{db}.auth", + db, + state.workspace / "mkosi.esl"], + root=state.config.tools_tree) def install_base_trees(state: MkosiState) -> None: @@ -502,7 +520,8 @@ def install_base_trees(state: MkosiState) -> None: elif path.suffix == ".tar": shutil.unpack_archive(path, state.root) elif path.suffix == ".raw": - run(["systemd-dissect", "--copy-from", path, "/", state.root]) + bwrap(["systemd-dissect", "--copy-from", path, "/", state.root], + root=state.config.tools_tree) else: die(f"Unsupported base tree source {path}") @@ -520,7 +539,7 @@ def install_skeleton_trees(state: MkosiState) -> None: t.parent.mkdir(mode=0o755, parents=True, exist_ok=True) if source.is_dir() or target: - copy_path(source, t, preserve_owner=False) + copy_path(source, t, preserve_owner=False, root=state.config.tools_tree) else: shutil.unpack_archive(source, t) @@ -538,7 +557,7 @@ def install_package_manager_trees(state: MkosiState) -> None: t.parent.mkdir(mode=0o755, parents=True, exist_ok=True) if source.is_dir() or target: - copy_path(source, t, preserve_owner=False) + copy_path(source, t, preserve_owner=False, root=state.config.tools_tree) else: shutil.unpack_archive(source, t) @@ -556,7 +575,7 @@ def install_extra_trees(state: MkosiState) -> None: t.parent.mkdir(mode=0o755, parents=True, exist_ok=True) if source.is_dir() or target: - copy_path(source, t, preserve_owner=False) + copy_path(source, t, preserve_owner=False, root=state.config.tools_tree) else: shutil.unpack_archive(source, t) @@ -566,7 +585,7 @@ def install_build_dest(state: MkosiState) -> None: return with complete_step("Copying in build tree…"): - copy_path(state.install_dir, state.root) + copy_path(state.install_dir, state.root, root=state.config.tools_tree) def gzip_binary() -> str: @@ -599,7 +618,7 @@ def make_tar(state: MkosiState) -> None: ] with complete_step("Creating archive…"): - run(cmd) + bwrap(cmd, root=state.config.tools_tree) def find_files(dir: Path, root: Path) -> Iterator[Path]: @@ -612,13 +631,21 @@ def make_initrd(state: MkosiState) -> None: if state.config.output_format != OutputFormat.cpio: return - make_cpio(state.root, find_files(state.root, state.root), state.staging / state.config.output_with_format) + make_cpio(state, find_files(state.root, state.root), state.staging / state.config.output_with_format) -def make_cpio(root: Path, files: Iterator[Path], output: Path) -> None: - with complete_step(f"Creating cpio {output}…"): +def make_cpio(state: MkosiState, files: Iterator[Path], output: Path) -> None: + with complete_step(f"Creating cpio {output}…"), bwrap_cmd(root=state.config.tools_tree) as bwrap: cmd: list[PathString] = [ - "cpio", "-o", "--reproducible", "--null", "-H", "newc", "--quiet", "-D", root, "-O", output + *bwrap, + "cpio", + "-o", + "--reproducible", + "--null", + "-H", "newc", + "--quiet", + "-D", state.root, + "-O", output ] with spawn(cmd, stdin=subprocess.PIPE, text=True) as cpio: @@ -685,22 +712,22 @@ def module_path_to_name(path: Path) -> str: return path.name.partition(".")[0] -def resolve_module_dependencies(root: Path, kver: str, modules: Sequence[str]) -> tuple[set[Path], set[Path]]: +def resolve_module_dependencies(state: MkosiState, kver: str, modules: Sequence[str]) -> tuple[set[Path], set[Path]]: """ Returns a tuple of lists containing the paths to the module and firmware dependencies of the given list of module names (including the given module paths themselves). The paths are returned relative to the - given root directory. + root directory. """ modulesd = Path("usr/lib/modules") / kver - builtin = set(module_path_to_name(Path(m)) for m in (root / modulesd / "modules.builtin").read_text().splitlines()) - allmodules = set((root / modulesd / "kernel").glob("**/*.ko*")) - nametofile = {module_path_to_name(m): m.relative_to(root) for m in allmodules} + builtin = set(module_path_to_name(Path(m)) for m in (state.root / modulesd / "modules.builtin").read_text().splitlines()) + allmodules = set((state.root / modulesd / "kernel").glob("**/*.ko*")) + nametofile = {module_path_to_name(m): m.relative_to(state.root) for m in allmodules} # We could run modinfo once for each module but that's slow. Luckily we can pass multiple modules to # modinfo and it'll process them all in a single go. We get the modinfo for all modules to build two maps # that map the path of the module to its module dependencies and its firmware dependencies respectively. - info = run(["modinfo", "--basedir", root, "--set-version", kver, "--null", *nametofile.keys(), *builtin], - text=True, stdout=subprocess.PIPE).stdout + info = bwrap(["modinfo", "--basedir", state.root, "--set-version", kver, "--null", *nametofile.keys(), *builtin], + stdout=subprocess.PIPE, root=state.config.tools_tree).stdout moddep = {} firmwaredep = {} @@ -716,7 +743,7 @@ def resolve_module_dependencies(root: Path, kver: str, modules: Sequence[str]) - depends += [d for d in value.strip().split(",") if d] elif key == "firmware": - firmware += [f.relative_to(root) for f in root.joinpath("usr/lib/firmware").glob(f"{value.strip()}*")] + firmware += [f.relative_to(state.root) for f in state.root.joinpath("usr/lib/firmware").glob(f"{value.strip()}*")] elif key == "name": name = value.strip() @@ -765,7 +792,7 @@ def gen_kernel_modules_initrd(state: MkosiState, kver: str) -> Path: state.config.kernel_modules_initrd_exclude) names = [module_path_to_name(m) for m in modules] - mods, firmware = resolve_module_dependencies(state.root, kver, names) + mods, firmware = resolve_module_dependencies(state, kver, names) for p in sorted(mods) + sorted(firmware): yield p @@ -785,7 +812,7 @@ def gen_kernel_modules_initrd(state: MkosiState, kver: str) -> Path: kmods = state.workspace / f"initramfs-kernel-modules-{kver}.img" with complete_step(f"Generating kernel modules initrd for kernel {kver}"): - make_cpio(state.root, files(), kmods) + make_cpio(state, files(), kmods) # Debian/Ubuntu do not compress their kernel modules, so we compress the initramfs instead. Note that # this is not ideal since the compressed kernel modules will all be decompressed on boot which @@ -807,7 +834,7 @@ def install_unified_kernel(state: MkosiState, roothash: Optional[str]) -> None: return for kver, kimg in gen_kernel_images(state): - copy_path(state.root / kimg, state.staging / state.config.output_split_kernel) + shutil.copy(state.root / kimg, state.staging / state.config.output_split_kernel) break if state.config.output_format == OutputFormat.cpio and state.config.bootable == ConfigFeature.auto: @@ -836,6 +863,7 @@ def install_unified_kernel(state: MkosiState, roothash: Optional[str]) -> None: "--repository-key-check", yes_no(state.config.repository_key_check), "--repositories", ",".join(state.config.repositories), "--package-manager-tree", ",".join(format_source_target(s, t) for s, t in state.config.package_manager_trees), + *(["--tools-tree", str(state.config.tools_tree)] if state.config.tools_tree else []), *(["--compress-output", str(state.config.compress_output)] if state.config.compress_output else []), "--with-network", yes_no(state.config.with_network), "--cache-only", yes_no(state.config.cache_only), @@ -943,7 +971,7 @@ def install_unified_kernel(state: MkosiState, roothash: Optional[str]) -> None: cmd += [ "--signtool", "pesign", "--secureboot-certificate-dir", state.workspace / "pesign", - "--secureboot-certificate-name", certificate_common_name(state.config.secure_boot_certificate), + "--secureboot-certificate-name", certificate_common_name(state, state.config.secure_boot_certificate), ] sign_expected_pcr = (state.config.sign_expected_pcr == ConfigFeature.enabled or @@ -961,10 +989,10 @@ def install_unified_kernel(state: MkosiState, roothash: Optional[str]) -> None: if state.config.kernel_modules_initrd: cmd += [gen_kernel_modules_initrd(state, kver)] - run(cmd) + bwrap(cmd, root=state.config.tools_tree) if not state.staging.joinpath(state.config.output_split_uki).exists(): - copy_path(boot_binary, state.staging / state.config.output_split_uki) + shutil.copy(boot_binary, state.staging / state.config.output_split_uki) print_output_size(boot_binary) @@ -999,7 +1027,8 @@ def maybe_compress(state: MkosiState, compression: Compression, src: Path, dst: src.unlink() # if src == dst, make sure dst doesn't truncate the src file but creates a new file. with dst.open("wb") as o: - run(compressor_command(compression), user=state.uid, group=state.gid, stdin=i, stdout=o) + bwrap(compressor_command(compression), user=state.uid, group=state.gid, stdin=i, stdout=o, + root=state.config.tools_tree) def copy_nspawn_settings(state: MkosiState) -> None: @@ -1007,7 +1036,7 @@ def copy_nspawn_settings(state: MkosiState) -> None: return None with complete_step("Copying nspawn settings file…"): - copy_path(state.config.nspawn_settings, state.staging / state.config.output_nspawn_settings) + shutil.copy(state.config.nspawn_settings, state.staging / state.config.output_nspawn_settings) def hash_file(of: TextIO, path: Path) -> None: @@ -1052,7 +1081,7 @@ def calculate_signature(state: MkosiState) -> None: state.staging / state.config.output_checksum, ] - run( + bwrap( cmdline, # Do not output warnings about keyring permissions stderr=subprocess.DEVNULL, @@ -1065,7 +1094,8 @@ def calculate_signature(state: MkosiState) -> None: 'GNUPGHOME', Path(os.environ['HOME']).joinpath('.gnupg') ) - } + }, + root=state.config.tools_tree, ) @@ -1192,6 +1222,8 @@ def check_inputs(config: MkosiConfig) -> None: for base in config.base_trees: check_tree_input(base) + check_tree_input(config.tools_tree) + for tree in (config.skeleton_trees, config.extra_trees): for item in tree: @@ -1444,7 +1476,7 @@ def process_kernel_modules(state: MkosiState, kver: str) -> None: state.config.kernel_modules_exclude) names = [module_path_to_name(m) for m in modules] - mods, firmware = resolve_module_dependencies(state.root, kver, names) + mods, firmware = resolve_module_dependencies(state, kver, names) allmodules = set(m.relative_to(state.root) for m in (state.root / modulesd).glob("**/*.ko*")) allfirmware = set(m.relative_to(state.root) for m in (state.root / "usr/lib/firmware").glob("**/*") if not m.is_dir()) @@ -1472,22 +1504,23 @@ def run_depmod(state: MkosiState) -> None: process_kernel_modules(state, kver) with complete_step(f"Running depmod for {kver}"): - run(["depmod", "--all", "--basedir", state.root, kver]) + bwrap(["depmod", "--all", "--basedir", state.root, kver], root=state.config.tools_tree) def run_sysusers(state: MkosiState) -> None: with complete_step("Generating system users"): - run(["systemd-sysusers", "--root", state.root]) + bwrap(["systemd-sysusers", "--root", state.root], root=state.config.tools_tree) def run_preset(state: MkosiState) -> None: with complete_step("Applying presets…"): - run(["systemctl", "--root", state.root, "preset-all"]) + bwrap(["systemctl", "--root", state.root, "preset-all"], root=state.config.tools_tree) def run_hwdb(state: MkosiState) -> None: with complete_step("Generating hardware database"): - run(["systemd-hwdb", "--root", state.root, "--usr", "--strict", "update"]) + bwrap(["systemd-hwdb", "--root", state.root, "--usr", "--strict", "update"], + root=state.config.tools_tree) def run_firstboot(state: MkosiState) -> None: @@ -1521,7 +1554,8 @@ def run_firstboot(state: MkosiState) -> None: return with complete_step("Applying first boot settings"): - run(["systemd-firstboot", "--root", state.root, "--force", *options]) + bwrap(["systemd-firstboot", "--root", state.root, "--force", *options], + root=state.config.tools_tree) # Initrds generally don't ship with only /usr so there's not much point in putting the credentials in # /usr/lib/credstore. @@ -1540,7 +1574,8 @@ def run_selinux_relabel(state: MkosiState) -> None: if not selinux.exists(): return - policy = run(["sh", "-c", f". {selinux} && echo $SELINUXTYPE"], text=True, stdout=subprocess.PIPE).stdout.strip() + policy = bwrap(["sh", "-c", f". {selinux} && echo $SELINUXTYPE"], + stdout=subprocess.PIPE, root=state.config.tools_tree).stdout.strip() if not policy: return @@ -1684,7 +1719,8 @@ def make_image(state: MkosiState, skip: Sequence[str] = [], split: bool = False) env[option] = value with complete_step("Generating disk image"): - output = json.loads(run(cmdline, stdout=subprocess.PIPE, env=env).stdout) + output = json.loads(bwrap(cmdline, stdout=subprocess.PIPE, env=env, + root=state.config.tools_tree).stdout) roothash = usrhash = None for p in output: @@ -1842,16 +1878,16 @@ def run_build_script(state: MkosiState) -> None: bwrap_params=bwrap, stdout=sys.stdout, env=env | state.environment) -def setfacl(root: Path, uid: int, allow: bool) -> None: - run(["setfacl", - "--physical", - "--modify" if allow else "--remove", - f"user:{uid}:rwx" if allow else f"user:{uid}", - "-"], - text=True, - # Supply files via stdin so we don't clutter --debug run output too much - input="\n".join([str(root), - *(e.path for e in cast(Iterator[os.DirEntry[str]], scandir_recursive(root)) if e.is_dir())]) +def setfacl(config: MkosiConfig, root: Path, uid: int, allow: bool) -> None: + bwrap(["setfacl", + "--physical", + "--modify" if allow else "--remove", + f"user:{uid}:rwx" if allow else f"user:{uid}", + "-"], + root=config.tools_tree, + # Supply files via stdin so we don't clutter --debug run output too much + input="\n".join([str(root), + *(e.path for e in cast(Iterator[os.DirEntry[str]], scandir_recursive(root)) if e.is_dir())]) ) @@ -1863,7 +1899,12 @@ def acl_maybe_toggle(config: MkosiConfig, root: Path, uid: int, *, always: bool) # getfacl complains about absolute paths so make sure we pass a relative one. if root.exists(): - has_acl = f"user:{uid}:rwx" in run(["getfacl", "-n", root.relative_to(Path.cwd())], stdout=subprocess.PIPE, text=True).stdout + has_acl = f"user:{uid}:rwx" in bwrap([ + "getfacl", "-n", root.relative_to(Path.cwd())], + stdout=subprocess.PIPE, + root=config.tools_tree, + ).stdout + if not has_acl and not always: yield return @@ -1873,13 +1914,13 @@ def acl_maybe_toggle(config: MkosiConfig, root: Path, uid: int, *, always: bool) try: if has_acl: with complete_step(f"Removing ACLs from {root}"): - setfacl(root, uid, allow=False) + setfacl(config, root, uid, allow=False) yield finally: if has_acl or always: with complete_step(f"Adding ACLs to {root}"): - setfacl(root, uid, allow=True) + setfacl(config, root, uid, allow=True) @contextlib.contextmanager @@ -1951,13 +1992,14 @@ def run_shell(args: MkosiArgs, config: MkosiConfig) -> None: fname = config.output_dir / config.output if config.output_format == OutputFormat.disk and args.verb == Verb.boot: - run(["systemd-repart", - "--image", fname, - "--size", "8G", - "--no-pager", - "--dry-run=no", - "--offline=no", - fname]) + bwrap(["systemd-repart", + "--image", fname, + "--size", "8G", + "--no-pager", + "--dry-run=no", + "--offline=no", + fname], + root=config.tools_tree) if config.output_format == OutputFormat.directory: cmdline += ["--directory", fname] @@ -1981,7 +2023,12 @@ def run_shell(args: MkosiArgs, config: MkosiConfig) -> None: stack.enter_context(acl_toggle_boot(config)) - run(cmdline, stdin=sys.stdin, stdout=sys.stdout, env=os.environ, log=False) + bwrap(cmdline, + stdin=sys.stdin, + stdout=sys.stdout, + env=os.environ, + log=False, + root=config.tools_tree) def run_ssh(args: MkosiArgs, config: MkosiConfig) -> None: @@ -1997,7 +2044,7 @@ def run_ssh(args: MkosiArgs, config: MkosiConfig) -> None: cmd += args.cmdline - run(cmd, stdin=sys.stdin, stdout=sys.stdout, env=os.environ, log=False) + bwrap(cmd, stdin=sys.stdin, stdout=sys.stdout, env=os.environ, log=False, root=config.tools_tree) def run_serve(config: MkosiConfig) -> None: diff --git a/mkosi/btrfs.py b/mkosi/btrfs.py index 32d95d279..552a638cf 100644 --- a/mkosi/btrfs.py +++ b/mkosi/btrfs.py @@ -8,18 +8,19 @@ from typing import cast from mkosi.config import ConfigFeature, MkosiConfig from mkosi.install import copy_path from mkosi.log import die -from mkosi.run import run +from mkosi.run import bwrap -def statfs(path: Path) -> str: - return cast(str, run(["stat", "--file-system", "--format", "%T", path.parent], text=True, stdout=subprocess.PIPE).stdout.strip()) +def statfs(config: MkosiConfig, path: Path) -> str: + return cast(str, bwrap(["stat", "--file-system", "--format", "%T", path.parent], + root=config.tools_tree, stdout=subprocess.PIPE).stdout.strip()) def btrfs_maybe_make_subvolume(config: MkosiConfig, path: Path, mode: int) -> None: if config.use_subvolumes == ConfigFeature.enabled and not shutil.which("btrfs"): die("Subvolumes requested but the btrfs command was not found") - if statfs(path.parent) != "btrfs": + if statfs(config, path.parent) != "btrfs": if config.use_subvolumes == ConfigFeature.enabled: die(f"Subvolumes requested but {path} is not located on a btrfs filesystem") @@ -27,8 +28,9 @@ def btrfs_maybe_make_subvolume(config: MkosiConfig, path: Path, mode: int) -> No return if config.use_subvolumes != ConfigFeature.disabled and shutil.which("btrfs") is not None: - result = run(["btrfs", "subvolume", "create", path], - check=config.use_subvolumes == ConfigFeature.enabled).returncode + result = bwrap(["btrfs", "subvolume", "create", path], + check=config.use_subvolumes == ConfigFeature.enabled, + root=config.tools_tree).returncode else: result = 1 @@ -46,18 +48,19 @@ def btrfs_maybe_snapshot_subvolume(config: MkosiConfig, src: Path, dst: Path) -> die("Subvolumes requested but the btrfs command was not found") # Subvolumes always have inode 256 so we can use that to check if a directory is a subvolume. - if not subvolume or statfs(src) != "btrfs" or src.stat().st_ino != 256 or (dst.exists() and any(dst.iterdir())): - return copy_path(src, dst) + if not subvolume or statfs(config, src) != "btrfs" or src.stat().st_ino != 256 or (dst.exists() and any(dst.iterdir())): + return copy_path(src, dst, root=config.tools_tree) # btrfs can't snapshot to an existing directory so make sure the destination does not exist. if dst.exists(): dst.rmdir() if shutil.which("btrfs"): - result = run(["btrfs", "subvolume", "snapshot", src, dst], - check=config.use_subvolumes == ConfigFeature.enabled).returncode + result = bwrap(["btrfs", "subvolume", "snapshot", src, dst], + check=config.use_subvolumes == ConfigFeature.enabled, + root=config.tools_tree).returncode else: result = 1 if result != 0: - copy_path(src, dst) + copy_path(src, dst, root=config.tools_tree) diff --git a/mkosi/config.py b/mkosi/config.py index e1b00364e..c47929ae1 100644 --- a/mkosi/config.py +++ b/mkosi/config.py @@ -658,6 +658,7 @@ class MkosiConfig: hostname: Optional[str] root_password: Optional[tuple[str, bool]] root_shell: Optional[str] + tools_tree: Optional[Path] # QEMU-specific options qemu_gui: bool @@ -1333,7 +1334,7 @@ class MkosiConfigParser: metavar="PATH", section="Host", parse=config_make_list_parser(delimiter=",", parse=make_path_parser()), - help="List of colon-separated paths to look for programs before looking in PATH", + help="List of comma-separated paths to look for programs before looking in PATH", ), MkosiConfigSetting( dest="qemu_gui", @@ -1430,6 +1431,15 @@ class MkosiConfigParser: parse=config_parse_boolean, help="Set ACLs on generated directories to permit the user running mkosi to remove them", ), + MkosiConfigSetting( + dest="tools_tree", + long="--tools-tree", + metavar="PATH", + section="Host", + parse=config_make_path_parser(required=False, absolute=False), + paths=("mkosi.tools",), + help="Look up programs to execute inside the given tree", + ), ) MATCHES = ( diff --git a/mkosi/distributions/arch.py b/mkosi/distributions/arch.py index c669d5f4f..7f2cecdd2 100644 --- a/mkosi/distributions/arch.py +++ b/mkosi/distributions/arch.py @@ -128,4 +128,7 @@ def invoke_pacman(state: MkosiState, packages: Sequence[str], apivfs: bool = Tru if state.config.bootable != ConfigFeature.disabled: cmdline += ["--assume-installed", "initramfs"] - bwrap(cmdline, apivfs=state.root if apivfs else None, env=dict(KERNEL_INSTALL_BYPASS="1") | state.environment) + bwrap(cmdline, + apivfs=state.root if apivfs else None, + env=dict(KERNEL_INSTALL_BYPASS="1") | state.environment, + root=state.config.tools_tree) diff --git a/mkosi/distributions/debian.py b/mkosi/distributions/debian.py index a792da12a..7a5097f50 100644 --- a/mkosi/distributions/debian.py +++ b/mkosi/distributions/debian.py @@ -9,7 +9,7 @@ from textwrap import dedent from mkosi.architecture import Architecture from mkosi.distributions import DistributionInstaller from mkosi.log import die -from mkosi.run import bwrap, run +from mkosi.run import bwrap from mkosi.state import MkosiState from mkosi.types import CompletedProcess, PathString @@ -93,8 +93,9 @@ class DebianInstaller(DistributionInstaller): for deb in essential: with tempfile.NamedTemporaryFile(dir=state.workspace) as f: - run(["dpkg-deb", "--fsys-tarfile", deb], stdout=f) - run(["tar", "-C", state.root, "--keep-directory-symlink", "--extract", "--file", f.name]) + bwrap(["dpkg-deb", "--fsys-tarfile", deb], stdout=f, root=state.config.tools_tree) + bwrap(["tar", "-C", state.root, "--keep-directory-symlink", "--extract", "--file", f.name], + root=state.config.tools_tree) # Finally, run apt to properly install packages in the chroot without having to worry that maintainer # scripts won't find basic tools that they depend on. @@ -244,7 +245,9 @@ def invoke_apt( ] return bwrap(["apt-get", *options, operation, *extra], - apivfs=state.root if apivfs else None, env=env | state.environment) + apivfs=state.root if apivfs else None, + env=env | state.environment, + root=state.config.tools_tree) def install_apt_sources(state: MkosiState, repos: Sequence[str]) -> None: diff --git a/mkosi/distributions/fedora.py b/mkosi/distributions/fedora.py index 2f1330a2f..4610ae02e 100644 --- a/mkosi/distributions/fedora.py +++ b/mkosi/distributions/fedora.py @@ -225,8 +225,10 @@ def invoke_dnf( cmdline += sort_packages(packages) - bwrap(cmdline, apivfs=state.root if apivfs else None, - env=dict(KERNEL_INSTALL_BYPASS="1") | env | state.environment) + bwrap(cmdline, + apivfs=state.root if apivfs else None, + env=dict(KERNEL_INSTALL_BYPASS="1") | env | state.environment, + root=state.config.tools_tree) fixup_rpmdb_location(state.root) diff --git a/mkosi/distributions/gentoo.py b/mkosi/distributions/gentoo.py index 5bbe3d262..0ded5e53c 100644 --- a/mkosi/distributions/gentoo.py +++ b/mkosi/distributions/gentoo.py @@ -12,7 +12,7 @@ from mkosi.distributions import DistributionInstaller from mkosi.install import copy_path from mkosi.log import ARG_DEBUG, complete_step, die from mkosi.remove import unlink_try_hard -from mkosi.run import run, run_workspace_command +from mkosi.run import bwrap, run_workspace_command from mkosi.state import MkosiState from mkosi.types import PathString @@ -117,7 +117,7 @@ class GentooInstaller(DistributionInstaller): if stage3_tar.exists(): cmd += ["--time-cond", stage3_tar] - run(cmd) + bwrap(cmd, root=state.config.tools_tree) if stage3_tar.stat().st_mtime > old: unlink_try_hard(stage3) @@ -126,21 +126,20 @@ class GentooInstaller(DistributionInstaller): if not any(stage3.iterdir()): with complete_step(f"Extracting {stage3_tar.name} to {stage3}"): - run([ - "tar", - "--numeric-owner", - "-C", stage3, - "--extract", - "--file", stage3_tar, - "--exclude", "./dev/*", - "--exclude", "./proc/*", - "--exclude", "./sys/*", - ]) + bwrap(["tar", + "--numeric-owner", + "-C", stage3, + "--extract", + "--file", stage3_tar, + "--exclude", "./dev/*", + "--exclude", "./proc/*", + "--exclude", "./sys/*"], + root=state.config.tools_tree) for d in ("binpkgs", "distfiles", "repos/gentoo"): (state.cache_dir / d).mkdir(parents=True, exist_ok=True) - copy_path(state.pkgmngr, stage3, preserve_owner=False) + copy_path(state.pkgmngr, stage3, preserve_owner=False, root=state.config.tools_tree) run_workspace_command( stage3, diff --git a/mkosi/distributions/opensuse.py b/mkosi/distributions/opensuse.py index 881cb808d..d6f4fc7e2 100644 --- a/mkosi/distributions/opensuse.py +++ b/mkosi/distributions/opensuse.py @@ -139,9 +139,10 @@ def invoke_zypper( *packages, ] - env = dict(ZYPP_CONF=str(state.pkgmngr / "etc/zypp/zypp.conf"), KERNEL_INSTALL_BYPASS="1") | state.environment - - bwrap(cmdline, apivfs=state.root if apivfs else None, env=env) + bwrap(cmdline, + apivfs=state.root if apivfs else None, + env=dict(ZYPP_CONF=str(state.pkgmngr / "etc/zypp/zypp.conf"), KERNEL_INSTALL_BYPASS="1") | state.environment, + root=state.config.tools_tree) fixup_rpmdb_location(state.root) diff --git a/mkosi/install.py b/mkosi/install.py index c3187d137..106c35b90 100644 --- a/mkosi/install.py +++ b/mkosi/install.py @@ -9,7 +9,7 @@ from collections.abc import Iterator from pathlib import Path from typing import Optional -from mkosi.run import run +from mkosi.run import bwrap def make_executable(path: Path) -> None: @@ -53,8 +53,9 @@ def copy_path( *, dereference: bool = False, preserve_owner: bool = True, + root: Optional[Path] = None, ) -> None: - run([ + bwrap([ "cp", "--recursive", f"--{'' if dereference else 'no-'}dereference", @@ -62,4 +63,4 @@ def copy_path( "--no-target-directory", "--reflink=auto", src, dst, - ]) + ], root=root) diff --git a/mkosi/manifest.py b/mkosi/manifest.py index 4e95bbec4..a0a5de155 100644 --- a/mkosi/manifest.py +++ b/mkosi/manifest.py @@ -9,7 +9,7 @@ from textwrap import dedent from typing import IO, Any, Optional from mkosi.config import MkosiConfig -from mkosi.run import run +from mkosi.run import bwrap from mkosi.util import Distribution, ManifestFormat, PackageType @@ -105,13 +105,13 @@ class Manifest: if not (root / dbpath).exists(): dbpath = "/var/lib/rpm" - c = run(["rpm", - f"--root={root}", - f"--dbpath={dbpath}", - "-qa", - "--qf", r"%{NEVRA}\t%{SOURCERPM}\t%{NAME}\t%{ARCH}\t%{LONGSIZE}\t%{INSTALLTIME}\n"], - stdout=PIPE, - text=True) + c = bwrap(["rpm", + f"--root={root}", + f"--dbpath={dbpath}", + "-qa", + "--qf", r"%{NEVRA}\t%{SOURCERPM}\t%{NAME}\t%{ARCH}\t%{LONGSIZE}\t%{INSTALLTIME}\n"], + stdout=PIPE, + root=self.config.tools_tree) packages = sorted(c.stdout.splitlines()) @@ -146,15 +146,15 @@ class Manifest: source = self.source_packages.get(srpm) if source is None: - c = run(["rpm", - f"--root={root}", - f"--dbpath={dbpath}", - "-q", - "--changelog", - nevra], - stdout=PIPE, - stderr=DEVNULL, - text=True) + c = bwrap(["rpm", + f"--root={root}", + f"--dbpath={dbpath}", + "-q", + "--changelog", + nevra], + stdout=PIPE, + stderr=DEVNULL, + root=self.config.tools_tree) changelog = c.stdout.strip() source = SourcePackageManifest(srpm, changelog) self.source_packages[srpm] = source @@ -162,14 +162,13 @@ class Manifest: source.add(package) def record_deb_packages(self, root: Path) -> None: - c = run(["dpkg-query", - f"--admindir={root}/var/lib/dpkg", - "--show", - "--showformat", - r'${Package}\t${source:Package}\t${Version}\t${Architecture}\t${Installed-Size}\t${db-fsys:Last-Modified}\n'], - stdout=PIPE, - text=True, - ) + c = bwrap(["dpkg-query", + f"--admindir={root}/var/lib/dpkg", + "--show", + "--showformat", + r'${Package}\t${source:Package}\t${Version}\t${Architecture}\t${Installed-Size}\t${db-fsys:Last-Modified}\n'], + stdout=PIPE, + root=self.config.tools_tree) packages = sorted(c.stdout.splitlines()) @@ -228,7 +227,7 @@ class Manifest: # We have to run from the root, because if we use the RootDir option to make # apt from the host look at the repositories in the image, it will also pick # the 'methods' executables from there, but the ABI might not be compatible. - result = run(cmd, text=True, stdout=PIPE) + result = bwrap(cmd, stdout=PIPE, root=self.config.tools_tree) source_package = SourcePackageManifest(source, result.stdout.strip()) self.source_packages[source] = source_package diff --git a/mkosi/mounts.py b/mkosi/mounts.py index 50af271ac..30e36bb7e 100644 --- a/mkosi/mounts.py +++ b/mkosi/mounts.py @@ -78,6 +78,9 @@ def mount( if options: cmd += ["--options", ",".join(options)] + # Ideally we'd run these with bwrap() but bubblewrap disables all mount propagation to the root so any + # mounts we do within bubblewrap aren't propagated to the overarching mount namespace. + try: run(cmd) yield where diff --git a/mkosi/qemu.py b/mkosi/qemu.py index abe940102..a4dd0bb9a 100644 --- a/mkosi/qemu.py +++ b/mkosi/qemu.py @@ -18,10 +18,9 @@ from typing import Iterator, Optional from mkosi.architecture import Architecture from mkosi.btrfs import btrfs_maybe_snapshot_subvolume from mkosi.config import ConfigFeature, MkosiArgs, MkosiConfig -from mkosi.install import copy_path from mkosi.log import die from mkosi.remove import unlink_try_hard -from mkosi.run import MkosiAsyncioThread, run, spawn +from mkosi.run import MkosiAsyncioThread, bwrap, bwrap_cmd, spawn from mkosi.types import PathString from mkosi.util import ( Distribution, @@ -138,10 +137,10 @@ def find_ovmf_vars(config: MkosiConfig) -> Path: @contextlib.contextmanager -def start_swtpm() -> Iterator[Optional[Path]]: - with tempfile.TemporaryDirectory() as state: +def start_swtpm(config: MkosiConfig) -> Iterator[Optional[Path]]: + with tempfile.TemporaryDirectory() as state, bwrap_cmd(root=config.tools_tree) as bwrap: sock = Path(state) / Path("sock") - proc = spawn(["swtpm", "socket", "--tpm2", "--tpmstate", f"dir={state}", "--ctrl", f"type=unixio,path={sock}"]) + proc = spawn([*bwrap, "swtpm", "socket", "--tpm2", "--tpmstate", f"dir={state}", "--ctrl", f"type=unixio,path={sock}"]) try: yield sock @@ -262,7 +261,7 @@ def run_qemu(args: MkosiArgs, config: MkosiConfig) -> None: with contextlib.ExitStack() as stack: if fw_supports_sb: ovmf_vars = stack.enter_context(tempfile.NamedTemporaryFile(prefix=".mkosi-", dir=tmp_dir())) - copy_path(find_ovmf_vars(config), Path(ovmf_vars.name), dereference=True) + shutil.copy(find_ovmf_vars(config), Path(ovmf_vars.name)) cmdline += [ "-global", "ICH9-LPC.disable_s3=1", "-global", "driver=cfi.pflash01,property=secure,value=on", @@ -275,7 +274,7 @@ def run_qemu(args: MkosiArgs, config: MkosiConfig) -> None: fname = config.output_dir / config.output if config.output_format == OutputFormat.disk: - run(["systemd-repart", "--definitions", "", "--no-pager", "--size", "8G", "--pretty", "no", fname]) + bwrap(["systemd-repart", "--definitions", "", "--no-pager", "--size", "8G", "--pretty", "no", fname]) # Debian images fail to boot with virtio-scsi, see: https://github.com/systemd/mkosi/issues/725 if config.output_format == OutputFormat.cpio: @@ -293,7 +292,7 @@ def run_qemu(args: MkosiArgs, config: MkosiConfig) -> None: "-device", "scsi-hd,drive=hd,bootindex=1"] if config.qemu_swtpm != ConfigFeature.disabled and shutil.which("swtpm") is not None: - sock = stack.enter_context(start_swtpm()) + sock = stack.enter_context(start_swtpm(config)) cmdline += ["-chardev", f"socket,id=chrtpm,path={sock}", "-tpmdev", "emulator,id=tpm0,chardev=chrtpm"] @@ -309,7 +308,12 @@ def run_qemu(args: MkosiArgs, config: MkosiConfig) -> None: cmdline += config.qemu_args cmdline += args.cmdline - run(cmdline, stdin=sys.stdin, stdout=sys.stdout, env=os.environ, log=False) + bwrap(cmdline, + stdin=sys.stdin, + stdout=sys.stdout, + env=os.environ, + log=False, + root=config.tools_tree) if status := int(notifications.get("EXIT_STATUS", 0)): raise subprocess.CalledProcessError(status, cmdline) diff --git a/mkosi/run.py b/mkosi/run.py index a1bd008fa..cdad973c5 100644 --- a/mkosi/run.py +++ b/mkosi/run.py @@ -2,6 +2,7 @@ import asyncio import asyncio.tasks +import contextlib import ctypes import ctypes.util import logging @@ -22,6 +23,7 @@ from typing import ( Any, Awaitable, Callable, + Iterator, Mapping, Optional, Sequence, @@ -293,23 +295,22 @@ def spawn( logging.error(f"\"{' '.join(str(s) for s in cmdline)}\" returned non-zero exit code {e.returncode}.") raise e - -def bwrap( - cmd: Sequence[PathString], +@contextlib.contextmanager +def bwrap_cmd( *, + root: Optional[Path] = None, apivfs: Optional[Path] = None, - stdout: _FILE = None, - env: Mapping[str, PathString] = {}, -) -> CompletedProcess: +) -> Iterator[list[PathString]]: cmdline: list[PathString] = [ "bwrap", - # Required to make chroot detection via /proc/1/root work properly. - "--unshare-pid", "--dev-bind", "/", "/", "--chdir", Path.cwd(), "--die-with-parent", ] + if root: + cmdline += ["--bind", root / "usr", "/usr"] + if apivfs: if not (apivfs / "etc/machine-id").exists(): # Uninitialized means we want it to get initialized on first boot. @@ -347,18 +348,33 @@ def bwrap( ] try: - result = run([*cmdline, *cmd], text=True, stdout=stdout, env=env, log=False) + yield cmdline + finally: + # Clean up some stuff that might get written by package manager post install scripts. + if apivfs: + for f in ("var/lib/systemd/random-seed", "var/lib/systemd/credential.secret", "etc/machine-info"): + apivfs.joinpath(f).unlink(missing_ok=True) + + +def bwrap( + cmd: Sequence[PathString], + *, + root: Optional[Path] = None, + apivfs: Optional[Path] = None, + env: Mapping[str, PathString] = {}, + log: bool = True, + **kwargs: Any, +) -> CompletedProcess: + with bwrap_cmd(root=root, apivfs=apivfs) as bwrap: + try: + result = run([*bwrap, *cmd], text=True, env=env, log=False, **kwargs) except subprocess.CalledProcessError as e: - logging.error(f"\"{' '.join(str(s) for s in cmd)}\" returned non-zero exit code {e.returncode}.") + if log: + logging.error(f"\"{' '.join(str(s) for s in cmd)}\" returned non-zero exit code {e.returncode}.") if ARG_DEBUG_SHELL.get(): - run([*cmdline, "sh"], stdin=sys.stdin, check=False, env=env, log=False) + run([*bwrap, "sh"], stdin=sys.stdin, check=False, env=env, log=False) raise e - # Clean up some stuff that might get written by package manager post install scripts. - if apivfs: - for f in ("var/lib/systemd/random-seed", "var/lib/systemd/credential.secret", "etc/machine-info"): - apivfs.joinpath(f).unlink(missing_ok=True) - return result -- 2.47.2