From: Daan De Meyer Date: Mon, 17 Jul 2023 07:50:28 +0000 (+0200) Subject: Add our own "which" implementation X-Git-Tag: v15~76^2 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=refs%2Fpull%2F1677%2Fhead;p=thirdparty%2Fmkosi.git Add our own "which" implementation shutil.which() doesn't take into account any configured tools tree so let's implement our own which() that does take the tools tree into account. --- diff --git a/mkosi/__init__.py b/mkosi/__init__.py index 0738d7b76..55b8828d3 100644 --- a/mkosi/__init__.py +++ b/mkosi/__init__.py @@ -45,6 +45,7 @@ from mkosi.run import ( fork_and_wait, run, spawn, + which, ) from mkosi.state import MkosiState from mkosi.types import PathString @@ -438,7 +439,7 @@ def install_boot_loader(state: MkosiState) -> None: if not any(gen_kernel_images(state)) and state.config.bootable == ConfigFeature.auto: return - if not shutil.which("bootctl"): + if not which("bootctl", tools=state.config.tools_tree): if state.config.bootable == ConfigFeature.enabled: die("A bootable image was requested but bootctl was not found") return @@ -460,7 +461,7 @@ def install_boot_loader(state: MkosiState) -> None: if (state.config.secure_boot_sign_tool == SecureBootSignTool.sbsign or state.config.secure_boot_sign_tool == SecureBootSignTool.auto and - shutil.which("sbsign") is not None): + which("sbsign", state.config.tools_tree) is not None): bwrap(["sbsign", "--key", state.config.secure_boot_key, "--cert", state.config.secure_boot_certificate, @@ -469,7 +470,7 @@ def install_boot_loader(state: MkosiState) -> None: tools=state.config.tools_tree) elif (state.config.secure_boot_sign_tool == SecureBootSignTool.pesign or state.config.secure_boot_sign_tool == SecureBootSignTool.auto and - shutil.which("pesign") is not None): + which("pesign", tools=state.config.tools_tree) is not None): pesign_prepare(state) bwrap(["pesign", "--certdir", state.workspace / "pesign", @@ -600,11 +601,11 @@ def install_build_dest(state: MkosiState) -> None: copy_path(state.install_dir, state.root, tools=state.config.tools_tree) -def gzip_binary() -> str: - return "pigz" if shutil.which("pigz") else "gzip" +def gzip_binary(config: MkosiConfig) -> str: + return "pigz" if which("pigz", tools=config.tools_tree) else "gzip" -def tar_binary() -> str: +def tar_binary(config: MkosiConfig) -> str: # Some distros (Mandriva) install BSD tar as "tar", hence prefer # "gtar" if it exists, which should be GNU tar wherever it exists. # We are interested in exposing same behaviour everywhere hence @@ -612,7 +613,7 @@ def tar_binary() -> str: # everywhere. In particular given the limited/different SELinux # support in BSD tar and the different command line syntax # compared to GNU tar. - return "gtar" if shutil.which("gtar") else "tar" + return "gtar" if which("gtar", tools=config.tools_tree) else "tar" def make_tar(state: MkosiState) -> None: @@ -620,7 +621,7 @@ def make_tar(state: MkosiState) -> None: return cmd: list[PathString] = [ - tar_binary(), + tar_binary(state.config), "-C", state.root, "-c", "--xattrs", "--xattrs-include=*", @@ -951,7 +952,7 @@ def install_unified_kernel(state: MkosiState, roothash: Optional[str]) -> None: die(f"sd-stub not found at /{stub.relative_to(state.root)} in the image") cmd: list[PathString] = [ - shutil.which("ukify") or "/usr/lib/systemd/ukify", + which("ukify", tools=state.config.tools_tree) or "/usr/lib/systemd/ukify", "--cmdline", f"@{state.workspace / 'cmdline'}", "--os-release", f"@{state.root / 'usr/lib/os-release'}", "--stub", stub, @@ -988,7 +989,7 @@ def install_unified_kernel(state: MkosiState, roothash: Optional[str]) -> None: sign_expected_pcr = (state.config.sign_expected_pcr == ConfigFeature.enabled or (state.config.sign_expected_pcr == ConfigFeature.auto and - shutil.which("systemd-measure") is not None)) + which("systemd-measure", tools=state.config.tools_tree) is not None)) if sign_expected_pcr: cmd += [ @@ -1012,11 +1013,11 @@ def install_unified_kernel(state: MkosiState, roothash: Optional[str]) -> None: die("A bootable image was requested but no kernel was found") -def compressor_command(compression: Compression) -> list[PathString]: +def compressor_command(config: MkosiConfig, compression: Compression) -> list[PathString]: """Returns a command suitable for compressing archives.""" if compression == Compression.gz: - return [gzip_binary(), "--fast", "--stdout", "-"] + return [gzip_binary(config), "--fast", "--stdout", "-"] elif compression == Compression.xz: return ["xz", "--check=crc32", "--fast", "-T0", "--stdout", "-"] elif compression == Compression.zst: @@ -1039,7 +1040,7 @@ def maybe_compress(state: MkosiState, compression: Compression, src: Path, dst: src.unlink() # if src == dst, make sure dst doesn't truncate the src file but creates a new file. with dst.open("wb") as o: - bwrap(compressor_command(compression), stdin=i, stdout=o, tools=state.config.tools_tree) + bwrap(compressor_command(state.config, compression), stdin=i, stdout=o, tools=state.config.tools_tree) os.chown(dst, uid=state.uid, gid=state.gid) diff --git a/mkosi/btrfs.py b/mkosi/btrfs.py index 5b18a2f7a..56140a738 100644 --- a/mkosi/btrfs.py +++ b/mkosi/btrfs.py @@ -1,6 +1,5 @@ # SPDX-License-Identifier: LGPL-2.1+ -import shutil import subprocess from pathlib import Path from typing import cast @@ -8,7 +7,7 @@ from typing import cast from mkosi.config import ConfigFeature, MkosiConfig from mkosi.install import copy_path from mkosi.log import die -from mkosi.run import bwrap +from mkosi.run import bwrap, which def statfs(config: MkosiConfig, path: Path) -> str: @@ -17,7 +16,7 @@ def statfs(config: MkosiConfig, path: Path) -> str: def btrfs_maybe_make_subvolume(config: MkosiConfig, path: Path, mode: int) -> None: - if config.use_subvolumes == ConfigFeature.enabled and not shutil.which("btrfs"): + if config.use_subvolumes == ConfigFeature.enabled and not which("btrfs", tools=config.tools_tree): die("Subvolumes requested but the btrfs command was not found") if statfs(config, path.parent) != "btrfs": @@ -27,7 +26,7 @@ def btrfs_maybe_make_subvolume(config: MkosiConfig, path: Path, mode: int) -> No path.mkdir(mode) return - if config.use_subvolumes != ConfigFeature.disabled and shutil.which("btrfs") is not None: + if config.use_subvolumes != ConfigFeature.disabled and which("btrfs", tools=config.tools_tree) is not None: result = bwrap(["btrfs", "subvolume", "create", path], check=config.use_subvolumes == ConfigFeature.enabled, tools=config.tools_tree).returncode @@ -42,9 +41,9 @@ def btrfs_maybe_make_subvolume(config: MkosiConfig, path: Path, mode: int) -> No def btrfs_maybe_snapshot_subvolume(config: MkosiConfig, src: Path, dst: Path) -> None: subvolume = (config.use_subvolumes == ConfigFeature.enabled or - config.use_subvolumes == ConfigFeature.auto and shutil.which("btrfs") is not None) + config.use_subvolumes == ConfigFeature.auto and which("btrfs", tools=config.tools_tree) is not None) - if config.use_subvolumes == ConfigFeature.enabled and not shutil.which("btrfs"): + if config.use_subvolumes == ConfigFeature.enabled and not which("btrfs", tools=config.tools_tree): die("Subvolumes requested but the btrfs command was not found") # Subvolumes always have inode 256 so we can use that to check if a directory is a subvolume. @@ -55,7 +54,7 @@ def btrfs_maybe_snapshot_subvolume(config: MkosiConfig, src: Path, dst: Path) -> if dst.exists(): dst.rmdir() - if shutil.which("btrfs"): + if which("btrfs", config.tools_tree): result = bwrap(["btrfs", "subvolume", "snapshot", src, dst], check=config.use_subvolumes == ConfigFeature.enabled, tools=config.tools_tree).returncode diff --git a/mkosi/distributions/debian.py b/mkosi/distributions/debian.py index 36ede4b64..5938f83bf 100644 --- a/mkosi/distributions/debian.py +++ b/mkosi/distributions/debian.py @@ -1,6 +1,5 @@ # SPDX-License-Identifier: LGPL-2.1+ -import shutil import tempfile from collections.abc import Sequence from pathlib import Path @@ -9,7 +8,7 @@ from textwrap import dedent from mkosi.architecture import Architecture from mkosi.distributions import DistributionInstaller from mkosi.log import die -from mkosi.run import bwrap +from mkosi.run import bwrap, which from mkosi.state import MkosiState from mkosi.types import CompletedProcess, PathString @@ -232,7 +231,7 @@ def invoke_apt( "-o", f"Dir::Etc::trusted={trustedkeys}", "-o", f"Dir::Etc::trustedparts={trustedkeys_dir}", "-o", f"Dir::Log={state.pkgmngr / 'var/log/apt'}", - "-o", f"Dir::Bin::dpkg={shutil.which('dpkg')}", + "-o", f"Dir::Bin::dpkg={which('dpkg', tools=state.config.tools_tree)}", "-o", "Debug::NoLocking=true", "-o", f"DPkg::Options::=--root={state.root}", "-o", f"DPkg::Options::=--log={state.pkgmngr / 'var/log/apt/dpkg.log'}", diff --git a/mkosi/distributions/fedora.py b/mkosi/distributions/fedora.py index 1b2cdea78..caa76e82b 100644 --- a/mkosi/distributions/fedora.py +++ b/mkosi/distributions/fedora.py @@ -12,7 +12,7 @@ from mkosi.architecture import Architecture from mkosi.distributions import DistributionInstaller from mkosi.log import die from mkosi.remove import unlink_try_hard -from mkosi.run import bwrap +from mkosi.run import bwrap, which from mkosi.state import MkosiState from mkosi.util import Distribution, detect_distribution, sort_packages @@ -173,8 +173,8 @@ def invoke_dnf( state.pkgmngr.joinpath("var/lib/dnf").mkdir(exist_ok=True, parents=True) # dnf5 does not support building for foreign architectures yet (missing --forcearch) - dnf = shutil.which("dnf5") if state.config.architecture.is_native() else None - dnf = dnf or shutil.which("dnf") or "yum" + dnf = which("dnf5", tools=state.config.tools_tree) if state.config.architecture.is_native() else None + dnf = dnf or which("dnf", tools=state.config.tools_tree) or "yum" cmdline = [ dnf, diff --git a/mkosi/distributions/opensuse.py b/mkosi/distributions/opensuse.py index 6b4ff8a82..e0c7ecd77 100644 --- a/mkosi/distributions/opensuse.py +++ b/mkosi/distributions/opensuse.py @@ -1,6 +1,5 @@ # SPDX-License-Identifier: LGPL-2.1+ -import shutil import textwrap import urllib.request import xml.etree.ElementTree as ElementTree @@ -10,7 +9,7 @@ from mkosi.architecture import Architecture from mkosi.distributions import DistributionInstaller from mkosi.distributions.fedora import Repo, fixup_rpmdb_location, invoke_dnf, setup_dnf from mkosi.log import die -from mkosi.run import bwrap +from mkosi.run import bwrap, which from mkosi.state import MkosiState @@ -44,7 +43,7 @@ class OpensuseInstaller(DistributionInstaller): release_url = f"{state.config.mirror}/distribution/leap/{release}/repo/oss/" updates_url = f"{state.config.mirror}/update/leap/{release}/oss/" - zypper = shutil.which("zypper") + zypper = which("zypper", tools=state.config.tools_tree) # If we need to use a local mirror, create a temporary repository definition # that doesn't get in the image, as it is valid only at image build time. @@ -64,7 +63,7 @@ class OpensuseInstaller(DistributionInstaller): @classmethod def remove_packages(cls, state: MkosiState, packages: Sequence[str]) -> None: - if shutil.which("zypper"): + if which("zypper", tools=state.config.tools_tree): invoke_zypper(state, "remove", packages, ["--clean-deps"]) else: invoke_dnf(state, "remove", packages) diff --git a/mkosi/qemu.py b/mkosi/qemu.py index 38d3c0a9a..e449fd68e 100644 --- a/mkosi/qemu.py +++ b/mkosi/qemu.py @@ -20,7 +20,7 @@ from mkosi.btrfs import btrfs_maybe_snapshot_subvolume from mkosi.config import ConfigFeature, MkosiArgs, MkosiConfig from mkosi.log import die from mkosi.remove import unlink_try_hard -from mkosi.run import MkosiAsyncioThread, bwrap, bwrap_cmd, spawn +from mkosi.run import MkosiAsyncioThread, bwrap, bwrap_cmd, spawn, which from mkosi.types import PathString from mkosi.util import ( Distribution, @@ -41,7 +41,7 @@ def machine_cid(config: MkosiConfig) -> int: def find_qemu_binary(config: MkosiConfig) -> str: binaries = ["qemu", "qemu-kvm", f"qemu-system-{config.architecture.to_qemu()}"] for binary in binaries: - if shutil.which(binary) is not None: + if which(binary, tools=config.tools_tree) is not None: return binary die("Couldn't find QEMU/KVM binary") @@ -291,7 +291,7 @@ def run_qemu(args: MkosiArgs, config: MkosiConfig) -> None: "-device", "virtio-scsi-pci,id=scsi", "-device", "scsi-hd,drive=hd,bootindex=1"] - if config.qemu_swtpm != ConfigFeature.disabled and shutil.which("swtpm") is not None: + if config.qemu_swtpm != ConfigFeature.disabled and which("swtpm", tools=config.tools_tree) is not None: sock = stack.enter_context(start_swtpm(config)) cmdline += ["-chardev", f"socket,id=chrtpm,path={sock}", "-tpmdev", "emulator,id=tpm0,chardev=chrtpm"] diff --git a/mkosi/run.py b/mkosi/run.py index a1e5291c6..9f9cdbf63 100644 --- a/mkosi/run.py +++ b/mkosi/run.py @@ -11,6 +11,7 @@ import os import pwd import queue import shlex +import shutil import signal import subprocess import sys @@ -486,6 +487,10 @@ def chroot_cmd(root: Path, *, options: Sequence[PathString] = (), network: bool return cmdline +def which(program: str, tools: Optional[Path]) -> Optional[str]: + return shutil.which(program, path=f"{tools}/usr/bin:{tools}/usr/sbin" if tools else None) + + class MkosiAsyncioThread(threading.Thread): """ The default threading.Thread() is not interruptable, so we make our own version by using the concurrency