From 102a4df5b1ad9edd4f6090a766d23eea39b290e7 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Sun, 16 Apr 2023 01:14:27 +0200 Subject: [PATCH] Get rid of debootstrap dependency Let's get rid of our dependency on debootstrap by replicating its core functionality ourselves. To make sure the necessary tools for maintainer scripts to run are available in the chroot, we have to extract the essential debs manually first before installing all the essential debs. This also allows us to get rid of our skeleton tree hack we added for debootstrap. --- NEWS.md | 9 ++- README.md | 2 +- action.yaml | 1 - mkosi.md | 24 ++++---- mkosi/__init__.py | 24 ++++++-- mkosi/backend.py | 1 - mkosi/distributions/__init__.py | 2 - mkosi/distributions/debian.py | 103 +++++++++++++++++++++----------- mkosi/install.py | 26 -------- mkosi/run.py | 5 +- 10 files changed, 108 insertions(+), 89 deletions(-) diff --git a/NEWS.md b/NEWS.md index e0166786a..bd0e4b689 100644 --- a/NEWS.md +++ b/NEWS.md @@ -60,8 +60,9 @@ image and enabling systemd-networkd. - If `mkosi.extra/` or `mkosi.skeleton/` exist, they are now always used instead of only when no explicit extra/skeleton trees are defined. -- mkosi doesn't install any default packages anymore aside from the base filesystem layout package. In - practice, this means systemd and other basic tools have to be installed explicitly from now on. +- mkosi doesn't install any default packages anymore aside from packages required by the distro or the base + filesystem layout package if there are no required packages. In practice, this means systemd and other + basic tools have to be installed explicitly from now on. - Removed `--base-packages` as it's not needed anymore since we don't install any packages by default anymore aside from the base filesystem layout package. - Removed `--qcow2` option in favor of supporting only raw disk images as the disk image output format. @@ -74,6 +75,10 @@ CentOS Stream 8 image with a RPM db in sqlite format, rewrite the db in bdb format using `rpm --rebuilddb --define _db_backend bdb`. - Repositories are now only written to /etc/apt/sources.list if apt is installed in the image. +- Removed the dependency on `debootstrap` to build Ubuntu or Debian images. +- Apt now uses the keyring from the host instead of the keyring from the image. This means + `debian-archive-keyring` or `ubuntu-archive-keyring` are now required to be installed to build Debian or + Ubuntu images respectively. ## v14 diff --git a/README.md b/README.md index f51b47713..424da5271 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # mkosi — Build Bespoke OS Images -A fancy wrapper around `dnf --installroot`, `debootstrap`, `pacman` +A fancy wrapper around `dnf --installroot`, `apt`, `pacman` and `zypper` that generates customized disk images with a number of bells and whistles. diff --git a/action.yaml b/action.yaml index a48556c90..6c0c23d76 100644 --- a/action.yaml +++ b/action.yaml @@ -12,7 +12,6 @@ runs: sudo add-apt-repository ppa:michel-slm/kernel-utils sudo apt-get update sudo apt-get install --assume-yes --no-install-recommends \ - debootstrap \ zypper \ dnf \ pacman-package-manager \ diff --git a/mkosi.md b/mkosi.md index a21621096..ac7c81b1f 100644 --- a/mkosi.md +++ b/mkosi.md @@ -33,9 +33,8 @@ mkosi — Build Bespoke OS Images # DESCRIPTION `mkosi` is a tool for easily building customized OS images. It's a -fancy wrapper around `dnf --installroot`, `debootstrap`, `pacman` -and `zypper` that may generate disk images with a number of bells and -whistles. +fancy wrapper around `dnf --installroot`, `apt`, `pacman` and `zypper` +that may generate disk images with a number of bells and whistles. ## Command Line Verbs @@ -950,11 +949,11 @@ following operating systems: In theory, any distribution may be used on the host for building images containing any other distribution, as long as the necessary tools are available. Specifically, any distribution that packages -`debootstrap` and `apt` may be used to build *Debian* or *Ubuntu* images. Any -distribution that packages `dnf` may be used to build *CentOS*, *Alma Linux*, -*Rocky Linux*, *Fedora Linux*, *Mageia* or *OpenMandriva* images. Any distro -that packages `pacman` may be used to build *Arch Linux* images. Any distribution -that packages `zypper` may be used to build *openSUSE* images. Any distribution +`apt` may be used to build *Debian* or *Ubuntu* images. Any distribution that +packages `dnf` may be used to build *CentOS*, *Alma Linux*, *Rocky Linux*, +*Fedora Linux*, *Mageia* or *OpenMandriva* images. Any distro that packages +`pacman` may be used to build *Arch Linux* images. Any distribution that +packages `zypper` may be used to build *openSUSE* images. Any distribution that packages `emerge` may be used to build *Gentoo* images. Currently, *Fedora Linux* packages all relevant tools as of Fedora 28. @@ -1328,14 +1327,13 @@ When not using distribution packages make sure to install the necessary dependencies. For example, on *Fedora Linux* you need: ```bash -dnf install bubblewrap btrfs-progs apt debootstrap dosfstools mtools edk2-ovmf e2fsprogs squashfs-tools gnupg python3 tar xfsprogs xz zypper sbsigntools +dnf install bubblewrap btrfs-progs apt dosfstools mtools edk2-ovmf e2fsprogs squashfs-tools gnupg python3 tar xfsprogs xz zypper sbsigntools ``` On Debian/Ubuntu it might be necessary to install the `ubuntu-keyring`, `ubuntu-archive-keyring` and/or `debian-archive-keyring` packages explicitly, -in addition to `apt` and `debootstrap`, depending on what kind of distribution images -you want to build. `debootstrap` on Debian only pulls in the Debian keyring -on its own, and the version on Ubuntu only the one from Ubuntu. +in addition to `apt`, depending on what kind of distribution images you want +to build. Note that the minimum required Python version is 3.9. @@ -1345,4 +1343,4 @@ Note that the minimum required Python version is 3.9. * [The mkosi OS generation tool](https://lwn.net/Articles/726655/) story on LWN # SEE ALSO -`systemd-nspawn(1)`, `dnf(8)`, `debootstrap(8)` +`systemd-nspawn(1)`, `dnf(8)`, diff --git a/mkosi/__init__.py b/mkosi/__init__.py index 8971898e5..fa69dea75 100644 --- a/mkosi/__init__.py +++ b/mkosi/__init__.py @@ -41,12 +41,7 @@ from mkosi.backend import ( should_compress_output, tmp_dir, ) -from mkosi.install import ( - add_dropin_config_from_resource, - copy_path, - flock, - install_skeleton_trees, -) +from mkosi.install import add_dropin_config_from_resource, copy_path, flock from mkosi.log import ( ARG_DEBUG, MkosiException, @@ -537,6 +532,23 @@ def install_boot_loader(state: MkosiState) -> None: ) +def install_skeleton_trees(state: MkosiState, cached: bool) -> None: + if not state.config.skeleton_trees or cached: + return + + with complete_step("Copying in skeleton file trees…"): + for source, target in state.config.skeleton_trees: + t = state.root + if target: + t = state.root / target.relative_to("/") + + t.mkdir(mode=0o755, parents=True, exist_ok=True) + if source.is_dir(): + copy_path(source, t, preserve_owner=False) + else: + shutil.unpack_archive(source, t) + + def install_extra_trees(state: MkosiState) -> None: if not state.config.extra_trees: return diff --git a/mkosi/backend.py b/mkosi/backend.py index bec6eea93..d2f6b0643 100644 --- a/mkosi/backend.py +++ b/mkosi/backend.py @@ -145,7 +145,6 @@ def detect_distribution() -> tuple[Optional[Distribution], Optional[str]]: break if d in {Distribution.debian, Distribution.ubuntu} and (version_codename or extracted_codename): - # debootstrap needs release codenames, not version numbers version_id = version_codename or extracted_codename return d, version_id diff --git a/mkosi/distributions/__init__.py b/mkosi/distributions/__init__.py index f32fc486c..49eb0c752 100644 --- a/mkosi/distributions/__init__.py +++ b/mkosi/distributions/__init__.py @@ -9,8 +9,6 @@ if TYPE_CHECKING: class DistributionInstaller: - needs_skeletons_after_bootstrap = False - @classmethod def install(cls, state: "MkosiState") -> None: raise NotImplementedError diff --git a/mkosi/distributions/debian.py b/mkosi/distributions/debian.py index 021b61218..6ec35427f 100644 --- a/mkosi/distributions/debian.py +++ b/mkosi/distributions/debian.py @@ -2,20 +2,18 @@ import os import subprocess +import tempfile from collections.abc import Sequence from pathlib import Path from textwrap import dedent from mkosi.backend import MkosiState from mkosi.distributions import DistributionInstaller -from mkosi.install import install_skeleton_trees from mkosi.run import run, run_with_apivfs -from mkosi.types import CompletedProcess, PathString +from mkosi.types import _FILE, CompletedProcess, PathString class DebianInstaller(DistributionInstaller): - needs_skeletons_after_bootstrap = True - @classmethod def filesystem(cls) -> str: return "ext4" @@ -52,36 +50,72 @@ class DebianInstaller(DistributionInstaller): @classmethod def install(cls, state: MkosiState) -> None: - repos = {"main", *state.config.repositories} - - cmdline: list[PathString] = [ - "debootstrap", - "--variant=minbase", - "--merged-usr", - f"--cache-dir={state.cache}", - f"--components={','.join(repos)}", - ] + # Instead of using debootstrap, we replicate its core functionality here. Because dpkg does not have + # an option to delay running pre-install maintainer scripts when it installs a package, it's + # impossible to use apt directly to bootstrap a Debian chroot since dpkg will try to run a maintainer + # script which depends on some basic tool to be available in the chroot from a deb which hasn't been + # unpacked yet, causing the script to fail. To avoid these issues, we have to extract all the + # essential debs first, and only then run the maintainer scripts for them. + + # First, we set up merged usr. + # This list is taken from https://salsa.debian.org/installer-team/debootstrap/-/blob/master/functions#L1369. + subdirs = ["bin", "sbin", "lib"] + { + "amd64" : ["lib32", "lib64", "libx32"], + "i386" : ["lib64", "libx32"], + "mips" : ["lib32", "lib64"], + "mipsel" : ["lib32", "lib64"], + "mips64el" : ["lib32", "lib64", "libo32"], + "loongarch64" : ["lib32", "lib64"], + "powerpc" : ["lib64"], + "ppc64" : ["lib32", "lib64"], + "ppc64el" : ["lib64"], + "s390x" : ["lib32"], + "sparc" : ["lib64"], + "sparc64" : ["lib32", "lib64"], + "x32" : ["lib32", "lib64", "libx32"], + }.get(DEBIAN_ARCHITECTURES[state.config.architecture], []) + + state.root.joinpath("usr").mkdir(mode=0o755) + for d in subdirs: + state.root.joinpath(d).symlink_to(f"usr/{d}") + state.root.joinpath(f"usr/{d}").mkdir(mode=0o755) + + # Next, we download the essential debs. We add usr-is-merged to assert the system is usr-merged + # already and to prevent usrmerge from being installed and pulling in all its dependencies. + setup_apt(state, cls.repositories(state)) + invoke_apt(state, "update") + invoke_apt(state, "install", ["--download-only", "?essential", "?name(usr-is-merged)"]) - debarch = DEBIAN_ARCHITECTURES[state.config.architecture] - cmdline += [f"--arch={debarch}"] + # Next, invoke apt install with an info fd to which it writes the debs it's operating on. However, by + # passing "-oDebug::pkgDpkgPm=1", apt will not actually execute any dpkg commands, which turns the + # install command into a noop that tells us the full paths to the essential debs and any dependencies + # that apt would install in the apt cache. + with tempfile.TemporaryFile(dir=state.workspace, mode="w+") as f: + os.set_inheritable(f.fileno(), True) - # Let's use --no-check-valid-until only if debootstrap knows it - if debootstrap_knows_arg("--no-check-valid-until"): - cmdline += ["--no-check-valid-until"] + invoke_apt(state, "install", [ + "-oDebug::pkgDpkgPm=1", + f"-oAPT::Keep-Fds::={f.fileno()}", + f"-oDPkg::Tools::options::'cat >&$fd'::InfoFD={f.fileno()}", + f"-oDpkg::Pre-Install-Pkgs::=cat >&{f.fileno()}", + "?essential", "?name(usr-is-merged)", + ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - if not state.config.repository_key_check: - cmdline += ["--no-check-gpg"] + f.seek(0) + essential = f.read().strip().splitlines() - mirror = state.config.local_mirror or state.config.mirror - assert mirror is not None - cmdline += [state.config.release, state.root, mirror] + # Now, extract the debs to the chroot by first extracting the sources tar file out of the deb and + # then extracting the tar file into the chroot. - # Pretend we're lxc so debootstrap skips its mknod check. - run_with_apivfs(state, cmdline, env=dict(container="lxc", DPKG_FORCE="unsafe-io")) + for deb in essential: + with tempfile.NamedTemporaryFile(dir=state.workspace) as f: + run(["dpkg-deb", "--fsys-tarfile", deb], stdout=f) + run(["tar", "-C", state.root, "--keep-directory-symlink", "--extract", "--file", f.name]) - install_skeleton_trees(state, False, late=True) + # Finally, run apt to properly install packages in the chroot without having to worry that maintainer + # scripts won't find basic tools that they depend on. - cls.install_packages(state, ["base-passwd"]) + cls.install_packages(state, [Path(deb).name.partition("_")[0] for deb in essential]) # Ensure /efi exists so that the ESP is mounted there, and we never run dpkg -i on vfat state.root.joinpath("efi").mkdir(mode=0o755, exist_ok=True) @@ -114,8 +148,7 @@ class DebianInstaller(DistributionInstaller): invoke_apt(state, "purge", packages) -# Debian calls their architectures differently, so when calling debootstrap we -# will have to map to their names +# Debian calls their architectures differently, so when calling apt we will have to map to their names. # uname -m -> dpkg --print-architecture DEBIAN_ARCHITECTURES = { "aarch64": "arm64", @@ -134,11 +167,6 @@ DEBIAN_ARCHITECTURES = { } -def debootstrap_knows_arg(arg: str) -> bool: - return bytes("invalid option", "UTF-8") not in run(["debootstrap", arg], - stdout=subprocess.PIPE, check=False).stdout - - def setup_apt(state: MkosiState, repos: Sequence[str]) -> None: state.workspace.joinpath("apt").mkdir(exist_ok=True) state.workspace.joinpath("apt/apt.conf.d").mkdir(exist_ok=True) @@ -160,6 +188,7 @@ def setup_apt(state: MkosiState, repos: Sequence[str]) -> None: APT::Install-Recommends "false"; APT::Get::Assume-Yes "true"; APT::Get::AutomaticRemove "true"; + APT::Sandbox::User "root"; Dir::Cache "{state.cache}"; Dir::State "{state.workspace / "apt"}"; Dir::State::status "{state.root / "var/lib/dpkg/status"}"; @@ -172,7 +201,9 @@ def setup_apt(state: MkosiState, repos: Sequence[str]) -> None: DPkg::Options:: "--root={state.root}"; DPkg::Options:: "--log={state.workspace / "apt/dpkg.log"}"; DPkg::Options:: "--force-unsafe-io"; + Dpkg::Use-Pty "false"; DPkg::Install::Recursive::Minimum "1000"; + pkgCacheGen::ForceEssential ","; """ ) ) @@ -186,6 +217,8 @@ def invoke_apt( state: MkosiState, operation: str, extra: Sequence[str] = tuple(), + stdout: _FILE = None, + stderr: _FILE = None, ) -> CompletedProcess: env: dict[str, PathString] = dict( APT_CONFIG=state.workspace / "apt/apt.conf", @@ -195,4 +228,4 @@ def invoke_apt( INITRD="No", ) - return run_with_apivfs(state, ["apt-get", operation, *extra], env=env) + return run_with_apivfs(state, ["apt-get", operation, *extra], env=env, stdout=stdout, stderr=stderr) diff --git a/mkosi/install.py b/mkosi/install.py index ec3557227..4b02e0193 100644 --- a/mkosi/install.py +++ b/mkosi/install.py @@ -4,14 +4,11 @@ import contextlib import fcntl import importlib.resources import os -import shutil import stat from collections.abc import Iterator from pathlib import Path from typing import Optional -from mkosi.backend import MkosiState -from mkosi.log import complete_step from mkosi.run import run @@ -60,26 +57,3 @@ def copy_path(src: Path, dst: Path, preserve_owner: bool = True) -> None: "--reflink=auto", src, dst, ]) - - -def install_skeleton_trees(state: MkosiState, cached: bool, *, late: bool=False) -> None: - if not state.config.skeleton_trees: - return - - if cached: - return - - if not late and state.installer.needs_skeletons_after_bootstrap: - return - - with complete_step("Copying in skeleton file trees…"): - for source, target in state.config.skeleton_trees: - t = state.root - if target: - t = state.root / target.relative_to("/") - - t.mkdir(mode=0o755, parents=True, exist_ok=True) - if source.is_dir(): - copy_path(source, t, preserve_owner=False) - else: - shutil.unpack_archive(source, t) diff --git a/mkosi/run.py b/mkosi/run.py index 30f4230f7..6fcb9dc5e 100644 --- a/mkosi/run.py +++ b/mkosi/run.py @@ -221,7 +221,7 @@ def run( try: return subprocess.run(cmdline, check=check, stdout=stdout, stderr=stderr, env=env, **kwargs, - preexec_fn=foreground) + preexec_fn=foreground, close_fds=False) except FileNotFoundError: die(f"{cmdline[0]} not found in PATH.") @@ -252,6 +252,7 @@ def run_with_apivfs( cmd: Sequence[PathString], bwrap_params: Sequence[PathString] = tuple(), stdout: _FILE = None, + stderr: _FILE = None, env: Mapping[str, PathString] = {}, ) -> CompletedProcess: cmdline: list[PathString] = [ @@ -290,7 +291,7 @@ def run_with_apivfs( try: return run([*cmdline, template.format(shlex.join(str(s) for s in cmd))], - text=True, stdout=stdout, env=env) + text=True, stdout=stdout, stderr=stderr, env=env) except subprocess.CalledProcessError as e: if "run" in ARG_DEBUG: run([*cmdline, template.format("sh")], check=False, env=env) -- 2.47.2