From: Daan De Meyer Date: Sun, 6 Aug 2023 09:40:31 +0000 (+0200) Subject: Introduce archive.py X-Git-Tag: v15~27^2~3 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=75b5365cc5cf5891aa367671d2ceb321c6d4b21f;p=thirdparty%2Fmkosi.git Introduce archive.py Let's gather the cpio and tar functions in archive.py. We also get rid of the make_tar(), make_initrd() and make_directory() output functions and make sure we also mount the root's passwd and related files when creating cpios. --- diff --git a/mkosi/__init__.py b/mkosi/__init__.py index c752efda4..706fc3e04 100644 --- a/mkosi/__init__.py +++ b/mkosi/__init__.py @@ -20,6 +20,7 @@ from pathlib import Path from textwrap import dedent from typing import ContextManager, Optional, TextIO, Union, cast +from mkosi.archive import extract_tar, make_cpio, make_tar from mkosi.config import ( Compression, ConfigFeature, @@ -43,14 +44,7 @@ from mkosi.pager import page from mkosi.qemu import copy_ephemeral, machine_cid, run_qemu from mkosi.run import become_root, bwrap, chroot_cmd, init_mount_namespace, run from mkosi.state import MkosiState -from mkosi.tree import ( - archive_tree, - copy_tree, - extract_tree, - install_tree, - move_tree, - rmtree, -) +from mkosi.tree import copy_tree, install_tree, move_tree, rmtree from mkosi.types import PathString from mkosi.util import ( InvokingUser, @@ -79,7 +73,7 @@ def mount_image(state: MkosiState) -> Iterator[None]: if path.is_dir(): bases += [path] elif path.suffix == ".tar": - extract_tree(path, d) + extract_tar(path, d) bases += [d] elif path.suffix == ".raw": run(["systemd-dissect", "-M", path, d]) @@ -576,42 +570,6 @@ def gzip_binary() -> str: return "pigz" if shutil.which("pigz") else "gzip" -def make_tar(state: MkosiState) -> None: - if state.config.output_format != OutputFormat.tar: - return - - with complete_step("Creating archive…"): - archive_tree(state.root, state.staging / state.config.output_with_format) - - -def make_initrd(state: MkosiState) -> None: - if state.config.output_format != OutputFormat.cpio: - return - - make_cpio(state.root, state.root.rglob("*"), state.staging / state.config.output_with_format) - - -def make_cpio(root: Path, files: Iterator[Path], output: Path) -> None: - with complete_step(f"Creating cpio {output}…"): - run([ - "cpio", - "-o", - "--reproducible", - "--null", - "-H", "newc", - "--quiet", - "-D", root, - "-O", output, - ], input="\0".join(os.fspath(f.relative_to(root)) for f in files)) - - -def make_directory(state: MkosiState) -> None: - if state.config.output_format != OutputFormat.directory: - return - - state.root.rename(state.staging / state.config.output_with_format) - - def gen_kernel_images(state: MkosiState) -> Iterator[tuple[str, Path]]: if not (state.root / "usr/lib/modules").exists(): return @@ -759,7 +717,7 @@ def gen_kernel_modules_initrd(state: MkosiState, kver: str) -> Path: for p in (state.root / modulesd / "vdso").iterdir(): yield p - make_cpio(state.root, files(), kmods) + make_cpio(state.root, kmods, files()) # Debian/Ubuntu do not compress their kernel modules, so we compress the initramfs instead. Note that # this is not ideal since the compressed kernel modules will all be decompressed on boot which @@ -1618,9 +1576,12 @@ def build_image(args: MkosiArgs, config: MkosiConfig) -> None: for p in split_paths: maybe_compress(state.config, state.config.compress_output, p) - make_tar(state) - make_initrd(state) - make_directory(state) + if state.config.output_format == OutputFormat.tar: + make_tar(state.root, state.staging / state.config.output_with_format) + elif state.config.output_format == OutputFormat.cpio: + make_cpio(state.root, state.staging / state.config.output_with_format) + elif state.config.output_format == OutputFormat.directory: + state.root.rename(state.staging / state.config.output_with_format) maybe_compress(state.config, state.config.compress_output, state.staging / state.config.output_with_format, diff --git a/mkosi/archive.py b/mkosi/archive.py new file mode 100644 index 000000000..52754a269 --- /dev/null +++ b/mkosi/archive.py @@ -0,0 +1,88 @@ +# SPDX-License-Identifier: LGPL-2.1+ + +import os +from collections.abc import Iterable +from pathlib import Path +from typing import Optional + +from mkosi.log import log_step +from mkosi.run import bwrap, finalize_passwd_mounts +from mkosi.util import tar_binary + + +def tar_exclude_apivfs_tmp() -> list[str]: + return [ + "--exclude", "./dev/*", + "--exclude", "./proc/*", + "--exclude", "./sys/*", + "--exclude", "./tmp/*", + "--exclude", "./run/*", + "--exclude", "./var/tmp/*", + ] + + +def make_tar(src: Path, dst: Path) -> None: + log_step(f"Creating tar archive {dst}…") + bwrap( + [ + tar_binary(), + "--create", + "--file", dst, + "--directory", src, + "--acls", + "--selinux", + "--xattrs", + "--sparse", + "--force-local", + *tar_exclude_apivfs_tmp(), + ".", + ], + # Make sure tar uses user/group information from the root directory instead of the host. + options=finalize_passwd_mounts(src) if (src / "etc/passwd").exists() else [], + ) + + +def extract_tar(src: Path, dst: Path) -> None: + log_step(f"Extracting tar archive {src}…") + bwrap( + [ + tar_binary(), + "--extract", + "--file", src, + "--directory", dst, + "--keep-directory-symlink", + "--no-overwrite-dir", + "--same-permissions", + "--same-owner" if (dst / "etc/passwd").exists() else "--numeric-owner", + "--same-order", + "--acls", + "--selinux", + "--xattrs", + "--force-local", + *tar_exclude_apivfs_tmp(), + ], + # Make sure tar uses user/group information from the root directory instead of the host. + options=finalize_passwd_mounts(dst) if (dst / "etc/passwd").exists() else [], + ) + + +def make_cpio(src: Path, dst: Path, files: Optional[Iterable[Path]] = None) -> None: + if not files: + files = src.rglob("*") + + log_step(f"Creating cpio archive {dst}…") + bwrap( + [ + "cpio", + "--create", + "--reproducible", + "--null", + "--format=newc", + "--quiet", + "--directory", src, + "-O", dst, + ], + input="\0".join(os.fspath(f.relative_to(src)) for f in files), + # Make sure tar uses user/group information from the root directory instead of the host. + options=finalize_passwd_mounts(dst), + ) diff --git a/mkosi/distributions/debian.py b/mkosi/distributions/debian.py index 823d14d4e..7acf8d6aa 100644 --- a/mkosi/distributions/debian.py +++ b/mkosi/distributions/debian.py @@ -6,12 +6,12 @@ from collections.abc import Sequence from pathlib import Path from mkosi.architecture import Architecture +from mkosi.archive import extract_tar from mkosi.distributions import DistributionInstaller, PackageType from mkosi.installer.apt import invoke_apt, setup_apt from mkosi.log import die from mkosi.run import run from mkosi.state import MkosiState -from mkosi.tree import extract_tree from mkosi.util import umask @@ -114,7 +114,7 @@ class DebianInstaller(DistributionInstaller): for deb in essential: with tempfile.NamedTemporaryFile() as f: run(["dpkg-deb", "--fsys-tarfile", deb], stdout=f) - extract_tree(Path(f.name), state.root) + extract_tar(Path(f.name), state.root) # Finally, run apt to properly install packages in the chroot without having to worry that maintainer # scripts won't find basic tools that they depend on. diff --git a/mkosi/distributions/gentoo.py b/mkosi/distributions/gentoo.py index 52ee62055..4453443ff 100644 --- a/mkosi/distributions/gentoo.py +++ b/mkosi/distributions/gentoo.py @@ -8,11 +8,12 @@ from collections.abc import Sequence from pathlib import Path from mkosi.architecture import Architecture +from mkosi.archive import extract_tar from mkosi.distributions import DistributionInstaller, PackageType from mkosi.log import ARG_DEBUG, complete_step, die from mkosi.run import apivfs_cmd, bwrap, chroot_cmd, run from mkosi.state import MkosiState -from mkosi.tree import copy_tree, extract_tree, rmtree +from mkosi.tree import copy_tree, rmtree from mkosi.types import PathString from mkosi.util import flatten, sort_packages @@ -111,7 +112,7 @@ class GentooInstaller(DistributionInstaller): if not any(stage3.iterdir()): with complete_step(f"Extracting {stage3_tar.name} to {stage3}"): - extract_tree(stage3_tar, stage3) + extract_tar(stage3_tar, stage3) for d in ("binpkgs", "distfiles", "repos/gentoo"): (state.cache_dir / d).mkdir(parents=True, exist_ok=True) diff --git a/mkosi/run.py b/mkosi/run.py index 0fe242b9b..076b2516b 100644 --- a/mkosi/run.py +++ b/mkosi/run.py @@ -257,6 +257,7 @@ def bwrap( scripts: Mapping[str, Sequence[PathString]] = {}, env: Mapping[str, str] = {}, stdin: _FILE = None, + input: Optional[str] = None, ) -> CompletedProcess: cmdline: list[PathString] = [ "bwrap", @@ -302,7 +303,7 @@ def bwrap( cmdline += ["sh", "-c", "chmod 1777 /tmp /dev/shm && exec $0 \"$@\""] try: - result = run([*cmdline, *cmd], env=env, log=False, stdin=stdin) + result = run([*cmdline, *cmd], env=env, log=False, stdin=stdin, input=input) except subprocess.CalledProcessError as e: if log: logging.error(f"\"{' '.join(str(s) for s in cmd)}\" returned non-zero exit code {e.returncode}.") diff --git a/mkosi/tree.py b/mkosi/tree.py index 2c1ddea3b..6c3266af0 100644 --- a/mkosi/tree.py +++ b/mkosi/tree.py @@ -6,11 +6,12 @@ import subprocess from pathlib import Path from typing import Optional, Sequence, cast +from mkosi.archive import extract_tar from mkosi.config import ConfigFeature, MkosiConfig from mkosi.log import die -from mkosi.run import bwrap, finalize_passwd_mounts, run +from mkosi.run import run from mkosi.types import PathString -from mkosi.util import tar_binary, umask +from mkosi.util import umask def statfs(path: Path) -> str: @@ -99,60 +100,6 @@ def move_tree(config: MkosiConfig, src: Path, dst: Path) -> None: rmtree(src) -def tar_exclude_apivfs_tmp() -> list[str]: - return [ - "--exclude", "./dev/*", - "--exclude", "./proc/*", - "--exclude", "./sys/*", - "--exclude", "./tmp/*", - "--exclude", "./run/*", - "--exclude", "./var/tmp/*", - ] - - -def archive_tree(src: Path, dst: Path) -> None: - bwrap( - [ - tar_binary(), - "--create", - "--file", dst, - "--directory", src, - "--acls", - "--selinux", - "--xattrs", - "--sparse", - "--force-local", - *tar_exclude_apivfs_tmp(), - ".", - ], - # Make sure tar uses user/group information from the root directory instead of the host. - options=finalize_passwd_mounts(src) if (src / "etc/passwd").exists() else [], - ) - - -def extract_tree(src: Path, dst: Path) -> None: - bwrap( - [ - tar_binary(), - "--extract", - "--file", src, - "--directory", dst, - "--keep-directory-symlink", - "--no-overwrite-dir", - "--same-permissions", - "--same-owner" if (dst / "etc/passwd").exists() else "--numeric-owner", - "--same-order", - "--acls", - "--selinux", - "--xattrs", - "--force-local", - *tar_exclude_apivfs_tmp(), - ], - # Make sure tar uses user/group information from the root directory instead of the host. - options=finalize_passwd_mounts(dst) if (dst / "etc/passwd").exists() else [], - ) - - def install_tree(config: MkosiConfig, src: Path, dst: Path, target: Optional[Path] = None) -> None: t = dst if target: @@ -164,7 +111,7 @@ def install_tree(config: MkosiConfig, src: Path, dst: Path, target: Optional[Pat if src.is_dir(): copy_tree(config, src, t, preserve_owner=False) elif src.suffix == ".tar": - extract_tree(src, t) + extract_tar(src, t) elif src.suffix == ".raw": run(["systemd-dissect", "--copy-from", src, "/", t]) else: