Introduce archive.py

Let's gather the cpio and tar functions in archive.py. We also get
rid of the make_tar(), make_initrd() and make_directory() output
functions and make sure we also mount the root's passwd and related
files when creating cpios.

diff --git a/mkosi/__init__.py b/mkosi/__init__.py
index c752efda450822e10869429e6f969f4663932622..706fc3e046ed8ddb771b798a4625768556df57fb 100644 (file)
--- a/mkosi/__init__.py
+++ b/mkosi/__init__.py
@@ -20,6 +20,7 @@ from pathlib import Path
 from textwrap import dedent
 from typing import ContextManager, Optional, TextIO, Union, cast
 
+from mkosi.archive import extract_tar, make_cpio, make_tar
 from mkosi.config import (
     Compression,
     ConfigFeature,
@@ -43,14 +44,7 @@ from mkosi.pager import page
 from mkosi.qemu import copy_ephemeral, machine_cid, run_qemu
 from mkosi.run import become_root, bwrap, chroot_cmd, init_mount_namespace, run
 from mkosi.state import MkosiState
-from mkosi.tree import (
-    archive_tree,
-    copy_tree,
-    extract_tree,
-    install_tree,
-    move_tree,
-    rmtree,
-)
+from mkosi.tree import copy_tree, install_tree, move_tree, rmtree
 from mkosi.types import PathString
 from mkosi.util import (
     InvokingUser,
@@ -79,7 +73,7 @@ def mount_image(state: MkosiState) -> Iterator[None]:
                 if path.is_dir():
                     bases += [path]
                 elif path.suffix == ".tar":
-                    extract_tree(path, d)
+                    extract_tar(path, d)
                     bases += [d]
                 elif path.suffix == ".raw":
                     run(["systemd-dissect", "-M", path, d])
@@ -576,42 +570,6 @@ def gzip_binary() -> str:
     return "pigz" if shutil.which("pigz") else "gzip"
 
 
-def make_tar(state: MkosiState) -> None:
-    if state.config.output_format != OutputFormat.tar:
-        return
-
-    with complete_step("Creating archive…"):
-        archive_tree(state.root, state.staging / state.config.output_with_format)
-
-
-def make_initrd(state: MkosiState) -> None:
-    if state.config.output_format != OutputFormat.cpio:
-        return
-
-    make_cpio(state.root, state.root.rglob("*"), state.staging / state.config.output_with_format)
-
-
-def make_cpio(root: Path, files: Iterator[Path], output: Path) -> None:
-    with complete_step(f"Creating cpio {output}…"):
-        run([
-            "cpio",
-            "-o",
-            "--reproducible",
-            "--null",
-            "-H", "newc",
-            "--quiet",
-            "-D", root,
-            "-O", output,
-        ], input="\0".join(os.fspath(f.relative_to(root)) for f in files))
-
-
-def make_directory(state: MkosiState) -> None:
-    if state.config.output_format != OutputFormat.directory:
-        return
-
-    state.root.rename(state.staging / state.config.output_with_format)
-
-
 def gen_kernel_images(state: MkosiState) -> Iterator[tuple[str, Path]]:
     if not (state.root / "usr/lib/modules").exists():
         return
@@ -759,7 +717,7 @@ def gen_kernel_modules_initrd(state: MkosiState, kver: str) -> Path:
                 for p in (state.root / modulesd / "vdso").iterdir():
                     yield p
 
-        make_cpio(state.root, files(), kmods)
+        make_cpio(state.root, kmods, files())
 
         # Debian/Ubuntu do not compress their kernel modules, so we compress the initramfs instead. Note that
         # this is not ideal since the compressed kernel modules will all be decompressed on boot which
@@ -1618,9 +1576,12 @@ def build_image(args: MkosiArgs, config: MkosiConfig) -> None:
         for p in split_paths:
             maybe_compress(state.config, state.config.compress_output, p)
 
-        make_tar(state)
-        make_initrd(state)
-        make_directory(state)
+        if state.config.output_format == OutputFormat.tar:
+            make_tar(state.root, state.staging / state.config.output_with_format)
+        elif state.config.output_format == OutputFormat.cpio:
+            make_cpio(state.root, state.staging / state.config.output_with_format)
+        elif state.config.output_format == OutputFormat.directory:
+            state.root.rename(state.staging / state.config.output_with_format)
 
         maybe_compress(state.config, state.config.compress_output,
                        state.staging / state.config.output_with_format,

diff --git a/mkosi/archive.py b/mkosi/archive.py
new file mode 100644 (file)
index 0000000..52754a2
--- /dev/null
+++ b/mkosi/archive.py
@@ -0,0 +1,88 @@
+# SPDX-License-Identifier: LGPL-2.1+
+
+import os
+from collections.abc import Iterable
+from pathlib import Path
+from typing import Optional
+
+from mkosi.log import log_step
+from mkosi.run import bwrap, finalize_passwd_mounts
+from mkosi.util import tar_binary
+
+
+def tar_exclude_apivfs_tmp() -> list[str]:
+    return [
+        "--exclude", "./dev/*",
+        "--exclude", "./proc/*",
+        "--exclude", "./sys/*",
+        "--exclude", "./tmp/*",
+        "--exclude", "./run/*",
+        "--exclude", "./var/tmp/*",
+    ]
+
+
+def make_tar(src: Path, dst: Path) -> None:
+    log_step(f"Creating tar archive {dst}…")
+    bwrap(
+        [
+            tar_binary(),
+            "--create",
+            "--file", dst,
+            "--directory", src,
+            "--acls",
+            "--selinux",
+            "--xattrs",
+            "--sparse",
+            "--force-local",
+            *tar_exclude_apivfs_tmp(),
+            ".",
+        ],
+        # Make sure tar uses user/group information from the root directory instead of the host.
+        options=finalize_passwd_mounts(src) if (src / "etc/passwd").exists() else [],
+    )
+
+
+def extract_tar(src: Path, dst: Path) -> None:
+    log_step(f"Extracting tar archive {src}…")
+    bwrap(
+        [
+            tar_binary(),
+            "--extract",
+            "--file", src,
+            "--directory", dst,
+            "--keep-directory-symlink",
+            "--no-overwrite-dir",
+            "--same-permissions",
+            "--same-owner" if (dst / "etc/passwd").exists() else "--numeric-owner",
+            "--same-order",
+            "--acls",
+            "--selinux",
+            "--xattrs",
+            "--force-local",
+            *tar_exclude_apivfs_tmp(),
+        ],
+        # Make sure tar uses user/group information from the root directory instead of the host.
+        options=finalize_passwd_mounts(dst) if (dst / "etc/passwd").exists() else [],
+    )
+
+
+def make_cpio(src: Path, dst: Path, files: Optional[Iterable[Path]] = None) -> None:
+    if not files:
+        files = src.rglob("*")
+
+    log_step(f"Creating cpio archive {dst}…")
+    bwrap(
+        [
+            "cpio",
+            "--create",
+            "--reproducible",
+            "--null",
+            "--format=newc",
+            "--quiet",
+            "--directory", src,
+            "-O", dst,
+        ],
+        input="\0".join(os.fspath(f.relative_to(src)) for f in files),
+        # Make sure tar uses user/group information from the root directory instead of the host.
+        options=finalize_passwd_mounts(dst),
+    )

diff --git a/mkosi/distributions/debian.py b/mkosi/distributions/debian.py
index 823d14d4eade2052f4b8c462b6fc3432061632f5..7acf8d6aa4e544123084fbc694cf4b122da3f834 100644 (file)
--- a/mkosi/distributions/debian.py
+++ b/mkosi/distributions/debian.py
@@ -6,12 +6,12 @@ from collections.abc import Sequence
 from pathlib import Path
 
 from mkosi.architecture import Architecture
+from mkosi.archive import extract_tar
 from mkosi.distributions import DistributionInstaller, PackageType
 from mkosi.installer.apt import invoke_apt, setup_apt
 from mkosi.log import die
 from mkosi.run import run
 from mkosi.state import MkosiState
-from mkosi.tree import extract_tree
 from mkosi.util import umask
 
 
@@ -114,7 +114,7 @@ class DebianInstaller(DistributionInstaller):
         for deb in essential:
             with tempfile.NamedTemporaryFile() as f:
                 run(["dpkg-deb", "--fsys-tarfile", deb], stdout=f)
-                extract_tree(Path(f.name), state.root)
+                extract_tar(Path(f.name), state.root)
 
         # Finally, run apt to properly install packages in the chroot without having to worry that maintainer
         # scripts won't find basic tools that they depend on.

diff --git a/mkosi/distributions/gentoo.py b/mkosi/distributions/gentoo.py
index 52ee620555dba93e03ba614f9532f2c938de5478..4453443ff0daf9df733980b5521ed2ac802cdb2c 100644 (file)
--- a/mkosi/distributions/gentoo.py
+++ b/mkosi/distributions/gentoo.py
@@ -8,11 +8,12 @@ from collections.abc import Sequence
 from pathlib import Path
 
 from mkosi.architecture import Architecture
+from mkosi.archive import extract_tar
 from mkosi.distributions import DistributionInstaller, PackageType
 from mkosi.log import ARG_DEBUG, complete_step, die
 from mkosi.run import apivfs_cmd, bwrap, chroot_cmd, run
 from mkosi.state import MkosiState
-from mkosi.tree import copy_tree, extract_tree, rmtree
+from mkosi.tree import copy_tree, rmtree
 from mkosi.types import PathString
 from mkosi.util import flatten, sort_packages
 
@@ -111,7 +112,7 @@ class GentooInstaller(DistributionInstaller):
 
         if not any(stage3.iterdir()):
             with complete_step(f"Extracting {stage3_tar.name} to {stage3}"):
-                extract_tree(stage3_tar, stage3)
+                extract_tar(stage3_tar, stage3)
 
         for d in ("binpkgs", "distfiles", "repos/gentoo"):
             (state.cache_dir / d).mkdir(parents=True, exist_ok=True)

diff --git a/mkosi/run.py b/mkosi/run.py
index 0fe242b9b5d46dc31ab702c3129d1fe6031c1574..076b2516b5308a7fc1c2b0a486a9dec4eddaa33d 100644 (file)
--- a/mkosi/run.py
+++ b/mkosi/run.py
@@ -257,6 +257,7 @@ def bwrap(
     scripts: Mapping[str, Sequence[PathString]] = {},
     env: Mapping[str, str] = {},
     stdin: _FILE = None,
+    input: Optional[str] = None,
 ) -> CompletedProcess:
     cmdline: list[PathString] = [
         "bwrap",
@@ -302,7 +303,7 @@ def bwrap(
         cmdline += ["sh", "-c", "chmod 1777 /tmp /dev/shm && exec $0 \"$@\""]
 
         try:
-            result = run([*cmdline, *cmd], env=env, log=False, stdin=stdin)
+            result = run([*cmdline, *cmd], env=env, log=False, stdin=stdin, input=input)
         except subprocess.CalledProcessError as e:
             if log:
                 logging.error(f"\"{' '.join(str(s) for s in cmd)}\" returned non-zero exit code {e.returncode}.")

diff --git a/mkosi/tree.py b/mkosi/tree.py
index 2c1ddea3b2bfa6c5f1f8d26b58abab287da76890..6c3266af0be4316c57abf8a4acab3da5dbf5aacc 100644 (file)
--- a/mkosi/tree.py
+++ b/mkosi/tree.py
@@ -6,11 +6,12 @@ import subprocess
 from pathlib import Path
 from typing import Optional, Sequence, cast
 
+from mkosi.archive import extract_tar
 from mkosi.config import ConfigFeature, MkosiConfig
 from mkosi.log import die
-from mkosi.run import bwrap, finalize_passwd_mounts, run
+from mkosi.run import run
 from mkosi.types import PathString
-from mkosi.util import tar_binary, umask
+from mkosi.util import umask
 
 
 def statfs(path: Path) -> str:
@@ -99,60 +100,6 @@ def move_tree(config: MkosiConfig, src: Path, dst: Path) -> None:
         rmtree(src)
 
 
-def tar_exclude_apivfs_tmp() -> list[str]:
-    return [
-        "--exclude", "./dev/*",
-        "--exclude", "./proc/*",
-        "--exclude", "./sys/*",
-        "--exclude", "./tmp/*",
-        "--exclude", "./run/*",
-        "--exclude", "./var/tmp/*",
-    ]
-
-
-def archive_tree(src: Path, dst: Path) -> None:
-    bwrap(
-        [
-            tar_binary(),
-            "--create",
-            "--file", dst,
-            "--directory", src,
-            "--acls",
-            "--selinux",
-            "--xattrs",
-            "--sparse",
-            "--force-local",
-            *tar_exclude_apivfs_tmp(),
-            ".",
-        ],
-        # Make sure tar uses user/group information from the root directory instead of the host.
-        options=finalize_passwd_mounts(src) if (src / "etc/passwd").exists() else [],
-    )
-
-
-def extract_tree(src: Path, dst: Path) -> None:
-    bwrap(
-        [
-            tar_binary(),
-            "--extract",
-            "--file", src,
-            "--directory", dst,
-            "--keep-directory-symlink",
-            "--no-overwrite-dir",
-            "--same-permissions",
-            "--same-owner" if (dst / "etc/passwd").exists() else "--numeric-owner",
-            "--same-order",
-            "--acls",
-            "--selinux",
-            "--xattrs",
-            "--force-local",
-            *tar_exclude_apivfs_tmp(),
-        ],
-        # Make sure tar uses user/group information from the root directory instead of the host.
-        options=finalize_passwd_mounts(dst) if (dst / "etc/passwd").exists() else [],
-    )
-
-
 def install_tree(config: MkosiConfig, src: Path, dst: Path, target: Optional[Path] = None) -> None:
     t = dst
     if target:
@@ -164,7 +111,7 @@ def install_tree(config: MkosiConfig, src: Path, dst: Path, target: Optional[Pat
     if src.is_dir():
         copy_tree(config, src, t, preserve_owner=False)
     elif src.suffix == ".tar":
-        extract_tree(src, t)
+        extract_tar(src, t)
     elif src.suffix == ".raw":
         run(["systemd-dissect", "--copy-from", src, "/", t])
     else: