From: Daan De Meyer Date: Fri, 19 Dec 2025 19:53:09 +0000 (+0100) Subject: util: Add reflink file-copying helpers X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=f04f702d4d985d0a07e2fd8af9c9e80c71529ed7;p=thirdparty%2Fmkosi.git util: Add reflink file-copying helpers shutil.copyfile() doesn't do reflinks internally, so let's add our own helpers which do. --- diff --git a/mkosi/__init__.py b/mkosi/__init__.py index a57fec481..d93a83931 100644 --- a/mkosi/__init__.py +++ b/mkosi/__init__.py @@ -155,6 +155,8 @@ from mkosi.user import INVOKING_USER, become_root_cmd from mkosi.util import ( PathString, chdir, + copyfile, + copyfile2, flatten, flock_or_die, format_rlimit, @@ -417,7 +419,7 @@ def configure_os_release(context: Context) -> None: newosrelease.rename(osrelease) if ArtifactOutput.os_release in context.config.split_artifacts: - shutil.copy(osrelease, context.staging / context.config.output_split_os_release) + copyfile(osrelease, context.staging / context.config.output_split_os_release) def configure_extension_release(context: Context) -> None: @@ -524,7 +526,7 @@ def configure_verity_certificate(context: Context) -> None: dest = veritydir / context.config.verity_certificate.with_suffix(".crt").name with umask(~0o644): - shutil.copy(context.config.verity_certificate, dest) + copyfile(context.config.verity_certificate, dest) def configure_mountpoints(context: Context) -> None: @@ -1234,9 +1236,9 @@ def install_sandbox_trees(config: Config, dst: Path) -> None: install_tree(config, tree.source, dst, target=tree.target, preserve=False) if Path("/etc/passwd").exists(): - shutil.copy("/etc/passwd", dst / "etc/passwd") + copyfile("/etc/passwd", dst / "etc/passwd") if Path("/etc/group").exists(): - shutil.copy("/etc/group", dst / "etc/group") + copyfile("/etc/group", dst / "etc/group") if not (dst / "etc/mtab").is_symlink(): (dst / "etc/mtab").symlink_to("../proc/self/mounts") @@ -1268,7 +1270,7 @@ def install_sandbox_trees(config: Config, dst: Path) -> None: ) if not (dst / "etc/hosts").exists() and Path("/etc/hosts").exists(): - shutil.copy("/etc/hosts", dst / "etc/hosts") + copyfile("/etc/hosts", dst / "etc/hosts") Path(dst / "etc/static").unlink(missing_ok=True) if (config.tools() / "etc/static").is_symlink(): @@ -1310,7 +1312,7 @@ def install_package_directories(context: Context, directories: Sequence[Path]) - context.config ).package_globs() ): - shutil.copy(p, context.repository, follow_symlinks=True) + copyfile(p, context.repository) def install_extra_trees(context: Context) -> None: @@ -1396,7 +1398,7 @@ def fixup_vmlinuz_location(context: Context) -> None: if vmlinuz.is_symlink() and vmlinuz.resolve().is_relative_to("/boot"): vmlinuz.unlink() if not vmlinuz.exists(): - shutil.copy2(d, vmlinuz) + copyfile2(d, vmlinuz) def want_initrd(context: Context) -> bool: @@ -1566,7 +1568,7 @@ def build_kernel_modules_initrd(context: Context, kver: str) -> Path: maybe_compress(context, compression, kmods, kmods) if ArtifactOutput.kernel_modules_initrd in context.config.split_artifacts: - shutil.copy(kmods, context.staging / context.config.output_split_kernel_modules_initrd) + copyfile(kmods, context.staging / context.config.output_split_kernel_modules_initrd) return kmods @@ -1999,14 +2001,14 @@ def install_type1( ): kimg = sign_efi_binary(context, kimg, dst / "vmlinuz") else: - kimg = Path(shutil.copy2(context.root / kimg, dst / "vmlinuz")) + kimg = Path(copyfile2(context.root / kimg, dst / "vmlinuz")) - initrds = [ - Path(shutil.copy2(initrd, dst.parent / initrd.name)) for initrd in microcode + initrds - ] + [Path(shutil.copy2(kmods, dst / "kernel-modules.initrd"))] + initrds = [Path(copyfile2(initrd, dst.parent / initrd.name)) for initrd in microcode + initrds] + [ + Path(copyfile2(kmods, dst / "kernel-modules.initrd")) + ] if dtb and source_dtb: - shutil.copy2(source_dtb, dtb) + copyfile2(source_dtb, dtb) with entry.open("w") as f: f.write( @@ -2132,7 +2134,7 @@ def install_uki( ) or context.config.unified_kernel_images == UnifiedKernelImage.signed: for p in (context.root / "usr/lib/modules" / kver).glob("*.efi"): log_step(f"Installing prebuilt UKI at {p} to {boot_binary}") - shutil.copy2(p, boot_binary) + copyfile2(p, boot_binary) break else: if context.config.bootable == ConfigFeature.enabled: @@ -2385,7 +2387,7 @@ def copy_nspawn_settings(context: Context) -> None: return None with complete_step("Copying nspawn settings file…"): - shutil.copy2(context.config.nspawn_settings, context.staging / context.config.output_nspawn_settings) + copyfile2(context.config.nspawn_settings, context.staging / context.config.output_nspawn_settings) def get_uki_path(context: Context) -> Optional[Path]: @@ -2422,7 +2424,7 @@ def copy_uki(context: Context) -> None: return if uki := get_uki_path(context): - shutil.copy(uki, context.staging / context.config.output_split_uki) + copyfile(uki, context.staging / context.config.output_split_uki) def copy_vmlinuz(context: Context) -> None: @@ -2439,7 +2441,7 @@ def copy_vmlinuz(context: Context) -> None: return for _, kimg in gen_kernel_images(context): - shutil.copy(context.root / kimg, context.staging / context.config.output_split_kernel) + copyfile(context.root / kimg, context.staging / context.config.output_split_kernel) break @@ -3348,7 +3350,7 @@ def save_esp_components( if not stub.exists(): die(f"sd-stub not found at /{stub.relative_to(context.root)} in the image") - return Path(shutil.copy2(stub, context.workspace)), None, None, [] + return Path(copyfile2(stub, context.workspace)), None, None, [] if context.config.output_format not in (OutputFormat.uki, OutputFormat.esp): return None, None, None, [] @@ -3367,7 +3369,7 @@ def save_esp_components( return None, None, None, [] - kimg = Path(shutil.copy2(context.root / kimg, context.workspace)) + kimg = Path(copyfile2(context.root / kimg, context.workspace)) if not context.config.architecture.to_efi(): die(f"Architecture {context.config.architecture} does not support UEFI") @@ -3376,7 +3378,7 @@ def save_esp_components( if not stub.exists(): die(f"sd-stub not found at /{stub.relative_to(context.root)} in the image") - stub = Path(shutil.copy2(stub, context.workspace)) + stub = Path(copyfile2(stub, context.workspace)) microcode = build_microcode_initrd(context) return stub, kver, kimg, microcode @@ -4238,7 +4240,7 @@ def run_shell(args: Args, config: Config) -> None: stack.callback( lambda: (config.output_dir_or_cwd() / f"{name}.nspawn").unlink(missing_ok=True) ) - shutil.copy2(config.nspawn_settings, config.output_dir_or_cwd() / f"{name}.nspawn") + copyfile2(config.nspawn_settings, config.output_dir_or_cwd() / f"{name}.nspawn") # If we're booting a directory image that wasn't built by root, we always make an ephemeral # copy to avoid ending up with files not owned by the directory image owner in the diff --git a/mkosi/bootloader.py b/mkosi/bootloader.py index 7d434bb47..6e22f5f00 100644 --- a/mkosi/bootloader.py +++ b/mkosi/bootloader.py @@ -4,7 +4,6 @@ import enum import itertools import logging import os -import shutil import subprocess import sys import tempfile @@ -33,7 +32,7 @@ from mkosi.log import complete_step, die, log_step from mkosi.partition import Partition from mkosi.run import CompletedProcess, run, workdir from mkosi.sandbox import umask -from mkosi.util import _FILE, PathString, StrEnum, flatten +from mkosi.util import _FILE, PathString, StrEnum, copyfile2, flatten from mkosi.versioncomp import GenericVersion @@ -387,7 +386,7 @@ def install_grub(context: Context) -> None: rel = output.relative_to(context.root) log_step(f"Installing signed grub EFI binary from /{signed.relative_to(context.root)} to /{rel}") - shutil.copy2(signed, output) + copyfile2(signed, output) else: if context.config.secure_boot and context.config.shim_bootloader != ShimBootloader.none: if not (signed := find_signed_grub_image(context)): @@ -414,7 +413,7 @@ def install_grub(context: Context) -> None: for d in ("grub", "grub2"): unicode = context.root / "usr/share" / d / "unicode.pf2" if unicode.exists(): - shutil.copy2(unicode, dst) + copyfile2(unicode, dst) def grub_bios_setup(context: Context, partitions: Sequence[Partition]) -> None: @@ -638,7 +637,7 @@ def find_and_install_shim_binary( output = output.with_name(f"{left_stem}.efi") log_step(f"Installing signed {name} EFI binary from /{rel} to /{output}") - shutil.copy2(p, context.root / output) + copyfile2(p, context.root / output) return if context.config.bootable == ConfigFeature.enabled: @@ -661,7 +660,7 @@ def find_and_install_shim_binary( sign_efi_binary(context, p, context.root / output) else: log_step(f"Installing unsigned {name} EFI binary /{rel} to /{output}") - shutil.copy2(p, context.root / output) + copyfile2(p, context.root / output) return @@ -755,7 +754,7 @@ def install_systemd_boot(context: Context) -> None: Path(context.root / "efi/loader/random-seed").unlink(missing_ok=True) if context.config.shim_bootloader != ShimBootloader.none: - shutil.copy2( + copyfile2( context.root / f"efi/EFI/systemd/systemd-boot{context.config.architecture.to_efi()}.efi", context.root / shim_second_stage_binary(context), ) diff --git a/mkosi/distribution/postmarketos.py b/mkosi/distribution/postmarketos.py index 7673e1aaa..1bf9b34ff 100644 --- a/mkosi/distribution/postmarketos.py +++ b/mkosi/distribution/postmarketos.py @@ -1,6 +1,5 @@ # SPDX-License-Identifier: LGPL-2.1-or-later -import shutil from collections.abc import Iterable from mkosi.config import Architecture, Config @@ -9,6 +8,7 @@ from mkosi.distribution import Distribution, DistributionInstaller, PackageType from mkosi.installer import PackageManager from mkosi.installer.apk import Apk, ApkRepository from mkosi.log import complete_step, die +from mkosi.util import copyfile class Installer(DistributionInstaller, distribution=Distribution.postmarketos): @@ -59,7 +59,7 @@ class Installer(DistributionInstaller, distribution=Distribution.postmarketos): if dest.exists(): continue - shutil.copy2(key, dest) + copyfile(key, dest) Apk.setup(context, list(cls.repositories(context))) diff --git a/mkosi/initrd.py b/mkosi/initrd.py index fc064d1a6..1f28b6ea8 100644 --- a/mkosi/initrd.py +++ b/mkosi/initrd.py @@ -20,7 +20,7 @@ from mkosi.log import ARG_DEBUG, ARG_DEBUG_SHELL, die, log_notice, log_setup from mkosi.run import find_binary, run, uncaught_exception_handler from mkosi.sandbox import __version__, umask from mkosi.tree import copy_tree, move_tree, rmtree -from mkosi.util import PathString, mandatory_variable, resource_path +from mkosi.util import PathString, copyfile2, mandatory_variable, resource_path @dataclasses.dataclass(frozen=True) @@ -407,7 +407,7 @@ def main() -> None: (Path(sandbox_tree) / "etc" / p).parent.mkdir(parents=True, exist_ok=True) if (Path("/etc") / p).resolve().is_file(): - shutil.copy2(Path("/etc") / p, Path(sandbox_tree) / "etc" / p) + copyfile2(Path("/etc") / p, Path(sandbox_tree) / "etc" / p) else: shutil.copytree( Path("/etc") / p, diff --git a/mkosi/installer/apk.py b/mkosi/installer/apk.py index 6489cac97..6a7d64772 100644 --- a/mkosi/installer/apk.py +++ b/mkosi/installer/apk.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: LGPL-2.1-or-later import dataclasses -import shutil from collections.abc import Sequence from pathlib import Path @@ -10,7 +9,7 @@ from mkosi.context import Context from mkosi.installer import PackageManager from mkosi.run import CompletedProcess, run, workdir from mkosi.tree import rmtree -from mkosi.util import _FILE, PathString +from mkosi.util import _FILE, PathString, copyfile @dataclasses.dataclass(frozen=True) @@ -171,7 +170,7 @@ class Apk(PackageManager): ) # fmt: skip keys = context.sandbox_tree / "etc/apk/keys" keys.mkdir(parents=True, exist_ok=True) - shutil.copy2(pub, keys / pub.name) + copyfile(pub, keys / pub.name) run( [ diff --git a/mkosi/qemu.py b/mkosi/qemu.py index 4d1bec156..dc10b62a7 100644 --- a/mkosi/qemu.py +++ b/mkosi/qemu.py @@ -57,6 +57,7 @@ from mkosi.user import INVOKING_USER, become_root_in_subuid_range, become_root_i from mkosi.util import ( PathString, StrEnum, + copyfile, flock, flock_or_die, groupby, @@ -724,7 +725,7 @@ def finalize_firmware_variables( ) if not vars.exists(): die(f"Firmware variables file {vars} does not exist") - shutil.copy(vars, ovmf_vars) + copyfile(vars, ovmf_vars) return ovmf_vars, ovmf_vars_format diff --git a/mkosi/sandbox.py b/mkosi/sandbox.py index 362349a8b..f03029d48 100755 --- a/mkosi/sandbox.py +++ b/mkosi/sandbox.py @@ -53,6 +53,7 @@ ENOSYS = 38 F_DUPFD = 0 F_GETFD = 1 FD_CLOEXEC = 1 +FICLONE = 0x40049409 FS_IOC_GETFLAGS = 0x80086601 FS_IOC_SETFLAGS = 0x40086602 FS_NOCOW_FL = 0x00800000 @@ -423,6 +424,15 @@ def btrfs_subvol_delete(path: str) -> None: btrfs_subvol_ioctl(path, BTRFS_IOC_SNAP_DESTROY_V2) +def reflink(src: str, dst: str) -> None: + with ( + close(os.open(src, os.O_CLOEXEC | os.O_RDONLY)) as src_fd, + close(os.open(dst, os.O_CLOEXEC | os.O_WRONLY | os.O_CREAT | os.O_TRUNC)) as dst_fd, + ): + if libc.ioctl(dst_fd, FICLONE, src_fd) < 0: + oserror("ioctl", src) + + def join_new_session_keyring() -> None: libkeyutils = ctypes.CDLL("libkeyutils.so.1") if libkeyutils is None: diff --git a/mkosi/util.py b/mkosi/util.py index 53e0a08eb..db705700b 100644 --- a/mkosi/util.py +++ b/mkosi/util.py @@ -15,6 +15,7 @@ import logging import os import re import resource +import shutil import stat import tempfile from collections.abc import Hashable, Iterable, Iterator, Mapping, Sequence @@ -24,6 +25,7 @@ from typing import IO, Any, Callable, Optional, Protocol, TypeVar, Union from mkosi.log import die from mkosi.resources import as_file +from mkosi.sandbox import reflink T = TypeVar("T") V = TypeVar("V") @@ -252,3 +254,25 @@ def mandatory_variable(name: str) -> str: return os.environ[name] except KeyError: die(f"${name} must be set in the environment") + + +def copyfile(src: PathString, dst: PathString) -> PathString: + if os.path.isdir(dst): + dst = os.path.join(dst, os.path.basename(src)) + + try: + reflink(os.fspath(src), os.fspath(dst)) + except OSError as e: + if e.errno not in (errno.EBADF, errno.EINVAL, errno.EXDEV, errno.ENOTTY, errno.EOPNOTSUPP): + raise e + + shutil.copyfile(src, dst) + + shutil.copymode(src, dst) + return dst + + +def copyfile2(src: PathString, dst: PathString) -> PathString: + dst = copyfile(src, dst) + shutil.copystat(src, dst) + return dst