From 892f3b2352bc64601cc5fd78bd764150ec225d00 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= Date: Fri, 27 Oct 2017 17:26:10 +0200 Subject: [PATCH] Add an internal tree copy implementation The general idea is to copy everything, preserving access mode, times, xattrs on files. An attempt to create the new object is first made, followed by an unlink and a repeated attempt. This optimizes for the case where the output tree is empty, which seems to be the common thing in our case. Reflink is attempted for normal files. Symlinks are not adjusted: if they pointed outside of the tree being copied, they might be dangling now. Block and character device nodes and any other special files are not copied. --- mkosi | 106 +++++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 97 insertions(+), 9 deletions(-) diff --git a/mkosi b/mkosi index d604ac4f1..5fa811ab1 100755 --- a/mkosi +++ b/mkosi @@ -6,9 +6,12 @@ import configparser import contextlib import ctypes, ctypes.util import crypt +import errno +import fcntl import getpass import hashlib import os +import pathlib import platform import shutil import stat @@ -133,6 +136,94 @@ def mkdir_last(path, mode=0o777): raise return path +_IOC_NRBITS = 8 +_IOC_TYPEBITS = 8 +_IOC_SIZEBITS = 14 +_IOC_DIRBITS = 2 + +_IOC_NRSHIFT = 0 +_IOC_TYPESHIFT = _IOC_NRSHIFT + _IOC_NRBITS +_IOC_SIZESHIFT = _IOC_TYPESHIFT + _IOC_TYPEBITS +_IOC_DIRSHIFT = _IOC_SIZESHIFT + _IOC_SIZEBITS + +_IOC_NONE = 0 +_IOC_WRITE = 1 +_IOC_READ = 2 + +def _IOC(dir, type, nr, argtype): + size = {'int':4, 'size_t':8}[argtype] + return dir<<_IOC_DIRSHIFT | type<<_IOC_TYPESHIFT | nr<<_IOC_NRSHIFT | size<<_IOC_SIZESHIFT +def _IOW(type, nr, size): + return _IOC(_IOC_WRITE, type, nr, size) + +FICLONE = _IOW(0x94, 9, 'int') + +@contextlib.contextmanager +def open_close(path, flags, mode=0o664): + fd = os.open(path, flags | os.O_CLOEXEC, mode) + try: + yield fd + finally: + os.close(fd) + +def _reflink(oldfd, newfd): + fcntl.ioctl(newfd, FICLONE, oldfd) + +def _copy_file(oldfd, newfd): + try: + _reflink(oldfd, newfd) + except OSError as e: + if e.errno not in {errno.EXDEV, errno.EOPNOTSUPP}: + raise + shutil.copyfileobj(open(oldfd, 'rb', closefd=False), + open(newfd, 'wb', closefd=False)) + +def copy_file(oldpath, newpath): + with open_close(oldpath, os.O_RDONLY) as oldfd: + st = os.stat(oldfd) + + try: + with open_close(newpath, os.O_WRONLY|os.O_CREAT, st.st_mode) as newfd: + _copy_file(oldfd, newfd) + except FileExistsError: + os.unlink(newpath) + with open_close(newpath, os.O_WRONLY|os.O_CREAT, st.st_mode) as newfd: + _copy_file(oldfd, newfd) + +def symlink_f(target, path): + try: + os.symlink(target, path) + except FileExistsError: + os.unlink(path) + os.symlink(target, path) + +def copy(oldpath, newpath): + if not isinstance(newpath, pathlib.Path): + newpath = pathlib.Path(newpath) + + try: + mkdir_last(newpath) + except FileExistsError: + # something that is not a directory already exists + os.unlink(path) + mkdir_last(newpath) + + for entry in os.scandir(oldpath): + newentry = newpath / entry.name + if entry.is_dir(follow_symlinks=False): + copy(entry.path, newentry) + elif entry.is_symlink(): + target = os.readlink(entry.path) + symlink_f(target, newentry) + else: + st = entry.stat(follow_symlinks=False) + if stat.S_ISREG(st.st_mode): + copy_file(entry.path, newentry) + else: + print('Ignoring', entry.path) + continue + shutil.copystat(entry.path, newentry, follow_symlinks=False) + @contextlib.contextmanager def complete_step(text, text2=None): print_step(text + '...') @@ -1430,9 +1521,6 @@ def install_boot_loader(args, workspace, cached): if args.distribution == Distribution.opensuse: install_boot_loader_opensuse(args, workspace) -def enumerate_and_copy(source, dest): - subprocess.run(["cp", "--reflink=auto", "--recursive", "--no-dereference", "--preserve=all", "--no-target-directory", source, dest], check=True) - def install_extra_trees(args, workspace, for_cache): if args.extra_trees is None: return @@ -1442,7 +1530,7 @@ def install_extra_trees(args, workspace, for_cache): with complete_step('Copying in extra file trees'): for d in args.extra_trees: - enumerate_and_copy(d, os.path.join(workspace, "root")) + copy(d, os.path.join(workspace, "root")) def copy_git_files(src, dest, *, git_files): subprocess.run(['git', 'clone', '--depth=1', '--recursive', '--shallow-submodules', src, dest], @@ -1476,7 +1564,7 @@ def copy_git_files(src, dest, *, git_files): directory = os.path.dirname(dest_path) os.makedirs(directory, exist_ok=True) - shutil.copy2(src_path, dest_path, follow_symlinks=False) + copy_file(src_path, dest_path) def install_build_src(args, workspace, run_build_script, for_cache): if not run_build_script: @@ -1488,8 +1576,8 @@ def install_build_src(args, workspace, run_build_script, for_cache): return with complete_step('Copying in build script and sources'): - shutil.copy(args.build_script, - os.path.join(workspace, "root", "root", os.path.basename(args.build_script))) + copy_file(args.build_script, + os.path.join(workspace, "root", "root", os.path.basename(args.build_script))) if args.build_sources is not None: target = os.path.join(workspace, "root", "root/src") @@ -1519,7 +1607,7 @@ def install_build_dest(args, workspace, run_build_script, for_cache): return with complete_step('Copying in build tree'): - enumerate_and_copy(os.path.join(workspace, "dest"), os.path.join(workspace, "root")) + copy(os.path.join(workspace, "dest"), os.path.join(workspace, "root")) def make_read_only(args, workspace, for_cache): if not args.read_only: @@ -2899,7 +2987,7 @@ def reuse_cache_tree(args, workspace, run_build_script, for_cache, cached): with complete_step('Copying in cached tree ' + fname): try: - enumerate_and_copy(fname, os.path.join(workspace, "root")) + copy(fname, os.path.join(workspace, "root")) except FileNotFoundError: return False -- 2.47.2