copy: support getting progress feedback from the various copy functions

This adds two optional functions that may be passed to the various copy
functions. One is invoked whenever we start copying a new file object,
the other while we copy file payload in each loop iteration.

When the caller passes one or both they can get notifications about copy
progress, for example to log where things are.

diff --git a/src/basic/btrfs-util.c b/src/basic/btrfs-util.c
index 48e819a7cbeac33c31dde8605798de2ec8767af0..3252b75802251a7bd2450e433f1eac3bb27d0fb2 100644 (file)
--- a/src/basic/btrfs-util.c
+++ b/src/basic/btrfs-util.c
@@ -1503,7 +1503,12 @@ static int copy_subtree_quota_limits(int fd, uint64_t old_subvol, uint64_t new_s
         return changed;
 }
 
-static int subvol_snapshot_children(int old_fd, int new_fd, const char *subvolume, uint64_t old_subvol_id, BtrfsSnapshotFlags flags) {
+static int subvol_snapshot_children(
+                int old_fd,
+                int new_fd,
+                const char *subvolume,
+                uint64_t old_subvol_id,
+                BtrfsSnapshotFlags flags) {
 
         struct btrfs_ioctl_search_args args = {
                 .key.tree_id = BTRFS_ROOT_TREE_OBJECTID,
@@ -1683,7 +1688,14 @@ static int subvol_snapshot_children(int old_fd, int new_fd, const char *subvolum
         return 0;
 }
 
-int btrfs_subvol_snapshot_fd(int old_fd, const char *new_path, BtrfsSnapshotFlags flags) {
+int btrfs_subvol_snapshot_fd_full(
+                int old_fd,
+                const char *new_path,
+                BtrfsSnapshotFlags flags,
+                copy_progress_path_t progress_path,
+                copy_progress_bytes_t progress_bytes,
+                void *userdata) {
+
         _cleanup_close_ int new_fd = -1;
         const char *subvolume;
         int r;
@@ -1711,7 +1723,7 @@ int btrfs_subvol_snapshot_fd(int old_fd, const char *new_path, BtrfsSnapshotFlag
                 } else if (r < 0)
                         return r;
 
-                r = copy_directory_fd(old_fd, new_path, COPY_MERGE|COPY_REFLINK);
+                r = copy_directory_fd_full(old_fd, new_path, COPY_MERGE|COPY_REFLINK, progress_path, progress_bytes, userdata);
                 if (r < 0)
                         goto fallback_fail;
 
@@ -1748,7 +1760,14 @@ int btrfs_subvol_snapshot_fd(int old_fd, const char *new_path, BtrfsSnapshotFlag
         return subvol_snapshot_children(old_fd, new_fd, subvolume, 0, flags);
 }
 
-int btrfs_subvol_snapshot(const char *old_path, const char *new_path, BtrfsSnapshotFlags flags) {
+int btrfs_subvol_snapshot_full(
+                const char *old_path,
+                const char *new_path,
+                BtrfsSnapshotFlags flags,
+                copy_progress_path_t progress_path,
+                copy_progress_bytes_t progress_bytes,
+                void *userdata) {
+
         _cleanup_close_ int old_fd = -1;
 
         assert(old_path);
@@ -1758,7 +1777,7 @@ int btrfs_subvol_snapshot(const char *old_path, const char *new_path, BtrfsSnaps
         if (old_fd < 0)
                 return -errno;
 
-        return btrfs_subvol_snapshot_fd(old_fd, new_path, flags);
+        return btrfs_subvol_snapshot_fd_full(old_fd, new_path, flags, progress_path, progress_bytes, userdata);
 }
 
 int btrfs_qgroup_find_parents(int fd, uint64_t qgroupid, uint64_t **ret) {

diff --git a/src/basic/btrfs-util.h b/src/basic/btrfs-util.h
index b0cf6739f7755b77fd53ef5b6740a494d67f4794..3adb51e02d6fc6a497eb3a1f973e5fec18cb18fc 100644 (file)
--- a/src/basic/btrfs-util.h
+++ b/src/basic/btrfs-util.h
@@ -7,6 +7,7 @@
 
 #include "sd-id128.h"
 
+#include "copy.h"
 #include "time-util.h"
 
 typedef struct BtrfsSubvolInfo {
@@ -67,8 +68,15 @@ int btrfs_resize_loopback(const char *path, uint64_t size, bool grow_only);
 int btrfs_subvol_make(const char *path);
 int btrfs_subvol_make_fd(int fd, const char *subvolume);
 
-int btrfs_subvol_snapshot_fd(int old_fd, const char *new_path, BtrfsSnapshotFlags flags);
-int btrfs_subvol_snapshot(const char *old_path, const char *new_path, BtrfsSnapshotFlags flags);
+int btrfs_subvol_snapshot_fd_full(int old_fd, const char *new_path, BtrfsSnapshotFlags flags, copy_progress_path_t progress_path, copy_progress_bytes_t progress_bytes, void *userdata);
+static inline int btrfs_subvol_snapshot_fd(int old_fd, const char *new_path, BtrfsSnapshotFlags flags) {
+        return btrfs_subvol_snapshot_fd_full(old_fd, new_path, flags, NULL, NULL, NULL);
+}
+
+int btrfs_subvol_snapshot_full(const char *old_path, const char *new_path, BtrfsSnapshotFlags flags, copy_progress_path_t progress_path, copy_progress_bytes_t progress_bytes, void *userdata);
+static inline int btrfs_subvol_snapshot(const char *old_path, const char *new_path, BtrfsSnapshotFlags flags) {
+        return btrfs_subvol_snapshot_full(old_path, new_path, flags, NULL, NULL, NULL);
+}
 
 int btrfs_subvol_remove(const char *path, BtrfsRemoveFlags flags);
 int btrfs_subvol_remove_fd(int fd, const char *subvolume, BtrfsRemoveFlags flags);

diff --git a/src/basic/copy.c b/src/basic/copy.c
index 3efd9041c0a6b0cc5ad8456618e7575caea080e6..5cc3a9572bce2fbbf835a1ca8cadaa98d9af0824 100644 (file)
--- a/src/basic/copy.c
+++ b/src/basic/copy.c
@@ -90,7 +90,9 @@ int copy_bytes_full(
                 uint64_t max_bytes,
                 CopyFlags copy_flags,
                 void **ret_remains,
-                size_t *ret_remains_size) {
+                size_t *ret_remains_size,
+                copy_progress_bytes_t progress,
+                void *userdata) {
 
         bool try_cfr = true, try_sendfile = true, try_splice = true;
         int r, nonblock_pipe = -1;
@@ -308,10 +310,17 @@ int copy_bytes_full(
                 }
 
         next:
+                if (progress) {
+                        r = progress(n, userdata);
+                        if (r < 0)
+                                return r;
+                }
+
                 if (max_bytes != (uint64_t) -1) {
                         assert(max_bytes >= (uint64_t) n);
                         max_bytes -= n;
                 }
+
                 /* sendfile accepts at most SSIZE_MAX-offset bytes to copy,
                  * so reduce our maximum by the amount we already copied,
                  * but don't go below our copy buffer size, unless we are
@@ -363,7 +372,9 @@ static int fd_copy_regular(
                 const char *to,
                 uid_t override_uid,
                 gid_t override_gid,
-                CopyFlags copy_flags) {
+                CopyFlags copy_flags,
+                copy_progress_bytes_t progress,
+                void *userdata) {
 
         _cleanup_close_ int fdf = -1, fdt = -1;
         struct timespec ts[2];
@@ -381,7 +392,7 @@ static int fd_copy_regular(
         if (fdt < 0)
                 return -errno;
 
-        r = copy_bytes(fdf, fdt, (uint64_t) -1, copy_flags);
+        r = copy_bytes_full(fdf, fdt, (uint64_t) -1, copy_flags, NULL, NULL, progress, userdata);
         if (r < 0) {
                 (void) unlinkat(dt, to, 0);
                 return r;
@@ -483,7 +494,11 @@ static int fd_copy_directory(
                 unsigned depth_left,
                 uid_t override_uid,
                 gid_t override_gid,
-                CopyFlags copy_flags) {
+                CopyFlags copy_flags,
+                const char *display_path,
+                copy_progress_path_t progress_path,
+                copy_progress_bytes_t progress_bytes,
+                void *userdata) {
 
         _cleanup_close_ int fdf = -1, fdt = -1;
         _cleanup_closedir_ DIR *d = NULL;
@@ -524,6 +539,8 @@ static int fd_copy_directory(
         r = 0;
 
         FOREACH_DIRENT_ALL(de, d, return -errno) {
+                const char *child_display_path = NULL;
+                _cleanup_free_ char *dp = NULL;
                 struct stat buf;
                 int q;
 
@@ -535,6 +552,17 @@ static int fd_copy_directory(
                         continue;
                 }
 
+                if (progress_path) {
+                        if (display_path)
+                                child_display_path = dp = strjoin(display_path, "/", de->d_name);
+                        else
+                                child_display_path = de->d_name;
+
+                        r = progress_path(child_display_path, &buf, userdata);
+                        if (r < 0)
+                                return r;
+                }
+
                 if (S_ISDIR(buf.st_mode)) {
                         /*
                          * Don't descend into directories on other file systems, if this is requested. We do a simple
@@ -566,9 +594,9 @@ static int fd_copy_directory(
                                         continue;
                         }
 
-                        q = fd_copy_directory(dirfd(d), de->d_name, &buf, fdt, de->d_name, original_device, depth_left-1, override_uid, override_gid, copy_flags);
+                        q = fd_copy_directory(dirfd(d), de->d_name, &buf, fdt, de->d_name, original_device, depth_left-1, override_uid, override_gid, copy_flags, child_display_path, progress_path, progress_bytes, userdata);
                 } else if (S_ISREG(buf.st_mode))
-                        q = fd_copy_regular(dirfd(d), de->d_name, &buf, fdt, de->d_name, override_uid, override_gid, copy_flags);
+                        q = fd_copy_regular(dirfd(d), de->d_name, &buf, fdt, de->d_name, override_uid, override_gid, copy_flags, progress_bytes, userdata);
                 else if (S_ISLNK(buf.st_mode))
                         q = fd_copy_symlink(dirfd(d), de->d_name, &buf, fdt, de->d_name, override_uid, override_gid, copy_flags);
                 else if (S_ISFIFO(buf.st_mode))
@@ -606,7 +634,18 @@ static int fd_copy_directory(
         return r;
 }
 
-int copy_tree_at(int fdf, const char *from, int fdt, const char *to, uid_t override_uid, gid_t override_gid, CopyFlags copy_flags) {
+int copy_tree_at_full(
+                int fdf,
+                const char *from,
+                int fdt,
+                const char *to,
+                uid_t override_uid,
+                gid_t override_gid,
+                CopyFlags copy_flags,
+                copy_progress_path_t progress_path,
+                copy_progress_bytes_t progress_bytes,
+                void *userdata) {
+
         struct stat st;
 
         assert(from);
@@ -616,9 +655,9 @@ int copy_tree_at(int fdf, const char *from, int fdt, const char *to, uid_t overr
                 return -errno;
 
         if (S_ISREG(st.st_mode))
-                return fd_copy_regular(fdf, from, &st, fdt, to, override_uid, override_gid, copy_flags);
+                return fd_copy_regular(fdf, from, &st, fdt, to, override_uid, override_gid, copy_flags, progress_bytes, userdata);
         else if (S_ISDIR(st.st_mode))
-                return fd_copy_directory(fdf, from, &st, fdt, to, st.st_dev, COPY_DEPTH_MAX, override_uid, override_gid, copy_flags);
+                return fd_copy_directory(fdf, from, &st, fdt, to, st.st_dev, COPY_DEPTH_MAX, override_uid, override_gid, copy_flags, NULL, progress_path, progress_bytes, userdata);
         else if (S_ISLNK(st.st_mode))
                 return fd_copy_symlink(fdf, from, &st, fdt, to, override_uid, override_gid, copy_flags);
         else if (S_ISFIFO(st.st_mode))
@@ -629,11 +668,14 @@ int copy_tree_at(int fdf, const char *from, int fdt, const char *to, uid_t overr
                 return -EOPNOTSUPP;
 }
 
-int copy_tree(const char *from, const char *to, uid_t override_uid, gid_t override_gid, CopyFlags copy_flags) {
-        return copy_tree_at(AT_FDCWD, from, AT_FDCWD, to, override_uid, override_gid, copy_flags);
-}
+int copy_directory_fd_full(
+                int dirfd,
+                const char *to,
+                CopyFlags copy_flags,
+                copy_progress_path_t progress_path,
+                copy_progress_bytes_t progress_bytes,
+                void *userdata) {
 
-int copy_directory_fd(int dirfd, const char *to, CopyFlags copy_flags) {
         struct stat st;
 
         assert(dirfd >= 0);
@@ -645,10 +687,17 @@ int copy_directory_fd(int dirfd, const char *to, CopyFlags copy_flags) {
         if (!S_ISDIR(st.st_mode))
                 return -ENOTDIR;
 
-        return fd_copy_directory(dirfd, NULL, &st, AT_FDCWD, to, st.st_dev, COPY_DEPTH_MAX, UID_INVALID, GID_INVALID, copy_flags);
+        return fd_copy_directory(dirfd, NULL, &st, AT_FDCWD, to, st.st_dev, COPY_DEPTH_MAX, UID_INVALID, GID_INVALID, copy_flags, NULL, progress_path, progress_bytes, userdata);
 }
 
-int copy_directory(const char *from, const char *to, CopyFlags copy_flags) {
+int copy_directory_full(
+                const char *from,
+                const char *to,
+                CopyFlags copy_flags,
+                copy_progress_path_t progress_path,
+                copy_progress_bytes_t progress_bytes,
+                void *userdata) {
+
         struct stat st;
 
         assert(from);
@@ -660,10 +709,16 @@ int copy_directory(const char *from, const char *to, CopyFlags copy_flags) {
         if (!S_ISDIR(st.st_mode))
                 return -ENOTDIR;
 
-        return fd_copy_directory(AT_FDCWD, from, &st, AT_FDCWD, to, st.st_dev, COPY_DEPTH_MAX, UID_INVALID, GID_INVALID, copy_flags);
+        return fd_copy_directory(AT_FDCWD, from, &st, AT_FDCWD, to, st.st_dev, COPY_DEPTH_MAX, UID_INVALID, GID_INVALID, copy_flags, NULL, progress_path, progress_bytes, userdata);
 }
 
-int copy_file_fd(const char *from, int fdt, CopyFlags copy_flags) {
+int copy_file_fd_full(
+                const char *from,
+                int fdt,
+                CopyFlags copy_flags,
+                copy_progress_bytes_t progress_bytes,
+                void *userdata) {
+
         _cleanup_close_ int fdf = -1;
         int r;
 
@@ -674,7 +729,7 @@ int copy_file_fd(const char *from, int fdt, CopyFlags copy_flags) {
         if (fdf < 0)
                 return -errno;
 
-        r = copy_bytes(fdf, fdt, (uint64_t) -1, copy_flags);
+        r = copy_bytes_full(fdf, fdt, (uint64_t) -1, copy_flags, NULL, NULL, progress_bytes, userdata);
 
         (void) copy_times(fdf, fdt);
         (void) copy_xattr(fdf, fdt);
@@ -682,7 +737,16 @@ int copy_file_fd(const char *from, int fdt, CopyFlags copy_flags) {
         return r;
 }
 
-int copy_file(const char *from, const char *to, int flags, mode_t mode, unsigned chattr_flags, CopyFlags copy_flags) {
+int copy_file_full(
+                const char *from,
+                const char *to,
+                int flags,
+                mode_t mode,
+                unsigned chattr_flags,
+                CopyFlags copy_flags,
+                copy_progress_bytes_t progress_bytes,
+                void *userdata) {
+
         int fdt = -1, r;
 
         assert(from);
@@ -697,7 +761,7 @@ int copy_file(const char *from, const char *to, int flags, mode_t mode, unsigned
         if (chattr_flags != 0)
                 (void) chattr_fd(fdt, chattr_flags, (unsigned) -1, NULL);
 
-        r = copy_file_fd(from, fdt, copy_flags);
+        r = copy_file_fd_full(from, fdt, copy_flags, progress_bytes, userdata);
         if (r < 0) {
                 close(fdt);
                 (void) unlink(to);
@@ -712,7 +776,15 @@ int copy_file(const char *from, const char *to, int flags, mode_t mode, unsigned
         return 0;
 }
 
-int copy_file_atomic(const char *from, const char *to, mode_t mode, unsigned chattr_flags, CopyFlags copy_flags) {
+int copy_file_atomic_full(
+                const char *from,
+                const char *to,
+                mode_t mode,
+                unsigned chattr_flags,
+                CopyFlags copy_flags,
+                copy_progress_bytes_t progress_bytes,
+                void *userdata) {
+
         _cleanup_(unlink_and_freep) char *t = NULL;
         _cleanup_close_ int fdt = -1;
         int r;
@@ -745,7 +817,7 @@ int copy_file_atomic(const char *from, const char *to, mode_t mode, unsigned cha
         if (chattr_flags != 0)
                 (void) chattr_fd(fdt, chattr_flags, (unsigned) -1, NULL);
 
-        r = copy_file_fd(from, fdt, copy_flags);
+        r = copy_file_fd_full(from, fdt, copy_flags, progress_bytes, userdata);
         if (r < 0)
                 return r;
 

diff --git a/src/basic/copy.h b/src/basic/copy.h
index 6a0a6bc9b399418f97d15085dfc05c5a337186ae..a41b44c70ab9a82477e74a5463c1e55f15cd7cd0 100644 (file)
--- a/src/basic/copy.h
+++ b/src/basic/copy.h
@@ -1,9 +1,11 @@
 /* SPDX-License-Identifier: LGPL-2.1+ */
 #pragma once
 
+#include <fcntl.h>
 #include <inttypes.h>
 #include <stdbool.h>
 #include <stdint.h>
+#include <sys/stat.h>
 #include <sys/types.h>
 
 typedef enum CopyFlags {
@@ -13,16 +15,46 @@ typedef enum CopyFlags {
         COPY_SAME_MOUNT = 1 << 3, /* Don't descend recursively into other file systems, across mount point boundaries */
 } CopyFlags;
 
-int copy_file_fd(const char *from, int to, CopyFlags copy_flags);
-int copy_file(const char *from, const char *to, int open_flags, mode_t mode, unsigned chattr_flags, CopyFlags copy_flags);
-int copy_file_atomic(const char *from, const char *to, mode_t mode, unsigned chattr_flags, CopyFlags copy_flags);
-int copy_tree(const char *from, const char *to, uid_t override_uid, gid_t override_gid, CopyFlags copy_flags);
-int copy_tree_at(int fdf, const char *from, int fdt, const char *to, uid_t override_uid, gid_t override_gid, CopyFlags copy_flags);
-int copy_directory_fd(int dirfd, const char *to, CopyFlags copy_flags);
-int copy_directory(const char *from, const char *to, CopyFlags copy_flags);
-int copy_bytes_full(int fdf, int fdt, uint64_t max_bytes, CopyFlags copy_flags, void **ret_remains, size_t *ret_remains_size);
+typedef int (*copy_progress_bytes_t)(uint64_t n_bytes, void *userdata);
+typedef int (*copy_progress_path_t)(const char *path, const struct stat *st, void *userdata);
+
+int copy_file_fd_full(const char *from, int to, CopyFlags copy_flags, copy_progress_bytes_t progress, void *userdata);
+static inline int copy_file_fd(const char *from, int to, CopyFlags copy_flags) {
+        return copy_file_fd_full(from, to, copy_flags, NULL, NULL);
+}
+
+int copy_file_full(const char *from, const char *to, int open_flags, mode_t mode, unsigned chattr_flags, CopyFlags copy_flags, copy_progress_bytes_t progress, void *userdata);
+static inline int copy_file(const char *from, const char *to, int open_flags, mode_t mode, unsigned chattr_flags, CopyFlags copy_flags) {
+        return copy_file_full(from, to, open_flags, mode, chattr_flags, copy_flags, NULL, NULL);
+}
+
+int copy_file_atomic_full(const char *from, const char *to, mode_t mode, unsigned chattr_flags, CopyFlags copy_flags, copy_progress_bytes_t progress, void *userdata);
+static inline int copy_file_atomic(const char *from, const char *to, mode_t mode, unsigned chattr_flags, CopyFlags copy_flags) {
+        return copy_file_atomic_full(from, to, mode, chattr_flags, copy_flags, NULL, NULL);
+}
+
+int copy_tree_at_full(int fdf, const char *from, int fdt, const char *to, uid_t override_uid, gid_t override_gid, CopyFlags copy_flags, copy_progress_path_t progress_path, copy_progress_bytes_t progress_bytes, void *userdata);
+static inline int copy_tree_at(int fdf, const char *from, int fdt, const char *to, uid_t override_uid, gid_t override_gid, CopyFlags copy_flags) {
+        return copy_tree_at_full(fdf, from, fdt, to, override_uid, override_gid, copy_flags, NULL, NULL, NULL);
+}
+static inline int copy_tree(const char *from, const char *to, uid_t override_uid, gid_t override_gid, CopyFlags copy_flags) {
+        return copy_tree_at_full(AT_FDCWD, from, AT_FDCWD, to, override_uid, override_gid, copy_flags, NULL, NULL, NULL);
+}
+
+int copy_directory_fd_full(int dirfd, const char *to, CopyFlags copy_flags, copy_progress_path_t progress_path, copy_progress_bytes_t progress_bytes, void *userdata);
+static inline int copy_directory_fd(int dirfd, const char *to, CopyFlags copy_flags) {
+        return copy_directory_fd_full(dirfd, to, copy_flags, NULL, NULL, NULL);
+}
+
+int copy_directory_full(const char *from, const char *to, CopyFlags copy_flags, copy_progress_path_t progress_path, copy_progress_bytes_t progress_bytes, void *userdata);
+static inline int copy_directory(const char *from, const char *to, CopyFlags copy_flags) {
+        return copy_directory_full(from, to, copy_flags, NULL, NULL, NULL);
+}
+
+int copy_bytes_full(int fdf, int fdt, uint64_t max_bytes, CopyFlags copy_flags, void **ret_remains, size_t *ret_remains_size, copy_progress_bytes_t progress, void *userdata);
 static inline int copy_bytes(int fdf, int fdt, uint64_t max_bytes, CopyFlags copy_flags) {
-        return copy_bytes_full(fdf, fdt, max_bytes, copy_flags, NULL, NULL);
+        return copy_bytes_full(fdf, fdt, max_bytes, copy_flags, NULL, NULL, NULL, NULL);
 }
+
 int copy_times(int fdf, int fdt);
 int copy_xattr(int fdf, int fdt);

diff --git a/src/basic/fd-util.c b/src/basic/fd-util.c
index 5775a13e3e8067298195143d9ba8b88ae35eb98c..97dc40357ccbfff3721a8a44a6a7f8faf23232de 100644 (file)
--- a/src/basic/fd-util.c
+++ b/src/basic/fd-util.c
@@ -648,7 +648,7 @@ int fd_duplicate_data_fd(int fd) {
 
                         if ((size_t) isz >= DATA_FD_MEMORY_LIMIT) {
 
-                                r = copy_bytes_full(fd, pipefds[1], DATA_FD_MEMORY_LIMIT, 0, &remains, &remains_size);
+                                r = copy_bytes_full(fd, pipefds[1], DATA_FD_MEMORY_LIMIT, 0, &remains, &remains_size, NULL, NULL);
                                 if (r < 0 && r != -EAGAIN)
                                         return r; /* If we get EAGAIN it could be because of the source or because of
                                                    * the destination fd, we can't know, as sendfile() and friends won't