]> git.ipfire.org Git - pakfire.git/commitdiff
pakfire: Experimental implementation to use overlayfs for snapshots
authorMichael Tremer <michael.tremer@ipfire.org>
Sun, 13 Oct 2024 13:24:44 +0000 (13:24 +0000)
committerMichael Tremer <michael.tremer@ipfire.org>
Sun, 13 Oct 2024 13:24:44 +0000 (13:24 +0000)
Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
src/cli/lib/build.c
src/cli/lib/pakfire.c
src/cli/lib/shell.c
src/libpakfire/build.c
src/libpakfire/include/pakfire/pakfire.h
src/libpakfire/include/pakfire/snapshot.h
src/libpakfire/pakfire.c
src/libpakfire/snapshot.c

index c3d23e1f444b14e4c2499efa6ba259cff23a7801..b787f040b616192a3d6f459606d77cb241c2a431 100644 (file)
@@ -139,7 +139,7 @@ int cli_build(void* data, int argc, char* argv[]) {
        struct config config = {
                .id     = NULL,
                .target = NULL,
-               .flags  = PAKFIRE_BUILD_INTERACTIVE,
+               .flags  = PAKFIRE_USE_SNAPSHOT,
        };
        int r;
 
index b3c497869628d20f63fe259ac307132ff946e284..402c207a708a9daed1a13a59b43a0076ecf1aa22 100644 (file)
@@ -94,8 +94,10 @@ int cli_setup_pakfire(struct pakfire** pakfire, struct cli_config* config) {
 
        // Initialize Pakfire
        r = pakfire_create(&p, config->ctx, config->root, config->arch, f, config->flags);
-       if (r)
+       if (r < 0) {
+               CTX_ERROR(config->ctx, "Could not initialize Pakfire: %s\n", strerror(-r));
                goto ERROR;
+       }
 
        // Enable repositories
        for (unsigned int i = 0; i < config->num_enable_repos; i++)
index c9e6f3f4b8a3478964231962cbc70ef0944d94de..31a43e8e97d5bef2cf01f9c0b1096d106998c2c5 100644 (file)
@@ -92,6 +92,9 @@ int cli_shell(void* data, int argc, char* argv[]) {
 
        struct cli_config* cli_config = data;
 
+       // XXX DEBUG
+       cli_config->flags |= PAKFIRE_USE_SNAPSHOT;
+
        struct config config = {
                .flags        = PAKFIRE_BUILD_INTERACTIVE,
                .argv         = {},
index 88a1fbd278ffb462df44b28c8371f20dde03e00c..57809f23717f07cc5ec158c0a30b296026fd83a6 100644 (file)
@@ -1732,18 +1732,21 @@ static int pakfire_build_init(struct pakfire_build* build) {
        // Tells us whether we need to (re-)create the snapshot
        int snapshot_needs_update = 0;
 
+#if 0
        // Extract snapshot
        if (use_snapshot) {
                r = pakfire_snapshot_restore(build->pakfire);
                if (r)
                        return r;
        }
+#endif
 
        // Install or update any build dependencies
        r = pakfire_build_install_packages(build, &snapshot_needs_update);
        if (r)
                return r;
 
+#if 0
        // Update the snapshot if there were changes
        if (use_snapshot && snapshot_needs_update) {
                // Store the snapshot
@@ -1751,6 +1754,7 @@ static int pakfire_build_init(struct pakfire_build* build) {
                if (r)
                        return r;
        }
+#endif
 
        // Mark as ready
        build->state = PAKFIRE_BUILD_READY;
index 32836b3cec1c49f87b4eaaf734f67591a4ee6eb5..ff79d10b62bba5053b892f9e38cc8480598ed3e9 100644 (file)
@@ -41,6 +41,7 @@ struct pakfire;
 
 enum pakfire_flags {
        PAKFIRE_FLAGS_BUILD             = (1 << 0),
+       PAKFIRE_USE_SNAPSHOT            = (1 << 1),
 };
 
 // Callbacks
index 7fb5b6d01c4b4d87ddef3c311beb6d4384500566..6aa18201e55232f201daa537367c8c6b4e90f125 100644 (file)
 
 #include <pakfire/pakfire.h>
 
+struct pakfire_snapshot;
+
+int pakfire_snapshot_create(
+       struct pakfire_snapshot** snapshot, struct pakfire_ctx* ctx, const char* path);
+
+struct pakfire_snapshot* pakfire_snapshot_ref(struct pakfire_snapshot* snapshot);
+struct pakfire_snapshot* pakfire_snapshot_unref(struct pakfire_snapshot* snapshot);
+
+int pakfire_snapshot_find(struct pakfire_snapshot** snapshot, struct pakfire* pakfire);
+
+const char* pakfire_snapshot_path(struct pakfire_snapshot* snapshot);
+
+int pakfire_snapshot_mount(struct pakfire_snapshot* snapshot);
+int pakfire_snapshot_umount(struct pakfire_snapshot* snapshot);
+
 int pakfire_snapshot_store(struct pakfire* pakfire);
 int pakfire_snapshot_restore(struct pakfire* pakfire);
 
index b443e0a80ed63ce1cf5c49d2e108f640d86ba45d..8223d77a01ecd4602149673a3960ef13cc489ab7 100644 (file)
@@ -60,6 +60,7 @@
 #include <pakfire/private.h>
 #include <pakfire/pwd.h>
 #include <pakfire/repo.h>
+#include <pakfire/snapshot.h>
 #include <pakfire/string.h>
 #include <pakfire/transaction.h>
 #include <pakfire/util.h>
@@ -103,6 +104,9 @@ struct pakfire {
        // Distro
        struct pakfire_distro distro;
 
+       // Snapshot
+       struct pakfire_snapshot* snapshot;
+
        // States
        unsigned int destroy_on_free:1;
        unsigned int pool_ready:1;
@@ -384,6 +388,10 @@ static void pakfire_free(struct pakfire* pakfire) {
                pakfire_repo_unref(repo);
        }
 
+       // Umount the snapshot
+       if (pakfire->snapshot)
+               pakfire_snapshot_unref(pakfire->snapshot);
+
        if (pakfire->destroy_on_free && *pakfire->path) {
                CTX_DEBUG(pakfire->ctx, "Destroying %s\n", pakfire->path);
 
@@ -637,10 +645,12 @@ static int pakfire_read_config(struct pakfire* pakfire, FILE* f) {
        if (r)
                return r;
 
+#if 0
        // Read repository configuration
        r = pakfire_read_repo_config(pakfire);
        if (r)
                return r;
+#endif
 
        return 0;
 }
@@ -785,7 +795,29 @@ PAKFIRE_EXPORT int pakfire_create(struct pakfire** pakfire, struct pakfire_ctx*
        p->arches.effective = pakfire_arch_is_supported_by_host(arch);
        if (!p->arches.effective) {
                CTX_ERROR(p->ctx, "Unsupported architecture: %s\n", arch);
-               r = errno;
+               r = -errno;
+               goto ERROR;
+       }
+
+       // Setup user/group
+       r = pakfire_setup_user(p);
+       if (r)
+               goto ERROR;
+
+       // Initialise configuration
+       r = pakfire_config_create(&p->config);
+       if (r < 0)
+               goto ERROR;
+
+       // Read configuration file
+       r = pakfire_read_config(p, conf);
+       if (r < 0)
+               goto ERROR;
+
+       // Set cache path
+       r = pakfire_set_cache_path(p);
+       if (r < 0) {
+               CTX_ERROR(p->ctx, "Could not set cache path: %m\n");
                goto ERROR;
        }
 
@@ -796,6 +828,34 @@ PAKFIRE_EXPORT int pakfire_create(struct pakfire** pakfire, struct pakfire_ctx*
                goto ERROR;
        }
 
+       // If using a snapshot, path cannot be set
+       if (p->flags & PAKFIRE_USE_SNAPSHOT) {
+               if (path) {
+                       CTX_ERROR(p->ctx, "Cannot use snapshots with path\n");
+                       r = -EINVAL;
+                       goto ERROR;
+               }
+
+               // Find the most recent snapshot
+               r = pakfire_snapshot_find(&p->snapshot, p);
+               if (r < 0) {
+                       CTX_ERROR(p->ctx, "Could not find a snapshot: %s\n", strerror(-r));
+                       goto ERROR;
+               }
+
+               // Mount the snapshot
+               if (p->snapshot) {
+                       r = pakfire_snapshot_mount(p->snapshot);
+                       if (r < 0) {
+                               CTX_ERROR(p->ctx, "Could not mount snapshot: %s\n", strerror(-r));
+                               goto ERROR;
+                       }
+
+                       // Fetch the path where the snapshot was mounted
+                       path = pakfire_snapshot_path(p->snapshot);
+               }
+       }
+
        // Create a ramdisk if no path was given
        if (!path) {
                r = pakfire_make_ramdisk(p, tempdir, NULL);
@@ -812,16 +872,6 @@ PAKFIRE_EXPORT int pakfire_create(struct pakfire** pakfire, struct pakfire_ctx*
        // Set path
        pakfire_string_set(p->path, path);
 
-       // Setup user/group
-       r = pakfire_setup_user(p);
-       if (r)
-               goto ERROR;
-
-       // Initialise configuration
-       r = pakfire_config_create(&p->config);
-       if (r)
-               goto ERROR;
-
        // Read /etc/os-release
        r = pakfire_read_os_release(p);
        if (r && errno != ENOENT)
@@ -847,11 +897,6 @@ PAKFIRE_EXPORT int pakfire_create(struct pakfire** pakfire, struct pakfire_ctx*
        if (r)
                goto ERROR;
 
-       // Read configuration file
-       r = pakfire_read_config(p, conf);
-       if (r)
-               goto ERROR;
-
        // Dump distribution configuration
        CTX_DEBUG(p->ctx, "  Distribution: %s\n", p->distro.pretty_name);
        CTX_DEBUG(p->ctx, "    name       = %s\n", p->distro.name);
@@ -865,13 +910,6 @@ PAKFIRE_EXPORT int pakfire_create(struct pakfire** pakfire, struct pakfire_ctx*
        if (*p->distro.slogan)
                CTX_DEBUG(p->ctx, "    slogan     = %s\n", p->distro.slogan);
 
-       // Set cache path
-       r = pakfire_set_cache_path(p);
-       if (r) {
-               CTX_ERROR(p->ctx, "Could not set cache path: %m\n");
-               goto ERROR;
-       }
-
        // Make path for private files
        r = pakfire_path(p, private_dir, "%s", PAKFIRE_PRIVATE_DIR);
        if (r)
@@ -969,11 +1007,11 @@ int __pakfire_cache_path(struct pakfire* pakfire, char* path, size_t length,
 
        // Format input into buffer
        va_start(args, format);
-       r = __pakfire_string_vformat(buffer, sizeof(buffer), format, args);
+       r = pakfire_string_vformat(buffer, format, args);
        va_end(args);
 
        // Break on any errors
-       if (r)
+       if (r < 0)
                return r;
 
        // Join paths together
index d4362d8a786bda4d6f2467bce870e1b4ab8b28c2..86056529e71481be319410be250a26f005dcbadb 100644 (file)
 #############################################################################*/
 
 #include <errno.h>
+#include <dirent.h>
+#include <fcntl.h>
 #include <linux/limits.h>
 #include <stdio.h>
 #include <stdlib.h>
+#include <sys/mount.h>
 
 #include <archive.h>
 
 #include <pakfire/filelist.h>
 #include <pakfire/i18n.h>
 #include <pakfire/logging.h>
+#include <pakfire/path.h>
 #include <pakfire/private.h>
 #include <pakfire/repo.h>
 #include <pakfire/snapshot.h>
 #include <pakfire/string.h>
 #include <pakfire/util.h>
 
+struct pakfire_snapshot_fs {
+       char path[PATH_MAX];
+       int fd;
+};
+
+struct pakfire_snapshot {
+       struct pakfire_ctx* ctx;
+       int nrefs;
+
+       // Path
+       char path[PATH_MAX];
+       int fd;
+
+       // File Systems
+       struct pakfire_snapshot_fs overlayfs;
+       struct pakfire_snapshot_fs tmpfs;
+
+       char upperdir[PATH_MAX];
+       char workdir[PATH_MAX];
+};
+
+static void pakfire_snapshot_free(struct pakfire_snapshot* snapshot) {
+       pakfire_snapshot_umount(snapshot);
+
+       if (snapshot->overlayfs.fd >= 0)
+               close(snapshot->overlayfs.fd);
+       if (snapshot->tmpfs.fd >= 0)
+               close(snapshot->tmpfs.fd);
+       if (snapshot->fd >= 0)
+               close(snapshot->fd);
+
+       if (snapshot->ctx)
+               pakfire_ctx_unref(snapshot->ctx);
+       free(snapshot);
+}
+
+int pakfire_snapshot_create(
+               struct pakfire_snapshot** snapshot, struct pakfire_ctx* ctx, const char* path) {
+       struct pakfire_snapshot* s = NULL;
+       int r;
+
+       // Allocate a new snapshot
+       s = calloc(1, sizeof(*s));
+       if (!s)
+               return -errno;
+
+       // Store a reference to the context
+       s->ctx = pakfire_ctx_ref(ctx);
+
+       // Initialize the reference counter
+       s->nrefs = 1;
+
+       // Store the path
+       r = pakfire_string_set(s->path, path);
+       if (r < 0)
+               goto ERROR;
+
+       // Open the directory
+       s->fd = open(path, O_DIRECTORY|O_CLOEXEC);
+       if (s->fd < 0) {
+               r = -errno;
+               goto ERROR;
+       }
+
+       // Initialize the file descriptors
+       s->overlayfs.fd = s->tmpfs.fd = -EBADF;
+
+       // XXX Check and lock the snapshot
+
+       // Return the snapshot
+       *snapshot = pakfire_snapshot_ref(s);
+
+ERROR:
+       if (s)
+               pakfire_snapshot_unref(s);
+
+       return r;
+}
+
+struct pakfire_snapshot* pakfire_snapshot_ref(struct pakfire_snapshot* snapshot) {
+       ++snapshot->nrefs;
+
+       return snapshot;
+}
+
+struct pakfire_snapshot* pakfire_snapshot_unref(struct pakfire_snapshot* snapshot) {
+       if (--snapshot->nrefs > 0)
+               return snapshot;
+
+       pakfire_snapshot_free(snapshot);
+       return NULL;
+}
+
+static int pakfire_snapshot_filter(const struct dirent* dirent) {
+       // Skip any hidden directories
+       if (*dirent->d_name == '.')
+               return 0;
+
+       // Select the rest
+       return 1;
+}
+
+/*
+       Finds and returns the latest snapshot
+*/
+int pakfire_snapshot_find(struct pakfire_snapshot** snapshot, struct pakfire* pakfire) {
+       struct dirent** paths = NULL;
+       int num_paths = 0;
+       char path[PATH_MAX];
+       int fd = -EBADF;
+       int r;
+
+       struct pakfire_ctx* ctx = pakfire_ctx(pakfire);
+
+       // Make the path
+       r = pakfire_cache_path(pakfire, path, "%s", "snapshots");
+       if (r < 0)
+               return r;
+
+       // Open the path
+       fd = open(path, O_DIRECTORY|O_CLOEXEC);
+       if (fd < 0) {
+               switch (errno) {
+                       // If the directory does not exist, we cannot find a snapshot
+                       case ENOENT:
+                               r = 0;
+                               goto ERROR;
+
+                       default:
+                               r = -errno;
+                               goto ERROR;
+               }
+       }
+
+       // Find all snapshots in the directory
+       num_paths = scandirat(fd, ".", &paths, pakfire_snapshot_filter, versionsort);
+       if (num_paths < 0) {
+               r = num_paths;
+               goto ERROR;
+
+       // We are done if we did not find anything
+       } else if (num_paths == 0) {
+               r = 0;
+               goto ERROR;
+       }
+
+       // List what we have found
+       for (int i = 0; i < num_paths; i++) {
+               CTX_DEBUG(ctx, "Found snapshot: %s\n", paths[i]->d_name);
+       }
+
+       // Build the path to the snapshot
+       r = pakfire_path_append(path, path, paths[num_paths - 1]->d_name);
+       if (r < 0)
+               goto ERROR;
+
+       CTX_DEBUG(ctx, "Selecting snapshot %s\n", path);
+
+       // Open the snapshot
+       r = pakfire_snapshot_create(snapshot, ctx, path);
+       if (r < 0)
+               goto ERROR;
+
+ERROR:
+       if (paths) {
+               for (int i = 0; i < num_paths; i++)
+                       free(paths[i]);
+               free(paths);
+       }
+       if (fd >= 0)
+               close(fd);
+       if (ctx)
+               pakfire_ctx_unref(ctx);
+
+       return r;
+}
+
+const char* pakfire_snapshot_path(struct pakfire_snapshot* snapshot) {
+       return snapshot->overlayfs.path;
+}
+
+static int pakfire_snapshot_mount_tmpfs(struct pakfire_snapshot* snapshot) {
+       char* path = NULL;
+       int r;
+
+       // Make path
+       r = pakfire_string_set(snapshot->tmpfs.path, PAKFIRE_TMP_DIR "/pakfire-tmpfs.XXXXXX");
+       if (r < 0)
+               return r;
+
+       // Create a temporary directory
+       path = pakfire_mkdtemp(snapshot->tmpfs.path);
+       if (!path)
+               return -errno;
+
+       // Perform mount
+       r = mount("pakfire_tmpfs", snapshot->tmpfs.path, "tmpfs", 0, NULL);
+       if (r < 0)
+               return -errno;
+
+       // Make the upper directory
+       r = pakfire_path_append(snapshot->upperdir, snapshot->tmpfs.path, "upper");
+       if (r < 0)
+               return r;
+
+       // Create the upper directory
+       r = pakfire_mkdir(snapshot->upperdir, 0755);
+       if (r < 0)
+               return r;
+
+       // Make the work directory
+       r = pakfire_path_append(snapshot->workdir, snapshot->tmpfs.path, "work");
+       if (r < 0)
+               return r;
+
+       // Create the work directory
+       r = pakfire_mkdir(snapshot->workdir, 0755);
+       if (r < 0)
+               return r;
+
+       return 0;
+}
+
+int pakfire_snapshot_mount(struct pakfire_snapshot* snapshot) {
+       const char* path = NULL;
+       int mountfd = -EBADF;
+       int fsfd = -EBADF;
+       int r;
+
+       CTX_DEBUG(snapshot->ctx, "Mounting snapshot %s\n", snapshot->path);
+
+       // Mount the tmpfs
+       r = pakfire_snapshot_mount_tmpfs(snapshot);
+       if (r < 0)
+               goto ERROR;
+
+       // Create a new overlayfs
+       fsfd = fsopen("overlay", FSOPEN_CLOEXEC);
+       if (fsfd < 0) {
+               r = -errno;
+               goto ERROR;
+       }
+
+       // Set the lower directory
+       r = fsconfig(fsfd, FSCONFIG_SET_STRING, "lowerdir", snapshot->path, 0);
+       if (r < 0)
+               goto ERROR;
+
+       // Set the upper directory
+       r = fsconfig(fsfd, FSCONFIG_SET_STRING, "upperdir", snapshot->upperdir, 0);
+       if (r < 0)
+               goto ERROR;
+
+       // Set some work directory
+       r = fsconfig(fsfd, FSCONFIG_SET_STRING, "workdir", snapshot->workdir, 0);
+       if (r < 0)
+               goto ERROR;
+
+       // Create a new filesystem
+       r = fsconfig(fsfd, FSCONFIG_CMD_CREATE, NULL, NULL, 0);
+       if (r < 0)
+               goto ERROR;
+
+       // Perform mount
+       mountfd = fsmount(fsfd, FSMOUNT_CLOEXEC, 0);
+       if (mountfd < 0) {
+               r = -errno;
+               goto ERROR;
+       }
+
+       // Make path
+       r = pakfire_string_set(snapshot->overlayfs.path, PAKFIRE_TMP_DIR "/pakfire-root.XXXXXX");
+       if (r < 0)
+               goto ERROR;
+
+       // Create a temporary directory
+       path = pakfire_mkdtemp(snapshot->overlayfs.path);
+       if (!path)
+               goto ERROR;
+
+       // Move the snapshot to the right place
+       r = move_mount(mountfd, "", AT_FDCWD, path, MOVE_MOUNT_F_EMPTY_PATH);
+       if (r < 0)
+               goto ERROR;
+
+ERROR:
+       if (mountfd >= 0)
+               close(mountfd);
+       if (fsfd >= 0)
+               close(fsfd);
+
+       return r;
+}
+
+int pakfire_snapshot_umount(struct pakfire_snapshot* snapshot) {
+       int r;
+
+       // Umount the overlayfs
+       if (*snapshot->overlayfs.path) {
+               r = umount(snapshot->overlayfs.path);
+               if (r < 0)
+                       return r;
+       }
+
+       // Umount the tmpfs
+       if (*snapshot->tmpfs.path) {
+               r = umount(snapshot->tmpfs.path);
+               if (r < 0)
+                       return r;
+       }
+
+       return 0;
+}
+
 #define pakfire_snapshot_path(pakfire, path) \
        __pakfire_snapshot_path(pakfire, path, sizeof(path))