From: Michael Tremer Date: Sun, 13 Oct 2024 13:24:44 +0000 (+0000) Subject: pakfire: Experimental implementation to use overlayfs for snapshots X-Git-Tag: 0.9.30~1041 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=574393d69c212d9d39c6b3c69c94702fb7f2fd63;p=pakfire.git pakfire: Experimental implementation to use overlayfs for snapshots Signed-off-by: Michael Tremer --- diff --git a/src/cli/lib/build.c b/src/cli/lib/build.c index c3d23e1f4..b787f040b 100644 --- a/src/cli/lib/build.c +++ b/src/cli/lib/build.c @@ -139,7 +139,7 @@ int cli_build(void* data, int argc, char* argv[]) { struct config config = { .id = NULL, .target = NULL, - .flags = PAKFIRE_BUILD_INTERACTIVE, + .flags = PAKFIRE_USE_SNAPSHOT, }; int r; diff --git a/src/cli/lib/pakfire.c b/src/cli/lib/pakfire.c index b3c497869..402c207a7 100644 --- a/src/cli/lib/pakfire.c +++ b/src/cli/lib/pakfire.c @@ -94,8 +94,10 @@ int cli_setup_pakfire(struct pakfire** pakfire, struct cli_config* config) { // Initialize Pakfire r = pakfire_create(&p, config->ctx, config->root, config->arch, f, config->flags); - if (r) + if (r < 0) { + CTX_ERROR(config->ctx, "Could not initialize Pakfire: %s\n", strerror(-r)); goto ERROR; + } // Enable repositories for (unsigned int i = 0; i < config->num_enable_repos; i++) diff --git a/src/cli/lib/shell.c b/src/cli/lib/shell.c index c9e6f3f4b..31a43e8e9 100644 --- a/src/cli/lib/shell.c +++ b/src/cli/lib/shell.c @@ -92,6 +92,9 @@ int cli_shell(void* data, int argc, char* argv[]) { struct cli_config* cli_config = data; + // XXX DEBUG + cli_config->flags |= PAKFIRE_USE_SNAPSHOT; + struct config config = { .flags = PAKFIRE_BUILD_INTERACTIVE, .argv = {}, diff --git a/src/libpakfire/build.c b/src/libpakfire/build.c index 88a1fbd27..57809f237 100644 --- a/src/libpakfire/build.c +++ b/src/libpakfire/build.c @@ -1732,18 +1732,21 @@ static int pakfire_build_init(struct pakfire_build* build) { // Tells us whether we need to (re-)create the snapshot int snapshot_needs_update = 0; +#if 0 // Extract snapshot if (use_snapshot) { r = pakfire_snapshot_restore(build->pakfire); if (r) return r; } +#endif // Install or update any build dependencies r = pakfire_build_install_packages(build, &snapshot_needs_update); if (r) return r; +#if 0 // Update the snapshot if there were changes if (use_snapshot && snapshot_needs_update) { // Store the snapshot @@ -1751,6 +1754,7 @@ static int pakfire_build_init(struct pakfire_build* build) { if (r) return r; } +#endif // Mark as ready build->state = PAKFIRE_BUILD_READY; diff --git a/src/libpakfire/include/pakfire/pakfire.h b/src/libpakfire/include/pakfire/pakfire.h index 32836b3ce..ff79d10b6 100644 --- a/src/libpakfire/include/pakfire/pakfire.h +++ b/src/libpakfire/include/pakfire/pakfire.h @@ -41,6 +41,7 @@ struct pakfire; enum pakfire_flags { PAKFIRE_FLAGS_BUILD = (1 << 0), + PAKFIRE_USE_SNAPSHOT = (1 << 1), }; // Callbacks diff --git a/src/libpakfire/include/pakfire/snapshot.h b/src/libpakfire/include/pakfire/snapshot.h index 7fb5b6d01..6aa18201e 100644 --- a/src/libpakfire/include/pakfire/snapshot.h +++ b/src/libpakfire/include/pakfire/snapshot.h @@ -27,6 +27,21 @@ #include +struct pakfire_snapshot; + +int pakfire_snapshot_create( + struct pakfire_snapshot** snapshot, struct pakfire_ctx* ctx, const char* path); + +struct pakfire_snapshot* pakfire_snapshot_ref(struct pakfire_snapshot* snapshot); +struct pakfire_snapshot* pakfire_snapshot_unref(struct pakfire_snapshot* snapshot); + +int pakfire_snapshot_find(struct pakfire_snapshot** snapshot, struct pakfire* pakfire); + +const char* pakfire_snapshot_path(struct pakfire_snapshot* snapshot); + +int pakfire_snapshot_mount(struct pakfire_snapshot* snapshot); +int pakfire_snapshot_umount(struct pakfire_snapshot* snapshot); + int pakfire_snapshot_store(struct pakfire* pakfire); int pakfire_snapshot_restore(struct pakfire* pakfire); diff --git a/src/libpakfire/pakfire.c b/src/libpakfire/pakfire.c index b443e0a80..8223d77a0 100644 --- a/src/libpakfire/pakfire.c +++ b/src/libpakfire/pakfire.c @@ -60,6 +60,7 @@ #include #include #include +#include #include #include #include @@ -103,6 +104,9 @@ struct pakfire { // Distro struct pakfire_distro distro; + // Snapshot + struct pakfire_snapshot* snapshot; + // States unsigned int destroy_on_free:1; unsigned int pool_ready:1; @@ -384,6 +388,10 @@ static void pakfire_free(struct pakfire* pakfire) { pakfire_repo_unref(repo); } + // Umount the snapshot + if (pakfire->snapshot) + pakfire_snapshot_unref(pakfire->snapshot); + if (pakfire->destroy_on_free && *pakfire->path) { CTX_DEBUG(pakfire->ctx, "Destroying %s\n", pakfire->path); @@ -637,10 +645,12 @@ static int pakfire_read_config(struct pakfire* pakfire, FILE* f) { if (r) return r; +#if 0 // Read repository configuration r = pakfire_read_repo_config(pakfire); if (r) return r; +#endif return 0; } @@ -785,7 +795,29 @@ PAKFIRE_EXPORT int pakfire_create(struct pakfire** pakfire, struct pakfire_ctx* p->arches.effective = pakfire_arch_is_supported_by_host(arch); if (!p->arches.effective) { CTX_ERROR(p->ctx, "Unsupported architecture: %s\n", arch); - r = errno; + r = -errno; + goto ERROR; + } + + // Setup user/group + r = pakfire_setup_user(p); + if (r) + goto ERROR; + + // Initialise configuration + r = pakfire_config_create(&p->config); + if (r < 0) + goto ERROR; + + // Read configuration file + r = pakfire_read_config(p, conf); + if (r < 0) + goto ERROR; + + // Set cache path + r = pakfire_set_cache_path(p); + if (r < 0) { + CTX_ERROR(p->ctx, "Could not set cache path: %m\n"); goto ERROR; } @@ -796,6 +828,34 @@ PAKFIRE_EXPORT int pakfire_create(struct pakfire** pakfire, struct pakfire_ctx* goto ERROR; } + // If using a snapshot, path cannot be set + if (p->flags & PAKFIRE_USE_SNAPSHOT) { + if (path) { + CTX_ERROR(p->ctx, "Cannot use snapshots with path\n"); + r = -EINVAL; + goto ERROR; + } + + // Find the most recent snapshot + r = pakfire_snapshot_find(&p->snapshot, p); + if (r < 0) { + CTX_ERROR(p->ctx, "Could not find a snapshot: %s\n", strerror(-r)); + goto ERROR; + } + + // Mount the snapshot + if (p->snapshot) { + r = pakfire_snapshot_mount(p->snapshot); + if (r < 0) { + CTX_ERROR(p->ctx, "Could not mount snapshot: %s\n", strerror(-r)); + goto ERROR; + } + + // Fetch the path where the snapshot was mounted + path = pakfire_snapshot_path(p->snapshot); + } + } + // Create a ramdisk if no path was given if (!path) { r = pakfire_make_ramdisk(p, tempdir, NULL); @@ -812,16 +872,6 @@ PAKFIRE_EXPORT int pakfire_create(struct pakfire** pakfire, struct pakfire_ctx* // Set path pakfire_string_set(p->path, path); - // Setup user/group - r = pakfire_setup_user(p); - if (r) - goto ERROR; - - // Initialise configuration - r = pakfire_config_create(&p->config); - if (r) - goto ERROR; - // Read /etc/os-release r = pakfire_read_os_release(p); if (r && errno != ENOENT) @@ -847,11 +897,6 @@ PAKFIRE_EXPORT int pakfire_create(struct pakfire** pakfire, struct pakfire_ctx* if (r) goto ERROR; - // Read configuration file - r = pakfire_read_config(p, conf); - if (r) - goto ERROR; - // Dump distribution configuration CTX_DEBUG(p->ctx, " Distribution: %s\n", p->distro.pretty_name); CTX_DEBUG(p->ctx, " name = %s\n", p->distro.name); @@ -865,13 +910,6 @@ PAKFIRE_EXPORT int pakfire_create(struct pakfire** pakfire, struct pakfire_ctx* if (*p->distro.slogan) CTX_DEBUG(p->ctx, " slogan = %s\n", p->distro.slogan); - // Set cache path - r = pakfire_set_cache_path(p); - if (r) { - CTX_ERROR(p->ctx, "Could not set cache path: %m\n"); - goto ERROR; - } - // Make path for private files r = pakfire_path(p, private_dir, "%s", PAKFIRE_PRIVATE_DIR); if (r) @@ -969,11 +1007,11 @@ int __pakfire_cache_path(struct pakfire* pakfire, char* path, size_t length, // Format input into buffer va_start(args, format); - r = __pakfire_string_vformat(buffer, sizeof(buffer), format, args); + r = pakfire_string_vformat(buffer, format, args); va_end(args); // Break on any errors - if (r) + if (r < 0) return r; // Join paths together diff --git a/src/libpakfire/snapshot.c b/src/libpakfire/snapshot.c index d4362d8a7..86056529e 100644 --- a/src/libpakfire/snapshot.c +++ b/src/libpakfire/snapshot.c @@ -19,9 +19,12 @@ #############################################################################*/ #include +#include +#include #include #include #include +#include #include @@ -34,12 +37,327 @@ #include #include #include +#include #include #include #include #include #include +struct pakfire_snapshot_fs { + char path[PATH_MAX]; + int fd; +}; + +struct pakfire_snapshot { + struct pakfire_ctx* ctx; + int nrefs; + + // Path + char path[PATH_MAX]; + int fd; + + // File Systems + struct pakfire_snapshot_fs overlayfs; + struct pakfire_snapshot_fs tmpfs; + + char upperdir[PATH_MAX]; + char workdir[PATH_MAX]; +}; + +static void pakfire_snapshot_free(struct pakfire_snapshot* snapshot) { + pakfire_snapshot_umount(snapshot); + + if (snapshot->overlayfs.fd >= 0) + close(snapshot->overlayfs.fd); + if (snapshot->tmpfs.fd >= 0) + close(snapshot->tmpfs.fd); + if (snapshot->fd >= 0) + close(snapshot->fd); + + if (snapshot->ctx) + pakfire_ctx_unref(snapshot->ctx); + free(snapshot); +} + +int pakfire_snapshot_create( + struct pakfire_snapshot** snapshot, struct pakfire_ctx* ctx, const char* path) { + struct pakfire_snapshot* s = NULL; + int r; + + // Allocate a new snapshot + s = calloc(1, sizeof(*s)); + if (!s) + return -errno; + + // Store a reference to the context + s->ctx = pakfire_ctx_ref(ctx); + + // Initialize the reference counter + s->nrefs = 1; + + // Store the path + r = pakfire_string_set(s->path, path); + if (r < 0) + goto ERROR; + + // Open the directory + s->fd = open(path, O_DIRECTORY|O_CLOEXEC); + if (s->fd < 0) { + r = -errno; + goto ERROR; + } + + // Initialize the file descriptors + s->overlayfs.fd = s->tmpfs.fd = -EBADF; + + // XXX Check and lock the snapshot + + // Return the snapshot + *snapshot = pakfire_snapshot_ref(s); + +ERROR: + if (s) + pakfire_snapshot_unref(s); + + return r; +} + +struct pakfire_snapshot* pakfire_snapshot_ref(struct pakfire_snapshot* snapshot) { + ++snapshot->nrefs; + + return snapshot; +} + +struct pakfire_snapshot* pakfire_snapshot_unref(struct pakfire_snapshot* snapshot) { + if (--snapshot->nrefs > 0) + return snapshot; + + pakfire_snapshot_free(snapshot); + return NULL; +} + +static int pakfire_snapshot_filter(const struct dirent* dirent) { + // Skip any hidden directories + if (*dirent->d_name == '.') + return 0; + + // Select the rest + return 1; +} + +/* + Finds and returns the latest snapshot +*/ +int pakfire_snapshot_find(struct pakfire_snapshot** snapshot, struct pakfire* pakfire) { + struct dirent** paths = NULL; + int num_paths = 0; + char path[PATH_MAX]; + int fd = -EBADF; + int r; + + struct pakfire_ctx* ctx = pakfire_ctx(pakfire); + + // Make the path + r = pakfire_cache_path(pakfire, path, "%s", "snapshots"); + if (r < 0) + return r; + + // Open the path + fd = open(path, O_DIRECTORY|O_CLOEXEC); + if (fd < 0) { + switch (errno) { + // If the directory does not exist, we cannot find a snapshot + case ENOENT: + r = 0; + goto ERROR; + + default: + r = -errno; + goto ERROR; + } + } + + // Find all snapshots in the directory + num_paths = scandirat(fd, ".", &paths, pakfire_snapshot_filter, versionsort); + if (num_paths < 0) { + r = num_paths; + goto ERROR; + + // We are done if we did not find anything + } else if (num_paths == 0) { + r = 0; + goto ERROR; + } + + // List what we have found + for (int i = 0; i < num_paths; i++) { + CTX_DEBUG(ctx, "Found snapshot: %s\n", paths[i]->d_name); + } + + // Build the path to the snapshot + r = pakfire_path_append(path, path, paths[num_paths - 1]->d_name); + if (r < 0) + goto ERROR; + + CTX_DEBUG(ctx, "Selecting snapshot %s\n", path); + + // Open the snapshot + r = pakfire_snapshot_create(snapshot, ctx, path); + if (r < 0) + goto ERROR; + +ERROR: + if (paths) { + for (int i = 0; i < num_paths; i++) + free(paths[i]); + free(paths); + } + if (fd >= 0) + close(fd); + if (ctx) + pakfire_ctx_unref(ctx); + + return r; +} + +const char* pakfire_snapshot_path(struct pakfire_snapshot* snapshot) { + return snapshot->overlayfs.path; +} + +static int pakfire_snapshot_mount_tmpfs(struct pakfire_snapshot* snapshot) { + char* path = NULL; + int r; + + // Make path + r = pakfire_string_set(snapshot->tmpfs.path, PAKFIRE_TMP_DIR "/pakfire-tmpfs.XXXXXX"); + if (r < 0) + return r; + + // Create a temporary directory + path = pakfire_mkdtemp(snapshot->tmpfs.path); + if (!path) + return -errno; + + // Perform mount + r = mount("pakfire_tmpfs", snapshot->tmpfs.path, "tmpfs", 0, NULL); + if (r < 0) + return -errno; + + // Make the upper directory + r = pakfire_path_append(snapshot->upperdir, snapshot->tmpfs.path, "upper"); + if (r < 0) + return r; + + // Create the upper directory + r = pakfire_mkdir(snapshot->upperdir, 0755); + if (r < 0) + return r; + + // Make the work directory + r = pakfire_path_append(snapshot->workdir, snapshot->tmpfs.path, "work"); + if (r < 0) + return r; + + // Create the work directory + r = pakfire_mkdir(snapshot->workdir, 0755); + if (r < 0) + return r; + + return 0; +} + +int pakfire_snapshot_mount(struct pakfire_snapshot* snapshot) { + const char* path = NULL; + int mountfd = -EBADF; + int fsfd = -EBADF; + int r; + + CTX_DEBUG(snapshot->ctx, "Mounting snapshot %s\n", snapshot->path); + + // Mount the tmpfs + r = pakfire_snapshot_mount_tmpfs(snapshot); + if (r < 0) + goto ERROR; + + // Create a new overlayfs + fsfd = fsopen("overlay", FSOPEN_CLOEXEC); + if (fsfd < 0) { + r = -errno; + goto ERROR; + } + + // Set the lower directory + r = fsconfig(fsfd, FSCONFIG_SET_STRING, "lowerdir", snapshot->path, 0); + if (r < 0) + goto ERROR; + + // Set the upper directory + r = fsconfig(fsfd, FSCONFIG_SET_STRING, "upperdir", snapshot->upperdir, 0); + if (r < 0) + goto ERROR; + + // Set some work directory + r = fsconfig(fsfd, FSCONFIG_SET_STRING, "workdir", snapshot->workdir, 0); + if (r < 0) + goto ERROR; + + // Create a new filesystem + r = fsconfig(fsfd, FSCONFIG_CMD_CREATE, NULL, NULL, 0); + if (r < 0) + goto ERROR; + + // Perform mount + mountfd = fsmount(fsfd, FSMOUNT_CLOEXEC, 0); + if (mountfd < 0) { + r = -errno; + goto ERROR; + } + + // Make path + r = pakfire_string_set(snapshot->overlayfs.path, PAKFIRE_TMP_DIR "/pakfire-root.XXXXXX"); + if (r < 0) + goto ERROR; + + // Create a temporary directory + path = pakfire_mkdtemp(snapshot->overlayfs.path); + if (!path) + goto ERROR; + + // Move the snapshot to the right place + r = move_mount(mountfd, "", AT_FDCWD, path, MOVE_MOUNT_F_EMPTY_PATH); + if (r < 0) + goto ERROR; + +ERROR: + if (mountfd >= 0) + close(mountfd); + if (fsfd >= 0) + close(fsfd); + + return r; +} + +int pakfire_snapshot_umount(struct pakfire_snapshot* snapshot) { + int r; + + // Umount the overlayfs + if (*snapshot->overlayfs.path) { + r = umount(snapshot->overlayfs.path); + if (r < 0) + return r; + } + + // Umount the tmpfs + if (*snapshot->tmpfs.path) { + r = umount(snapshot->tmpfs.path); + if (r < 0) + return r; + } + + return 0; +} + #define pakfire_snapshot_path(pakfire, path) \ __pakfire_snapshot_path(pakfire, path, sizeof(path))