From: Michael Tremer Date: Tue, 16 Aug 2022 14:32:24 +0000 (+0000) Subject: build: Move snapshot extraction X-Git-Tag: 0.9.28~501 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=c0f2502ab3714ef298332c19b5be8a6126971bc6;p=pakfire.git build: Move snapshot extraction Signed-off-by: Michael Tremer --- diff --git a/src/_pakfire/pakfire.c b/src/_pakfire/pakfire.c index f4975ce24..a71512907 100644 --- a/src/_pakfire/pakfire.c +++ b/src/_pakfire/pakfire.c @@ -143,7 +143,6 @@ static int Pakfire_init(PakfireObject* self, PyObject* args, PyObject* kwds) { "conf", "build", "disable_ccache", - "disable_snapshot", "confirm_callback", NULL, }; @@ -154,11 +153,10 @@ static int Pakfire_init(PakfireObject* self, PyObject* args, PyObject* kwds) { int offline = 0; int build = 0; int disable_ccache = 1; - int disable_snapshot = 1; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|zzOppzpppO", kwlist, + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|zzOppzppO", kwlist, &path, &arch, &self->callbacks.log, &interactive, &offline, &conf, &build, - &disable_ccache, &disable_snapshot, &self->callbacks.confirm)) + &disable_ccache, &self->callbacks.confirm)) return -1; // Check if log callback is callable @@ -189,9 +187,6 @@ static int Pakfire_init(PakfireObject* self, PyObject* args, PyObject* kwds) { if (disable_ccache) flags |= PAKFIRE_FLAGS_DISABLE_CCACHE; - - if (disable_snapshot) - flags |= PAKFIRE_FLAGS_DISABLE_SNAPSHOT; } // Configure callbacks @@ -1108,17 +1103,26 @@ static PyObject* Pakfire_build(PakfireObject* self, PyObject* args, PyObject* kw char* kwlist[] = { "path", "build_id", + "disable_snapshot", NULL, }; const char* path = NULL; const char* build_id = NULL; + int disable_snapshot = 0; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s|z", kwlist, &path, &build_id)) + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s|zp", kwlist, &path, &build_id, + &disable_snapshot)) return NULL; + int flags = 0; + + // Disable snapshot if requested + if (disable_snapshot) + flags |= PAKFIRE_BUILD_DISABLE_SNAPSHOT; + // Run build - int r = pakfire_build(self->pakfire, path, NULL, build_id, 0); + int r = pakfire_build(self->pakfire, path, NULL, build_id, flags); return execute_return_value(r); } diff --git a/src/libpakfire/build.c b/src/libpakfire/build.c index 4723f2f90..4e5f2eba4 100644 --- a/src/libpakfire/build.c +++ b/src/libpakfire/build.c @@ -38,6 +38,7 @@ #include #include #include +#include #include // We guarantee 2 GiB of memory to every build container @@ -84,6 +85,10 @@ static const char* stages[] = { "\n" \ "exit 0\n" +static int pakfire_build_has_flag(struct pakfire_build* build, int flag) { + return build->flags & flag; +} + static int pakfire_build_run_script(struct pakfire_build* build, const char* filename, const char* args[], char** output) { char* script = NULL; @@ -925,6 +930,128 @@ PAKFIRE_EXPORT int pakfire_build_set_target( return 0; } +static int pakfire_build_install_packages(struct pakfire_build* build, + int* snapshot_needs_update) { + char** packages = NULL; + int r = 1; + + // Fetch configuration + struct pakfire_config* config = pakfire_get_config(build->pakfire); + if (!config) { + ERROR(build->pakfire, "Could not fetch configuration: %m\n"); + r = 1; + goto ERROR; + } + + // Fetch build environment + const char* requires = pakfire_config_get(config, "build", "requires", NULL); + if (!requires) { + ERROR(build->pakfire, "No build requirements have been defined\n"); + goto ERROR; + } + + // Split requirements into packages + packages = pakfire_split_string(requires, ','); + if (!packages) + goto ERROR; + + int changed = 0; + + // Install everything + r = pakfire_install(build->pakfire, 0, 0, (const char**)packages, NULL, 0, + &changed, NULL, NULL); + if (r) { + ERROR(build->pakfire, "Could not install build dependencies: %m\n"); + goto ERROR; + } + + // Mark snapshot as changed if new packages were installed + if (changed) + *snapshot_needs_update = 1; + + // Update everything + r = pakfire_sync(build->pakfire, 0, 0, &changed, NULL, NULL); + if (r) { + ERROR(build->pakfire, "Could not update packages: %m\n"); + goto ERROR; + } + + // Has anything changed? + if (changed) + *snapshot_needs_update = 1; + + // Success + r = 0; + +ERROR: + if (config) + pakfire_config_unref(config); + + if (packages) { + for (char** package = packages; *package; package++) + free(*package); + free(packages); + } + + return r; +} + +static int pakfire_build_extract_snapshot(struct pakfire_build* build) { + char path[PATH_MAX]; + int r; + + // Check if the user wants a snapshot extracted + if (!pakfire_build_has_flag(build, PAKFIRE_BUILD_DISABLE_SNAPSHOT)) { + DEBUG(build->pakfire, "Snapshot extraction has been disabled for this build\n"); + return 0; + } + + // Extract snapshot + r = pakfire_make_cache_path(build->pakfire, path, "%s", "snapshot.tar.zst"); + if (r < 0) { + ERROR(build->pakfire, "Could not compose snapshot path: %m\n"); + return 1; + } + + // Open the snapshot + FILE* f = fopen(path, "r"); + + // Try restoring the snapshot + if (f) { + r = pakfire_snapshot_restore(build->pakfire, f); + if (r) { + fclose(f); + return r; + } + } + + // Tells us whether we need to (re-)create the snapshot + int snapshot_needs_update = 0; + + // Install or update any build dependencies + r = pakfire_build_install_packages(build, &snapshot_needs_update); + if (r) + return r; + + if (snapshot_needs_update) { + // Open snapshot file for writing + f = fopen(path, "w"); + if (!f) { + ERROR(build->pakfire, "Could not open snapshot file for writing: %m\n"); + return 1; + } + + // Create a new snapshot + r = pakfire_snapshot_create(build->pakfire, f); + fclose(f); + + if (r) + return r; + } + + return 0; +} + PAKFIRE_EXPORT int pakfire_build_exec(struct pakfire_build* build, const char* path) { struct pakfire_archive* archive = NULL; struct pakfire_package* package = NULL; @@ -952,6 +1079,13 @@ PAKFIRE_EXPORT int pakfire_build_exec(struct pakfire_build* build, const char* p INFO(build->pakfire, "Building %s...\n", nevra); + // Extract the snapshot + r = pakfire_build_extract_snapshot(build); + if (r) { + ERROR(build->pakfire, "Could not extract snapshot: %m\n"); + goto ERROR; + } + // Setup build environment r = pakfire_build_setup(build->pakfire); if (r) diff --git a/src/libpakfire/include/pakfire/build.h b/src/libpakfire/include/pakfire/build.h index 0f313dc30..3b1c96844 100644 --- a/src/libpakfire/include/pakfire/build.h +++ b/src/libpakfire/include/pakfire/build.h @@ -25,6 +25,10 @@ struct pakfire_build; +enum pakfire_build_flags { + PAKFIRE_BUILD_DISABLE_SNAPSHOT = (1 << 0), +}; + int pakfire_build_create(struct pakfire_build** build, struct pakfire* pakfire, const char* id, int flags); diff --git a/src/libpakfire/include/pakfire/pakfire.h b/src/libpakfire/include/pakfire/pakfire.h index da9d4c675..5c3433bc6 100644 --- a/src/libpakfire/include/pakfire/pakfire.h +++ b/src/libpakfire/include/pakfire/pakfire.h @@ -47,7 +47,6 @@ enum pakfire_flags { PAKFIRE_FLAGS_OFFLINE = (1 << 1), PAKFIRE_FLAGS_BUILD = (1 << 2), PAKFIRE_FLAGS_DISABLE_CCACHE = (1 << 3), - PAKFIRE_FLAGS_DISABLE_SNAPSHOT = (1 << 4), }; // Callbacks diff --git a/src/libpakfire/pakfire.c b/src/libpakfire/pakfire.c index 4d24238ed..743cb4570 100644 --- a/src/libpakfire/pakfire.c +++ b/src/libpakfire/pakfire.c @@ -56,7 +56,6 @@ #include #include #include -#include #include #include #include @@ -1932,59 +1931,6 @@ PAKFIRE_EXPORT int pakfire_sync(struct pakfire* pakfire, int solver_flags, int f // Build Stuff -static int pakfire_build_install_packages(struct pakfire* pakfire, int* snapshot_needs_update) { - char** packages = NULL; - int r = 1; - - // Fetch build environment - const char* requires = pakfire_config_get(pakfire->config, "build", "requires", NULL); - if (!requires) { - ERROR(pakfire, "No build requirements have been defined\n"); - goto ERROR; - } - - // Split requirements into packages - packages = pakfire_split_string(requires, ','); - if (!packages) - goto ERROR; - - int changed = 0; - - // Install everything - r = pakfire_install(pakfire, 0, 0, (const char**)packages, NULL, 0, &changed, NULL, NULL); - if (r) { - ERROR(pakfire, "Could not install build dependencies: %m\n"); - goto ERROR; - } - - // Mark snapshot as changed if new packages were installed - if (changed) - *snapshot_needs_update = 1; - - // Update everything - r = pakfire_sync(pakfire, 0, 0, &changed, NULL, NULL); - if (r) { - ERROR(pakfire, "Could not update packages: %m\n"); - goto ERROR; - } - - // Has anything changed? - if (changed) - *snapshot_needs_update = 1; - - // Success - r = 0; - -ERROR: - if (packages) { - for (char** package = packages; *package; package++) - free(*package); - free(packages); - } - - return r; -} - int pakfire_build_setup(struct pakfire* pakfire) { // This function can only be called when in build mode if (!pakfire_has_flag(pakfire, PAKFIRE_FLAGS_BUILD)) { @@ -2019,49 +1965,6 @@ int pakfire_build_setup(struct pakfire* pakfire) { } } - // Extract snapshot - if (!pakfire_has_flag(pakfire, PAKFIRE_FLAGS_DISABLE_SNAPSHOT)) { - r = pakfire_make_cache_path(pakfire, path, "%s", "snapshot.tar.zst"); - if (r < 0) - return r; - - // Open the snapshot - FILE* f = fopen(path, "r"); - - // Try restoring the snapshot - if (f) { - r = pakfire_snapshot_restore(pakfire, f); - if (r) { - fclose(f); - return r; - } - } - - // Tells us whether we need to (re-)create the snapshot - int snapshot_needs_update = 0; - - // Install or update any build dependencies - r = pakfire_build_install_packages(pakfire, &snapshot_needs_update); - if (r) - return r; - - if (snapshot_needs_update) { - // Open snapshot file for writing - f = fopen(path, "w"); - if (!f) { - ERROR(pakfire, "Could not open snapshot file for writing: %m\n"); - return 1; - } - - // Create a new snapshot - r = pakfire_snapshot_create(pakfire, f); - fclose(f); - - if (r) - return r; - } - } - // Build setup done pakfire->build_setup = 1; diff --git a/src/pakfire/daemon.py b/src/pakfire/daemon.py index fdc76ea03..961ccd944 100644 --- a/src/pakfire/daemon.py +++ b/src/pakfire/daemon.py @@ -272,9 +272,8 @@ class Worker(multiprocessing.Process): arch=arch, logger=logger, - # Enable build mode and disable snapshots + # Enable build mode build=True, - disable_snapshot=True, interactive=False, ) finally: @@ -282,7 +281,7 @@ class Worker(multiprocessing.Process): os.unlink(self.pakfire_conf) # Run the build in a new thread - thread = asyncio.to_thread(p.build, pkg, **kwargs) + thread = asyncio.to_thread(p.build, pkg, disable_snapshot=True, **kwargs) # Return a task return asyncio.create_task(thread) diff --git a/src/scripts/pakfire-builder.in b/src/scripts/pakfire-builder.in index eee098804..0bd7bc373 100644 --- a/src/scripts/pakfire-builder.in +++ b/src/scripts/pakfire-builder.in @@ -162,7 +162,6 @@ class Cli(object): # Enable build mode build=build, - disable_snapshot=ns.disable_snapshot, ) def __call__(self): @@ -208,6 +207,7 @@ class Cli(object): p.build( package, build_id="%s" % ns.build_id if ns.build_id else None, + disable_snapshot=ns.disable_snapshot, ) # Cleanup the temporary directory