From 26452aefabe253284ca8f3588149d5d57affc3be Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Sat, 5 Jun 2021 11:59:23 +0000 Subject: [PATCH] builder: Move setup of build environment into libpakfire Signed-off-by: Michael Tremer --- src/_pakfire/pakfire.c | 24 +++- src/libpakfire/build.c | 155 +++++++++++++++++++++++ src/libpakfire/include/pakfire/build.h | 2 + src/libpakfire/include/pakfire/pakfire.h | 3 + src/libpakfire/pakfire.c | 8 +- src/pakfire/base.py | 4 +- src/pakfire/builder.py | 37 +----- 7 files changed, 193 insertions(+), 40 deletions(-) diff --git a/src/_pakfire/pakfire.c b/src/_pakfire/pakfire.c index 8f27060d5..f1f2a76bd 100644 --- a/src/_pakfire/pakfire.c +++ b/src/_pakfire/pakfire.c @@ -51,21 +51,37 @@ static PyObject* Pakfire_new(PyTypeObject* type, PyObject* args, PyObject* kwds) } static int Pakfire_init(PakfireObject* self, PyObject* args, PyObject* kwds) { - char* kwlist[] = { "path", "arch", "offline", "conf", NULL }; + char* kwlist[] = { "path", "arch", "offline", "conf", "build", + "enable_ccache", "enable_snapshot", NULL }; const char* path = NULL; const char* arch = NULL; const char* conf = NULL; int offline = 0; - int flags = 0; + int build = 0; + int enable_ccache = 1; + int enable_snapshot = 1; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|zzpz", kwlist, - &path, &arch, &offline, &conf)) + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|zzpzppp", kwlist, + &path, &arch, &offline, &conf, &build, &enable_ccache, &enable_snapshot)) return -1; + int flags = 0; + // Enable offline mode if (offline) flags |= PAKFIRE_FLAGS_OFFLINE; + // Enable build mode + if (build) { + flags |= PAKFIRE_FLAGS_BUILD; + + if (!enable_ccache) + flags |= PAKFIRE_FLAGS_DISABLE_CCACHE; + + if (!enable_snapshot) + flags |= PAKFIRE_FLAGS_DISABLE_SNAPSHOT; + } + // Create a new Pakfire instance int r = pakfire_create(&self->pakfire, path, arch, conf, flags); if (r) { diff --git a/src/libpakfire/build.c b/src/libpakfire/build.c index f09df6dd2..43141ca7d 100644 --- a/src/libpakfire/build.c +++ b/src/libpakfire/build.c @@ -20,6 +20,7 @@ #include #include +#include #include #include @@ -30,10 +31,15 @@ #include #include #include +#include #include +#include +#include #include #include +#define CCACHE_DIR "/var/cache/ccache" + static const char* stages[] = { "prepare", "build", @@ -52,6 +58,155 @@ static const char* stages[] = { "\n" \ "exit 0\n" +static int pakfire_build_install_packages(Pakfire pakfire, int* snapshot_needs_update) { + struct pakfire_request* request = NULL; + struct pakfire_transaction* transaction = NULL; + char** packages = NULL; + int r = 1; + + struct pakfire_config* config = pakfire_get_config(pakfire); + + // Fetch build environment + const char* requires = pakfire_config_get(config, "build", "requires", NULL); + if (!requires) { + ERROR(pakfire, "No build requirements have been defined\n"); + goto ERROR; + } + + // Split requirements into packages + packages = pakfire_split_string(requires, ','); + if (!packages) + goto ERROR; + + // Create a new request + r = pakfire_request_create(&request, pakfire, 0); + if (r) + goto ERROR; + + // Install all packages + for (char** package = packages; *package; package++) { + r = pakfire_request_install(request, *package); + if (r) { + ERROR(pakfire, "Cannot install '%s': %m\n", *package); + goto ERROR; + } + } + + // XXX update everything + + // Solve the request + r = pakfire_request_solve(request, 0); + if (r) + goto ERROR; + + // Fetch the transaction + transaction = pakfire_request_get_transaction(request); + if (!transaction) + goto ERROR; + + // Mark the snapshot for update + if (pakfire_transaction_count(transaction)) + *snapshot_needs_update = 1; + + // Download + r = pakfire_transaction_download(transaction); + if (r) + goto ERROR; + + // Run the transaction + r = pakfire_transaction_run(transaction); + if (r) + goto ERROR; + + // Success + r = 0; + +ERROR: + if (transaction) + pakfire_transaction_unref(transaction); + if (request) + pakfire_request_unref(request); + if (packages) { + for (char** package = packages; *package; package++) + free(*package); + free(packages); + } + if (config) + pakfire_config_unref(config); + + return r; +} + +int pakfire_build_setup(Pakfire pakfire) { + char path[PATH_MAX]; + int r; + + // Mount ccache + if (!pakfire_has_flag(pakfire, PAKFIRE_FLAGS_DISABLE_CCACHE)) { + r = pakfire_make_cache_path(pakfire, path, "%s", "ccache"); + if (r < 0) + return r; + + // Ensure path exists + r = pakfire_mkdir(path, 0); + if (r && errno != EEXIST) { + ERROR(pakfire, "Could not create ccache directory %s: %m\n", path); + return r; + } + + r = pakfire_bind(pakfire, path, CCACHE_DIR, MS_NOSUID|MS_NOEXEC|MS_NODEV); + if (r) { + ERROR(pakfire, "Could not mount ccache: %m\n"); + return r; + } + } + + // Extract snapshot + if (!pakfire_has_flag(pakfire, PAKFIRE_FLAGS_DISABLE_SNAPSHOT)) { + r = pakfire_make_cache_path(pakfire, path, "%s", "snapshot.tar.zst"); + if (r < 0) + return r; + + // Open the snapshot + FILE* f = fopen(path, "r"); + + // Try restoring the snapshot + if (f) { + r = pakfire_snapshot_restore(pakfire, f); + if (r) { + fclose(f); + return r; + } + } + + // Tells us whether we need to (re-)create the snapshot + int snapshot_needs_update = 0; + + // Install or update any build dependencies + r = pakfire_build_install_packages(pakfire, &snapshot_needs_update); + if (r) + return r; + + if (snapshot_needs_update) { + // Open snapshot file for writing + f = fopen(path, "w"); + if (!f) { + ERROR(pakfire, "Could not open snapshot file for writing: %m\n"); + return 1; + } + + // Create a new snapshot + r = pakfire_snapshot_create(pakfire, f); + fclose(f); + + if (r) + return r; + } + } + + return 0; +} + static int pakfire_build_run_script(Pakfire pakfire, const char* filename, const char* args[], pakfire_execute_logging_callback logging_callback, void* data) { char* script = NULL; diff --git a/src/libpakfire/include/pakfire/build.h b/src/libpakfire/include/pakfire/build.h index 8fb99d367..90edc86d2 100644 --- a/src/libpakfire/include/pakfire/build.h +++ b/src/libpakfire/include/pakfire/build.h @@ -28,6 +28,8 @@ enum pakfire_build_flags { PAKFIRE_BUILD_INTERACTIVE = (1 << 0), }; +int pakfire_build_setup(Pakfire pakfire); + int pakfire_build(Pakfire pakfire, const char* path, const char* target, int flags, pakfire_execute_logging_callback logging_callback, void* data); int pakfire_shell(Pakfire pakfire); diff --git a/src/libpakfire/include/pakfire/pakfire.h b/src/libpakfire/include/pakfire/pakfire.h index ff4451a02..59e9c29c2 100644 --- a/src/libpakfire/include/pakfire/pakfire.h +++ b/src/libpakfire/include/pakfire/pakfire.h @@ -33,6 +33,9 @@ enum pakfire_flags { PAKFIRE_FLAGS_OFFLINE = (1 << 0), PAKFIRE_FLAGS_NON_INTERACTIVE = (1 << 1), + PAKFIRE_FLAGS_BUILD = (1 << 2), + PAKFIRE_FLAGS_DISABLE_CCACHE = (1 << 3), + PAKFIRE_FLAGS_DISABLE_SNAPSHOT = (1 << 4), }; int pakfire_create(Pakfire* pakfire, const char* path, const char* arch, diff --git a/src/libpakfire/pakfire.c b/src/libpakfire/pakfire.c index c1d6785a2..ec7d272e9 100644 --- a/src/libpakfire/pakfire.c +++ b/src/libpakfire/pakfire.c @@ -859,7 +859,6 @@ PAKFIRE_EXPORT int pakfire_create(Pakfire* pakfire, const char* path, const char DEBUG(p, " arch = %s\n", pakfire_get_arch(p)); DEBUG(p, " path = %s\n", pakfire_get_path(p)); - // Automatically disable interactive mode if (!pakfire_has_flag(p, PAKFIRE_FLAGS_NON_INTERACTIVE)) { if (pakfire_tty_is_noninteractive()) { @@ -934,6 +933,13 @@ PAKFIRE_EXPORT int pakfire_create(Pakfire* pakfire, const char* path, const char if (r) goto ERROR; + // Setup build stuff + if (pakfire_has_flag(p, PAKFIRE_FLAGS_BUILD)) { + r = pakfire_build_setup(p); + if (r) + goto ERROR; + } + *pakfire = p; return 0; diff --git a/src/pakfire/base.py b/src/pakfire/base.py index 0bc6ba55e..3486d7af7 100644 --- a/src/pakfire/base.py +++ b/src/pakfire/base.py @@ -36,8 +36,8 @@ from .i18n import _ class Pakfire(_pakfire.Pakfire): __version__ = PAKFIRE_VERSION - def __init__(self, path=None, arch=None, conf=None, offline=False): - _pakfire.Pakfire.__init__(self, path, arch, conf=conf, offline=offline) + def __init__(self, path=None, arch=None, conf=None, offline=False, **kwargs): + _pakfire.Pakfire.__init__(self, path, arch, conf=conf, offline=offline, **kwargs) # Initialise logging system self.log = self._setup_logger() diff --git a/src/pakfire/builder.py b/src/pakfire/builder.py index 6c39d9824..3164b16a9 100644 --- a/src/pakfire/builder.py +++ b/src/pakfire/builder.py @@ -99,11 +99,11 @@ class Builder(object): pakfire = base.Pakfire( arch=self.arch, conf=self.conf, - ) - # Mount ccache - if self.settings.get("enable_ccache"): - pakfire.bind(CCACHE_CACHE_DIR, "/var/cache/ccache") + # Enable build mode + build=True, + **self.settings, + ) # Setup domain name resolution in chroot for i in ("/etc/resolv.conf", "/etc/hosts"): @@ -144,29 +144,6 @@ class BuilderContext(object): # Get a reference to the logger self.log = self.builder.log - def _setup(self): - """ - Sets up the environment by installing some basic packages - """ - enable_snapshot = self.builder.settings.get("enable_snapshot", True) - - snapshot_path = self.pakfire.make_cache_path("snapshot.tar.zst") - - # Restore the snapshot if available - if enable_snapshot: - try: - self.pakfire.restore_snapshot(snapshot_path) - - # Ignore if no snapshot was present - except FileNotFoundError: - pass - - # Install any updates and essential packages - # If there have been updates, or on a fresh install, re-create the snapshot - if self._install(BUILD_PACKAGES): - if enable_snapshot: - self.pakfire.create_snapshot(snapshot_path) - def _install(self, packages): self.log.debug(_("Installing packages in build environment:")) for package in packages: @@ -193,9 +170,6 @@ class BuilderContext(object): return ret def build(self, path, shell=True): - # Setup the environment - self._setup() - # Install the package self._install([path]) @@ -203,9 +177,6 @@ class BuilderContext(object): self.pakfire.build(makefile, logging_callback=self.log.log, interactive=shell) def shell(self, packages=[], install=None): - # Setup the environment - self._setup() - if install: packages += install -- 2.47.2