From ef009305096cbec1190ff0c0071c07a41c667f94 Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Fri, 28 Apr 2023 16:06:29 +0000 Subject: [PATCH] build: Allow setting a different ccache path Signed-off-by: Michael Tremer --- src/_pakfire/pakfire.c | 16 +++++- src/libpakfire/build.c | 78 ++++++++++++++++++-------- src/libpakfire/include/pakfire/build.h | 1 + src/libpakfire/libpakfire.sym | 1 + src/pakfire/daemon.py | 42 +++++++++++++- 5 files changed, 111 insertions(+), 27 deletions(-) diff --git a/src/_pakfire/pakfire.c b/src/_pakfire/pakfire.c index 80f3c5ce5..e4c704503 100644 --- a/src/_pakfire/pakfire.c +++ b/src/_pakfire/pakfire.c @@ -1214,6 +1214,7 @@ static PyObject* Pakfire_build(PakfireObject* self, PyObject* args, PyObject* kw "path", "target", "build_id", + "ccache_path", "interactive", "disable_snapshot", "disable_ccache", @@ -1224,14 +1225,16 @@ static PyObject* Pakfire_build(PakfireObject* self, PyObject* args, PyObject* kw const char* path = NULL; const char* target = NULL; const char* build_id = NULL; + const char* ccache_path = NULL; int interactive = 0; int disable_snapshot = 0; int disable_ccache = 0; int disable_tests = 0; int r; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s|zzpppp", kwlist, &path, &target, - &build_id, &interactive, &disable_snapshot, &disable_ccache, &disable_tests)) + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s|zzzpppp", kwlist, + &path, &target, &build_id, &ccache_path, &interactive, + &disable_snapshot, &disable_ccache, &disable_tests)) return NULL; int flags = 0; @@ -1267,6 +1270,15 @@ static PyObject* Pakfire_build(PakfireObject* self, PyObject* args, PyObject* kw } } + // Set ccache path + if (ccache_path) { + r = pakfire_build_set_ccache_path(build, ccache_path); + if (r) { + PyErr_SetFromErrno(PyExc_OSError); + goto ERROR; + } + } + Py_BEGIN_ALLOW_THREADS // Run build diff --git a/src/libpakfire/build.c b/src/libpakfire/build.c index 22e76c6fc..f0245f2f6 100644 --- a/src/libpakfire/build.c +++ b/src/libpakfire/build.c @@ -95,6 +95,9 @@ struct pakfire_build { // Buildroot char buildroot[PATH_MAX]; + // ccache path + char ccache_path[PATH_MAX]; + // States int init:1; }; @@ -1459,8 +1462,32 @@ static int pakfire_build_setup_jail(struct pakfire_build* build) { /* Sets up the ccache for this build */ +static int pakfire_build_mount_ccache(struct pakfire_build* build) { + int r; + + // Do nothing if the ccache is disabled + if (pakfire_build_has_flag(build, PAKFIRE_BUILD_DISABLE_CCACHE)) + return 0; + + // Check that the path is set + if (!*build->ccache_path) { + errno = ENOTSUP; + return 1; + } + + // Make sure the path exists + r = pakfire_mkdir(build->ccache_path, 0755); + if (r) { + ERROR(build->pakfire, "Could not create %s: %m\n", build->ccache_path); + return r; + } + + // Bind-mount the directory + return pakfire_jail_bind(build->jail, build->ccache_path, CCACHE_DIR, + MS_NOSUID|MS_NOEXEC|MS_NODEV); +} + static int pakfire_build_setup_ccache(struct pakfire_build* build) { - char path[PATH_MAX]; int r; // Check if we want a ccache @@ -1477,28 +1504,10 @@ static int pakfire_build_setup_ccache(struct pakfire_build* build) { return 0; } - // Compose path - r = pakfire_cache_path(build->pakfire, path, "%s", "ccache"); - if (r) { - ERROR(build->pakfire, "Could not compose ccache path: %m\n"); - return 1; - } - - DEBUG(build->pakfire, "Mounting ccache from %s\n", path); - - // Ensure path exists - r = pakfire_mkdir(path, 0755); - if (r && errno != EEXIST) { - ERROR(build->pakfire, "Could not create ccache directory %s: %m\n", path); + // Set a default path + r = pakfire_cache_path(build->pakfire, build->ccache_path, "%s", "ccache"); + if (r) return r; - } - - // Bind-mount the directory - r = pakfire_jail_bind(build->jail, path, CCACHE_DIR, MS_NOSUID|MS_NOEXEC|MS_NODEV); - if (r) { - ERROR(build->pakfire, "Could not mount ccache: %m\n"); - return r; - } return 0; } @@ -1619,6 +1628,24 @@ PAKFIRE_EXPORT struct pakfire_build* pakfire_build_unref(struct pakfire_build* b return NULL; } +PAKFIRE_EXPORT int pakfire_build_set_ccache_path( + struct pakfire_build* build, const char* path) { + // Check if this can be called + if (pakfire_build_has_flag(build, PAKFIRE_BUILD_DISABLE_CCACHE)) { + errno = EPERM; + return 1; + } + + // Check input value + if (!path || !*path) { + errno = EINVAL; + return 1; + } + + // Store the path + return pakfire_string_set(build->ccache_path, path); +} + PAKFIRE_EXPORT int pakfire_build_set_target( struct pakfire_build* build, const char* target) { return pakfire_string_set(build->target, target); @@ -2108,6 +2135,13 @@ PAKFIRE_EXPORT int pakfire_build_exec(struct pakfire_build* build, const char* p goto ERROR; } + // Mount the ccache + r = pakfire_build_mount_ccache(build); + if (r) { + ERROR(build->pakfire, "Could not mount the ccache: %m\n"); + goto ERROR; + } + // Create BUILDROOT buildroot = pakfire_mkdtemp(build->buildroot); if (!buildroot) { diff --git a/src/libpakfire/include/pakfire/build.h b/src/libpakfire/include/pakfire/build.h index 56749e78b..7f470e1ac 100644 --- a/src/libpakfire/include/pakfire/build.h +++ b/src/libpakfire/include/pakfire/build.h @@ -38,6 +38,7 @@ int pakfire_build_create(struct pakfire_build** build, struct pakfire_build* pakfire_build_ref(struct pakfire_build* build); struct pakfire_build* pakfire_build_unref(struct pakfire_build* build); +int pakfire_build_set_ccache_path(struct pakfire_build* build, const char* path); int pakfire_build_set_target(struct pakfire_build* build, const char* target); int pakfire_build_exec(struct pakfire_build* build, const char* path); diff --git a/src/libpakfire/libpakfire.sym b/src/libpakfire/libpakfire.sym index a68edebb3..7378b053c 100644 --- a/src/libpakfire/libpakfire.sym +++ b/src/libpakfire/libpakfire.sym @@ -67,6 +67,7 @@ global: pakfire_build_create; pakfire_build_exec; pakfire_build_ref; + pakfire_build_set_ccache_path; pakfire_build_set_target; pakfire_build_unref; pakfire_shell; diff --git a/src/pakfire/daemon.py b/src/pakfire/daemon.py index 43f79971d..830f2c7aa 100644 --- a/src/pakfire/daemon.py +++ b/src/pakfire/daemon.py @@ -7,6 +7,7 @@ import json import logging import logging.handlers import multiprocessing +import os.path import setproctitle import signal import socket @@ -50,6 +51,13 @@ class Daemon(object): # Stats Connection self.stats = None + @property + def ccache_path(self): + """ + Returns the ccache path + """ + return self.config.get("daemon", "ccache_path", "/var/cache/pakfire/ccache") + def connect_to_hub(self): url = self.config.get("daemon", "server", PAKFIRE_HUB) @@ -200,6 +208,27 @@ class Worker(multiprocessing.Process): self.log.debug("Worker %s terminated gracefully" % self.pid) + @property + def ccache(self): + """ + ccache settings + """ + return self.data.get("ccache", {}) + + @property + def ccache_enabled(self): + return self.ccache.get("enabled", False) + + @property + def ccache_path(self): + """ + The ccache path for this job + """ + path = self.ccache.get("path", None) + + if path: + return os.path.join(self.daemon.ccache_path, path) + async def _work(self): """ Called from the async IO loop doing all the work @@ -239,7 +268,15 @@ class Worker(multiprocessing.Process): # Run the build try: build = self._build(pkg, arch=arch, target=target, - logger=logger._log, build_id=job_id) + logger=logger._log, build_id=job_id, + + # Always disable using snapshots + disable_snapshot=True, + + # ccache + disable_ccache=not self.ccache_enabled, + ccache_path=self.ccache_path, + ) # Wait until the build process is done and stream the log in the meantime while not build.done(): @@ -282,8 +319,7 @@ class Worker(multiprocessing.Process): os.unlink(self.pakfire_conf) # Run the build in a new thread - thread = asyncio.to_thread(p.build, pkg, - disable_ccache=True, disable_snapshot=True, **kwargs) + thread = asyncio.to_thread(p.build, pkg, **kwargs) # Return a task return asyncio.create_task(thread) -- 2.39.5