From ccdd2e95bf6758478ef25b29ab6ca620b22fccb8 Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Tue, 6 Dec 2022 18:34:10 +0000 Subject: [PATCH] build: Pass filelists into dependency scripts Signed-off-by: Michael Tremer --- src/_pakfire/pakfire.c | 2 +- src/libpakfire/build.c | 164 ++++++++++++++++------ src/libpakfire/filelist.c | 16 --- src/libpakfire/include/pakfire/filelist.h | 1 - src/libpakfire/include/pakfire/jail.h | 24 +++- src/libpakfire/jail.c | 32 +++-- src/libpakfire/scriptlet.c | 2 +- src/scripts/find-provides | 9 +- src/scripts/find-requires | 9 +- tests/libpakfire/cgroup.c | 2 +- tests/libpakfire/jail.c | 17 +-- 11 files changed, 167 insertions(+), 111 deletions(-) diff --git a/src/_pakfire/pakfire.c b/src/_pakfire/pakfire.c index ea25f3ed2..68b97bdeb 100644 --- a/src/_pakfire/pakfire.c +++ b/src/_pakfire/pakfire.c @@ -963,7 +963,7 @@ static PyObject* Pakfire_execute(PakfireObject* self, PyObject* args, PyObject* } // Execute command - r = pakfire_jail_exec_communicate(jail, argv, + r = pakfire_jail_exec(jail, argv, NULL, Pakfire_execute_output_callback, callback); // If the return code was negative, we had some internal error diff --git a/src/libpakfire/build.c b/src/libpakfire/build.c index 6cd2f44ce..de9706672 100644 --- a/src/libpakfire/build.c +++ b/src/libpakfire/build.c @@ -183,59 +183,137 @@ static int pakfire_build_shell(struct pakfire_build* build) { return pakfire_jail_shell(build->jail); } -static int pakfire_build_run_script(struct pakfire_build* build, const char* filename, - const char* args[], char** output) { - char* script = NULL; - size_t size = 0; +static int pakfire_build_read_script(struct pakfire_build* build, + const char* filename, char** buffer, size_t* length) { char path[PATH_MAX]; + FILE* f = NULL; + int r; - DEBUG(build->pakfire, "Running build script '%s'...\n", filename); + // Compose the source path + r = pakfire_path_join(path, PAKFIRE_SCRIPTS_DIR, filename); + if (r) { + ERROR(build->pakfire, "Could not compose path for script '%s': %m\n", filename); + goto ERROR; + } - // Make the source path - pakfire_path_join(path, PAKFIRE_SCRIPTS_DIR, filename); + DEBUG(build->pakfire, "Reading script from %s...\n", path); - // Open the source script - FILE* f = fopen(path, "r"); + // Open the file + f = fopen(path, "r"); if (!f) { - ERROR(build->pakfire, "Could not open %s: %m\n", path); - return 1; + ERROR(build->pakfire, "Could not open script %s: %m\n", path); + goto ERROR; } - // Read the script into memory - int r = pakfire_read_file_into_buffer(f, &script, &size); + // Read the file into a the buffer + r = pakfire_read_file_into_buffer(f, buffer, length); if (r) { - ERROR(build->pakfire, "Could not read script into memory: %m\n"); + ERROR(build->pakfire, "Could not read script: %m\n"); goto ERROR; } +ERROR: + if (f) + fclose(f); + + return r; +} + +static int pakfire_build_run_script( + struct pakfire_build* build, + const char* filename, + const char* args[], + pakfire_jail_communicate_in communicate_in, + pakfire_jail_communicate_out communicate_out, + void* data) { + int r; + + char* script = NULL; + size_t length = 0; + + DEBUG(build->pakfire, "Running build script '%s'...\n", filename); + + // Read the script + r = pakfire_build_read_script(build, filename, &script, &length); + if (r) { + ERROR(build->pakfire, "Could not read script %s: %m\n", filename); + return r; + } + // Execute the script - r = pakfire_jail_exec_script(build->jail, script, size, args, output); + r = pakfire_jail_exec_script(build->jail, script, length, args, + communicate_in, communicate_out, data); if (r) { ERROR(build->pakfire, "Script '%s' failed with status %d\n", filename, r); } -ERROR: if (script) free(script); return r; } +struct pakfire_build_filelist_streamer { + struct pakfire_filelist* filelist; + char* output; +}; + +static int pakfire_build_filelist_streamer_one(struct pakfire* pakfire, + struct pakfire_file* file, void* data) { + int* fd = (int*)data; + int r; + + const char* path = pakfire_file_get_path(file); + if (!path) { + ERROR(pakfire, "Received a file with an empty path\n"); + return 1; + } + + // Write path + r = dprintf(*fd, "%s\n", path); + if (r < 0) + return r; + + return 0; +} + +static int pakfire_build_filelist_streamer_in(struct pakfire* pakfire, void* data, int fd) { + struct pakfire_build_filelist_streamer* streamer = + (struct pakfire_build_filelist_streamer*)data; + + return pakfire_filelist_export(streamer->filelist, pakfire_build_filelist_streamer_one, &fd); +} + +static int pakfire_build_filelist_streamer_out(struct pakfire* pakfire, + void* data, int priority, const char* buffer, const size_t length) { + struct pakfire_build_filelist_streamer* streamer = + (struct pakfire_build_filelist_streamer*)data; + + return pakfire_jail_capture_stdout(pakfire, streamer->output, priority, buffer, length); +} + static int pakfire_build_run_dependency_script(struct pakfire_build* build, - struct pakfire_package* pkg, int dep, const char* script, const char** args) { - char* output = NULL; + struct pakfire_package* pkg, int dep, const char* script, const char** args, + struct pakfire_filelist* filelist) { + struct pakfire_build_filelist_streamer streamer = { + .filelist = filelist, + .output = NULL, + }; int r; + printf("streamer 1 = %p\n", &streamer); + // Run the script - r = pakfire_build_run_script(build, script, args, &output); + r = pakfire_build_run_script(build, script, args, + pakfire_build_filelist_streamer_in, pakfire_build_filelist_streamer_out, &streamer); if (r) { ERROR(build->pakfire, "%s returned with error %d\n", script, r); goto ERROR; } // Add all output to the package - if (output) { - r = pakfire_str2deps(build->pakfire, pkg, dep, output); + if (streamer.output) { + r = pakfire_str2deps(build->pakfire, pkg, dep, streamer.output); if (r) { ERROR(build->pakfire, "Could not add dependencies: %m\n"); goto ERROR; @@ -243,8 +321,8 @@ static int pakfire_build_run_dependency_script(struct pakfire_build* build, } ERROR: - if (output) - free(output); + if (streamer.output) + free(streamer.output); return r; } @@ -252,25 +330,7 @@ ERROR: static int pakfire_build_find_dependencies(struct pakfire_build* build, struct pakfire_package* pkg, struct pakfire_filelist* filelist, const char* buildroot) { char path[PATH_MAX]; - - // Allocate path to write the filelist to - int r = pakfire_path(build->pakfire, path, "%s", - PAKFIRE_TMP_DIR "/pakfire-filelist.XXXXXX"); - if (r) - return 1; - - // Create a temporary file - FILE* f = pakfire_mktemp(path, 0); - if (!f) - goto ERROR; - - // Write filelist to the temporary file - r = pakfire_filelist_export(filelist, f); - fclose(f); - if (r) { - ERROR(build->pakfire, "Could not export filelist: %m\n"); - goto ERROR; - } + int r; const char* root = pakfire_get_path(build->pakfire); @@ -281,13 +341,25 @@ static int pakfire_build_find_dependencies(struct pakfire_build* build, // Find all provides r = pakfire_build_run_dependency_script(build, pkg, - PAKFIRE_PKG_PROVIDES, "find-provides", args); + PAKFIRE_PKG_PROVIDES, "find-provides", args, filelist); + if (r) + goto ERROR; + + // Find all Perl provides + r = pakfire_build_run_dependency_script(build, pkg, + PAKFIRE_PKG_PROVIDES, "perl.prov", args, filelist); if (r) goto ERROR; // Find all requires r = pakfire_build_run_dependency_script(build, pkg, - PAKFIRE_PKG_REQUIRES, "find-requires", args); + PAKFIRE_PKG_REQUIRES, "find-requires", args, filelist); + if (r) + goto ERROR; + + // Find all Perl requires + r = pakfire_build_run_dependency_script(build, pkg, + PAKFIRE_PKG_REQUIRES, "perl.req", args, filelist); if (r) goto ERROR; @@ -744,7 +816,7 @@ static int pakfire_build_stage(struct pakfire_build* build, } // Run the script - r = pakfire_jail_exec_script(build->jail, script, strlen(script), NULL, NULL); + r = pakfire_jail_exec_script(build->jail, script, strlen(script), NULL, NULL, NULL, NULL); if (r) { ERROR(build->pakfire, "Build stage '%s' failed with status %d\n", stage, r); } @@ -788,7 +860,7 @@ static int pakfire_build_run_post_build_scripts(struct pakfire_build* build) { // Run them one by one for (const char** script = post_build_scripts; *script; script++) { - int r = pakfire_build_run_script(build, *script, args, NULL); + int r = pakfire_build_run_script(build, *script, args, NULL, NULL, NULL); if (r) return r; } diff --git a/src/libpakfire/filelist.c b/src/libpakfire/filelist.c index a3722f31a..f3d131ac3 100644 --- a/src/libpakfire/filelist.c +++ b/src/libpakfire/filelist.c @@ -349,22 +349,6 @@ int pakfire_filelist_contains(struct pakfire_filelist* list, const char* pattern return 0; } -int pakfire_filelist_export(struct pakfire_filelist* list, FILE* f) { - for (unsigned int i = 0; i < list->size; i++) { - struct pakfire_file* file = list->elements[i]; - - const char* path = pakfire_file_get_path(file); - if (!path) - return 1; - - int r = fprintf(f, "%s\n", path); - if (r < 0) - return r; - } - - return 0; -} - int pakfire_filelist_walk(struct pakfire_filelist* list, pakfire_filelist_walk_callback callback, void* data) { struct pakfire_file* file = NULL; diff --git a/src/libpakfire/include/pakfire/filelist.h b/src/libpakfire/include/pakfire/filelist.h index 519ce82fc..6cf963223 100644 --- a/src/libpakfire/include/pakfire/filelist.h +++ b/src/libpakfire/include/pakfire/filelist.h @@ -49,7 +49,6 @@ int pakfire_filelist_scan(struct pakfire_filelist* list, const char* root, const char** includes, const char** excludes); int pakfire_filelist_contains(struct pakfire_filelist* list, const char* pattern); -int pakfire_filelist_export(struct pakfire_filelist* list, FILE* f); typedef int (*pakfire_filelist_walk_callback) (struct pakfire* pakfire, struct pakfire_file* file, void* data); diff --git a/src/libpakfire/include/pakfire/jail.h b/src/libpakfire/include/pakfire/jail.h index b6b1eb901..dc1396cb6 100644 --- a/src/libpakfire/include/pakfire/jail.h +++ b/src/libpakfire/include/pakfire/jail.h @@ -47,34 +47,44 @@ int pakfire_jail_set_env(struct pakfire_jail* jail, const char* key, const char* int pakfire_jail_import_env(struct pakfire_jail* jail, const char* env[]); // Execute -int pakfire_jail_exec(struct pakfire_jail* jail, const char* argv[], char** output); -int pakfire_jail_exec_script(struct pakfire_jail* jail, - const char* script, const size_t size, const char* args[], char** output); - -// Communicate typedef int (*pakfire_jail_communicate_in) (struct pakfire* pakfire, void* data, int fd); typedef int (*pakfire_jail_communicate_out) (struct pakfire* pakfire, void* data, int priority, const char* line, const size_t length); -int pakfire_jail_exec_communicate( +int pakfire_jail_exec( struct pakfire_jail* jail, const char* argv[], pakfire_jail_communicate_in callback_in, pakfire_jail_communicate_out callback_out, void* data); +int pakfire_jail_exec_script(struct pakfire_jail* jail, + const char* script, + const size_t size, + const char* args[], + pakfire_jail_communicate_in callback_in, + pakfire_jail_communicate_out callback_out, + void* data); + #ifdef PAKFIRE_PRIVATE #include +// Capture +int pakfire_jail_capture_stdout(struct pakfire* pakfire, void* data, + int priority, const char* line, size_t length); + // Resource limits int pakfire_jail_set_cgroup(struct pakfire_jail* jail, struct pakfire_cgroup* cgroup); // Convenience functions int pakfire_jail_run(struct pakfire* pakfire, const char* argv[], int flags, char** output); int pakfire_jail_run_script(struct pakfire* pakfire, - const char* script, const size_t length, const char* argv[], int flags, char** output); + const char* script, const size_t length, const char* argv[], int flags, + pakfire_jail_communicate_in communicate_in, + pakfire_jail_communicate_out communicate_out, + void* data); int pakfire_jail_shell(struct pakfire_jail* jail); int pakfire_jail_ldconfig(struct pakfire* pakfire); diff --git a/src/libpakfire/jail.c b/src/libpakfire/jail.c index ba9a10059..54b8a0807 100644 --- a/src/libpakfire/jail.c +++ b/src/libpakfire/jail.c @@ -708,8 +708,8 @@ ERROR: return r; } -static int pakfire_jail_capture_stdout(struct pakfire* pakfire, void* data, int priority, - const char* line, size_t length) { +int pakfire_jail_capture_stdout(struct pakfire* pakfire, void* data, + int priority, const char* line, size_t length) { char** output = (char**)data; int r; @@ -1547,7 +1547,7 @@ ERROR: return exit; } -PAKFIRE_EXPORT int pakfire_jail_exec_communicate( +PAKFIRE_EXPORT int pakfire_jail_exec( struct pakfire_jail* jail, const char* argv[], pakfire_jail_communicate_in callback_in, @@ -1556,11 +1556,6 @@ PAKFIRE_EXPORT int pakfire_jail_exec_communicate( return __pakfire_jail_exec(jail, argv, 0, callback_in, callback_out, data); } -PAKFIRE_EXPORT int pakfire_jail_exec(struct pakfire_jail* jail, - const char* argv[], char** output) { - return __pakfire_jail_exec(jail, argv, 0, NULL, pakfire_jail_capture_stdout, output); -} - static int pakfire_jail_exec_interactive( struct pakfire_jail* jail, const char* argv[]) { int r; @@ -1573,8 +1568,13 @@ static int pakfire_jail_exec_interactive( return __pakfire_jail_exec(jail, argv, 1, NULL, NULL, NULL); } -PAKFIRE_EXPORT int pakfire_jail_exec_script(struct pakfire_jail* jail, - const char* script, const size_t size, const char* args[], char** output) { +int pakfire_jail_exec_script(struct pakfire_jail* jail, + const char* script, + const size_t size, + const char* args[], + pakfire_jail_communicate_in callback_in, + pakfire_jail_communicate_out callback_out, + void* data) { char path[PATH_MAX]; const char** argv = NULL; FILE* f = NULL; @@ -1633,7 +1633,7 @@ PAKFIRE_EXPORT int pakfire_jail_exec_script(struct pakfire_jail* jail, argv[i] = args[i-1]; // Run the script - r = pakfire_jail_exec(jail, argv, output); + r = pakfire_jail_exec(jail, argv, callback_in, callback_out, data); ERROR: if (argv) @@ -1662,7 +1662,7 @@ int pakfire_jail_run(struct pakfire* pakfire, const char* argv[], int flags, cha goto ERROR; // Execute the command - r = pakfire_jail_exec(jail, argv, output); + r = pakfire_jail_exec(jail, argv, NULL, pakfire_jail_capture_stdout, output); ERROR: if (jail) @@ -1672,7 +1672,10 @@ ERROR: } int pakfire_jail_run_script(struct pakfire* pakfire, - const char* script, const size_t length, const char* argv[], int flags, char** output) { + const char* script, const size_t length, const char* argv[], int flags, + pakfire_jail_communicate_in communicate_in, + pakfire_jail_communicate_out communicate_out, + void* data) { struct pakfire_jail* jail = NULL; int r; @@ -1682,7 +1685,8 @@ int pakfire_jail_run_script(struct pakfire* pakfire, goto ERROR; // Execute the command - r = pakfire_jail_exec_script(jail, script, length, argv, output); + r = pakfire_jail_exec_script(jail, script, length, argv, + communicate_in, communicate_out, data); ERROR: if (jail) diff --git a/src/libpakfire/scriptlet.c b/src/libpakfire/scriptlet.c index e9c2aae97..919c4ba54 100644 --- a/src/libpakfire/scriptlet.c +++ b/src/libpakfire/scriptlet.c @@ -170,7 +170,7 @@ int pakfire_scriptlet_execute(struct pakfire_scriptlet* scriptlet) { // Detect what kind of script this is and run it if (pakfire_scriptlet_is_shell_script(scriptlet)) return pakfire_jail_run_script(scriptlet->pakfire, - scriptlet->data, scriptlet->size, NULL, 0, NULL); + scriptlet->data, scriptlet->size, NULL, 0, NULL, NULL, NULL); ERROR(scriptlet->pakfire, "Scriptlet is of an unknown kind\n"); errno = ENOTSUP; diff --git a/src/scripts/find-provides b/src/scripts/find-provides index d319f1768..d3379ae5e 100644 --- a/src/scripts/find-provides +++ b/src/scripts/find-provides @@ -71,7 +71,6 @@ shared_object_provides() { main() { local buildroot="${1}" - local filelist="${2}" # Check if BUILDROOT exists if [ ! -d "${buildroot}" ]; then @@ -79,12 +78,6 @@ main() { return 1 fi - # Check if the filelist exists - if [ ! -r "${filelist}" ]; then - error "Cannot read filelist '${filelist}'" - return 1 - fi - local file while read -r file; do # Filter out what we don't need @@ -131,7 +124,7 @@ main() { shared_object_provides "${path}" ;; esac - done < "${filelist}" | sort -u + done | sort -u return 0 } diff --git a/src/scripts/find-requires b/src/scripts/find-requires index 63992e213..7db19da79 100644 --- a/src/scripts/find-requires +++ b/src/scripts/find-requires @@ -131,7 +131,6 @@ find_script_interpreter() { main() { local buildroot="${1}" - local filelist="${2}" # Check if BUILDROOT exists if [ ! -d "${buildroot}" ]; then @@ -139,12 +138,6 @@ main() { return 1 fi - # Check if the filelist exists - if [ ! -r "${filelist}" ]; then - error "Cannot read filelist '${filelist}'" - return 1 - fi - # Tell pkg-config to search in the newly created files, too export PKG_CONFIG_PATH="${buildroot}/usr/lib64/pkgconfig:${buildroot}/usr/lib/pkgconfig:${buildroot}/usr/share/pkgconfig" @@ -244,7 +237,7 @@ main() { if ! find_script_interpreter "${path}"; then return 1 fi - done < "${filelist}" | sort -u + done | sort -u return 0 } diff --git a/tests/libpakfire/cgroup.c b/tests/libpakfire/cgroup.c index c37084281..e90a8f4b6 100644 --- a/tests/libpakfire/cgroup.c +++ b/tests/libpakfire/cgroup.c @@ -74,7 +74,7 @@ static int test_stats(const struct test* t) { // Run a few things for (unsigned int i = 0; i < 3; i++) { - ASSERT_SUCCESS(pakfire_jail_exec(jail, argv, NULL)); + ASSERT_SUCCESS(pakfire_jail_exec(jail, argv, NULL, NULL, NULL)); } // Try reading the stats into some invalid space diff --git a/tests/libpakfire/jail.c b/tests/libpakfire/jail.c index 0bb0348c1..8de3dce26 100644 --- a/tests/libpakfire/jail.c +++ b/tests/libpakfire/jail.c @@ -91,7 +91,7 @@ static int test_exec(const struct test* t) { ASSERT_SUCCESS(pakfire_jail_create(&jail, t->pakfire, 0)); // Try to execute something - ASSERT_SUCCESS(pakfire_jail_exec(jail, cmd_hello_world, NULL)); + ASSERT_SUCCESS(pakfire_jail_exec(jail, cmd_hello_world, NULL, NULL, NULL)); // Destroy it ASSERT_NULL(pakfire_jail_unref(jail)); @@ -117,7 +117,7 @@ static int test_launch_into_cgroup(const struct test* t) { ASSERT_SUCCESS(pakfire_jail_set_cgroup(jail, cgroup)); // Run command - ASSERT(pakfire_jail_exec(jail, cmd_hello_world, NULL) == 0); + ASSERT(pakfire_jail_exec(jail, cmd_hello_world, NULL, NULL, NULL) == 0); r = EXIT_SUCCESS; @@ -152,7 +152,8 @@ static int test_nice(const struct test* t) { ASSERT_SUCCESS(pakfire_jail_nice(jail, 5)); // Check if the nice level has been set - ASSERT_SUCCESS(pakfire_jail_exec(jail, argv, &output)); + ASSERT_SUCCESS(pakfire_jail_exec(jail, argv, + NULL, pakfire_jail_capture_stdout, &output)); ASSERT_STRING_EQUALS(output, "5\n"); // Success @@ -186,10 +187,10 @@ static int test_memory_limit(const struct test* t) { ASSERT_SUCCESS(pakfire_cgroup_set_memory_limit(cgroup, 100 * 1024 * 1024)); // Try to exhaust all memory - ASSERT_FAILURE(pakfire_jail_exec(jail, cmd_exhaust_memory, NULL)); + ASSERT_FAILURE(pakfire_jail_exec(jail, cmd_exhaust_memory, NULL, NULL, NULL)); // A fork bomb should also exhaust all memory - ASSERT_FAILURE(pakfire_jail_exec(jail, cmd_fork_bomb, NULL)); + ASSERT_FAILURE(pakfire_jail_exec(jail, cmd_fork_bomb, NULL, NULL, NULL)); // Success r = EXIT_SUCCESS; @@ -223,7 +224,7 @@ static int test_pid_limit(const struct test* t) { ASSERT_SUCCESS(pakfire_cgroup_set_pid_limit(cgroup, 100)); // Try to fork as many processes as possible - ASSERT_FAILURE(pakfire_jail_exec(jail, cmd_fork_bomb, NULL)); + ASSERT_FAILURE(pakfire_jail_exec(jail, cmd_fork_bomb, NULL, NULL, NULL)); // Success r = EXIT_SUCCESS; @@ -282,7 +283,7 @@ static int test_bind(const struct test* t) { ASSERT_SUCCESS(pakfire_jail_bind(jail, source, target, MS_RDONLY)); // Check if the mount actually works - ASSERT_SUCCESS(pakfire_jail_exec(jail, argv, NULL)); + ASSERT_SUCCESS(pakfire_jail_exec(jail, argv, NULL, NULL, NULL)); // Success r = EXIT_SUCCESS; @@ -322,7 +323,7 @@ static int test_communicate(const struct test* t) { ASSERT_SUCCESS(pakfire_jail_create(&jail, t->pakfire, 0)); // Check if the mount actually works - ASSERT_SUCCESS(pakfire_jail_exec_communicate(jail, argv, callback_stdin, NULL, NULL)); + ASSERT_SUCCESS(pakfire_jail_exec(jail, argv, callback_stdin, NULL, NULL)); // Success r = EXIT_SUCCESS; -- 2.39.5