[people/stevee/pakfire.git] / src / libpakfire / build.c

/*#############################################################################
#                                                                             #
# Pakfire - The IPFire package management system                              #
# Copyright (C) 2021 Pakfire development team                                 #
#                                                                             #
# This program is free software: you can redistribute it and/or modify        #
# it under the terms of the GNU General Public License as published by        #
# the Free Software Foundation, either version 3 of the License, or           #
# (at your option) any later version.                                         #
#                                                                             #
# This program is distributed in the hope that it will be useful,             #
# but WITHOUT ANY WARRANTY; without even the implied warranty of              #
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
# GNU General Public License for more details.                                #
#                                                                             #
# You should have received a copy of the GNU General Public License           #
# along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
#                                                                             #
#############################################################################*/

#include <errno.h>
#include <stdlib.h>
#include <sys/mount.h>

#include <pakfire/build.h>
#include <pakfire/execute.h>
#include <pakfire/dist.h>
#include <pakfire/file.h>
#include <pakfire/logging.h>
#include <pakfire/package.h>
#include <pakfire/packager.h>
#include <pakfire/parser.h>
#include <pakfire/private.h>
#include <pakfire/scriptlet.h>
#include <pakfire/snapshot.h>
#include <pakfire/types.h>
#include <pakfire/util.h>

#define CCACHE_DIR "/var/cache/ccache"

static const char* stages[] = {
	"prepare",
	"build",
	"test",
	"install",
	NULL,
};

#define TEMPLATE \
	"#!/bin/bash --login\n" \
	"\n" \
	"set -e\n" \
	"set -x\n" \
	"\n" \
	"%%{_%s}\n" \
	"\n" \
	"exit 0\n"

static int pakfire_build_install_packages(Pakfire pakfire, int* snapshot_needs_update) {
	char** packages = NULL;
	int r = 1;

	struct pakfire_config* config = pakfire_get_config(pakfire);

	// Fetch build environment
	const char* requires = pakfire_config_get(config, "build", "requires", NULL);
	if (!requires) {
		ERROR(pakfire, "No build requirements have been defined\n");
		goto ERROR;
	}

	// Split requirements into packages
	packages = pakfire_split_string(requires, ',');
	if (!packages)
		goto ERROR;

	int changed = 0;

	// Install everything
	r = pakfire_install(pakfire, (const char**)packages, 0, &changed);
	if (r) {
		ERROR(pakfire, "Could not install build dependencies\n");
		goto ERROR;
	}

	// Mark snapshot as changed if new packages were installed
	if (changed)
		*snapshot_needs_update = 1;

	// XXX Update everything

	// Success
	r = 0;

ERROR:
	if (packages) {
		for (char** package = packages; *package; package++)
			free(*package);
		free(packages);
	}
	if (config)
		pakfire_config_unref(config);

	return r;
}

int pakfire_build_setup(Pakfire pakfire) {
	char path[PATH_MAX];
	int r;

	// Mount ccache
	if (!pakfire_has_flag(pakfire, PAKFIRE_FLAGS_DISABLE_CCACHE)) {
		r = pakfire_make_cache_path(pakfire, path, "%s", "ccache");
		if (r < 0)
			return r;

		// Ensure path exists
		r = pakfire_mkdir(path, 0);
		if (r && errno != EEXIST) {
			ERROR(pakfire, "Could not create ccache directory %s: %m\n", path);
			return r;
		}

		r = pakfire_bind(pakfire, path, CCACHE_DIR, MS_NOSUID|MS_NOEXEC|MS_NODEV);
		if (r) {
			ERROR(pakfire, "Could not mount ccache: %m\n");
			return r;
		}
	}

	// Extract snapshot
	if (!pakfire_has_flag(pakfire, PAKFIRE_FLAGS_DISABLE_SNAPSHOT)) {
		r = pakfire_make_cache_path(pakfire, path, "%s", "snapshot.tar.zst");
		if (r < 0)
			return r;

		// Open the snapshot
		FILE* f = fopen(path, "r");

		// Try restoring the snapshot
		if (f) {
			r = pakfire_snapshot_restore(pakfire, f);
			if (r) {
				fclose(f);
				return r;
			}
		}

		// Tells us whether we need to (re-)create the snapshot
		int snapshot_needs_update = 0;

		// Install or update any build dependencies
		r = pakfire_build_install_packages(pakfire, &snapshot_needs_update);
		if (r)
			return r;

		if (snapshot_needs_update) {
			// Open snapshot file for writing
			f = fopen(path, "w");
			if (!f) {
				ERROR(pakfire, "Could not open snapshot file for writing: %m\n");
				return 1;
			}

			// Create a new snapshot
			r = pakfire_snapshot_create(pakfire, f);
			fclose(f);

			if (r)
				return r;
		}
	}

	return 0;
}

static int pakfire_build_run_script(Pakfire pakfire, const char* filename, const char* args[],
		pakfire_execute_logging_callback logging_callback, void* data) {
	char* script = NULL;
	size_t size = 0;
	char path[PATH_MAX];

	DEBUG(pakfire, "Running build script '%s'...\n", filename);

	// Make the source path
	pakfire_path_join(path, PAKFIRE_SCRIPTS_DIR, filename);

	// Open the source script
	FILE* f = fopen(path, "r");
	if (!f) {
		ERROR(pakfire, "Could not open %s: %s\n", path, strerror(errno));
		return 1;
	}

	// Read the script into memory
	int r = pakfire_read_file_into_buffer(f, &script, &size);
	if (r) {
		ERROR(pakfire, "Could not read script into memory: %s\n", strerror(errno));
		goto ERROR;
	}

	// Execute it
	r = pakfire_execute_script(pakfire, script, size, args, NULL, 0, logging_callback, data);
	if (r) {
		ERROR(pakfire, "Script '%s' failed with status %d\n", filename, r);
	}

ERROR:
	if (script)
		free(script);

	return r;
}

static int find_dependency(char** haystack, const char* needle) {
	if (!needle) {
		errno = EINVAL;
		return -1;
	}

	if (!haystack)
		return 0;

	for (char** element = haystack; *element; element++) {
		if (strcmp(needle, *element) == 0)
			return 1;
	}

	return 0;
}

static int pakfire_build_find_dependencies(Pakfire pakfire,
		PakfirePackage pkg, PakfireFilelist filelist, const char* buildroot) {
	char** provides = NULL;
	char** requires = NULL;
	char path[PATH_MAX];

	// Allocate path to write the filelist to
	int r = pakfire_make_path(pakfire, path, "tmp/.pakfire-filelist.XXXXXX");
	if (r < 0)
		return 1;

	// Create a temporary file
	FILE* f = pakfire_mktemp(path);
	if (!f)
		goto ERROR;

	// Write filelist to the temporary file
	r = pakfire_filelist_export(filelist, f);
	fclose(f);
	if (r) {
		ERROR(pakfire, "Could not export filelist: %s\n", strerror(errno));
		goto ERROR;
	}

	const char* root = pakfire_get_path(pakfire);

	// Pass buildroot and the filelist as arguments
	const char* args[] = {
		buildroot, pakfire_path_relpath(root, path), NULL
	};

	// Find all provides
	r = pakfire_build_run_script(pakfire, "find-provides", args,
		pakfire_execute_capture_stdout_to_array, &provides);
	if (r) {
		ERROR(pakfire, "find-provides returned with error %d\n", r);
		goto ERROR;
	}

	// Find all requires
	r = pakfire_build_run_script(pakfire, "find-requires", args,
		pakfire_execute_capture_stdout_to_array, &requires);
	if (r) {
		ERROR(pakfire, "find-requires returned with error %d\n", r);
		goto ERROR;
	}

	// Add all provides to the package
	if (provides) {
		for (char** element = provides; *element; element++) {
			DEBUG(pakfire, "Adding provides: %s\n", *element);
			pakfire_package_add_provides(pkg, *element);
		}
	}

	// Add all requires to the package
	if (requires) {
		for (char** element = requires; *element; element++) {
			// Skip adding this requirement if also provided by this package
			if (find_dependency(provides, *element))
				continue;

			DEBUG(pakfire, "Adding requires: %s\n", *element);
			pakfire_package_add_requires(pkg, *element);
		}
	}

	// Success
	r = 0;

ERROR:
	if (provides) {
		for (char** element = provides; *element; element++)
			free(*element);
		free(provides);
	}
	if (requires) {
		for (char** element = requires; *element; element++)
			free(*element);
		free(requires);
	}
	unlink(path);

	return r;
}

static int append_to_array(const char*** array, const char* s) {
	unsigned int length = 0;

	// Determine the length of the existing array
	if (*array) {
		for (const char** element = *array; *element; element++)
			length++;
	}

	// Allocate space
	*array = reallocarray(*array, length + 2, sizeof(**array));
	if (!*array)
		return 1;

	// Append s and terminate the array
	(*array)[length] = s;
	(*array)[length + 1] = NULL;

	return 0;
}

static int pakfire_build_package_add_files(Pakfire pakfire, PakfireParser makefile,
		const char* buildroot, const char* namespace, PakfirePackage pkg,
		struct pakfire_packager* packager) {
	PakfireFilelist filelist = NULL;
	char path[PATH_MAX];
	int r = 1;

	char** files = NULL;
	const char** includes = NULL;
	const char** excludes = NULL;

	// Fetch filelist from makefile
	files = pakfire_parser_get_split(makefile, namespace, "files", '\n');

	// No files to package?
	if (!files)
		return 0;

	// Convert to absolute path
	pakfire_make_path(pakfire, path, buildroot);

	// Split into includes and excludes
	for (char** file = files; *file; file++) {
		if (**file == '!')
			r = append_to_array(&excludes, *file);
		else
			r = append_to_array(&includes, *file);
		if (r)
			goto ERROR;
	}

	// Allocate a new filelist
	r = pakfire_filelist_create(&filelist, pakfire);
	if (r)
		goto ERROR;

	// Scan for files
	r = pakfire_filelist_scan(filelist, path, includes, excludes);
	if (r)
		goto ERROR;

	const size_t length = pakfire_filelist_size(filelist);
	DEBUG(pakfire, "%zu file(s) found\n", length);

	// Nothing to do if the filelist is empty
	if (!length)
		goto ERROR;

	// Find dependencies
	r = pakfire_build_find_dependencies(pakfire, pkg, filelist, buildroot);
	if (r) {
		ERROR(pakfire, "Finding dependencies failed: %s\n", strerror(errno));
		goto ERROR;
	}

	// Add all files to the package
	for (unsigned int i = 0; i < length; i++) {
		struct pakfire_file* file = pakfire_filelist_get(filelist, i);

		// Add the file to the package
		r = pakfire_packager_add(
			packager,
			pakfire_file_get_abspath(file),
			pakfire_file_get_path(file)
		);
		if (r) {
			pakfire_file_unref(file);
			goto ERROR;
		}

		// Remove the file after it was packaged
		r = pakfire_file_cleanup(file);
		if (r) {
			pakfire_file_unref(file);
			goto ERROR;
		}

		pakfire_file_unref(file);
	}

ERROR:
	if (filelist)
		pakfire_filelist_unref(filelist);
	if (files) {
		for (char** file = files; *file; file++)
			free(*file);
		free(files);
	}
	if (includes)
		free(includes);
	if (excludes)
		free(excludes);

	return r;
}

static int pakfire_build_add_scriptlet_requires(Pakfire pakfire, PakfirePackage pkg,
		struct pakfire_scriptlet* scriptlet) {
	char** prerequires = NULL;
	char path[PATH_MAX];
	size_t size;
	int r;

	const char* root = pakfire_get_path(pakfire);

	// Make filename
	r = pakfire_make_path(pakfire, path, "tmp/.pakfire-scriptlet.XXXXXX");
	if (r < 0)
		return r;

	// Fetch scriptlet payload
	const char* data = pakfire_scriptlet_get_data(scriptlet, &size);

	// Create a temporary file
	FILE* f = pakfire_mktemp(path);
	if (!f)
		return 1;

	// Write scriptlet
	ssize_t bytes_written = fwrite(data, 1, size, f);
	if (bytes_written < 0) {
		ERROR(pakfire, "Could not write to %s: %m\n", path);
		fclose(f);
		goto ERROR;
	}

	// Close file
	fclose(f);

	// Build commandline
	const char* args[] = {
		pakfire_path_relpath(root, path),
		NULL,
	};

	// Find all pre-requires
	r = pakfire_build_run_script(pakfire, "find-prerequires", args,
		pakfire_execute_capture_stdout_to_array, &prerequires);
	if (r)
		goto ERROR;

	// Add all pre-requires to the package
	if (prerequires) {
		for (char** element = prerequires; *element; element++) {
			DEBUG(pakfire, "Adding pre-requires: %s\n", *element);
			pakfire_package_add_prerequires(pkg, *element);
		}
	}

ERROR:
	if (r && *path)
		unlink(path);
	if (prerequires) {
		for (char** element = prerequires; *element; element++)
			free(*element);
		free(prerequires);
	}
	return r;
}

static int pakfire_build_package_add_scriptlet(Pakfire pakfire, PakfirePackage pkg,
		struct pakfire_packager* packager, const char* type, const char* data) {
	struct pakfire_scriptlet* scriptlet = NULL;
	char* shell = NULL;
	int r;

	// Wrap scriptlet into a shell script
	r = asprintf(&shell, "#!/bin/sh\n\nset -e\n\n%s\n\nexit 0\n", data);
	if (r < 0)
		goto ERROR;

	// Create a scriptlet
	r = pakfire_scriptlet_create(&scriptlet, pakfire, type, shell, 0);
	if (r)
		goto ERROR;

	// Add it to the package
	r = pakfire_packager_add_scriptlet(packager, scriptlet);
	if (r) {
		ERROR(pakfire, "Could not add scriptlet %s\n", type);
		goto ERROR;
	}

	// Add scriptlet requirements
	r = pakfire_build_add_scriptlet_requires(pakfire, pkg, scriptlet);
	if (r) {
		ERROR(pakfire, "Could not add scriptlet requirements: %s\n", strerror(errno));
		goto ERROR;
	}

	// Success
	r = 0;

ERROR:
	if (scriptlet)
		pakfire_scriptlet_unref(scriptlet);
	if (shell)
		free(shell);

	return r;
}

static int pakfire_build_package_add_scriptlets(Pakfire pakfire, PakfireParser makefile,
		const char* namespace, PakfirePackage pkg, struct pakfire_packager* packager) {
	char name[NAME_MAX];
	int r;

	for (const char** type = pakfire_scriptlet_types; *type; type++) {
		r = pakfire_string_format(name, "scriptlet:%s", *type);
		if (r < 0)
			return r;

		// Fetch the scriptlet
		char* data = pakfire_parser_get(makefile, namespace, name);
		if (!data)
			continue;

		// Add it to the package
		r = pakfire_build_package_add_scriptlet(pakfire, pkg, packager, *type, data);
		if (r) {
			free(data);
			return r;
		}

		free(data);
	}

	return 0;
}

static int pakfire_build_package(Pakfire pakfire, PakfireParser makefile,
		const char* id, const char* buildroot, const char* namespace, const char* target) {
	PakfireRepo repo = NULL;
	PakfirePackage pkg = NULL;
	struct pakfire_packager* packager = NULL;

	int r = 1;

	// Expand the handle into the package name
	char* name = pakfire_parser_expand(makefile, "packages", namespace);
	if (!name) {
		ERROR(pakfire, "Could not get package name: %s\n", strerror(errno));
		goto ERROR;
	}

	INFO(pakfire, "Building package '%s'...\n", name);

	// Fetch build architecture
	const char* arch = pakfire_get_arch(pakfire);
	if (!arch)
		goto ERROR;

	// Fetch the dummy repository
	repo = pakfire_get_repo(pakfire, "@dummy");
	if (!repo)
		goto ERROR;

	// Fetch package from makefile
	r = pakfire_parser_create_package(makefile, &pkg, repo, namespace, arch);
	if (r) {
		ERROR(pakfire, "Could not create package from makefile: %s\n", strerror(errno));
		goto ERROR;
	}

	// Set build ID
	if (id)
		pakfire_package_set_build_id(pkg, id);

	// Create a packager
	r = pakfire_packager_create(&packager, pkg);
	if (r)
		goto ERROR;

	// Add files
	r = pakfire_build_package_add_files(pakfire, makefile, buildroot, namespace,
		pkg, packager);
	if (r)
		goto ERROR;

	// Add scriptlets
	r = pakfire_build_package_add_scriptlets(pakfire, makefile, namespace, pkg, packager);
	if (r)
		goto ERROR;

	// Write the finished package
	r = pakfire_packager_finish_to_directory(packager, target);
	if (r) {
		ERROR(pakfire, "pakfire_packager_finish() failed: %s\n", strerror(errno));
		goto ERROR;
	}

#ifdef ENABLE_DEBUG
	char* dump = pakfire_package_dump(pkg, PAKFIRE_PKG_DUMP_LONG);
	if (dump) {
		DEBUG(pakfire, "%s\n", dump);
		free(dump);
	}
#endif

	// Success
	r = 0;

ERROR:
	if (packager)
		pakfire_packager_unref(packager);
	if (repo)
		pakfire_repo_unref(repo);
	if (pkg)
		pakfire_package_unref(pkg);
	if (name)
		free(name);

	return r;
}

static int pakfire_build_packages(Pakfire pakfire, PakfireParser makefile,
		const char* id, const char* buildroot, const char* target) {
	DEBUG(pakfire, "Creating packages...");
	int r = 1;

	// Fetch a list all all packages
	char** packages = pakfire_parser_list_namespaces(makefile, "packages.package:*");
	if (!packages) {
		ERROR(pakfire, "Could not find any packages: %s\n", strerror(errno));
		goto ERROR;
	}

	unsigned int num_packages = 0;

	// Count how many packages we have
	for (char** package = packages; *package; package++)
		num_packages++;

	DEBUG(pakfire, "Found %d package(s)\n", num_packages);

	// Build packages in reverse order
	for (int i = num_packages - 1; i >= 0; i--) {
		r = pakfire_build_package(pakfire, makefile, id, buildroot, packages[i], target);
		if (r)
			goto ERROR;
	}

	// Success
	r = 0;

ERROR:
	if (packages)
		free(packages);

	return r;
}

static int pakfire_build_stage(Pakfire pakfire, PakfireParser makefile, const char* stage,
		pakfire_execute_logging_callback logging_callback, void* data) {
	char template[1024];

	// Prepare template for this stage
	int r = pakfire_string_format(template, TEMPLATE, stage);
	if (r < 0)
		return r;

	// Fetch the environment
	char** envp = pakfire_parser_make_environ(makefile);

	// Create the build script
	char* script = pakfire_parser_expand(makefile, "build", template);
	if (!script) {
		ERROR(pakfire, "Could not generate the build script for stage '%s': %s\n",
			stage, strerror(errno));
		goto ERROR;
	}

	INFO(pakfire, "Running build stage '%s'\n", stage);

	r = pakfire_execute_script(pakfire, script, strlen(script), NULL, envp, 0,
		logging_callback, data);
	if (r) {
		ERROR(pakfire, "Build stage '%s' failed with status %d\n", stage, r);
	}

ERROR:
	if (envp) {
		for (char** e = envp; *e; e++)
			free(*e);
		free(envp);
	}
	if (script)
		free(script);

	return r;
}

static const char* post_build_scripts[] = {
	"remove-static-libs",
	"check-symlinks",
	"check-unsafe-files",
	"check-libraries",
	"check-rpaths",
	"check-buildroot",
	"check-include",
	"check-hardening",
	"check-interpreters",
	"check-fhs",
	"compress-man-pages",
	"strip",
	NULL,
};

static int pakfire_build_run_post_build_scripts(Pakfire pakfire, const char* buildroot,
		pakfire_execute_logging_callback logging_callback, void* data) {
	// Set default arguments for build scripts
	const char* args[] = {
		buildroot, NULL
	};

	// Run them one by one
	for (const char** script = post_build_scripts; *script; script++) {
		int r = pakfire_build_run_script(pakfire, *script, args, logging_callback, data);
		if (r)
			return r;
	}

	return 0;
}

PAKFIRE_EXPORT int pakfire_build(Pakfire pakfire, const char* path,
		const char* target, const char* id, int flags,
		pakfire_execute_logging_callback logging_callback, void* data) {
	PakfireParser makefile = NULL;
	char buildroot[PATH_MAX];
	struct pakfire_parser_error* error = NULL;
	char* generated_id = NULL;
	int r;

	// Check for valid input
	if (!path) {
		errno = EINVAL;
		return 1;
	}

	// The default target is the local repository path
	if (!target) {
		PakfireRepo repo = pakfire_get_repo(pakfire, "local");
		if (!repo) {
			errno = EINVAL;
			return 1;
		}

		target = pakfire_repo_get_path(repo);
		pakfire_repo_unref(repo);
	}

	const char* packages[] = {
		path, NULL
	};

	// Install the package into the build environment
	r = pakfire_install(pakfire, packages, 0, NULL);
	if (r) {
		ERROR(pakfire, "Could not install %s\n", path);
		goto ERROR;
	}

	const char* root = pakfire_get_path(pakfire);

	// Create BUILDROOT
	pakfire_make_path(pakfire, buildroot, "/tmp/.buildroot.XXXXXX");
	if (!pakfire_mkdtemp(buildroot))
		goto ERROR;

	// Make relative BUILDROOT path
	const char* buildroot_rel = pakfire_path_relpath(root, buildroot);
	if (!buildroot_rel)
		goto ERROR;

	// Read makefile
	r = pakfire_read_makefile(&makefile, pakfire, path, &error);
	if (r) {
		if (error) {
			ERROR(pakfire, "Could not parse makefile %s: %s\n", path,
				pakfire_parser_error_get_message(error));
			pakfire_parser_error_unref(error);
		} else {
			ERROR(pakfire, "Could not parse makefile %s: %s\n", path,
				strerror(errno));
		}

		goto ERROR;
	}

	// If no build ID was passed, we generate a random one
	if (!id) {
		generated_id = pakfire_generate_uuid();
		if (!generated_id)
			goto ERROR;

		id = generated_id;
	}

	// Set BUILDROOT
	pakfire_parser_set(makefile, NULL, "BUILDROOT", buildroot_rel, 0);

	// Run through all build stages
	for (const char** stage = stages; *stage; stage++) {
		int r = pakfire_build_stage(pakfire, makefile, *stage, logging_callback, data);
		if (r) {
			// Drop to a shell for debugging
			if (flags & PAKFIRE_BUILD_INTERACTIVE)
				pakfire_shell(pakfire);

			goto ERROR;
		}
	}

	// Run post build scripts
	r = pakfire_build_run_post_build_scripts(pakfire, buildroot_rel, logging_callback, data);
	if (r)
		goto ERROR;

	// Create the packages
	r = pakfire_build_packages(pakfire, makefile, id, buildroot_rel, target);
	if (r) {
		ERROR(pakfire, "Could not create packages: %s\n", strerror(errno));
		goto ERROR;
	}

ERROR:
	if (generated_id)
		free(generated_id);
	if (makefile)
		pakfire_parser_unref(makefile);

	// Remove buildroot
	pakfire_rmtree(buildroot, 0);

	return r;
}

PAKFIRE_EXPORT int pakfire_shell(Pakfire pakfire) {
	const char* argv[] = {
		"/bin/bash", "--login", NULL,
	};

	const int flags =
		PAKFIRE_EXECUTE_INTERACTIVE | PAKFIRE_EXECUTE_ENABLE_NETWORK;

	return pakfire_execute(pakfire, argv, NULL, flags, NULL, NULL);
}