packager: Add new function that accepts a file object

[people/ms/pakfire.git] / src / libpakfire / packager.c

/*#############################################################################
#                                                                             #
# Pakfire - The IPFire package management system                              #
# Copyright (C) 2021 Pakfire development team                                 #
#                                                                             #
# This program is free software: you can redistribute it and/or modify        #
# it under the terms of the GNU General Public License as published by        #
# the Free Software Foundation, either version 3 of the License, or           #
# (at your option) any later version.                                         #
#                                                                             #
# This program is distributed in the hope that it will be useful,             #
# but WITHOUT ANY WARRANTY; without even the implied warranty of              #
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
# GNU General Public License for more details.                                #
#                                                                             #
# You should have received a copy of the GNU General Public License           #
# along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
#                                                                             #
#############################################################################*/

#include <errno.h>
#include <fcntl.h>
#include <linux/limits.h>
#include <stdlib.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <time.h>
#include <unistd.h>

#include <archive.h>
#include <archive_entry.h>

#include <json.h>

#include <pakfire/archive.h>
#include <pakfire/constants.h>
#include <pakfire/logging.h>
#include <pakfire/package.h>
#include <pakfire/packager.h>
#include <pakfire/pakfire.h>
#include <pakfire/pwd.h>
#include <pakfire/string.h>
#include <pakfire/util.h>

struct pakfire_packager {
	struct pakfire* pakfire;
	int nrefs;
	time_t time_created;

	struct pakfire_package* pkg;
	char filename[PATH_MAX];

	// Reader
	struct archive* reader;

	// Payload
	struct archive* payload;
	FILE* fpayload;
	size_t installsize;
	unsigned int files;

	struct pakfire_scriptlet** scriptlets;
	unsigned int num_scriptlets;
};

static int pakfire_packager_create_payload(struct pakfire_packager* p) {
	char path[] = PAKFIRE_TMP_DIR "/pakfire-payload.XXXXXX";

	p->payload = archive_write_new();
	if (!p->payload) {
		ERROR(p->pakfire, "archive_write_new() failed\n");
		return 1;
	}

	// Use the PAX format
	int r = archive_write_set_format_pax(p->payload);
	if (r) {
		ERROR(p->pakfire, "Could not set format to PAX: %s\n",
			archive_error_string(p->payload));
		return r;
	}

	// Create a new temporary file
	p->fpayload = pakfire_mktemp(path);
	if (!p->fpayload)
		return 1;

	// Unlink the file straight away
	unlink(path);

	// Write archive to file
	r = archive_write_open_FILE(p->payload, p->fpayload);
	if (r)
		return r;

	// Add a requirement for the cryptographic algorithms we are using
	pakfire_package_add_requires(p->pkg, "pakfire(Digest-SHA512)");
	pakfire_package_add_requires(p->pkg, "pakfire(Digest-SHA256)");

	return 0;
}

static void pakfire_packager_free(struct pakfire_packager* packager) {
	// Scriptlets
	if (packager->scriptlets) {
		for (unsigned int i = 0; i < packager->num_scriptlets; i++)
			pakfire_scriptlet_unref(packager->scriptlets[i]);
		free(packager->scriptlets);
	}

	// Payload
	if (packager->payload)
		archive_write_free(packager->payload);

	if (packager->fpayload)
		fclose(packager->fpayload);

	if (packager->reader)
		archive_read_free(packager->reader);

	pakfire_package_unref(packager->pkg);
	pakfire_unref(packager->pakfire);
	free(packager);
}

int pakfire_packager_create(struct pakfire_packager** packager,
		struct pakfire* pakfire, struct pakfire_package* pkg) {
	struct pakfire_packager* p = calloc(1, sizeof(*p));
	if (!p)
		return ENOMEM;

	int r = 1;

	// Save creation time
	p->time_created = time(NULL);

	// Initialize reference counting
	p->nrefs = 1;

	// Store a reference to Pakfire
	p->pakfire = pakfire_ref(pakfire);

	// Store a reference to the package
	p->pkg = pakfire_package_ref(pkg);

	// Set build host
	pakfire_package_set_build_host(pkg, pakfire_hostname());

	// Set build time
	pakfire_package_set_build_time(pkg, p->time_created);

	// Create reader
	p->reader = pakfire_make_archive_disk_reader(p->pakfire, 1);
	if (!p->reader)
		goto ERROR;

	// Start payload
	r = pakfire_packager_create_payload(p);
	if (r)
		goto ERROR;

	*packager = p;

	return 0;

ERROR:
	pakfire_packager_free(p);

	return r;
}

struct pakfire_packager* pakfire_packager_ref(
		struct pakfire_packager* packager) {
	++packager->nrefs;

	return packager;
}

struct pakfire_packager* pakfire_packager_unref(
		struct pakfire_packager* packager) {
	if (--packager->nrefs > 0)
		return packager;

	pakfire_packager_free(packager);

	return NULL;
}

const char* pakfire_packager_filename(struct pakfire_packager* packager) {
	if (!*packager->filename) {
		const char* filename = pakfire_package_get_filename(packager->pkg);

		// Add arch
		if (pakfire_package_is_source(packager->pkg))
			pakfire_string_set(packager->filename, filename);
		else {
			const char* arch = pakfire_package_get_arch(packager->pkg);

			pakfire_string_format(packager->filename, "%s/%s", arch, filename);
		}
	}

	return packager->filename;
}

static struct archive_entry* pakfire_packager_create_file(
		struct pakfire_packager* packager, const char* filename, size_t size, mode_t mode) {
	// Create a new file entry
	struct archive_entry* entry = archive_entry_new();
	if (!entry)
		return NULL;

	// Set filename
	archive_entry_set_pathname(entry, filename);

	// This is a regular file
    archive_entry_set_filetype(entry, AE_IFREG);
    archive_entry_set_perm(entry, mode);

	// Set size
	archive_entry_set_size(entry, size);

	// Set ownership
	archive_entry_set_uname(entry, "root");
	archive_entry_set_uid(entry, 0);
	archive_entry_set_gname(entry, "root");
	archive_entry_set_gid(entry, 0);

	// Set times
	archive_entry_set_birthtime(entry, packager->time_created, 0);
	archive_entry_set_ctime(entry, packager->time_created, 0);
	archive_entry_set_mtime(entry, packager->time_created, 0);
	archive_entry_set_atime(entry, packager->time_created, 0);

	return entry;
}

static int pakfire_packager_write_file_from_buffer(struct pakfire_packager* packager,
		struct archive* a, const char* filename, mode_t mode, const char* buffer) {
	size_t size = strlen(buffer);

	// Create a new file
	struct archive_entry* entry = pakfire_packager_create_file(packager, filename, size, mode);
	if (!entry) {
		ERROR(packager->pakfire, "Could not create file '%s'\n", filename);
		return 1;
	}

	// This is the end of the header
	int r = archive_write_header(a, entry);
	if (r) {
		ERROR(packager->pakfire, "Error writing header: %s\n", archive_error_string(a));
		goto ERROR;
	}

	// Write content
	r = archive_write_data(a, buffer, strlen(buffer));
	if (r < 0) {
		ERROR(packager->pakfire, "Error writing data: %s\n", archive_error_string(a));
		goto ERROR;
	}

	// Success
	r = 0;

ERROR:
	archive_entry_free(entry);

	return r;
}

static int pakfire_packager_write_format(struct pakfire_packager* packager,
		struct archive* a) {
	const char buffer[] = TO_STRING(PACKAGE_FORMAT) "\n";

	DEBUG(packager->pakfire, "Writing package format\n");

	int r = pakfire_packager_write_file_from_buffer(packager, a,
		"pakfire-format", 0444, buffer);
	if (r)
		return r;

	// Add package format marker
	pakfire_package_add_requires(packager->pkg,
		"pakfire(PackageFormat-" TO_STRING(PACKAGE_FORMAT) ")");

	return 0;
}

static char* pakfire_packager_make_metadata(struct pakfire_packager* packager) {
	char* result = NULL;

	// Convert all package metadata to JSON
	struct json_object* md = pakfire_package_to_json(packager->pkg);
	if (!md)
		goto ERROR;

	// Serialize JSON to file
	const char* s = json_object_to_json_string_ext(md, 0);
	if (!s)
		goto ERROR;

	// Copy result onto heap
	result = strdup(s);
	if (!result)
		goto ERROR;

ERROR:
	// Free metadata
	if (md)
		json_object_put(md);

	return result;
}

static int pakfire_packager_write_metadata(struct pakfire_packager* packager,
		struct archive* a) {
	// Make metadata
	char* buffer = pakfire_packager_make_metadata(packager);
	if (!buffer)
		return 1;

	DEBUG(packager->pakfire, "Generated package metadata:\n%s\n", buffer);

	// Write buffer
	int r = pakfire_packager_write_file_from_buffer(packager, a,
		".PKGINFO", 0444, buffer);

	free(buffer);

	return r;
}

static int pakfire_packager_copy_archive(struct pakfire_packager* packager,
		struct archive* a, FILE* f) {
	struct archive* payload = NULL;
	struct archive_entry* entry = NULL;
	int r;

	DEBUG(packager->pakfire, "Copying payload\n");

	// Reset fd to beginning of the file
	rewind(f);

	// Re-open the payload archive for reading
	payload = archive_read_new();
	if (!payload) {
		ERROR(packager->pakfire, "Could not create archive reader: %m\n");
		r = 1;
		goto ERROR;
	}

	// We expect a tar archive
	r = archive_read_support_format_tar(payload);
	if (r) {
		ERROR(packager->pakfire, "Could not add support for tar: %s\n",
			archive_error_string(payload));
		goto ERROR;
	}

	// Open the file handle
	r = archive_read_open_FILE(payload, f);
	if (r) {
		ERROR(packager->pakfire, "Could not open payload: %s\n",
			archive_error_string(payload));
		goto ERROR;
	}

	// Copy all files into our archive
	for (;;) {
		r = archive_read_next_header(payload, &entry);

		// End when we have reached the end of the archive
		if (r == ARCHIVE_EOF) {
			r = 0;
			break;
		}

		// Raise any other errors
		else if (r) {
			ERROR(packager->pakfire, "Could not read next header: %s\n",
				archive_error_string(payload));
			goto ERROR;
		}

		// Write the entry
		r = archive_write_header(a, entry);
		if (r) {
			ERROR(packager->pakfire, "Could not write entry: %s\n",
				archive_error_string(a));
			goto ERROR;
		}

		// Copy the data
		r = pakfire_archive_copy_data(packager->pakfire, payload, a);
		if (r)
			goto ERROR;
	}

	// Success
	r = 0;

ERROR:
	if (payload)
		archive_read_free(payload);

	return r;
}

static int pakfire_packager_write_scriptlet(struct pakfire_packager* packager,
		struct archive* a, struct pakfire_scriptlet* scriptlet) {
	char filename[PATH_MAX];
	size_t size;
	int r;

	// Fetch type
	const char* type = pakfire_scriptlet_get_type(scriptlet);

	DEBUG(packager->pakfire, "Writing scriptlet '%s' to package\n", type);

	// Make filename
	r = pakfire_string_format(filename, ".scriptlets/%s", type);
	if (r)
		return r;

	// Fetch scriptlet
	const char* data = pakfire_scriptlet_get_data(scriptlet, &size);

	// Write file
	return pakfire_packager_write_file_from_buffer(packager, a, filename, 0544, data);
}

/*
	This function is being called at the end when all data has been added to the package.

	It will create a new archive and write the package to the given file descriptor.
*/
int pakfire_packager_finish(struct pakfire_packager* packager, FILE* f) {
	struct archive* a = NULL;
	int r = 1;

	// Close the payload
	if (packager->payload) {
		r = archive_write_free(packager->payload);
		if (r) {
			ERROR(packager->pakfire, "Could not close payload: %s\n",
				archive_error_string(packager->payload));
			goto ERROR;
		}
		packager->payload = NULL;
	}

	// Add requires feature markers
	if (pakfire_package_has_rich_deps(packager->pkg))
		pakfire_package_add_requires(packager->pkg, "pakfire(RichDependencies)");

	// Store total install size
	pakfire_package_set_installsize(packager->pkg, packager->installsize);

	// Dump package metadata
	char* dump = pakfire_package_dump(packager->pkg, PAKFIRE_PKG_DUMP_LONG);
	if (dump) {
		INFO(packager->pakfire, "%s\n", dump);
		free(dump);
	}

	// Open a new archive
	a = archive_write_new();
	if (!a) {
		ERROR(packager->pakfire, "archive_write_new() failed\n");
		goto ERROR;
	}

	// Use the PAX format
	r = archive_write_set_format_pax(a);
	if (r) {
		ERROR(packager->pakfire, "Could not set format to PAX: %s\n",
			archive_error_string(a));
		goto ERROR;
	}

	// Store any extended attributes in the SCHILY headers
	r = archive_write_set_format_option(a, "pax", "xattrheader", "SCHILY");
	if (r) {
		ERROR(packager->pakfire, "Could not set xattrheader option: %s\n",
			archive_error_string(a));
		return r;
	}

	// Enable Zstd
	r = archive_write_add_filter_zstd(a);
	if (r) {
		ERROR(packager->pakfire, "Could not enable Zstandard compression: %s\n",
			archive_error_string(a));
		return r;
	}

	// Set compression level to highest
	r = archive_write_set_filter_option(a, NULL, "compression-level", "22");
	if (r) {
		ERROR(packager->pakfire, "Could not set Zstandard compression level: %s\n",
			archive_error_string(a));
		return r;
	}

	// Add feature marker
	pakfire_package_add_requires(packager->pkg, "pakfire(Compress-Zstandard)");

	// Do not pad the last block
	archive_write_set_bytes_in_last_block(a, 1);

	// Write archive to f
	r = archive_write_open_FILE(a, f);
	if (r) {
		ERROR(packager->pakfire, "archive_write_open_FILE() failed: %s\n",
			archive_error_string(a));
		goto ERROR;
	}

	// Start with the format file
	r = pakfire_packager_write_format(packager, a);
	if (r) {
		ERROR(packager->pakfire, "Could not add format file to archive: %s\n",
			archive_error_string(a));
		goto ERROR;
	}

	// Write the metadata
	r = pakfire_packager_write_metadata(packager, a);
	if (r) {
		ERROR(packager->pakfire, "Could not add metadata file to archive: %s\n",
			archive_error_string(a));
		goto ERROR;
	}

	// Write scriptlets
	for (unsigned int i = 0; i < packager->num_scriptlets; i++) {
		r = pakfire_packager_write_scriptlet(packager, a, packager->scriptlets[i]);
		if (r) {
			ERROR(packager->pakfire, "Could not add scriptlet to the archive: %m\n");
			goto ERROR;
		}
	}

	// Write the payload
	if (packager->files) {
		r = pakfire_packager_copy_archive(packager, a, packager->fpayload);
		if (r) {
			ERROR(packager->pakfire, "Could not add payload to archive: %s\n",
				archive_error_string(a));
			goto ERROR;
		}
	}

	// Success
	r = 0;

ERROR:
	if (a)
		archive_write_free(a);

	return r;
}

int pakfire_packager_finish_to_directory(struct pakfire_packager* packager,
		const char* target, char** result) {
	char path[PATH_MAX];
	char tmppath[PATH_MAX];
	int r = 1;

	// target cannot be empty
	if (!target) {
		errno = EINVAL;
		return 1;
	}

	// Get the filename of the package
	const char* filename = pakfire_packager_filename(packager);
	if (!filename) {
		ERROR(packager->pakfire, "Could not generate filename for package: %m\n");
		r = 1;
		goto ERROR;
	}

	// Make the package path
	r = pakfire_string_format(path, "%s/%s", target, filename);
	if (r)
		goto ERROR;

	// Create the parent directory
	r = pakfire_mkparentdir(path, 0755);
	if (r)
		goto ERROR;

	// Create a temporary file in the target directory
	r = pakfire_string_format(tmppath, "%s.XXXXXX", path);
	if (r)
		goto ERROR;

	// Create a temporary result file
	FILE* f = pakfire_mktemp(tmppath);
	if (!f)
		goto ERROR;

	// Write the finished package
	r = pakfire_packager_finish(packager, f);
	fclose(f);

	if (r) {
		ERROR(packager->pakfire, "pakfire_packager_finish() failed: %m\n");
		goto ERROR;
	}

	// Move the temporary file to destination
	r = rename(tmppath, path);
	if (r) {
		ERROR(packager->pakfire, "Could not move %s to %s: %m\n", tmppath, path);
		goto ERROR;
	}

	INFO(packager->pakfire, "Package written to %s\n", path);

	// Store result path if requested
	if (result) {
		*result = strdup(path);
		if (!*result) {
			r = 1;
			goto ERROR;
		}
	}

	// Success
	r = 0;

ERROR:
	// Remove temporary file
	if (r && *tmppath)
		unlink(tmppath);

	return r;
}

static int pakfire_packager_add_file(struct pakfire_packager* packager,
		struct pakfire_file* file) {
	struct archive_entry* entry = NULL;
	FILE* f = NULL;
	int r;

	// Check input
	if (!file) {
		errno = EINVAL;
		return 1;
	}

	// Payload has already been closed
	if (!packager->payload) {
		ERROR(packager->pakfire, "Payload has already been closed\n");
		errno = EPERM;
		return 1;
	}

	// Fetch path
	const char* path = pakfire_file_get_path(file);

	// Fetch filetype
	const mode_t filetype = pakfire_file_get_type(file);

	// Files cannot have an empty path
	if (!*path) {
		ERROR(packager->pakfire, "Cannot add a file with an empty path\n");
		errno = EPERM;
		return 1;

	// Hidden files cannot be added
	} else if (*path == '.') {
		ERROR(packager->pakfire, "Hidden files cannot be added to a package: %s\n", path);
		errno = EPERM;
		return 1;
	}

	DEBUG(packager->pakfire, "Adding file to payload: %s\n", path);

	// Overwrite a couple of things for source archives
	if (pakfire_package_is_source(packager->pkg)) {
#if 0
		// Reset permissions
		pakfire_file_set_perms(file, 0644);
#endif

		// Reset file ownership
		pakfire_file_set_user(file, "root");
		pakfire_file_set_group(file, "root");
	}

	// Open the file
	f = pakfire_file_open(file);
	if (!f)
		goto ERROR;

#if 0
	// Add digests for regular files
	if (filetype == AE_IFREG) {
		r = pakfire_packager_compute_digests(packager, entry, f);
		if (r) {
			ERROR(packager->pakfire, "Could not compute digests: %m\n")
			goto ERROR;
		}

		// Rewind the file descriptor
		rewind(f);
	}
#endif

	// Generate file metadata into an archive entry
	entry = pakfire_file_archive_entry(file);
	if (!entry)
		goto ERROR;

	// Write the header
	r = archive_write_header(packager->payload, entry);
	if (r) {
		ERROR(packager->pakfire, "Error writing file header: %s\n",
			archive_error_string(packager->payload));
		goto ERROR;
	}

	// Copy the data of regular files
	if (filetype == S_IFREG) {
		// Copy the payload into the archive
		r = pakfire_archive_copy_data_from_file(packager->pakfire, packager->payload, f);
		if (r)
			goto ERROR;
	}

	// Increment installsize
	packager->installsize += pakfire_file_get_size(file);

	// Increment file counter
	packager->files++;

ERROR:
	if (entry)
		archive_entry_free(entry);
	if (f)
		fclose(f);

	return r;
}

int pakfire_packager_add(struct pakfire_packager* packager,
		const char* sourcepath, const char* path) {
	struct pakfire_file* file = NULL;
	int r = 1;

	// Check if path is set
	if (!sourcepath) {
		errno = EINVAL;
		return 1;
	}

	// Use basename if path isn't set
	if (!path) {
		path = strrchr(sourcepath, '/');
		if (path)
			path++;
	}

	// Create a new file entry
	struct archive_entry* entry = archive_entry_new();
	if (!entry)
		return 1;

	// Set the source path
	archive_entry_copy_sourcepath(entry, sourcepath);

	// Set path in archive
	if (path)
		archive_entry_set_pathname(entry, path);

	// Read all attributes from file
	r = archive_read_disk_entry_from_file(packager->reader, entry, -1, NULL);
	if (r) {
		ERROR(packager->pakfire, "Could not read attributes from %s: %m\n", path);
		goto ERROR;
	}

	// Convert to file
	r = pakfire_file_create_from_archive_entry(&file, packager->pakfire, entry);
	if (r)
		goto ERROR;

	// Call the main function
	r = pakfire_packager_add_file(packager, file);

ERROR:
	if (file)
		pakfire_file_unref(file);

	return r;
}

int pakfire_packager_add_scriptlet(struct pakfire_packager* packager,
		struct pakfire_scriptlet* scriptlet) {
	if (!scriptlet) {
		errno = EINVAL;
		return 1;
	}

	// Extend array
	packager->scriptlets = reallocarray(packager->scriptlets,
		packager->num_scriptlets + 1, sizeof(*packager->scriptlets));
	if (!packager->scriptlets)
		return 1;

	// Append scriptlet
	packager->scriptlets[packager->num_scriptlets++] = pakfire_scriptlet_ref(scriptlet);

	return 0;
}