compress: Show how large a package is

[people/ms/pakfire.git] / src / libpakfire / compress.c

/*#############################################################################
#                                                                             #
# Pakfire - The IPFire package management system                              #
# Copyright (C) 2021 Pakfire development team                                 #
#                                                                             #
# This program is free software: you can redistribute it and/or modify        #
# it under the terms of the GNU General Public License as published by        #
# the Free Software Foundation, either version 3 of the License, or           #
# (at your option) any later version.                                         #
#                                                                             #
# This program is distributed in the hope that it will be useful,             #
# but WITHOUT ANY WARRANTY; without even the implied warranty of              #
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
# GNU General Public License for more details.                                #
#                                                                             #
# You should have received a copy of the GNU General Public License           #
# along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
#                                                                             #
#############################################################################*/

#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include <archive.h>
#include <lzma.h>
#include <zstd.h>

#include <pakfire/compress.h>
#include <pakfire/file.h>
#include <pakfire/filelist.h>
#include <pakfire/logging.h>
#include <pakfire/progressbar.h>
#include <pakfire/util.h>

// Read up to N bytes for analyze the magic
#define MAX_MAGIC_LENGTH	6

// Compression/Decompression buffer size
#define BUFFER_SIZE			64 * 1024

// Settings for XZ
#define XZ_COMPRESSION_LEVEL	6

// Settings for ZSTD
#define ZSTD_COMPRESSION_LEVEL	7

const struct compressor {
	char magic[MAX_MAGIC_LENGTH];
	size_t magic_length;
	FILE* (*open)(FILE* f, const char* mode);
} compressors[] = {
	// XZ
	{ { 0xFD, '7', 'z', 'X', 'Z', 0x00 }, 6, pakfire_xzfopen, },
	// ZSTD
	{ { 0x28, 0xb5, 0x2f, 0xfd }, 4, pakfire_zstdfopen, },
	// End
	{ "", 0, NULL, },
};

// Try to guess the compression
FILE* pakfire_xfopen(FILE* f, const char* mode) {
	char buffer[MAX_MAGIC_LENGTH];

	if (!f) {
		errno = EBADFD;
		return NULL;
	}

	if (!mode) {
		errno = EINVAL;
		return NULL;
	}

	// This only works for reading files
	if (*mode != 'r') {
		errno = ENOTSUP;
		return NULL;
	}

	fpos_t pos;

	// Store the position
	int r = fgetpos(f, &pos);
	if (r < 0)
		return NULL;

	// Read a couple of bytes
	size_t bytes_read = fread(buffer, 1, sizeof(buffer), f);

	// Reset position
	r = fsetpos(f, &pos);
	if (r < 0)
		return NULL;

	// Check if we could read anything
	if (!bytes_read || bytes_read < sizeof(buffer))
		return f;

	// Analyze magic
	for (const struct compressor* c = compressors; c->open; c++) {
		// Check if we have read enough data
		if (bytes_read < c->magic_length)
			continue;

		// Compare the magic value
		r = memcmp(c->magic, buffer, c->magic_length);
		if (r)
			continue;

		// We found a match!
		return c->open(f, mode);
	}

	// Nothing seems to match
	errno = ENOTSUP;
	return f;
}

struct xz_cookie {
	FILE* f;
	char mode;
	lzma_stream stream;
	int done;

	uint8_t buffer[BUFFER_SIZE];
};

static ssize_t xz_read(void* data, char* buffer, size_t size) {
	struct xz_cookie* cookie = (struct xz_cookie*)data;
	if (!cookie)
		return -1;

	// Do not read when mode is "w"
	if (cookie->mode == 'w')
		return -1;

	lzma_action action = LZMA_RUN;

	// Set output to allocated buffer
	cookie->stream.next_out  = (uint8_t *)buffer;
	cookie->stream.avail_out = size;

	while (1) {
		// Read something when the input buffer is empty
		if (cookie->stream.avail_in == 0) {
			cookie->stream.next_in  = cookie->buffer;
			cookie->stream.avail_in = fread(cookie->buffer,
				1, sizeof(cookie->buffer), cookie->f);

			// Break if the input file could not be read
			if (ferror(cookie->f))
				return -1;

			// Finish after we have reached the end of the input file
			if (feof(cookie->f))
				cookie->done = 1;
		}

		lzma_ret ret = lzma_code(&cookie->stream, action);

		// If the stream has ended, we just send the
		// remaining output and mark that we are done.
		if (ret == LZMA_STREAM_END) {
			cookie->done = 1;
			return size - cookie->stream.avail_out;
		}

		// Break on all other unexpected errors
		if (ret != LZMA_OK)
			return -1;

		// When we have read enough to fill the entire output buffer, we return
		if (cookie->stream.avail_out == 0)
			return size;

		if (cookie->done)
			return -1;
	}
}

static ssize_t xz_write(void* data, const char* buffer, size_t size) {
	struct xz_cookie* cookie = (struct xz_cookie*)data;
	if (!cookie)
		return -1;

	// Do not write when mode is "r"
	if (cookie->mode == 'r')
		return -1;

	// Return nothing when there is no input
	if (size == 0)
		return 0;

	// Set input to allocated buffer
	cookie->stream.next_in  = (uint8_t *)buffer;
	cookie->stream.avail_in = size;

	while (1) {
		cookie->stream.next_out  = cookie->buffer;
		cookie->stream.avail_out = sizeof(cookie->buffer);

		lzma_ret ret = lzma_code(&cookie->stream, LZMA_RUN);
		if (ret != LZMA_OK)
			return -1;

		size_t bytes_to_write = sizeof(cookie->buffer) - cookie->stream.avail_out;
		if (bytes_to_write) {
			size_t bytes_written = fwrite(cookie->buffer, 1, bytes_to_write, cookie->f);

			if (bytes_written != bytes_to_write)
				return -1;
		}

		// Report that all data has been written
		if (cookie->stream.avail_in == 0)
			return size;
	}
}

static int xz_close(void* data) {
	struct xz_cookie* cookie = (struct xz_cookie*)data;
	if (!cookie)
		return -1;

	if (cookie->mode == 'w') {
		while (1) {
			cookie->stream.next_out  = cookie->buffer;
			cookie->stream.avail_out = sizeof(cookie->buffer);

			lzma_ret ret = lzma_code(&cookie->stream, LZMA_FINISH);
			if (ret != LZMA_OK && ret != LZMA_STREAM_END)
				return -1;

			size_t bytes_to_write = sizeof(cookie->buffer) - cookie->stream.avail_out;
			if (bytes_to_write) {
				size_t bytes_written = fwrite(cookie->buffer, 1, bytes_to_write, cookie->f);

				if (bytes_written != bytes_to_write)
					return -1;
			}

			if (ret == LZMA_STREAM_END)
				break;
		}
	}

	lzma_end(&cookie->stream);

	// Close input file
	int r = fclose(cookie->f);
	free(cookie);

	return r;
}

static cookie_io_functions_t xz_functions = {
	.read  = xz_read,
	.write = xz_write,
	.seek  = NULL,
	.close = xz_close,
};

FILE* pakfire_xzfopen(FILE* f, const char* mode) {
	lzma_ret ret;

	if (!f) {
		errno = EBADFD;
		return NULL;
	}

	if (!mode) {
		errno = EINVAL;
		return NULL;
	}

	struct xz_cookie* cookie = calloc(1, sizeof(*cookie));
	if (!cookie)
		return NULL;

	cookie->f = f;
	cookie->mode = *mode;

	switch (cookie->mode) {
		case 'r':
			ret = lzma_stream_decoder(&cookie->stream, UINT64_MAX, 0);
			break;

		case 'w':
			ret = lzma_easy_encoder(&cookie->stream, XZ_COMPRESSION_LEVEL, LZMA_CHECK_SHA256);
			break;

		default:
			errno = ENOTSUP;
			goto ERROR;
	}

	if (ret != LZMA_OK)
		goto ERROR;

	return fopencookie(cookie, mode, xz_functions);

ERROR:
	free(cookie);
	return NULL;
}

// ZSTD

struct zstd_cookie {
	FILE* f;
	char mode;
	int done;

	// Encoder
	ZSTD_CStream* cstream;
	ZSTD_inBuffer in;

	// Decoder
	ZSTD_DStream* dstream;
	ZSTD_outBuffer out;

	uint8_t buffer[BUFFER_SIZE];
};

static ssize_t zstd_read(void* data, char* buffer, size_t size) {
	struct zstd_cookie* cookie = (struct zstd_cookie*)data;
	if (!cookie)
		return -1;

	// Do not read when mode is "w"
	if (cookie->mode == 'w')
		return -1;

	if (cookie->done)
		return 0;

	size_t r = 0;

	// Configure output buffer
	cookie->out.dst = buffer;
	cookie->out.pos = 0;
	cookie->out.size = size;

	while (1) {
		if (!feof(cookie->f) && (cookie->in.pos == cookie->in.size)) {
			cookie->in.pos = 0;
			cookie->in.size = fread(cookie->buffer, 1, sizeof(cookie->buffer), cookie->f);
		}

		if (r || cookie->in.size)
			r = ZSTD_decompressStream(cookie->dstream, &cookie->out, &cookie->in);

		if (r == 0 && feof(cookie->f)) {
			cookie->done = 1;
			return cookie->out.pos;
		}

		if (ZSTD_isError(r))
			return -1;

		// Buffer full
		if (cookie->out.pos == size)
			return size;
	}
}

static ssize_t zstd_write(void* data, const char* buffer, size_t size) {
	struct zstd_cookie* cookie = (struct zstd_cookie*)data;
	if (!cookie)
		return -1;

	// Do not write when mode is "r"
	if (cookie->mode == 'r')
		return -1;

	// Return nothing when there is no input
	if (size == 0)
		return 0;

	// Configure input buffer
	cookie->in.src = buffer;
	cookie->in.pos = 0;
	cookie->in.size = size;

	while (1) {
		cookie->out.pos = 0;

		size_t r = ZSTD_compressStream(cookie->cstream, &cookie->out, &cookie->in);
		if (ZSTD_isError(r))
			return -1;

		if (cookie->out.pos > 0) {
			size_t bytes_written = fwrite(cookie->buffer, 1, cookie->out.pos, cookie->f);

			if (bytes_written != cookie->out.pos)
				return -1;
		}

		// Return when all input has been written
		if (cookie->in.pos == size)
			return size;
	}
}

static int zstd_close(void* data) {
	struct zstd_cookie* cookie = (struct zstd_cookie*)data;
	if (!cookie)
		return -1;

	if (cookie->mode == 'w') {
		while (1) {
			// Reset output buffer
			cookie->out.pos = 0;

			size_t r = ZSTD_endStream(cookie->cstream, &cookie->out);
			if (ZSTD_isError(r))
				return -1;

			if (cookie->out.pos > 0) {
				size_t bytes_written = fwrite(cookie->buffer, 1, cookie->out.pos, cookie->f);

				if (bytes_written != cookie->out.pos)
					return -1;
			}

			if (r == 0)
				break;
		}
	}

	int r = fclose(cookie->f);

	// Free everything
	if (cookie->cstream)
		ZSTD_freeCStream(cookie->cstream);
	if (cookie->dstream)
		ZSTD_freeDStream(cookie->dstream);
	free(cookie);

	return r;
}

static cookie_io_functions_t zstd_functions = {
	.read  = zstd_read,
	.write = zstd_write,
	.seek  = NULL,
	.close = zstd_close,
};

FILE* pakfire_zstdfopen(FILE* f, const char* mode) {
	if (!f) {
		errno = EBADFD;
		return NULL;
	}

	if (!mode) {
		errno = EINVAL;
		return NULL;
	}

	struct zstd_cookie* cookie = calloc(1, sizeof(*cookie));
	if (!cookie)
		return NULL;

	cookie->f = f;
	cookie->mode = *mode;

	size_t r;
	switch (cookie->mode) {
		case 'r':
			// Allocate stream
			cookie->dstream = ZSTD_createDStream();
			if (!cookie->dstream)
				goto ERROR;

			// Initialize stream
			r = ZSTD_initDStream(cookie->dstream);
			if (ZSTD_isError(r))
				goto ERROR;

			cookie->in.src = cookie->buffer;
			cookie->in.pos = 0;
			cookie->in.size = 0;
			break;

		case 'w':
			// Allocate stream
			cookie->cstream = ZSTD_createCStream();
			if (!cookie->cstream)
				goto ERROR;

			// Initialize stream
			r = ZSTD_initCStream(cookie->cstream, ZSTD_COMPRESSION_LEVEL);
			if (ZSTD_isError(r))
				goto ERROR;

			cookie->out.dst = cookie->buffer;
			cookie->out.pos = 0;
			cookie->out.size = sizeof(cookie->buffer);
			break;

		default:
			errno = ENOTSUP;
			goto ERROR;
	}

	return fopencookie(cookie, mode, zstd_functions);

ERROR:
	if (cookie->cstream)
		ZSTD_freeCStream(cookie->cstream);
	if (cookie->dstream)
		ZSTD_freeDStream(cookie->dstream);
	free(cookie);

	return NULL;
}

/*
	Helper function to conditionally walk through an archive
	and perform actions based on the callback.
*/
int pakfire_walk(struct pakfire* pakfire, struct archive* archive,
		pakfire_walk_callback callback, pakfire_walk_filter_callback filter_callback,
		void* p) {
	struct archive_entry* entry = NULL;
	const char* path = NULL;
	int r;

	// Walk through the archive
	for (;;) {
		r = archive_read_next_header(archive, &entry);

		// Handle the return code
		switch (r) {
			// Fall through if everything is okay
			case ARCHIVE_OK:
				break;

			// Return OK when we reached the end of the archive
			case ARCHIVE_EOF:
				return 0;

			// Raise any other errors
			default:
				return r;
		}

		path = archive_entry_pathname(entry);

		DEBUG(pakfire, "Walking through %s...\n", path);

		// Call the filter callback before we call the actual callback
		if (filter_callback) {
			r = filter_callback(pakfire, archive, entry, p);

			// Handle the return code
			switch (r) {
				case PAKFIRE_WALK_OK:
					break;

				case PAKFIRE_WALK_DONE:
					DEBUG(pakfire, "Filter callback sent DONE\n");
					return 0;

				case PAKFIRE_WALK_SKIP:
					DEBUG(pakfire, "Filter callback sent SKIP\n");
					continue;

				// Raise any other errors
				default:
					DEBUG(pakfire, "Filter callback returned an error: %d\n", r);
					return r;
			}
		}

		// Run callback
		if (callback) {
			r = callback(pakfire, archive, entry, p);

			// Handle the return code
			switch (r) {
				case PAKFIRE_WALK_OK:
					break;

				case PAKFIRE_WALK_DONE:
					DEBUG(pakfire, "Callback sent DONE\n");
					return 0;

				// Raise any other errors
				default:
					return r;
			}
		}
	}

	return 0;
}

// Common extraction

struct pakfire_extract {
	// Reference to Pakfire
	struct pakfire* pakfire;

	// Flags
	int flags;

	// The archive to extract
	struct archive* archive;

	// The filelist of all extracted files
	struct pakfire_filelist* filelist;

	// Prepend this prefix
	char prefix[PATH_MAX];

	// The writer
	struct archive* writer;

	// The progressbar
	struct pakfire_progressbar* progressbar;
};

static int pakfire_extract_progressbar_create(struct pakfire_progressbar** progressbar,
		const char* message, int flags) {
	int r;

	// Create the progressbar
	r = pakfire_progressbar_create(progressbar, NULL);
	if (r)
		return r;

	// Add message
	if (message) {
		r = pakfire_progressbar_add_string(*progressbar, "%s", message);
		if (r)
			return r;
	}

	// Add bar
	r = pakfire_progressbar_add_bar(*progressbar);
	if (r)
		return r;

	// Add throughput
	if (flags & PAKFIRE_EXTRACT_SHOW_THROUGHPUT) {
		r = pakfire_progressbar_add_transfer_speed(*progressbar);
		if (r)
			return r;
	}

	// Add percentage
	r = pakfire_progressbar_add_percentage(*progressbar);
	if (r)
		return r;

	// Success
	return 0;
}

static void pakfire_extract_progress(void* p) {
	struct pakfire_extract* data = (struct pakfire_extract*)p;

	// Fetch how many bytes have been read
	const size_t position = archive_filter_bytes(data->archive, -1);

	// Update progressbar
	pakfire_progressbar_update(data->progressbar, position);
}

static int __pakfire_extract(struct pakfire* pakfire, struct archive* a,
		struct archive_entry* entry, void* p) {
	struct pakfire_file* file = NULL;
	char buffer[PATH_MAX];
	int r;

	struct pakfire_extract* data = (struct pakfire_extract*)p;

	// Fetch path
	const char* path = archive_entry_pathname(entry);

	// Add entry to filelist (if requested)
	if (data->filelist) {
		r = pakfire_file_create_from_archive_entry(&file, pakfire, entry);
		if (r)
			goto ERROR;

		// Append the file to the list
		r = pakfire_filelist_append(data->filelist, file);
		if (r)
			goto ERROR;
	}

	// Prepend the prefix
	if (*data->prefix) {
		// Compose file path
		r = pakfire_path_join(buffer, data->prefix, path);
		if (r) {
			ERROR(pakfire, "Could not compose file path: %m\n");
			return r;
		}

		// Set file path
		archive_entry_set_pathname(entry, buffer);

		// Update hardlink destination
		const char* link = archive_entry_hardlink(entry);
		if (link) {
			r = pakfire_path_join(buffer, data->prefix, link);
			if (r) {
				ERROR(pakfire, "Could not compose hardlink path: %m\n");
				return r;
			}

			// Set hardlink path
			archive_entry_set_hardlink(entry, buffer);
		}
	}

	// Create file & extract payload
	if (data->writer) {
		// Fetch path again since we changed it
		path = archive_entry_pathname(entry);

		DEBUG(pakfire, "Extracting %s\n", path);

		r = archive_read_extract2(data->archive, entry, data->writer);
		switch (r) {
			case ARCHIVE_OK:
				r = 0;
				break;

			case ARCHIVE_WARN:
				ERROR(pakfire, "%s\n", archive_error_string(data->writer));

				// Pretend everything has been okay
				r = 0;
				break;

			case ARCHIVE_FATAL:
				ERROR(pakfire, "%s\n", archive_error_string(data->writer));
				r = 1;
				break;
		}
	}

ERROR:
	if (file)
		pakfire_file_unref(file);

	return r;
}

int pakfire_extract(struct pakfire* pakfire, struct archive* archive,
		size_t size, struct pakfire_filelist* filelist,
		const char* prefix, const char* message, int flags) {
	int r = 1;

	// Use an empty string if no prefix set
	if (!prefix)
		prefix = "";

	struct pakfire_extract data = {
		.pakfire  = pakfire,
		.archive  = archive,
		.filelist = filelist,
		.flags    = flags,
		.writer   = NULL,
	};

	// Is this a dry run?
	const int dry_run = flags & PAKFIRE_EXTRACT_DRY_RUN;

	// Should we show a progress bar?
	const int no_progress = flags & PAKFIRE_EXTRACT_NO_PROGRESS;

	// Set prefix (including pakfire path)
	r = pakfire_path(pakfire, data.prefix, "%s", prefix);
	if (r)
		goto ERROR;

	// Allocate writer
	if (!dry_run) {
		data.writer = pakfire_make_archive_disk_writer(pakfire, 1);
		if (!data.writer) {
			ERROR(pakfire, "Could not create disk writer: %m\n");
			r = 1;
			goto ERROR;
		}
	}

	// Create the progressbar
	if (!no_progress) {
		r = pakfire_extract_progressbar_create(&data.progressbar, message, flags);
		if (r)
			goto ERROR;

		// Register progress callback
		archive_read_extract_set_progress_callback(data.archive,
			pakfire_extract_progress, &data);

		// Start progressbar
		pakfire_progressbar_start(data.progressbar, size);
	}

	// Walk through the entire archive
	r = pakfire_walk(pakfire, archive, __pakfire_extract, NULL, &data);
	if (r)
		goto ERROR;

	// Finish the progressbar
	if (data.progressbar)
		pakfire_progressbar_finish(data.progressbar);

ERROR:
	if (data.progressbar)
		pakfire_progressbar_unref(data.progressbar);
	if (data.writer)
		archive_write_free(data.writer);

	return r;
}

// Common compression

struct pakfire_compress {
	// Reference to Pakfire
	struct pakfire* pakfire;

	// Flags
	int flags;

	// The archive to write to
	struct archive* archive;

	// The filelist of all files to write
	struct pakfire_filelist* filelist;

	// The progressbar
	struct pakfire_progressbar* progressbar;

	// Digests to write to the archive
	int digests;
};

static int pakfire_compress_progressbar_create(struct pakfire_progressbar** progressbar,
		const char* message, int flags) {
	int r;

	// Create the progressbar
	r = pakfire_progressbar_create(progressbar, NULL);
	if (r)
		return r;

	// Add message
	if (message) {
		r = pakfire_progressbar_add_string(*progressbar, "%s", message);
		if (r)
			return r;
	}

	// Add bar
	r = pakfire_progressbar_add_bar(*progressbar);
	if (r)
		return r;

	// Add throughput
	if (flags & PAKFIRE_COMPRESS_SHOW_THROUGHPUT) {
		r = pakfire_progressbar_add_transfer_speed(*progressbar);
		if (r)
			return r;
	}

	// Show how many bytes have been written
	r = pakfire_progressbar_add_bytes_transferred(*progressbar);
	if (r)
		return r;

	// Add percentage
	r = pakfire_progressbar_add_percentage(*progressbar);
	if (r)
		return r;

	// Success
	return 0;
}

static int __pakfire_compress(struct pakfire* pakfire, struct pakfire_file* file, void* p) {
	struct archive_entry* entry = NULL;
	FILE* f = NULL;
	int r = 1;

	struct pakfire_compress* data = (struct pakfire_compress*)p;

	// Fetch type
	const mode_t type = pakfire_file_get_type(file);

	// Fetch the filelist
	const size_t size = pakfire_file_get_size(file);

	// Generate file metadata into an archive entry
	entry = pakfire_file_archive_entry(file, data->digests);
	if (!entry) {
		r = 1;
		goto ERROR;
	}

	// Write the header
	r = archive_write_header(data->archive, entry);
	if (r) {
		ERROR(pakfire, "Error writing file header: %s\n",
			archive_error_string(data->archive));
		goto ERROR;
	}

	// Copy the data of regular files
	if (type == S_IFREG) {
		// Open the file
		f = pakfire_file_open(file);
		if (!f) {
			r = 1;
			goto ERROR;
		}

		// Copy the payload into the archive
		r = pakfire_archive_copy_data_from_file(pakfire, data->archive, f);
		if (r)
			goto ERROR;
	}

	// Write trailer
	r = archive_write_finish_entry(data->archive);
	if (r)
		goto ERROR;

	// Update the progressbar
	if (data->progressbar)
		pakfire_progressbar_increment(data->progressbar, size);

ERROR:
	if (f)
		fclose(f);
	if (entry)
		archive_entry_free(entry);

	return r;
}

int pakfire_compress(struct pakfire* pakfire, struct archive* archive,
		struct pakfire_filelist* filelist, const char* message, int flags, int digests) {
	int r = 1;

	struct pakfire_compress data = {
		.pakfire  = pakfire,
		.archive  = archive,
		.filelist = filelist,
		.flags    = flags,
		.digests  = digests,
	};

	// Should we show a progress bar?
	const int no_progress = flags & PAKFIRE_COMPRESS_NO_PROGRESS;

	// Fetch the length of the filelist
	const size_t size = pakfire_filelist_total_size(filelist);

	// Create the progressbar
	if (!no_progress) {
		r = pakfire_compress_progressbar_create(&data.progressbar, message, flags);
		if (r)
			goto ERROR;

		// Start progressbar
		pakfire_progressbar_start(data.progressbar, size);
	}

	// Walk through the entire filelist
	r = pakfire_filelist_walk(filelist, __pakfire_compress, &data);
	if (r)
		goto ERROR;

	// Finish the progressbar
	if (data.progressbar)
		pakfire_progressbar_finish(data.progressbar);

ERROR:
	if (data.progressbar)
		pakfire_progressbar_unref(data.progressbar);

	return r;
}