[people/ms/pakfire.git] / src / libpakfire / compress.c

/*#############################################################################
#                                                                             #
# Pakfire - The IPFire package management system                              #
# Copyright (C) 2021 Pakfire development team                                 #
#                                                                             #
# This program is free software: you can redistribute it and/or modify        #
# it under the terms of the GNU General Public License as published by        #
# the Free Software Foundation, either version 3 of the License, or           #
# (at your option) any later version.                                         #
#                                                                             #
# This program is distributed in the hope that it will be useful,             #
# but WITHOUT ANY WARRANTY; without even the implied warranty of              #
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
# GNU General Public License for more details.                                #
#                                                                             #
# You should have received a copy of the GNU General Public License           #
# along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
#                                                                             #
#############################################################################*/

#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include <archive.h>
#include <lzma.h>
#include <zstd.h>

#include <pakfire/compress.h>
#include <pakfire/file.h>
#include <pakfire/filelist.h>
#include <pakfire/logging.h>
#include <pakfire/progressbar.h>
#include <pakfire/util.h>

// Read up to N bytes for analyze the magic
#define MAX_MAGIC_LENGTH	6

// Compression/Decompression buffer size
#define BUFFER_SIZE			64 * 1024

// Settings for XZ
#define XZ_COMPRESSION_LEVEL	6

// Settings for ZSTD
#define ZSTD_COMPRESSION_LEVEL	7

const struct compressor {
	char magic[MAX_MAGIC_LENGTH];
	size_t magic_length;
	FILE* (*open)(FILE* f, const char* mode);
} compressors[] = {
	// XZ
	{ { 0xFD, '7', 'z', 'X', 'Z', 0x00 }, 6, pakfire_xzfopen, },
	// ZSTD
	{ { 0x28, 0xb5, 0x2f, 0xfd }, 4, pakfire_zstdfopen, },
	// End
	{ "", 0, NULL, },
};

// Try to guess the compression
FILE* pakfire_xfopen(FILE* f, const char* mode) {
	char buffer[MAX_MAGIC_LENGTH];

	if (!f) {
		errno = EBADFD;
		return NULL;
	}

	if (!mode) {
		errno = EINVAL;
		return NULL;
	}

	// This only works for reading files
	if (*mode != 'r') {
		errno = ENOTSUP;
		return NULL;
	}

	fpos_t pos;

	// Store the position
	int r = fgetpos(f, &pos);
	if (r < 0)
		return NULL;

	// Read a couple of bytes
	size_t bytes_read = fread(buffer, 1, sizeof(buffer), f);

	// Reset position
	r = fsetpos(f, &pos);
	if (r < 0)
		return NULL;

	// Check if we could read anything
	if (!bytes_read || bytes_read < sizeof(buffer))
		return f;

	// Analyze magic
	for (const struct compressor* c = compressors; c->open; c++) {
		// Check if we have read enough data
		if (bytes_read < c->magic_length)
			continue;

		// Compare the magic value
		r = memcmp(c->magic, buffer, c->magic_length);
		if (r)
			continue;

		// We found a match!
		return c->open(f, mode);
	}

	// Nothing seems to match
	errno = ENOTSUP;
	return f;
}

struct xz_cookie {
	FILE* f;
	char mode;
	lzma_stream stream;
	int done;

	uint8_t buffer[BUFFER_SIZE];
};

static ssize_t xz_read(void* data, char* buffer, size_t size) {
	struct xz_cookie* cookie = (struct xz_cookie*)data;
	if (!cookie)
		return -1;

	// Do not read when mode is "w"
	if (cookie->mode == 'w')
		return -1;

	lzma_action action = LZMA_RUN;

	// Set output to allocated buffer
	cookie->stream.next_out  = (uint8_t *)buffer;
	cookie->stream.avail_out = size;

	while (1) {
		// Read something when the input buffer is empty
		if (cookie->stream.avail_in == 0) {
			cookie->stream.next_in  = cookie->buffer;
			cookie->stream.avail_in = fread(cookie->buffer,
				1, sizeof(cookie->buffer), cookie->f);

			// Break if the input file could not be read
			if (ferror(cookie->f))
				return -1;

			// Finish after we have reached the end of the input file
			if (feof(cookie->f))
				cookie->done = 1;
		}

		lzma_ret ret = lzma_code(&cookie->stream, action);

		// If the stream has ended, we just send the
		// remaining output and mark that we are done.
		if (ret == LZMA_STREAM_END) {
			cookie->done = 1;
			return size - cookie->stream.avail_out;
		}

		// Break on all other unexpected errors
		if (ret != LZMA_OK)
			return -1;

		// When we have read enough to fill the entire output buffer, we return
		if (cookie->stream.avail_out == 0)
			return size;

		if (cookie->done)
			return -1;
	}
}

static ssize_t xz_write(void* data, const char* buffer, size_t size) {
	struct xz_cookie* cookie = (struct xz_cookie*)data;
	if (!cookie)
		return -1;

	// Do not write when mode is "r"
	if (cookie->mode == 'r')
		return -1;

	// Return nothing when there is no input
	if (size == 0)
		return 0;

	// Set input to allocated buffer
	cookie->stream.next_in  = (uint8_t *)buffer;
	cookie->stream.avail_in = size;

	while (1) {
		cookie->stream.next_out  = cookie->buffer;
		cookie->stream.avail_out = sizeof(cookie->buffer);

		lzma_ret ret = lzma_code(&cookie->stream, LZMA_RUN);
		if (ret != LZMA_OK)
			return -1;

		size_t bytes_to_write = sizeof(cookie->buffer) - cookie->stream.avail_out;
		if (bytes_to_write) {
			size_t bytes_written = fwrite(cookie->buffer, 1, bytes_to_write, cookie->f);

			if (bytes_written != bytes_to_write)
				return -1;
		}

		// Report that all data has been written
		if (cookie->stream.avail_in == 0)
			return size;
	}
}

static int xz_close(void* data) {
	struct xz_cookie* cookie = (struct xz_cookie*)data;
	if (!cookie)
		return -1;

	if (cookie->mode == 'w') {
		while (1) {
			cookie->stream.next_out  = cookie->buffer;
			cookie->stream.avail_out = sizeof(cookie->buffer);

			lzma_ret ret = lzma_code(&cookie->stream, LZMA_FINISH);
			if (ret != LZMA_OK && ret != LZMA_STREAM_END)
				return -1;

			size_t bytes_to_write = sizeof(cookie->buffer) - cookie->stream.avail_out;
			if (bytes_to_write) {
				size_t bytes_written = fwrite(cookie->buffer, 1, bytes_to_write, cookie->f);

				if (bytes_written != bytes_to_write)
					return -1;
			}

			if (ret == LZMA_STREAM_END)
				break;
		}
	}

	lzma_end(&cookie->stream);

	// Close input file
	int r = fclose(cookie->f);
	free(cookie);

	return r;
}

static cookie_io_functions_t xz_functions = {
	.read  = xz_read,
	.write = xz_write,
	.seek  = NULL,
	.close = xz_close,
};

FILE* pakfire_xzfopen(FILE* f, const char* mode) {
	lzma_ret ret;

	if (!f) {
		errno = EBADFD;
		return NULL;
	}

	if (!mode) {
		errno = EINVAL;
		return NULL;
	}

	struct xz_cookie* cookie = calloc(1, sizeof(*cookie));
	if (!cookie)
		return NULL;

	cookie->f = f;
	cookie->mode = *mode;

	switch (cookie->mode) {
		case 'r':
			ret = lzma_stream_decoder(&cookie->stream, UINT64_MAX, 0);
			break;

		case 'w':
			ret = lzma_easy_encoder(&cookie->stream, XZ_COMPRESSION_LEVEL, LZMA_CHECK_SHA256);
			break;

		default:
			errno = ENOTSUP;
			goto ERROR;
	}

	if (ret != LZMA_OK)
		goto ERROR;

	return fopencookie(cookie, mode, xz_functions);

ERROR:
	free(cookie);
	return NULL;
}

// ZSTD

struct zstd_cookie {
	FILE* f;
	char mode;
	int done;

	// Encoder
	ZSTD_CStream* cstream;
	ZSTD_inBuffer in;

	// Decoder
	ZSTD_DStream* dstream;
	ZSTD_outBuffer out;

	uint8_t buffer[BUFFER_SIZE];
};

static ssize_t zstd_read(void* data, char* buffer, size_t size) {
	struct zstd_cookie* cookie = (struct zstd_cookie*)data;
	if (!cookie)
		return -1;

	// Do not read when mode is "w"
	if (cookie->mode == 'w')
		return -1;

	if (cookie->done)
		return 0;

	size_t r = 0;

	// Configure output buffer
	cookie->out.dst = buffer;
	cookie->out.pos = 0;
	cookie->out.size = size;

	while (1) {
		if (!feof(cookie->f) && (cookie->in.pos == cookie->in.size)) {
			cookie->in.pos = 0;
			cookie->in.size = fread(cookie->buffer, 1, sizeof(cookie->buffer), cookie->f);
		}

		if (r || cookie->in.size)
			r = ZSTD_decompressStream(cookie->dstream, &cookie->out, &cookie->in);

		if (r == 0 && feof(cookie->f)) {
			cookie->done = 1;
			return cookie->out.pos;
		}

		if (ZSTD_isError(r))
			return -1;

		// Buffer full
		if (cookie->out.pos == size)
			return size;
	}
}

static ssize_t zstd_write(void* data, const char* buffer, size_t size) {
	struct zstd_cookie* cookie = (struct zstd_cookie*)data;
	if (!cookie)
		return -1;

	// Do not write when mode is "r"
	if (cookie->mode == 'r')
		return -1;

	// Return nothing when there is no input
	if (size == 0)
		return 0;

	// Configure input buffer
	cookie->in.src = buffer;
	cookie->in.pos = 0;
	cookie->in.size = size;

	while (1) {
		cookie->out.pos = 0;

		size_t r = ZSTD_compressStream(cookie->cstream, &cookie->out, &cookie->in);
		if (ZSTD_isError(r))
			return -1;

		if (cookie->out.pos > 0) {
			size_t bytes_written = fwrite(cookie->buffer, 1, cookie->out.pos, cookie->f);

			if (bytes_written != cookie->out.pos)
				return -1;
		}

		// Return when all input has been written
		if (cookie->in.pos == size)
			return size;
	}
}

static int zstd_close(void* data) {
	struct zstd_cookie* cookie = (struct zstd_cookie*)data;
	if (!cookie)
		return -1;

	if (cookie->mode == 'w') {
		while (1) {
			// Reset output buffer
			cookie->out.pos = 0;

			size_t r = ZSTD_endStream(cookie->cstream, &cookie->out);
			if (ZSTD_isError(r))
				return -1;

			if (cookie->out.pos > 0) {
				size_t bytes_written = fwrite(cookie->buffer, 1, cookie->out.pos, cookie->f);

				if (bytes_written != cookie->out.pos)
					return -1;
			}

			if (r == 0)
				break;
		}
	}

	int r = fclose(cookie->f);

	// Free everything
	if (cookie->cstream)
		ZSTD_freeCStream(cookie->cstream);
	if (cookie->dstream)
		ZSTD_freeDStream(cookie->dstream);
	free(cookie);

	return r;
}

static cookie_io_functions_t zstd_functions = {
	.read  = zstd_read,
	.write = zstd_write,
	.seek  = NULL,
	.close = zstd_close,
};

FILE* pakfire_zstdfopen(FILE* f, const char* mode) {
	if (!f) {
		errno = EBADFD;
		return NULL;
	}

	if (!mode) {
		errno = EINVAL;
		return NULL;
	}

	struct zstd_cookie* cookie = calloc(1, sizeof(*cookie));
	if (!cookie)
		return NULL;

	cookie->f = f;
	cookie->mode = *mode;

	size_t r;
	switch (cookie->mode) {
		case 'r':
			// Allocate stream
			cookie->dstream = ZSTD_createDStream();
			if (!cookie->dstream)
				goto ERROR;

			// Initialize stream
			r = ZSTD_initDStream(cookie->dstream);
			if (ZSTD_isError(r))
				goto ERROR;

			cookie->in.src = cookie->buffer;
			cookie->in.pos = 0;
			cookie->in.size = 0;
			break;

		case 'w':
			// Allocate stream
			cookie->cstream = ZSTD_createCStream();
			if (!cookie->cstream)
				goto ERROR;

			// Initialize stream
			r = ZSTD_initCStream(cookie->cstream, ZSTD_COMPRESSION_LEVEL);
			if (ZSTD_isError(r))
				goto ERROR;

			cookie->out.dst = cookie->buffer;
			cookie->out.pos = 0;
			cookie->out.size = sizeof(cookie->buffer);
			break;

		default:
			errno = ENOTSUP;
			goto ERROR;
	}

	return fopencookie(cookie, mode, zstd_functions);

ERROR:
	if (cookie->cstream)
		ZSTD_freeCStream(cookie->cstream);
	if (cookie->dstream)
		ZSTD_freeDStream(cookie->dstream);
	free(cookie);

	return NULL;
}

/*
	Helper function to conditionally walk through an archive
	and perform actions based on the callback.
*/
int pakfire_walk(struct pakfire* pakfire, struct archive* archive,
		pakfire_walk_callback callback, pakfire_walk_filter_callback filter_callback,
		void* p) {
	struct archive_entry* entry = NULL;
	int r;

	// Walk through the archive
	for (;;) {
		r = archive_read_next_header(archive, &entry);

		// Handle the return code
		switch (r) {
			// Return OK when we reached the end of the archive
			case ARCHIVE_EOF:
				return 0;

			// Raise any other errors
			default:
				return r;
		}

		// Call the filter callback before we call the actual callback
		if (filter_callback) {
			r = filter_callback(pakfire, archive, entry, p);

			// Handle the return code
			switch (r) {
				case PAKFIRE_WALK_OK:
					break;

				case PAKFIRE_WALK_DONE:
					DEBUG(pakfire, "Filter callback sent DONE\n");
					return 0;

				case PAKFIRE_WALK_SKIP:
					DEBUG(pakfire, "Filter callback sent SKIP\n");
					continue;

				// Raise any other errors
				default:
					DEBUG(pakfire, "Filter callback received an error: %d\n", r);
					return r;
			}
		}

		// Run callback
		if (callback) {
			r = callback(pakfire, archive, entry, p);

			// Handle the return code
			switch (r) {
				case PAKFIRE_WALK_OK:
					break;

				case PAKFIRE_WALK_DONE:
					DEBUG(pakfire, "Callback sent DONE\n");
					return 0;

				// Raise any other errors
				default:
					return r;
			}
		}
	}

	return 0;
}

// Common extraction

struct pakfire_extract {
	// Reference to Pakfire
	struct pakfire* pakfire;

	// Flags
	int flags;

	// The archive to extract
	struct archive* archive;

	// The filelist of all extracted files
	struct pakfire_filelist* filelist;

	// Prepend this prefix
	char prefix[PATH_MAX];

	// The writer
	struct archive* writer;

	// The progressbar
	struct pakfire_progressbar* progressbar;
};

static int pakfire_extract_progressbar_create(struct pakfire_progressbar** progressbar,
		const char* message, int flags) {
	int r;

	// Create the progressbar
	r = pakfire_progressbar_create(progressbar, NULL);
	if (r)
		return r;

	// Add message
	if (message) {
		r = pakfire_progressbar_add_string(*progressbar, "%s", message);
		if (r)
			return r;
	}

	// Add bar
	r = pakfire_progressbar_add_bar(*progressbar);
	if (r)
		return r;

	// Add throughput
	if (flags & PAKFIRE_EXTRACT_SHOW_THROUGHPUT) {
		r = pakfire_progressbar_add_transfer_speed(*progressbar);
		if (r)
			return r;
	}

	// Add percentage
	r = pakfire_progressbar_add_percentage(*progressbar);
	if (r)
		return r;

	// Success
	return 0;
}

static void pakfire_extract_progress(void* p) {
	struct pakfire_extract* data = (struct pakfire_extract*)p;

	// Fetch how many bytes have been read
	const size_t position = archive_filter_bytes(data->archive, -1);

	// Update progressbar
	pakfire_progressbar_update(data->progressbar, position);
}

static int __pakfire_extract(struct pakfire* pakfire, struct archive* a,
		struct archive_entry* entry, void* p) {
	struct pakfire_file* file = NULL;
	char buffer[PATH_MAX];
	int r;

	struct pakfire_extract* data = (struct pakfire_extract*)p;

	// Fetch path
	const char* path = archive_entry_pathname(entry);

	// Prepend the prefix
	if (*data->prefix) {
		// Compose file path
		r = pakfire_path_join(buffer, data->prefix, path);
		if (r) {
			ERROR(pakfire, "Could not compose file path: %m\n");
			return r;
		}

		// Set file path
		archive_entry_set_pathname(entry, buffer);

		// Update hardlink destination
		const char* link = archive_entry_hardlink(entry);
		if (link) {
			r = pakfire_path_join(buffer, data->prefix, link);
			if (r) {
				ERROR(pakfire, "Could not compose hardlink path: %m\n");
				return r;
			}

			// Set hardlink path
			archive_entry_set_hardlink(entry, buffer);
		}
	}

	// Add entry to filelist (if requested)
	if (data->filelist) {
		r = pakfire_file_create_from_archive_entry(&file, pakfire, entry);
		if (r)
			goto ERROR;

		// Append the file to the list
		r = pakfire_filelist_append(data->filelist, file);
		if (r)
			goto ERROR;
	}

	// Create file & extract payload
	if (data->writer) {
		DEBUG(pakfire, "Extracting %s\n", path);

		r = archive_read_extract2(data->archive, entry, data->writer);
		switch (r) {
			case ARCHIVE_OK:
				r = 0;
				break;

			case ARCHIVE_WARN:
				ERROR(pakfire, "%s\n", archive_error_string(data->writer));

				// Pretend everything has been okay
				r = 0;
				break;

			case ARCHIVE_FATAL:
				ERROR(pakfire, "%s\n", archive_error_string(data->writer));
				r = 1;
				break;
		}
	}

ERROR:
	if (file)
		pakfire_file_unref(file);

	return r;
}

int pakfire_extract(struct pakfire* pakfire, struct archive* archive,
		size_t size, struct pakfire_filelist* filelist,
		const char* prefix, const char* message, int flags) {
	int r = 1;

	struct pakfire_extract data = {
		.pakfire  = pakfire,
		.archive  = archive,
		.filelist = filelist,
		.flags    = flags,
		.writer   = NULL,
	};

	// Is this a dry run?
	const int dry_run = flags & PAKFIRE_EXTRACT_DRY_RUN;

	// Should we show a progress bar?
	const int no_progress = flags & PAKFIRE_EXTRACT_NO_PROGRESS;

	// Set prefix (including pakfire path)
	r = pakfire_path(pakfire, data.prefix, "%s", prefix);
	if (r)
		goto ERROR;

	// Allocate writer
	if (!dry_run) {
		data.writer = pakfire_make_archive_disk_writer(pakfire, 1);
		if (!data.writer) {
			ERROR(pakfire, "Could not create disk writer: %m\n");
			r = 1;
			goto ERROR;
		}
	}

	// Create the progressbar
	if (!no_progress) {
		r = pakfire_extract_progressbar_create(&data.progressbar, message, flags);
		if (r)
			goto ERROR;

		// Register progress callback
		archive_read_extract_set_progress_callback(data.archive,
			pakfire_extract_progress, &data);

		// Start progressbar
		pakfire_progressbar_start(data.progressbar, size);
	}

	// Walk through the entire archive
	r = pakfire_walk(pakfire, archive, __pakfire_extract, NULL, &data);
	if (r)
		goto ERROR;

	// Finish the progressbar
	if (data.progressbar)
		pakfire_progressbar_finish(data.progressbar);

ERROR:
	if (data.progressbar)
		pakfire_progressbar_unref(data.progressbar);
	if (data.writer)
		archive_write_free(data.writer);

	return r;
}
Commit	Line	Data
c352b776 MT	1	/*#############################################################################
	2	# #
	3	# Pakfire - The IPFire package management system #
	4	# Copyright (C) 2021 Pakfire development team #
	5	# #
	6	# This program is free software: you can redistribute it and/or modify #
	7	# it under the terms of the GNU General Public License as published by #
	8	# the Free Software Foundation, either version 3 of the License, or #
	9	# (at your option) any later version. #
	10	# #
	11	# This program is distributed in the hope that it will be useful, #
	12	# but WITHOUT ANY WARRANTY; without even the implied warranty of #
	13	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
	14	# GNU General Public License for more details. #
	15	# #
	16	# You should have received a copy of the GNU General Public License #
	17	# along with this program. If not, see <http://www.gnu.org/licenses/>. #
	18	# #
	19	#############################################################################*/
	20
64f977a3	21	#include <errno.h>
c352b776	22	#include <stdio.h>
99018421	23	#include <stdlib.h>
c352b776 MT	24	#include <string.h>
c352b776 MT	25
79824416	26	#include <archive.h>
c352b776	27	#include <lzma.h>
9476d502	28	#include <zstd.h>
c352b776 MT	29
c352b776 MT	30	#include <pakfire/compress.h>
bb7f09cd MT	31	#include <pakfire/file.h>
bb7f09cd MT	32	#include <pakfire/filelist.h>
79824416 MT	33	#include <pakfire/logging.h>
	34	#include <pakfire/progressbar.h>
	35	#include <pakfire/util.h>
c352b776 MT	36
	37	// Read up to N bytes for analyze the magic
	38	#define MAX_MAGIC_LENGTH 6
	39
5cd454df MT	40	// Compression/Decompression buffer size
	41	#define BUFFER_SIZE 64 * 1024
	42
0bed1e1d MT	43	// Settings for XZ
	44	#define XZ_COMPRESSION_LEVEL 6
	45
9476d502 MT	46	// Settings for ZSTD
	47	#define ZSTD_COMPRESSION_LEVEL 7
	48
c352b776 MT	49	const struct compressor {
	50	char magic[MAX_MAGIC_LENGTH];
	51	size_t magic_length;
	52	FILE* (open)(FILE f, const char* mode);
	53	} compressors[] = {
	54	// XZ
	55	{ { 0xFD, '7', 'z', 'X', 'Z', 0x00 }, 6, pakfire_xzfopen, },
60732b6e MT	56	// ZSTD
	57	{ { 0x28, 0xb5, 0x2f, 0xfd }, 4, pakfire_zstdfopen, },
	58	// End
c352b776 MT	59	{ "", 0, NULL, },
	60	};
	61
c352b776 MT	62	// Try to guess the compression
	63	FILE* pakfire_xfopen(FILE* f, const char* mode) {
	64	char buffer[MAX_MAGIC_LENGTH];
	65
64f977a3 MT	66	if (!f) {
64f977a3 MT	67	errno = EBADFD;
c352b776	68	return NULL;
64f977a3 MT	69	}
	70
	71	if (!mode) {
	72	errno = EINVAL;
	73	return NULL;
	74	}
	75
	76	// This only works for reading files
	77	if (*mode != 'r') {
	78	errno = ENOTSUP;
	79	return NULL;
	80	}
c352b776 MT	81
	82	fpos_t pos;
	83
	84	// Store the position
	85	int r = fgetpos(f, &pos);
	86	if (r < 0)
	87	return NULL;
	88
	89	// Read a couple of bytes
	90	size_t bytes_read = fread(buffer, 1, sizeof(buffer), f);
	91
	92	// Reset position
	93	r = fsetpos(f, &pos);
	94	if (r < 0)
	95	return NULL;
	96
	97	// Check if we could read anything
	98	if (!bytes_read \|\| bytes_read < sizeof(buffer))
	99	return f;
	100
	101	// Analyze magic
	102	for (const struct compressor* c = compressors; c->open; c++) {
	103	// Check if we have read enough data
	104	if (bytes_read < c->magic_length)
	105	continue;
	106
	107	// Compare the magic value
	108	r = memcmp(c->magic, buffer, c->magic_length);
	109	if (r)
	110	continue;
	111
	112	// We found a match!
	113	return c->open(f, mode);
	114	}
	115
	116	// Nothing seems to match
49492077	117	errno = ENOTSUP;
c352b776 MT	118	return f;
	119	}
	120
5cd454df MT	121	struct xz_cookie {
	122	FILE* f;
	123	char mode;
	124	lzma_stream stream;
	125	int done;
	126
	127	uint8_t buffer[BUFFER_SIZE];
	128	};
	129
c352b776 MT	130	static ssize_t xz_read(void* data, char* buffer, size_t size) {
	131	struct xz_cookie* cookie = (struct xz_cookie*)data;
	132	if (!cookie)
	133	return -1;
	134
48187ff3 MT	135	// Do not read when mode is "w"
48187ff3 MT	136	if (cookie->mode == 'w')
6923552a	137	return -1;
48187ff3	138
c352b776 MT	139	lzma_action action = LZMA_RUN;
	140
	141	// Set output to allocated buffer
	142	cookie->stream.next_out = (uint8_t *)buffer;
	143	cookie->stream.avail_out = size;
	144
	145	while (1) {
	146	// Read something when the input buffer is empty
	147	if (cookie->stream.avail_in == 0) {
	148	cookie->stream.next_in = cookie->buffer;
	149	cookie->stream.avail_in = fread(cookie->buffer,
5cd454df	150	1, sizeof(cookie->buffer), cookie->f);
c352b776 MT	151
	152	// Break if the input file could not be read
	153	if (ferror(cookie->f))
	154	return -1;
	155
	156	// Finish after we have reached the end of the input file
5e12758c	157	if (feof(cookie->f))
c352b776	158	cookie->done = 1;
c352b776 MT	159	}
	160
	161	lzma_ret ret = lzma_code(&cookie->stream, action);
	162
	163	// If the stream has ended, we just send the
	164	// remaining output and mark that we are done.
	165	if (ret == LZMA_STREAM_END) {
	166	cookie->done = 1;
	167	return size - cookie->stream.avail_out;
	168	}
	169
	170	// Break on all other unexpected errors
	171	if (ret != LZMA_OK)
	172	return -1;
	173
	174	// When we have read enough to fill the entire output buffer, we return
	175	if (cookie->stream.avail_out == 0)
	176	return size;
	177
	178	if (cookie->done)
	179	return -1;
	180	}
	181	}
	182
0bed1e1d MT	183	static ssize_t xz_write(void* data, const char* buffer, size_t size) {
	184	struct xz_cookie* cookie = (struct xz_cookie*)data;
	185	if (!cookie)
	186	return -1;
	187
	188	// Do not write when mode is "r"
	189	if (cookie->mode == 'r')
6923552a	190	return -1;
0bed1e1d MT	191
	192	// Return nothing when there is no input
	193	if (size == 0)
	194	return 0;
	195
	196	// Set input to allocated buffer
9d816df3 MT	197	cookie->stream.next_in = (uint8_t *)buffer;
9d816df3 MT	198	cookie->stream.avail_in = size;
0bed1e1d MT	199
	200	while (1) {
	201	cookie->stream.next_out = cookie->buffer;
	202	cookie->stream.avail_out = sizeof(cookie->buffer);
	203
	204	lzma_ret ret = lzma_code(&cookie->stream, LZMA_RUN);
	205	if (ret != LZMA_OK)
	206	return -1;
	207
	208	size_t bytes_to_write = sizeof(cookie->buffer) - cookie->stream.avail_out;
	209	if (bytes_to_write) {
	210	size_t bytes_written = fwrite(cookie->buffer, 1, bytes_to_write, cookie->f);
	211
	212	if (bytes_written != bytes_to_write)
	213	return -1;
	214	}
	215
	216	// Report that all data has been written
	217	if (cookie->stream.avail_in == 0)
	218	return size;
	219	}
	220	}
	221
c352b776 MT	222	static int xz_close(void* data) {
c352b776 MT	223	struct xz_cookie* cookie = (struct xz_cookie*)data;
99018421 MT	224	if (!cookie)
	225	return -1;
	226
	227	if (cookie->mode == 'w') {
	228	while (1) {
	229	cookie->stream.next_out = cookie->buffer;
	230	cookie->stream.avail_out = sizeof(cookie->buffer);
	231
	232	lzma_ret ret = lzma_code(&cookie->stream, LZMA_FINISH);
	233	if (ret != LZMA_OK && ret != LZMA_STREAM_END)
	234	return -1;
	235
	236	size_t bytes_to_write = sizeof(cookie->buffer) - cookie->stream.avail_out;
	237	if (bytes_to_write) {
	238	size_t bytes_written = fwrite(cookie->buffer, 1, bytes_to_write, cookie->f);
	239
	240	if (bytes_written != bytes_to_write)
	241	return -1;
	242	}
	243
	244	if (ret == LZMA_STREAM_END)
	245	break;
	246	}
	247	}
c352b776	248
c352b776 MT	249	lzma_end(&cookie->stream);
	250
	251	// Close input file
99018421 MT	252	int r = fclose(cookie->f);
99018421 MT	253	free(cookie);
c352b776	254
99018421	255	return r;
c352b776 MT	256	}
c352b776 MT	257
99018421 MT	258	static cookie_io_functions_t xz_functions = {
	259	.read = xz_read,
	260	.write = xz_write,
	261	.seek = NULL,
	262	.close = xz_close,
	263	};
	264
c352b776	265	FILE* pakfire_xzfopen(FILE* f, const char* mode) {
99018421 MT	266	lzma_ret ret;
99018421 MT	267
48187ff3 MT	268	if (!f) {
	269	errno = EBADFD;
	270	return NULL;
	271	}
	272
	273	if (!mode) {
	274	errno = EINVAL;
	275	return NULL;
	276	}
	277
99018421 MT	278	struct xz_cookie* cookie = calloc(1, sizeof(*cookie));
	279	if (!cookie)
	280	return NULL;
c352b776	281
99018421 MT	282	cookie->f = f;
99018421 MT	283	cookie->mode = *mode;
0bed1e1d	284
99018421	285	switch (cookie->mode) {
0bed1e1d	286	case 'r':
99018421	287	ret = lzma_stream_decoder(&cookie->stream, UINT64_MAX, 0);
0bed1e1d MT	288	break;
	289
	290	case 'w':
99018421	291	ret = lzma_easy_encoder(&cookie->stream, XZ_COMPRESSION_LEVEL, LZMA_CHECK_SHA256);
0bed1e1d MT	292	break;
	293
	294	default:
	295	errno = ENOTSUP;
2d4af759	296	goto ERROR;
0bed1e1d MT	297	}
0bed1e1d MT	298
c352b776	299	if (ret != LZMA_OK)
2d4af759	300	goto ERROR;
c352b776	301
99018421	302	return fopencookie(cookie, mode, xz_functions);
2d4af759 MT	303
	304	ERROR:
	305	free(cookie);
	306	return NULL;
c352b776	307	}
9476d502 MT	308
	309	// ZSTD
	310
	311	struct zstd_cookie {
	312	FILE* f;
	313	char mode;
	314	int done;
	315
	316	// Encoder
	317	ZSTD_CStream* cstream;
	318	ZSTD_inBuffer in;
	319
	320	// Decoder
	321	ZSTD_DStream* dstream;
	322	ZSTD_outBuffer out;
	323
	324	uint8_t buffer[BUFFER_SIZE];
	325	};
	326
	327	static ssize_t zstd_read(void* data, char* buffer, size_t size) {
	328	struct zstd_cookie* cookie = (struct zstd_cookie*)data;
	329	if (!cookie)
	330	return -1;
	331
	332	// Do not read when mode is "w"
	333	if (cookie->mode == 'w')
	334	return -1;
	335
	336	if (cookie->done)
	337	return 0;
	338
	339	size_t r = 0;
	340
	341	// Configure output buffer
	342	cookie->out.dst = buffer;
	343	cookie->out.pos = 0;
	344	cookie->out.size = size;
	345
	346	while (1) {
	347	if (!feof(cookie->f) && (cookie->in.pos == cookie->in.size)) {
	348	cookie->in.pos = 0;
	349	cookie->in.size = fread(cookie->buffer, 1, sizeof(cookie->buffer), cookie->f);
	350	}
	351
	352	if (r \|\| cookie->in.size)
	353	r = ZSTD_decompressStream(cookie->dstream, &cookie->out, &cookie->in);
	354
	355	if (r == 0 && feof(cookie->f)) {
	356	cookie->done = 1;
	357	return cookie->out.pos;
	358	}
	359
	360	if (ZSTD_isError(r))
	361	return -1;
	362
	363	// Buffer full
	364	if (cookie->out.pos == size)
	365	return size;
	366	}
	367	}
	368
99a1527a MT	369	static ssize_t zstd_write(void* data, const char* buffer, size_t size) {
	370	struct zstd_cookie* cookie = (struct zstd_cookie*)data;
	371	if (!cookie)
	372	return -1;
	373
	374	// Do not write when mode is "r"
	375	if (cookie->mode == 'r')
	376	return -1;
	377
	378	// Return nothing when there is no input
	379	if (size == 0)
	380	return 0;
	381
	382	// Configure input buffer
	383	cookie->in.src = buffer;
	384	cookie->in.pos = 0;
	385	cookie->in.size = size;
	386
	387	while (1) {
	388	cookie->out.pos = 0;
	389
	390	size_t r = ZSTD_compressStream(cookie->cstream, &cookie->out, &cookie->in);
	391	if (ZSTD_isError(r))
	392	return -1;
	393
	394	if (cookie->out.pos > 0) {
	395	size_t bytes_written = fwrite(cookie->buffer, 1, cookie->out.pos, cookie->f);
	396
	397	if (bytes_written != cookie->out.pos)
	398	return -1;
	399	}
	400
	401	// Return when all input has been written
	402	if (cookie->in.pos == size)
	403	return size;
	404	}
	405	}
	406
9476d502 MT	407	static int zstd_close(void* data) {
	408	struct zstd_cookie* cookie = (struct zstd_cookie*)data;
	409	if (!cookie)
	410	return -1;
	411
99a1527a MT	412	if (cookie->mode == 'w') {
	413	while (1) {
	414	// Reset output buffer
	415	cookie->out.pos = 0;
	416
	417	size_t r = ZSTD_endStream(cookie->cstream, &cookie->out);
	418	if (ZSTD_isError(r))
	419	return -1;
	420
	421	if (cookie->out.pos > 0) {
	422	size_t bytes_written = fwrite(cookie->buffer, 1, cookie->out.pos, cookie->f);
	423
	424	if (bytes_written != cookie->out.pos)
	425	return -1;
	426	}
	427
	428	if (r == 0)
	429	break;
	430	}
	431	}
9476d502 MT	432
	433	int r = fclose(cookie->f);
	434
	435	// Free everything
	436	if (cookie->cstream)
	437	ZSTD_freeCStream(cookie->cstream);
	438	if (cookie->dstream)
	439	ZSTD_freeDStream(cookie->dstream);
	440	free(cookie);
	441
	442	return r;
	443	}
	444
	445	static cookie_io_functions_t zstd_functions = {
	446	.read = zstd_read,
99a1527a	447	.write = zstd_write,
9476d502 MT	448	.seek = NULL,
	449	.close = zstd_close,
	450	};
	451
	452	FILE* pakfire_zstdfopen(FILE* f, const char* mode) {
	453	if (!f) {
	454	errno = EBADFD;
	455	return NULL;
	456	}
	457
	458	if (!mode) {
	459	errno = EINVAL;
	460	return NULL;
	461	}
	462
	463	struct zstd_cookie* cookie = calloc(1, sizeof(*cookie));
	464	if (!cookie)
	465	return NULL;
	466
	467	cookie->f = f;
	468	cookie->mode = *mode;
	469
	470	size_t r;
	471	switch (cookie->mode) {
	472	case 'r':
	473	// Allocate stream
	474	cookie->dstream = ZSTD_createDStream();
	475	if (!cookie->dstream)
	476	goto ERROR;
	477
	478	// Initialize stream
	479	r = ZSTD_initDStream(cookie->dstream);
	480	if (ZSTD_isError(r))
	481	goto ERROR;
	482
	483	cookie->in.src = cookie->buffer;
	484	cookie->in.pos = 0;
	485	cookie->in.size = 0;
	486	break;
	487
	488	case 'w':
	489	// Allocate stream
	490	cookie->cstream = ZSTD_createCStream();
	491	if (!cookie->cstream)
	492	goto ERROR;
	493
	494	// Initialize stream
	495	r = ZSTD_initCStream(cookie->cstream, ZSTD_COMPRESSION_LEVEL);
	496	if (ZSTD_isError(r))
	497	goto ERROR;
	498
	499	cookie->out.dst = cookie->buffer;
	500	cookie->out.pos = 0;
	501	cookie->out.size = sizeof(cookie->buffer);
	502	break;
	503
	504	default:
	505	errno = ENOTSUP;
	506	goto ERROR;
	507	}
	508
	509	return fopencookie(cookie, mode, zstd_functions);
	510
	511	ERROR:
512	if (cookie->cstream)
513	ZSTD_freeCStream(cookie->cstream);
514	if (cookie->dstream)
515	ZSTD_freeDStream(cookie->dstream);
516	free(cookie);
517
518	return NULL;
519	}
79824416	520
d9302cc3 MT	521	/*
	522	Helper function to conditionally walk through an archive
	523	and perform actions based on the callback.
	524	*/
	525	int pakfire_walk(struct pakfire* pakfire, struct archive* archive,
	526	pakfire_walk_callback callback, pakfire_walk_filter_callback filter_callback,
	527	void* p) {
	528	struct archive_entry* entry = NULL;
	529	int r;
	530
	531	// Walk through the archive
	532	for (;;) {
	533	r = archive_read_next_header(archive, &entry);
	534
	535	// Handle the return code
	536	switch (r) {
	537	// Return OK when we reached the end of the archive
	538	case ARCHIVE_EOF:
	539	return 0;
	540
	541	// Raise any other errors
	542	default:
	543	return r;
	544	}
	545
	546	// Call the filter callback before we call the actual callback
	547	if (filter_callback) {
	548	r = filter_callback(pakfire, archive, entry, p);
	549
	550	// Handle the return code
	551	switch (r) {
	552	case PAKFIRE_WALK_OK:
	553	break;
	554
	555	case PAKFIRE_WALK_DONE:
	556	DEBUG(pakfire, "Filter callback sent DONE\n");
	557	return 0;
	558
	559	case PAKFIRE_WALK_SKIP:
	560	DEBUG(pakfire, "Filter callback sent SKIP\n");
	561	continue;
	562
	563	// Raise any other errors
	564	default:
	565	DEBUG(pakfire, "Filter callback received an error: %d\n", r);
	566	return r;
	567	}
	568	}
	569
	570	// Run callback
	571	if (callback) {
	572	r = callback(pakfire, archive, entry, p);
	573
	574	// Handle the return code
	575	switch (r) {
	576	case PAKFIRE_WALK_OK:
	577	break;
	578
	579	case PAKFIRE_WALK_DONE:
	580	DEBUG(pakfire, "Callback sent DONE\n");
	581	return 0;
	582
	583	// Raise any other errors
	584	default:
585	return r;
586	}
587	}
588	}
589
590	return 0;
591	}
592
79824416 MT	593	// Common extraction
	594
	595	struct pakfire_extract {
	596	// Reference to Pakfire
	597	struct pakfire* pakfire;
	598
a3b276d1 MT	599	// Flags
	600	int flags;
	601
79824416 MT	602	// The archive to extract
	603	struct archive* archive;
	604
d05af3ba MT	605	// The filelist of all extracted files
	606	struct pakfire_filelist* filelist;
	607
79824416 MT	608	// Prepend this prefix
	609	char prefix[PATH_MAX];
	610
	611	// The writer
	612	struct archive* writer;
	613
	614	// The progressbar
	615	struct pakfire_progressbar* progressbar;
	616	};
	617
	618	static int pakfire_extract_progressbar_create(struct pakfire_progressbar** progressbar,
	619	const char* message, int flags) {
	620	int r;
	621
	622	// Create the progressbar
	623	r = pakfire_progressbar_create(progressbar, NULL);
	624	if (r)
	625	return r;
	626
	627	// Add message
	628	if (message) {
	629	r = pakfire_progressbar_add_string(*progressbar, "%s", message);
	630	if (r)
	631	return r;
	632	}
	633
	634	// Add bar
	635	r = pakfire_progressbar_add_bar(*progressbar);
	636	if (r)
	637	return r;
	638
95a232a7 MT	639	// Add throughput
	640	if (flags & PAKFIRE_EXTRACT_SHOW_THROUGHPUT) {
	641	r = pakfire_progressbar_add_transfer_speed(*progressbar);
	642	if (r)
	643	return r;
	644	}
	645
79824416 MT	646	// Add percentage
	647	r = pakfire_progressbar_add_percentage(*progressbar);
	648	if (r)
	649	return r;
	650
	651	// Success
	652	return 0;
	653	}
	654
	655	static void pakfire_extract_progress(void* p) {
	656	struct pakfire_extract* data = (struct pakfire_extract*)p;
	657
	658	// Fetch how many bytes have been read
	659	const size_t position = archive_filter_bytes(data->archive, -1);
	660
	661	// Update progressbar
	662	pakfire_progressbar_update(data->progressbar, position);
	663	}
	664
14eedd80 MT	665	static int __pakfire_extract(struct pakfire* pakfire, struct archive* a,
14eedd80 MT	666	struct archive_entry* entry, void* p) {
bb7f09cd	667	struct pakfire_file* file = NULL;
79824416 MT	668	char buffer[PATH_MAX];
	669	int r;
	670
14eedd80 MT	671	struct pakfire_extract* data = (struct pakfire_extract*)p;
14eedd80 MT	672
79824416 MT	673	// Fetch path
	674	const char* path = archive_entry_pathname(entry);
	675
79824416 MT	676	// Prepend the prefix
	677	if (*data->prefix) {
	678	// Compose file path
	679	r = pakfire_path_join(buffer, data->prefix, path);
56796f84	680	if (r) {
79824416 MT	681	ERROR(pakfire, "Could not compose file path: %m\n");
	682	return r;
	683	}
	684
	685	// Set file path
	686	archive_entry_set_pathname(entry, buffer);
	687
	688	// Update hardlink destination
	689	const char* link = archive_entry_hardlink(entry);
	690	if (link) {
	691	r = pakfire_path_join(buffer, data->prefix, link);
56796f84	692	if (r) {
79824416 MT	693	ERROR(pakfire, "Could not compose hardlink path: %m\n");
	694	return r;
	695	}
	696
	697	// Set hardlink path
	698	archive_entry_set_hardlink(entry, buffer);
	699	}
	700	}
	701
bb7f09cd	702	// Add entry to filelist (if requested)
d05af3ba	703	if (data->filelist) {
bb7f09cd MT	704	r = pakfire_file_create_from_archive_entry(&file, pakfire, entry);
	705	if (r)
	706	goto ERROR;
	707
	708	// Append the file to the list
d05af3ba	709	r = pakfire_filelist_append(data->filelist, file);
bb7f09cd MT	710	if (r)
	711	goto ERROR;
	712	}
	713
79824416	714	// Create file & extract payload
a3b276d1	715	if (data->writer) {
cf71c992 MT	716	DEBUG(pakfire, "Extracting %s\n", path);
cf71c992 MT	717
a3b276d1 MT	718	r = archive_read_extract2(data->archive, entry, data->writer);
	719	switch (r) {
	720	case ARCHIVE_OK:
	721	r = 0;
	722	break;
79824416	723
a3b276d1 MT	724	case ARCHIVE_WARN:
a3b276d1 MT	725	ERROR(pakfire, "%s\n", archive_error_string(data->writer));
79824416	726
a3b276d1 MT	727	// Pretend everything has been okay
	728	r = 0;
	729	break;
79824416	730
a3b276d1 MT	731	case ARCHIVE_FATAL:
	732	ERROR(pakfire, "%s\n", archive_error_string(data->writer));
	733	r = 1;
	734	break;
	735	}
79824416 MT	736	}
79824416 MT	737
bb7f09cd MT	738	ERROR:
	739	if (file)
	740	pakfire_file_unref(file);
	741
79824416 MT	742	return r;
	743	}
	744
	745	int pakfire_extract(struct pakfire* pakfire, struct archive* archive,
bb7f09cd MT	746	size_t size, struct pakfire_filelist* filelist,
bb7f09cd MT	747	const char* prefix, const char* message, int flags) {
79824416 MT	748	int r = 1;
	749
	750	struct pakfire_extract data = {
d05af3ba MT	751	.pakfire = pakfire,
	752	.archive = archive,
	753	.filelist = filelist,
	754	.flags = flags,
	755	.writer = NULL,
79824416 MT	756	};
79824416 MT	757
a3b276d1 MT	758	// Is this a dry run?
	759	const int dry_run = flags & PAKFIRE_EXTRACT_DRY_RUN;
	760
5880b6ac MT	761	// Should we show a progress bar?
	762	const int no_progress = flags & PAKFIRE_EXTRACT_NO_PROGRESS;
	763
79824416	764	// Set prefix (including pakfire path)
77e26129 MT	765	r = pakfire_path(pakfire, data.prefix, "%s", prefix);
	766	if (r)
	767	goto ERROR;
79824416 MT	768
79824416 MT	769	// Allocate writer
a3b276d1 MT	770	if (!dry_run) {
	771	data.writer = pakfire_make_archive_disk_writer(pakfire, 1);
	772	if (!data.writer) {
	773	ERROR(pakfire, "Could not create disk writer: %m\n");
	774	r = 1;
	775	goto ERROR;
	776	}
79824416 MT	777	}
	778
	779	// Create the progressbar
5880b6ac MT	780	if (!no_progress) {
	781	r = pakfire_extract_progressbar_create(&data.progressbar, message, flags);
	782	if (r)
	783	goto ERROR;
79824416	784
5880b6ac	785	// Register progress callback
79824416 MT	786	archive_read_extract_set_progress_callback(data.archive,
	787	pakfire_extract_progress, &data);
	788
5880b6ac	789	// Start progressbar
79824416	790	pakfire_progressbar_start(data.progressbar, size);
5880b6ac	791	}
79824416	792
14eedd80 MT	793	// Walk through the entire archive
	794	r = pakfire_walk(pakfire, archive, __pakfire_extract, NULL, &data);
	795	if (r)
	796	goto ERROR;
79824416 MT	797
	798	// Finish the progressbar
	799	if (data.progressbar)
	800	pakfire_progressbar_finish(data.progressbar);
	801
	802	ERROR:
	803	if (data.progressbar)
	804	pakfire_progressbar_unref(data.progressbar);
	805	if (data.writer)
	806	archive_write_free(data.writer);
	807
	808	return r;
	809	}