[people/ms/pakfire.git] / src / libpakfire / compress.c

/*#############################################################################
#                                                                             #
# Pakfire - The IPFire package management system                              #
# Copyright (C) 2021 Pakfire development team                                 #
#                                                                             #
# This program is free software: you can redistribute it and/or modify        #
# it under the terms of the GNU General Public License as published by        #
# the Free Software Foundation, either version 3 of the License, or           #
# (at your option) any later version.                                         #
#                                                                             #
# This program is distributed in the hope that it will be useful,             #
# but WITHOUT ANY WARRANTY; without even the implied warranty of              #
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
# GNU General Public License for more details.                                #
#                                                                             #
# You should have received a copy of the GNU General Public License           #
# along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
#                                                                             #
#############################################################################*/

#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include <archive.h>
#include <lzma.h>
#include <zstd.h>

#include <pakfire/compress.h>
#include <pakfire/file.h>
#include <pakfire/filelist.h>
#include <pakfire/logging.h>
#include <pakfire/progressbar.h>
#include <pakfire/util.h>

// Read up to N bytes for analyze the magic
#define MAX_MAGIC_LENGTH	6

// Compression/Decompression buffer size
#define BUFFER_SIZE			64 * 1024

// Settings for XZ
#define XZ_COMPRESSION_LEVEL	6

// Settings for ZSTD
#define ZSTD_COMPRESSION_LEVEL	7

const struct compressor {
	char magic[MAX_MAGIC_LENGTH];
	size_t magic_length;
	FILE* (*open)(FILE* f, const char* mode);
} compressors[] = {
	// XZ
	{ { 0xFD, '7', 'z', 'X', 'Z', 0x00 }, 6, pakfire_xzfopen, },
	// ZSTD
	{ { 0x28, 0xb5, 0x2f, 0xfd }, 4, pakfire_zstdfopen, },
	// End
	{ "", 0, NULL, },
};

// Try to guess the compression
FILE* pakfire_xfopen(FILE* f, const char* mode) {
	char buffer[MAX_MAGIC_LENGTH];

	if (!f) {
		errno = EBADFD;
		return NULL;
	}

	if (!mode) {
		errno = EINVAL;
		return NULL;
	}

	// This only works for reading files
	if (*mode != 'r') {
		errno = ENOTSUP;
		return NULL;
	}

	fpos_t pos;

	// Store the position
	int r = fgetpos(f, &pos);
	if (r < 0)
		return NULL;

	// Read a couple of bytes
	size_t bytes_read = fread(buffer, 1, sizeof(buffer), f);

	// Reset position
	r = fsetpos(f, &pos);
	if (r < 0)
		return NULL;

	// Check if we could read anything
	if (!bytes_read || bytes_read < sizeof(buffer))
		return f;

	// Analyze magic
	for (const struct compressor* c = compressors; c->open; c++) {
		// Check if we have read enough data
		if (bytes_read < c->magic_length)
			continue;

		// Compare the magic value
		r = memcmp(c->magic, buffer, c->magic_length);
		if (r)
			continue;

		// We found a match!
		return c->open(f, mode);
	}

	// Nothing seems to match
	errno = ENOTSUP;
	return f;
}

struct xz_cookie {
	FILE* f;
	char mode;
	lzma_stream stream;
	int done;

	uint8_t buffer[BUFFER_SIZE];
};

static ssize_t xz_read(void* data, char* buffer, size_t size) {
	struct xz_cookie* cookie = (struct xz_cookie*)data;
	if (!cookie)
		return -1;

	// Do not read when mode is "w"
	if (cookie->mode == 'w')
		return -1;

	lzma_action action = LZMA_RUN;

	// Set output to allocated buffer
	cookie->stream.next_out  = (uint8_t *)buffer;
	cookie->stream.avail_out = size;

	while (1) {
		// Read something when the input buffer is empty
		if (cookie->stream.avail_in == 0) {
			cookie->stream.next_in  = cookie->buffer;
			cookie->stream.avail_in = fread(cookie->buffer,
				1, sizeof(cookie->buffer), cookie->f);

			// Break if the input file could not be read
			if (ferror(cookie->f))
				return -1;

			// Finish after we have reached the end of the input file
			if (feof(cookie->f))
				cookie->done = 1;
		}

		lzma_ret ret = lzma_code(&cookie->stream, action);

		// If the stream has ended, we just send the
		// remaining output and mark that we are done.
		if (ret == LZMA_STREAM_END) {
			cookie->done = 1;
			return size - cookie->stream.avail_out;
		}

		// Break on all other unexpected errors
		if (ret != LZMA_OK)
			return -1;

		// When we have read enough to fill the entire output buffer, we return
		if (cookie->stream.avail_out == 0)
			return size;

		if (cookie->done)
			return -1;
	}
}

static ssize_t xz_write(void* data, const char* buffer, size_t size) {
	struct xz_cookie* cookie = (struct xz_cookie*)data;
	if (!cookie)
		return -1;

	// Do not write when mode is "r"
	if (cookie->mode == 'r')
		return -1;

	// Return nothing when there is no input
	if (size == 0)
		return 0;

	// Set input to allocated buffer
	cookie->stream.next_in  = (uint8_t *)buffer;
	cookie->stream.avail_in = size;

	while (1) {
		cookie->stream.next_out  = cookie->buffer;
		cookie->stream.avail_out = sizeof(cookie->buffer);

		lzma_ret ret = lzma_code(&cookie->stream, LZMA_RUN);
		if (ret != LZMA_OK)
			return -1;

		size_t bytes_to_write = sizeof(cookie->buffer) - cookie->stream.avail_out;
		if (bytes_to_write) {
			size_t bytes_written = fwrite(cookie->buffer, 1, bytes_to_write, cookie->f);

			if (bytes_written != bytes_to_write)
				return -1;
		}

		// Report that all data has been written
		if (cookie->stream.avail_in == 0)
			return size;
	}
}

static int xz_close(void* data) {
	struct xz_cookie* cookie = (struct xz_cookie*)data;
	if (!cookie)
		return -1;

	if (cookie->mode == 'w') {
		while (1) {
			cookie->stream.next_out  = cookie->buffer;
			cookie->stream.avail_out = sizeof(cookie->buffer);

			lzma_ret ret = lzma_code(&cookie->stream, LZMA_FINISH);
			if (ret != LZMA_OK && ret != LZMA_STREAM_END)
				return -1;

			size_t bytes_to_write = sizeof(cookie->buffer) - cookie->stream.avail_out;
			if (bytes_to_write) {
				size_t bytes_written = fwrite(cookie->buffer, 1, bytes_to_write, cookie->f);

				if (bytes_written != bytes_to_write)
					return -1;
			}

			if (ret == LZMA_STREAM_END)
				break;
		}
	}

	lzma_end(&cookie->stream);

	// Close input file
	int r = fclose(cookie->f);
	free(cookie);

	return r;
}

static cookie_io_functions_t xz_functions = {
	.read  = xz_read,
	.write = xz_write,
	.seek  = NULL,
	.close = xz_close,
};

FILE* pakfire_xzfopen(FILE* f, const char* mode) {
	lzma_ret ret;

	if (!f) {
		errno = EBADFD;
		return NULL;
	}

	if (!mode) {
		errno = EINVAL;
		return NULL;
	}

	struct xz_cookie* cookie = calloc(1, sizeof(*cookie));
	if (!cookie)
		return NULL;

	cookie->f = f;
	cookie->mode = *mode;

	switch (cookie->mode) {
		case 'r':
			ret = lzma_stream_decoder(&cookie->stream, UINT64_MAX, 0);
			break;

		case 'w':
			ret = lzma_easy_encoder(&cookie->stream, XZ_COMPRESSION_LEVEL, LZMA_CHECK_SHA256);
			break;

		default:
			errno = ENOTSUP;
			goto ERROR;
	}

	if (ret != LZMA_OK)
		goto ERROR;

	return fopencookie(cookie, mode, xz_functions);

ERROR:
	free(cookie);
	return NULL;
}

// ZSTD

struct zstd_cookie {
	FILE* f;
	char mode;
	int done;

	// Encoder
	ZSTD_CStream* cstream;
	ZSTD_inBuffer in;

	// Decoder
	ZSTD_DStream* dstream;
	ZSTD_outBuffer out;

	uint8_t buffer[BUFFER_SIZE];
};

static ssize_t zstd_read(void* data, char* buffer, size_t size) {
	struct zstd_cookie* cookie = (struct zstd_cookie*)data;
	if (!cookie)
		return -1;

	// Do not read when mode is "w"
	if (cookie->mode == 'w')
		return -1;

	if (cookie->done)
		return 0;

	size_t r = 0;

	// Configure output buffer
	cookie->out.dst = buffer;
	cookie->out.pos = 0;
	cookie->out.size = size;

	while (1) {
		if (!feof(cookie->f) && (cookie->in.pos == cookie->in.size)) {
			cookie->in.pos = 0;
			cookie->in.size = fread(cookie->buffer, 1, sizeof(cookie->buffer), cookie->f);
		}

		if (r || cookie->in.size)
			r = ZSTD_decompressStream(cookie->dstream, &cookie->out, &cookie->in);

		if (r == 0 && feof(cookie->f)) {
			cookie->done = 1;
			return cookie->out.pos;
		}

		if (ZSTD_isError(r))
			return -1;

		// Buffer full
		if (cookie->out.pos == size)
			return size;
	}
}

static ssize_t zstd_write(void* data, const char* buffer, size_t size) {
	struct zstd_cookie* cookie = (struct zstd_cookie*)data;
	if (!cookie)
		return -1;

	// Do not write when mode is "r"
	if (cookie->mode == 'r')
		return -1;

	// Return nothing when there is no input
	if (size == 0)
		return 0;

	// Configure input buffer
	cookie->in.src = buffer;
	cookie->in.pos = 0;
	cookie->in.size = size;

	while (1) {
		cookie->out.pos = 0;

		size_t r = ZSTD_compressStream(cookie->cstream, &cookie->out, &cookie->in);
		if (ZSTD_isError(r))
			return -1;

		if (cookie->out.pos > 0) {
			size_t bytes_written = fwrite(cookie->buffer, 1, cookie->out.pos, cookie->f);

			if (bytes_written != cookie->out.pos)
				return -1;
		}

		// Return when all input has been written
		if (cookie->in.pos == size)
			return size;
	}
}

static int zstd_close(void* data) {
	struct zstd_cookie* cookie = (struct zstd_cookie*)data;
	if (!cookie)
		return -1;

	if (cookie->mode == 'w') {
		while (1) {
			// Reset output buffer
			cookie->out.pos = 0;

			size_t r = ZSTD_endStream(cookie->cstream, &cookie->out);
			if (ZSTD_isError(r))
				return -1;

			if (cookie->out.pos > 0) {
				size_t bytes_written = fwrite(cookie->buffer, 1, cookie->out.pos, cookie->f);

				if (bytes_written != cookie->out.pos)
					return -1;
			}

			if (r == 0)
				break;
		}
	}

	int r = fclose(cookie->f);

	// Free everything
	if (cookie->cstream)
		ZSTD_freeCStream(cookie->cstream);
	if (cookie->dstream)
		ZSTD_freeDStream(cookie->dstream);
	free(cookie);

	return r;
}

static cookie_io_functions_t zstd_functions = {
	.read  = zstd_read,
	.write = zstd_write,
	.seek  = NULL,
	.close = zstd_close,
};

FILE* pakfire_zstdfopen(FILE* f, const char* mode) {
	if (!f) {
		errno = EBADFD;
		return NULL;
	}

	if (!mode) {
		errno = EINVAL;
		return NULL;
	}

	struct zstd_cookie* cookie = calloc(1, sizeof(*cookie));
	if (!cookie)
		return NULL;

	cookie->f = f;
	cookie->mode = *mode;

	size_t r;
	switch (cookie->mode) {
		case 'r':
			// Allocate stream
			cookie->dstream = ZSTD_createDStream();
			if (!cookie->dstream)
				goto ERROR;

			// Initialize stream
			r = ZSTD_initDStream(cookie->dstream);
			if (ZSTD_isError(r))
				goto ERROR;

			cookie->in.src = cookie->buffer;
			cookie->in.pos = 0;
			cookie->in.size = 0;
			break;

		case 'w':
			// Allocate stream
			cookie->cstream = ZSTD_createCStream();
			if (!cookie->cstream)
				goto ERROR;

			// Initialize stream
			r = ZSTD_initCStream(cookie->cstream, ZSTD_COMPRESSION_LEVEL);
			if (ZSTD_isError(r))
				goto ERROR;

			cookie->out.dst = cookie->buffer;
			cookie->out.pos = 0;
			cookie->out.size = sizeof(cookie->buffer);
			break;

		default:
			errno = ENOTSUP;
			goto ERROR;
	}

	return fopencookie(cookie, mode, zstd_functions);

ERROR:
	if (cookie->cstream)
		ZSTD_freeCStream(cookie->cstream);
	if (cookie->dstream)
		ZSTD_freeDStream(cookie->dstream);
	free(cookie);

	return NULL;
}

// Common extraction

struct pakfire_extract {
	// Reference to Pakfire
	struct pakfire* pakfire;

	// Flags
	int flags;

	// The archive to extract
	struct archive* archive;

	// Prepend this prefix
	char prefix[PATH_MAX];

	// The writer
	struct archive* writer;

	// The progressbar
	struct pakfire_progressbar* progressbar;
};

static int pakfire_extract_progressbar_create(struct pakfire_progressbar** progressbar,
		const char* message, int flags) {
	int r;

	// Create the progressbar
	r = pakfire_progressbar_create(progressbar, NULL);
	if (r)
		return r;

	// Add message
	if (message) {
		r = pakfire_progressbar_add_string(*progressbar, "%s", message);
		if (r)
			return r;
	}

	// Add bar
	r = pakfire_progressbar_add_bar(*progressbar);
	if (r)
		return r;

	// Add throughput
	if (flags & PAKFIRE_EXTRACT_SHOW_THROUGHPUT) {
		r = pakfire_progressbar_add_transfer_speed(*progressbar);
		if (r)
			return r;
	}

	// Add percentage
	r = pakfire_progressbar_add_percentage(*progressbar);
	if (r)
		return r;

	// Success
	return 0;
}

static void pakfire_extract_progress(void* p) {
	struct pakfire_extract* data = (struct pakfire_extract*)p;

	// Fetch how many bytes have been read
	const size_t position = archive_filter_bytes(data->archive, -1);

	// Update progressbar
	pakfire_progressbar_update(data->progressbar, position);
}

static int __pakfire_extract_entry(struct pakfire* pakfire, struct pakfire_extract* data,
		struct archive_entry* entry, struct pakfire_filelist* filelist) {
	struct pakfire_file* file = NULL;
	char buffer[PATH_MAX];
	int r;

	// Fetch path
	const char* path = archive_entry_pathname(entry);

	DEBUG(pakfire, "Extracting %s\n", path);

	// Prepend the prefix
	if (*data->prefix) {
		// Compose file path
		r = pakfire_path_join(buffer, data->prefix, path);
		if (r) {
			ERROR(pakfire, "Could not compose file path: %m\n");
			return r;
		}

		// Set file path
		archive_entry_set_pathname(entry, buffer);

		// Update hardlink destination
		const char* link = archive_entry_hardlink(entry);
		if (link) {
			r = pakfire_path_join(buffer, data->prefix, link);
			if (r) {
				ERROR(pakfire, "Could not compose hardlink path: %m\n");
				return r;
			}

			// Set hardlink path
			archive_entry_set_hardlink(entry, buffer);
		}
	}

	// Add entry to filelist (if requested)
	if (filelist) {
		r = pakfire_file_create_from_archive_entry(&file, pakfire, entry);
		if (r)
			goto ERROR;

		// Append the file to the list
		r = pakfire_filelist_append(filelist, file);
		if (r)
			goto ERROR;
	}

	// Create file & extract payload
	if (data->writer) {
		r = archive_read_extract2(data->archive, entry, data->writer);
		switch (r) {
			case ARCHIVE_OK:
				r = 0;
				break;

			case ARCHIVE_WARN:
				ERROR(pakfire, "%s\n", archive_error_string(data->writer));

				// Pretend everything has been okay
				r = 0;
				break;

			case ARCHIVE_FATAL:
				ERROR(pakfire, "%s\n", archive_error_string(data->writer));
				r = 1;
				break;
		}
	}

ERROR:
	if (file)
		pakfire_file_unref(file);

	return r;
}

int pakfire_extract(struct pakfire* pakfire, struct archive* archive,
		size_t size, struct pakfire_filelist* filelist,
		const char* prefix, const char* message, int flags) {
	int r = 1;

	struct pakfire_extract data = {
		.pakfire = pakfire,
		.archive = archive,
		.flags   = flags,
		.writer  = NULL,
	};

	// Is this a dry run?
	const int dry_run = flags & PAKFIRE_EXTRACT_DRY_RUN;

	// Set prefix (including pakfire path)
	r = pakfire_path(pakfire, data.prefix, "%s", prefix);
	if (r)
		goto ERROR;

	// Allocate writer
	if (!dry_run) {
		data.writer = pakfire_make_archive_disk_writer(pakfire, 1);
		if (!data.writer) {
			ERROR(pakfire, "Could not create disk writer: %m\n");
			r = 1;
			goto ERROR;
		}
	}

	// Create the progressbar
	r = pakfire_extract_progressbar_create(&data.progressbar, message, flags);
	if (r)
		goto ERROR;

	// Register progress callback
	if (data.progressbar)
		archive_read_extract_set_progress_callback(data.archive,
			pakfire_extract_progress, &data);

	// Start progressbar
	if (data.progressbar)
		pakfire_progressbar_start(data.progressbar, size);

	struct archive_entry* entry = NULL;

	// Walk through the archive
	while (1) {
		r = archive_read_next_header(archive, &entry);

		// End when we have reached the end of the archive
		if (r == ARCHIVE_EOF) {
			r = 0;
			break;
		}

		// Raise any other errors
		else if (r) {
			ERROR(pakfire, "Could not read next header: %s\n",
				archive_error_string(archive));
			goto ERROR;
		}

		// Extract the entry
		r = __pakfire_extract_entry(pakfire, &data, entry, filelist);
		if (r)
			goto ERROR;
	}

	// Finish the progressbar
	if (data.progressbar)
		pakfire_progressbar_finish(data.progressbar);

ERROR:
	if (data.progressbar)
		pakfire_progressbar_unref(data.progressbar);
	if (data.writer)
		archive_write_free(data.writer);

	return r;
}
Commit	Line	Data
c352b776 MT	1	/*#############################################################################
	2	# #
	3	# Pakfire - The IPFire package management system #
	4	# Copyright (C) 2021 Pakfire development team #
	5	# #
	6	# This program is free software: you can redistribute it and/or modify #
	7	# it under the terms of the GNU General Public License as published by #
	8	# the Free Software Foundation, either version 3 of the License, or #
	9	# (at your option) any later version. #
	10	# #
	11	# This program is distributed in the hope that it will be useful, #
	12	# but WITHOUT ANY WARRANTY; without even the implied warranty of #
	13	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
	14	# GNU General Public License for more details. #
	15	# #
	16	# You should have received a copy of the GNU General Public License #
	17	# along with this program. If not, see <http://www.gnu.org/licenses/>. #
	18	# #
	19	#############################################################################*/
	20
64f977a3	21	#include <errno.h>
c352b776	22	#include <stdio.h>
99018421	23	#include <stdlib.h>
c352b776 MT	24	#include <string.h>
c352b776 MT	25
79824416	26	#include <archive.h>
c352b776	27	#include <lzma.h>
9476d502	28	#include <zstd.h>
c352b776 MT	29
c352b776 MT	30	#include <pakfire/compress.h>
bb7f09cd MT	31	#include <pakfire/file.h>
bb7f09cd MT	32	#include <pakfire/filelist.h>
79824416 MT	33	#include <pakfire/logging.h>
	34	#include <pakfire/progressbar.h>
	35	#include <pakfire/util.h>
c352b776 MT	36
	37	// Read up to N bytes for analyze the magic
	38	#define MAX_MAGIC_LENGTH 6
	39
5cd454df MT	40	// Compression/Decompression buffer size
	41	#define BUFFER_SIZE 64 * 1024
	42
0bed1e1d MT	43	// Settings for XZ
	44	#define XZ_COMPRESSION_LEVEL 6
	45
9476d502 MT	46	// Settings for ZSTD
	47	#define ZSTD_COMPRESSION_LEVEL 7
	48
c352b776 MT	49	const struct compressor {
	50	char magic[MAX_MAGIC_LENGTH];
	51	size_t magic_length;
	52	FILE* (open)(FILE f, const char* mode);
	53	} compressors[] = {
	54	// XZ
	55	{ { 0xFD, '7', 'z', 'X', 'Z', 0x00 }, 6, pakfire_xzfopen, },
60732b6e MT	56	// ZSTD
	57	{ { 0x28, 0xb5, 0x2f, 0xfd }, 4, pakfire_zstdfopen, },
	58	// End
c352b776 MT	59	{ "", 0, NULL, },
	60	};
	61
c352b776 MT	62	// Try to guess the compression
	63	FILE* pakfire_xfopen(FILE* f, const char* mode) {
	64	char buffer[MAX_MAGIC_LENGTH];
	65
64f977a3 MT	66	if (!f) {
64f977a3 MT	67	errno = EBADFD;
c352b776	68	return NULL;
64f977a3 MT	69	}
	70
	71	if (!mode) {
	72	errno = EINVAL;
	73	return NULL;
	74	}
	75
	76	// This only works for reading files
	77	if (*mode != 'r') {
	78	errno = ENOTSUP;
	79	return NULL;
	80	}
c352b776 MT	81
	82	fpos_t pos;
	83
	84	// Store the position
	85	int r = fgetpos(f, &pos);
	86	if (r < 0)
	87	return NULL;
	88
	89	// Read a couple of bytes
	90	size_t bytes_read = fread(buffer, 1, sizeof(buffer), f);
	91
	92	// Reset position
	93	r = fsetpos(f, &pos);
	94	if (r < 0)
	95	return NULL;
	96
	97	// Check if we could read anything
	98	if (!bytes_read \|\| bytes_read < sizeof(buffer))
	99	return f;
	100
	101	// Analyze magic
	102	for (const struct compressor* c = compressors; c->open; c++) {
	103	// Check if we have read enough data
	104	if (bytes_read < c->magic_length)
	105	continue;
	106
	107	// Compare the magic value
	108	r = memcmp(c->magic, buffer, c->magic_length);
	109	if (r)
	110	continue;
	111
	112	// We found a match!
	113	return c->open(f, mode);
	114	}
	115
	116	// Nothing seems to match
49492077	117	errno = ENOTSUP;
c352b776 MT	118	return f;
	119	}
	120
5cd454df MT	121	struct xz_cookie {
	122	FILE* f;
	123	char mode;
	124	lzma_stream stream;
	125	int done;
	126
	127	uint8_t buffer[BUFFER_SIZE];
	128	};
	129
c352b776 MT	130	static ssize_t xz_read(void* data, char* buffer, size_t size) {
	131	struct xz_cookie* cookie = (struct xz_cookie*)data;
	132	if (!cookie)
	133	return -1;
	134
48187ff3 MT	135	// Do not read when mode is "w"
48187ff3 MT	136	if (cookie->mode == 'w')
6923552a	137	return -1;
48187ff3	138
c352b776 MT	139	lzma_action action = LZMA_RUN;
	140
	141	// Set output to allocated buffer
	142	cookie->stream.next_out = (uint8_t *)buffer;
	143	cookie->stream.avail_out = size;
	144
	145	while (1) {
	146	// Read something when the input buffer is empty
	147	if (cookie->stream.avail_in == 0) {
	148	cookie->stream.next_in = cookie->buffer;
	149	cookie->stream.avail_in = fread(cookie->buffer,
5cd454df	150	1, sizeof(cookie->buffer), cookie->f);
c352b776 MT	151
	152	// Break if the input file could not be read
	153	if (ferror(cookie->f))
	154	return -1;
	155
	156	// Finish after we have reached the end of the input file
5e12758c	157	if (feof(cookie->f))
c352b776	158	cookie->done = 1;
c352b776 MT	159	}
	160
	161	lzma_ret ret = lzma_code(&cookie->stream, action);
	162
	163	// If the stream has ended, we just send the
	164	// remaining output and mark that we are done.
	165	if (ret == LZMA_STREAM_END) {
	166	cookie->done = 1;
	167	return size - cookie->stream.avail_out;
	168	}
	169
	170	// Break on all other unexpected errors
	171	if (ret != LZMA_OK)
	172	return -1;
	173
	174	// When we have read enough to fill the entire output buffer, we return
	175	if (cookie->stream.avail_out == 0)
	176	return size;
	177
	178	if (cookie->done)
	179	return -1;
	180	}
	181	}
	182
0bed1e1d MT	183	static ssize_t xz_write(void* data, const char* buffer, size_t size) {
	184	struct xz_cookie* cookie = (struct xz_cookie*)data;
	185	if (!cookie)
	186	return -1;
	187
	188	// Do not write when mode is "r"
	189	if (cookie->mode == 'r')
6923552a	190	return -1;
0bed1e1d MT	191
	192	// Return nothing when there is no input
	193	if (size == 0)
	194	return 0;
	195
	196	// Set input to allocated buffer
9d816df3 MT	197	cookie->stream.next_in = (uint8_t *)buffer;
9d816df3 MT	198	cookie->stream.avail_in = size;
0bed1e1d MT	199
	200	while (1) {
	201	cookie->stream.next_out = cookie->buffer;
	202	cookie->stream.avail_out = sizeof(cookie->buffer);
	203
	204	lzma_ret ret = lzma_code(&cookie->stream, LZMA_RUN);
	205	if (ret != LZMA_OK)
	206	return -1;
	207
	208	size_t bytes_to_write = sizeof(cookie->buffer) - cookie->stream.avail_out;
	209	if (bytes_to_write) {
	210	size_t bytes_written = fwrite(cookie->buffer, 1, bytes_to_write, cookie->f);
	211
	212	if (bytes_written != bytes_to_write)
	213	return -1;
	214	}
	215
	216	// Report that all data has been written
	217	if (cookie->stream.avail_in == 0)
	218	return size;
	219	}
	220	}
	221
c352b776 MT	222	static int xz_close(void* data) {
c352b776 MT	223	struct xz_cookie* cookie = (struct xz_cookie*)data;
99018421 MT	224	if (!cookie)
	225	return -1;
	226
	227	if (cookie->mode == 'w') {
	228	while (1) {
	229	cookie->stream.next_out = cookie->buffer;
	230	cookie->stream.avail_out = sizeof(cookie->buffer);
	231
	232	lzma_ret ret = lzma_code(&cookie->stream, LZMA_FINISH);
	233	if (ret != LZMA_OK && ret != LZMA_STREAM_END)
	234	return -1;
	235
	236	size_t bytes_to_write = sizeof(cookie->buffer) - cookie->stream.avail_out;
	237	if (bytes_to_write) {
	238	size_t bytes_written = fwrite(cookie->buffer, 1, bytes_to_write, cookie->f);
	239
	240	if (bytes_written != bytes_to_write)
	241	return -1;
	242	}
	243
	244	if (ret == LZMA_STREAM_END)
	245	break;
	246	}
	247	}
c352b776	248
c352b776 MT	249	lzma_end(&cookie->stream);
	250
	251	// Close input file
99018421 MT	252	int r = fclose(cookie->f);
99018421 MT	253	free(cookie);
c352b776	254
99018421	255	return r;
c352b776 MT	256	}
c352b776 MT	257
99018421 MT	258	static cookie_io_functions_t xz_functions = {
	259	.read = xz_read,
	260	.write = xz_write,
	261	.seek = NULL,
	262	.close = xz_close,
	263	};
	264
c352b776	265	FILE* pakfire_xzfopen(FILE* f, const char* mode) {
99018421 MT	266	lzma_ret ret;
99018421 MT	267
48187ff3 MT	268	if (!f) {
	269	errno = EBADFD;
	270	return NULL;
	271	}
	272
	273	if (!mode) {
	274	errno = EINVAL;
	275	return NULL;
	276	}
	277
99018421 MT	278	struct xz_cookie* cookie = calloc(1, sizeof(*cookie));
	279	if (!cookie)
	280	return NULL;
c352b776	281
99018421 MT	282	cookie->f = f;
99018421 MT	283	cookie->mode = *mode;
0bed1e1d	284
99018421	285	switch (cookie->mode) {
0bed1e1d	286	case 'r':
99018421	287	ret = lzma_stream_decoder(&cookie->stream, UINT64_MAX, 0);
0bed1e1d MT	288	break;
	289
	290	case 'w':
99018421	291	ret = lzma_easy_encoder(&cookie->stream, XZ_COMPRESSION_LEVEL, LZMA_CHECK_SHA256);
0bed1e1d MT	292	break;
	293
	294	default:
	295	errno = ENOTSUP;
2d4af759	296	goto ERROR;
0bed1e1d MT	297	}
0bed1e1d MT	298
c352b776	299	if (ret != LZMA_OK)
2d4af759	300	goto ERROR;
c352b776	301
99018421	302	return fopencookie(cookie, mode, xz_functions);
2d4af759 MT	303
	304	ERROR:
	305	free(cookie);
	306	return NULL;
c352b776	307	}
9476d502 MT	308
	309	// ZSTD
	310
	311	struct zstd_cookie {
	312	FILE* f;
	313	char mode;
	314	int done;
	315
	316	// Encoder
	317	ZSTD_CStream* cstream;
	318	ZSTD_inBuffer in;
	319
	320	// Decoder
	321	ZSTD_DStream* dstream;
	322	ZSTD_outBuffer out;
	323
	324	uint8_t buffer[BUFFER_SIZE];
	325	};
	326
	327	static ssize_t zstd_read(void* data, char* buffer, size_t size) {
	328	struct zstd_cookie* cookie = (struct zstd_cookie*)data;
	329	if (!cookie)
	330	return -1;
	331
	332	// Do not read when mode is "w"
	333	if (cookie->mode == 'w')
	334	return -1;
	335
	336	if (cookie->done)
	337	return 0;
	338
	339	size_t r = 0;
	340
	341	// Configure output buffer
	342	cookie->out.dst = buffer;
	343	cookie->out.pos = 0;
	344	cookie->out.size = size;
	345
	346	while (1) {
	347	if (!feof(cookie->f) && (cookie->in.pos == cookie->in.size)) {
	348	cookie->in.pos = 0;
	349	cookie->in.size = fread(cookie->buffer, 1, sizeof(cookie->buffer), cookie->f);
	350	}
	351
	352	if (r \|\| cookie->in.size)
	353	r = ZSTD_decompressStream(cookie->dstream, &cookie->out, &cookie->in);
	354
	355	if (r == 0 && feof(cookie->f)) {
	356	cookie->done = 1;
	357	return cookie->out.pos;
	358	}
	359
	360	if (ZSTD_isError(r))
	361	return -1;
	362
	363	// Buffer full
	364	if (cookie->out.pos == size)
	365	return size;
	366	}
	367	}
	368
99a1527a MT	369	static ssize_t zstd_write(void* data, const char* buffer, size_t size) {
	370	struct zstd_cookie* cookie = (struct zstd_cookie*)data;
	371	if (!cookie)
	372	return -1;
	373
	374	// Do not write when mode is "r"
	375	if (cookie->mode == 'r')
	376	return -1;
	377
	378	// Return nothing when there is no input
	379	if (size == 0)
	380	return 0;
	381
	382	// Configure input buffer
	383	cookie->in.src = buffer;
	384	cookie->in.pos = 0;
	385	cookie->in.size = size;
	386
	387	while (1) {
	388	cookie->out.pos = 0;
	389
	390	size_t r = ZSTD_compressStream(cookie->cstream, &cookie->out, &cookie->in);
	391	if (ZSTD_isError(r))
	392	return -1;
	393
	394	if (cookie->out.pos > 0) {
	395	size_t bytes_written = fwrite(cookie->buffer, 1, cookie->out.pos, cookie->f);
	396
	397	if (bytes_written != cookie->out.pos)
	398	return -1;
	399	}
	400
	401	// Return when all input has been written
	402	if (cookie->in.pos == size)
	403	return size;
	404	}
	405	}
	406
9476d502 MT	407	static int zstd_close(void* data) {
	408	struct zstd_cookie* cookie = (struct zstd_cookie*)data;
	409	if (!cookie)
	410	return -1;
	411
99a1527a MT	412	if (cookie->mode == 'w') {
	413	while (1) {
	414	// Reset output buffer
	415	cookie->out.pos = 0;
	416
	417	size_t r = ZSTD_endStream(cookie->cstream, &cookie->out);
	418	if (ZSTD_isError(r))
	419	return -1;
	420
	421	if (cookie->out.pos > 0) {
	422	size_t bytes_written = fwrite(cookie->buffer, 1, cookie->out.pos, cookie->f);
	423
	424	if (bytes_written != cookie->out.pos)
	425	return -1;
	426	}
	427
	428	if (r == 0)
	429	break;
	430	}
	431	}
9476d502 MT	432
	433	int r = fclose(cookie->f);
	434
	435	// Free everything
	436	if (cookie->cstream)
	437	ZSTD_freeCStream(cookie->cstream);
	438	if (cookie->dstream)
	439	ZSTD_freeDStream(cookie->dstream);
	440	free(cookie);
	441
	442	return r;
	443	}
	444
	445	static cookie_io_functions_t zstd_functions = {
	446	.read = zstd_read,
99a1527a	447	.write = zstd_write,
9476d502 MT	448	.seek = NULL,
	449	.close = zstd_close,
	450	};
	451
	452	FILE* pakfire_zstdfopen(FILE* f, const char* mode) {
	453	if (!f) {
	454	errno = EBADFD;
	455	return NULL;
	456	}
	457
	458	if (!mode) {
	459	errno = EINVAL;
	460	return NULL;
	461	}
	462
	463	struct zstd_cookie* cookie = calloc(1, sizeof(*cookie));
	464	if (!cookie)
	465	return NULL;
	466
	467	cookie->f = f;
	468	cookie->mode = *mode;
	469
	470	size_t r;
	471	switch (cookie->mode) {
	472	case 'r':
	473	// Allocate stream
	474	cookie->dstream = ZSTD_createDStream();
	475	if (!cookie->dstream)
	476	goto ERROR;
	477
	478	// Initialize stream
	479	r = ZSTD_initDStream(cookie->dstream);
	480	if (ZSTD_isError(r))
	481	goto ERROR;
	482
	483	cookie->in.src = cookie->buffer;
	484	cookie->in.pos = 0;
	485	cookie->in.size = 0;
	486	break;
	487
	488	case 'w':
	489	// Allocate stream
	490	cookie->cstream = ZSTD_createCStream();
	491	if (!cookie->cstream)
	492	goto ERROR;
	493
	494	// Initialize stream
	495	r = ZSTD_initCStream(cookie->cstream, ZSTD_COMPRESSION_LEVEL);
	496	if (ZSTD_isError(r))
	497	goto ERROR;
	498
	499	cookie->out.dst = cookie->buffer;
	500	cookie->out.pos = 0;
	501	cookie->out.size = sizeof(cookie->buffer);
	502	break;
	503
	504	default:
	505	errno = ENOTSUP;
	506	goto ERROR;
	507	}
	508
	509	return fopencookie(cookie, mode, zstd_functions);
	510
	511	ERROR:
512	if (cookie->cstream)
513	ZSTD_freeCStream(cookie->cstream);
514	if (cookie->dstream)
515	ZSTD_freeDStream(cookie->dstream);
516	free(cookie);
517
518	return NULL;
519	}
79824416 MT	520
	521	// Common extraction
	522
	523	struct pakfire_extract {
	524	// Reference to Pakfire
	525	struct pakfire* pakfire;
	526
a3b276d1 MT	527	// Flags
	528	int flags;
	529
79824416 MT	530	// The archive to extract
	531	struct archive* archive;
	532
	533	// Prepend this prefix
	534	char prefix[PATH_MAX];
	535
	536	// The writer
	537	struct archive* writer;
	538
	539	// The progressbar
	540	struct pakfire_progressbar* progressbar;
	541	};
	542
	543	static int pakfire_extract_progressbar_create(struct pakfire_progressbar** progressbar,
	544	const char* message, int flags) {
	545	int r;
	546
	547	// Create the progressbar
	548	r = pakfire_progressbar_create(progressbar, NULL);
	549	if (r)
	550	return r;
	551
	552	// Add message
	553	if (message) {
	554	r = pakfire_progressbar_add_string(*progressbar, "%s", message);
	555	if (r)
	556	return r;
	557	}
	558
	559	// Add bar
	560	r = pakfire_progressbar_add_bar(*progressbar);
	561	if (r)
	562	return r;
	563
95a232a7 MT	564	// Add throughput
	565	if (flags & PAKFIRE_EXTRACT_SHOW_THROUGHPUT) {
	566	r = pakfire_progressbar_add_transfer_speed(*progressbar);
	567	if (r)
	568	return r;
	569	}
	570
79824416 MT	571	// Add percentage
	572	r = pakfire_progressbar_add_percentage(*progressbar);
	573	if (r)
	574	return r;
	575
	576	// Success
	577	return 0;
	578	}
	579
	580	static void pakfire_extract_progress(void* p) {
	581	struct pakfire_extract* data = (struct pakfire_extract*)p;
	582
	583	// Fetch how many bytes have been read
	584	const size_t position = archive_filter_bytes(data->archive, -1);
	585
	586	// Update progressbar
	587	pakfire_progressbar_update(data->progressbar, position);
	588	}
	589
	590	static int __pakfire_extract_entry(struct pakfire* pakfire, struct pakfire_extract* data,
bb7f09cd MT	591	struct archive_entry* entry, struct pakfire_filelist* filelist) {
bb7f09cd MT	592	struct pakfire_file* file = NULL;
79824416 MT	593	char buffer[PATH_MAX];
	594	int r;
	595
	596	// Fetch path
	597	const char* path = archive_entry_pathname(entry);
	598
	599	DEBUG(pakfire, "Extracting %s\n", path);
	600
	601	// Prepend the prefix
	602	if (*data->prefix) {
	603	// Compose file path
	604	r = pakfire_path_join(buffer, data->prefix, path);
56796f84	605	if (r) {
79824416 MT	606	ERROR(pakfire, "Could not compose file path: %m\n");
	607	return r;
	608	}
	609
	610	// Set file path
	611	archive_entry_set_pathname(entry, buffer);
	612
	613	// Update hardlink destination
	614	const char* link = archive_entry_hardlink(entry);
	615	if (link) {
	616	r = pakfire_path_join(buffer, data->prefix, link);
56796f84	617	if (r) {
79824416 MT	618	ERROR(pakfire, "Could not compose hardlink path: %m\n");
	619	return r;
	620	}
	621
	622	// Set hardlink path
	623	archive_entry_set_hardlink(entry, buffer);
	624	}
	625	}
	626
bb7f09cd MT	627	// Add entry to filelist (if requested)
	628	if (filelist) {
	629	r = pakfire_file_create_from_archive_entry(&file, pakfire, entry);
	630	if (r)
	631	goto ERROR;
	632
	633	// Append the file to the list
	634	r = pakfire_filelist_append(filelist, file);
	635	if (r)
	636	goto ERROR;
	637	}
	638
79824416	639	// Create file & extract payload
a3b276d1 MT	640	if (data->writer) {
	641	r = archive_read_extract2(data->archive, entry, data->writer);
	642	switch (r) {
	643	case ARCHIVE_OK:
	644	r = 0;
	645	break;
79824416	646
a3b276d1 MT	647	case ARCHIVE_WARN:
a3b276d1 MT	648	ERROR(pakfire, "%s\n", archive_error_string(data->writer));
79824416	649
a3b276d1 MT	650	// Pretend everything has been okay
	651	r = 0;
	652	break;
79824416	653
a3b276d1 MT	654	case ARCHIVE_FATAL:
	655	ERROR(pakfire, "%s\n", archive_error_string(data->writer));
	656	r = 1;
	657	break;
	658	}
79824416 MT	659	}
79824416 MT	660
bb7f09cd MT	661	ERROR:
	662	if (file)
	663	pakfire_file_unref(file);
	664
79824416 MT	665	return r;
	666	}
	667
	668	int pakfire_extract(struct pakfire* pakfire, struct archive* archive,
bb7f09cd MT	669	size_t size, struct pakfire_filelist* filelist,
bb7f09cd MT	670	const char* prefix, const char* message, int flags) {
79824416 MT	671	int r = 1;
	672
	673	struct pakfire_extract data = {
	674	.pakfire = pakfire,
	675	.archive = archive,
a3b276d1 MT	676	.flags = flags,
a3b276d1 MT	677	.writer = NULL,
79824416 MT	678	};
79824416 MT	679
a3b276d1 MT	680	// Is this a dry run?
	681	const int dry_run = flags & PAKFIRE_EXTRACT_DRY_RUN;
	682
79824416	683	// Set prefix (including pakfire path)
77e26129 MT	684	r = pakfire_path(pakfire, data.prefix, "%s", prefix);
	685	if (r)
	686	goto ERROR;
79824416 MT	687
79824416 MT	688	// Allocate writer
a3b276d1 MT	689	if (!dry_run) {
	690	data.writer = pakfire_make_archive_disk_writer(pakfire, 1);
	691	if (!data.writer) {
	692	ERROR(pakfire, "Could not create disk writer: %m\n");
	693	r = 1;
	694	goto ERROR;
	695	}
79824416 MT	696	}
	697
	698	// Create the progressbar
	699	r = pakfire_extract_progressbar_create(&data.progressbar, message, flags);
	700	if (r)
	701	goto ERROR;
	702
	703	// Register progress callback
	704	if (data.progressbar)
	705	archive_read_extract_set_progress_callback(data.archive,
	706	pakfire_extract_progress, &data);
	707
	708	// Start progressbar
	709	if (data.progressbar)
	710	pakfire_progressbar_start(data.progressbar, size);
	711
	712	struct archive_entry* entry = NULL;
	713
	714	// Walk through the archive
	715	while (1) {
	716	r = archive_read_next_header(archive, &entry);
	717
	718	// End when we have reached the end of the archive
	719	if (r == ARCHIVE_EOF) {
	720	r = 0;
	721	break;
	722	}
	723
	724	// Raise any other errors
	725	else if (r) {
	726	ERROR(pakfire, "Could not read next header: %s\n",
	727	archive_error_string(archive));
	728	goto ERROR;
	729	}
	730
	731	// Extract the entry
bb7f09cd	732	r = __pakfire_extract_entry(pakfire, &data, entry, filelist);
79824416 MT	733	if (r)
	734	goto ERROR;
	735	}
	736
	737	// Finish the progressbar
	738	if (data.progressbar)
	739	pakfire_progressbar_finish(data.progressbar);
	740
	741	ERROR:
	742	if (data.progressbar)
	743	pakfire_progressbar_unref(data.progressbar);
	744	if (data.writer)
	745	archive_write_free(data.writer);
	746
	747	return r;
	748	}