[thirdparty/libarchive.git] / libarchive / archive_write_add_filter_zstd.c

/*-
 * Copyright (c) 2017 Sean Purcell
 * Copyright (c) 2023-2024 Klara, Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "archive_platform.h"

#ifdef HAVE_ERRNO_H
#include <errno.h>
#endif
#ifdef HAVE_STDINT_H
#include <stdint.h>
#endif
#ifdef HAVE_STDLIB_H
#include <stdlib.h>
#endif
#ifdef HAVE_STRING_H
#include <string.h>
#endif
#ifdef HAVE_ZSTD_H
#include <zstd.h>
#endif

#include "archive.h"
#include "archive_private.h"
#include "archive_string.h"
#include "archive_write_private.h"

/* Don't compile this if we don't have zstd.h */

struct private_data {
	int		 compression_level;
	int		 threads;
	int		 long_distance;
#if HAVE_ZSTD_H && HAVE_ZSTD_compressStream
	enum {
		running,
		finishing,
		resetting,
	} state;
	int		 frame_per_file;
	size_t		 min_frame_in;
	size_t		 max_frame_in;
	size_t		 min_frame_out;
	size_t		 max_frame_out;
	size_t		 cur_frame;
	size_t		 cur_frame_in;
	size_t		 cur_frame_out;
	size_t		 total_in;
	ZSTD_CStream	*cstream;
	ZSTD_outBuffer	 out;
#else
	struct archive_write_program_data *pdata;
#endif
};

/* If we don't have the library use default range values (zstdcli.c v1.4.0) */
#define CLEVEL_MIN -99
#define CLEVEL_STD_MIN 0 /* prior to 1.3.4 and more recent without using --fast */
#define CLEVEL_DEFAULT 3
#define CLEVEL_STD_MAX 19 /* without using --ultra */
#define CLEVEL_MAX 22

#define LONG_STD 27

#define MINVER_NEGCLEVEL 10304
#define MINVER_MINCLEVEL 10306
#define MINVER_LONG 10302

static int archive_compressor_zstd_options(struct archive_write_filter *,
		    const char *, const char *);
static int archive_compressor_zstd_open(struct archive_write_filter *);
static int archive_compressor_zstd_write(struct archive_write_filter *,
		    const void *, size_t);
static int archive_compressor_zstd_flush(struct archive_write_filter *);
static int archive_compressor_zstd_close(struct archive_write_filter *);
static int archive_compressor_zstd_free(struct archive_write_filter *);
#if HAVE_ZSTD_H && HAVE_ZSTD_compressStream
static int drive_compressor(struct archive_write_filter *,
		    struct private_data *, int, const void *, size_t);
#endif


/*
 * Add a zstd compression filter to this write handle.
 */
int
archive_write_add_filter_zstd(struct archive *_a)
{
	struct archive_write *a = (struct archive_write *)_a;
	struct archive_write_filter *f = __archive_write_allocate_filter(_a);
	struct private_data *data;
	archive_check_magic(&a->archive, ARCHIVE_WRITE_MAGIC,
	    ARCHIVE_STATE_NEW, "archive_write_add_filter_zstd");

	data = calloc(1, sizeof(*data));
	if (data == NULL) {
		archive_set_error(&a->archive, ENOMEM, "Out of memory");
		return (ARCHIVE_FATAL);
	}
	f->data = data;
	f->open = &archive_compressor_zstd_open;
	f->options = &archive_compressor_zstd_options;
	f->flush = &archive_compressor_zstd_flush;
	f->close = &archive_compressor_zstd_close;
	f->free = &archive_compressor_zstd_free;
	f->code = ARCHIVE_FILTER_ZSTD;
	f->name = "zstd";
	data->compression_level = CLEVEL_DEFAULT;
	data->threads = 0;
	data->long_distance = 0;
#if HAVE_ZSTD_H && HAVE_ZSTD_compressStream
	data->frame_per_file = 0;
	data->min_frame_in = 0;
	data->max_frame_in = SIZE_MAX;
	data->min_frame_out = 0;
	data->max_frame_out = SIZE_MAX;
	data->cur_frame_in = 0;
	data->cur_frame_out = 0;
	data->cstream = ZSTD_createCStream();
	if (data->cstream == NULL) {
		free(data);
		archive_set_error(&a->archive, ENOMEM,
		    "Failed to allocate zstd compressor object");
		return (ARCHIVE_FATAL);
	}

	return (ARCHIVE_OK);
#else
	data->pdata = __archive_write_program_allocate("zstd");
	if (data->pdata == NULL) {
		free(data);
		archive_set_error(&a->archive, ENOMEM, "Out of memory");
		return (ARCHIVE_FATAL);
	}
	archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC,
	    "Using external zstd program");
	return (ARCHIVE_WARN);
#endif
}

static int
archive_compressor_zstd_free(struct archive_write_filter *f)
{
	struct private_data *data = (struct private_data *)f->data;
#if HAVE_ZSTD_H && HAVE_ZSTD_compressStream
	ZSTD_freeCStream(data->cstream);
	free(data->out.dst);
#else
	__archive_write_program_free(data->pdata);
#endif
	free(data);
	f->data = NULL;
	return (ARCHIVE_OK);
}

static int
string_to_number(const char *string, intmax_t *numberp)
{
	char *end;

	if (string == NULL || *string == '\0')
		return (ARCHIVE_WARN);
	*numberp = strtoimax(string, &end, 10);
	if (end == string || *end != '\0' || errno == EOVERFLOW) {
		*numberp = 0;
		return (ARCHIVE_WARN);
	}
	return (ARCHIVE_OK);
}

static int
string_to_size(const char *string, size_t *numberp)
{
	uintmax_t number;
	char *end;
	unsigned int shift = 0;

	if (string == NULL || *string == '\0' || *string == '-')
		return (ARCHIVE_WARN);
	number = strtoumax(string, &end, 10);
	if (end > string) {
		if (*end == 'K' || *end == 'k') {
			shift = 10;
			end++;
		} else if (*end == 'M' || *end == 'm') {
			shift = 20;
			end++;
		} else if (*end == 'G' || *end == 'g') {
			shift = 30;
			end++;
		}
		if (*end == 'B' || *end == 'b') {
			end++;
		}
	}
	if (end == string || *end != '\0' || errno == EOVERFLOW) {
		return (ARCHIVE_WARN);
	}
	if (number > (uintmax_t)SIZE_MAX >> shift) {
		return (ARCHIVE_WARN);
	}
	*numberp = (size_t)(number << shift);
	return (ARCHIVE_OK);
}

/*
 * Set write options.
 */
static int
archive_compressor_zstd_options(struct archive_write_filter *f, const char *key,
    const char *value)
{
	struct private_data *data = (struct private_data *)f->data;

	if (strcmp(key, "compression-level") == 0) {
		intmax_t level;
		if (string_to_number(value, &level) != ARCHIVE_OK) {
			return (ARCHIVE_WARN);
		}
		/* If we don't have the library, hard-code the max level */
		int minimum = CLEVEL_MIN;
		int maximum = CLEVEL_MAX;
#if HAVE_ZSTD_H && HAVE_ZSTD_compressStream
		maximum = ZSTD_maxCLevel();
#if ZSTD_VERSION_NUMBER >= MINVER_MINCLEVEL
		if (ZSTD_versionNumber() >= MINVER_MINCLEVEL) {
			minimum = ZSTD_minCLevel();
		}
		else
#endif
		if (ZSTD_versionNumber() < MINVER_NEGCLEVEL) {
			minimum = CLEVEL_STD_MIN;
		}
#endif
		if (level < minimum || level > maximum) {
			return (ARCHIVE_WARN);
		}
		data->compression_level = (int)level;
		return (ARCHIVE_OK);
	} else if (strcmp(key, "threads") == 0) {
		intmax_t threads;
		if (string_to_number(value, &threads) != ARCHIVE_OK) {
			return (ARCHIVE_WARN);
		}
		if (threads < 0) {
			return (ARCHIVE_WARN);
		}
		data->threads = (int)threads;
		return (ARCHIVE_OK);
#if HAVE_ZSTD_H && HAVE_ZSTD_compressStream
	} else if (strcmp(key, "frame-per-file") == 0) {
		data->frame_per_file = 1;
		return (ARCHIVE_OK);
	} else if (strcmp(key, "min-frame-in") == 0) {
		if (string_to_size(value, &data->min_frame_in) != ARCHIVE_OK) {
			return (ARCHIVE_WARN);
		}
		return (ARCHIVE_OK);
	} else if (strcmp(key, "min-frame-out") == 0 ||
	    strcmp(key, "min-frame-size") == 0) {
		if (string_to_size(value, &data->min_frame_out) != ARCHIVE_OK) {
			return (ARCHIVE_WARN);
		}
		return (ARCHIVE_OK);
	} else if (strcmp(key, "max-frame-in") == 0 ||
	    strcmp(key, "max-frame-size") == 0) {
		if (string_to_size(value, &data->max_frame_in) != ARCHIVE_OK ||
		    data->max_frame_in < 1024) {
			return (ARCHIVE_WARN);
		}
		return (ARCHIVE_OK);
	} else if (strcmp(key, "max-frame-out") == 0) {
		if (string_to_size(value, &data->max_frame_out) != ARCHIVE_OK ||
		    data->max_frame_out < 1024) {
			return (ARCHIVE_WARN);
		}
		return (ARCHIVE_OK);
#endif
	}
	else if (strcmp(key, "long") == 0) {
		intmax_t long_distance;
		if (string_to_number(value, &long_distance) != ARCHIVE_OK) {
			return (ARCHIVE_WARN);
		}
#if HAVE_ZSTD_H && HAVE_ZSTD_compressStream && ZSTD_VERSION_NUMBER >= MINVER_LONG
		ZSTD_bounds bounds = ZSTD_cParam_getBounds(ZSTD_c_windowLog);
		if (ZSTD_isError(bounds.error)) {
			int max_distance = ((int)(sizeof(size_t) == 4 ? 30 : 31));
			if (((int)long_distance) < 10 || (int)long_distance > max_distance)
				return (ARCHIVE_WARN);
		} else {
			if ((int)long_distance < bounds.lowerBound || (int)long_distance > bounds.upperBound)
				return (ARCHIVE_WARN);
		}
#else
		int max_distance = ((int)(sizeof(size_t) == 4 ? 30 : 31));
		if (((int)long_distance) < 10 || (int)long_distance > max_distance)
		    return (ARCHIVE_WARN);
#endif
		data->long_distance = (int)long_distance;
		return (ARCHIVE_OK);
	}

	/* Note: The "warn" return is just to inform the options
	 * supervisor that we didn't handle it.  It will generate
	 * a suitable error if no one used this option. */
	return (ARCHIVE_WARN);
}

#if HAVE_ZSTD_H && HAVE_ZSTD_compressStream
/*
 * Setup callback.
 */
static int
archive_compressor_zstd_open(struct archive_write_filter *f)
{
	struct private_data *data = (struct private_data *)f->data;

	if (data->out.dst == NULL) {
		size_t bs = ZSTD_CStreamOutSize(), bpb;
		if (f->archive->magic == ARCHIVE_WRITE_MAGIC) {
			/* Buffer size should be a multiple number of
			 * the of bytes per block for performance. */
			bpb = archive_write_get_bytes_per_block(f->archive);
			if (bpb > bs)
				bs = bpb;
			else if (bpb != 0)
				bs -= bs % bpb;
		}
		data->out.size = bs;
		data->out.pos = 0;
		data->out.dst
		    = (unsigned char *)malloc(data->out.size);
		if (data->out.dst == NULL) {
			archive_set_error(f->archive, ENOMEM,
			    "Can't allocate data for compression buffer");
			return (ARCHIVE_FATAL);
		}
	}

	f->write = archive_compressor_zstd_write;

	if (ZSTD_isError(ZSTD_initCStream(data->cstream,
	    data->compression_level))) {
		archive_set_error(f->archive, ARCHIVE_ERRNO_MISC,
		    "Internal error initializing zstd compressor object");
		return (ARCHIVE_FATAL);
	}

	ZSTD_CCtx_setParameter(data->cstream, ZSTD_c_nbWorkers, data->threads);

#if ZSTD_VERSION_NUMBER >= MINVER_LONG
	ZSTD_CCtx_setParameter(data->cstream, ZSTD_c_windowLog, data->long_distance);
#endif

	return (ARCHIVE_OK);
}

/*
 * Write data to the compressed stream.
 */
static int
archive_compressor_zstd_write(struct archive_write_filter *f, const void *buff,
    size_t length)
{
	struct private_data *data = (struct private_data *)f->data;

	return (drive_compressor(f, data, 0, buff, length));
}

/*
 * Flush the compressed stream.
 */
static int
archive_compressor_zstd_flush(struct archive_write_filter *f)
{
	struct private_data *data = (struct private_data *)f->data;

	if (data->frame_per_file && data->state == running) {
		if (data->cur_frame_in > data->min_frame_in &&
		    data->cur_frame_out > data->min_frame_out) {
			data->state = finishing;
		}
	}
	return (drive_compressor(f, data, 1, NULL, 0));
}

/*
 * Finish the compression...
 */
static int
archive_compressor_zstd_close(struct archive_write_filter *f)
{
	struct private_data *data = (struct private_data *)f->data;

	if (data->state == running)
		data->state = finishing;
	return (drive_compressor(f, data, 1, NULL, 0));
}

/*
 * Utility function to push input data through compressor,
 * writing full output blocks as necessary.
 */
static int
drive_compressor(struct archive_write_filter *f,
    struct private_data *data, int flush, const void *src, size_t length)
{
	ZSTD_inBuffer in = { .src = src, .size = length, .pos = 0 };
	size_t ipos, opos, zstdret = 0;
	int ret;

	for (;;) {
		ipos = in.pos;
		opos = data->out.pos;
		switch (data->state) {
		case running:
			if (in.pos == in.size)
				return (ARCHIVE_OK);
			zstdret = ZSTD_compressStream(data->cstream,
			    &data->out, &in);
			if (ZSTD_isError(zstdret))
				goto zstd_fatal;
			break;
		case finishing:
			zstdret = ZSTD_endStream(data->cstream, &data->out);
			if (ZSTD_isError(zstdret))
				goto zstd_fatal;
			if (zstdret == 0)
				data->state = resetting;
			break;
		case resetting:
			ZSTD_CCtx_reset(data->cstream, ZSTD_reset_session_only);
			data->cur_frame++;
			data->cur_frame_in = 0;
			data->cur_frame_out = 0;
			data->state = running;
			break;
		}
		data->total_in += in.pos - ipos;
		data->cur_frame_in += in.pos - ipos;
		data->cur_frame_out += data->out.pos - opos;
		if (data->state == running) {
			if (data->cur_frame_in >= data->max_frame_in ||
			    data->cur_frame_out >= data->max_frame_out) {
				data->state = finishing;
			}
		}
		if (data->out.pos == data->out.size ||
		    (flush && data->out.pos > 0)) {
			ret = __archive_write_filter(f->next_filter,
			    data->out.dst, data->out.pos);
			if (ret != ARCHIVE_OK)
				goto fatal;
			data->out.pos = 0;
		}
	}
zstd_fatal:
	archive_set_error(f->archive, ARCHIVE_ERRNO_MISC,
	    "Zstd compression failed: %s",
	    ZSTD_getErrorName(zstdret));
fatal:
	return (ARCHIVE_FATAL);
}

#else /* HAVE_ZSTD_H && HAVE_ZSTD_compressStream */

static int
archive_compressor_zstd_open(struct archive_write_filter *f)
{
	struct private_data *data = (struct private_data *)f->data;
	struct archive_string as;
	int r;

	archive_string_init(&as);
	/* --no-check matches library default */
	archive_strcpy(&as, "zstd --no-check");

	if (data->compression_level < CLEVEL_STD_MIN) {
		archive_string_sprintf(&as, " --fast=%d", -data->compression_level);
	} else {
		archive_string_sprintf(&as, " -%d", data->compression_level);
	}

	if (data->compression_level > CLEVEL_STD_MAX) {
		archive_strcat(&as, " --ultra");
	}

	if (data->threads != 0) {
		archive_string_sprintf(&as, " --threads=%d", data->threads);
	}

	if (data->long_distance != 0) {
		archive_string_sprintf(&as, " --long=%d", data->long_distance);
	}

	f->write = archive_compressor_zstd_write;
	r = __archive_write_program_open(f, data->pdata, as.s);
	archive_string_free(&as);
	return (r);
}

static int
archive_compressor_zstd_write(struct archive_write_filter *f, const void *buff,
    size_t length)
{
	struct private_data *data = (struct private_data *)f->data;

	return __archive_write_program_write(f, data->pdata, buff, length);
}

static int
archive_compressor_zstd_flush(struct archive_write_filter *f)
{
	(void)f; /* UNUSED */

	return (ARCHIVE_OK);
}

static int
archive_compressor_zstd_close(struct archive_write_filter *f)
{
	struct private_data *data = (struct private_data *)f->data;

	return __archive_write_program_close(f, data->pdata);
}

#endif /* HAVE_ZSTD_H && HAVE_ZSTD_compressStream */
Commit	Line	Data
f59fec02 SP	1	/*-
f59fec02 SP	2	* Copyright (c) 2017 Sean Purcell
ac417cce	3	* Copyright (c) 2023-2024 Klara, Inc.
f59fec02 SP	4	* All rights reserved.
	5	*
	6	* Redistribution and use in source and binary forms, with or without
	7	* modification, are permitted provided that the following conditions
	8	* are met:
	9	* 1. Redistributions of source code must retain the above copyright
	10	* notice, this list of conditions and the following disclaimer.
	11	* 2. Redistributions in binary form must reproduce the above copyright
	12	* notice, this list of conditions and the following disclaimer in the
	13	* documentation and/or other materials provided with the distribution.
	14	*
	15	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
	16	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	17	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	18	* IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
	19	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
	20	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	21	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	22	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	23	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
	24	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	25	*/
	26
	27	#include "archive_platform.h"
	28
f59fec02 SP	29	#ifdef HAVE_ERRNO_H
	30	#include <errno.h>
	31	#endif
c7b51f04 DES	32	#ifdef HAVE_STDINT_H
	33	#include <stdint.h>
	34	#endif
f59fec02 SP	35	#ifdef HAVE_STDLIB_H
	36	#include <stdlib.h>
	37	#endif
	38	#ifdef HAVE_STRING_H
	39	#include <string.h>
	40	#endif
	41	#ifdef HAVE_ZSTD_H
	42	#include <zstd.h>
	43	#endif
	44
	45	#include "archive.h"
	46	#include "archive_private.h"
	47	#include "archive_string.h"
	48	#include "archive_write_private.h"
	49
	50	/* Don't compile this if we don't have zstd.h */
	51
	52	struct private_data {
	53	int compression_level;
c7b51f04	54	int threads;
c0cdbb1f	55	int long_distance;
771f434c	56	#if HAVE_ZSTD_H && HAVE_ZSTD_compressStream
a6b488c5 DES	57	enum {
	58	running,
	59	finishing,
	60	resetting,
	61	} state;
	62	int frame_per_file;
ac417cce DES	63	size_t min_frame_in;
	64	size_t max_frame_in;
	65	size_t min_frame_out;
	66	size_t max_frame_out;
a6b488c5 DES	67	size_t cur_frame;
	68	size_t cur_frame_in;
	69	size_t cur_frame_out;
	70	size_t total_in;
f59fec02	71	ZSTD_CStream *cstream;
f59fec02 SP	72	ZSTD_outBuffer out;
	73	#else
	74	struct archive_write_program_data *pdata;
	75	#endif
	76	};
	77
c13d55f2 HM	78	/* If we don't have the library use default range values (zstdcli.c v1.4.0) */
	79	#define CLEVEL_MIN -99
	80	#define CLEVEL_STD_MIN 0 /* prior to 1.3.4 and more recent without using --fast */
	81	#define CLEVEL_DEFAULT 3
f6480495	82	#define CLEVEL_STD_MAX 19 /* without using --ultra */
c13d55f2 HM	83	#define CLEVEL_MAX 22
c13d55f2 HM	84
c0cdbb1f	85	#define LONG_STD 27
c0cdbb1f	86
c13d55f2 HM	87	#define MINVER_NEGCLEVEL 10304
c13d55f2 HM	88	#define MINVER_MINCLEVEL 10306
c0cdbb1f	89	#define MINVER_LONG 10302
f6480495	90
f59fec02 SP	91	static int archive_compressor_zstd_options(struct archive_write_filter *,
	92	const char , const char );
	93	static int archive_compressor_zstd_open(struct archive_write_filter *);
	94	static int archive_compressor_zstd_write(struct archive_write_filter *,
	95	const void *, size_t);
a6b488c5	96	static int archive_compressor_zstd_flush(struct archive_write_filter *);
f59fec02 SP	97	static int archive_compressor_zstd_close(struct archive_write_filter *);
f59fec02 SP	98	static int archive_compressor_zstd_free(struct archive_write_filter *);
771f434c	99	#if HAVE_ZSTD_H && HAVE_ZSTD_compressStream
f59fec02 SP	100	static int drive_compressor(struct archive_write_filter *,
	101	struct private_data , int, const void , size_t);
	102	#endif
	103
	104
	105	/*
	106	* Add a zstd compression filter to this write handle.
	107	*/
	108	int
	109	archive_write_add_filter_zstd(struct archive *_a)
	110	{
	111	struct archive_write a = (struct archive_write )_a;
	112	struct archive_write_filter *f = __archive_write_allocate_filter(_a);
	113	struct private_data *data;
	114	archive_check_magic(&a->archive, ARCHIVE_WRITE_MAGIC,
	115	ARCHIVE_STATE_NEW, "archive_write_add_filter_zstd");
	116
	117	data = calloc(1, sizeof(*data));
	118	if (data == NULL) {
	119	archive_set_error(&a->archive, ENOMEM, "Out of memory");
	120	return (ARCHIVE_FATAL);
	121	}
	122	f->data = data;
	123	f->open = &archive_compressor_zstd_open;
	124	f->options = &archive_compressor_zstd_options;
a6b488c5	125	f->flush = &archive_compressor_zstd_flush;
f59fec02 SP	126	f->close = &archive_compressor_zstd_close;
	127	f->free = &archive_compressor_zstd_free;
	128	f->code = ARCHIVE_FILTER_ZSTD;
	129	f->name = "zstd";
c13d55f2	130	data->compression_level = CLEVEL_DEFAULT;
fd447e33	131	data->threads = 0;
c0cdbb1f	132	data->long_distance = 0;
771f434c	133	#if HAVE_ZSTD_H && HAVE_ZSTD_compressStream
a6b488c5	134	data->frame_per_file = 0;
ac417cce DES	135	data->min_frame_in = 0;
	136	data->max_frame_in = SIZE_MAX;
	137	data->min_frame_out = 0;
	138	data->max_frame_out = SIZE_MAX;
a6b488c5 DES	139	data->cur_frame_in = 0;
a6b488c5 DES	140	data->cur_frame_out = 0;
f59fec02 SP	141	data->cstream = ZSTD_createCStream();
	142	if (data->cstream == NULL) {
	143	free(data);
	144	archive_set_error(&a->archive, ENOMEM,
	145	"Failed to allocate zstd compressor object");
	146	return (ARCHIVE_FATAL);
	147	}
	148
	149	return (ARCHIVE_OK);
	150	#else
	151	data->pdata = __archive_write_program_allocate("zstd");
	152	if (data->pdata == NULL) {
	153	free(data);
	154	archive_set_error(&a->archive, ENOMEM, "Out of memory");
	155	return (ARCHIVE_FATAL);
	156	}
f59fec02 SP	157	archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC,
	158	"Using external zstd program");
	159	return (ARCHIVE_WARN);
	160	#endif
	161	}
	162
	163	static int
	164	archive_compressor_zstd_free(struct archive_write_filter *f)
	165	{
	166	struct private_data data = (struct private_data )f->data;
771f434c	167	#if HAVE_ZSTD_H && HAVE_ZSTD_compressStream
f59fec02 SP	168	ZSTD_freeCStream(data->cstream);
	169	free(data->out.dst);
	170	#else
	171	__archive_write_program_free(data->pdata);
	172	#endif
	173	free(data);
	174	f->data = NULL;
	175	return (ARCHIVE_OK);
	176	}
	177
ac417cce DES	178	static int
ac417cce DES	179	string_to_number(const char string, intmax_t numberp)
c13d55f2	180	{
c7b51f04 DES	181	char *end;
c7b51f04 DES	182
a6b488c5 DES	183	if (string == NULL \|\| *string == '\0')
a6b488c5 DES	184	return (ARCHIVE_WARN);
c7b51f04 DES	185	*numberp = strtoimax(string, &end, 10);
	186	if (end == string \|\| *end != '\0' \|\| errno == EOVERFLOW) {
	187	*numberp = 0;
	188	return (ARCHIVE_WARN);
	189	}
	190	return (ARCHIVE_OK);
c13d55f2 HM	191	}
c13d55f2 HM	192
ac417cce DES	193	static int
	194	string_to_size(const char string, size_t numberp)
	195	{
	196	uintmax_t number;
	197	char *end;
	198	unsigned int shift = 0;
	199
	200	if (string == NULL \|\| string == '\0' \|\| string == '-')
	201	return (ARCHIVE_WARN);
	202	number = strtoumax(string, &end, 10);
	203	if (end > string) {
	204	if (end == 'K' \|\| end == 'k') {
	205	shift = 10;
	206	end++;
	207	} else if (end == 'M' \|\| end == 'm') {
	208	shift = 20;
	209	end++;
	210	} else if (end == 'G' \|\| end == 'g') {
	211	shift = 30;
	212	end++;
	213	}
	214	if (end == 'B' \|\| end == 'b') {
	215	end++;
	216	}
	217	}
	218	if (end == string \|\| *end != '\0' \|\| errno == EOVERFLOW) {
	219	return (ARCHIVE_WARN);
	220	}
	221	if (number > (uintmax_t)SIZE_MAX >> shift) {
	222	return (ARCHIVE_WARN);
	223	}
	224	*numberp = (size_t)(number << shift);
	225	return (ARCHIVE_OK);
	226	}
	227
f59fec02 SP	228	/*
	229	* Set write options.
	230	*/
	231	static int
	232	archive_compressor_zstd_options(struct archive_write_filter f, const char key,
	233	const char *value)
	234	{
	235	struct private_data data = (struct private_data )f->data;
	236
	237	if (strcmp(key, "compression-level") == 0) {
c7b51f04 DES	238	intmax_t level;
	239	if (string_to_number(value, &level) != ARCHIVE_OK) {
	240	return (ARCHIVE_WARN);
	241	}
60635a89	242	/* If we don't have the library, hard-code the max level */
c13d55f2 HM	243	int minimum = CLEVEL_MIN;
c13d55f2 HM	244	int maximum = CLEVEL_MAX;
771f434c	245	#if HAVE_ZSTD_H && HAVE_ZSTD_compressStream
c13d55f2	246	maximum = ZSTD_maxCLevel();
09e78e00	247	#if ZSTD_VERSION_NUMBER >= MINVER_MINCLEVEL
c13d55f2 HM	248	if (ZSTD_versionNumber() >= MINVER_MINCLEVEL) {
	249	minimum = ZSTD_minCLevel();
	250	}
09e78e00 HM	251	else
	252	#endif
	253	if (ZSTD_versionNumber() < MINVER_NEGCLEVEL) {
c13d55f2 HM	254	minimum = CLEVEL_STD_MIN;
c13d55f2 HM	255	}
60635a89	256	#endif
c13d55f2	257	if (level < minimum \|\| level > maximum) {
f59fec02 SP	258	return (ARCHIVE_WARN);
f59fec02 SP	259	}
3131c6f2	260	data->compression_level = (int)level;
f59fec02	261	return (ARCHIVE_OK);
fd447e33	262	} else if (strcmp(key, "threads") == 0) {
c7b51f04 DES	263	intmax_t threads;
c7b51f04 DES	264	if (string_to_number(value, &threads) != ARCHIVE_OK) {
fd447e33 RG	265	return (ARCHIVE_WARN);
fd447e33 RG	266	}
c7b51f04	267	if (threads < 0) {
fd447e33 RG	268	return (ARCHIVE_WARN);
fd447e33 RG	269	}
3131c6f2	270	data->threads = (int)threads;
fd447e33	271	return (ARCHIVE_OK);
771f434c	272	#if HAVE_ZSTD_H && HAVE_ZSTD_compressStream
a6b488c5 DES	273	} else if (strcmp(key, "frame-per-file") == 0) {
	274	data->frame_per_file = 1;
	275	return (ARCHIVE_OK);
ac417cce DES	276	} else if (strcmp(key, "min-frame-in") == 0) {
ac417cce DES	277	if (string_to_size(value, &data->min_frame_in) != ARCHIVE_OK) {
a6b488c5 DES	278	return (ARCHIVE_WARN);
a6b488c5 DES	279	}
ac417cce DES	280	return (ARCHIVE_OK);
	281	} else if (strcmp(key, "min-frame-out") == 0 \|\|
	282	strcmp(key, "min-frame-size") == 0) {
	283	if (string_to_size(value, &data->min_frame_out) != ARCHIVE_OK) {
a6b488c5 DES	284	return (ARCHIVE_WARN);
a6b488c5 DES	285	}
a6b488c5	286	return (ARCHIVE_OK);
ac417cce DES	287	} else if (strcmp(key, "max-frame-in") == 0 \|\|
	288	strcmp(key, "max-frame-size") == 0) {
	289	if (string_to_size(value, &data->max_frame_in) != ARCHIVE_OK \|\|
	290	data->max_frame_in < 1024) {
a6b488c5 DES	291	return (ARCHIVE_WARN);
a6b488c5 DES	292	}
ac417cce DES	293	return (ARCHIVE_OK);
	294	} else if (strcmp(key, "max-frame-out") == 0) {
	295	if (string_to_size(value, &data->max_frame_out) != ARCHIVE_OK \|\|
	296	data->max_frame_out < 1024) {
a6b488c5 DES	297	return (ARCHIVE_WARN);
a6b488c5 DES	298	}
a6b488c5 DES	299	return (ARCHIVE_OK);
a6b488c5 DES	300	#endif
f59fec02	301	}
c0cdbb1f	302	else if (strcmp(key, "long") == 0) {
	303	intmax_t long_distance;
	304	if (string_to_number(value, &long_distance) != ARCHIVE_OK) {
	305	return (ARCHIVE_WARN);
	306	}
771f434c	307	#if HAVE_ZSTD_H && HAVE_ZSTD_compressStream && ZSTD_VERSION_NUMBER >= MINVER_LONG
c0cdbb1f	308	ZSTD_bounds bounds = ZSTD_cParam_getBounds(ZSTD_c_windowLog);
	309	if (ZSTD_isError(bounds.error)) {
	310	int max_distance = ((int)(sizeof(size_t) == 4 ? 30 : 31));
	311	if (((int)long_distance) < 10 \|\| (int)long_distance > max_distance)
	312	return (ARCHIVE_WARN);
	313	} else {
	314	if ((int)long_distance < bounds.lowerBound \|\| (int)long_distance > bounds.upperBound)
	315	return (ARCHIVE_WARN);
	316	}
	317	#else
	318	int max_distance = ((int)(sizeof(size_t) == 4 ? 30 : 31));
	319	if (((int)long_distance) < 10 \|\| (int)long_distance > max_distance)
	320	return (ARCHIVE_WARN);
	321	#endif
	322	data->long_distance = (int)long_distance;
	323	return (ARCHIVE_OK);
	324	}
f59fec02 SP	325
	326	/* Note: The "warn" return is just to inform the options
	327	* supervisor that we didn't handle it. It will generate
	328	* a suitable error if no one used this option. */
	329	return (ARCHIVE_WARN);
	330	}
	331
771f434c	332	#if HAVE_ZSTD_H && HAVE_ZSTD_compressStream
f59fec02 SP	333	/*
	334	* Setup callback.
	335	*/
	336	static int
	337	archive_compressor_zstd_open(struct archive_write_filter *f)
	338	{
	339	struct private_data data = (struct private_data )f->data;
f59fec02 SP	340
	341	if (data->out.dst == NULL) {
	342	size_t bs = ZSTD_CStreamOutSize(), bpb;
	343	if (f->archive->magic == ARCHIVE_WRITE_MAGIC) {
	344	/* Buffer size should be a multiple number of
	345	* the of bytes per block for performance. */
	346	bpb = archive_write_get_bytes_per_block(f->archive);
	347	if (bpb > bs)
	348	bs = bpb;
	349	else if (bpb != 0)
	350	bs -= bs % bpb;
	351	}
	352	data->out.size = bs;
	353	data->out.pos = 0;
	354	data->out.dst
	355	= (unsigned char *)malloc(data->out.size);
	356	if (data->out.dst == NULL) {
	357	archive_set_error(f->archive, ENOMEM,
	358	"Can't allocate data for compression buffer");
	359	return (ARCHIVE_FATAL);
	360	}
	361	}
	362
	363	f->write = archive_compressor_zstd_write;
	364
	365	if (ZSTD_isError(ZSTD_initCStream(data->cstream,
	366	data->compression_level))) {
	367	archive_set_error(f->archive, ARCHIVE_ERRNO_MISC,
	368	"Internal error initializing zstd compressor object");
	369	return (ARCHIVE_FATAL);
	370	}
	371
fd447e33 RG	372	ZSTD_CCtx_setParameter(data->cstream, ZSTD_c_nbWorkers, data->threads);
fd447e33 RG	373
c0cdbb1f	374	#if ZSTD_VERSION_NUMBER >= MINVER_LONG
	375	ZSTD_CCtx_setParameter(data->cstream, ZSTD_c_windowLog, data->long_distance);
	376	#endif
	377
f59fec02 SP	378	return (ARCHIVE_OK);
	379	}
	380
	381	/*
	382	* Write data to the compressed stream.
	383	*/
	384	static int
	385	archive_compressor_zstd_write(struct archive_write_filter f, const void buff,
	386	size_t length)
	387	{
	388	struct private_data data = (struct private_data )f->data;
f59fec02	389
a6b488c5 DES	390	return (drive_compressor(f, data, 0, buff, length));
a6b488c5 DES	391	}
f59fec02	392
a6b488c5 DES	393	/*
	394	* Flush the compressed stream.
	395	*/
	396	static int
	397	archive_compressor_zstd_flush(struct archive_write_filter *f)
	398	{
	399	struct private_data data = (struct private_data )f->data;
f59fec02	400
ac417cce DES	401	if (data->frame_per_file && data->state == running) {
	402	if (data->cur_frame_in > data->min_frame_in &&
	403	data->cur_frame_out > data->min_frame_out) {
	404	data->state = finishing;
	405	}
	406	}
a6b488c5	407	return (drive_compressor(f, data, 1, NULL, 0));
f59fec02 SP	408	}
	409
	410	/*
	411	* Finish the compression...
	412	*/
	413	static int
	414	archive_compressor_zstd_close(struct archive_write_filter *f)
	415	{
	416	struct private_data data = (struct private_data )f->data;
f59fec02	417
a6b488c5 DES	418	if (data->state == running)
	419	data->state = finishing;
	420	return (drive_compressor(f, data, 1, NULL, 0));
f59fec02 SP	421	}
	422
	423	/*
	424	* Utility function to push input data through compressor,
	425	* writing full output blocks as necessary.
f59fec02 SP	426	*/
	427	static int
	428	drive_compressor(struct archive_write_filter *f,
a6b488c5	429	struct private_data data, int flush, const void src, size_t length)
f59fec02	430	{
c7b51f04	431	ZSTD_inBuffer in = { .src = src, .size = length, .pos = 0 };
a6b488c5	432	size_t ipos, opos, zstdret = 0;
c7b51f04	433	int ret;
f59fec02 SP	434
f59fec02 SP	435	for (;;) {
a6b488c5 DES	436	ipos = in.pos;
	437	opos = data->out.pos;
	438	switch (data->state) {
	439	case running:
	440	if (in.pos == in.size)
	441	return (ARCHIVE_OK);
	442	zstdret = ZSTD_compressStream(data->cstream,
	443	&data->out, &in);
	444	if (ZSTD_isError(zstdret))
	445	goto zstd_fatal;
	446	break;
	447	case finishing:
	448	zstdret = ZSTD_endStream(data->cstream, &data->out);
	449	if (ZSTD_isError(zstdret))
	450	goto zstd_fatal;
	451	if (zstdret == 0)
	452	data->state = resetting;
	453	break;
	454	case resetting:
	455	ZSTD_CCtx_reset(data->cstream, ZSTD_reset_session_only);
	456	data->cur_frame++;
	457	data->cur_frame_in = 0;
	458	data->cur_frame_out = 0;
	459	data->state = running;
	460	break;
f59fec02	461	}
a6b488c5 DES	462	data->total_in += in.pos - ipos;
	463	data->cur_frame_in += in.pos - ipos;
	464	data->cur_frame_out += data->out.pos - opos;
ac417cce DES	465	if (data->state == running) {
	466	if (data->cur_frame_in >= data->max_frame_in \|\|
	467	data->cur_frame_out >= data->max_frame_out) {
	468	data->state = finishing;
	469	}
c7b51f04	470	}
a6b488c5 DES	471	if (data->out.pos == data->out.size \|\|
a6b488c5 DES	472	(flush && data->out.pos > 0)) {
c7b51f04 DES	473	ret = __archive_write_filter(f->next_filter,
c7b51f04 DES	474	data->out.dst, data->out.pos);
a6b488c5 DES	475	if (ret != ARCHIVE_OK)
	476	goto fatal;
	477	data->out.pos = 0;
f59fec02 SP	478	}
f59fec02 SP	479	}
a6b488c5 DES	480	zstd_fatal:
	481	archive_set_error(f->archive, ARCHIVE_ERRNO_MISC,
	482	"Zstd compression failed: %s",
	483	ZSTD_getErrorName(zstdret));
	484	fatal:
	485	return (ARCHIVE_FATAL);
f59fec02 SP	486	}
f59fec02 SP	487
771f434c	488	#else /* HAVE_ZSTD_H && HAVE_ZSTD_compressStream */
f59fec02 SP	489
	490	static int
	491	archive_compressor_zstd_open(struct archive_write_filter *f)
	492	{
	493	struct private_data data = (struct private_data )f->data;
	494	struct archive_string as;
	495	int r;
	496
	497	archive_string_init(&as);
7e2b609f	498	/* --no-check matches library default */
c13d55f2 HM	499	archive_strcpy(&as, "zstd --no-check");
	500
	501	if (data->compression_level < CLEVEL_STD_MIN) {
c7b51f04	502	archive_string_sprintf(&as, " --fast=%d", -data->compression_level);
c13d55f2	503	} else {
c7b51f04	504	archive_string_sprintf(&as, " -%d", data->compression_level);
c13d55f2	505	}
f59fec02	506
f6480495 HM	507	if (data->compression_level > CLEVEL_STD_MAX) {
	508	archive_strcat(&as, " --ultra");
	509	}
	510
fd447e33	511	if (data->threads != 0) {
c7b51f04	512	archive_string_sprintf(&as, " --threads=%d", data->threads);
fd447e33 RG	513	}
fd447e33 RG	514
c0cdbb1f	515	if (data->long_distance != 0) {
	516	archive_string_sprintf(&as, " --long=%d", data->long_distance);
	517	}
	518
f59fec02 SP	519	f->write = archive_compressor_zstd_write;
	520	r = __archive_write_program_open(f, data->pdata, as.s);
	521	archive_string_free(&as);
	522	return (r);
	523	}
	524
	525	static int
	526	archive_compressor_zstd_write(struct archive_write_filter f, const void buff,
	527	size_t length)
	528	{
	529	struct private_data data = (struct private_data )f->data;
	530
	531	return __archive_write_program_write(f, data->pdata, buff, length);
	532	}
	533
a6b488c5 DES	534	static int
	535	archive_compressor_zstd_flush(struct archive_write_filter *f)
	536	{
f1ae9980	537	(void)f; /* UNUSED */
a6b488c5 DES	538
	539	return (ARCHIVE_OK);
	540	}
	541
f59fec02 SP	542	static int
	543	archive_compressor_zstd_close(struct archive_write_filter *f)
	544	{
	545	struct private_data data = (struct private_data )f->data;
	546
	547	return __archive_write_program_close(f, data->pdata);
	548	}
	549
771f434c	550	#endif /* HAVE_ZSTD_H && HAVE_ZSTD_compressStream */