From 8c4be28fe717b20c772ee4cb6d4cc77fb98e27aa Mon Sep 17 00:00:00 2001 From: Michihiro NAKAJIMA Date: Thu, 14 Aug 2014 08:31:49 +0900 Subject: [PATCH] Add support for lz4 write filter. --- Makefile.am | 4 + cpio/bsdcpio.1 | 7 +- cpio/cmdline.c | 1 + cpio/cpio.c | 4 + cpio/cpio.h | 1 + cpio/test/CMakeLists.txt | 1 + cpio/test/test_option_lz4.c | 56 ++ libarchive/CMakeLists.txt | 1 + libarchive/archive.h | 1 + libarchive/archive_util.c | 11 +- libarchive/archive_write_add_filter.c | 1 + libarchive/archive_write_add_filter_by_name.c | 1 + libarchive/archive_write_add_filter_lz4.c | 642 ++++++++++++++++++ libarchive/test/CMakeLists.txt | 1 + libarchive/test/test_write_filter_lz4.c | 392 +++++++++++ tar/bsdtar.1 | 21 +- tar/bsdtar.c | 2 + tar/bsdtar.h | 1 + tar/cmdline.c | 1 + tar/creation_set.c | 1 + tar/test/CMakeLists.txt | 1 + tar/test/test_option_lz4.c | 56 ++ 22 files changed, 1203 insertions(+), 4 deletions(-) create mode 100644 cpio/test/test_option_lz4.c create mode 100644 libarchive/archive_write_add_filter_lz4.c create mode 100644 libarchive/test/test_write_filter_lz4.c create mode 100644 tar/test/test_option_lz4.c diff --git a/Makefile.am b/Makefile.am index d288ea965..0797e179a 100644 --- a/Makefile.am +++ b/Makefile.am @@ -195,6 +195,7 @@ libarchive_la_SOURCES= \ libarchive/archive_write_add_filter_grzip.c \ libarchive/archive_write_add_filter_gzip.c \ libarchive/archive_write_add_filter_lrzip.c \ + libarchive/archive_write_add_filter_lz4.c \ libarchive/archive_write_add_filter_lzop.c \ libarchive/archive_write_add_filter_none.c \ libarchive/archive_write_add_filter_program.c \ @@ -492,6 +493,7 @@ libarchive_test_SOURCES= \ libarchive/test/test_write_filter_gzip.c \ libarchive/test/test_write_filter_gzip_timestamp.c \ libarchive/test/test_write_filter_lrzip.c \ + libarchive/test/test_write_filter_lz4.c \ libarchive/test/test_write_filter_lzip.c \ libarchive/test/test_write_filter_lzma.c \ libarchive/test/test_write_filter_lzop.c \ @@ -882,6 +884,7 @@ bsdtar_test_SOURCES= \ tar/test/test_option_k.c \ tar/test/test_option_keep_newer_files.c \ tar/test/test_option_lrzip.c \ + tar/test/test_option_lz4.c \ tar/test/test_option_lzma.c \ tar/test/test_option_lzop.c \ tar/test/test_option_n.c \ @@ -1028,6 +1031,7 @@ bsdcpio_test_SOURCES= \ cpio/test/test_option_help.c \ cpio/test/test_option_l.c \ cpio/test/test_option_lrzip.c \ + cpio/test/test_option_lz4.c \ cpio/test/test_option_lzma.c \ cpio/test/test_option_lzop.c \ cpio/test/test_option_m.c \ diff --git a/cpio/bsdcpio.1 b/cpio/bsdcpio.1 index ba0f50369..9dee0eee8 100644 --- a/cpio/bsdcpio.1 +++ b/cpio/bsdcpio.1 @@ -24,7 +24,7 @@ .\" .\" $FreeBSD$ .\" -.Dd October 7, 2012 +.Dd August 14, 2014 .Dt CPIO 1 .Os .Sh NAME @@ -181,6 +181,11 @@ instead of copying. Compress the resulting archive with .Xr lrzip 1 . In input mode, this option is ignored. +.It Fl Fl lz4 +(o mode only) +Compress the archive with lz4-compatible compression before writing it. +In input mode, this option is ignored; lz4 compression is recognized +automatically on input. .It Fl Fl lzma (o mode only) Compress the file with lzma-compatible compression before writing it. diff --git a/cpio/cmdline.c b/cpio/cmdline.c index 0121fd139..da44574f0 100644 --- a/cpio/cmdline.c +++ b/cpio/cmdline.c @@ -73,6 +73,7 @@ static const struct option { { "link", 0, 'l' }, { "list", 0, 't' }, { "lrzip", 0, OPTION_LRZIP }, + { "lz4", 0, OPTION_LZ4 }, { "lzma", 0, OPTION_LZMA }, { "lzop", 0, OPTION_LZOP }, { "make-directories", 0, 'd' }, diff --git a/cpio/cpio.c b/cpio/cpio.c index ac12c8a24..f1317505a 100644 --- a/cpio/cpio.c +++ b/cpio/cpio.c @@ -261,6 +261,7 @@ main(int argc, char *argv[]) cpio->option_link = 1; break; case OPTION_LRZIP: + case OPTION_LZ4: case OPTION_LZMA: /* GNU tar, others */ case OPTION_LZOP: /* GNU tar, others */ cpio->compress = opt; @@ -526,6 +527,9 @@ mode_out(struct cpio *cpio) case OPTION_LRZIP: r = archive_write_add_filter_lrzip(cpio->archive); break; + case OPTION_LZ4: + r = archive_write_add_filter_lz4(cpio->archive); + break; case OPTION_LZMA: r = archive_write_add_filter_lzma(cpio->archive); break; diff --git a/cpio/cpio.h b/cpio/cpio.h index 3e951ce74..c70ec5781 100644 --- a/cpio/cpio.h +++ b/cpio/cpio.h @@ -101,6 +101,7 @@ enum { OPTION_GRZIP, OPTION_INSECURE, OPTION_LRZIP, + OPTION_LZ4, OPTION_LZMA, OPTION_LZOP, OPTION_NO_PRESERVE_OWNER, diff --git a/cpio/test/CMakeLists.txt b/cpio/test/CMakeLists.txt index 09ca2c7d9..8280986cb 100644 --- a/cpio/test/CMakeLists.txt +++ b/cpio/test/CMakeLists.txt @@ -39,6 +39,7 @@ IF(ENABLE_CPIO AND ENABLE_TEST) test_option_help.c test_option_l.c test_option_lrzip.c + test_option_lz4.c test_option_lzma.c test_option_lzop.c test_option_m.c diff --git a/cpio/test/test_option_lz4.c b/cpio/test/test_option_lz4.c new file mode 100644 index 000000000..5889d2262 --- /dev/null +++ b/cpio/test/test_option_lz4.c @@ -0,0 +1,56 @@ +/*- + * Copyright (c) 2014 Michihiro NAKAJIMA + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include "test.h" +__FBSDID("$FreeBSD$"); + +DEFINE_TEST(test_option_lz4) +{ + char *p; + int r; + size_t s; + + /* Create a file. */ + assertMakeFile("f", 0644, "a"); + + /* Archive it with lz4 compression. */ + r = systemf("echo f | %s -o --lz4 >archive.out 2>archive.err", + testprog); + p = slurpfile(&s, "archive.err"); + p[s] = '\0'; + if (r != 0) { + if (strstr(p, "compression not available") != NULL) { + skipping("This version of bsdcpio was compiled " + "without lz4 support"); + return; + } + failure("--lz4 option is broken"); + assertEqualInt(r, 0); + return; + } + /* Check that the archive file has an lz4 signature. */ + p = slurpfile(&s, "archive.out"); + assert(s > 2); + assertEqualMem(p, "\x04\x22\x4d\x18", 4); +} diff --git a/libarchive/CMakeLists.txt b/libarchive/CMakeLists.txt index 7d2da3f63..8e5dc545b 100644 --- a/libarchive/CMakeLists.txt +++ b/libarchive/CMakeLists.txt @@ -115,6 +115,7 @@ SET(libarchive_SOURCES archive_write_add_filter_grzip.c archive_write_add_filter_gzip.c archive_write_add_filter_lrzip.c + archive_write_add_filter_lz4.c archive_write_add_filter_lzop.c archive_write_add_filter_none.c archive_write_add_filter_program.c diff --git a/libarchive/archive.h b/libarchive/archive.h index 78a139f28..2fb78b8b7 100644 --- a/libarchive/archive.h +++ b/libarchive/archive.h @@ -715,6 +715,7 @@ __LA_DECL int archive_write_add_filter_compress(struct archive *); __LA_DECL int archive_write_add_filter_grzip(struct archive *); __LA_DECL int archive_write_add_filter_gzip(struct archive *); __LA_DECL int archive_write_add_filter_lrzip(struct archive *); +__LA_DECL int archive_write_add_filter_lz4(struct archive *); __LA_DECL int archive_write_add_filter_lzip(struct archive *); __LA_DECL int archive_write_add_filter_lzma(struct archive *); __LA_DECL int archive_write_add_filter_lzop(struct archive *); diff --git a/libarchive/archive_util.c b/libarchive/archive_util.c index 643eef871..1b685d109 100644 --- a/libarchive/archive_util.c +++ b/libarchive/archive_util.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2009-2012 Michihiro NAKAJIMA + * Copyright (c) 2009-2012,2014 Michihiro NAKAJIMA * Copyright (c) 2003-2007 Tim Kientzle * All rights reserved. * @@ -54,6 +54,9 @@ __FBSDID("$FreeBSD: head/lib/libarchive/archive_util.c 201098 2009-12-28 02:58:1 #ifdef HAVE_BZLIB_H #include #endif +#ifdef HAVE_LZ4_H +#include +#endif #include "archive.h" #include "archive_private.h" @@ -113,7 +116,11 @@ archive_version_details(void) archive_strncat(&str, p, sep - p); } #endif - } +#if defined(HAVE_LZ4_H) && defined(HAVE_LIBLZ4) + archive_string_sprintf(&str, " liblz4/%d.%d.%d", + LZ4_VERSION_MAJOR, LZ4_VERSION_MINOR, LZ4_VERSION_RELEASE); +#endif + } return str.s; } diff --git a/libarchive/archive_write_add_filter.c b/libarchive/archive_write_add_filter.c index 81dd683aa..ad5dc832f 100644 --- a/libarchive/archive_write_add_filter.c +++ b/libarchive/archive_write_add_filter.c @@ -47,6 +47,7 @@ struct { int code; int (*setter)(struct archive *); } codes[] = { ARCHIVE_FILTER_COMPRESS, archive_write_add_filter_compress }, { ARCHIVE_FILTER_GRZIP, archive_write_add_filter_grzip }, { ARCHIVE_FILTER_LRZIP, archive_write_add_filter_lrzip }, + { ARCHIVE_FILTER_LZ4, archive_write_add_filter_lz4 }, { ARCHIVE_FILTER_LZIP, archive_write_add_filter_lzip }, { ARCHIVE_FILTER_LZMA, archive_write_add_filter_lzma }, { ARCHIVE_FILTER_LZOP, archive_write_add_filter_lzip }, diff --git a/libarchive/archive_write_add_filter_by_name.c b/libarchive/archive_write_add_filter_by_name.c index e4cba4afa..eac4011cb 100644 --- a/libarchive/archive_write_add_filter_by_name.c +++ b/libarchive/archive_write_add_filter_by_name.c @@ -51,6 +51,7 @@ struct { const char *name; int (*setter)(struct archive *); } names[] = { "grzip", archive_write_add_filter_grzip }, { "gzip", archive_write_add_filter_gzip }, { "lrzip", archive_write_add_filter_lrzip }, + { "lz4", archive_write_add_filter_lz4 }, { "lzip", archive_write_add_filter_lzip }, { "lzma", archive_write_add_filter_lzma }, { "lzop", archive_write_add_filter_lzop }, diff --git a/libarchive/archive_write_add_filter_lz4.c b/libarchive/archive_write_add_filter_lz4.c new file mode 100644 index 000000000..b532889f2 --- /dev/null +++ b/libarchive/archive_write_add_filter_lz4.c @@ -0,0 +1,642 @@ +/*- + * Copyright (c) 2014 Michihiro NAKAJIMA + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "archive_platform.h" + +__FBSDID("$FreeBSD$"); + +#ifdef HAVE_ERRNO_H +#include +#endif +#include +#ifdef HAVE_STDLIB_H +#include +#endif +#ifdef HAVE_STRING_H +#include +#endif +#ifdef HAVE_LZ4_H +#include +#endif +#ifdef HAVE_LZ4HC_H +#include +#endif + +#include "archive.h" +#include "archive_endian.h" +#include "archive_private.h" +#include "archive_write_private.h" +#include "archive_xxhash.h" + +#define LZ4_MAGICNUMBER 0x184d2204 + +struct private_data { + int compression_level; + uint8_t header_written:1; + uint8_t version_number:1; + uint8_t block_independence:1; + uint8_t block_checksum:1; + uint8_t stream_size:1; + uint8_t stream_checksum:1; + uint8_t preset_dictionary:1; + uint8_t block_maximum_size:3; +#if defined(HAVE_LIBLZ4) && LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 2 + int64_t total_in; + char *out; + char *out_buffer; + size_t out_buffer_size; + size_t out_block_size; + char *in; + char *in_buffer_allocated; + char *in_buffer; + size_t in_buffer_size; + size_t block_size; + + void *xxh32_state; + void *lz4_stream; +#else + struct archive_write_program_data *pdata; +#endif +}; + +static int archive_filter_lz4_close(struct archive_write_filter *); +static int archive_filter_lz4_free(struct archive_write_filter *); +static int archive_filter_lz4_open(struct archive_write_filter *); +static int archive_filter_lz4_options(struct archive_write_filter *, + const char *, const char *); +static int archive_filter_lz4_write(struct archive_write_filter *, + const void *, size_t); + +/* + * Add a lz4 compression filter to this write handle. + */ +int +archive_write_add_filter_lz4(struct archive *_a) +{ + struct archive_write *a = (struct archive_write *)_a; + struct archive_write_filter *f = __archive_write_allocate_filter(_a); + struct private_data *data; + + archive_check_magic(&a->archive, ARCHIVE_WRITE_MAGIC, + ARCHIVE_STATE_NEW, "archive_write_add_filter_lz4"); + + data = calloc(1, sizeof(*data)); + if (data == NULL) { + archive_set_error(&a->archive, ENOMEM, "Out of memory"); + return (ARCHIVE_FATAL); + } + + /* + * Setup default settings. + */ + data->compression_level = 1; + data->version_number = 0x01; + data->block_independence = 1; + data->block_checksum = 0; + data->stream_size = 0; + data->stream_checksum = 1; + data->preset_dictionary = 0; + data->block_maximum_size = 7; + + /* + * Setup a filter setting. + */ + f->data = data; + f->options = &archive_filter_lz4_options; + f->close = &archive_filter_lz4_close; + f->free = &archive_filter_lz4_free; + f->open = &archive_filter_lz4_open; + f->code = ARCHIVE_FILTER_LZ4; + f->name = "lz4"; +#if defined(HAVE_LIBLZ4) && LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 2 + return (ARCHIVE_OK); +#else + /* + * We don't have lz4 library, and execute external lz4 program + * instead. + */ + data->pdata = __archive_write_program_allocate(); + if (data->pdata == NULL) { + free(data); + archive_set_error(&a->archive, ENOMEM, "Out of memory"); + return (ARCHIVE_FATAL); + } + data->compression_level = 0; + archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, + "Using external lz4 program"); + return (ARCHIVE_WARN); +#endif +} + +/* + * Set write options. + */ +static int +archive_filter_lz4_options(struct archive_write_filter *f, + const char *key, const char *value) +{ + struct private_data *data = (struct private_data *)f->data; + + if (strcmp(key, "compression-level") == 0) { + if (value == NULL || !(value[0] >= '1' && value[0] <= '9') || + value[1] != '\0') + return (ARCHIVE_WARN); + data->compression_level = value[0] - '0'; + return (ARCHIVE_OK); + } + if (strcmp(key, "stream-checksum") == 0) { + data->stream_checksum = value != NULL; + return (ARCHIVE_OK); + } + if (strcmp(key, "block-checksum") == 0) { + data->block_checksum = value != NULL; + return (ARCHIVE_OK); + } + if (strcmp(key, "block-size") == 0) { + if (value == NULL || !(value[0] >= '4' && value[0] <= '7') || + value[1] != '\0') + return (ARCHIVE_WARN); + data->block_maximum_size = value[0] - '0'; + return (ARCHIVE_OK); + } + if (strcmp(key, "block-dependence") == 0) { + data->block_independence = value == NULL; + return (ARCHIVE_OK); + } + + /* Note: The "warn" return is just to inform the options + * supervisor that we didn't handle it. It will generate + * a suitable error if no one used this option. */ + return (ARCHIVE_WARN); +} + +#if defined(HAVE_LIBLZ4) && LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 2 +/* Don't compile this if we don't have liblz4. */ + +static int drive_compressor(struct archive_write_filter *, const char *, + size_t); +static int drive_compressor_independence(struct archive_write_filter *, + const char *, size_t); +static int drive_compressor_dependence(struct archive_write_filter *, + const char *, size_t); +static int lz4_write_stream_descriptor(struct archive_write_filter *); +static ssize_t lz4_write_one_block(struct archive_write_filter *, const char *, + size_t); + + +/* + * Setup callback. + */ +static int +archive_filter_lz4_open(struct archive_write_filter *f) +{ + struct private_data *data = (struct private_data *)f->data; + int ret; + size_t required_size; + static size_t bkmap[] = { 64 * 1024, 256 * 1024, 1 * 1024 * 1024, + 4 * 1024 * 1024 }; + size_t pre_block_size; + + ret = __archive_write_open_filter(f->next_filter); + if (ret != 0) + return (ret); + + if (data->block_maximum_size < 4) + data->block_size = bkmap[0]; + else + data->block_size = bkmap[data->block_maximum_size - 4]; + + required_size = 4 + 15 + 4 + data->block_size + 4 + 4; + if (data->out_buffer_size < required_size) { + size_t bs = required_size, bpb; + free(data->out_buffer); + if (f->archive->magic == ARCHIVE_WRITE_MAGIC) { + /* Buffer size should be a multiple number of + * the of bytes per block for performance. */ + bpb = archive_write_get_bytes_per_block(f->archive); + if (bpb > bs) + bs = bpb; + else if (bpb != 0) { + bs += bpb; + bs -= bs % bpb; + } + } + data->out_block_size = bs; + bs += required_size; + data->out_buffer = malloc(bs); + data->out = data->out_buffer; + data->out_buffer_size = bs; + } + + pre_block_size = (data->block_independence)? 0: 64 * 1024; + if (data->in_buffer_size < data->block_size + pre_block_size) { + free(data->in_buffer_allocated); + data->in_buffer_size = data->block_size; + data->in_buffer_allocated = + malloc(data->in_buffer_size + pre_block_size); + data->in_buffer = data->in_buffer_allocated + pre_block_size; + if (!data->block_independence && data->compression_level >= 3) + data->in_buffer = data->in_buffer_allocated; + data->in = data->in_buffer; + data->in_buffer_size = data->block_size; + } + + if (data->out_buffer == NULL || data->in_buffer_allocated == NULL) { + archive_set_error(f->archive, ENOMEM, + "Can't allocate data for compression buffer"); + return (ARCHIVE_FATAL); + } + + f->write = archive_filter_lz4_write; + + return (ARCHIVE_OK); +} + +/* + * Write data to the out stream. + * + * Returns ARCHIVE_OK if all data written, error otherwise. + */ +static int +archive_filter_lz4_write(struct archive_write_filter *f, + const void *buff, size_t length) +{ + struct private_data *data = (struct private_data *)f->data; + int ret = ARCHIVE_OK; + const char *p; + size_t remaining; + ssize_t size; + + /* If we haven't written a stream descriptor, we have to do it first. */ + if (!data->header_written) { + ret = lz4_write_stream_descriptor(f); + if (ret != ARCHIVE_OK) + return (ret); + data->header_written = 1; + } + + /* Update statistics */ + data->total_in += length; + + p = (const char *)buff; + remaining = length; + while (remaining) { + size_t l; + /* Compress input data to output buffer */ + size = lz4_write_one_block(f, p, remaining); + if (size < ARCHIVE_OK) + return (ARCHIVE_FATAL); + l = data->out - data->out_buffer; + if (l >= data->out_block_size) { + ret = __archive_write_filter(f->next_filter, + data->out_buffer, data->out_block_size); + l -= data->out_block_size; + memcpy(data->out_buffer, + data->out_buffer + data->out_block_size, l); + data->out = data->out_buffer + l; + if (ret < ARCHIVE_WARN) + break; + } + p += size; + remaining -= size; + } + + return (ret); +} + +/* + * Finish the compression. + */ +static int +archive_filter_lz4_close(struct archive_write_filter *f) +{ + struct private_data *data = (struct private_data *)f->data; + int ret, r1; + + /* Finish compression cycle. */ + ret = (int)lz4_write_one_block(f, NULL, 0); + if (ret >= 0) { + /* + * Write the last block and the end of the stream data. + */ + + /* Write End Of Stream. */ + memset(data->out, 0, 4); data->out += 4; + /* Write Stream checksum if needed. */ + if (data->stream_checksum) { + unsigned int checksum; + checksum = __archive_xxhash.XXH32_digest( + data->xxh32_state); + data->xxh32_state = NULL; + archive_le32enc(data->out, checksum); + data->out += 4; + } + ret = __archive_write_filter(f->next_filter, + data->out_buffer, data->out - data->out_buffer); + } + + r1 = __archive_write_close_filter(f->next_filter); + return (r1 < ret ? r1 : ret); +} + +static int +archive_filter_lz4_free(struct archive_write_filter *f) +{ + struct private_data *data = (struct private_data *)f->data; + + if (data->lz4_stream != NULL) { + if (data->compression_level < 3) + LZ4_free(data->lz4_stream); + else + LZ4_freeHC(data->lz4_stream); + } + free(data->out_buffer); + free(data->in_buffer_allocated); + free(data->xxh32_state); + free(data); + f->data = NULL; + return (ARCHIVE_OK); +} + +static int +lz4_write_stream_descriptor(struct archive_write_filter *f) +{ + struct private_data *data = (struct private_data *)f->data; + uint8_t *sd; + + sd = (uint8_t *)data->out; + /* Write Magic Number. */ + archive_le32enc(&sd[0], LZ4_MAGICNUMBER); + /* FLG */ + sd[4] = (data->version_number << 6) + | (data->block_independence << 5) + | (data->block_checksum << 4) + | (data->stream_size << 3) + | (data->stream_checksum << 2) + | (data->preset_dictionary << 0); + /* BD */ + sd[5] = (data->block_maximum_size << 4); + sd[6] = (__archive_xxhash.XXH32(&sd[4], 2, 0) >> 8) & 0xff; + data->out += 7; + if (data->stream_checksum) + data->xxh32_state = __archive_xxhash.XXH32_init(0); + else + data->xxh32_state = NULL; + return (ARCHIVE_OK); +} + +static ssize_t +lz4_write_one_block(struct archive_write_filter *f, const char *p, + size_t length) +{ + struct private_data *data = (struct private_data *)f->data; + ssize_t r; + + if (p == NULL && length == 0) { + /* Compress remaining uncompressed data. */ + if (data->in_buffer == data->in) + return 0; + else { + size_t l = data->in - data->in_buffer; + r = drive_compressor(f, data->in_buffer, l); + if (r == ARCHIVE_OK) + r = (ssize_t)l; + } + } else if ((data->block_independence || data->compression_level < 3) && + data->in_buffer == data->in && length >= data->block_size) { + r = drive_compressor(f, p, data->block_size); + if (r == ARCHIVE_OK) + r = (ssize_t)data->block_size; + } else { + size_t remaining_size = data->in_buffer_size - + (data->in - data->in_buffer); + size_t l = (remaining_size > length)? length: remaining_size; + memcpy(data->in, p, l); + data->in += l; + if (l == remaining_size) { + r = drive_compressor(f, data->in_buffer, + data->block_size); + if (r == ARCHIVE_OK) + r = (ssize_t)l; + data->in = data->in_buffer; + } else + r = (ssize_t)l; + } + + return (r); +} + + +/* + * Utility function to push input data through compressor, writing + * full output blocks as necessary. + * + * Note that this handles both the regular write case (finishing == + * false) and the end-of-archive case (finishing == true). + */ +static int +drive_compressor(struct archive_write_filter *f, const char *p, size_t length) +{ + struct private_data *data = (struct private_data *)f->data; + + if (data->stream_checksum) + __archive_xxhash.XXH32_update(data->xxh32_state, + p, (int)length); + if (data->block_independence) + return drive_compressor_independence(f, p, length); + else + return drive_compressor_dependence(f, p, length); +} + +static int +drive_compressor_independence(struct archive_write_filter *f, const char *p, + size_t length) +{ + struct private_data *data = (struct private_data *)f->data; + unsigned int outsize; + + if (data->compression_level < 4) + outsize = LZ4_compress_limitedOutput(p, data->out + 4, + (int)length, (int)data->block_size); + else + outsize = LZ4_compressHC2_limitedOutput(p, data->out + 4, + (int)length, (int)data->block_size, + data->compression_level); + + if (outsize) { + /* The buffer is compressed. */ + archive_le32enc(data->out, outsize); + data->out += 4; + } else { + /* The buffer is not compressed. The commpressed size was + * bigger than its uncompressed size. */ + archive_le32enc(data->out, length | 0x80000000); + data->out += 4; + memcpy(data->out, p, length); + outsize = length; + } + data->out += outsize; + if (data->block_checksum) { + unsigned int checksum = + __archive_xxhash.XXH32(data->out - outsize, outsize, 0); + archive_le32enc(data->out, checksum); + data->out += 4; + } + return (ARCHIVE_OK); +} + +static int +drive_compressor_dependence(struct archive_write_filter *f, const char *p, + size_t length) +{ + struct private_data *data = (struct private_data *)f->data; + int outsize; + + if (data->compression_level < 3) { + if (data->lz4_stream == NULL) { + data->lz4_stream = LZ4_createStream(); + if (data->lz4_stream == NULL) { + archive_set_error(f->archive, ENOMEM, + "Can't allocate data for compression" + " buffer"); + return (ARCHIVE_FATAL); + } + } + outsize = LZ4_compress_limitedOutput_continue( + data->lz4_stream, p, data->out + 4, (int)length, + (int)data->block_size); + } else { + if (data->lz4_stream == NULL) { + data->lz4_stream = + LZ4_createHC(data->in_buffer_allocated); + if (data->lz4_stream == NULL) { + archive_set_error(f->archive, ENOMEM, + "Can't allocate data for compression" + " buffer"); + return (ARCHIVE_FATAL); + } + } + outsize = LZ4_compressHC2_limitedOutput_continue( + data->lz4_stream, p, data->out + 4, (int)length, + (int)data->block_size, data->compression_level); + } + + if (outsize) { + /* The buffer is compressed. */ + archive_le32enc(data->out, outsize); + data->out += 4; + } else { + /* The buffer is not compressed. The commpressed size was + * bigger than its uncompressed size. */ + archive_le32enc(data->out, length | 0x80000000); + data->out += 4; + memcpy(data->out, p, length); + outsize = length; + } + data->out += outsize; + if (data->block_checksum) { + unsigned int checksum = + __archive_xxhash.XXH32(data->out - outsize, outsize, 0); + archive_le32enc(data->out, checksum); + data->out += 4; + } + + if (length == data->block_size) { +#define DICT_SIZE (64 * 1024) + if (data->compression_level < 3) + LZ4_saveDict(data->lz4_stream, + data->in_buffer_allocated, DICT_SIZE); + else { + LZ4_slideInputBufferHC(data->lz4_stream); + data->in_buffer = data->in_buffer_allocated + DICT_SIZE; + } +#undef DICT_SIZE + } + return (ARCHIVE_OK); +} + +#else /* HAVE_LIBLZ4 */ + +static int +archive_filter_lz4_open(struct archive_write_filter *f) +{ + struct private_data *data = (struct private_data *)f->data; + struct archive_string as; + int r; + + archive_string_init(&as); + archive_strcpy(&as, "lz4 -z -q -q"); + + /* Specify a compression level. */ + if (data->compression_level > 0) { + archive_strcat(&as, " -"); + archive_strappend_char(&as, '0' + data->compression_level); + } + /* Specify a block size. */ + archive_strcat(&as, " -B"); + archive_strappend_char(&as, '0' + data->block_maximum_size); + + if (data->block_checksum) + archive_strcat(&as, " -BX"); + if (data->stream_checksum == 0) + archive_strcat(&as, " -Sx"); + if (data->block_independence == 0) + archive_strcat(&as, " -BD"); + + f->write = archive_filter_lz4_write; + + r = __archive_write_program_open(f, data->pdata, as.s); + archive_string_free(&as); + return (r); +} + +static int +archive_filter_lz4_write(struct archive_write_filter *f, const void *buff, + size_t length) +{ + struct private_data *data = (struct private_data *)f->data; + + return __archive_write_program_write(f, data->pdata, buff, length); +} + +static int +archive_filter_lz4_close(struct archive_write_filter *f) +{ + struct private_data *data = (struct private_data *)f->data; + + return __archive_write_program_close(f, data->pdata); +} + +static int +archive_filter_lz4_free(struct archive_write_filter *f) +{ + struct private_data *data = (struct private_data *)f->data; + + __archive_write_program_free(data->pdata); + free(data); + return (ARCHIVE_OK); +} + +#endif /* HAVE_LIBLZ4 */ diff --git a/libarchive/test/CMakeLists.txt b/libarchive/test/CMakeLists.txt index 2be3dd407..359a90f1f 100644 --- a/libarchive/test/CMakeLists.txt +++ b/libarchive/test/CMakeLists.txt @@ -192,6 +192,7 @@ IF(ENABLE_TEST) test_write_filter_gzip.c test_write_filter_gzip_timestamp.c test_write_filter_lrzip.c + test_write_filter_lz4.c test_write_filter_lzip.c test_write_filter_lzma.c test_write_filter_lzop.c diff --git a/libarchive/test/test_write_filter_lz4.c b/libarchive/test/test_write_filter_lz4.c new file mode 100644 index 000000000..2c20b0d7a --- /dev/null +++ b/libarchive/test/test_write_filter_lz4.c @@ -0,0 +1,392 @@ +/*- + * Copyright (c) 2014 Michihiro NAKAJIMA + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer + * in this position and unchanged. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "test.h" +__FBSDID("$FreeBSD$"); + +/* + * A basic exercise of lz4 reading and writing. + */ + +DEFINE_TEST(test_write_filter_lz4) +{ + struct archive_entry *ae; + struct archive* a; + char *buff, *data; + size_t buffsize, datasize; + char path[16]; + size_t used1, used2; + int i, r, use_prog = 0, filecount; + + assert((a = archive_write_new()) != NULL); + r = archive_write_add_filter_lz4(a); + if (r != ARCHIVE_OK) { + if (canLzop() && r == ARCHIVE_WARN) + use_prog = 1; + else { + skipping("lz4 writing not supported on this platform"); + assertEqualInt(ARCHIVE_OK, archive_write_free(a)); + return; + } + } + + buffsize = 2000000; + assert(NULL != (buff = (char *)malloc(buffsize))); + + datasize = 10000; + assert(NULL != (data = (char *)calloc(1, datasize))); + filecount = 10; + + /* + * Write a filecount files and read them all back. + */ + assert((a = archive_write_new()) != NULL); + assertEqualIntA(a, ARCHIVE_OK, archive_write_set_format_ustar(a)); + assertEqualIntA(a, (use_prog)?ARCHIVE_WARN:ARCHIVE_OK, + archive_write_add_filter_lz4(a)); + assertEqualIntA(a, ARCHIVE_OK, + archive_write_set_bytes_per_block(a, 1024)); + assertEqualIntA(a, ARCHIVE_OK, + archive_write_set_bytes_in_last_block(a, 1024)); + assertEqualInt(ARCHIVE_FILTER_LZ4, archive_filter_code(a, 0)); + assertEqualString("lz4", archive_filter_name(a, 0)); + assertEqualIntA(a, ARCHIVE_OK, + archive_write_open_memory(a, buff, buffsize, &used1)); + assert((ae = archive_entry_new()) != NULL); + archive_entry_set_filetype(ae, AE_IFREG); + archive_entry_set_size(ae, datasize); + for (i = 0; i < filecount; i++) { + sprintf(path, "file%03d", i); + archive_entry_copy_pathname(ae, path); + assertEqualIntA(a, ARCHIVE_OK, archive_write_header(a, ae)); + assertA(datasize + == (size_t)archive_write_data(a, data, datasize)); + } + archive_entry_free(ae); + assertEqualIntA(a, ARCHIVE_OK, archive_write_close(a)); + assertEqualInt(ARCHIVE_OK, archive_write_free(a)); + + assert((a = archive_read_new()) != NULL); + assertEqualIntA(a, ARCHIVE_OK, archive_read_support_format_all(a)); + r = archive_read_support_filter_lz4(a); + if (r == ARCHIVE_WARN) { + skipping("Can't verify lz4 writing by reading back;" + " lz4 reading not fully supported on this platform"); + } else { + assertEqualIntA(a, ARCHIVE_OK, + archive_read_open_memory(a, buff, used1)); + for (i = 0; i < filecount; i++) { + sprintf(path, "file%03d", i); + if (!assertEqualInt(ARCHIVE_OK, + archive_read_next_header(a, &ae))) + break; + assertEqualString(path, archive_entry_pathname(ae)); + assertEqualInt((int)datasize, archive_entry_size(ae)); + } + assertEqualIntA(a, ARCHIVE_OK, archive_read_close(a)); + } + assertEqualInt(ARCHIVE_OK, archive_read_free(a)); + + /* + * Repeat the cycle again, this time setting some compression + * options. + */ + assert((a = archive_write_new()) != NULL); + assertEqualIntA(a, ARCHIVE_OK, archive_write_set_format_ustar(a)); + assertEqualIntA(a, ARCHIVE_OK, + archive_write_set_bytes_per_block(a, 10)); + assertEqualIntA(a, (use_prog)?ARCHIVE_WARN:ARCHIVE_OK, + archive_write_add_filter_lz4(a)); + assertEqualIntA(a, ARCHIVE_FAILED, + archive_write_set_options(a, "lz4:nonexistent-option=0")); + assertEqualIntA(a, ARCHIVE_OK, + archive_write_set_options(a, "lz4:compression-level=1")); + assertEqualIntA(a, ARCHIVE_OK, + archive_write_set_filter_option(a, NULL, "compression-level", "9")); + assertEqualIntA(a, ARCHIVE_FAILED, + archive_write_set_filter_option(a, NULL, "compression-level", "abc")); + assertEqualIntA(a, ARCHIVE_FAILED, + archive_write_set_filter_option(a, NULL, "compression-level", "99")); + assertEqualIntA(a, ARCHIVE_OK, + archive_write_set_options(a, "lz4:compression-level=9")); + assertEqualIntA(a, ARCHIVE_OK, + archive_write_open_memory(a, buff, buffsize, &used2)); + for (i = 0; i < filecount; i++) { + sprintf(path, "file%03d", i); + assert((ae = archive_entry_new()) != NULL); + archive_entry_copy_pathname(ae, path); + archive_entry_set_size(ae, datasize); + archive_entry_set_filetype(ae, AE_IFREG); + assertEqualIntA(a, ARCHIVE_OK, archive_write_header(a, ae)); + assertA(datasize == (size_t)archive_write_data( + a, data, datasize)); + archive_entry_free(ae); + } + assertEqualIntA(a, ARCHIVE_OK, archive_write_close(a)); + assertEqualInt(ARCHIVE_OK, archive_write_free(a)); + + failure("compression-level=9 wrote %d bytes, default wrote %d bytes", + (int)used2, (int)used1); + assert(used2 < used1); + + assert((a = archive_read_new()) != NULL); + assertEqualIntA(a, ARCHIVE_OK, archive_read_support_format_all(a)); + r = archive_read_support_filter_lz4(a); + if (r != ARCHIVE_OK && !use_prog) { + skipping("lz4 reading not fully supported on this platform"); + } else { + assertEqualIntA(a, ARCHIVE_OK, + archive_read_support_filter_all(a)); + assertEqualIntA(a, ARCHIVE_OK, + archive_read_open_memory(a, buff, used2)); + for (i = 0; i < filecount; i++) { + sprintf(path, "file%03d", i); + if (!assertEqualInt(ARCHIVE_OK, + archive_read_next_header(a, &ae))) + break; + assertEqualString(path, archive_entry_pathname(ae)); + assertEqualInt((int)datasize, archive_entry_size(ae)); + } + assertEqualIntA(a, ARCHIVE_OK, archive_read_close(a)); + } + assertEqualInt(ARCHIVE_OK, archive_read_free(a)); + + /* + * Repeat again, with much lower compression. + */ + assert((a = archive_write_new()) != NULL); + assertEqualIntA(a, ARCHIVE_OK, archive_write_set_format_ustar(a)); + assertEqualIntA(a, ARCHIVE_OK, + archive_write_set_bytes_per_block(a, 10)); + assertEqualIntA(a, (use_prog)?ARCHIVE_WARN:ARCHIVE_OK, + archive_write_add_filter_lz4(a)); + assertEqualIntA(a, ARCHIVE_OK, + archive_write_set_filter_option(a, NULL, "compression-level", "1")); + assertEqualIntA(a, ARCHIVE_OK, + archive_write_open_memory(a, buff, buffsize, &used2)); + for (i = 0; i < filecount; i++) { + sprintf(path, "file%03d", i); + assert((ae = archive_entry_new()) != NULL); + archive_entry_copy_pathname(ae, path); + archive_entry_set_size(ae, datasize); + archive_entry_set_filetype(ae, AE_IFREG); + assertEqualIntA(a, ARCHIVE_OK, archive_write_header(a, ae)); + failure("Writing file %s", path); + assertEqualIntA(a, datasize, + (size_t)archive_write_data(a, data, datasize)); + archive_entry_free(ae); + } + assertEqualIntA(a, ARCHIVE_OK, archive_write_close(a)); + assertEqualInt(ARCHIVE_OK, archive_write_free(a)); + +#if 0 + failure("Compression-level=1 wrote %d bytes; default wrote %d bytes", + (int)used2, (int)used1); + assert(used2 > used1); +#endif + + assert((a = archive_read_new()) != NULL); + assertEqualIntA(a, ARCHIVE_OK, archive_read_support_format_all(a)); + assertEqualIntA(a, ARCHIVE_OK, archive_read_support_filter_all(a)); + r = archive_read_support_filter_lz4(a); + if (r == ARCHIVE_WARN) { + skipping("lz4 reading not fully supported on this platform"); + } else { + assertEqualIntA(a, ARCHIVE_OK, + archive_read_open_memory(a, buff, used2)); + for (i = 0; i < filecount; i++) { + sprintf(path, "file%03d", i); + if (!assertEqualInt(ARCHIVE_OK, + archive_read_next_header(a, &ae))) + break; + assertEqualString(path, archive_entry_pathname(ae)); + assertEqualInt((int)datasize, archive_entry_size(ae)); + } + assertEqualIntA(a, ARCHIVE_OK, archive_read_close(a)); + } + assertEqualInt(ARCHIVE_OK, archive_read_free(a)); + + /* + * Test various premature shutdown scenarios to make sure we + * don't crash or leak memory. + */ + assert((a = archive_write_new()) != NULL); + assertEqualIntA(a, (use_prog)?ARCHIVE_WARN:ARCHIVE_OK, + archive_write_add_filter_lz4(a)); + assertEqualInt(ARCHIVE_OK, archive_write_free(a)); + + assert((a = archive_write_new()) != NULL); + assertEqualIntA(a, (use_prog)?ARCHIVE_WARN:ARCHIVE_OK, + archive_write_add_filter_lz4(a)); + assertEqualInt(ARCHIVE_OK, archive_write_close(a)); + assertEqualInt(ARCHIVE_OK, archive_write_free(a)); + + assert((a = archive_write_new()) != NULL); + assertEqualIntA(a, ARCHIVE_OK, archive_write_set_format_ustar(a)); + assertEqualIntA(a, (use_prog)?ARCHIVE_WARN:ARCHIVE_OK, + archive_write_add_filter_lz4(a)); + assertEqualInt(ARCHIVE_OK, archive_write_close(a)); + assertEqualInt(ARCHIVE_OK, archive_write_free(a)); + + assert((a = archive_write_new()) != NULL); + assertEqualIntA(a, ARCHIVE_OK, archive_write_set_format_ustar(a)); + assertEqualIntA(a, (use_prog)?ARCHIVE_WARN:ARCHIVE_OK, + archive_write_add_filter_lz4(a)); + assertEqualIntA(a, ARCHIVE_OK, + archive_write_open_memory(a, buff, buffsize, &used2)); + assertEqualInt(ARCHIVE_OK, archive_write_close(a)); + assertEqualInt(ARCHIVE_OK, archive_write_free(a)); + + /* + * Clean up. + */ + free(data); + free(buff); +} + +static void +test_options(const char *options) +{ + struct archive_entry *ae; + struct archive* a; + char *buff, *data; + size_t buffsize, datasize; + char path[16]; + size_t used1; + int i, r, use_prog = 0, filecount; + + assert((a = archive_write_new()) != NULL); + r = archive_write_add_filter_lz4(a); + if (r != ARCHIVE_OK) { + if (canLzop() && r == ARCHIVE_WARN) + use_prog = 1; + else { + skipping("lz4 writing not supported on this platform"); + assertEqualInt(ARCHIVE_OK, archive_write_free(a)); + return; + } + } + + buffsize = 2000000; + assert(NULL != (buff = (char *)malloc(buffsize))); + + datasize = 10000; + assert(NULL != (data = (char *)calloc(1, datasize))); + filecount = 10; + + /* + * Write a filecount files and read them all back. + */ + assert((a = archive_write_new()) != NULL); + assertEqualIntA(a, ARCHIVE_OK, archive_write_set_format_ustar(a)); + assertEqualIntA(a, (use_prog)?ARCHIVE_WARN:ARCHIVE_OK, + archive_write_add_filter_lz4(a)); + assertEqualIntA(a, ARCHIVE_OK, + archive_write_set_options(a, options)); + assertEqualIntA(a, ARCHIVE_OK, + archive_write_set_bytes_per_block(a, 1024)); + assertEqualIntA(a, ARCHIVE_OK, + archive_write_set_bytes_in_last_block(a, 1024)); + assertEqualInt(ARCHIVE_FILTER_LZ4, archive_filter_code(a, 0)); + assertEqualString("lz4", archive_filter_name(a, 0)); + assertEqualIntA(a, ARCHIVE_OK, + archive_write_open_memory(a, buff, buffsize, &used1)); + assert((ae = archive_entry_new()) != NULL); + archive_entry_set_filetype(ae, AE_IFREG); + archive_entry_set_size(ae, datasize); + for (i = 0; i < filecount; i++) { + sprintf(path, "file%03d", i); + archive_entry_copy_pathname(ae, path); + assertEqualIntA(a, ARCHIVE_OK, archive_write_header(a, ae)); + assertA(datasize + == (size_t)archive_write_data(a, data, datasize)); + } + archive_entry_free(ae); + assertEqualIntA(a, ARCHIVE_OK, archive_write_close(a)); + assertEqualInt(ARCHIVE_OK, archive_write_free(a)); + + assert((a = archive_read_new()) != NULL); + assertEqualIntA(a, ARCHIVE_OK, archive_read_support_format_all(a)); + r = archive_read_support_filter_lz4(a); + if (r == ARCHIVE_WARN) { + skipping("Can't verify lz4 writing by reading back;" + " lz4 reading not fully supported on this platform"); + } else { + assertEqualIntA(a, ARCHIVE_OK, + archive_read_open_memory(a, buff, used1)); + for (i = 0; i < filecount; i++) { + sprintf(path, "file%03d", i); + if (!assertEqualInt(ARCHIVE_OK, + archive_read_next_header(a, &ae))) + break; + assertEqualString(path, archive_entry_pathname(ae)); + assertEqualInt((int)datasize, archive_entry_size(ae)); + } + assertEqualIntA(a, ARCHIVE_OK, archive_read_close(a)); + } + assertEqualInt(ARCHIVE_OK, archive_read_free(a)); + + /* + * Clean up. + */ + free(data); + free(buff); +} + +DEFINE_TEST(test_write_filter_lz4_disable_stream_checksum) +{ + test_options("lz4:!stream-checksum"); +} + +DEFINE_TEST(test_write_filter_lz4_enable_block_checksum) +{ + test_options("lz4:block-checksum"); +} + +DEFINE_TEST(test_write_filter_lz4_block_size_4) +{ + test_options("lz4:block-size=4"); +} + +DEFINE_TEST(test_write_filter_lz4_block_size_5) +{ + test_options("lz4:block-size=5"); +} + +DEFINE_TEST(test_write_filter_lz4_block_size_6) +{ + test_options("lz4:block-size=6"); +} + +DEFINE_TEST(test_write_filter_lz4_block_dependence) +{ + test_options("lz4:block-dependence"); +} + +DEFINE_TEST(test_write_filter_lz4_block_dependence_hc) +{ + test_options("lz4:block-dependence,lz4:compression-level=9"); +} diff --git a/tar/bsdtar.1 b/tar/bsdtar.1 index 34fb1210f..47162e641 100644 --- a/tar/bsdtar.1 +++ b/tar/bsdtar.1 @@ -24,7 +24,7 @@ .\" .\" $FreeBSD$ .\" -.Dd November 1, 2012 +.Dd August 14, 2014 .Dt TAR 1 .Os .Sh NAME @@ -318,6 +318,11 @@ Issue a warning message unless all links to each file are archived. Compress the resulting archive with .Xr lrzip 1 . In extract or list modes, this option is ignored. +.It Fl Fl lz4 +(c mode only) +Compress the archive with lz4-compatible compression before writing it. +In input mode, this option is ignored; lz4 compression is recognized +automatically on input. .It Fl Fl lzma (c mode only) Compress the resulting archive with the original LZMA algorithm. Use of this option is discouraged and new archives should be created with @@ -498,6 +503,20 @@ Supported values are bzip2, gzip, lzo (ultra fast), and zpaq (best, extremely slow). .It Cm lrzip:compression-level A decimal integer from 1 to 9 specifying the lrzip compression level. +.It Cm lz4:compression-level +A decimal integer from 1 to 9 specifying the lzop compression level. +.It Cm lz4:stream-checksum +Enalbe stream checksum. This is by default, use +.Cm lz4:!stream-checksum +to disable. +.It Cm lz4:block-checksum +Enable block checksum (Disabled by default). +.It Cm lz4:block-size +A decimal integer from 4 to 7 specifying the lz4 compression block size +(7 is set by default). +.It Cm lz4:block-dependence +Use the previous block of the block being compressed for +a compression dictionary to improve compression ratio. .It Cm lzop:compression-level A decimal integer from 1 to 9 specifying the lzop compression level. .It Cm xz:compression-level diff --git a/tar/bsdtar.c b/tar/bsdtar.c index 205634dd6..30fbe0d7a 100644 --- a/tar/bsdtar.c +++ b/tar/bsdtar.c @@ -397,6 +397,7 @@ main(int argc, char **argv) bsdtar->option_warn_links = 1; break; case OPTION_LRZIP: + case OPTION_LZ4: case OPTION_LZIP: /* GNU tar beginning with 1.23 */ case OPTION_LZMA: /* GNU tar beginning with 1.20 */ case OPTION_LZOP: /* GNU tar beginning with 1.21 */ @@ -407,6 +408,7 @@ main(int argc, char **argv) compression = opt; switch (opt) { case OPTION_LRZIP: compression_name = "lrzip"; break; + case OPTION_LZ4: compression_name = "lz4"; break; case OPTION_LZIP: compression_name = "lzip"; break; case OPTION_LZMA: compression_name = "lzma"; break; case OPTION_LZOP: compression_name = "lzop"; break; diff --git a/tar/bsdtar.h b/tar/bsdtar.h index f8aae14cf..53745adfa 100644 --- a/tar/bsdtar.h +++ b/tar/bsdtar.h @@ -129,6 +129,7 @@ enum { OPTION_INCLUDE, OPTION_KEEP_NEWER_FILES, OPTION_LRZIP, + OPTION_LZ4, OPTION_LZIP, OPTION_LZMA, OPTION_LZOP, diff --git a/tar/cmdline.c b/tar/cmdline.c index cb0968658..a3562ffcd 100644 --- a/tar/cmdline.c +++ b/tar/cmdline.c @@ -102,6 +102,7 @@ static const struct bsdtar_option { { "keep-old-files", 0, 'k' }, { "list", 0, 't' }, { "lrzip", 0, OPTION_LRZIP }, + { "lz4", 0, OPTION_LZ4 }, { "lzip", 0, OPTION_LZIP }, { "lzma", 0, OPTION_LZMA }, { "lzop", 0, OPTION_LZOP }, diff --git a/tar/creation_set.c b/tar/creation_set.c index dd3751114..87d561b35 100644 --- a/tar/creation_set.c +++ b/tar/creation_set.c @@ -75,6 +75,7 @@ get_filter_code(const char *suffix) { ".grz", "grzip" }, { ".lrz", "lrzip" }, { ".lz", "lzip" }, + { ".lz4", "lz4" }, { ".lzo", "lzop" }, { ".lzma", "lzma" }, { ".uu", "uuencode" }, diff --git a/tar/test/CMakeLists.txt b/tar/test/CMakeLists.txt index 98f49e292..f19eb548b 100644 --- a/tar/test/CMakeLists.txt +++ b/tar/test/CMakeLists.txt @@ -18,6 +18,7 @@ IF(ENABLE_TAR AND ENABLE_TEST) test_extract_tar_gz.c test_extract_tar_lrz.c test_extract_tar_lz.c + test_extract_tar_lz4.c test_extract_tar_lzma.c test_extract_tar_lzo.c test_extract_tar_xz.c diff --git a/tar/test/test_option_lz4.c b/tar/test/test_option_lz4.c new file mode 100644 index 000000000..3ebe343a3 --- /dev/null +++ b/tar/test/test_option_lz4.c @@ -0,0 +1,56 @@ +/*- + * Copyright (c) 2014 Michihiro NAKAJIMA + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include "test.h" +__FBSDID("$FreeBSD$"); + +DEFINE_TEST(test_option_lz4) +{ + char *p; + int r; + size_t s; + + /* Create a file. */ + assertMakeFile("f", 0644, "a"); + + /* Archive it with lz4 compression. */ + r = systemf("%s -cf - --lz4 f >archive.out 2>archive.err", + testprog); + p = slurpfile(&s, "archive.err"); + p[s] = '\0'; + if (r != 0) { + if (strstr(p, "Unsupported compression") != NULL) { + skipping("This version of bsdtar was compiled " + "without lz4 support"); + return; + } + failure("--lz4 option is broken"); + assertEqualInt(r, 0); + return; + } + /* Check that the archive file has an lz4 signature. */ + p = slurpfile(&s, "archive.out"); + assert(s > 2); + assertEqualMem(p, "\x04\x22\x4d\x18", 4); +} -- 2.47.2