From: Michihiro NAKAJIMA Date: Tue, 4 May 2010 14:33:46 +0000 (-0400) Subject: Add support for lzip (read filter). X-Git-Tag: v3.0.0a~1026 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=d8961d393aafcab69144ab5f1fb5f7b308781757;p=thirdparty%2Flibarchive.git Add support for lzip (read filter). SVN-Revision: 2368 --- diff --git a/Makefile.am b/Makefile.am index 99f37d39f..b4346af47 100644 --- a/Makefile.am +++ b/Makefile.am @@ -271,6 +271,7 @@ libarchive_test_SOURCES= \ libarchive/test/test_read_format_cpio_bin_be.c \ libarchive/test/test_read_format_cpio_bin_bz2.c \ libarchive/test/test_read_format_cpio_bin_gz.c \ + libarchive/test/test_read_format_cpio_bin_lzip.c \ libarchive/test/test_read_format_cpio_bin_lzma.c \ libarchive/test/test_read_format_cpio_bin_xz.c \ libarchive/test/test_read_format_cpio_odc.c \ diff --git a/libarchive/archive.h b/libarchive/archive.h index b06200cb6..fe0ca2264 100644 --- a/libarchive/archive.h +++ b/libarchive/archive.h @@ -247,6 +247,7 @@ typedef int archive_close_callback(struct archive *, void *_client_data); #define ARCHIVE_FILTER_XZ 6 #define ARCHIVE_FILTER_UU 7 #define ARCHIVE_FILTER_RPM 8 +#define ARCHIVE_FILTER_LZIP 9 #if ARCHIVE_VERSION_NUMBER < 4000000 #define ARCHIVE_COMPRESSION_NONE ARCHIVE_FILTER_NONE @@ -258,6 +259,7 @@ typedef int archive_close_callback(struct archive *, void *_client_data); #define ARCHIVE_COMPRESSION_XZ ARCHIVE_FILTER_XZ #define ARCHIVE_COMPRESSION_UU ARCHIVE_FILTER_UU #define ARCHIVE_COMPRESSION_RPM ARCHIVE_FILTER_RPM +#define ARCHIVE_COMPRESSION_LZIP ARCHIVE_FILTER_LZIP #endif /* @@ -331,6 +333,7 @@ __LA_DECL int archive_read_support_compression_all(struct archive *); __LA_DECL int archive_read_support_compression_bzip2(struct archive *); __LA_DECL int archive_read_support_compression_compress(struct archive *); __LA_DECL int archive_read_support_compression_gzip(struct archive *); +__LA_DECL int archive_read_support_compression_lzip(struct archive *); __LA_DECL int archive_read_support_compression_lzma(struct archive *); __LA_DECL int archive_read_support_compression_none(struct archive *); __LA_DECL int archive_read_support_compression_program(struct archive *, diff --git a/libarchive/archive_read_private.h b/libarchive/archive_read_private.h index 8ecc3ecf7..6429db1d4 100644 --- a/libarchive/archive_read_private.h +++ b/libarchive/archive_read_private.h @@ -148,7 +148,7 @@ struct archive_read { struct archive_read_client client; /* Registered filter bidders. */ - struct archive_read_filter_bidder bidders[8]; + struct archive_read_filter_bidder bidders[9]; /* Last filter in chain */ struct archive_read_filter *filter; diff --git a/libarchive/archive_read_support_compression_all.c b/libarchive/archive_read_support_compression_all.c index a6db7364d..483312319 100644 --- a/libarchive/archive_read_support_compression_all.c +++ b/libarchive/archive_read_support_compression_all.c @@ -37,6 +37,8 @@ archive_read_support_compression_all(struct archive *a) archive_read_support_compression_compress(a); /* Gzip decompress falls back to "gunzip" command-line. */ archive_read_support_compression_gzip(a); + /* Lzip falls back to "unlzip" command-line program. */ + archive_read_support_compression_lzip(a); /* The LZMA file format has a very weak signature, so it * may not be feasible to keep this here, but we'll try. * This will come back out if there are problems. */ diff --git a/libarchive/archive_read_support_compression_xz.c b/libarchive/archive_read_support_compression_xz.c index 28c4e2d9f..9b3450cf1 100644 --- a/libarchive/archive_read_support_compression_xz.c +++ b/libarchive/archive_read_support_compression_xz.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2009 Michihiro NAKAJIMA + * Copyright (c) 2009,2010 Michihiro NAKAJIMA * Copyright (c) 2003-2008 Tim Kientzle and Miklos Vajna * All rights reserved. * @@ -60,6 +60,13 @@ struct private_data { size_t out_block_size; int64_t total_out; char eof; /* True = found end of compressed data. */ + char in_stream; + + /* Following variables is used for lzip only */ + char lzip_ver; + uint32_t crc32; + int64_t member_in; + int64_t member_out; }; /* Combined lzma/xz filter */ @@ -94,6 +101,10 @@ static int xz_bidder_init(struct archive_read_filter *); static int lzma_bidder_bid(struct archive_read_filter_bidder *, struct archive_read_filter *); static int lzma_bidder_init(struct archive_read_filter *); +static int lzip_has_member(struct archive_read_filter *); +static int lzip_bidder_bid(struct archive_read_filter_bidder *, + struct archive_read_filter *); +static int lzip_bidder_init(struct archive_read_filter *); int archive_read_support_compression_xz(struct archive *_a) @@ -145,6 +156,30 @@ archive_read_support_compression_lzma(struct archive *_a) #endif } +int +archive_read_support_compression_lzip(struct archive *_a) +{ + struct archive_read *a = (struct archive_read *)_a; + struct archive_read_filter_bidder *bidder = __archive_read_get_bidder(a); + + archive_clear_error(_a); + if (bidder == NULL) + return (ARCHIVE_FATAL); + + bidder->data = NULL; + bidder->bid = lzip_bidder_bid; + bidder->init = lzip_bidder_init; + bidder->options = NULL; + bidder->free = NULL; +#if HAVE_LZMA_H && HAVE_LIBLZMA + return (ARCHIVE_OK); +#else + archive_set_error(_a, ARCHIVE_ERRNO_MISC, + "Using external lzip program for lzip decompression"); + return (ARCHIVE_WARN); +#endif +} + /* * Test whether we can handle this data. */ @@ -307,6 +342,58 @@ lzma_bidder_bid(struct archive_read_filter_bidder *self, return (bits_checked); } +static int +lzip_has_member(struct archive_read_filter *filter) +{ + const unsigned char *buffer; + ssize_t avail; + int bits_checked; + int log2dic; + + buffer = __archive_read_filter_ahead(filter, 6, &avail); + if (buffer == NULL) + return (0); + + /* + * Verify Header Magic Bytes : 4C 5A 49 50 (`LZIP') + */ + bits_checked = 0; + if (buffer[0] != 0x4C) + return (0); + bits_checked += 8; + if (buffer[1] != 0x5A) + return (0); + bits_checked += 8; + if (buffer[2] != 0x49) + return (0); + bits_checked += 8; + if (buffer[3] != 0x50) + return (0); + bits_checked += 8; + + /* A version number must be 0 or 1 */ + if (buffer[4] != 0 && buffer[4] != 1) + return (0); + bits_checked += 8; + + /* Dictionary size. */ + log2dic = buffer[5] & 0x1f; + if (log2dic < 12 || log2dic > 27) + return (0); + bits_checked += 8; + + return (bits_checked); +} + +static int +lzip_bidder_bid(struct archive_read_filter_bidder *self, + struct archive_read_filter *filter) +{ + + (void)self; /* UNUSED */ + return (lzip_has_member(filter)); +} + #if HAVE_LZMA_H && HAVE_LIBLZMA /* @@ -328,39 +415,24 @@ lzma_bidder_init(struct archive_read_filter *self) return (xz_lzma_bidder_init(self)); } +static int +lzip_bidder_init(struct archive_read_filter *self) +{ + self->code = ARCHIVE_COMPRESSION_LZIP; + self->name = "lzip"; + return (xz_lzma_bidder_init(self)); +} + /* - * Setup the callbacks. + * Initialize lzma library. */ static int -xz_lzma_bidder_init(struct archive_read_filter *self) +init_lzma_stream(struct archive_read_filter *self) { - static const size_t out_block_size = 64 * 1024; - void *out_block; struct private_data *state; int ret; - state = (struct private_data *)calloc(sizeof(*state), 1); - out_block = (unsigned char *)malloc(out_block_size); - if (state == NULL || out_block == NULL) { - archive_set_error(&self->archive->archive, ENOMEM, - "Can't allocate data for xz decompression"); - free(out_block); - free(state); - return (ARCHIVE_FATAL); - } - - self->data = state; - state->out_block_size = out_block_size; - state->out_block = out_block; - self->read = xz_filter_read; - self->skip = NULL; /* not supported */ - self->close = xz_filter_close; - - state->stream.avail_in = 0; - - state->stream.next_out = state->out_block; - state->stream.avail_out = state->out_block_size; - + state = (struct private_data *)self->data; /* Initialize compression library. * TODO: I don't know what value is best for memlimit. * maybe, it needs to check memory size which @@ -395,6 +467,50 @@ xz_lzma_bidder_init(struct archive_read_filter *self) "Internal error initializing lzma library"); break; } + return (ARCHIVE_FATAL); +} + +/* + * Setup the callbacks. + */ +static int +xz_lzma_bidder_init(struct archive_read_filter *self) +{ + static const size_t out_block_size = 64 * 1024; + void *out_block; + struct private_data *state; + + state = (struct private_data *)calloc(sizeof(*state), 1); + out_block = (unsigned char *)malloc(out_block_size); + if (state == NULL || out_block == NULL) { + archive_set_error(&self->archive->archive, ENOMEM, + "Can't allocate data for xz decompression"); + free(out_block); + free(state); + return (ARCHIVE_FATAL); + } + + self->data = state; + state->out_block_size = out_block_size; + state->out_block = out_block; + self->read = xz_filter_read; + self->skip = NULL; /* not supported */ + self->close = xz_filter_close; + + state->stream.avail_in = 0; + + state->stream.next_out = state->out_block; + state->stream.avail_out = state->out_block_size; + + if (self->code == ARCHIVE_COMPRESSION_LZIP) + state->in_stream = 0; + else + state->in_stream = 1; + state->crc32 = 0; + + /* Initialize compression library. */ + if (init_lzma_stream(self) == ARCHIVE_OK) + return (ARCHIVE_OK); free(state->out_block); free(state); @@ -402,6 +518,66 @@ xz_lzma_bidder_init(struct archive_read_filter *self) return (ARCHIVE_FATAL); } +static int +lzip_tail(struct archive_read_filter *self) +{ + struct private_data *state; + const unsigned char *f; + ssize_t avail_in; + int tail; + + state = (struct private_data *)self->data; + if (state->lzip_ver == 0) + tail = 12; + else + tail = 20; + f = __archive_read_filter_ahead(self->upstream, tail, &avail_in); + if (f == NULL && avail_in < 0) + return (ARCHIVE_FATAL); + if (avail_in < tail) { + archive_set_error(&self->archive->archive, ARCHIVE_ERRNO_MISC, + "Lzip: Remaining data is less bytes"); + return (ARCHIVE_FAILED); + } + + /* Check the crc32 value of the uncompressed data of the current + * member */ + if (state->crc32 != archive_le32dec(f)) { + archive_set_error(&self->archive->archive, ARCHIVE_ERRNO_MISC, + "Lzip: CRC32 error"); + return (ARCHIVE_FAILED); + } + + /* Check the uncompressed size of the current member */ + if ((uint64_t)state->member_out != archive_le64dec(f + 4)) { + archive_set_error(&self->archive->archive, ARCHIVE_ERRNO_MISC, + "Lzip: Uncompressed size error"); + return (ARCHIVE_FAILED); + } + + /* Check the total size of the current member */ + if (state->lzip_ver == 1 && + (uint64_t)state->member_in + tail != archive_le64dec(f + 12)) { + archive_set_error(&self->archive->archive, ARCHIVE_ERRNO_MISC, + "Lzip: Member size error"); + return (ARCHIVE_FAILED); + } + __archive_read_filter_consume(self->upstream, tail); + + /* If current lzip data consists of multi member, try decompressing + * a next member. */ + if (lzip_has_member(self->upstream) != 0) { + state->in_stream = 0; + state->crc32 = 0; + state->member_out = 0; + state->member_in = 0; + state->eof = 0; + if (init_lzma_stream(self) != ARCHIVE_OK) + return (ARCHIVE_FATAL); + } + return (ARCHIVE_OK); +} + /* * Return the next block of decompressed data. */ @@ -411,6 +587,7 @@ xz_filter_read(struct archive_read_filter *self, const void **p) struct private_data *state; size_t decompressed; ssize_t avail_in; + unsigned char props[13]; int ret; state = (struct private_data *)self->data; @@ -421,10 +598,40 @@ xz_filter_read(struct archive_read_filter *self, const void **p) /* Try to fill the output buffer. */ while (state->stream.avail_out > 0 && !state->eof) { - state->stream.next_in = - __archive_read_filter_ahead(self->upstream, 1, &avail_in); - if (state->stream.next_in == NULL && avail_in < 0) - return (ARCHIVE_FATAL); + if (!state->in_stream) { + /* + * Make a lzma header from a lzip header + */ + uint32_t dicsize; + const unsigned char *h; + int log2dic; + + h = __archive_read_filter_ahead(self->upstream, 6, + &avail_in); + if (h == NULL && avail_in < 0) + return (ARCHIVE_FATAL); + + state->lzip_ver = h[4]; + props[0] = 0x5d; + log2dic = h[5] & 0x1f; + if (log2dic < 12 || log2dic > 27) + return (ARCHIVE_FATAL); + dicsize = 1U << log2dic; + if (log2dic > 12) + dicsize -= (dicsize / 16) * (h[5] >> 5); + archive_le32enc(props+1, dicsize); + memset(props+5, 0xff, 8); + state->stream.next_in = props; + avail_in = sizeof(props); + __archive_read_filter_consume(self->upstream, 6); + state->member_in = 6; + } else { + state->stream.next_in = + __archive_read_filter_ahead(self->upstream, 1, + &avail_in); + if (state->stream.next_in == NULL && avail_in < 0) + return (ARCHIVE_FATAL); + } state->stream.avail_in = avail_in; /* Decompress as much as we can in one pass. */ @@ -435,8 +642,14 @@ xz_filter_read(struct archive_read_filter *self, const void **p) state->eof = 1; /* FALL THROUGH */ case LZMA_OK: /* Decompressor made some progress. */ - __archive_read_filter_consume(self->upstream, - avail_in - state->stream.avail_in); + if (!state->in_stream) + state->in_stream = 1; + else { + __archive_read_filter_consume(self->upstream, + avail_in - state->stream.avail_in); + state->member_in += + avail_in - state->stream.avail_in; + } break; case LZMA_MEM_ERROR: archive_set_error(&self->archive->archive, ENOMEM, @@ -477,10 +690,21 @@ xz_filter_read(struct archive_read_filter *self, const void **p) decompressed = state->stream.next_out - state->out_block; state->total_out += decompressed; + state->member_out += decompressed; if (decompressed == 0) *p = NULL; - else + else { *p = state->out_block; + if (self->code == ARCHIVE_COMPRESSION_LZIP) { + state->crc32 = lzma_crc32(state->out_block, + decompressed, state->crc32); + if (state->eof) { + ret = lzip_tail(self); + if (ret != ARCHIVE_OK) + return (ret); + } + } + } return (decompressed); } @@ -704,5 +928,19 @@ xz_bidder_init(struct archive_read_filter *self) return (r); } +static int +lzip_bidder_init(struct archive_read_filter *self) +{ + int r; + + r = __archive_read_program(self, "unlzip"); + /* Note: We set the format here even if __archive_read_program() + * above fails. We do, after all, know what the format is + * even if we weren't able to read it. */ + self->code = ARCHIVE_COMPRESSION_LZIP; + self->name = "lzip"; + return (r); +} + #endif /* HAVE_LZMA_H */ diff --git a/libarchive/test/CMakeLists.txt b/libarchive/test/CMakeLists.txt index 57009f9aa..e6282a90e 100644 --- a/libarchive/test/CMakeLists.txt +++ b/libarchive/test/CMakeLists.txt @@ -46,6 +46,7 @@ IF(ENABLE_TEST) test_read_format_cpio_bin_be.c test_read_format_cpio_bin_bz2.c test_read_format_cpio_bin_gz.c + test_read_format_cpio_bin_lzip.c test_read_format_cpio_bin_lzma.c test_read_format_cpio_bin_xz.c test_read_format_cpio_odc.c diff --git a/libarchive/test/test_read_format_cpio_bin_lzip.c b/libarchive/test/test_read_format_cpio_bin_lzip.c new file mode 100644 index 000000000..bb92f18ec --- /dev/null +++ b/libarchive/test/test_read_format_cpio_bin_lzip.c @@ -0,0 +1,61 @@ +/*- + * Copyright (c) 2010 Michihiro NAKAJIMA + * Copyright (c) 2003-2007 Tim Kientzle + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include "test.h" +__FBSDID("$FreeBSD$"); + +static unsigned char archive[] = { + 76, 90, 73, 80, 1, 12, 0, 99,156, 62,160, 67,124,230, 93,220, +235,118, 29, 75, 27,226,158, 67,149,151, 96, 22, 54,198,209, 63, +104,209,148,249,238, 71,187,201,243,162, 1, 42, 47, 43,178, 35, + 90, 6,156,208, 74,107, 91,229,126, 5, 85,255,136,255, 64, 0, +170,199,228,195, 0, 2, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, + 0, 0, 0, 0 +}; + +DEFINE_TEST(test_read_format_cpio_bin_lzip) +{ + struct archive_entry *ae; + struct archive *a; + int r; + + assert((a = archive_read_new()) != NULL); + assertEqualIntA(a, ARCHIVE_OK, archive_read_support_compression_all(a)); + r = archive_read_support_compression_lzip(a); + if (r == ARCHIVE_WARN) { + skipping("lzip reading not fully supported on this platform"); + assertEqualInt(ARCHIVE_OK, archive_read_free(a)); + return; + } + assertEqualIntA(a, ARCHIVE_OK, archive_read_support_format_all(a)); + assertEqualIntA(a, ARCHIVE_OK, + archive_read_open_memory(a, archive, sizeof(archive))); + assertEqualIntA(a, ARCHIVE_OK, archive_read_next_header(a, &ae)); + assertEqualInt(archive_compression(a), ARCHIVE_COMPRESSION_LZIP); + assertEqualInt(archive_format(a), ARCHIVE_FORMAT_CPIO_BIN_LE); + assertEqualIntA(a, ARCHIVE_OK, archive_read_close(a)); + assertEqualInt(ARCHIVE_OK, archive_read_free(a)); +} +