From: Michihiro NAKAJIMA Date: Sat, 31 Jan 2009 06:31:15 +0000 (-0500) Subject: Add support xz format and compressing lzma format. X-Git-Tag: v2.7.0~372 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=c25e0da2e8d6091bea7ff1c86940808a0bd79eef;p=thirdparty%2Flibarchive.git Add support xz format and compressing lzma format. SVN-Revision: 521 --- diff --git a/CMakeLists.txt b/CMakeLists.txt index a4d981910..628ebf1e1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -154,10 +154,15 @@ ENDIF(BZIP2_FOUND) # FIND_PACKAGE(LZMA) IF(LZMA_FOUND) - SET(HAVE_LIBLZMADEC 1) - SET(HAVE_LZMADEC_H 1) + SET(HAVE_LIBLZMA 1) + SET(HAVE_LZMA_H 1) INCLUDE_DIRECTORIES(${LZMA_INCLUDE_DIR}) LIST(APPEND ADDITIONAL_LIBS ${LZMA_LIBRARIES}) +ELSEIF(LZMADEC_FOUND) + SET(HAVE_LIBLZMADEC 1) + SET(HAVE_LZMADEC_H 1) + INCLUDE_DIRECTORIES(${LZMADEC_INCLUDE_DIR}) + LIST(APPEND ADDITIONAL_LIBS ${LZMADEC_LIBRARIES}) ENDIF(LZMA_FOUND) # diff --git a/Makefile.am b/Makefile.am index 53faaa59f..25452f501 100644 --- a/Makefile.am +++ b/Makefile.am @@ -103,6 +103,7 @@ libarchive_la_SOURCES= \ libarchive/archive_read_support_compression_none.c \ libarchive/archive_read_support_compression_program.c \ libarchive/archive_read_support_compression_lzma.c \ + libarchive/archive_read_support_compression_xz.c \ libarchive/archive_read_support_format_all.c \ libarchive/archive_read_support_format_ar.c \ libarchive/archive_read_support_format_cpio.c \ @@ -130,6 +131,7 @@ libarchive_la_SOURCES= \ libarchive/archive_write_set_compression_gzip.c \ libarchive/archive_write_set_compression_none.c \ libarchive/archive_write_set_compression_program.c \ + libarchive/archive_write_set_compression_xz.c \ libarchive/archive_write_set_format.c \ libarchive/archive_write_set_format_ar.c \ libarchive/archive_write_set_format_by_name.c \ diff --git a/cmake/FindLZMA.cmake b/cmake/FindLZMA.cmake index a33c66734..1d065c4ab 100644 --- a/cmake/FindLZMA.cmake +++ b/cmake/FindLZMA.cmake @@ -1,17 +1,20 @@ -# - Find lzma +# - Find lzma and lzmadec # Find the native LZMA includes and library # -# LZMA_INCLUDE_DIR - where to find zlib.h, etc. -# LZMA_LIBRARIES - List of libraries when using zlib. -# LZMA_FOUND - True if zlib found. +# LZMA_INCLUDE_DIR - where to find lzma.h, etc. +# LZMA_LIBRARIES - List of libraries when using liblzma. +# LZMA_FOUND - True if liblzma found. +# LZMADEC_INCLUDE_DIR - where to find lzmadec.h, etc. +# LZMADEC_LIBRARIES - List of libraries when using liblzmadec. +# LZMADEC_FOUND - True if liblzmadec found. IF (LZMA_INCLUDE_DIR) # Already in cache, be silent SET(LZMA_FIND_QUIETLY TRUE) ENDIF (LZMA_INCLUDE_DIR) -FIND_PATH(LZMA_INCLUDE_DIR lzmadec.h) -FIND_LIBRARY(LZMA_LIBRARY NAMES lzmadec ) +FIND_PATH(LZMA_INCLUDE_DIR lzma.h) +FIND_LIBRARY(LZMA_LIBRARY NAMES lzma ) # handle the QUIETLY and REQUIRED arguments and set LZMA_FOUND to TRUE if # all listed variables are TRUE @@ -22,6 +25,28 @@ IF(LZMA_FOUND) SET( LZMA_LIBRARIES ${LZMA_LIBRARY} ) ELSE(LZMA_FOUND) SET( LZMA_LIBRARIES ) + + IF (LZMADEC_INCLUDE_DIR) + # Already in cache, be silent + SET(LZMADEC_FIND_QUIETLY TRUE) + ENDIF (LZMADEC_INCLUDE_DIR) + + FIND_PATH(LZMADEC_INCLUDE_DIR lzmadec.h) + FIND_LIBRARY(LZMADEC_LIBRARY NAMES lzmadec ) + + # handle the QUIETLY and REQUIRED arguments and set LZMADEC_FOUND to TRUE if + # all listed variables are TRUE + INCLUDE(FindPackageHandleStandardArgs) + FIND_PACKAGE_HANDLE_STANDARD_ARGS(LZMADEC DEFAULT_MSG LZMADEC_LIBRARY + LZMADEC_INCLUDE_DIR) + + IF(LZMADEC_FOUND) + SET( LZMADEC_LIBRARIES ${LZMADEC_LIBRARY} ) + ELSE(LZMADEC_FOUND) + SET( LZMADEC_LIBRARIES ) + ENDIF(LZMADEC_FOUND) ENDIF(LZMA_FOUND) -MARK_AS_ADVANCED( LZMA_LIBRARY LZMA_INCLUDE_DIR ) + +MARK_AS_ADVANCED( LZMA_LIBRARY LZMA_INCLUDE_DIR + LZMADEC_LIBRARY LZMADEC_INCLUDE_DIR ) diff --git a/cmake/config.h.in b/cmake/config.h.in index 892a0ec9d..2a66fd5d3 100644 --- a/cmake/config.h.in +++ b/cmake/config.h.in @@ -211,6 +211,9 @@ /* Define to 1 if you have the `bz2' library (-lbz2). */ #cmakedefine HAVE_LIBBZ2 1 +/* Define to 1 if you have the `lzma' library (-llzma). */ +#cmakedefine HAVE_LIBLZMA 1 + /* Define to 1 if you have the `lzmadec' library (-llzmadec). */ #cmakedefine HAVE_LIBLZMADEC 1 @@ -248,6 +251,9 @@ /* Define to 1 if you have the header file. */ #cmakedefine HAVE_LZMADEC_H 1 +/* Define to 1 if you have the header file. */ +#cmakedefine HAVE_LZMA_H 1 + /* Define to 1 if you have the `MD5' functions. */ #cmakedefine HAVE_MD5 1 diff --git a/configure.ac b/configure.ac index 40806f70c..6e4bc050f 100644 --- a/configure.ac +++ b/configure.ac @@ -195,6 +195,14 @@ if test "x$with_lzmadec" != "xno"; then AC_CHECK_LIB(lzmadec,lzmadec_decode) fi +AC_ARG_WITH([lzma], + AS_HELP_STRING([--without-lzma], [Don't build support for xz through lzma])) + +if test "x$with_lzma" != "xno"; then + AC_CHECK_HEADERS([lzma.h]) + AC_CHECK_LIB(lzma,lzma_stream_decoder) +fi + AC_CHECK_HEADERS([openssl/md5.h openssl/ripemd.h openssl/sha.h]) AC_CHECK_HEADERS([md5.h md5global.h]) AC_CHECK_HEADERS([ripemd.h rmd160.h]) diff --git a/libarchive/archive.h b/libarchive/archive.h index 8e0ad3ebe..efca42b2e 100644 --- a/libarchive/archive.h +++ b/libarchive/archive.h @@ -231,6 +231,7 @@ typedef int archive_close_callback(struct archive *, void *_client_data); #define ARCHIVE_COMPRESSION_COMPRESS 3 #define ARCHIVE_COMPRESSION_PROGRAM 4 #define ARCHIVE_COMPRESSION_LZMA 5 +#define ARCHIVE_COMPRESSION_XZ 6 /* * Codes returned by archive_format. @@ -301,6 +302,7 @@ __LA_DECL int archive_read_support_compression_lzma(struct archive *); __LA_DECL int archive_read_support_compression_none(struct archive *); __LA_DECL int archive_read_support_compression_program(struct archive *, const char *command); +__LA_DECL int archive_read_support_compression_xz(struct archive *); __LA_DECL int archive_read_support_format_all(struct archive *); __LA_DECL int archive_read_support_format_ar(struct archive *); @@ -496,9 +498,11 @@ __LA_DECL int archive_write_set_skip_file(struct archive *, dev_t, ino_t); __LA_DECL int archive_write_set_compression_bzip2(struct archive *); __LA_DECL int archive_write_set_compression_compress(struct archive *); __LA_DECL int archive_write_set_compression_gzip(struct archive *); +__LA_DECL int archive_write_set_compression_lzma(struct archive *); __LA_DECL int archive_write_set_compression_none(struct archive *); __LA_DECL int archive_write_set_compression_program(struct archive *, const char *cmd); +__LA_DECL int archive_write_set_compression_xz(struct archive *); /* A convenience function to set the format based on the code or name. */ __LA_DECL int archive_write_set_format(struct archive *, int format_code); __LA_DECL int archive_write_set_format_by_name(struct archive *, diff --git a/libarchive/archive_read_support_compression_all.c b/libarchive/archive_read_support_compression_all.c index b56951620..50dca38e9 100644 --- a/libarchive/archive_read_support_compression_all.c +++ b/libarchive/archive_read_support_compression_all.c @@ -44,6 +44,9 @@ archive_read_support_compression_all(struct archive *a) * the LZMA file format has a very weak signature. It * may not be feasible to include LZMA detection here. */ /* archive_read_support_compression_lzma(a); */ +#endif +#if HAVE_LZMA_H + archive_read_support_compression_xz(a); #endif return (ARCHIVE_OK); } diff --git a/libarchive/archive_read_support_compression_lzma.c b/libarchive/archive_read_support_compression_lzma.c index b19b70156..7b91d71d0 100644 --- a/libarchive/archive_read_support_compression_lzma.c +++ b/libarchive/archive_read_support_compression_lzma.c @@ -48,6 +48,7 @@ __FBSDID("$FreeBSD$"); #include "archive_private.h" #include "archive_read_private.h" +#ifndef HAVE_LZMA_H #if HAVE_LZMADEC_H struct private_data { lzmadec_stream stream; @@ -352,3 +353,4 @@ lzma_filter_close(struct archive_read_filter *self) } #endif /* HAVE_LZMADEC_H */ +#endif /* HAVE_LZMA_H */ diff --git a/libarchive/archive_read_support_compression_xz.c b/libarchive/archive_read_support_compression_xz.c new file mode 100644 index 000000000..0b424c393 --- /dev/null +++ b/libarchive/archive_read_support_compression_xz.c @@ -0,0 +1,459 @@ +/*- + * Copyright (c) 2009 Michihiro NAKAJIMA + * Copyright (c) 2003-2008 Tim Kientzle and Miklos Vajna + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "archive_platform.h" + +__FBSDID("$FreeBSD$"); + +#ifdef HAVE_ERRNO_H +#include +#endif +#include +#ifdef HAVE_STDLIB_H +#include +#endif +#ifdef HAVE_STRING_H +#include +#endif +#ifdef HAVE_UNISTD_H +#include +#endif +#ifdef HAVE_LZMA_H +#include +#endif + +#include "archive.h" +#include "archive_private.h" +#include "archive_read_private.h" + +#if HAVE_LZMA_H +struct private_data { + lzma_stream stream; + unsigned char *out_block; + size_t out_block_size; + int64_t total_out; + char eof; /* True = found end of compressed data. */ +}; + +/* Lzma filter */ +static ssize_t xz_filter_read(struct archive_read_filter *, const void **); +static int xz_filter_close(struct archive_read_filter *); +#endif + +/* + * Note that we can detect xz archives even if we can't decompress + * them. (In fact, we like detecting them because we can give better + * error messages.) So the bid framework here gets compiled even + * if lzma is unavailable. + */ +static int xz_lzma_bidder_init(struct archive_read_filter *, int code); +#ifndef HAVE_LZMADEC_H +static int lzma_bidder_bid(struct archive_read_filter_bidder *, + struct archive_read_filter *); +static int lzma_bidder_init(struct archive_read_filter *); +#endif +static int xz_bidder_bid(struct archive_read_filter_bidder *, + struct archive_read_filter *); +static int xz_bidder_init(struct archive_read_filter *); +static int xz_bidder_free(struct archive_read_filter_bidder *); + +int +archive_read_support_compression_xz(struct archive *_a) +{ + struct archive_read *a = (struct archive_read *)_a; + struct archive_read_filter_bidder *bidder = __archive_read_get_bidder(a); + + if (bidder == NULL) + return (ARCHIVE_FATAL); + + bidder->data = NULL; + bidder->bid = xz_bidder_bid; + bidder->init = xz_bidder_init; + bidder->options = NULL; + bidder->free = xz_bidder_free; + return (ARCHIVE_OK); +} + +#ifndef HAVE_LZMADEC_H +int +archive_read_support_compression_lzma(struct archive *_a) +{ + struct archive_read *a = (struct archive_read *)_a; + struct archive_read_filter_bidder *bidder = __archive_read_get_bidder(a); + + if (bidder == NULL) + return (ARCHIVE_FATAL); + + bidder->data = NULL; + bidder->bid = lzma_bidder_bid; + bidder->init = lzma_bidder_init; + bidder->options = NULL; + bidder->free = xz_bidder_free; + return (ARCHIVE_OK); +} +#endif + +static int +xz_bidder_free(struct archive_read_filter_bidder *self){ + (void)self; /* UNUSED */ + return (ARCHIVE_OK); +} + +/* + * Test whether we can handle this data. + * + * This logic returns zero if any part of the signature fails. It + * also tries to Do The Right Thing if a very short buffer prevents us + * from verifying as much as we would like. + * + */ +static int +xz_bidder_bid(struct archive_read_filter_bidder *self, + struct archive_read_filter *filter) +{ + const unsigned char *buffer; + size_t avail; + int bits_checked; + + (void)self; /* UNUSED */ + + buffer = __archive_read_filter_ahead(filter, 6, &avail); + if (buffer == NULL) + return (0); + + /* + * Verify Header Magic Bytes : FD 37 7A 58 5A 00 + */ + bits_checked = 0; + if (buffer[0] != 0xFD) + return (0); + bits_checked += 8; + if (buffer[1] != 0x37) + return (0); + bits_checked += 8; + if (buffer[2] != 0x7A) + return (0); + bits_checked += 8; + if (buffer[3] != 0x58) + return (0); + bits_checked += 8; + if (buffer[4] != 0x5A) + return (0); + bits_checked += 8; + if (buffer[5] != 0x00) + return (0); + bits_checked += 8; + + return (bits_checked); +} + +#ifndef HAVE_LZMADEC_H +/* + * Test whether we can handle this data. + * + * This logic returns zero if any part of the signature fails. It + * also tries to Do The Right Thing if a very short buffer prevents us + * from verifying as much as we would like. + * + * LZMA has a rather poor file signature. Zeros do not + * make good signature bytes as a rule, and the only non-zero byte + * here is an ASCII character. For example, an uncompressed tar + * archive whose first file is ']' would satisfy this check. It may + * be necessary to exclude LZMA from compression_all() because of + * this. Clients of libarchive would then have to explicitly enable + * LZMA checking instead of (or in addition to) compression_all() when + * they have other evidence (file name, command-line option) to go on. + */ +static int +lzma_bidder_bid(struct archive_read_filter_bidder *self, + struct archive_read_filter *filter) +{ + const unsigned char *buffer; + size_t avail; + int bits_checked; + + (void)self; /* UNUSED */ + + buffer = __archive_read_filter_ahead(filter, 6, &avail); + if (buffer == NULL) + return (0); + + /* First byte of raw LZMA stream is always 0x5d. */ + bits_checked = 0; + if (buffer[0] != 0x5d) + return (0); + bits_checked += 8; + + /* Second through fifth bytes are dictionary code, stored in + * little-endian order. The two least-significant bytes are + * always zero. */ + if (buffer[1] != 0 || buffer[2] != 0) + return (0); + bits_checked += 16; + + /* ??? TODO: Fix this. ??? */ + /* NSIS format check uses this, but I've seen tar.lzma + * archives where this byte is 0xff, not 0. Can it + * ever be anything other than 0 or 0xff? + */ +#if 0 + if (buffer[5] != 0) + return (0); + bits_checked += 8; +#endif + + /* TODO: The above test is still very weak. It would be + * good to do better. */ + + return (bits_checked); +} +#endif + +#ifndef HAVE_LZMA_H + +/* + * If we don't have the library on this system, we can't actually do the + * decompression. We can, however, still detect compressed archives + * and emit a useful message. + */ +static int +xz_bidder_init(struct archive_read_filter *filter) +{ + (void)filter; /* UNUSED */ + + archive_set_error(&filter->archive->archive, -1, + "This version of libarchive was compiled without xz support"); + return (ARCHIVE_FATAL); +} + + +#else + +static int +xz_bidder_init(struct archive_read_filter *self) +{ + return (xz_lzma_bidder_init(self, ARCHIVE_COMPRESSION_XZ)); +} + +#ifndef HAVE_LZMADEC_H +static int +lzma_bidder_init(struct archive_read_filter *self) +{ + return (xz_lzma_bidder_init(self, ARCHIVE_COMPRESSION_LZMA)); +} +#endif + +/* + * Setup the callbacks. + */ +static int +xz_lzma_bidder_init(struct archive_read_filter *self, int code) +{ + static const size_t out_block_size = 64 * 1024; + void *out_block; + const char *buff; + struct private_data *state; + int ret; + + self->code = code; + self->name = (code == ARCHIVE_COMPRESSION_XZ)?"xz": "lzma"; + + state = (struct private_data *)calloc(sizeof(*state), 1); + out_block = (unsigned char *)malloc(out_block_size); + if (state == NULL || out_block == NULL) { + archive_set_error(&self->archive->archive, ENOMEM, + "Can't allocate data for xz decompression"); + free(out_block); + free(state); + return (ARCHIVE_FATAL); + } + + self->data = state; + state->out_block_size = out_block_size; + state->out_block = out_block; + self->read = xz_filter_read; + self->skip = NULL; /* not supported */ + self->close = xz_filter_close; + + state->stream.next_in = buff; + state->stream.avail_in = ret; + + state->stream.next_out = state->out_block; + state->stream.avail_out = state->out_block_size; + + /* Initialize compression library. + * TODO: I don't know what value is best for memlimit. + * maybe, it needs to check memory size which + * running system has. + */ + if (code == ARCHIVE_COMPRESSION_XZ) + ret = lzma_stream_decoder(&(state->stream), + (1U << 23) + (1U << 21),/* memlimit */ + LZMA_CONCATENATED); + else + lzma_alone_decoder(&(state->stream), + (1U << 23) + (1U << 21));/* memlimit */ + + if (ret == LZMA_OK) + return (ARCHIVE_OK); + + /* Library setup failed: Clean up. */ + archive_set_error(&self->archive->archive, ARCHIVE_ERRNO_MISC, + "Internal error initializing lzma library"); + + /* Override the error message if we know what really went wrong. */ + switch (ret) { + case LZMA_MEM_ERROR: + archive_set_error(&self->archive->archive, ENOMEM, + "Internal error initializing compression library: " + "Cannot allocate memory"); + break; + case LZMA_OPTIONS_ERROR: + archive_set_error(&self->archive->archive, + ARCHIVE_ERRNO_MISC, + "Internal error initializing compression library: " + "Invalid or unsupported options"); + break; + } + + free(state->out_block); + free(state); + self->data = NULL; + return (ARCHIVE_FATAL); +} + +/* + * Return the next block of decompressed data. + */ +static ssize_t +xz_filter_read(struct archive_read_filter *self, const void **p) +{ + struct private_data *state; + size_t read_avail, decompressed; + const void *read_buf; + int ret; + + state = (struct private_data *)self->data; + read_avail = 0; + + /* Empty our output buffer. */ + state->stream.next_out = state->out_block; + state->stream.avail_out = state->out_block_size; + + /* Try to fill the output buffer. */ + while (state->stream.avail_out > 0 && !state->eof) { + /* If the last upstream block is done, get another one. */ + if (state->stream.avail_in == 0) { + read_buf = __archive_read_filter_ahead(self->upstream, + 1, &ret); + if (ret == 0) { + state->eof = 1; + break; + } + if (read_buf == NULL || ret < 0) + return (ARCHIVE_FATAL); + /* stream.next_in is really const, but lzma + * doesn't declare it so. */ + state->stream.next_in + = (unsigned char *)(uintptr_t)read_buf; + state->stream.avail_in = ret; + __archive_read_filter_consume(self->upstream, ret); + } + + /* Decompress as much as we can in one pass. */ + ret = lzma_code(&(state->stream), + (state->stream.avail_in == 0)? LZMA_FINISH: LZMA_RUN); + switch (ret) { + case LZMA_STREAM_END: /* Found end of stream. */ + state->eof = 1; + case LZMA_OK: /* Decompressor made some progress. */ + break; + case LZMA_MEM_ERROR: + archive_set_error(&self->archive->archive, ENOMEM, + "Internal error decompressing compression library: " + "Cannot allocate memory"); + break; + case LZMA_MEMLIMIT_ERROR: + archive_set_error(&self->archive->archive, ENOMEM, + "Internal error decompressing compression library: " + "Memory usage limit was reached"); + break; + case LZMA_FORMAT_ERROR: + archive_set_error(&self->archive->archive, + ARCHIVE_ERRNO_MISC, + "Internal error decompressing compression library: " + "File format not recognized"); + break; + case LZMA_OPTIONS_ERROR: + archive_set_error(&self->archive->archive, + ARCHIVE_ERRNO_MISC, + "Internal error decompressing compression library: " + "Invalid or unsupported options"); + break; + case LZMA_DATA_ERROR: + archive_set_error(&self->archive->archive, + ARCHIVE_ERRNO_MISC, + "Internal error decompressing compression library: " + "Data is corrupt"); + break; + case LZMA_BUF_ERROR: + archive_set_error(&self->archive->archive, + ARCHIVE_ERRNO_MISC, + "Internal error decompressing compression library: " + "No progress is possible"); + return (ARCHIVE_FATAL); + default: + /* Return an error. */ + archive_set_error(&self->archive->archive, + ARCHIVE_ERRNO_MISC, + "%s decompression failed", + self->archive->archive.compression_name); + return (ARCHIVE_FATAL); + } + } + + *p = state->out_block; + decompressed = state->stream.next_out - state->out_block; + state->total_out += decompressed; + return (decompressed); +} + +/* + * Clean up the decompressor. + */ +static int +xz_filter_close(struct archive_read_filter *self) +{ + struct private_data *state; + + state = (struct private_data *)self->data; + lzma_end(&(state->stream)); + free(state->out_block); + free(state); + return (ARCHIVE_OK); +} + +#endif /* HAVE_LZMA_H */ diff --git a/libarchive/archive_write_set_compression_xz.c b/libarchive/archive_write_set_compression_xz.c new file mode 100644 index 000000000..55c10fe19 --- /dev/null +++ b/libarchive/archive_write_set_compression_xz.c @@ -0,0 +1,453 @@ +/*- + * Copyright (c) 2009 Michihiro NAKAJIMA + * Copyright (c) 2003-2007 Tim Kientzle + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "archive_platform.h" + +__FBSDID("$FreeBSD$"); + +#ifdef HAVE_CTYPE_H +#include +#endif +#ifdef HAVE_ERRNO_H +#include +#endif +#ifdef HAVE_STDLIB_H +#include +#endif +#ifdef HAVE_STRING_H +#include +#endif +#include +#ifdef HAVE_LZMA_H +#include +#endif + +#include "archive.h" +#include "archive_private.h" +#include "archive_write_private.h" + +#ifndef HAVE_LZMA_H +int +archive_write_set_compression_xz(struct archive *_a) +{ + /* Unsupported xz compression, we don't have liblzma */ + return (ARCHIVE_FATAL); +} +#else +/* Don't compile this if we don't have liblzma. */ + +struct private_data { + lzma_stream stream; + lzma_filter lzmafilters[2]; + lzma_options_lzma lzma_opt; + int64_t total_in; + unsigned char *compressed; + size_t compressed_buffer_size; + /* Options */ + uint32_t compression_level; +}; + + +static int archive_compressor_xz_lzma_init(struct archive_write *, int); +static int archive_compressor_lzma_init(struct archive_write *); +static int archive_compressor_xz_finish(struct archive_write *); +static int archive_compressor_xz_init(struct archive_write *); +static int archive_compressor_xz_options(struct archive_write *, + const char *, const char *); +static int archive_compressor_xz_write(struct archive_write *, + const void *, size_t); +static int drive_compressor(struct archive_write *, struct private_data *, + int finishing); + + +/* + * Allocate, initialize and return a archive object. + */ +int +archive_write_set_compression_xz(struct archive *_a) +{ + struct archive_write *a = (struct archive_write *)_a; + __archive_check_magic(&a->archive, ARCHIVE_WRITE_MAGIC, + ARCHIVE_STATE_NEW, "archive_write_set_compression_xz"); + a->compressor.init = &archive_compressor_xz_init; + return (ARCHIVE_OK); +} + +int +archive_write_set_compression_lzma(struct archive *_a) +{ + struct archive_write *a = (struct archive_write *)_a; + __archive_check_magic(&a->archive, ARCHIVE_WRITE_MAGIC, + ARCHIVE_STATE_NEW, "archive_write_set_compression_lzma"); + a->compressor.init = &archive_compressor_lzma_init; + return (ARCHIVE_OK); +} + +static int +archive_compressor_xz_init(struct archive_write *a) +{ + return (archive_compressor_xz_lzma_init(a, ARCHIVE_COMPRESSION_XZ)); +} + +static int +archive_compressor_lzma_init(struct archive_write *a) +{ + return (archive_compressor_xz_lzma_init(a, ARCHIVE_COMPRESSION_LZMA)); +} + +static int +archive_compressor_xz_init_stream(struct archive_write *a) +{ + struct private_data *state; + int ret; + + state = (struct private_data *)a->compressor.data; + + state->stream = (lzma_stream)LZMA_STREAM_INIT; + state->stream.next_out = state->compressed; + state->stream.avail_out = state->compressed_buffer_size; + if (a->archive.compression_code == ARCHIVE_COMPRESSION_XZ) + ret = lzma_stream_encoder(&(state->stream), + state->lzmafilters, LZMA_CHECK_CRC64); + else + ret = lzma_alone_encoder(&(state->stream), &state->lzma_opt); + if (ret == LZMA_OK) + return (ARCHIVE_OK); + + switch (ret) { + case LZMA_MEM_ERROR: + archive_set_error(&a->archive, ENOMEM, + "Internal error initializing compression library: " + "Cannot allocate memory"); + break; + default: + archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, + "Internal error initializing compression library: " + "It's a bug in liblzma"); + break; + } + return (ARCHIVE_FATAL); +} + +/* + * Setup callback. + */ +static int +archive_compressor_xz_lzma_init(struct archive_write *a, int code) +{ + int ret; + struct private_data *state; + time_t t; + + a->archive.compression_code = code; + a->archive.compression_name = + (code == ARCHIVE_COMPRESSION_XZ)? "xz" : "lzma"; + + if (a->client_opener != NULL) { + ret = (a->client_opener)(&a->archive, a->client_data); + if (ret != ARCHIVE_OK) + return (ret); + } + + state = (struct private_data *)malloc(sizeof(*state)); + if (state == NULL) { + archive_set_error(&a->archive, ENOMEM, + "Can't allocate data for compression"); + return (ARCHIVE_FATAL); + } + memset(state, 0, sizeof(*state)); + + /* + * See comment above. We should set compressed_buffer_size to + * max(bytes_per_block, 65536), but the code can't handle that yet. + */ + state->compressed_buffer_size = a->bytes_per_block; + state->compressed = (unsigned char *)malloc(state->compressed_buffer_size); + state->compression_level = LZMA_PRESET_DEFAULT; + + if (state->compressed == NULL) { + archive_set_error(&a->archive, ENOMEM, + "Can't allocate data for compression buffer"); + free(state); + return (ARCHIVE_FATAL); + } + a->compressor.options = archive_compressor_xz_options; + a->compressor.write = archive_compressor_xz_write; + a->compressor.finish = archive_compressor_xz_finish; + + /* Initialize compression library. */ + if (lzma_lzma_preset(&state->lzma_opt, state->compression_level)) { + archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, + "Internal error initializing compression library"); + free(state->compressed); + free(state); + } + state->lzmafilters[0].id = LZMA_FILTER_LZMA2; + state->lzmafilters[0].options = &state->lzma_opt; + state->lzmafilters[1].id = LZMA_VLI_UNKNOWN;/* Terminate */ + ret = archive_compressor_xz_init_stream(a); + if (ret == LZMA_OK) { + a->compressor.data = state; + return (0); + } + /* Library setup failed: clean up. */ + free(state->compressed); + free(state); + + return (ARCHIVE_FATAL); +} + +/* + * Set write options. + */ +static int +archive_compressor_xz_options(struct archive_write *a, const char *key, + const char *value) +{ + struct private_data *state; + int ret; + + state = (struct private_data *)a->compressor.data; + if (strcmp(key, "compression-level") == 0) { + /* + * compression level : 0 .. 9 + */ + int level; + + if (value == NULL || !isdigit(value[0]) || value[1] != '\0') + return (ARCHIVE_WARN); + level = value[0] - '0'; + if (level == state->compression_level) + return (ARCHIVE_OK); + if (lzma_lzma_preset(&state->lzma_opt, level)) { + archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, + "Internal error compression library"); + return (ARCHIVE_FATAL); + } + /* + * Reinitialize + */ + lzma_end(&(state->stream)); + ret = archive_compressor_xz_init_stream(a); + if (ret == LZMA_OK) { + state->compression_level = level; + return (ARCHIVE_OK); + } + return (ARCHIVE_FATAL); + } + + return (ARCHIVE_WARN); +} + +/* + * Write data to the compressed stream. + */ +static int +archive_compressor_xz_write(struct archive_write *a, const void *buff, + size_t length) +{ + struct private_data *state; + int ret; + + state = (struct private_data *)a->compressor.data; + if (a->client_writer == NULL) { + archive_set_error(&a->archive, ARCHIVE_ERRNO_PROGRAMMER, + "No write callback is registered? " + "This is probably an internal programming error."); + return (ARCHIVE_FATAL); + } + + /* Update statistics */ + state->total_in += length; + + /* Compress input data to output buffer */ + state->stream.next_in = buff; + state->stream.avail_in = length; + if ((ret = drive_compressor(a, state, 0)) != ARCHIVE_OK) + return (ret); + + a->archive.file_position += length; + return (ARCHIVE_OK); +} + + +/* + * Finish the compression... + */ +static int +archive_compressor_xz_finish(struct archive_write *a) +{ + ssize_t block_length, target_block_length, bytes_written; + int ret; + struct private_data *state; + unsigned tocopy; + unsigned char trailer[8]; + + state = (struct private_data *)a->compressor.data; + ret = 0; + if (a->client_writer == NULL) { + archive_set_error(&a->archive, ARCHIVE_ERRNO_PROGRAMMER, + "No write callback is registered? " + "This is probably an internal programming error."); + ret = ARCHIVE_FATAL; + goto cleanup; + } + + /* By default, always pad the uncompressed data. */ + if (a->pad_uncompressed) { + tocopy = a->bytes_per_block - + (state->total_in % a->bytes_per_block); + while (tocopy > 0 && tocopy < (unsigned)a->bytes_per_block) { + state->stream.next_in = a->nulls; + state->stream.avail_in = tocopy < a->null_length ? + tocopy : a->null_length; + state->total_in += state->stream.avail_in; + tocopy -= state->stream.avail_in; + ret = drive_compressor(a, state, 0); + if (ret != ARCHIVE_OK) + goto cleanup; + } + } + + /* Finish compression cycle */ + if (((ret = drive_compressor(a, state, 1))) != ARCHIVE_OK) + goto cleanup; + + /* Optionally, pad the final compressed block. */ + block_length = state->stream.next_out - state->compressed; + + /* Tricky calculation to determine size of last block. */ + target_block_length = block_length; + if (a->bytes_in_last_block <= 0) + /* Default or Zero: pad to full block */ + target_block_length = a->bytes_per_block; + else + /* Round length to next multiple of bytes_in_last_block. */ + target_block_length = a->bytes_in_last_block * + ( (block_length + a->bytes_in_last_block - 1) / + a->bytes_in_last_block); + if (target_block_length > a->bytes_per_block) + target_block_length = a->bytes_per_block; + if (block_length < target_block_length) { + memset(state->stream.next_out, 0, + target_block_length - block_length); + block_length = target_block_length; + } + + /* Write the last block */ + bytes_written = (a->client_writer)(&a->archive, a->client_data, + state->compressed, block_length); + if (bytes_written <= 0) { + ret = ARCHIVE_FATAL; + goto cleanup; + } + a->archive.raw_position += bytes_written; + + /* Cleanup: shut down compressor, release memory, etc. */ +cleanup: + lzma_end(&(state->stream)); + free(state->compressed); + free(state); + return (ret); +} + +/* + * Utility function to push input data through compressor, + * writing full output blocks as necessary. + * + * Note that this handles both the regular write case (finishing == + * false) and the end-of-archive case (finishing == true). + */ +static int +drive_compressor(struct archive_write *a, struct private_data *state, int finishing) +{ + ssize_t bytes_written; + int ret; + + for (;;) { + if (state->stream.avail_out == 0) { + bytes_written = (a->client_writer)(&a->archive, + a->client_data, state->compressed, + state->compressed_buffer_size); + if (bytes_written <= 0) { + /* TODO: Handle this write failure */ + return (ARCHIVE_FATAL); + } else if ((size_t)bytes_written < state->compressed_buffer_size) { + /* Short write: Move remaining to + * front of block and keep filling */ + memmove(state->compressed, + state->compressed + bytes_written, + state->compressed_buffer_size - bytes_written); + } + a->archive.raw_position += bytes_written; + state->stream.next_out + = state->compressed + + state->compressed_buffer_size - bytes_written; + state->stream.avail_out = bytes_written; + } + + /* If there's nothing to do, we're done. */ + if (!finishing && state->stream.avail_in == 0) + return (ARCHIVE_OK); + + ret = lzma_code(&(state->stream), + finishing ? LZMA_FINISH : LZMA_RUN ); + + switch (ret) { + case LZMA_OK: + /* In non-finishing case, check if compressor + * consumed everything */ + if (!finishing && state->stream.avail_in == 0) + return (ARCHIVE_OK); + /* In finishing case, this return always means + * there's more work */ + break; + case LZMA_STREAM_END: + /* This return can only occur in finishing case. */ + if (finishing) + return (ARCHIVE_OK); + archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, + "lzma compression data error"); + return (ARCHIVE_FATAL); + case LZMA_MEMLIMIT_ERROR: + archive_set_error(&a->archive, ENOMEM, + "lzma compression error: " + "%ju MiB would have been needed", + (lzma_memusage(&(state->stream)) + 1024 * 1024 -1) + / (1024 * 1024)); + return (ARCHIVE_FATAL); + default: + /* Any other return value indicates an error. */ + archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, + "lzma compression failed:" + " lzma_code() call returned status %d", + ret); + return (ARCHIVE_FATAL); + } + } +} + +#endif /* HAVE_LZMA_H */ diff --git a/tar/bsdtar.c b/tar/bsdtar.c index 65975dff9..0fc837505 100644 --- a/tar/bsdtar.c +++ b/tar/bsdtar.c @@ -249,6 +249,19 @@ main(int argc, char **argv) bsdtar_warnc(bsdtar, 0, "bzip2 compression not supported by this version of bsdtar"); usage(bsdtar); +#endif + break; + case 'J': /* GNU tar 1.21 and later */ +#if HAVE_LIBLZMA + if (bsdtar->create_compression != '\0') + bsdtar_errc(bsdtar, 1, 0, + "Can't specify both -%c and -%c", opt, + bsdtar->create_compression); + bsdtar->create_compression = opt; +#else + bsdtar_warnc(bsdtar, 0, + "lzma compression not supported by this version of bsdtar"); + usage(bsdtar); #endif break; case 'k': /* GNU tar */ @@ -420,6 +433,19 @@ main(int argc, char **argv) case 'x': /* SUSv2 */ set_mode(bsdtar, opt); break; + case OPTION_XZ: +#if HAVE_LIBLZMA + if (bsdtar->create_compression != '\0') + bsdtar_errc(bsdtar, 1, 0, + "Can't specify both -%c and -%c", opt, + bsdtar->create_compression); + bsdtar->create_compression = opt; +#else + bsdtar_warnc(bsdtar, 0, + "xz compression not supported by this version of bsdtar"); + usage(bsdtar); +#endif + break; case 'y': /* FreeBSD version of GNU tar */ #if HAVE_LIBBZ2 if (bsdtar->create_compression != '\0') diff --git a/tar/bsdtar.h b/tar/bsdtar.h index a427b9a39..8315453e4 100644 --- a/tar/bsdtar.h +++ b/tar/bsdtar.h @@ -129,7 +129,8 @@ enum { OPTION_STRIP_COMPONENTS, OPTION_TOTALS, OPTION_USE_COMPRESS_PROGRAM, - OPTION_VERSION + OPTION_VERSION, + OPTION_XZ }; diff --git a/tar/cmdline.c b/tar/cmdline.c index ad8b487a6..c24b3d54e 100644 --- a/tar/cmdline.c +++ b/tar/cmdline.c @@ -46,7 +46,7 @@ __FBSDID("$FreeBSD$"); * Short options for tar. Please keep this sorted. */ static const char *short_options - = "Bb:C:cf:HhI:jkLlmnOoPpqrSs:T:tUuvW:wX:xyZz"; + = "Bb:C:cf:HhI:JjkLlmnOoPpqrSs:T:tUuvW:wX:xyZz"; /* * Long options for tar. Please keep this list sorted. @@ -93,6 +93,7 @@ static struct option { { "keep-newer-files", 0, OPTION_KEEP_NEWER_FILES }, { "keep-old-files", 0, 'k' }, { "list", 0, 't' }, + { "lzma", 0, 'J' }, { "modification-time", 0, 'm' }, { "newer", 1, OPTION_NEWER_CTIME }, { "newer-ctime", 1, OPTION_NEWER_CTIME }, @@ -122,6 +123,7 @@ static struct option { { "use-compress-program", 1, OPTION_USE_COMPRESS_PROGRAM }, { "verbose", 0, 'v' }, { "version", 0, OPTION_VERSION }, + { "xz", 0, OPTION_XZ }, { NULL, 0, 0 } }; diff --git a/tar/write.c b/tar/write.c index 6bb317cd2..fa143d6b4 100644 --- a/tar/write.c +++ b/tar/write.c @@ -182,6 +182,14 @@ tar_mode_c(struct bsdtar *bsdtar) archive_write_set_compression_bzip2(a); break; #endif +#ifdef HAVE_LIBLZMA + case 'J': + archive_write_set_compression_lzma(a); + break; + case OPTION_XZ: + archive_write_set_compression_xz(a); + break; +#endif #ifdef HAVE_LIBZ case 'z': archive_write_set_compression_gzip(a);