From: Josef 'Jeff' Sipek Date: Fri, 22 Jan 2021 19:00:21 +0000 (-0500) Subject: lib-compression: Remove xz/lzma write support X-Git-Tag: 2.3.14~2 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=52e6c6f4e02b623c4c812e1226d36137fc723b7a;p=thirdparty%2Fdovecot%2Fcore.git lib-compression: Remove xz/lzma write support For now, we keep the read support to allow for a more gradual migration from lzma/xz. The code has bugs. Additionally, there is no benefit in keeping it around. bzip2 or zstd (at maximum compression level) are likely better choices for archival. For non-archival usecases, a light compression algorithm like zstd (at low to medium compression level) or lz4 are better choices. --- diff --git a/configure.ac b/configure.ac index b7fe4e24f9..9cd6c8c876 100644 --- a/configure.ac +++ b/configure.ac @@ -172,7 +172,7 @@ AS_HELP_STRING([--with-bzlib], [Build with bzlib compression support (auto)]), want_bzlib=auto) AC_ARG_WITH(lzma, -AS_HELP_STRING([--with-lzma], [Build with LZMA compression support (auto)]), +AS_HELP_STRING([--with-lzma], [Build with LZMA decompression support (auto)]), TEST_WITH(lzma, $withval), want_lzma=auto) diff --git a/run-test-valgrind.supp b/run-test-valgrind.supp index 96544884b5..ae6bd57c07 100644 --- a/run-test-valgrind.supp +++ b/run-test-valgrind.supp @@ -4,17 +4,6 @@ fun:malloc obj:*/bash } -{ - - Memcheck:Cond - obj:/lib/x86_64-linux-gnu/liblzma.so.5.* - obj:/lib/x86_64-linux-gnu/liblzma.so.5.* - obj:/lib/x86_64-linux-gnu/liblzma.so.5.* - obj:/lib/x86_64-linux-gnu/liblzma.so.5.* - obj:/lib/x86_64-linux-gnu/liblzma.so.5.* - fun:lzma_stream_encoder - fun:lzma_easy_encoder -} { Memcheck:Leak diff --git a/src/lib-compression/Makefile.am b/src/lib-compression/Makefile.am index 88f38558a3..f689b0c783 100644 --- a/src/lib-compression/Makefile.am +++ b/src/lib-compression/Makefile.am @@ -13,7 +13,6 @@ libcompression_la_SOURCES = \ istream-zlib.c \ istream-bzlib.c \ istream-zstd.c \ - ostream-lzma.c \ ostream-lz4.c \ ostream-zlib.c \ ostream-bzlib.c \ diff --git a/src/lib-compression/compression.c b/src/lib-compression/compression.c index c03d9b1661..bf70222e1f 100644 --- a/src/lib-compression/compression.c +++ b/src/lib-compression/compression.c @@ -19,7 +19,6 @@ #endif #ifndef HAVE_LZMA # define i_stream_create_lzma NULL -# define o_stream_create_lzma NULL #endif #ifndef HAVE_LZ4 # define i_stream_create_lz4 NULL @@ -162,7 +161,7 @@ const struct compression_handler compression_handlers[] = { { "deflate", NULL, NULL, i_stream_create_deflate, o_stream_create_deflate }, { "xz", ".xz", is_compressed_xz, - i_stream_create_lzma, o_stream_create_lzma }, + i_stream_create_lzma, NULL }, { "lz4", ".lz4", is_compressed_lz4, i_stream_create_lz4, o_stream_create_lz4 }, { "zstd", ".zstd", is_compressed_zstd, diff --git a/src/lib-compression/ostream-lzma.c b/src/lib-compression/ostream-lzma.c deleted file mode 100644 index fe8bc161ad..0000000000 --- a/src/lib-compression/ostream-lzma.c +++ /dev/null @@ -1,263 +0,0 @@ -/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */ - -#include "lib.h" - -#ifdef HAVE_LZMA - -#include "ostream-private.h" -#include "ostream-zlib.h" -#include - -#define CHUNK_SIZE (1024*64) - -struct lzma_ostream { - struct ostream_private ostream; - lzma_stream strm; - - unsigned char outbuf[CHUNK_SIZE]; - unsigned int outbuf_offset, outbuf_used; - - bool flushed:1; -}; - -static void o_stream_lzma_close(struct iostream_private *stream, - bool close_parent) -{ - struct lzma_ostream *zstream = (struct lzma_ostream *)stream; - i_assert(zstream->ostream.finished || - zstream->ostream.ostream.stream_errno != 0 || - zstream->ostream.error_handling_disabled); - lzma_end(&zstream->strm); - if (close_parent) - o_stream_close(zstream->ostream.parent); -} - -static int o_stream_zlib_send_outbuf(struct lzma_ostream *zstream) -{ - ssize_t ret; - size_t size; - - if (zstream->outbuf_used == 0) - return 1; - - size = zstream->outbuf_used - zstream->outbuf_offset; - i_assert(size > 0); - ret = o_stream_send(zstream->ostream.parent, - zstream->outbuf + zstream->outbuf_offset, size); - if (ret < 0) { - o_stream_copy_error_from_parent(&zstream->ostream); - return -1; - } - if ((size_t)ret != size) { - zstream->outbuf_offset += ret; - return 0; - } - zstream->outbuf_offset = 0; - zstream->outbuf_used = 0; - return 1; -} - -static ssize_t -o_stream_lzma_send_chunk(struct lzma_ostream *zstream, - const void *data, size_t size) -{ - lzma_stream *zs = &zstream->strm; - int ret; - - i_assert(zstream->outbuf_used == 0); - - zs->next_in = (void *)data; - zs->avail_in = size; - while (zs->avail_in > 0) { - if (zs->avail_out == 0) { - /* previous block was compressed. send it and start - compression for a new block. */ - zs->next_out = zstream->outbuf; - zs->avail_out = sizeof(zstream->outbuf); - - zstream->outbuf_used = sizeof(zstream->outbuf); - if ((ret = o_stream_zlib_send_outbuf(zstream)) < 0) - return -1; - if (ret == 0) { - /* parent stream's buffer full */ - break; - } - } - - ret = lzma_code(zs, LZMA_RUN); - switch (ret) { - case LZMA_OK: - break; - case LZMA_MEM_ERROR: - i_fatal_status(FATAL_OUTOFMEM, - "lzma.write(%s): Out of memory", - o_stream_get_name(&zstream->ostream.ostream)); - default: - i_panic("lzma.write(%s) failed with unexpected code %d", - o_stream_get_name(&zstream->ostream.ostream), ret); - } - } - size -= zs->avail_in; - - return size; -} - -static int o_stream_lzma_send_flush(struct lzma_ostream *zstream, bool final) -{ - lzma_stream *zs = &zstream->strm; - size_t len; - bool done = FALSE; - int ret; - - i_assert(zs->avail_in == 0); - - if (zstream->flushed) { - i_assert(zstream->outbuf_used == 0); - return 1; - } - - if ((ret = o_stream_flush_parent_if_needed(&zstream->ostream)) <= 0) - return ret; - if ((ret = o_stream_zlib_send_outbuf(zstream)) <= 0) - return ret; - - if (!final) - return 1; - - i_assert(zstream->outbuf_used == 0); - do { - len = sizeof(zstream->outbuf) - zs->avail_out; - if (len != 0) { - zs->next_out = zstream->outbuf; - zs->avail_out = sizeof(zstream->outbuf); - - zstream->outbuf_used = len; - if ((ret = o_stream_zlib_send_outbuf(zstream)) <= 0) - return ret; - if (done) - break; - } - ret = lzma_code(zs, LZMA_FINISH); - switch (ret) { - case LZMA_OK: - /* still unfinished - need to call lzma_code() again */ - break; - case LZMA_STREAM_END: - /* output is fully finished */ - done = TRUE; - break; - case LZMA_MEM_ERROR: - i_fatal_status(FATAL_OUTOFMEM, - "lzma.write(%s): Out of memory", - o_stream_get_name(&zstream->ostream.ostream)); - default: - i_panic("lzma.write(%s) flush failed with unexpected code %d", - o_stream_get_name(&zstream->ostream.ostream), ret); - } - } while (zs->avail_out != sizeof(zstream->outbuf)); - - if (final) - zstream->flushed = TRUE; - i_assert(zstream->outbuf_used == 0); - return 1; -} - -static int o_stream_lzma_flush(struct ostream_private *stream) -{ - struct lzma_ostream *zstream = (struct lzma_ostream *)stream; - int ret; - - if ((ret = o_stream_lzma_send_flush(zstream, stream->finished)) < 0) - return -1; - else if (ret > 0) - return o_stream_flush_parent(stream); - return ret; -} - -static size_t -o_stream_lzma_get_buffer_used_size(const struct ostream_private *stream) -{ - const struct lzma_ostream *zstream = - (const struct lzma_ostream *)stream; - - /* outbuf has already compressed data that we're trying to send to the - parent stream. We're not including lzma's internal compression - buffer size. */ - return (zstream->outbuf_used - zstream->outbuf_offset) + - o_stream_get_buffer_used_size(stream->parent); -} - -static size_t -o_stream_lzma_get_buffer_avail_size(const struct ostream_private *stream) -{ - /* FIXME: not correct - this is counting compressed size, which may be - too larger than uncompressed size in some situations. Fixing would - require some kind of additional buffering. */ - return o_stream_get_buffer_avail_size(stream->parent); -} - -static ssize_t -o_stream_lzma_sendv(struct ostream_private *stream, - const struct const_iovec *iov, unsigned int iov_count) -{ - struct lzma_ostream *zstream = (struct lzma_ostream *)stream; - ssize_t ret, bytes = 0; - unsigned int i; - - if ((ret = o_stream_zlib_send_outbuf(zstream)) <= 0) { - /* error / we still couldn't flush existing data to - parent stream. */ - return ret; - } - - for (i = 0; i < iov_count; i++) { - ret = o_stream_lzma_send_chunk(zstream, iov[i].iov_base, - iov[i].iov_len); - if (ret < 0) - return -1; - bytes += ret; - if ((size_t)ret != iov[i].iov_len) - break; - } - stream->ostream.offset += bytes; - - /* avail_in!=0 check is used to detect errors. if it's non-zero here - it simply means we didn't send all the data */ - zstream->strm.avail_in = 0; - return bytes; -} - -struct ostream *o_stream_create_lzma(struct ostream *output, int level) -{ - struct lzma_ostream *zstream; - lzma_ret ret; - - i_assert(level >= 1 && level <= 9); - - zstream = i_new(struct lzma_ostream, 1); - zstream->ostream.sendv = o_stream_lzma_sendv; - zstream->ostream.flush = o_stream_lzma_flush; - zstream->ostream.get_buffer_used_size = - o_stream_lzma_get_buffer_used_size; - zstream->ostream.get_buffer_avail_size = - o_stream_lzma_get_buffer_avail_size; - zstream->ostream.iostream.close = o_stream_lzma_close; - - ret = lzma_easy_encoder(&zstream->strm, level, LZMA_CHECK_CRC64); - switch (ret) { - case LZMA_OK: - break; - case LZMA_MEM_ERROR: - i_fatal_status(FATAL_OUTOFMEM, "lzma: Out of memory"); - case LZMA_OPTIONS_ERROR: - i_fatal("lzma: Invalid level"); - default: - i_fatal("lzma_easy_encoder() failed with %d", ret); - } - - zstream->strm.next_out = zstream->outbuf; - zstream->strm.avail_out = sizeof(zstream->outbuf); - return o_stream_create(&zstream->ostream, output, - o_stream_get_fd(output)); -} -#endif diff --git a/src/lib-compression/ostream-zlib.h b/src/lib-compression/ostream-zlib.h index 59b61ab901..101d1c7630 100644 --- a/src/lib-compression/ostream-zlib.h +++ b/src/lib-compression/ostream-zlib.h @@ -4,7 +4,6 @@ struct ostream *o_stream_create_gz(struct ostream *output, int level); struct ostream *o_stream_create_deflate(struct ostream *output, int level); struct ostream *o_stream_create_bz2(struct ostream *output, int level); -struct ostream *o_stream_create_lzma(struct ostream *output, int level); struct ostream *o_stream_create_lz4(struct ostream *output, int level); struct ostream *o_stream_create_zstd(struct ostream *output, int level); diff --git a/src/lib-compression/test-compression.c b/src/lib-compression/test-compression.c index 078a4def51..b7364b93fe 100644 --- a/src/lib-compression/test-compression.c +++ b/src/lib-compression/test-compression.c @@ -730,6 +730,7 @@ static void test_compression_int(bool autodetect) for (i = 0; compression_handlers[i].name != NULL; i++) { if (compression_handlers[i].create_istream != NULL && + compression_handlers[i].create_ostream != NULL && (!autodetect || compression_handlers[i].is_compressed != NULL)) T_BEGIN { if (compression_handlers[i].is_compressed != NULL &&