From 8b6142f9ba910926aa337eac734fcd77841fe13a Mon Sep 17 00:00:00 2001 From: Timo Sirainen Date: Thu, 1 Oct 2020 15:36:08 +0300 Subject: [PATCH] lib-compression: Add istream-decompress This stream detects the compression format and creates the proper istream afterwards. This is more efficient than creating each compression istream and using istream-try. --- src/lib-compression/Makefile.am | 1 + src/lib-compression/compression.h | 17 ++ src/lib-compression/istream-decompress.c | 254 +++++++++++++++++++++++ src/lib-compression/test-compression.c | 216 ++++++++++++++----- 4 files changed, 433 insertions(+), 55 deletions(-) create mode 100644 src/lib-compression/istream-decompress.c diff --git a/src/lib-compression/Makefile.am b/src/lib-compression/Makefile.am index 6c2e5e234e..88f38558a3 100644 --- a/src/lib-compression/Makefile.am +++ b/src/lib-compression/Makefile.am @@ -7,6 +7,7 @@ AM_CPPFLAGS = \ libcompression_la_SOURCES = \ compression.c \ + istream-decompress.c \ istream-lzma.c \ istream-lz4.c \ istream-zlib.c \ diff --git a/src/lib-compression/compression.h b/src/lib-compression/compression.h index b27d09e8bc..9b1f859020 100644 --- a/src/lib-compression/compression.h +++ b/src/lib-compression/compression.h @@ -1,6 +1,12 @@ #ifndef COMPRESSION_H #define COMPRESSION_H +enum istream_decompress_flags { + /* If stream isn't detected to be compressed, return it as passthrough + istream. */ + ISTREAM_DECOMPRESS_FLAG_TRY = BIT(0), +}; + /* Compressed input is always detected once at maximum this many bytes have been read. This value must be smaller than a typical istream max buffer size. */ @@ -29,4 +35,15 @@ compression_detect_handler(struct istream *input); int compression_lookup_handler_from_ext(const char *path, const struct compression_handler **handler_r); +/* Automatically detect the compression format. Note that using tee-istream as + one of the parent streams is dangerous here: A decompression istream may + have to read a lot of data (e.g. 8 kB isn't enough) before it returns even + the first byte as output. If the other tee children aren't read forward, + this can cause an infinite loop when i_stream_read() is always returning 0. + This is why ISTREAM_DECOMPRESS_FLAG_TRY should be used instead of attempting + to implement similar functionality with tee-istream. */ +struct istream * +i_stream_create_decompress(struct istream *input, + enum istream_decompress_flags flags); + #endif diff --git a/src/lib-compression/istream-decompress.c b/src/lib-compression/istream-decompress.c new file mode 100644 index 0000000000..1cd21e4562 --- /dev/null +++ b/src/lib-compression/istream-decompress.c @@ -0,0 +1,254 @@ +/* Copyright (c) 2020 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "istream-private.h" +#include "compression.h" + +struct decompress_istream { + struct istream_private istream; + struct istream *compressed_input; + struct istream *decompressed_input; + enum istream_decompress_flags flags; +}; + +static void copy_compressed_input_error(struct decompress_istream *zstream) +{ + struct istream_private *stream = &zstream->istream; + + stream->istream.stream_errno = zstream->compressed_input->stream_errno; + stream->istream.eof = zstream->compressed_input->eof; + if (zstream->compressed_input->real_stream->iostream.error != NULL) { + io_stream_set_error(&stream->iostream, "%s", + zstream->compressed_input->real_stream->iostream.error); + } +} + +static void copy_decompressed_input_error(struct decompress_istream *zstream) +{ + struct istream_private *stream = &zstream->istream; + + stream->istream.stream_errno = zstream->decompressed_input->stream_errno; + stream->istream.eof = zstream->decompressed_input->eof; + if (zstream->decompressed_input->real_stream->iostream.error != NULL) { + io_stream_set_error(&stream->iostream, "%s", + zstream->decompressed_input->real_stream->iostream.error); + } +} + +static void +i_stream_decompress_close(struct iostream_private *_stream, bool close_parent) +{ + struct istream_private *stream = + container_of(_stream, struct istream_private, iostream); + struct decompress_istream *zstream = + container_of(stream, struct decompress_istream, istream); + + if (zstream->decompressed_input != NULL) + i_stream_close(zstream->decompressed_input); + if (close_parent) + i_stream_close(zstream->compressed_input); +} + +static void +i_stream_decompress_destroy(struct iostream_private *_stream) +{ + struct istream_private *stream = + container_of(_stream, struct istream_private, iostream); + struct decompress_istream *zstream = + container_of(stream, struct decompress_istream, istream); + + i_stream_unref(&zstream->decompressed_input); + i_stream_unref(&zstream->compressed_input); +} + +static int +i_stream_decompress_not_compressed(struct decompress_istream *zstream) +{ + if ((zstream->flags & ISTREAM_DECOMPRESS_FLAG_TRY) == 0) { + zstream->istream.istream.stream_errno = EINVAL; + io_stream_set_error(&zstream->istream.iostream, + "Stream isn't compressed"); + return -1; + } else { + zstream->decompressed_input = zstream->compressed_input; + i_stream_ref(zstream->decompressed_input); + return 1; + } +} + +static int i_stream_decompress_detect(struct decompress_istream *zstream) +{ + const struct compression_handler *handler; + ssize_t ret; + + ret = i_stream_read(zstream->compressed_input); + handler = compression_detect_handler(zstream->compressed_input); + if (handler == NULL) { + switch (ret) { + case -1: + if (zstream->compressed_input->stream_errno != 0) { + copy_compressed_input_error(zstream); + return -1; + } + /* fall through */ + case -2: + /* we've read a full buffer or we reached EOF - + the stream isn't compressed */ + return i_stream_decompress_not_compressed(zstream); + case 0: + return 0; + default: + if (!zstream->istream.istream.blocking) + return 0; + return i_stream_decompress_detect(zstream); + } + } + if (handler->create_istream == NULL) { + zstream->istream.istream.stream_errno = EINVAL; + io_stream_set_error(&zstream->istream.iostream, + "Compression handler %s not supported", handler->name); + return -1; + } + + zstream->decompressed_input = + handler->create_istream(zstream->compressed_input, FALSE); + return 1; +} + +static ssize_t i_stream_decompress_read(struct istream_private *stream) +{ + struct decompress_istream *zstream = + container_of(stream, struct decompress_istream, istream); + ssize_t ret; + size_t pos; + + if (zstream->decompressed_input == NULL) { + if ((ret = i_stream_decompress_detect(zstream)) <= 0) + return ret; + } + + i_stream_seek(zstream->decompressed_input, stream->istream.v_offset); + stream->pos -= stream->skip; + stream->skip = 0; + + stream->buffer = i_stream_get_data(zstream->decompressed_input, &pos); + if (pos > stream->pos) + ret = 0; + else do { + ret = i_stream_read_memarea(zstream->decompressed_input); + copy_decompressed_input_error(zstream); + stream->buffer = i_stream_get_data(zstream->decompressed_input, + &pos); + } while (pos <= stream->pos && ret > 0); + if (ret == -2) + return -2; + + if (pos <= stream->pos) + ret = ret == 0 ? 0 : -1; + else + ret = (ssize_t)(pos - stream->pos); + stream->pos = pos; + i_assert(ret != -1 || stream->istream.eof || + stream->istream.stream_errno != 0); + return ret; +} + +static void i_stream_decompress_reset(struct istream_private *stream) +{ + stream->skip = stream->pos = 0; + stream->istream.v_offset = 0; + stream->istream.eof = FALSE; +} + +static void +i_stream_decompress_seek(struct istream_private *stream, + uoff_t v_offset, bool mark) +{ + struct decompress_istream *zstream = + container_of(stream, struct decompress_istream, istream); + + if (zstream->decompressed_input == NULL) { + if (!i_stream_nonseekable_try_seek(stream, v_offset)) + i_panic("seeking backwards before detecting compression format"); + } else { + i_stream_decompress_reset(stream); + stream->istream.v_offset = v_offset; + if (mark) + i_stream_seek_mark(zstream->decompressed_input, v_offset); + else + i_stream_seek(zstream->decompressed_input, v_offset); + copy_decompressed_input_error(zstream); + } +} + +static void i_stream_decompress_sync(struct istream_private *stream) +{ + struct decompress_istream *zstream = + container_of(stream, struct decompress_istream, istream); + + i_stream_decompress_reset(stream); + if (zstream->decompressed_input != NULL) + i_stream_sync(zstream->decompressed_input); +} + +static int i_stream_decompress_stat(struct istream_private *stream, bool exact) +{ + struct decompress_istream *zstream = + container_of(stream, struct decompress_istream, istream); + const struct stat *st; + + if (!exact) { + if (i_stream_stat(zstream->compressed_input, exact, &st) < 0) { + copy_compressed_input_error(zstream); + return -1; + } + stream->statbuf = *st; + return 0; + } + if (zstream->decompressed_input == NULL) { + (void)i_stream_read(&stream->istream); + if (zstream->decompressed_input == NULL) { + if (stream->istream.stream_errno == 0) { + zstream->istream.istream.stream_errno = EINVAL; + io_stream_set_error(&zstream->istream.iostream, + "Stream compression couldn't be detected during stat"); + } + return -1; + } + } + + if (i_stream_stat(zstream->decompressed_input, exact, &st) < 0) { + copy_decompressed_input_error(zstream); + return -1; + } + i_stream_decompress_reset(stream); + stream->statbuf = *st; + return 0; +} + +struct istream * +i_stream_create_decompress(struct istream *input, + enum istream_decompress_flags flags) +{ + struct decompress_istream *zstream; + + zstream = i_new(struct decompress_istream, 1); + zstream->compressed_input = input; + zstream->flags = flags; + i_stream_ref(input); + + zstream->istream.iostream.close = i_stream_decompress_close; + zstream->istream.iostream.destroy = i_stream_decompress_destroy; + zstream->istream.max_buffer_size = input->real_stream->max_buffer_size; + zstream->istream.read = i_stream_decompress_read; + zstream->istream.seek = i_stream_decompress_seek; + zstream->istream.sync = i_stream_decompress_sync; + zstream->istream.stat = i_stream_decompress_stat; + + zstream->istream.istream.readable_fd = FALSE; + zstream->istream.istream.blocking = input->blocking; + zstream->istream.istream.seekable = input->seekable; + + return i_stream_create(&zstream->istream, NULL, + i_stream_get_fd(input), 0); +} diff --git a/src/lib-compression/test-compression.c b/src/lib-compression/test-compression.c index 1bf97c48c9..8cf45caa38 100644 --- a/src/lib-compression/test-compression.c +++ b/src/lib-compression/test-compression.c @@ -63,7 +63,9 @@ static void test_compression_handler_detect(const struct compression_handler *ha test_end(); } -static void test_compression_handler_short(const struct compression_handler *handler) +static void +test_compression_handler_short(const struct compression_handler *handler, + bool autodetect) { const unsigned char *data; size_t len, size; @@ -76,7 +78,8 @@ static void test_compression_handler_short(const struct compression_handler *han struct istream *input; /* write some amount of data */ - test_begin(t_strdup_printf("compression handler %s (small)", handler->name)); + test_begin(t_strdup_printf("compression handler %s (small, autodetect=%s)", + handler->name, autodetect ? "yes" : "no")); len = i_rand_minmax(1, 1024); test_data = buffer_create_dynamic(default_pool, len); random_fill(buffer_append_space_unsafe(test_data, len), len); @@ -95,7 +98,8 @@ static void test_compression_handler_short(const struct compression_handler *han /* read data at once */ test_input = test_istream_create_data(buffer->data, buffer->used); - input = handler->create_istream(test_input, TRUE); + input = !autodetect ? handler->create_istream(test_input, TRUE) : + i_stream_create_decompress(test_input, 0); i_stream_unref(&test_input); test_assert(i_stream_read_more(input, &data, &size) > 0); @@ -110,7 +114,9 @@ static void test_compression_handler_short(const struct compression_handler *han test_end(); } -static void test_compression_handler_seek(const struct compression_handler *handler) +static void +test_compression_handler_seek(const struct compression_handler *handler, + bool autodetect) { const unsigned char *data,*ptr; size_t len, size, pos; @@ -123,7 +129,8 @@ static void test_compression_handler_seek(const struct compression_handler *hand struct istream *input; /* write some amount of data */ - test_begin(t_strdup_printf("compression handler %s (seek)", handler->name)); + test_begin(t_strdup_printf("compression handler %s (seek, autodetect=%s)", + handler->name, autodetect ? "yes" : "no")); len = i_rand_minmax(1024, 2048); test_data = buffer_create_dynamic(default_pool, len); random_fill(buffer_append_space_unsafe(test_data, len), len); @@ -141,7 +148,8 @@ static void test_compression_handler_seek(const struct compression_handler *hand o_stream_unref(&output); test_input = test_istream_create_data(buffer->data, buffer->used); - input = handler->create_istream(test_input, TRUE); + input = !autodetect ? handler->create_istream(test_input, TRUE) : + i_stream_create_decompress(test_input, 0); i_stream_unref(&test_input); /* seek forward */ @@ -170,7 +178,9 @@ static void test_compression_handler_seek(const struct compression_handler *hand test_end(); } -static void test_compression_handler_reset(const struct compression_handler *handler) +static void +test_compression_handler_reset(const struct compression_handler *handler, + bool autodetect) { const unsigned char *data; size_t len, size; @@ -183,7 +193,8 @@ static void test_compression_handler_reset(const struct compression_handler *han struct istream *input; /* write some amount of data */ - test_begin(t_strdup_printf("compression handler %s (reset)", handler->name)); + test_begin(t_strdup_printf("compression handler %s (reset, autodetect=%s)", + handler->name, autodetect ? "yes" : "no")); len = i_rand_minmax(1024, 2048); test_data = buffer_create_dynamic(default_pool, len); random_fill(buffer_append_space_unsafe(test_data, len), len); @@ -201,7 +212,8 @@ static void test_compression_handler_reset(const struct compression_handler *han o_stream_unref(&output); test_input = test_istream_create_data(buffer->data, buffer->used); - input = handler->create_istream(test_input, TRUE); + input = !autodetect ? handler->create_istream(test_input, TRUE) : + i_stream_create_decompress(test_input, 0); i_stream_unref(&test_input); /* seek forward */ @@ -227,7 +239,9 @@ static void test_compression_handler_reset(const struct compression_handler *han test_end(); } -static void test_compression_handler(const struct compression_handler *handler) +static void +test_compression_handler(const struct compression_handler *handler, + bool autodetect) { const char *path = "test-compression.tmp"; struct istream *file_input, *input; @@ -242,7 +256,8 @@ static void test_compression_handler(const struct compression_handler *handler) int fd; ssize_t ret; - test_begin(t_strdup_printf("compression handler %s", handler->name)); + test_begin(t_strdup_printf("compression handler %s (autodetect=%s)", + handler->name, autodetect ? "yes" : "no")); /* write compressed data */ fd = open(path, O_TRUNC | O_CREAT | O_RDWR, 0600); @@ -291,7 +306,8 @@ static void test_compression_handler(const struct compression_handler *handler) /* read and uncompress the data */ file_input = i_stream_create_fd(fd, IO_BLOCK_SIZE); - input = handler->create_istream(file_input, TRUE); + input = !autodetect ? handler->create_istream(file_input, TRUE) : + i_stream_create_decompress(file_input, 0); test_assert(i_stream_get_size(input, FALSE, &stream_size) == 1); test_assert(stream_size == compressed_size); @@ -326,9 +342,12 @@ static void test_compression_handler(const struct compression_handler *handler) test_end(); } -static void test_compression_handler_partial_parent_write(const struct compression_handler *handler) +static void +test_compression_handler_partial_parent_write(const struct compression_handler *handler, + bool autodetect) { - test_begin(t_strdup_printf("compression handler %s (partial parent writes)", handler->name)); + test_begin(t_strdup_printf("compression handler %s (partial parent writes, autodetect=%s)", + handler->name, autodetect ? "yes" : "no")); int ret; buffer_t *buffer = t_buffer_create(64); @@ -362,7 +381,9 @@ static void test_compression_handler_partial_parent_write(const struct compressi buffer_append(compressed_data, buffer->data, buffer->used); struct istream *is = test_istream_create_data(compressed_data->data, compressed_data->used); - struct istream *is_decompressed = handler->create_istream(is, TRUE); + struct istream *is_decompressed = + !autodetect ? handler->create_istream(is, TRUE) : + i_stream_create_decompress(is, 0); i_stream_unref(&is); const unsigned char *data; @@ -386,7 +407,8 @@ static void test_compression_handler_partial_parent_write(const struct compressi } static void -test_compression_handler_random_io(const struct compression_handler *handler) +test_compression_handler_random_io(const struct compression_handler *handler, + bool autodetect) { unsigned char in_buf[8192]; size_t in_buf_size; @@ -397,8 +419,8 @@ test_compression_handler_random_io(const struct compression_handler *handler) enc_buf = buffer_create_dynamic(default_pool, sizeof(in_buf)); dec_buf = buffer_create_dynamic(default_pool, sizeof(in_buf)); - test_begin(t_strdup_printf("compression handler %s (random I/O)", - handler->name)); + test_begin(t_strdup_printf("compression handler %s (random I/O, autodetect=%s)", + handler->name, autodetect ? "yes" : "no")); for (i = 0; !test_has_failed() && i < 300; i++) { struct istream *input1, *input2; @@ -474,7 +496,8 @@ test_compression_handler_random_io(const struct compression_handler *handler) i_stream_set_name(input1, "[compressed-data]"); /* Create decompressor stream */ - input2 = handler->create_istream(input1, TRUE); + input2 = !autodetect ? handler->create_istream(input1, TRUE) : + i_stream_create_decompress(input1, 0); i_stream_set_name(input2, "[decompressor]"); /* Assign random buffer sizes */ @@ -528,7 +551,8 @@ test_compression_handler_random_io(const struct compression_handler *handler) } static void -test_compression_handler_large_random_io(const struct compression_handler *handler) +test_compression_handler_large_random_io(const struct compression_handler *handler, + bool autodetect) { #define RANDOMNESS_SIZE (1024*1024) unsigned char *randomness; @@ -538,7 +562,8 @@ test_compression_handler_large_random_io(const struct compression_handler *handl size_t size; int ret; - test_begin(t_strdup_printf("compression handler %s (large random io)", handler->name)); + test_begin(t_strdup_printf("compression handler %s (large random io, autodetect=%s)", + handler->name, autodetect ? "yes" : "no")); randomness = i_malloc(RANDOMNESS_SIZE); random_fill(randomness, RANDOMNESS_SIZE); @@ -569,7 +594,8 @@ test_compression_handler_large_random_io(const struct compression_handler *handl /* verify that reading the input works */ - dec_input = handler->create_istream(input, FALSE); + dec_input = !autodetect ? handler->create_istream(input, TRUE) : + i_stream_create_decompress(input, 0); while ((ret = i_stream_read_more(dec_input, &data, &size)) > 0) { test_assert(memcmp(data, randomness + dec_input->v_offset, size) == 0); @@ -585,13 +611,18 @@ test_compression_handler_large_random_io(const struct compression_handler *handl test_end(); } -static void test_compression_handler_errors(const struct compression_handler *handler) +static void +test_compression_handler_errors(const struct compression_handler *handler, + bool autodetect) { - test_begin(t_strdup_printf("compression handler %s (errors)", handler->name)); + test_begin(t_strdup_printf("compression handler %s (errors, autodetect=%s)", + handler->name, autodetect ? "yes" : "no")); /* test that zero stream reading errors out */ struct istream *is = test_istream_create(""); - struct istream *input = handler->create_istream(is, FALSE); + struct istream *input = + !autodetect ? handler->create_istream(is, FALSE) : + i_stream_create_decompress(is, 0); i_stream_unref(&is); test_assert(i_stream_read(input) == -1 && input->eof); i_stream_unref(&input); @@ -600,7 +631,9 @@ static void test_compression_handler_errors(const struct compression_handler *ha is = test_istream_create("dedededededededededededededede" "dedededeededdedededededededede" "dedededededededededededededede"); - input = handler->create_istream(is, FALSE); + is->blocking = TRUE; + input = !autodetect ? handler->create_istream(is, FALSE) : + i_stream_create_decompress(is, 0); i_stream_unref(&is); test_assert(i_stream_read(input) == -1 && input->eof); i_stream_unref(&input); @@ -622,7 +655,8 @@ static void test_compression_handler_errors(const struct compression_handler *ha /* truncate buffer */ is = test_istream_create_data(odata->data, odata->used - sizeof(buf)*2 - 1); - input = handler->create_istream(is, FALSE); + input = !autodetect ? handler->create_istream(is, FALSE) : + i_stream_create_decompress(is, 0); i_stream_unref(&is); const unsigned char *data ATTR_UNUSED; @@ -639,7 +673,8 @@ static void test_compression_handler_errors(const struct compression_handler *ha that should not match any handlers' header */ for (size_t i = 0; i < 32; i++) { is = test_istream_create_data("dededededededededededededededede", i); - input = handler->create_istream(is, FALSE); + input = !autodetect ? handler->create_istream(is, FALSE) : + i_stream_create_decompress(is, 0); i_stream_unref(&is); while (i_stream_read_more(input, &data, &size) >= 0) { test_assert_idx(size == 0, i); @@ -653,27 +688,71 @@ static void test_compression_handler_errors(const struct compression_handler *ha test_end(); } -static void test_compression(void) +static void test_compression_int(bool autodetect) { unsigned int i; for (i = 0; compression_handlers[i].name != NULL; i++) { - if (compression_handlers[i].create_istream != NULL) T_BEGIN { - test_compression_handler_short(&compression_handlers[i]); - test_compression_handler(&compression_handlers[i]); - if (compression_handlers[i].is_compressed != NULL) + if (compression_handlers[i].create_istream != NULL && + (!autodetect || + compression_handlers[i].is_compressed != NULL)) T_BEGIN { + if (compression_handlers[i].is_compressed != NULL && + !autodetect) test_compression_handler_detect(&compression_handlers[i]); - test_compression_handler_seek(&compression_handlers[i]); - test_compression_handler_reset(&compression_handlers[i]); - test_compression_handler_partial_parent_write(&compression_handlers[i]); - test_compression_handler_random_io(&compression_handlers[i]); - test_compression_handler_large_random_io(&compression_handlers[i]); - test_compression_handler_errors(&compression_handlers[i]); + test_compression_handler_short(&compression_handlers[i], autodetect); + test_compression_handler(&compression_handlers[i], autodetect); + test_compression_handler_seek(&compression_handlers[i], autodetect); + test_compression_handler_reset(&compression_handlers[i], autodetect); + test_compression_handler_partial_parent_write(&compression_handlers[i], autodetect); + test_compression_handler_random_io(&compression_handlers[i], autodetect); + test_compression_handler_large_random_io(&compression_handlers[i], autodetect); + test_compression_handler_errors(&compression_handlers[i], autodetect); } T_END; } } -static void test_gz(const char *str1, const char *str2) +static void test_compression(void) +{ + test_compression_int(FALSE); + test_compression_int(TRUE); +} + +static void test_istream_decompression_try(void) +{ + const char *tests[] = { + "", + "1", + "12", + "12345678901234567890123456789012345678901234567890", + }; + struct istream *is, *input; + const unsigned char *data; + size_t size; + + test_begin("istream-decompression try"); + + for (unsigned int i = 0; i < N_ELEMENTS(tests); i++) { + size_t test_len = strlen(tests[i]); + is = i_stream_create_from_data(tests[i], test_len); + input = i_stream_create_decompress(is, ISTREAM_DECOMPRESS_FLAG_TRY); + i_stream_unref(&is); + + ssize_t ret = i_stream_read(input); + test_assert_idx((test_len == 0 && ret == -1) || + (test_len > 0 && ret == (ssize_t)test_len), i); + data = i_stream_get_data(input, &size); + test_assert_idx(size == test_len && + memcmp(data, tests[i], size) == 0, i); + + i_stream_skip(input, size); + test_assert_idx(i_stream_read(input) == -1, i); + test_assert_idx(input->stream_errno == 0, i); + i_stream_unref(&input); + } + test_end(); +} + +static void test_gz(const char *str1, const char *str2, bool autodetect) { const struct compression_handler *gz; struct ostream *buf_output, *output; @@ -706,7 +785,8 @@ static void test_gz(const char *str1, const char *str2) size_t size; test_input = test_istream_create_data(buf->data, buf->used); test_istream_set_allow_eof(test_input, FALSE); - input = gz->create_istream(test_input, TRUE); + input = !autodetect ? gz->create_istream(test_input, TRUE) : + i_stream_create_decompress(test_input, 0); for (size_t i = 0; i <= buf->used; i++) { test_istream_set_size(test_input, i); test_assert(i_stream_read(input) >= 0); @@ -725,19 +805,27 @@ static void test_gz(const char *str1, const char *str2) static void test_gz_concat(void) { - test_begin("gz concat"); - test_gz("hello", "world"); + test_begin("gz concat (autodetect=no)"); + test_gz("hello", "world", FALSE); + test_end(); + + test_begin("gz concat (autodetect=yes)"); + test_gz("hello", "world", TRUE); test_end(); } static void test_gz_no_concat(void) { - test_begin("gz no concat"); - test_gz("hello", ""); + test_begin("gz no concat (autodetect=no)"); + test_gz("hello", "", FALSE); + test_end(); + + test_begin("gz no concat (autodetect=yes)"); + test_gz("hello", "", TRUE); test_end(); } -static void test_gz_header(void) +static void test_gz_header_int(bool autodetect) { const struct compression_handler *gz; const char *input_strings[] = { @@ -752,12 +840,14 @@ static void test_gz_header(void) if (compression_lookup_handler("gz", &gz) <= 0 ) return; /* not compiled in or unkown*/ - test_begin("gz header"); + test_begin(t_strdup_printf( + "gz header (autodetect=%s)", autodetect ? "yes" : "no")); for (unsigned int i = 0; i < N_ELEMENTS(input_strings); i++) { file_input = test_istream_create_data(input_strings[i], strlen(input_strings[i])); file_input->blocking = TRUE; - input = gz->create_istream(file_input, FALSE); + input = !autodetect ? gz->create_istream(file_input, FALSE) : + i_stream_create_decompress(file_input, 0); test_assert_idx(i_stream_read(input) == -1, i); test_assert_idx(input->stream_errno == EINVAL, i); i_stream_unref(&input); @@ -766,7 +856,13 @@ static void test_gz_header(void) test_end(); } -static void test_gz_large_header(void) +static void test_gz_header(void) +{ + test_gz_header_int(FALSE); + test_gz_header_int(TRUE); +} + +static void test_gz_large_header_int(bool autodetect) { const struct compression_handler *gz; static const unsigned char gz_input[] = { @@ -780,7 +876,8 @@ static void test_gz_large_header(void) if (compression_lookup_handler("gz", &gz) <= 0 ) return; /* not compiled in or unkown*/ - test_begin("gz large header"); + test_begin(t_strdup_printf( + "gz large header (autodetect=%s)", autodetect ? "yes" : "no")); /* max buffer size smaller than gz header */ for (i = 1; i < sizeof(gz_input); i++) { @@ -788,7 +885,8 @@ static void test_gz_large_header(void) test_istream_set_size(file_input, i); test_istream_set_max_buffer_size(file_input, i); - input = gz->create_istream(file_input, FALSE); + input = !autodetect ? gz->create_istream(file_input, FALSE) : + i_stream_create_decompress(file_input, 0); test_assert_idx(i_stream_read(input) == 0, i); test_assert_idx(i_stream_read(input) == -1 && input->stream_errno == EINVAL, i); @@ -798,10 +896,11 @@ static void test_gz_large_header(void) /* max buffer size is exactly the gz header */ file_input = test_istream_create_data(gz_input, sizeof(gz_input)); - input = gz->create_istream(file_input, TRUE); - test_istream_set_size(input, i); - test_istream_set_allow_eof(input, FALSE); - test_istream_set_max_buffer_size(input, i); + input = !autodetect ? gz->create_istream(file_input, TRUE) : + i_stream_create_decompress(file_input, 0); + test_istream_set_size(file_input, i); + test_istream_set_allow_eof(file_input, FALSE); + test_istream_set_max_buffer_size(file_input, i); test_assert(i_stream_read(input) == 0); i_stream_unref(&input); i_stream_unref(&file_input); @@ -809,6 +908,12 @@ static void test_gz_large_header(void) test_end(); } +static void test_gz_large_header(void) +{ + test_gz_large_header_int(FALSE); + test_gz_large_header_int(TRUE); +} + static void test_uncompress_file(const char *path) { const struct compression_handler *handler; @@ -915,6 +1020,7 @@ int main(int argc, char *argv[]) { static void (*const test_functions[])(void) = { test_compression, + test_istream_decompression_try, test_gz_concat, test_gz_no_concat, test_gz_header, -- 2.47.3