lib-compression: Add istream-decompress

This stream detects the compression format and creates the proper istream
afterwards. This is more efficient than creating each compression istream
and using istream-try.

diff --git a/src/lib-compression/Makefile.am b/src/lib-compression/Makefile.am
index 6c2e5e234eafbb304170a7cb27020619d02513b7..88f38558a35f7e760f1d8778e0bee46fc5956fe9 100644 (file)
--- a/src/lib-compression/Makefile.am
+++ b/src/lib-compression/Makefile.am
@@ -7,6 +7,7 @@ AM_CPPFLAGS = \
 
 libcompression_la_SOURCES = \
        compression.c \
+       istream-decompress.c \
        istream-lzma.c \
        istream-lz4.c \
        istream-zlib.c \

diff --git a/src/lib-compression/compression.h b/src/lib-compression/compression.h
index b27d09e8bc864a559c2bc7fd5d43b6f813c25b7b..9b1f859020cca2741de79f665b819ce4187cfd47 100644 (file)
--- a/src/lib-compression/compression.h
+++ b/src/lib-compression/compression.h
@@ -1,6 +1,12 @@
 #ifndef COMPRESSION_H
 #define COMPRESSION_H
 
+enum istream_decompress_flags {
+       /* If stream isn't detected to be compressed, return it as passthrough
+          istream. */
+       ISTREAM_DECOMPRESS_FLAG_TRY = BIT(0),
+};
+
 /* Compressed input is always detected once at maximum this many bytes have
    been read. This value must be smaller than a typical istream max buffer
    size. */
@@ -29,4 +35,15 @@ compression_detect_handler(struct istream *input);
 int compression_lookup_handler_from_ext(const char *path,
                                        const struct compression_handler **handler_r);
 
+/* Automatically detect the compression format. Note that using tee-istream as
+   one of the parent streams is dangerous here: A decompression istream may
+   have to read a lot of data (e.g. 8 kB isn't enough) before it returns even
+   the first byte as output. If the other tee children aren't read forward,
+   this can cause an infinite loop when i_stream_read() is always returning 0.
+   This is why ISTREAM_DECOMPRESS_FLAG_TRY should be used instead of attempting
+   to implement similar functionality with tee-istream. */
+struct istream *
+i_stream_create_decompress(struct istream *input,
+                          enum istream_decompress_flags flags);
+
 #endif

diff --git a/src/lib-compression/istream-decompress.c b/src/lib-compression/istream-decompress.c
new file mode 100644 (file)
index 0000000..1cd21e4
--- /dev/null
+++ b/src/lib-compression/istream-decompress.c
@@ -0,0 +1,254 @@
+/* Copyright (c) 2020 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "istream-private.h"
+#include "compression.h"
+
+struct decompress_istream {
+       struct istream_private istream;
+       struct istream *compressed_input;
+       struct istream *decompressed_input;
+       enum istream_decompress_flags flags;
+};
+
+static void copy_compressed_input_error(struct decompress_istream *zstream)
+{
+       struct istream_private *stream = &zstream->istream;
+
+       stream->istream.stream_errno = zstream->compressed_input->stream_errno;
+       stream->istream.eof = zstream->compressed_input->eof;
+       if (zstream->compressed_input->real_stream->iostream.error != NULL) {
+               io_stream_set_error(&stream->iostream, "%s",
+                       zstream->compressed_input->real_stream->iostream.error);
+       }
+}
+
+static void copy_decompressed_input_error(struct decompress_istream *zstream)
+{
+       struct istream_private *stream = &zstream->istream;
+
+       stream->istream.stream_errno = zstream->decompressed_input->stream_errno;
+       stream->istream.eof = zstream->decompressed_input->eof;
+       if (zstream->decompressed_input->real_stream->iostream.error != NULL) {
+               io_stream_set_error(&stream->iostream, "%s",
+                       zstream->decompressed_input->real_stream->iostream.error);
+       }
+}
+
+static void
+i_stream_decompress_close(struct iostream_private *_stream, bool close_parent)
+{
+       struct istream_private *stream =
+               container_of(_stream, struct istream_private, iostream);
+       struct decompress_istream *zstream =
+               container_of(stream, struct decompress_istream, istream);
+
+       if (zstream->decompressed_input != NULL)
+               i_stream_close(zstream->decompressed_input);
+       if (close_parent)
+               i_stream_close(zstream->compressed_input);
+}
+
+static void
+i_stream_decompress_destroy(struct iostream_private *_stream)
+{
+       struct istream_private *stream =
+               container_of(_stream, struct istream_private, iostream);
+       struct decompress_istream *zstream =
+               container_of(stream, struct decompress_istream, istream);
+
+       i_stream_unref(&zstream->decompressed_input);
+       i_stream_unref(&zstream->compressed_input);
+}
+
+static int
+i_stream_decompress_not_compressed(struct decompress_istream *zstream)
+{
+       if ((zstream->flags & ISTREAM_DECOMPRESS_FLAG_TRY) == 0) {
+               zstream->istream.istream.stream_errno = EINVAL;
+               io_stream_set_error(&zstream->istream.iostream,
+                                   "Stream isn't compressed");
+               return -1;
+       } else {
+               zstream->decompressed_input = zstream->compressed_input;
+               i_stream_ref(zstream->decompressed_input);
+               return 1;
+       }
+}
+
+static int i_stream_decompress_detect(struct decompress_istream *zstream)
+{
+       const struct compression_handler *handler;
+       ssize_t ret;
+
+       ret = i_stream_read(zstream->compressed_input);
+       handler = compression_detect_handler(zstream->compressed_input);
+       if (handler == NULL) {
+               switch (ret) {
+               case -1:
+                       if (zstream->compressed_input->stream_errno != 0) {
+                               copy_compressed_input_error(zstream);
+                               return -1;
+                       }
+                       /* fall through */
+               case -2:
+                       /* we've read a full buffer or we reached EOF -
+                          the stream isn't compressed */
+                       return i_stream_decompress_not_compressed(zstream);
+               case 0:
+                       return 0;
+               default:
+                       if (!zstream->istream.istream.blocking)
+                               return 0;
+                       return i_stream_decompress_detect(zstream);
+               }
+       }
+       if (handler->create_istream == NULL) {
+               zstream->istream.istream.stream_errno = EINVAL;
+               io_stream_set_error(&zstream->istream.iostream,
+                       "Compression handler %s not supported", handler->name);
+               return -1;
+       }
+
+       zstream->decompressed_input =
+               handler->create_istream(zstream->compressed_input, FALSE);
+       return 1;
+}
+
+static ssize_t i_stream_decompress_read(struct istream_private *stream)
+{
+       struct decompress_istream *zstream =
+               container_of(stream, struct decompress_istream, istream);
+       ssize_t ret;
+       size_t pos;
+
+       if (zstream->decompressed_input == NULL) {
+               if ((ret = i_stream_decompress_detect(zstream)) <= 0)
+                       return ret;
+       }
+
+       i_stream_seek(zstream->decompressed_input, stream->istream.v_offset);
+       stream->pos -= stream->skip;
+       stream->skip = 0;
+
+       stream->buffer = i_stream_get_data(zstream->decompressed_input, &pos);
+       if (pos > stream->pos)
+               ret = 0;
+       else do {
+               ret = i_stream_read_memarea(zstream->decompressed_input);
+               copy_decompressed_input_error(zstream);
+               stream->buffer = i_stream_get_data(zstream->decompressed_input,
+                                                  &pos);
+       } while (pos <= stream->pos && ret > 0);
+       if (ret == -2)
+               return -2;
+
+       if (pos <= stream->pos)
+               ret = ret == 0 ? 0 : -1;
+       else
+               ret = (ssize_t)(pos - stream->pos);
+       stream->pos = pos;
+       i_assert(ret != -1 || stream->istream.eof ||
+                stream->istream.stream_errno != 0);
+       return ret;
+}
+
+static void i_stream_decompress_reset(struct istream_private *stream)
+{
+       stream->skip = stream->pos = 0;
+       stream->istream.v_offset = 0;
+       stream->istream.eof = FALSE;
+}
+
+static void
+i_stream_decompress_seek(struct istream_private *stream,
+                        uoff_t v_offset, bool mark)
+{
+       struct decompress_istream *zstream =
+               container_of(stream, struct decompress_istream, istream);
+
+       if (zstream->decompressed_input == NULL) {
+               if (!i_stream_nonseekable_try_seek(stream, v_offset))
+                       i_panic("seeking backwards before detecting compression format");
+       } else {
+               i_stream_decompress_reset(stream);
+               stream->istream.v_offset = v_offset;
+               if (mark)
+                       i_stream_seek_mark(zstream->decompressed_input, v_offset);
+               else
+                       i_stream_seek(zstream->decompressed_input, v_offset);
+               copy_decompressed_input_error(zstream);
+       }
+}
+
+static void i_stream_decompress_sync(struct istream_private *stream)
+{
+       struct decompress_istream *zstream =
+               container_of(stream, struct decompress_istream, istream);
+
+       i_stream_decompress_reset(stream);
+       if (zstream->decompressed_input != NULL)
+               i_stream_sync(zstream->decompressed_input);
+}
+
+static int i_stream_decompress_stat(struct istream_private *stream, bool exact)
+{
+       struct decompress_istream *zstream =
+               container_of(stream, struct decompress_istream, istream);
+       const struct stat *st;
+
+       if (!exact) {
+               if (i_stream_stat(zstream->compressed_input, exact, &st) < 0) {
+                       copy_compressed_input_error(zstream);
+                       return -1;
+               }
+               stream->statbuf = *st;
+               return 0;
+       }
+       if (zstream->decompressed_input == NULL) {
+               (void)i_stream_read(&stream->istream);
+               if (zstream->decompressed_input == NULL) {
+                       if (stream->istream.stream_errno == 0) {
+                               zstream->istream.istream.stream_errno = EINVAL;
+                               io_stream_set_error(&zstream->istream.iostream,
+                                       "Stream compression couldn't be detected during stat");
+                       }
+                       return -1;
+               }
+       }
+
+       if (i_stream_stat(zstream->decompressed_input, exact, &st) < 0) {
+               copy_decompressed_input_error(zstream);
+               return -1;
+       }
+       i_stream_decompress_reset(stream);
+       stream->statbuf = *st;
+       return 0;
+}
+
+struct istream *
+i_stream_create_decompress(struct istream *input,
+                          enum istream_decompress_flags flags)
+{
+       struct decompress_istream *zstream;
+
+       zstream = i_new(struct decompress_istream, 1);
+       zstream->compressed_input = input;
+       zstream->flags = flags;
+       i_stream_ref(input);
+
+       zstream->istream.iostream.close = i_stream_decompress_close;
+       zstream->istream.iostream.destroy = i_stream_decompress_destroy;
+       zstream->istream.max_buffer_size = input->real_stream->max_buffer_size;
+       zstream->istream.read = i_stream_decompress_read;
+       zstream->istream.seek = i_stream_decompress_seek;
+       zstream->istream.sync = i_stream_decompress_sync;
+       zstream->istream.stat = i_stream_decompress_stat;
+
+       zstream->istream.istream.readable_fd = FALSE;
+       zstream->istream.istream.blocking = input->blocking;
+       zstream->istream.istream.seekable = input->seekable;
+
+       return i_stream_create(&zstream->istream, NULL,
+                              i_stream_get_fd(input), 0);
+}

diff --git a/src/lib-compression/test-compression.c b/src/lib-compression/test-compression.c
index 1bf97c48c903b9da500b313f852135407c9f122b..8cf45caa38abea17dff288bac8d3d185fe440582 100644 (file)
--- a/src/lib-compression/test-compression.c
+++ b/src/lib-compression/test-compression.c
@@ -63,7 +63,9 @@ static void test_compression_handler_detect(const struct compression_handler *ha
        test_end();
 }
 
-static void test_compression_handler_short(const struct compression_handler *handler)
+static void
+test_compression_handler_short(const struct compression_handler *handler,
+                              bool autodetect)
 {
        const unsigned char *data;
        size_t len, size;
@@ -76,7 +78,8 @@ static void test_compression_handler_short(const struct compression_handler *han
        struct istream *input;
 
        /* write some amount of data */
-       test_begin(t_strdup_printf("compression handler %s (small)", handler->name));
+       test_begin(t_strdup_printf("compression handler %s (small, autodetect=%s)",
+                                  handler->name, autodetect ? "yes" : "no"));
        len = i_rand_minmax(1, 1024);
        test_data = buffer_create_dynamic(default_pool, len);
        random_fill(buffer_append_space_unsafe(test_data, len), len);
@@ -95,7 +98,8 @@ static void test_compression_handler_short(const struct compression_handler *han
 
        /* read data at once */
        test_input = test_istream_create_data(buffer->data, buffer->used);
-       input = handler->create_istream(test_input, TRUE);
+       input = !autodetect ? handler->create_istream(test_input, TRUE) :
+               i_stream_create_decompress(test_input, 0);
        i_stream_unref(&test_input);
 
        test_assert(i_stream_read_more(input, &data, &size) > 0);
@@ -110,7 +114,9 @@ static void test_compression_handler_short(const struct compression_handler *han
        test_end();
 }
 
-static void test_compression_handler_seek(const struct compression_handler *handler)
+static void
+test_compression_handler_seek(const struct compression_handler *handler,
+                             bool autodetect)
 {
        const unsigned char *data,*ptr;
        size_t len, size, pos;
@@ -123,7 +129,8 @@ static void test_compression_handler_seek(const struct compression_handler *hand
        struct istream *input;
 
        /* write some amount of data */
-       test_begin(t_strdup_printf("compression handler %s (seek)", handler->name));
+       test_begin(t_strdup_printf("compression handler %s (seek, autodetect=%s)",
+                                  handler->name, autodetect ? "yes" : "no"));
        len = i_rand_minmax(1024, 2048);
        test_data = buffer_create_dynamic(default_pool, len);
        random_fill(buffer_append_space_unsafe(test_data, len), len);
@@ -141,7 +148,8 @@ static void test_compression_handler_seek(const struct compression_handler *hand
        o_stream_unref(&output);
 
        test_input = test_istream_create_data(buffer->data, buffer->used);
-       input = handler->create_istream(test_input, TRUE);
+       input = !autodetect ? handler->create_istream(test_input, TRUE) :
+               i_stream_create_decompress(test_input, 0);
        i_stream_unref(&test_input);
 
        /* seek forward */
@@ -170,7 +178,9 @@ static void test_compression_handler_seek(const struct compression_handler *hand
        test_end();
 }
 
-static void test_compression_handler_reset(const struct compression_handler *handler)
+static void
+test_compression_handler_reset(const struct compression_handler *handler,
+                              bool autodetect)
 {
        const unsigned char *data;
        size_t len, size;
@@ -183,7 +193,8 @@ static void test_compression_handler_reset(const struct compression_handler *han
        struct istream *input;
 
        /* write some amount of data */
-       test_begin(t_strdup_printf("compression handler %s (reset)", handler->name));
+       test_begin(t_strdup_printf("compression handler %s (reset, autodetect=%s)",
+                                  handler->name, autodetect ? "yes" : "no"));
        len = i_rand_minmax(1024, 2048);
        test_data = buffer_create_dynamic(default_pool, len);
        random_fill(buffer_append_space_unsafe(test_data, len), len);
@@ -201,7 +212,8 @@ static void test_compression_handler_reset(const struct compression_handler *han
        o_stream_unref(&output);
 
        test_input = test_istream_create_data(buffer->data, buffer->used);
-       input = handler->create_istream(test_input, TRUE);
+       input = !autodetect ? handler->create_istream(test_input, TRUE) :
+               i_stream_create_decompress(test_input, 0);
        i_stream_unref(&test_input);
 
        /* seek forward */
@@ -227,7 +239,9 @@ static void test_compression_handler_reset(const struct compression_handler *han
        test_end();
 }
 
-static void test_compression_handler(const struct compression_handler *handler)
+static void
+test_compression_handler(const struct compression_handler *handler,
+                        bool autodetect)
 {
        const char *path = "test-compression.tmp";
        struct istream *file_input, *input;
@@ -242,7 +256,8 @@ static void test_compression_handler(const struct compression_handler *handler)
        int fd;
        ssize_t ret;
 
-       test_begin(t_strdup_printf("compression handler %s", handler->name));
+       test_begin(t_strdup_printf("compression handler %s (autodetect=%s)",
+                                  handler->name, autodetect ? "yes" : "no"));
 
        /* write compressed data */
        fd = open(path, O_TRUNC | O_CREAT | O_RDWR, 0600);
@@ -291,7 +306,8 @@ static void test_compression_handler(const struct compression_handler *handler)
 
        /* read and uncompress the data */
        file_input = i_stream_create_fd(fd, IO_BLOCK_SIZE);
-       input = handler->create_istream(file_input, TRUE);
+       input = !autodetect ? handler->create_istream(file_input, TRUE) :
+               i_stream_create_decompress(file_input, 0);
 
        test_assert(i_stream_get_size(input, FALSE, &stream_size) == 1);
        test_assert(stream_size == compressed_size);
@@ -326,9 +342,12 @@ static void test_compression_handler(const struct compression_handler *handler)
        test_end();
 }
 
-static void test_compression_handler_partial_parent_write(const struct compression_handler *handler)
+static void
+test_compression_handler_partial_parent_write(const struct compression_handler *handler,
+                                             bool autodetect)
 {
-       test_begin(t_strdup_printf("compression handler %s (partial parent writes)", handler->name));
+       test_begin(t_strdup_printf("compression handler %s (partial parent writes, autodetect=%s)",
+                                  handler->name, autodetect ? "yes" : "no"));
 
        int ret;
        buffer_t *buffer = t_buffer_create(64);
@@ -362,7 +381,9 @@ static void test_compression_handler_partial_parent_write(const struct compressi
                 buffer_append(compressed_data, buffer->data, buffer->used);
 
        struct istream *is = test_istream_create_data(compressed_data->data, compressed_data->used);
-       struct istream *is_decompressed = handler->create_istream(is, TRUE);
+       struct istream *is_decompressed =
+               !autodetect ? handler->create_istream(is, TRUE) :
+               i_stream_create_decompress(is, 0);
        i_stream_unref(&is);
 
        const unsigned char *data;
@@ -386,7 +407,8 @@ static void test_compression_handler_partial_parent_write(const struct compressi
 }
 
 static void
-test_compression_handler_random_io(const struct compression_handler *handler)
+test_compression_handler_random_io(const struct compression_handler *handler,
+                                  bool autodetect)
 {
        unsigned char in_buf[8192];
        size_t in_buf_size;
@@ -397,8 +419,8 @@ test_compression_handler_random_io(const struct compression_handler *handler)
        enc_buf = buffer_create_dynamic(default_pool, sizeof(in_buf));
        dec_buf = buffer_create_dynamic(default_pool, sizeof(in_buf));
 
-       test_begin(t_strdup_printf("compression handler %s (random I/O)",
-                                  handler->name));
+       test_begin(t_strdup_printf("compression handler %s (random I/O, autodetect=%s)",
+                                  handler->name, autodetect ? "yes" : "no"));
 
        for (i = 0; !test_has_failed() && i < 300; i++) {
                struct istream *input1, *input2;
@@ -474,7 +496,8 @@ test_compression_handler_random_io(const struct compression_handler *handler)
                i_stream_set_name(input1, "[compressed-data]");
 
                /* Create decompressor stream */
-               input2 = handler->create_istream(input1, TRUE);
+               input2 = !autodetect ? handler->create_istream(input1, TRUE) :
+                       i_stream_create_decompress(input1, 0);
                i_stream_set_name(input2, "[decompressor]");
 
                /* Assign random buffer sizes */
@@ -528,7 +551,8 @@ test_compression_handler_random_io(const struct compression_handler *handler)
 }
 
 static void
-test_compression_handler_large_random_io(const struct compression_handler *handler)
+test_compression_handler_large_random_io(const struct compression_handler *handler,
+                                        bool autodetect)
 {
 #define RANDOMNESS_SIZE (1024*1024)
        unsigned char *randomness;
@@ -538,7 +562,8 @@ test_compression_handler_large_random_io(const struct compression_handler *handl
        size_t size;
        int ret;
 
-       test_begin(t_strdup_printf("compression handler %s (large random io)", handler->name));
+       test_begin(t_strdup_printf("compression handler %s (large random io, autodetect=%s)",
+                                  handler->name, autodetect ? "yes" : "no"));
        randomness = i_malloc(RANDOMNESS_SIZE);
        random_fill(randomness, RANDOMNESS_SIZE);
 
@@ -569,7 +594,8 @@ test_compression_handler_large_random_io(const struct compression_handler *handl
 
        /* verify that reading the input works */
 
-       dec_input = handler->create_istream(input, FALSE);
+       dec_input = !autodetect ? handler->create_istream(input, TRUE) :
+               i_stream_create_decompress(input, 0);
 
        while ((ret = i_stream_read_more(dec_input, &data, &size)) > 0) {
                test_assert(memcmp(data, randomness + dec_input->v_offset, size) == 0);
@@ -585,13 +611,18 @@ test_compression_handler_large_random_io(const struct compression_handler *handl
        test_end();
 }
 
-static void test_compression_handler_errors(const struct compression_handler *handler)
+static void
+test_compression_handler_errors(const struct compression_handler *handler,
+                               bool autodetect)
 {
-       test_begin(t_strdup_printf("compression handler %s (errors)", handler->name));
+       test_begin(t_strdup_printf("compression handler %s (errors, autodetect=%s)",
+                                  handler->name, autodetect ? "yes" : "no"));
 
        /* test that zero stream reading errors out */
        struct istream *is = test_istream_create("");
-       struct istream *input = handler->create_istream(is, FALSE);
+       struct istream *input =
+               !autodetect ? handler->create_istream(is, FALSE) :
+               i_stream_create_decompress(is, 0);
        i_stream_unref(&is);
        test_assert(i_stream_read(input) == -1 && input->eof);
        i_stream_unref(&input);
@@ -600,7 +631,9 @@ static void test_compression_handler_errors(const struct compression_handler *ha
        is = test_istream_create("dedededededededededededededede"
                                 "dedededeededdedededededededede"
                                 "dedededededededededededededede");
-       input = handler->create_istream(is, FALSE);
+       is->blocking = TRUE;
+       input = !autodetect ? handler->create_istream(is, FALSE) :
+               i_stream_create_decompress(is, 0);
        i_stream_unref(&is);
        test_assert(i_stream_read(input) == -1 && input->eof);
        i_stream_unref(&input);
@@ -622,7 +655,8 @@ static void test_compression_handler_errors(const struct compression_handler *ha
 
        /* truncate buffer */
        is = test_istream_create_data(odata->data, odata->used - sizeof(buf)*2 - 1);
-       input = handler->create_istream(is, FALSE);
+       input = !autodetect ? handler->create_istream(is, FALSE) :
+               i_stream_create_decompress(is, 0);
        i_stream_unref(&is);
 
        const unsigned char *data ATTR_UNUSED;
@@ -639,7 +673,8 @@ static void test_compression_handler_errors(const struct compression_handler *ha
                   that should not match any handlers' header */
                for (size_t i = 0; i < 32; i++) {
                        is = test_istream_create_data("dededededededededededededededede", i);
-                       input = handler->create_istream(is, FALSE);
+                       input = !autodetect ? handler->create_istream(is, FALSE) :
+                               i_stream_create_decompress(is, 0);
                        i_stream_unref(&is);
                        while (i_stream_read_more(input, &data, &size) >= 0) {
                                test_assert_idx(size == 0, i);
@@ -653,27 +688,71 @@ static void test_compression_handler_errors(const struct compression_handler *ha
        test_end();
 }
 
-static void test_compression(void)
+static void test_compression_int(bool autodetect)
 {
        unsigned int i;
 
        for (i = 0; compression_handlers[i].name != NULL; i++) {
-               if (compression_handlers[i].create_istream != NULL) T_BEGIN {
-                       test_compression_handler_short(&compression_handlers[i]);
-                       test_compression_handler(&compression_handlers[i]);
-                       if (compression_handlers[i].is_compressed != NULL)
+               if (compression_handlers[i].create_istream != NULL &&
+                   (!autodetect ||
+                    compression_handlers[i].is_compressed != NULL)) T_BEGIN {
+                       if (compression_handlers[i].is_compressed != NULL &&
+                           !autodetect)
                                test_compression_handler_detect(&compression_handlers[i]);
-                       test_compression_handler_seek(&compression_handlers[i]);
-                       test_compression_handler_reset(&compression_handlers[i]);
-                       test_compression_handler_partial_parent_write(&compression_handlers[i]);
-                       test_compression_handler_random_io(&compression_handlers[i]);
-                       test_compression_handler_large_random_io(&compression_handlers[i]);
-                       test_compression_handler_errors(&compression_handlers[i]);
+                       test_compression_handler_short(&compression_handlers[i], autodetect);
+                       test_compression_handler(&compression_handlers[i], autodetect);
+                       test_compression_handler_seek(&compression_handlers[i], autodetect);
+                       test_compression_handler_reset(&compression_handlers[i], autodetect);
+                       test_compression_handler_partial_parent_write(&compression_handlers[i], autodetect);
+                       test_compression_handler_random_io(&compression_handlers[i], autodetect);
+                       test_compression_handler_large_random_io(&compression_handlers[i], autodetect);
+                       test_compression_handler_errors(&compression_handlers[i], autodetect);
                } T_END;
        }
 }
 
-static void test_gz(const char *str1, const char *str2)
+static void test_compression(void)
+{
+       test_compression_int(FALSE);
+       test_compression_int(TRUE);
+}
+
+static void test_istream_decompression_try(void)
+{
+       const char *tests[] = {
+               "",
+               "1",
+               "12",
+               "12345678901234567890123456789012345678901234567890",
+       };
+       struct istream *is, *input;
+       const unsigned char *data;
+       size_t size;
+
+       test_begin("istream-decompression try");
+
+       for (unsigned int i = 0; i < N_ELEMENTS(tests); i++) {
+               size_t test_len = strlen(tests[i]);
+               is = i_stream_create_from_data(tests[i], test_len);
+               input = i_stream_create_decompress(is, ISTREAM_DECOMPRESS_FLAG_TRY);
+               i_stream_unref(&is);
+
+               ssize_t ret = i_stream_read(input);
+               test_assert_idx((test_len == 0 && ret == -1) ||
+                               (test_len > 0 && ret == (ssize_t)test_len), i);
+               data = i_stream_get_data(input, &size);
+               test_assert_idx(size == test_len &&
+                               memcmp(data, tests[i], size) == 0, i);
+
+               i_stream_skip(input, size);
+               test_assert_idx(i_stream_read(input) == -1, i);
+               test_assert_idx(input->stream_errno == 0, i);
+               i_stream_unref(&input);
+       }
+       test_end();
+}
+
+static void test_gz(const char *str1, const char *str2, bool autodetect)
 {
        const struct compression_handler *gz;
        struct ostream *buf_output, *output;
@@ -706,7 +785,8 @@ static void test_gz(const char *str1, const char *str2)
        size_t size;
        test_input = test_istream_create_data(buf->data, buf->used);
        test_istream_set_allow_eof(test_input, FALSE);
-       input = gz->create_istream(test_input, TRUE);
+       input = !autodetect ? gz->create_istream(test_input, TRUE) :
+               i_stream_create_decompress(test_input, 0);
        for (size_t i = 0; i <= buf->used; i++) {
                test_istream_set_size(test_input, i);
                test_assert(i_stream_read(input) >= 0);
@@ -725,19 +805,27 @@ static void test_gz(const char *str1, const char *str2)
 
 static void test_gz_concat(void)
 {
-       test_begin("gz concat");
-       test_gz("hello", "world");
+       test_begin("gz concat (autodetect=no)");
+       test_gz("hello", "world", FALSE);
+       test_end();
+
+       test_begin("gz concat (autodetect=yes)");
+       test_gz("hello", "world", TRUE);
        test_end();
 }
 
 static void test_gz_no_concat(void)
 {
-       test_begin("gz no concat");
-       test_gz("hello", "");
+       test_begin("gz no concat (autodetect=no)");
+       test_gz("hello", "", FALSE);
+       test_end();
+
+       test_begin("gz no concat (autodetect=yes)");
+       test_gz("hello", "", TRUE);
        test_end();
 }
 
-static void test_gz_header(void)
+static void test_gz_header_int(bool autodetect)
 {
        const struct compression_handler *gz;
        const char *input_strings[] = {
@@ -752,12 +840,14 @@ static void test_gz_header(void)
        if (compression_lookup_handler("gz", &gz) <= 0 )
                return; /* not compiled in or unkown*/
 
-       test_begin("gz header");
+       test_begin(t_strdup_printf(
+               "gz header (autodetect=%s)", autodetect ? "yes" : "no"));
        for (unsigned int i = 0; i < N_ELEMENTS(input_strings); i++) {
                file_input = test_istream_create_data(input_strings[i],
                                                      strlen(input_strings[i]));
                file_input->blocking = TRUE;
-               input = gz->create_istream(file_input, FALSE);
+               input = !autodetect ? gz->create_istream(file_input, FALSE) :
+                       i_stream_create_decompress(file_input, 0);
                test_assert_idx(i_stream_read(input) == -1, i);
                test_assert_idx(input->stream_errno == EINVAL, i);
                i_stream_unref(&input);
@@ -766,7 +856,13 @@ static void test_gz_header(void)
        test_end();
 }
 
-static void test_gz_large_header(void)
+static void test_gz_header(void)
+{
+       test_gz_header_int(FALSE);
+       test_gz_header_int(TRUE);
+}
+
+static void test_gz_large_header_int(bool autodetect)
 {
        const struct compression_handler *gz;
        static const unsigned char gz_input[] = {
@@ -780,7 +876,8 @@ static void test_gz_large_header(void)
        if (compression_lookup_handler("gz", &gz) <= 0 )
                return; /* not compiled in or unkown*/
 
-       test_begin("gz large header");
+       test_begin(t_strdup_printf(
+               "gz large header (autodetect=%s)", autodetect ? "yes" : "no"));
 
        /* max buffer size smaller than gz header */
        for (i = 1; i < sizeof(gz_input); i++) {
@@ -788,7 +885,8 @@ static void test_gz_large_header(void)
                test_istream_set_size(file_input, i);
                test_istream_set_max_buffer_size(file_input, i);
 
-               input = gz->create_istream(file_input, FALSE);
+               input = !autodetect ? gz->create_istream(file_input, FALSE) :
+                       i_stream_create_decompress(file_input, 0);
                test_assert_idx(i_stream_read(input) == 0, i);
                test_assert_idx(i_stream_read(input) == -1 &&
                                input->stream_errno == EINVAL, i);
@@ -798,10 +896,11 @@ static void test_gz_large_header(void)
 
        /* max buffer size is exactly the gz header */
        file_input = test_istream_create_data(gz_input, sizeof(gz_input));
-       input = gz->create_istream(file_input, TRUE);
-       test_istream_set_size(input, i);
-       test_istream_set_allow_eof(input, FALSE);
-       test_istream_set_max_buffer_size(input, i);
+       input = !autodetect ? gz->create_istream(file_input, TRUE) :
+               i_stream_create_decompress(file_input, 0);
+       test_istream_set_size(file_input, i);
+       test_istream_set_allow_eof(file_input, FALSE);
+       test_istream_set_max_buffer_size(file_input, i);
        test_assert(i_stream_read(input) == 0);
        i_stream_unref(&input);
        i_stream_unref(&file_input);
@@ -809,6 +908,12 @@ static void test_gz_large_header(void)
        test_end();
 }
 
+static void test_gz_large_header(void)
+{
+       test_gz_large_header_int(FALSE);
+       test_gz_large_header_int(TRUE);
+}
+
 static void test_uncompress_file(const char *path)
 {
        const struct compression_handler *handler;
@@ -915,6 +1020,7 @@ int main(int argc, char *argv[])
 {
        static void (*const test_functions[])(void) = {
                test_compression,
+               test_istream_decompression_try,
                test_gz_concat,
                test_gz_no_concat,
                test_gz_header,