From: Timo Sirainen <tss@iki.fi>
Date: Tue, 14 Jan 2014 22:57:59 +0000 (+0200)
Subject: lib-compression: Added initial support for LZ4
X-Git-Tag: 2.2.11~37
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=a49d1c2ca3c134c0c62b37a94936c78e9849e044;p=thirdparty%2Fdovecot%2Fcore.git

lib-compression: Added initial support for LZ4
There's no standard file format for LZ4, so we created our own. The code has
had only minimal testing currently, so there may be bugs.
---

diff --git a/configure.ac b/configure.ac
index 95984bb224..c5623b9aea 100644
--- a/configure.ac
+++ b/configure.ac
@@ -184,6 +184,11 @@ AS_HELP_STRING([--with-lzma], [Build with LZMA compression support]),
   TEST_WITH(lzma, $withval),
   want_lzma=auto)
 
+AC_ARG_WITH(lz4,
+AS_HELP_STRING([--with-lz4], [Build with LZ4 compression support]),
+  TEST_WITH(lz4, $withval),
+  want_lz4=auto)
+
 AC_ARG_WITH(libcap,
 AS_HELP_STRING([--with-libcap], [Build with libcap support (Dropping capabilities).]),
   TEST_WITH(libcap, $withval),
@@ -2691,6 +2696,27 @@ if test "$want_lzma" != "no"; then
   ])
 fi
 AC_SUBST(COMPRESS_LIBS)
+
+if test "$want_lz4" != "no"; then
+  AC_CHECK_HEADER(lz4.h, [
+    AC_CHECK_LIB(lz4, LZ4_compress, [
+      have_lz4=yes
+      have_compress_lib=yes
+      AC_DEFINE(HAVE_LZ4,, Define if you have lz4 library)
+      COMPRESS_LIBS="$COMPRESS_LIBS -llz4"
+    ], [
+      if test "$want_lz4" = "yes"; then
+	AC_ERROR([Can't build with lz4 support: liblz4 not found])
+      fi
+    ])
+  ], [
+    if test "$want_lz4" = "yes"; then
+      AC_ERROR([Can't build with lz4 support: lz4.h not found])
+    fi
+  ])
+fi
+AC_SUBST(COMPRESS_LIBS)
+
 AM_CONDITIONAL(BUILD_ZLIB_PLUGIN, test "$have_compress_lib" = "yes")
 
 RPCGEN=${RPCGEN-rpcgen}
diff --git a/src/lib-compression/Makefile.am b/src/lib-compression/Makefile.am
index c353742aff..46414a49f0 100644
--- a/src/lib-compression/Makefile.am
+++ b/src/lib-compression/Makefile.am
@@ -7,9 +7,11 @@ AM_CPPFLAGS = \
 libcompression_la_SOURCES = \
 	compression.c \
 	istream-lzma.c \
+	istream-lz4.c \
 	istream-zlib.c \
 	istream-bzlib.c \
 	ostream-lzma.c \
+	ostream-lz4.c \
 	ostream-zlib.c \
 	ostream-bzlib.c
 libcompression_la_LIBADD = \
diff --git a/src/lib-compression/compression.c b/src/lib-compression/compression.c
index 8fc515145d..ed466a0fc9 100644
--- a/src/lib-compression/compression.c
+++ b/src/lib-compression/compression.c
@@ -4,6 +4,7 @@
 #include "istream.h"
 #include "istream-zlib.h"
 #include "ostream-zlib.h"
+#include "iostream-lz4.h"
 #include "compression.h"
 
 #ifndef HAVE_ZLIB
@@ -20,6 +21,10 @@
 #  define i_stream_create_lzma NULL
 #  define o_stream_create_lzma NULL
 #endif
+#ifndef HAVE_LZ4
+#  define i_stream_create_lz4 NULL
+#  define o_stream_create_lz4 NULL
+#endif
 
 static bool is_compressed_zlib(struct istream *input)
 {
@@ -63,6 +68,17 @@ static bool is_compressed_xz(struct istream *input)
 	return memcmp(data, "\xfd\x37\x7a\x58\x5a", 6) == 0;
 }
 
+static bool is_compressed_lz4(struct istream *input)
+{
+	const unsigned char *data;
+	size_t size;
+
+	if (i_stream_read_data(input, &data, &size, 6 - 1) <= 0)
+		return FALSE;
+	/* there is no standard LZ4 header, so we've created our own */
+	return memcmp(data, IOSTREAM_LZ4_MAGIC, IOSTREAM_LZ4_MAGIC_LEN) == 0;
+}
+
 const struct compression_handler *compression_lookup_handler(const char *name)
 {
 	unsigned int i;
@@ -113,5 +129,7 @@ const struct compression_handler compression_handlers[] = {
 	  i_stream_create_deflate, o_stream_create_deflate },
 	{ "xz", ".xz", is_compressed_xz,
 	  i_stream_create_lzma, o_stream_create_lzma },
+	{ "lz4", ".lz4", is_compressed_lz4,
+	  i_stream_create_lz4, o_stream_create_lz4 },
 	{ NULL, NULL, NULL, NULL, NULL }
 };
diff --git a/src/lib-compression/iostream-lz4.h b/src/lib-compression/iostream-lz4.h
new file mode 100644
index 0000000000..a0897f5904
--- /dev/null
+++ b/src/lib-compression/iostream-lz4.h
@@ -0,0 +1,30 @@
+#ifndef IOSTREAM_LZ4_H
+#define IOSTREAM_LZ4_H
+
+/*
+   Dovecot's LZ4 compressed files contain:
+
+   IOSTREAM_LZ4_HEADER
+   n x (4 byte big-endian: compressed chunk length, compressed chunk)
+*/
+
+#define IOSTREAM_LZ4_MAGIC "Dovecot-LZ4\x0d\x2a\x9b\xc5"
+#define IOSTREAM_LZ4_MAGIC_LEN (sizeof(IOSTREAM_LZ4_MAGIC)-1)
+
+struct iostream_lz4_header {
+	unsigned char magic[IOSTREAM_LZ4_MAGIC_LEN];
+	/* OSTREAM_LZ4_CHUNK_SIZE in big-endian */
+	unsigned char max_uncompressed_chunk_size[4];
+};
+
+/* How large chunks we're buffering into memory before compressing them */
+#define OSTREAM_LZ4_CHUNK_SIZE (1024*64)
+/* How large chunks we allow in input data before returning a failure.
+   This must be at least OSTREAM_LZ4_CHUNK_SIZE, but for future compatibility
+   should be somewhat higher (but not too high to avoid wasting memory for
+   corrupted files). */
+#define ISTREAM_LZ4_CHUNK_SIZE (1024*1024)
+
+#define IOSTREAM_LZ4_CHUNK_PREFIX_LEN 4 /* big-endian size of chunk */
+
+#endif
diff --git a/src/lib-compression/istream-lz4.c b/src/lib-compression/istream-lz4.c
new file mode 100644
index 0000000000..b98038d6d0
--- /dev/null
+++ b/src/lib-compression/istream-lz4.c
@@ -0,0 +1,318 @@
+/* Copyright (c) 2013-2014 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+
+#ifdef HAVE_LZ4
+
+#include "buffer.h"
+#include "istream-private.h"
+#include "istream-zlib.h"
+#include "iostream-lz4.h"
+#include <lz4.h>
+
+struct lz4_istream {
+	struct istream_private istream;
+
+	uoff_t stream_size;
+	struct stat last_parent_statbuf;
+
+	buffer_t *chunk_buf;
+	uint32_t chunk_size, chunk_left, max_uncompressed_chunk_size;
+
+	unsigned int log_errors:1;
+	unsigned int marked:1;
+	unsigned int header_read:1;
+};
+
+static void i_stream_lz4_close(struct iostream_private *stream,
+			       bool close_parent)
+{
+	struct lz4_istream *zstream = (struct lz4_istream *)stream;
+
+	buffer_free(&zstream->chunk_buf);
+	if (close_parent)
+		i_stream_close(zstream->istream.parent);
+}
+
+static void lz4_read_error(struct lz4_istream *zstream, const char *error)
+{
+	io_stream_set_error(&zstream->istream.iostream,
+			    "lz4.read(%s): %s at %"PRIuUOFF_T,
+			    i_stream_get_name(&zstream->istream.istream), error,
+			    zstream->istream.abs_start_offset +
+			    zstream->istream.istream.v_offset);
+	if (zstream->log_errors)
+		i_error("%s", zstream->istream.iostream.error);
+}
+
+static int i_stream_lz4_read_header(struct lz4_istream *zstream)
+{
+	const struct iostream_lz4_header *hdr;
+	const unsigned char *data;
+	size_t size;
+	int ret;
+
+	ret = i_stream_read_data(zstream->istream.parent, &data, &size,
+				 sizeof(*hdr)-1);
+	if (ret < 0) {
+		zstream->istream.istream.stream_errno =
+			zstream->istream.parent->stream_errno;
+		return ret;
+	}
+	if (ret == 0 && !zstream->istream.istream.eof)
+		return 0;
+	hdr = (const void *)data;
+	if (ret == 0 || memcmp(hdr->magic, IOSTREAM_LZ4_MAGIC,
+			       IOSTREAM_LZ4_MAGIC_LEN) != 0) {
+		lz4_read_error(zstream, "wrong magic in header (not lz4 file?)");
+		zstream->istream.istream.stream_errno = EINVAL;
+		return -1;
+	}
+	zstream->max_uncompressed_chunk_size =
+		((uint32_t)hdr->max_uncompressed_chunk_size[0] << 24) |
+		(hdr->max_uncompressed_chunk_size[1] << 16) |
+		(hdr->max_uncompressed_chunk_size[2] << 8) |
+		hdr->max_uncompressed_chunk_size[3];
+	if (zstream->max_uncompressed_chunk_size > ISTREAM_LZ4_CHUNK_SIZE) {
+		lz4_read_error(zstream, t_strdup_printf(
+			"lz4 max chunk size too large (%u > %u)",
+			zstream->max_uncompressed_chunk_size,
+			ISTREAM_LZ4_CHUNK_SIZE));
+		zstream->istream.istream.stream_errno = EINVAL;
+		return -1;
+	}
+	i_stream_skip(zstream->istream.parent, sizeof(*hdr));
+	return 1;
+}
+
+static ssize_t i_stream_lz4_read(struct istream_private *stream)
+{
+	struct lz4_istream *zstream = (struct lz4_istream *)stream;
+	const unsigned char *data;
+	size_t size, max_size;
+	int ret;
+
+	if (!zstream->header_read) {
+		if ((ret = i_stream_lz4_read_header(zstream)) <= 0)
+			return ret;
+		zstream->header_read = TRUE;
+	}
+
+	if (zstream->chunk_left == 0) {
+		ret = i_stream_read_data(stream->parent, &data, &size,
+					 IOSTREAM_LZ4_CHUNK_PREFIX_LEN);
+		if (ret < 0) {
+			stream->istream.stream_errno =
+				stream->parent->stream_errno;
+			if (stream->istream.stream_errno != 0) {
+				stream->istream.eof = TRUE;
+				zstream->stream_size = stream->istream.v_offset +
+					stream->pos - stream->skip;
+			}
+			return ret;
+		}
+		if (ret == 0 && !stream->istream.eof)
+			return 0;
+		zstream->chunk_size = zstream->chunk_left =
+			((uint32_t)data[0] << 24) |
+			(data[1] << 16) | (data[2] << 8) | data[3];
+		if (zstream->chunk_size == 0 ||
+		    zstream->chunk_size > ISTREAM_LZ4_CHUNK_SIZE) {
+			lz4_read_error(zstream, t_strdup_printf(
+				"invalid lz4 chunk size: %u", zstream->chunk_size));
+			stream->istream.stream_errno = EINVAL;
+			return -1;
+		}
+		i_stream_skip(stream->parent, IOSTREAM_LZ4_CHUNK_PREFIX_LEN);
+		buffer_set_used_size(zstream->chunk_buf, 0);
+	}
+
+	/* read the whole compressed chunk into memory */
+	while (zstream->chunk_left > 0 &&
+	       (ret = i_stream_read_data(zstream->istream.parent,
+					 &data, &size, 0)) > 0) {
+		if (size > zstream->chunk_left)
+			size = zstream->chunk_left;
+		buffer_append(zstream->chunk_buf, data, size);
+		i_stream_skip(zstream->istream.parent, size);
+		zstream->chunk_left -= size;
+	}
+	if (zstream->chunk_left > 0) {
+		if (ret == -1 && zstream->istream.parent->stream_errno == 0) {
+			lz4_read_error(zstream, "truncated lz4 chunk");
+			stream->istream.stream_errno = EINVAL;
+			return -1;
+		}
+		zstream->istream.istream.stream_errno =
+			zstream->istream.parent->stream_errno;
+		return ret;
+	}
+	/* if we already have max_buffer_size amount of data, fail here */
+	i_stream_compress(stream);
+	if (stream->pos >= stream->max_buffer_size)
+		return -2;
+	/* allocate enough space for the old data and the new
+	   decompressed chunk. we don't know the original compressed size,
+	   so just allocate the max amount of memory. */
+	max_size = stream->pos + zstream->max_uncompressed_chunk_size;
+	if (stream->buffer_size < max_size) {
+		stream->w_buffer = i_realloc(stream->w_buffer,
+					     stream->buffer_size, max_size);
+		stream->buffer_size = max_size;
+		stream->buffer = stream->w_buffer;
+	}
+	ret = LZ4_decompress_safe(zstream->chunk_buf->data,
+				  (void *)(stream->w_buffer + stream->pos),
+				  zstream->chunk_buf->used,
+				  stream->buffer_size - stream->pos);
+	i_assert(ret <= (int)zstream->max_uncompressed_chunk_size);
+	if (ret < 0) {
+		lz4_read_error(zstream, "corrupted lz4 chunk");
+		stream->istream.stream_errno = EINVAL;
+		return -1;
+	}
+	i_assert(ret > 0);
+	stream->pos += ret;
+	i_assert(stream->pos <= stream->buffer_size);
+	return ret;
+}
+
+static void i_stream_lz4_reset(struct lz4_istream *zstream)
+{
+	struct istream_private *stream = &zstream->istream;
+
+	i_stream_seek(stream->parent, stream->parent_start_offset);
+	zstream->header_read = FALSE;
+	zstream->chunk_size = zstream->chunk_left = 0;
+
+	stream->parent_expected_offset = stream->parent_start_offset;
+	stream->skip = stream->pos = 0;
+	stream->istream.v_offset = 0;
+}
+
+static void
+i_stream_lz4_seek(struct istream_private *stream, uoff_t v_offset, bool mark)
+{
+	struct lz4_istream *zstream = (struct lz4_istream *) stream;
+	uoff_t start_offset = stream->istream.v_offset - stream->skip;
+
+	if (v_offset < start_offset) {
+		/* have to seek backwards */
+		i_stream_lz4_reset(zstream);
+		start_offset = 0;
+	}
+
+	if (v_offset <= start_offset + stream->pos) {
+		/* seeking backwards within what's already cached */
+		stream->skip = v_offset - start_offset;
+		stream->istream.v_offset = v_offset;
+		stream->pos = stream->skip;
+	} else {
+		/* read and cache forward */
+		do {
+			size_t avail = stream->pos - stream->skip;
+
+			if (stream->istream.v_offset + avail >= v_offset) {
+				i_stream_skip(&stream->istream,
+					      v_offset -
+					      stream->istream.v_offset);
+				break;
+			}
+
+			i_stream_skip(&stream->istream, avail);
+		} while (i_stream_read(&stream->istream) >= 0);
+
+		if (stream->istream.v_offset != v_offset) {
+			/* some failure, we've broken it */
+			if (stream->istream.stream_errno != 0) {
+				i_error("lz4_istream.seek(%s) failed: %s",
+					i_stream_get_name(&stream->istream),
+					strerror(stream->istream.stream_errno));
+				i_stream_close(&stream->istream);
+			} else {
+				/* unexpected EOF. allow it since we may just
+				   want to check if there's anything.. */
+				i_assert(stream->istream.eof);
+			}
+		}
+	}
+
+	if (mark)
+		zstream->marked = TRUE;
+}
+
+static int
+i_stream_lz4_stat(struct istream_private *stream, bool exact)
+{
+	struct lz4_istream *zstream = (struct lz4_istream *) stream;
+	const struct stat *st;
+	size_t size;
+
+	if (i_stream_stat(stream->parent, exact, &st) < 0)
+		return -1;
+	stream->statbuf = *st;
+
+	/* when exact=FALSE always return the parent stat's size, even if we
+	   know the exact value. this is necessary because otherwise e.g. mbox
+	   code can see two different values and think that a compressed mbox
+	   file keeps changing. */
+	if (!exact)
+		return 0;
+
+	if (zstream->stream_size == (uoff_t)-1) {
+		uoff_t old_offset = stream->istream.v_offset;
+
+		do {
+			size = i_stream_get_data_size(&stream->istream);
+			i_stream_skip(&stream->istream, size);
+		} while (i_stream_read(&stream->istream) > 0);
+
+		i_stream_seek(&stream->istream, old_offset);
+		if (zstream->stream_size == (uoff_t)-1)
+			return -1;
+	}
+	stream->statbuf.st_size = zstream->stream_size;
+	return 0;
+}
+
+static void i_stream_lz4_sync(struct istream_private *stream)
+{
+	struct lz4_istream *zstream = (struct lz4_istream *) stream;
+	const struct stat *st;
+
+	if (i_stream_stat(stream->parent, FALSE, &st) < 0) {
+		if (memcmp(&zstream->last_parent_statbuf,
+			   st, sizeof(*st)) == 0) {
+			/* a compressed file doesn't change unexpectedly,
+			   don't clear our caches unnecessarily */
+			return;
+		}
+		zstream->last_parent_statbuf = *st;
+	}
+	i_stream_lz4_reset(zstream);
+}
+
+struct istream *i_stream_create_lz4(struct istream *input, bool log_errors)
+{
+	struct lz4_istream *zstream;
+
+	zstream = i_new(struct lz4_istream, 1);
+	zstream->stream_size = (uoff_t)-1;
+	zstream->log_errors = log_errors;
+
+	zstream->istream.iostream.close = i_stream_lz4_close;
+	zstream->istream.max_buffer_size = input->real_stream->max_buffer_size;
+	zstream->istream.read = i_stream_lz4_read;
+	zstream->istream.seek = i_stream_lz4_seek;
+	zstream->istream.stat = i_stream_lz4_stat;
+	zstream->istream.sync = i_stream_lz4_sync;
+
+	zstream->istream.istream.readable_fd = FALSE;
+	zstream->istream.istream.blocking = input->blocking;
+	zstream->istream.istream.seekable = input->seekable;
+	zstream->chunk_buf = buffer_create_dynamic(default_pool, 1024);
+
+	return i_stream_create(&zstream->istream, input,
+			       i_stream_get_fd(input));
+}
+#endif
diff --git a/src/lib-compression/istream-zlib.h b/src/lib-compression/istream-zlib.h
index f81a0e1eb4..831d4717e1 100644
--- a/src/lib-compression/istream-zlib.h
+++ b/src/lib-compression/istream-zlib.h
@@ -5,5 +5,6 @@ struct istream *i_stream_create_gz(struct istream *input, bool log_errors);
 struct istream *i_stream_create_deflate(struct istream *input, bool log_errors);
 struct istream *i_stream_create_bz2(struct istream *input, bool log_errors);
 struct istream *i_stream_create_lzma(struct istream *input, bool log_errors);
+struct istream *i_stream_create_lz4(struct istream *input, bool log_errors);
 
 #endif
diff --git a/src/lib-compression/ostream-lz4.c b/src/lib-compression/ostream-lz4.c
new file mode 100644
index 0000000000..90aa2abc01
--- /dev/null
+++ b/src/lib-compression/ostream-lz4.c
@@ -0,0 +1,188 @@
+/* Copyright (c) 2013-2014 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+
+#ifdef HAVE_LZ4
+
+#include "ostream-private.h"
+#include "ostream-zlib.h"
+#include "iostream-lz4.h"
+#include <lz4.h>
+
+#define CHUNK_SIZE OSTREAM_LZ4_CHUNK_SIZE
+
+struct lz4_ostream {
+	struct ostream_private ostream;
+
+	unsigned char compressbuf[CHUNK_SIZE];
+	unsigned int compressbuf_offset;
+
+	/* chunk size, followed by compressed data */
+	unsigned char outbuf[IOSTREAM_LZ4_CHUNK_PREFIX_LEN + CHUNK_SIZE];
+	unsigned int outbuf_offset, outbuf_used;
+};
+
+static void o_stream_lz4_close(struct iostream_private *stream,
+			       bool close_parent)
+{
+	struct lz4_ostream *zstream = (struct lz4_ostream *)stream;
+
+	(void)o_stream_flush(&zstream->ostream.ostream);
+	if (close_parent)
+		o_stream_close(zstream->ostream.parent);
+}
+
+static int o_stream_lz4_send_outbuf(struct lz4_ostream *zstream)
+{
+	ssize_t ret;
+	size_t size;
+
+	if (zstream->outbuf_used == 0)
+		return 1;
+
+	size = zstream->outbuf_used - zstream->outbuf_offset;
+	i_assert(size > 0);
+	ret = o_stream_send(zstream->ostream.parent,
+			    zstream->outbuf + zstream->outbuf_offset, size);
+	if (ret < 0) {
+		o_stream_copy_error_from_parent(&zstream->ostream);
+		return -1;
+	}
+	if ((size_t)ret != size) {
+		zstream->outbuf_offset += ret;
+		return 0;
+	}
+	zstream->outbuf_offset = 0;
+	zstream->outbuf_used = 0;
+	return 1;
+}
+
+static int o_stream_lz4_compress(struct lz4_ostream *zstream)
+{
+	uint32_t chunk_size;
+	int ret;
+
+	if (zstream->compressbuf_offset == 0)
+		return 1;
+	if ((ret = o_stream_lz4_send_outbuf(zstream)) <= 0)
+		return ret;
+
+	i_assert(zstream->outbuf_offset == 0);
+	i_assert(zstream->outbuf_used == 0);
+
+	zstream->outbuf_used = IOSTREAM_LZ4_CHUNK_PREFIX_LEN +
+		LZ4_compress((void *)zstream->compressbuf,
+			     (void *)(zstream->outbuf + IOSTREAM_LZ4_CHUNK_PREFIX_LEN),
+			     zstream->compressbuf_offset);
+	i_assert(zstream->outbuf_used > IOSTREAM_LZ4_CHUNK_PREFIX_LEN);
+	chunk_size = zstream->outbuf_used - IOSTREAM_LZ4_CHUNK_PREFIX_LEN;
+	zstream->outbuf[0] = (chunk_size & 0xff000000) >> 24;
+	zstream->outbuf[1] = (chunk_size & 0x00ff0000) >> 16;
+	zstream->outbuf[2] = (chunk_size & 0x0000ff00) >> 8;
+	zstream->outbuf[3] = (chunk_size & 0x000000ff);
+	zstream->compressbuf_offset = 0;
+	return 1;
+}
+
+static ssize_t
+o_stream_lz4_send_chunk(struct lz4_ostream *zstream,
+			const void *data, size_t size)
+{
+	size_t max_size;
+	ssize_t added_bytes = 0;
+	int ret;
+
+	i_assert(zstream->outbuf_used == 0);
+
+	do {
+		max_size = I_MIN(size, sizeof(zstream->compressbuf) -
+				 zstream->compressbuf_offset);
+		memcpy(zstream->compressbuf + zstream->compressbuf_offset,
+		       data, max_size);
+		zstream->compressbuf_offset += max_size;
+
+		data = CONST_PTR_OFFSET(data, max_size);
+		size -= max_size;
+		added_bytes += max_size;
+
+		if (zstream->compressbuf_offset == sizeof(zstream->compressbuf)) {
+			ret = o_stream_lz4_compress(zstream);
+			if (ret <= 0)
+				return added_bytes != 0 ? added_bytes : ret;
+		}
+	} while (size > 0);
+
+	return added_bytes;
+}
+
+static int o_stream_lz4_flush(struct ostream_private *stream)
+{
+	struct lz4_ostream *zstream = (struct lz4_ostream *)stream;
+	int ret;
+
+	if (o_stream_lz4_compress(zstream) < 0)
+		return -1;
+	if (o_stream_lz4_send_outbuf(zstream) < 0)
+		return -1;
+
+	ret = o_stream_flush(stream->parent);
+	if (ret < 0)
+		o_stream_copy_error_from_parent(stream);
+	return ret;
+}
+
+static ssize_t
+o_stream_lz4_sendv(struct ostream_private *stream,
+		    const struct const_iovec *iov, unsigned int iov_count)
+{
+	struct lz4_ostream *zstream = (struct lz4_ostream *)stream;
+	ssize_t ret, bytes = 0;
+	unsigned int i;
+
+	if ((ret = o_stream_lz4_send_outbuf(zstream)) <= 0) {
+		/* error / we still couldn't flush existing data to
+		   parent stream. */
+		return ret;
+	}
+
+	for (i = 0; i < iov_count; i++) {
+		ret = o_stream_lz4_send_chunk(zstream, iov[i].iov_base,
+					      iov[i].iov_len);
+		if (ret < 0)
+			return -1;
+		bytes += ret;
+		if ((size_t)ret != iov[i].iov_len)
+			break;
+	}
+	stream->ostream.offset += bytes;
+	return bytes;
+}
+
+struct ostream *o_stream_create_lz4(struct ostream *output, int level)
+{
+	struct iostream_lz4_header *hdr;
+	struct lz4_ostream *zstream;
+
+	i_assert(level >= 1 && level <= 9);
+
+	zstream = i_new(struct lz4_ostream, 1);
+	zstream->ostream.sendv = o_stream_lz4_sendv;
+	zstream->ostream.flush = o_stream_lz4_flush;
+	zstream->ostream.iostream.close = o_stream_lz4_close;
+
+	i_assert(sizeof(zstream->outbuf) >= sizeof(*hdr));
+	hdr = (void *)zstream->outbuf;
+	memcpy(hdr->magic, IOSTREAM_LZ4_MAGIC, sizeof(hdr->magic));
+	hdr->max_uncompressed_chunk_size[0] =
+		(OSTREAM_LZ4_CHUNK_SIZE & 0xff000000) >> 24;
+	hdr->max_uncompressed_chunk_size[1] =
+		(OSTREAM_LZ4_CHUNK_SIZE & 0x00ff0000) >> 16;
+	hdr->max_uncompressed_chunk_size[2] =
+		(OSTREAM_LZ4_CHUNK_SIZE & 0x0000ff00) >> 8;
+	hdr->max_uncompressed_chunk_size[3] =
+		(OSTREAM_LZ4_CHUNK_SIZE & 0x000000ff);
+	zstream->outbuf_used = sizeof(*hdr);
+	return o_stream_create(&zstream->ostream, output,
+			       o_stream_get_fd(output));
+}
+#endif
diff --git a/src/lib-compression/ostream-zlib.h b/src/lib-compression/ostream-zlib.h
index 8be57fead1..57cddf9070 100644
--- a/src/lib-compression/ostream-zlib.h
+++ b/src/lib-compression/ostream-zlib.h
@@ -5,5 +5,6 @@ struct ostream *o_stream_create_gz(struct ostream *output, int level);
 struct ostream *o_stream_create_deflate(struct ostream *output, int level);
 struct ostream *o_stream_create_bz2(struct ostream *output, int level);
 struct ostream *o_stream_create_lzma(struct ostream *output, int level);
+struct ostream *o_stream_create_lz4(struct ostream *output, int level);
 
 #endif