[thirdparty/systemd.git] / src / journal / compress.c

/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/

/***
  This file is part of systemd.

  Copyright 2011 Lennart Poettering

  systemd is free software; you can redistribute it and/or modify it
  under the terms of the GNU Lesser General Public License as published by
  the Free Software Foundation; either version 2.1 of the License, or
  (at your option) any later version.

  systemd is distributed in the hope that it will be useful, but
  WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  Lesser General Public License for more details.

  You should have received a copy of the GNU Lesser General Public License
  along with systemd; If not, see <http://www.gnu.org/licenses/>.
***/

#include <assert.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>

#ifdef HAVE_XZ
#  include <lzma.h>
#endif

#ifdef HAVE_LZ4
#  include <lz4.h>
#endif

#include "compress.h"
#include "macro.h"
#include "util.h"
#include "sparse-endian.h"
#include "journal-def.h"

#define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t))

static const char* const object_compressed_table[_OBJECT_COMPRESSED_MAX] = {
        [OBJECT_COMPRESSED_XZ] = "XZ",
        [OBJECT_COMPRESSED_LZ4] = "LZ4",
};

DEFINE_STRING_TABLE_LOOKUP(object_compressed, int);

int compress_blob_xz(const void *src, uint64_t src_size, void *dst, uint64_t *dst_size) {
#ifdef HAVE_XZ
        lzma_ret ret;
        size_t out_pos = 0;

        assert(src);
        assert(src_size > 0);
        assert(dst);
        assert(dst_size);

        /* Returns < 0 if we couldn't compress the data or the
         * compressed result is longer than the original */

        ret = lzma_easy_buffer_encode(LZMA_PRESET_DEFAULT, LZMA_CHECK_NONE, NULL,
                                      src, src_size, dst, &out_pos, src_size - 1);
        if (ret != LZMA_OK)
                return -ENOBUFS;

        *dst_size = out_pos;
        return 0;
#else
        return -EPROTONOSUPPORT;
#endif
}

int compress_blob_lz4(const void *src, uint64_t src_size, void *dst, uint64_t *dst_size) {
#ifdef HAVE_LZ4
        int r;

        assert(src);
        assert(src_size > 0);
        assert(dst);
        assert(dst_size);

        /* Returns < 0 if we couldn't compress the data or the
         * compressed result is longer than the original */

        if (src_size < 9)
                return -ENOBUFS;

        r = LZ4_compress_limitedOutput(src, dst + 8, src_size, src_size - 8 - 1);
        if (r <= 0)
                return -ENOBUFS;

        *(le64_t*) dst = htole64(src_size);
        *dst_size = r + 8;

        return 0;
#else
        return -EPROTONOSUPPORT;
#endif
}


int decompress_blob_xz(const void *src, uint64_t src_size,
                       void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max) {

#ifdef HAVE_XZ
        _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
        lzma_ret ret;
        uint64_t space;

        assert(src);
        assert(src_size > 0);
        assert(dst);
        assert(dst_alloc_size);
        assert(dst_size);
        assert(*dst_alloc_size == 0 || *dst);

        ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
        if (ret != LZMA_OK)
                return -ENOMEM;

        space = MIN(src_size * 2, dst_max ?: (uint64_t) -1);
        if (!greedy_realloc(dst, dst_alloc_size, space, 1))
                return false;

        s.next_in = src;
        s.avail_in = src_size;

        s.next_out = *dst;
        s.avail_out = space;

        for (;;) {
                uint64_t used;

                ret = lzma_code(&s, LZMA_FINISH);

                if (ret == LZMA_STREAM_END)
                        break;
                else if (ret != LZMA_OK)
                        return -ENOMEM;

                if (dst_max > 0 && (space - s.avail_out) >= dst_max)
                        break;
                else if (dst_max > 0 && space == dst_max)
                        return -ENOBUFS;

                used = space - s.avail_out;
                space = MIN(2 * space, dst_max ?: (uint64_t) -1);
                if (!greedy_realloc(dst, dst_alloc_size, space, 1))
                        return false;

                s.avail_out = space - used;
                s.next_out = *dst + used;
        }

        *dst_size = space - s.avail_out;
        return 0;
#else
        return -EPROTONOSUPPORT;
#endif
}

int decompress_blob_lz4(const void *src, uint64_t src_size,
                        void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max) {

#ifdef HAVE_LZ4
        char* out;
        uint64_t size;
        int r;

        assert(src);
        assert(src_size > 0);
        assert(dst);
        assert(dst_alloc_size);
        assert(dst_size);
        assert(*dst_alloc_size == 0 || *dst);

        if (src_size <= 8)
                return -EBADMSG;

        size = le64toh( *(le64_t*)src );
        if (size > *dst_alloc_size) {
                out = realloc(*dst, size);
                if (!out)
                        return -ENOMEM;
                *dst = out;
                *dst_alloc_size = size;
        } else
                out = *dst;

        r = LZ4_decompress_safe(src + 8, out, src_size - 8, size);
        if (r < 0 || (uint64_t) r != size)
                return -EBADMSG;

        *dst_size = size;
        return 0;
#else
        return -EPROTONOSUPPORT;
#endif
}

int decompress_blob(int compression,
                    const void *src, uint64_t src_size,
                    void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max) {
        if (compression == OBJECT_COMPRESSED_XZ)
                return decompress_blob_xz(src, src_size,
                                          dst, dst_alloc_size, dst_size, dst_max);
        else if (compression == OBJECT_COMPRESSED_LZ4)
                return decompress_blob_lz4(src, src_size,
                                           dst, dst_alloc_size, dst_size, dst_max);
        else
                return -EBADMSG;
}


int decompress_startswith_xz(const void *src, uint64_t src_size,
                             void **buffer, uint64_t *buffer_size,
                             const void *prefix, uint64_t prefix_len,
                             uint8_t extra) {

#ifdef HAVE_XZ
        _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
        lzma_ret ret;

        /* Checks whether the decompressed blob starts with the
         * mentioned prefix. The byte extra needs to follow the
         * prefix */

        assert(src);
        assert(src_size > 0);
        assert(buffer);
        assert(buffer_size);
        assert(prefix);
        assert(*buffer_size == 0 || *buffer);

        ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
        if (ret != LZMA_OK)
                return -EBADMSG;

        if (!(greedy_realloc(buffer, buffer_size, ALIGN_8(prefix_len + 1), 1)))
                return -ENOMEM;

        s.next_in = src;
        s.avail_in = src_size;

        s.next_out = *buffer;
        s.avail_out = *buffer_size;

        for (;;) {
                ret = lzma_code(&s, LZMA_FINISH);

                if (ret != LZMA_STREAM_END && ret != LZMA_OK)
                        return -EBADMSG;

                if (*buffer_size - s.avail_out >= prefix_len + 1)
                        return memcmp(*buffer, prefix, prefix_len) == 0 &&
                                ((const uint8_t*) *buffer)[prefix_len] == extra;

                if (ret == LZMA_STREAM_END)
                        return 0;

                s.avail_out += *buffer_size;

                if (!(greedy_realloc(buffer, buffer_size, *buffer_size * 2, 1)))
                        return -ENOMEM;

                s.next_out = *buffer + *buffer_size - s.avail_out;
        }

#else
        return -EPROTONOSUPPORT;
#endif
}

int decompress_startswith_lz4(const void *src, uint64_t src_size,
                              void **buffer, uint64_t *buffer_size,
                              const void *prefix, uint64_t prefix_len,
                              uint8_t extra) {
#ifdef HAVE_LZ4
        /* Checks whether the decompressed blob starts with the
         * mentioned prefix. The byte extra needs to follow the
         * prefix */

        int r;

        assert(src);
        assert(src_size > 0);
        assert(buffer);
        assert(buffer_size);
        assert(prefix);
        assert(*buffer_size == 0 || *buffer);

        if (src_size <= 8)
                return -EBADMSG;

        if (!(greedy_realloc(buffer, buffer_size, ALIGN_8(prefix_len + 1), 1)))
                return -ENOMEM;

        r = LZ4_decompress_safe_partial(src + 8, *buffer, src_size - 8,
                                        prefix_len + 1, *buffer_size);

        if (r < 0)
                return -EBADMSG;
        if ((unsigned) r >= prefix_len + 1)
                return memcmp(*buffer, prefix, prefix_len) == 0 &&
                        ((const uint8_t*) *buffer)[prefix_len] == extra;
        else
                return 0;

#else
        return -EPROTONOSUPPORT;
#endif
}

int decompress_startswith(int compression,
                          const void *src, uint64_t src_size,
                          void **buffer, uint64_t *buffer_size,
                          const void *prefix, uint64_t prefix_len,
                          uint8_t extra) {
        if (compression == OBJECT_COMPRESSED_XZ)
                return decompress_startswith_xz(src, src_size,
                                                buffer, buffer_size,
                                                prefix, prefix_len,
                                                extra);
        else if (compression == OBJECT_COMPRESSED_LZ4)
                return decompress_startswith_lz4(src, src_size,
                                                 buffer, buffer_size,
                                                 prefix, prefix_len,
                                                 extra);
        else
                return -EBADMSG;
}

int compress_stream_xz(int fdf, int fdt, off_t max_bytes) {
        _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
        lzma_ret ret;

        uint8_t buf[BUFSIZ], out[BUFSIZ];
        lzma_action action = LZMA_RUN;

        assert(fdf >= 0);
        assert(fdt >= 0);

        ret = lzma_easy_encoder(&s, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64);
        if (ret != LZMA_OK) {
                log_error("Failed to initialize XZ encoder: code %d", ret);
                return -EINVAL;
        }

        for (;;) {
                if (s.avail_in == 0 && action == LZMA_RUN) {
                        size_t m = sizeof(buf);
                        ssize_t n;

                        if (max_bytes != -1 && m > (size_t) max_bytes)
                                m = max_bytes;

                        n = read(fdf, buf, m);
                        if (n < 0)
                                return -errno;
                        if (n == 0)
                                action = LZMA_FINISH;
                        else {
                                s.next_in = buf;
                                s.avail_in = n;

                                if (max_bytes != -1) {
                                        assert(max_bytes >= n);
                                        max_bytes -= n;
                                }
                        }
                }

                if (s.avail_out == 0) {
                        s.next_out = out;
                        s.avail_out = sizeof(out);
                }

                ret = lzma_code(&s, action);
                if (ret != LZMA_OK && ret != LZMA_STREAM_END) {
                        log_error("Compression failed: code %d", ret);
                        return -EBADMSG;
                }

                if (s.avail_out == 0 || ret == LZMA_STREAM_END) {
                        ssize_t n, k;

                        n = sizeof(out) - s.avail_out;

                        errno = 0;
                        k = loop_write(fdt, out, n, false);
                        if (k < 0)
                                return k;
                        if (k != n)
                                return errno ? -errno : -EIO;

                        if (ret == LZMA_STREAM_END) {
                                log_debug("XZ compression finished (%zu -> %zu bytes, %.1f%%)",
                                          s.total_in, s.total_out,
                                          (double) s.total_out / s.total_in * 100);

                                return 0;
                        }
                }
        }
}

#define LZ4_BUFSIZE (512*1024)

int compress_stream_lz4(int fdf, int fdt, off_t max_bytes) {

#ifdef HAVE_LZ4

        _cleanup_free_ char *buf1 = NULL, *buf2 = NULL, *out = NULL;
        char *buf;
        LZ4_stream_t lz4_data = {};
        le32_t header;
        size_t total_in = 0, total_out = sizeof(header);
        ssize_t n;

        assert(fdf >= 0);
        assert(fdt >= 0);

        buf1 = malloc(LZ4_BUFSIZE);
        buf2 = malloc(LZ4_BUFSIZE);
        out = malloc(LZ4_COMPRESSBOUND(LZ4_BUFSIZE));
        if (!buf1 || !buf2 || !out)
                return log_oom();

        buf = buf1;
        for (;;) {
                size_t m;
                int r;

                m = LZ4_BUFSIZE;
                if (max_bytes != -1 && m > (size_t) max_bytes - total_in)
                        m = max_bytes - total_in;

                n = read(fdf, buf, m);
                if (n < 0)
                        return -errno;
                if (n == 0)
                        break;

                total_in += n;

                r = LZ4_compress_limitedOutput_continue(&lz4_data, buf, out, n, n);
                if (r == 0) {
                        log_debug("Compressed size exceeds original, aborting compression.");
                        return -ENOBUFS;
                }

                header = htole32(r);
                errno = 0;

                n = write(fdt, &header, sizeof(header));
                if (n < 0)
                        return -errno;
                if (n != sizeof(header))
                        return errno ? -errno : -EIO;

                n = loop_write(fdt, out, r, false);
                if (n < 0)
                        return n;
                if (n != r)
                        return errno ? -errno : -EIO;

                total_out += sizeof(header) + r;

                buf = buf == buf1 ? buf2 : buf1;
        }

        header = htole32(0);
        n = write(fdt, &header, sizeof(header));
        if (n < 0)
                return -errno;
        if (n != sizeof(header))
                return errno ? -errno : -EIO;

        log_debug("LZ4 compression finished (%zu -> %zu bytes, %.1f%%)",
                  total_in, total_out,
                  (double) total_out / total_in * 100);

        return 0;
#else
        return -EPROTONOSUPPORT;
#endif
}

int decompress_stream_xz(int fdf, int fdt, off_t max_bytes) {

#ifdef HAVE_XZ
        _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
        lzma_ret ret;

        uint8_t buf[BUFSIZ], out[BUFSIZ];
        lzma_action action = LZMA_RUN;

        assert(fdf >= 0);
        assert(fdt >= 0);

        ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
        if (ret != LZMA_OK) {
                log_error("Failed to initialize XZ decoder: code %d", ret);
                return -ENOMEM;
        }

        for (;;) {
                if (s.avail_in == 0 && action == LZMA_RUN) {
                        ssize_t n;

                        n = read(fdf, buf, sizeof(buf));
                        if (n < 0)
                                return -errno;
                        if (n == 0)
                                action = LZMA_FINISH;
                        else {
                                s.next_in = buf;
                                s.avail_in = n;
                        }
                }

                if (s.avail_out == 0) {
                        s.next_out = out;
                        s.avail_out = sizeof(out);
                }

                ret = lzma_code(&s, action);
                if (ret != LZMA_OK && ret != LZMA_STREAM_END) {
                        log_error("Decompression failed: code %d", ret);
                        return -EBADMSG;
                }

                if (s.avail_out == 0 || ret == LZMA_STREAM_END) {
                        ssize_t n, k;

                        n = sizeof(out) - s.avail_out;

                        if (max_bytes != -1) {
                                if (max_bytes < n)
                                        return -EFBIG;

                                max_bytes -= n;
                        }

                        errno = 0;
                        k = loop_write(fdt, out, n, false);
                        if (k < 0)
                                return k;
                        if (k != n)
                                return errno ? -errno : -EIO;

                        if (ret == LZMA_STREAM_END) {
                                log_debug("XZ decompression finished (%zu -> %zu bytes, %.1f%%)",
                                          s.total_in, s.total_out,
                                          (double) s.total_out / s.total_in * 100);

                                return 0;
                        }
                }
        }
#else
        log_error("Cannot decompress file. Compiled without XZ support.");
        return -EPROTONOSUPPORT;
#endif
}

int decompress_stream_lz4(int fdf, int fdt, off_t max_bytes) {

#ifdef HAVE_LZ4
        _cleanup_free_ char *buf = NULL, *out = NULL;
        size_t buf_size = 0;
        LZ4_streamDecode_t lz4_data = {};
        le32_t header;
        size_t total_in = sizeof(header), total_out = 0;

        assert(fdf >= 0);
        assert(fdt >= 0);

        out = malloc(4*LZ4_BUFSIZE);
        if (!out)
                return log_oom();

        for (;;) {
                ssize_t n, m;
                int r;

                n = read(fdf, &header, sizeof(header));
                if (n < 0)
                        return -errno;
                if (n != sizeof(header))
                        return errno ? -errno : -EIO;

                m = le32toh(header);
                if (m == 0)
                        break;

                /* We refuse to use a bigger decompression buffer than
                 * the one used for compression by 4 times. This means
                 * that compression buffer size can be enlarged 4
                 * times. This can be changed, but old binaries might
                 * not accept buffers compressed by newer binaries then.
                 */
                if (m > LZ4_COMPRESSBOUND(LZ4_BUFSIZE * 4)) {
                        log_error("Compressed stream block too big: %zd bytes", m);
                        return -EBADMSG;
                }

                total_in += sizeof(header) + m;

                if (!GREEDY_REALLOC(buf, buf_size, m))
                        return log_oom();

                errno = 0;
                n = loop_read(fdf, buf, m, false);
                if (n < 0)
                        return n;
                if (n != m)
                        return errno ? -errno : -EIO;

                r = LZ4_decompress_safe_continue(&lz4_data, buf, out, m, 4*LZ4_BUFSIZE);
                if (r <= 0)
                        log_error("LZ4 decompression failed.");

                total_out += r;

                if (max_bytes != -1 && total_out > (size_t) max_bytes) {
                        log_debug("Decompressed stream longer than %zd bytes", max_bytes);
                        return -EFBIG;
                }

                errno = 0;
                n = loop_write(fdt, out, r, false);
                if (n < 0)
                        return n;
                if (n != r)
                        return errno ? -errno : -EIO;
        }

        log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)",
                  total_in, total_out,
                  (double) total_out / total_in * 100);

        return 0;
#else
        log_error("Cannot decompress file. Compiled without LZ4 support.");
        return -EPROTONOSUPPORT;
#endif
}

int decompress_stream(const char *filename, int fdf, int fdt, off_t max_bytes) {

        if (endswith(filename, ".lz4"))
                return decompress_stream_lz4(fdf, fdt, max_bytes);
        else if (endswith(filename, ".xz"))
                return decompress_stream_xz(fdf, fdt, max_bytes);
        else
                return -EPROTONOSUPPORT;
}
Commit	Line	Data
e4e61fdb LP	1	/-- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil --/
	2
	3	/***
	4	This file is part of systemd.
	5
	6	Copyright 2011 Lennart Poettering
	7
	8	systemd is free software; you can redistribute it and/or modify it
5430f7f2 LP	9	under the terms of the GNU Lesser General Public License as published by
5430f7f2 LP	10	the Free Software Foundation; either version 2.1 of the License, or
e4e61fdb LP	11	(at your option) any later version.
	12
	13	systemd is distributed in the hope that it will be useful, but
	14	WITHOUT ANY WARRANTY; without even the implied warranty of
	15	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
5430f7f2	16	Lesser General Public License for more details.
e4e61fdb	17
5430f7f2	18	You should have received a copy of the GNU Lesser General Public License
e4e61fdb LP	19	along with systemd; If not, see <http://www.gnu.org/licenses/>.
	20	***/
	21
	22	#include <assert.h>
	23	#include <stdlib.h>
	24	#include <string.h>
355b59e2	25	#include <unistd.h>
d89c8fdf ZJS	26
	27	#ifdef HAVE_XZ
	28	# include <lzma.h>
	29	#endif
	30
	31	#ifdef HAVE_LZ4
	32	# include <lz4.h>
	33	#endif
e4e61fdb LP	34
e4e61fdb LP	35	#include "compress.h"
355b59e2 ZJS	36	#include "macro.h"
355b59e2 ZJS	37	#include "util.h"
d89c8fdf ZJS	38	#include "sparse-endian.h"
	39	#include "journal-def.h"
	40
	41	#define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t))
	42
	43	static const char* const object_compressed_table[_OBJECT_COMPRESSED_MAX] = {
	44	[OBJECT_COMPRESSED_XZ] = "XZ",
	45	[OBJECT_COMPRESSED_LZ4] = "LZ4",
	46	};
e4e61fdb	47
d89c8fdf ZJS	48	DEFINE_STRING_TABLE_LOOKUP(object_compressed, int);
	49
	50	int compress_blob_xz(const void src, uint64_t src_size, void dst, uint64_t *dst_size) {
	51	#ifdef HAVE_XZ
e4e61fdb	52	lzma_ret ret;
76cc0bf6	53	size_t out_pos = 0;
e4e61fdb LP	54
	55	assert(src);
	56	assert(src_size > 0);
	57	assert(dst);
	58	assert(dst_size);
	59
d89c8fdf	60	/* Returns < 0 if we couldn't compress the data or the
e4e61fdb LP	61	* compressed result is longer than the original */
e4e61fdb LP	62
76cc0bf6	63	ret = lzma_easy_buffer_encode(LZMA_PRESET_DEFAULT, LZMA_CHECK_NONE, NULL,
d89c8fdf	64	src, src_size, dst, &out_pos, src_size - 1);
e4e61fdb	65	if (ret != LZMA_OK)
d89c8fdf	66	return -ENOBUFS;
e4e61fdb	67
76cc0bf6	68	*dst_size = out_pos;
d89c8fdf ZJS	69	return 0;
	70	#else
	71	return -EPROTONOSUPPORT;
	72	#endif
e4e61fdb LP	73	}
e4e61fdb LP	74
d89c8fdf ZJS	75	int compress_blob_lz4(const void src, uint64_t src_size, void dst, uint64_t *dst_size) {
	76	#ifdef HAVE_LZ4
	77	int r;
	78
	79	assert(src);
	80	assert(src_size > 0);
	81	assert(dst);
	82	assert(dst_size);
	83
	84	/* Returns < 0 if we couldn't compress the data or the
	85	* compressed result is longer than the original */
	86
	87	if (src_size < 9)
	88	return -ENOBUFS;
e4e61fdb	89
d89c8fdf ZJS	90	r = LZ4_compress_limitedOutput(src, dst + 8, src_size, src_size - 8 - 1);
	91	if (r <= 0)
	92	return -ENOBUFS;
	93
	94	(le64_t) dst = htole64(src_size);
	95	*dst_size = r + 8;
	96
	97	return 0;
	98	#else
	99	return -EPROTONOSUPPORT;
	100	#endif
	101	}
	102
	103
	104	int decompress_blob_xz(const void *src, uint64_t src_size,
	105	void *dst, uint64_t dst_alloc_size, uint64_t* dst_size, uint64_t dst_max) {
	106
	107	#ifdef HAVE_XZ
5e592c66	108	_cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
e4e61fdb	109	lzma_ret ret;
93b73b06	110	uint64_t space;
e4e61fdb LP	111
	112	assert(src);
	113	assert(src_size > 0);
	114	assert(dst);
	115	assert(dst_alloc_size);
	116	assert(dst_size);
	117	assert(dst_alloc_size == 0 \|\| dst);
	118
	119	ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
	120	if (ret != LZMA_OK)
d89c8fdf	121	return -ENOMEM;
e4e61fdb	122
5e592c66 ZJS	123	space = MIN(src_size * 2, dst_max ?: (uint64_t) -1);
	124	if (!greedy_realloc(dst, dst_alloc_size, space, 1))
	125	return false;
e4e61fdb LP	126
	127	s.next_in = src;
	128	s.avail_in = src_size;
	129
	130	s.next_out = *dst;
93b73b06	131	s.avail_out = space;
e4e61fdb LP	132
e4e61fdb LP	133	for (;;) {
5e592c66	134	uint64_t used;
e4e61fdb LP	135
	136	ret = lzma_code(&s, LZMA_FINISH);
	137
	138	if (ret == LZMA_STREAM_END)
	139	break;
d89c8fdf ZJS	140	else if (ret != LZMA_OK)
d89c8fdf ZJS	141	return -ENOMEM;
e4e61fdb	142
93b73b06 LP	143	if (dst_max > 0 && (space - s.avail_out) >= dst_max)
93b73b06 LP	144	break;
d89c8fdf ZJS	145	else if (dst_max > 0 && space == dst_max)
d89c8fdf ZJS	146	return -ENOBUFS;
93b73b06	147
5e592c66 ZJS	148	used = space - s.avail_out;
	149	space = MIN(2 * space, dst_max ?: (uint64_t) -1);
	150	if (!greedy_realloc(dst, dst_alloc_size, space, 1))
	151	return false;
e4e61fdb	152
5e592c66 ZJS	153	s.avail_out = space - used;
5e592c66 ZJS	154	s.next_out = *dst + used;
e4e61fdb LP	155	}
e4e61fdb LP	156
93b73b06	157	*dst_size = space - s.avail_out;
d89c8fdf ZJS	158	return 0;
	159	#else
	160	return -EPROTONOSUPPORT;
	161	#endif
e4e61fdb LP	162	}
e4e61fdb LP	163
d89c8fdf ZJS	164	int decompress_blob_lz4(const void *src, uint64_t src_size,
	165	void *dst, uint64_t dst_alloc_size, uint64_t* dst_size, uint64_t dst_max) {
	166
	167	#ifdef HAVE_LZ4
	168	char* out;
	169	uint64_t size;
	170	int r;
e4e61fdb	171
d89c8fdf ZJS	172	assert(src);
	173	assert(src_size > 0);
	174	assert(dst);
	175	assert(dst_alloc_size);
	176	assert(dst_size);
	177	assert(dst_alloc_size == 0 \|\| dst);
	178
	179	if (src_size <= 8)
	180	return -EBADMSG;
	181
	182	size = le64toh( (le64_t)src );
	183	if (size > *dst_alloc_size) {
	184	out = realloc(*dst, size);
	185	if (!out)
	186	return -ENOMEM;
	187	*dst = out;
	188	*dst_alloc_size = size;
	189	} else
	190	out = *dst;
	191
	192	r = LZ4_decompress_safe(src + 8, out, src_size - 8, size);
	193	if (r < 0 \|\| (uint64_t) r != size)
	194	return -EBADMSG;
	195
	196	*dst_size = size;
	197	return 0;
	198	#else
	199	return -EPROTONOSUPPORT;
	200	#endif
	201	}
	202
	203	int decompress_blob(int compression,
	204	const void *src, uint64_t src_size,
	205	void *dst, uint64_t dst_alloc_size, uint64_t* dst_size, uint64_t dst_max) {
	206	if (compression == OBJECT_COMPRESSED_XZ)
	207	return decompress_blob_xz(src, src_size,
	208	dst, dst_alloc_size, dst_size, dst_max);
	209	else if (compression == OBJECT_COMPRESSED_LZ4)
	210	return decompress_blob_lz4(src, src_size,
	211	dst, dst_alloc_size, dst_size, dst_max);
	212	else
	213	return -EBADMSG;
	214	}
	215
	216
	217	int decompress_startswith_xz(const void *src, uint64_t src_size,
	218	void *buffer, uint64_t buffer_size,
	219	const void *prefix, uint64_t prefix_len,
	220	uint8_t extra) {
	221
	222	#ifdef HAVE_XZ
5e592c66	223	_cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
e4e61fdb	224	lzma_ret ret;
e4e61fdb	225
d89c8fdf	226	/* Checks whether the decompressed blob starts with the
e4e61fdb LP	227	* mentioned prefix. The byte extra needs to follow the
	228	* prefix */
	229
	230	assert(src);
	231	assert(src_size > 0);
	232	assert(buffer);
	233	assert(buffer_size);
	234	assert(prefix);
	235	assert(buffer_size == 0 \|\| buffer);
	236
	237	ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
	238	if (ret != LZMA_OK)
d89c8fdf	239	return -EBADMSG;
e4e61fdb	240
d89c8fdf ZJS	241	if (!(greedy_realloc(buffer, buffer_size, ALIGN_8(prefix_len + 1), 1)))
d89c8fdf ZJS	242	return -ENOMEM;
e4e61fdb LP	243
	244	s.next_in = src;
	245	s.avail_in = src_size;
	246
	247	s.next_out = *buffer;
	248	s.avail_out = *buffer_size;
	249
	250	for (;;) {
e4e61fdb LP	251	ret = lzma_code(&s, LZMA_FINISH);
	252
	253	if (ret != LZMA_STREAM_END && ret != LZMA_OK)
d89c8fdf	254	return -EBADMSG;
e4e61fdb	255
5e592c66 ZJS	256	if (*buffer_size - s.avail_out >= prefix_len + 1)
	257	return memcmp(*buffer, prefix, prefix_len) == 0 &&
	258	((const uint8_t) buffer)[prefix_len] == extra;
e4e61fdb LP	259
e4e61fdb LP	260	if (ret == LZMA_STREAM_END)
d89c8fdf	261	return 0;
e4e61fdb	262
e4e61fdb LP	263	s.avail_out += *buffer_size;
e4e61fdb LP	264
5e592c66	265	if (!(greedy_realloc(buffer, buffer_size, buffer_size 2, 1)))
d89c8fdf	266	return -ENOMEM;
e4e61fdb	267
5e592c66 ZJS	268	s.next_out = buffer + buffer_size - s.avail_out;
5e592c66 ZJS	269	}
d89c8fdf ZJS	270
	271	#else
	272	return -EPROTONOSUPPORT;
	273	#endif
	274	}
	275
	276	int decompress_startswith_lz4(const void *src, uint64_t src_size,
	277	void *buffer, uint64_t buffer_size,
	278	const void *prefix, uint64_t prefix_len,
	279	uint8_t extra) {
	280	#ifdef HAVE_LZ4
	281	/* Checks whether the decompressed blob starts with the
	282	* mentioned prefix. The byte extra needs to follow the
	283	* prefix */
	284
	285	int r;
	286
	287	assert(src);
	288	assert(src_size > 0);
	289	assert(buffer);
	290	assert(buffer_size);
	291	assert(prefix);
	292	assert(buffer_size == 0 \|\| buffer);
	293
	294	if (src_size <= 8)
	295	return -EBADMSG;
	296
	297	if (!(greedy_realloc(buffer, buffer_size, ALIGN_8(prefix_len + 1), 1)))
	298	return -ENOMEM;
	299
	300	r = LZ4_decompress_safe_partial(src + 8, *buffer, src_size - 8,
	301	prefix_len + 1, *buffer_size);
	302
	303	if (r < 0)
	304	return -EBADMSG;
	305	if ((unsigned) r >= prefix_len + 1)
	306	return memcmp(*buffer, prefix, prefix_len) == 0 &&
	307	((const uint8_t) buffer)[prefix_len] == extra;
	308	else
	309	return 0;
	310
	311	#else
	312	return -EPROTONOSUPPORT;
	313	#endif
e4e61fdb	314	}
355b59e2	315
d89c8fdf ZJS	316	int decompress_startswith(int compression,
	317	const void *src, uint64_t src_size,
	318	void *buffer, uint64_t buffer_size,
	319	const void *prefix, uint64_t prefix_len,
	320	uint8_t extra) {
	321	if (compression == OBJECT_COMPRESSED_XZ)
	322	return decompress_startswith_xz(src, src_size,
	323	buffer, buffer_size,
	324	prefix, prefix_len,
	325	extra);
	326	else if (compression == OBJECT_COMPRESSED_LZ4)
	327	return decompress_startswith_lz4(src, src_size,
	328	buffer, buffer_size,
	329	prefix, prefix_len,
	330	extra);
	331	else
	332	return -EBADMSG;
	333	}
	334
	335	int compress_stream_xz(int fdf, int fdt, off_t max_bytes) {
355b59e2 ZJS	336	_cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
	337	lzma_ret ret;
	338
	339	uint8_t buf[BUFSIZ], out[BUFSIZ];
	340	lzma_action action = LZMA_RUN;
	341
	342	assert(fdf >= 0);
	343	assert(fdt >= 0);
	344
d89c8fdf	345	ret = lzma_easy_encoder(&s, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64);
355b59e2 ZJS	346	if (ret != LZMA_OK) {
	347	log_error("Failed to initialize XZ encoder: code %d", ret);
	348	return -EINVAL;
	349	}
	350
	351	for (;;) {
	352	if (s.avail_in == 0 && action == LZMA_RUN) {
	353	size_t m = sizeof(buf);
	354	ssize_t n;
	355
	356	if (max_bytes != -1 && m > (size_t) max_bytes)
	357	m = max_bytes;
	358
	359	n = read(fdf, buf, m);
	360	if (n < 0)
	361	return -errno;
	362	if (n == 0)
	363	action = LZMA_FINISH;
	364	else {
	365	s.next_in = buf;
	366	s.avail_in = n;
	367
	368	if (max_bytes != -1) {
	369	assert(max_bytes >= n);
	370	max_bytes -= n;
	371	}
	372	}
	373	}
	374
	375	if (s.avail_out == 0) {
	376	s.next_out = out;
	377	s.avail_out = sizeof(out);
	378	}
	379
	380	ret = lzma_code(&s, action);
	381	if (ret != LZMA_OK && ret != LZMA_STREAM_END) {
	382	log_error("Compression failed: code %d", ret);
	383	return -EBADMSG;
	384	}
	385
	386	if (s.avail_out == 0 \|\| ret == LZMA_STREAM_END) {
	387	ssize_t n, k;
	388
	389	n = sizeof(out) - s.avail_out;
	390
	391	errno = 0;
	392	k = loop_write(fdt, out, n, false);
	393	if (k < 0)
	394	return k;
	395	if (k != n)
	396	return errno ? -errno : -EIO;
	397
	398	if (ret == LZMA_STREAM_END) {
d89c8fdf	399	log_debug("XZ compression finished (%zu -> %zu bytes, %.1f%%)",
355b59e2 ZJS	400	s.total_in, s.total_out,
	401	(double) s.total_out / s.total_in * 100);
	402
	403	return 0;
	404	}
	405	}
	406	}
	407	}
	408
d89c8fdf ZJS	409	#define LZ4_BUFSIZE (512*1024)
	410
	411	int compress_stream_lz4(int fdf, int fdt, off_t max_bytes) {
	412
	413	#ifdef HAVE_LZ4
	414
	415	_cleanup_free_ char buf1 = NULL, buf2 = NULL, *out = NULL;
	416	char *buf;
	417	LZ4_stream_t lz4_data = {};
	418	le32_t header;
	419	size_t total_in = 0, total_out = sizeof(header);
	420	ssize_t n;
	421
	422	assert(fdf >= 0);
	423	assert(fdt >= 0);
	424
	425	buf1 = malloc(LZ4_BUFSIZE);
	426	buf2 = malloc(LZ4_BUFSIZE);
	427	out = malloc(LZ4_COMPRESSBOUND(LZ4_BUFSIZE));
	428	if (!buf1 \|\| !buf2 \|\| !out)
	429	return log_oom();
	430
	431	buf = buf1;
	432	for (;;) {
	433	size_t m;
	434	int r;
	435
	436	m = LZ4_BUFSIZE;
	437	if (max_bytes != -1 && m > (size_t) max_bytes - total_in)
	438	m = max_bytes - total_in;
	439
	440	n = read(fdf, buf, m);
	441	if (n < 0)
	442	return -errno;
	443	if (n == 0)
	444	break;
	445
	446	total_in += n;
	447
	448	r = LZ4_compress_limitedOutput_continue(&lz4_data, buf, out, n, n);
	449	if (r == 0) {
	450	log_debug("Compressed size exceeds original, aborting compression.");
	451	return -ENOBUFS;
	452	}
	453
	454	header = htole32(r);
	455	errno = 0;
	456
	457	n = write(fdt, &header, sizeof(header));
	458	if (n < 0)
	459	return -errno;
	460	if (n != sizeof(header))
	461	return errno ? -errno : -EIO;
	462
	463	n = loop_write(fdt, out, r, false);
	464	if (n < 0)
	465	return n;
	466	if (n != r)
	467	return errno ? -errno : -EIO;
	468
	469	total_out += sizeof(header) + r;
	470
	471	buf = buf == buf1 ? buf2 : buf1;
	472	}
473
474	header = htole32(0);
475	n = write(fdt, &header, sizeof(header));
476	if (n < 0)
477	return -errno;
478	if (n != sizeof(header))
479	return errno ? -errno : -EIO;
480
481	log_debug("LZ4 compression finished (%zu -> %zu bytes, %.1f%%)",
482	total_in, total_out,
483	(double) total_out / total_in * 100);
484
485	return 0;
486	#else
487	return -EPROTONOSUPPORT;
488	#endif
489	}
490
491	int decompress_stream_xz(int fdf, int fdt, off_t max_bytes) {
492
493	#ifdef HAVE_XZ
355b59e2 ZJS	494	_cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
	495	lzma_ret ret;
	496
	497	uint8_t buf[BUFSIZ], out[BUFSIZ];
	498	lzma_action action = LZMA_RUN;
	499
	500	assert(fdf >= 0);
	501	assert(fdt >= 0);
	502
	503	ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
	504	if (ret != LZMA_OK) {
	505	log_error("Failed to initialize XZ decoder: code %d", ret);
d89c8fdf	506	return -ENOMEM;
355b59e2 ZJS	507	}
	508
	509	for (;;) {
	510	if (s.avail_in == 0 && action == LZMA_RUN) {
	511	ssize_t n;
	512
	513	n = read(fdf, buf, sizeof(buf));
	514	if (n < 0)
	515	return -errno;
	516	if (n == 0)
	517	action = LZMA_FINISH;
	518	else {
	519	s.next_in = buf;
	520	s.avail_in = n;
	521	}
	522	}
	523
	524	if (s.avail_out == 0) {
	525	s.next_out = out;
	526	s.avail_out = sizeof(out);
	527	}
	528
	529	ret = lzma_code(&s, action);
	530	if (ret != LZMA_OK && ret != LZMA_STREAM_END) {
	531	log_error("Decompression failed: code %d", ret);
	532	return -EBADMSG;
	533	}
	534
	535	if (s.avail_out == 0 \|\| ret == LZMA_STREAM_END) {
	536	ssize_t n, k;
	537
	538	n = sizeof(out) - s.avail_out;
	539
	540	if (max_bytes != -1) {
	541	if (max_bytes < n)
d89c8fdf	542	return -EFBIG;
355b59e2 ZJS	543
	544	max_bytes -= n;
	545	}
	546
	547	errno = 0;
	548	k = loop_write(fdt, out, n, false);
	549	if (k < 0)
	550	return k;
	551	if (k != n)
	552	return errno ? -errno : -EIO;
	553
	554	if (ret == LZMA_STREAM_END) {
d89c8fdf	555	log_debug("XZ decompression finished (%zu -> %zu bytes, %.1f%%)",
355b59e2 ZJS	556	s.total_in, s.total_out,
	557	(double) s.total_out / s.total_in * 100);
	558
	559	return 0;
	560	}
	561	}
	562	}
d89c8fdf ZJS	563	#else
	564	log_error("Cannot decompress file. Compiled without XZ support.");
	565	return -EPROTONOSUPPORT;
	566	#endif
	567	}
	568
	569	int decompress_stream_lz4(int fdf, int fdt, off_t max_bytes) {
	570
	571	#ifdef HAVE_LZ4
	572	_cleanup_free_ char buf = NULL, out = NULL;
	573	size_t buf_size = 0;
	574	LZ4_streamDecode_t lz4_data = {};
	575	le32_t header;
	576	size_t total_in = sizeof(header), total_out = 0;
	577
	578	assert(fdf >= 0);
	579	assert(fdt >= 0);
	580
	581	out = malloc(4*LZ4_BUFSIZE);
	582	if (!out)
	583	return log_oom();
	584
	585	for (;;) {
	586	ssize_t n, m;
	587	int r;
	588
	589	n = read(fdf, &header, sizeof(header));
	590	if (n < 0)
	591	return -errno;
	592	if (n != sizeof(header))
	593	return errno ? -errno : -EIO;
	594
	595	m = le32toh(header);
	596	if (m == 0)
	597	break;
	598
	599	/* We refuse to use a bigger decompression buffer than
	600	* the one used for compression by 4 times. This means
	601	* that compression buffer size can be enlarged 4
	602	* times. This can be changed, but old binaries might
	603	* not accept buffers compressed by newer binaries then.
	604	*/
	605	if (m > LZ4_COMPRESSBOUND(LZ4_BUFSIZE * 4)) {
	606	log_error("Compressed stream block too big: %zd bytes", m);
	607	return -EBADMSG;
	608	}
	609
	610	total_in += sizeof(header) + m;
	611
	612	if (!GREEDY_REALLOC(buf, buf_size, m))
	613	return log_oom();
	614
	615	errno = 0;
	616	n = loop_read(fdf, buf, m, false);
	617	if (n < 0)
	618	return n;
	619	if (n != m)
	620	return errno ? -errno : -EIO;
	621
	622	r = LZ4_decompress_safe_continue(&lz4_data, buf, out, m, 4*LZ4_BUFSIZE);
	623	if (r <= 0)
	624	log_error("LZ4 decompression failed.");
	625
	626	total_out += r;
627
628	if (max_bytes != -1 && total_out > (size_t) max_bytes) {
629	log_debug("Decompressed stream longer than %zd bytes", max_bytes);
630	return -EFBIG;
631	}
632
633	errno = 0;
634	n = loop_write(fdt, out, r, false);
635	if (n < 0)
636	return n;
637	if (n != r)
638	return errno ? -errno : -EIO;
639	}
640
641	log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)",
642	total_in, total_out,
643	(double) total_out / total_in * 100);
644
645	return 0;
646	#else
647	log_error("Cannot decompress file. Compiled without LZ4 support.");
648	return -EPROTONOSUPPORT;
649	#endif
650	}
651
652	int decompress_stream(const char *filename, int fdf, int fdt, off_t max_bytes) {
653
654	if (endswith(filename, ".lz4"))
655	return decompress_stream_lz4(fdf, fdt, max_bytes);
656	else if (endswith(filename, ".xz"))
657	return decompress_stream_xz(fdf, fdt, max_bytes);
658	else
659	return -EPROTONOSUPPORT;
355b59e2	660	}