[thirdparty/systemd.git] / src / journal / compress.c

/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/

/***
  This file is part of systemd.

  Copyright 2011 Lennart Poettering

  systemd is free software; you can redistribute it and/or modify it
  under the terms of the GNU Lesser General Public License as published by
  the Free Software Foundation; either version 2.1 of the License, or
  (at your option) any later version.

  systemd is distributed in the hope that it will be useful, but
  WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  Lesser General Public License for more details.

  You should have received a copy of the GNU Lesser General Public License
  along with systemd; If not, see <http://www.gnu.org/licenses/>.
***/

#include <assert.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>

#ifdef HAVE_XZ
#  include <lzma.h>
#endif

#ifdef HAVE_LZ4
#  include <lz4.h>
#endif

#include "compress.h"
#include "macro.h"
#include "util.h"
#include "sparse-endian.h"
#include "journal-def.h"

#define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t))

static const char* const object_compressed_table[_OBJECT_COMPRESSED_MAX] = {
        [OBJECT_COMPRESSED_XZ] = "XZ",
        [OBJECT_COMPRESSED_LZ4] = "LZ4",
};

DEFINE_STRING_TABLE_LOOKUP(object_compressed, int);

int compress_blob_xz(const void *src, uint64_t src_size, void *dst, size_t *dst_size) {
#ifdef HAVE_XZ
        static const lzma_options_lzma opt = {
                1u << 20u, NULL, 0, LZMA_LC_DEFAULT, LZMA_LP_DEFAULT,
                LZMA_PB_DEFAULT, LZMA_MODE_FAST, 128, LZMA_MF_HC3, 4};
        static const lzma_filter filters[2] = {
                {LZMA_FILTER_LZMA2, (lzma_options_lzma*) &opt},
                {LZMA_VLI_UNKNOWN, NULL}
        };
        lzma_ret ret;
        size_t out_pos = 0;

        assert(src);
        assert(src_size > 0);
        assert(dst);
        assert(dst_size);

        /* Returns < 0 if we couldn't compress the data or the
         * compressed result is longer than the original */

        if (src_size < 80)
                return -ENOBUFS;

        ret = lzma_stream_buffer_encode((lzma_filter*) filters, LZMA_CHECK_NONE, NULL,
                                        src, src_size, dst, &out_pos, src_size - 1);
        if (ret != LZMA_OK)
                return -ENOBUFS;

        *dst_size = out_pos;
        return 0;
#else
        return -EPROTONOSUPPORT;
#endif
}

int compress_blob_lz4(const void *src, uint64_t src_size, void *dst, size_t *dst_size) {
#ifdef HAVE_LZ4
        int r;

        assert(src);
        assert(src_size > 0);
        assert(dst);
        assert(dst_size);

        /* Returns < 0 if we couldn't compress the data or the
         * compressed result is longer than the original */

        if (src_size < 9)
                return -ENOBUFS;

        r = LZ4_compress_limitedOutput(src, dst + 8, src_size, src_size - 8 - 1);
        if (r <= 0)
                return -ENOBUFS;

        *(le64_t*) dst = htole64(src_size);
        *dst_size = r + 8;

        return 0;
#else
        return -EPROTONOSUPPORT;
#endif
}


int decompress_blob_xz(const void *src, uint64_t src_size,
                       void **dst, size_t *dst_alloc_size, size_t* dst_size, size_t dst_max) {

#ifdef HAVE_XZ
        _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
        lzma_ret ret;
        size_t space;

        assert(src);
        assert(src_size > 0);
        assert(dst);
        assert(dst_alloc_size);
        assert(dst_size);
        assert(*dst_alloc_size == 0 || *dst);

        ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
        if (ret != LZMA_OK)
                return -ENOMEM;

        space = MIN(src_size * 2, dst_max ?: (size_t) -1);
        if (!greedy_realloc(dst, dst_alloc_size, space, 1))
                return -ENOMEM;

        s.next_in = src;
        s.avail_in = src_size;

        s.next_out = *dst;
        s.avail_out = space;

        for (;;) {
                size_t used;

                ret = lzma_code(&s, LZMA_FINISH);

                if (ret == LZMA_STREAM_END)
                        break;
                else if (ret != LZMA_OK)
                        return -ENOMEM;

                if (dst_max > 0 && (space - s.avail_out) >= dst_max)
                        break;
                else if (dst_max > 0 && space == dst_max)
                        return -ENOBUFS;

                used = space - s.avail_out;
                space = MIN(2 * space, dst_max ?: (size_t) -1);
                if (!greedy_realloc(dst, dst_alloc_size, space, 1))
                        return -ENOMEM;

                s.avail_out = space - used;
                s.next_out = *dst + used;
        }

        *dst_size = space - s.avail_out;
        return 0;
#else
        return -EPROTONOSUPPORT;
#endif
}

int decompress_blob_lz4(const void *src, uint64_t src_size,
                        void **dst, size_t *dst_alloc_size, size_t* dst_size, size_t dst_max) {

#ifdef HAVE_LZ4
        char* out;
        int r, size; /* LZ4 uses int for size */

        assert(src);
        assert(src_size > 0);
        assert(dst);
        assert(dst_alloc_size);
        assert(dst_size);
        assert(*dst_alloc_size == 0 || *dst);

        if (src_size <= 8)
                return -EBADMSG;

        size = le64toh( *(le64_t*)src );
        if (size < 0 || (le64_t) size != *(le64_t*)src)
                return -EFBIG;
        if ((size_t) size > *dst_alloc_size) {
                out = realloc(*dst, size);
                if (!out)
                        return -ENOMEM;
                *dst = out;
                *dst_alloc_size = size;
        } else
                out = *dst;

        r = LZ4_decompress_safe(src + 8, out, src_size - 8, size);
        if (r < 0 || r != size)
                return -EBADMSG;

        *dst_size = size;
        return 0;
#else
        return -EPROTONOSUPPORT;
#endif
}

int decompress_blob(int compression,
                    const void *src, uint64_t src_size,
                    void **dst, size_t *dst_alloc_size, size_t* dst_size, size_t dst_max) {
        if (compression == OBJECT_COMPRESSED_XZ)
                return decompress_blob_xz(src, src_size,
                                          dst, dst_alloc_size, dst_size, dst_max);
        else if (compression == OBJECT_COMPRESSED_LZ4)
                return decompress_blob_lz4(src, src_size,
                                           dst, dst_alloc_size, dst_size, dst_max);
        else
                return -EBADMSG;
}


int decompress_startswith_xz(const void *src, uint64_t src_size,
                             void **buffer, size_t *buffer_size,
                             const void *prefix, size_t prefix_len,
                             uint8_t extra) {

#ifdef HAVE_XZ
        _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
        lzma_ret ret;

        /* Checks whether the decompressed blob starts with the
         * mentioned prefix. The byte extra needs to follow the
         * prefix */

        assert(src);
        assert(src_size > 0);
        assert(buffer);
        assert(buffer_size);
        assert(prefix);
        assert(*buffer_size == 0 || *buffer);

        ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
        if (ret != LZMA_OK)
                return -EBADMSG;

        if (!(greedy_realloc(buffer, buffer_size, ALIGN_8(prefix_len + 1), 1)))
                return -ENOMEM;

        s.next_in = src;
        s.avail_in = src_size;

        s.next_out = *buffer;
        s.avail_out = *buffer_size;

        for (;;) {
                ret = lzma_code(&s, LZMA_FINISH);

                if (ret != LZMA_STREAM_END && ret != LZMA_OK)
                        return -EBADMSG;

                if (*buffer_size - s.avail_out >= prefix_len + 1)
                        return memcmp(*buffer, prefix, prefix_len) == 0 &&
                                ((const uint8_t*) *buffer)[prefix_len] == extra;

                if (ret == LZMA_STREAM_END)
                        return 0;

                s.avail_out += *buffer_size;

                if (!(greedy_realloc(buffer, buffer_size, *buffer_size * 2, 1)))
                        return -ENOMEM;

                s.next_out = *buffer + *buffer_size - s.avail_out;
        }

#else
        return -EPROTONOSUPPORT;
#endif
}

int decompress_startswith_lz4(const void *src, uint64_t src_size,
                              void **buffer, size_t *buffer_size,
                              const void *prefix, size_t prefix_len,
                              uint8_t extra) {
#ifdef HAVE_LZ4
        /* Checks whether the decompressed blob starts with the
         * mentioned prefix. The byte extra needs to follow the
         * prefix */

        int r;

        assert(src);
        assert(src_size > 0);
        assert(buffer);
        assert(buffer_size);
        assert(prefix);
        assert(*buffer_size == 0 || *buffer);

        if (src_size <= 8)
                return -EBADMSG;

        if (!(greedy_realloc(buffer, buffer_size, ALIGN_8(prefix_len + 1), 1)))
                return -ENOMEM;

        r = LZ4_decompress_safe_partial(src + 8, *buffer, src_size - 8,
                                        prefix_len + 1, *buffer_size);

        if (r < 0)
                return -EBADMSG;
        if ((unsigned) r >= prefix_len + 1)
                return memcmp(*buffer, prefix, prefix_len) == 0 &&
                        ((const uint8_t*) *buffer)[prefix_len] == extra;
        else
                return 0;

#else
        return -EPROTONOSUPPORT;
#endif
}

int decompress_startswith(int compression,
                          const void *src, uint64_t src_size,
                          void **buffer, size_t *buffer_size,
                          const void *prefix, size_t prefix_len,
                          uint8_t extra) {
        if (compression == OBJECT_COMPRESSED_XZ)
                return decompress_startswith_xz(src, src_size,
                                                buffer, buffer_size,
                                                prefix, prefix_len,
                                                extra);
        else if (compression == OBJECT_COMPRESSED_LZ4)
                return decompress_startswith_lz4(src, src_size,
                                                 buffer, buffer_size,
                                                 prefix, prefix_len,
                                                 extra);
        else
                return -EBADMSG;
}

int compress_stream_xz(int fdf, int fdt, off_t max_bytes) {
#ifdef HAVE_XZ
        _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
        lzma_ret ret;

        uint8_t buf[BUFSIZ], out[BUFSIZ];
        lzma_action action = LZMA_RUN;

        assert(fdf >= 0);
        assert(fdt >= 0);

        ret = lzma_easy_encoder(&s, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64);
        if (ret != LZMA_OK) {
                log_error("Failed to initialize XZ encoder: code %d", ret);
                return -EINVAL;
        }

        for (;;) {
                if (s.avail_in == 0 && action == LZMA_RUN) {
                        size_t m = sizeof(buf);
                        ssize_t n;

                        if (max_bytes != -1 && m > (size_t) max_bytes)
                                m = max_bytes;

                        n = read(fdf, buf, m);
                        if (n < 0)
                                return -errno;
                        if (n == 0)
                                action = LZMA_FINISH;
                        else {
                                s.next_in = buf;
                                s.avail_in = n;

                                if (max_bytes != -1) {
                                        assert(max_bytes >= n);
                                        max_bytes -= n;
                                }
                        }
                }

                if (s.avail_out == 0) {
                        s.next_out = out;
                        s.avail_out = sizeof(out);
                }

                ret = lzma_code(&s, action);
                if (ret != LZMA_OK && ret != LZMA_STREAM_END) {
                        log_error("Compression failed: code %d", ret);
                        return -EBADMSG;
                }

                if (s.avail_out == 0 || ret == LZMA_STREAM_END) {
                        ssize_t n, k;

                        n = sizeof(out) - s.avail_out;

                        errno = 0;
                        k = loop_write(fdt, out, n, false);
                        if (k < 0)
                                return k;
                        if (k != n)
                                return errno ? -errno : -EIO;

                        if (ret == LZMA_STREAM_END) {
                                log_debug("XZ compression finished (%"PRIu64" -> %"PRIu64" bytes, %.1f%%)",
                                          s.total_in, s.total_out,
                                          (double) s.total_out / s.total_in * 100);

                                return 0;
                        }
                }
        }
#else
        return -EPROTONOSUPPORT;
#endif
}

#define LZ4_BUFSIZE (512*1024)

int compress_stream_lz4(int fdf, int fdt, off_t max_bytes) {

#ifdef HAVE_LZ4

        _cleanup_free_ char *buf1 = NULL, *buf2 = NULL, *out = NULL;
        char *buf;
        LZ4_stream_t lz4_data = {};
        le32_t header;
        size_t total_in = 0, total_out = sizeof(header);
        ssize_t n;

        assert(fdf >= 0);
        assert(fdt >= 0);

        buf1 = malloc(LZ4_BUFSIZE);
        buf2 = malloc(LZ4_BUFSIZE);
        out = malloc(LZ4_COMPRESSBOUND(LZ4_BUFSIZE));
        if (!buf1 || !buf2 || !out)
                return log_oom();

        buf = buf1;
        for (;;) {
                size_t m;
                int r;

                m = LZ4_BUFSIZE;
                if (max_bytes != -1 && m > (size_t) max_bytes - total_in)
                        m = max_bytes - total_in;

                n = read(fdf, buf, m);
                if (n < 0)
                        return -errno;
                if (n == 0)
                        break;

                total_in += n;

                r = LZ4_compress_limitedOutput_continue(&lz4_data, buf, out, n, n);
                if (r == 0) {
                        log_debug("Compressed size exceeds original, aborting compression.");
                        return -ENOBUFS;
                }

                header = htole32(r);
                errno = 0;

                n = write(fdt, &header, sizeof(header));
                if (n < 0)
                        return -errno;
                if (n != sizeof(header))
                        return errno ? -errno : -EIO;

                n = loop_write(fdt, out, r, false);
                if (n < 0)
                        return n;
                if (n != r)
                        return errno ? -errno : -EIO;

                total_out += sizeof(header) + r;

                buf = buf == buf1 ? buf2 : buf1;
        }

        header = htole32(0);
        n = write(fdt, &header, sizeof(header));
        if (n < 0)
                return -errno;
        if (n != sizeof(header))
                return errno ? -errno : -EIO;

        log_debug("LZ4 compression finished (%zu -> %zu bytes, %.1f%%)",
                  total_in, total_out,
                  (double) total_out / total_in * 100);

        return 0;
#else
        return -EPROTONOSUPPORT;
#endif
}

int decompress_stream_xz(int fdf, int fdt, off_t max_bytes) {

#ifdef HAVE_XZ
        _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
        lzma_ret ret;

        uint8_t buf[BUFSIZ], out[BUFSIZ];
        lzma_action action = LZMA_RUN;

        assert(fdf >= 0);
        assert(fdt >= 0);

        ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
        if (ret != LZMA_OK) {
                log_error("Failed to initialize XZ decoder: code %d", ret);
                return -ENOMEM;
        }

        for (;;) {
                if (s.avail_in == 0 && action == LZMA_RUN) {
                        ssize_t n;

                        n = read(fdf, buf, sizeof(buf));
                        if (n < 0)
                                return -errno;
                        if (n == 0)
                                action = LZMA_FINISH;
                        else {
                                s.next_in = buf;
                                s.avail_in = n;
                        }
                }

                if (s.avail_out == 0) {
                        s.next_out = out;
                        s.avail_out = sizeof(out);
                }

                ret = lzma_code(&s, action);
                if (ret != LZMA_OK && ret != LZMA_STREAM_END) {
                        log_error("Decompression failed: code %d", ret);
                        return -EBADMSG;
                }

                if (s.avail_out == 0 || ret == LZMA_STREAM_END) {
                        ssize_t n, k;

                        n = sizeof(out) - s.avail_out;

                        if (max_bytes != -1) {
                                if (max_bytes < n)
                                        return -EFBIG;

                                max_bytes -= n;
                        }

                        errno = 0;
                        k = loop_write(fdt, out, n, false);
                        if (k < 0)
                                return k;
                        if (k != n)
                                return errno ? -errno : -EIO;

                        if (ret == LZMA_STREAM_END) {
                                log_debug("XZ decompression finished (%"PRIu64" -> %"PRIu64" bytes, %.1f%%)",
                                          s.total_in, s.total_out,
                                          (double) s.total_out / s.total_in * 100);

                                return 0;
                        }
                }
        }
#else
        log_error("Cannot decompress file. Compiled without XZ support.");
        return -EPROTONOSUPPORT;
#endif
}

int decompress_stream_lz4(int fdf, int fdt, off_t max_bytes) {

#ifdef HAVE_LZ4
        _cleanup_free_ char *buf = NULL, *out = NULL;
        size_t buf_size = 0;
        LZ4_streamDecode_t lz4_data = {};
        le32_t header;
        size_t total_in = sizeof(header), total_out = 0;

        assert(fdf >= 0);
        assert(fdt >= 0);

        out = malloc(4*LZ4_BUFSIZE);
        if (!out)
                return log_oom();

        for (;;) {
                ssize_t n, m;
                int r;

                n = read(fdf, &header, sizeof(header));
                if (n < 0)
                        return -errno;
                if (n != sizeof(header))
                        return errno ? -errno : -EIO;

                m = le32toh(header);
                if (m == 0)
                        break;

                /* We refuse to use a bigger decompression buffer than
                 * the one used for compression by 4 times. This means
                 * that compression buffer size can be enlarged 4
                 * times. This can be changed, but old binaries might
                 * not accept buffers compressed by newer binaries then.
                 */
                if (m > LZ4_COMPRESSBOUND(LZ4_BUFSIZE * 4)) {
                        log_error("Compressed stream block too big: %zd bytes", m);
                        return -EBADMSG;
                }

                total_in += sizeof(header) + m;

                if (!GREEDY_REALLOC(buf, buf_size, m))
                        return log_oom();

                errno = 0;
                n = loop_read(fdf, buf, m, false);
                if (n < 0)
                        return n;
                if (n != m)
                        return errno ? -errno : -EIO;

                r = LZ4_decompress_safe_continue(&lz4_data, buf, out, m, 4*LZ4_BUFSIZE);
                if (r <= 0)
                        log_error("LZ4 decompression failed.");

                total_out += r;

                if (max_bytes != -1 && total_out > (size_t) max_bytes) {
                        log_debug("Decompressed stream longer than %zd bytes", max_bytes);
                        return -EFBIG;
                }

                errno = 0;
                n = loop_write(fdt, out, r, false);
                if (n < 0)
                        return n;
                if (n != r)
                        return errno ? -errno : -EIO;
        }

        log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)",
                  total_in, total_out,
                  (double) total_out / total_in * 100);

        return 0;
#else
        log_error("Cannot decompress file. Compiled without LZ4 support.");
        return -EPROTONOSUPPORT;
#endif
}

int decompress_stream(const char *filename, int fdf, int fdt, off_t max_bytes) {

        if (endswith(filename, ".lz4"))
                return decompress_stream_lz4(fdf, fdt, max_bytes);
        else if (endswith(filename, ".xz"))
                return decompress_stream_xz(fdf, fdt, max_bytes);
        else
                return -EPROTONOSUPPORT;
}
Commit	Line	Data
e4e61fdb LP	1	/-- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil --/
	2
	3	/***
	4	This file is part of systemd.
	5
	6	Copyright 2011 Lennart Poettering
	7
	8	systemd is free software; you can redistribute it and/or modify it
5430f7f2 LP	9	under the terms of the GNU Lesser General Public License as published by
5430f7f2 LP	10	the Free Software Foundation; either version 2.1 of the License, or
e4e61fdb LP	11	(at your option) any later version.
	12
	13	systemd is distributed in the hope that it will be useful, but
	14	WITHOUT ANY WARRANTY; without even the implied warranty of
	15	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
5430f7f2	16	Lesser General Public License for more details.
e4e61fdb	17
5430f7f2	18	You should have received a copy of the GNU Lesser General Public License
e4e61fdb LP	19	along with systemd; If not, see <http://www.gnu.org/licenses/>.
	20	***/
	21
	22	#include <assert.h>
	23	#include <stdlib.h>
	24	#include <string.h>
355b59e2	25	#include <unistd.h>
d89c8fdf ZJS	26
	27	#ifdef HAVE_XZ
	28	# include <lzma.h>
	29	#endif
	30
	31	#ifdef HAVE_LZ4
	32	# include <lz4.h>
	33	#endif
e4e61fdb LP	34
e4e61fdb LP	35	#include "compress.h"
355b59e2 ZJS	36	#include "macro.h"
355b59e2 ZJS	37	#include "util.h"
d89c8fdf ZJS	38	#include "sparse-endian.h"
	39	#include "journal-def.h"
	40
	41	#define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t))
	42
	43	static const char* const object_compressed_table[_OBJECT_COMPRESSED_MAX] = {
	44	[OBJECT_COMPRESSED_XZ] = "XZ",
	45	[OBJECT_COMPRESSED_LZ4] = "LZ4",
	46	};
e4e61fdb	47
d89c8fdf ZJS	48	DEFINE_STRING_TABLE_LOOKUP(object_compressed, int);
d89c8fdf ZJS	49
fa1c4b51	50	int compress_blob_xz(const void src, uint64_t src_size, void dst, size_t *dst_size) {
d89c8fdf	51	#ifdef HAVE_XZ
1930eed2 JS	52	static const lzma_options_lzma opt = {
	53	1u << 20u, NULL, 0, LZMA_LC_DEFAULT, LZMA_LP_DEFAULT,
	54	LZMA_PB_DEFAULT, LZMA_MODE_FAST, 128, LZMA_MF_HC3, 4};
	55	static const lzma_filter filters[2] = {
	56	{LZMA_FILTER_LZMA2, (lzma_options_lzma*) &opt},
	57	{LZMA_VLI_UNKNOWN, NULL}
	58	};
e4e61fdb	59	lzma_ret ret;
76cc0bf6	60	size_t out_pos = 0;
e4e61fdb LP	61
	62	assert(src);
	63	assert(src_size > 0);
	64	assert(dst);
	65	assert(dst_size);
	66
d89c8fdf	67	/* Returns < 0 if we couldn't compress the data or the
e4e61fdb LP	68	* compressed result is longer than the original */
e4e61fdb LP	69
1930eed2 JS	70	if (src_size < 80)
	71	return -ENOBUFS;
	72
	73	ret = lzma_stream_buffer_encode((lzma_filter*) filters, LZMA_CHECK_NONE, NULL,
	74	src, src_size, dst, &out_pos, src_size - 1);
e4e61fdb	75	if (ret != LZMA_OK)
d89c8fdf	76	return -ENOBUFS;
e4e61fdb	77
76cc0bf6	78	*dst_size = out_pos;
d89c8fdf ZJS	79	return 0;
	80	#else
	81	return -EPROTONOSUPPORT;
	82	#endif
e4e61fdb LP	83	}
e4e61fdb LP	84
fa1c4b51	85	int compress_blob_lz4(const void src, uint64_t src_size, void dst, size_t *dst_size) {
d89c8fdf ZJS	86	#ifdef HAVE_LZ4
	87	int r;
	88
	89	assert(src);
	90	assert(src_size > 0);
	91	assert(dst);
	92	assert(dst_size);
	93
	94	/* Returns < 0 if we couldn't compress the data or the
	95	* compressed result is longer than the original */
	96
	97	if (src_size < 9)
	98	return -ENOBUFS;
e4e61fdb	99
d89c8fdf ZJS	100	r = LZ4_compress_limitedOutput(src, dst + 8, src_size, src_size - 8 - 1);
	101	if (r <= 0)
	102	return -ENOBUFS;
	103
	104	(le64_t) dst = htole64(src_size);
	105	*dst_size = r + 8;
	106
	107	return 0;
	108	#else
	109	return -EPROTONOSUPPORT;
	110	#endif
	111	}
	112
	113
	114	int decompress_blob_xz(const void *src, uint64_t src_size,
fa1c4b51	115	void *dst, size_t dst_alloc_size, size_t* dst_size, size_t dst_max) {
d89c8fdf ZJS	116
d89c8fdf ZJS	117	#ifdef HAVE_XZ
5e592c66	118	_cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
e4e61fdb	119	lzma_ret ret;
fa1c4b51	120	size_t space;
e4e61fdb LP	121
	122	assert(src);
	123	assert(src_size > 0);
	124	assert(dst);
	125	assert(dst_alloc_size);
	126	assert(dst_size);
	127	assert(dst_alloc_size == 0 \|\| dst);
	128
	129	ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
	130	if (ret != LZMA_OK)
d89c8fdf	131	return -ENOMEM;
e4e61fdb	132
fa1c4b51	133	space = MIN(src_size * 2, dst_max ?: (size_t) -1);
5e592c66	134	if (!greedy_realloc(dst, dst_alloc_size, space, 1))
01c3322e	135	return -ENOMEM;
e4e61fdb LP	136
	137	s.next_in = src;
	138	s.avail_in = src_size;
	139
	140	s.next_out = *dst;
93b73b06	141	s.avail_out = space;
e4e61fdb LP	142
e4e61fdb LP	143	for (;;) {
fa1c4b51	144	size_t used;
e4e61fdb LP	145
	146	ret = lzma_code(&s, LZMA_FINISH);
	147
	148	if (ret == LZMA_STREAM_END)
	149	break;
d89c8fdf ZJS	150	else if (ret != LZMA_OK)
d89c8fdf ZJS	151	return -ENOMEM;
e4e61fdb	152
93b73b06 LP	153	if (dst_max > 0 && (space - s.avail_out) >= dst_max)
93b73b06 LP	154	break;
d89c8fdf ZJS	155	else if (dst_max > 0 && space == dst_max)
d89c8fdf ZJS	156	return -ENOBUFS;
93b73b06	157
5e592c66	158	used = space - s.avail_out;
fa1c4b51	159	space = MIN(2 * space, dst_max ?: (size_t) -1);
5e592c66	160	if (!greedy_realloc(dst, dst_alloc_size, space, 1))
01c3322e	161	return -ENOMEM;
e4e61fdb	162
5e592c66 ZJS	163	s.avail_out = space - used;
5e592c66 ZJS	164	s.next_out = *dst + used;
e4e61fdb LP	165	}
e4e61fdb LP	166
93b73b06	167	*dst_size = space - s.avail_out;
d89c8fdf ZJS	168	return 0;
	169	#else
	170	return -EPROTONOSUPPORT;
	171	#endif
e4e61fdb LP	172	}
e4e61fdb LP	173
d89c8fdf	174	int decompress_blob_lz4(const void *src, uint64_t src_size,
fa1c4b51	175	void *dst, size_t dst_alloc_size, size_t* dst_size, size_t dst_max) {
d89c8fdf ZJS	176
	177	#ifdef HAVE_LZ4
	178	char* out;
fa1c4b51	179	int r, size; /* LZ4 uses int for size */
e4e61fdb	180
d89c8fdf ZJS	181	assert(src);
	182	assert(src_size > 0);
	183	assert(dst);
	184	assert(dst_alloc_size);
	185	assert(dst_size);
	186	assert(dst_alloc_size == 0 \|\| dst);
	187
	188	if (src_size <= 8)
	189	return -EBADMSG;
	190
	191	size = le64toh( (le64_t)src );
fa1c4b51 ZJS	192	if (size < 0 \|\| (le64_t) size != (le64_t)src)
	193	return -EFBIG;
	194	if ((size_t) size > *dst_alloc_size) {
d89c8fdf ZJS	195	out = realloc(*dst, size);
	196	if (!out)
	197	return -ENOMEM;
	198	*dst = out;
	199	*dst_alloc_size = size;
	200	} else
	201	out = *dst;
	202
	203	r = LZ4_decompress_safe(src + 8, out, src_size - 8, size);
fa1c4b51	204	if (r < 0 \|\| r != size)
d89c8fdf ZJS	205	return -EBADMSG;
	206
	207	*dst_size = size;
	208	return 0;
	209	#else
	210	return -EPROTONOSUPPORT;
	211	#endif
	212	}
	213
	214	int decompress_blob(int compression,
	215	const void *src, uint64_t src_size,
fa1c4b51	216	void *dst, size_t dst_alloc_size, size_t* dst_size, size_t dst_max) {
d89c8fdf ZJS	217	if (compression == OBJECT_COMPRESSED_XZ)
	218	return decompress_blob_xz(src, src_size,
	219	dst, dst_alloc_size, dst_size, dst_max);
	220	else if (compression == OBJECT_COMPRESSED_LZ4)
	221	return decompress_blob_lz4(src, src_size,
	222	dst, dst_alloc_size, dst_size, dst_max);
	223	else
	224	return -EBADMSG;
	225	}
	226
	227
	228	int decompress_startswith_xz(const void *src, uint64_t src_size,
fa1c4b51 ZJS	229	void *buffer, size_t buffer_size,
fa1c4b51 ZJS	230	const void *prefix, size_t prefix_len,
d89c8fdf ZJS	231	uint8_t extra) {
	232
	233	#ifdef HAVE_XZ
5e592c66	234	_cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
e4e61fdb	235	lzma_ret ret;
e4e61fdb	236
d89c8fdf	237	/* Checks whether the decompressed blob starts with the
e4e61fdb LP	238	* mentioned prefix. The byte extra needs to follow the
	239	* prefix */
	240
	241	assert(src);
	242	assert(src_size > 0);
	243	assert(buffer);
	244	assert(buffer_size);
	245	assert(prefix);
	246	assert(buffer_size == 0 \|\| buffer);
	247
	248	ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
	249	if (ret != LZMA_OK)
d89c8fdf	250	return -EBADMSG;
e4e61fdb	251
d89c8fdf ZJS	252	if (!(greedy_realloc(buffer, buffer_size, ALIGN_8(prefix_len + 1), 1)))
d89c8fdf ZJS	253	return -ENOMEM;
e4e61fdb LP	254
	255	s.next_in = src;
	256	s.avail_in = src_size;
	257
	258	s.next_out = *buffer;
	259	s.avail_out = *buffer_size;
	260
	261	for (;;) {
e4e61fdb LP	262	ret = lzma_code(&s, LZMA_FINISH);
	263
	264	if (ret != LZMA_STREAM_END && ret != LZMA_OK)
d89c8fdf	265	return -EBADMSG;
e4e61fdb	266
5e592c66 ZJS	267	if (*buffer_size - s.avail_out >= prefix_len + 1)
	268	return memcmp(*buffer, prefix, prefix_len) == 0 &&
	269	((const uint8_t) buffer)[prefix_len] == extra;
e4e61fdb LP	270
e4e61fdb LP	271	if (ret == LZMA_STREAM_END)
d89c8fdf	272	return 0;
e4e61fdb	273
e4e61fdb LP	274	s.avail_out += *buffer_size;
e4e61fdb LP	275
5e592c66	276	if (!(greedy_realloc(buffer, buffer_size, buffer_size 2, 1)))
d89c8fdf	277	return -ENOMEM;
e4e61fdb	278
5e592c66 ZJS	279	s.next_out = buffer + buffer_size - s.avail_out;
5e592c66 ZJS	280	}
d89c8fdf ZJS	281
	282	#else
	283	return -EPROTONOSUPPORT;
	284	#endif
	285	}
	286
	287	int decompress_startswith_lz4(const void *src, uint64_t src_size,
fa1c4b51 ZJS	288	void *buffer, size_t buffer_size,
fa1c4b51 ZJS	289	const void *prefix, size_t prefix_len,
d89c8fdf ZJS	290	uint8_t extra) {
	291	#ifdef HAVE_LZ4
	292	/* Checks whether the decompressed blob starts with the
	293	* mentioned prefix. The byte extra needs to follow the
	294	* prefix */
	295
	296	int r;
	297
	298	assert(src);
	299	assert(src_size > 0);
	300	assert(buffer);
	301	assert(buffer_size);
	302	assert(prefix);
	303	assert(buffer_size == 0 \|\| buffer);
	304
	305	if (src_size <= 8)
	306	return -EBADMSG;
	307
	308	if (!(greedy_realloc(buffer, buffer_size, ALIGN_8(prefix_len + 1), 1)))
	309	return -ENOMEM;
	310
	311	r = LZ4_decompress_safe_partial(src + 8, *buffer, src_size - 8,
	312	prefix_len + 1, *buffer_size);
	313
	314	if (r < 0)
	315	return -EBADMSG;
	316	if ((unsigned) r >= prefix_len + 1)
	317	return memcmp(*buffer, prefix, prefix_len) == 0 &&
	318	((const uint8_t) buffer)[prefix_len] == extra;
	319	else
	320	return 0;
	321
	322	#else
	323	return -EPROTONOSUPPORT;
	324	#endif
e4e61fdb	325	}
355b59e2	326
d89c8fdf ZJS	327	int decompress_startswith(int compression,
d89c8fdf ZJS	328	const void *src, uint64_t src_size,
fa1c4b51 ZJS	329	void *buffer, size_t buffer_size,
fa1c4b51 ZJS	330	const void *prefix, size_t prefix_len,
d89c8fdf ZJS	331	uint8_t extra) {
	332	if (compression == OBJECT_COMPRESSED_XZ)
	333	return decompress_startswith_xz(src, src_size,
	334	buffer, buffer_size,
	335	prefix, prefix_len,
	336	extra);
	337	else if (compression == OBJECT_COMPRESSED_LZ4)
	338	return decompress_startswith_lz4(src, src_size,
	339	buffer, buffer_size,
	340	prefix, prefix_len,
	341	extra);
	342	else
	343	return -EBADMSG;
	344	}
	345
	346	int compress_stream_xz(int fdf, int fdt, off_t max_bytes) {
3b1a55e1	347	#ifdef HAVE_XZ
355b59e2 ZJS	348	_cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
	349	lzma_ret ret;
	350
	351	uint8_t buf[BUFSIZ], out[BUFSIZ];
	352	lzma_action action = LZMA_RUN;
	353
	354	assert(fdf >= 0);
	355	assert(fdt >= 0);
	356
d89c8fdf	357	ret = lzma_easy_encoder(&s, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64);
355b59e2 ZJS	358	if (ret != LZMA_OK) {
	359	log_error("Failed to initialize XZ encoder: code %d", ret);
	360	return -EINVAL;
	361	}
	362
	363	for (;;) {
	364	if (s.avail_in == 0 && action == LZMA_RUN) {
	365	size_t m = sizeof(buf);
	366	ssize_t n;
	367
	368	if (max_bytes != -1 && m > (size_t) max_bytes)
	369	m = max_bytes;
	370
	371	n = read(fdf, buf, m);
	372	if (n < 0)
	373	return -errno;
	374	if (n == 0)
	375	action = LZMA_FINISH;
	376	else {
	377	s.next_in = buf;
	378	s.avail_in = n;
	379
	380	if (max_bytes != -1) {
	381	assert(max_bytes >= n);
	382	max_bytes -= n;
	383	}
	384	}
	385	}
	386
	387	if (s.avail_out == 0) {
	388	s.next_out = out;
	389	s.avail_out = sizeof(out);
	390	}
	391
	392	ret = lzma_code(&s, action);
	393	if (ret != LZMA_OK && ret != LZMA_STREAM_END) {
	394	log_error("Compression failed: code %d", ret);
	395	return -EBADMSG;
	396	}
	397
	398	if (s.avail_out == 0 \|\| ret == LZMA_STREAM_END) {
	399	ssize_t n, k;
	400
	401	n = sizeof(out) - s.avail_out;
	402
	403	errno = 0;
	404	k = loop_write(fdt, out, n, false);
	405	if (k < 0)
	406	return k;
	407	if (k != n)
	408	return errno ? -errno : -EIO;
	409
	410	if (ret == LZMA_STREAM_END) {
fa1c4b51	411	log_debug("XZ compression finished (%"PRIu64" -> %"PRIu64" bytes, %.1f%%)",
355b59e2 ZJS	412	s.total_in, s.total_out,
	413	(double) s.total_out / s.total_in * 100);
	414
	415	return 0;
	416	}
	417	}
	418	}
3b1a55e1 ZJS	419	#else
	420	return -EPROTONOSUPPORT;
	421	#endif
355b59e2 ZJS	422	}
355b59e2 ZJS	423
d89c8fdf ZJS	424	#define LZ4_BUFSIZE (512*1024)
	425
	426	int compress_stream_lz4(int fdf, int fdt, off_t max_bytes) {
	427
	428	#ifdef HAVE_LZ4
	429
	430	_cleanup_free_ char buf1 = NULL, buf2 = NULL, *out = NULL;
	431	char *buf;
	432	LZ4_stream_t lz4_data = {};
	433	le32_t header;
	434	size_t total_in = 0, total_out = sizeof(header);
	435	ssize_t n;
	436
	437	assert(fdf >= 0);
	438	assert(fdt >= 0);
	439
	440	buf1 = malloc(LZ4_BUFSIZE);
	441	buf2 = malloc(LZ4_BUFSIZE);
	442	out = malloc(LZ4_COMPRESSBOUND(LZ4_BUFSIZE));
	443	if (!buf1 \|\| !buf2 \|\| !out)
	444	return log_oom();
	445
	446	buf = buf1;
	447	for (;;) {
	448	size_t m;
	449	int r;
	450
	451	m = LZ4_BUFSIZE;
	452	if (max_bytes != -1 && m > (size_t) max_bytes - total_in)
	453	m = max_bytes - total_in;
	454
	455	n = read(fdf, buf, m);
	456	if (n < 0)
	457	return -errno;
	458	if (n == 0)
	459	break;
	460
	461	total_in += n;
	462
	463	r = LZ4_compress_limitedOutput_continue(&lz4_data, buf, out, n, n);
	464	if (r == 0) {
	465	log_debug("Compressed size exceeds original, aborting compression.");
	466	return -ENOBUFS;
	467	}
	468
	469	header = htole32(r);
	470	errno = 0;
	471
	472	n = write(fdt, &header, sizeof(header));
	473	if (n < 0)
	474	return -errno;
	475	if (n != sizeof(header))
	476	return errno ? -errno : -EIO;
	477
	478	n = loop_write(fdt, out, r, false);
	479	if (n < 0)
	480	return n;
	481	if (n != r)
	482	return errno ? -errno : -EIO;
	483
	484	total_out += sizeof(header) + r;
	485
	486	buf = buf == buf1 ? buf2 : buf1;
	487	}
488
489	header = htole32(0);
490	n = write(fdt, &header, sizeof(header));
491	if (n < 0)
492	return -errno;
493	if (n != sizeof(header))
494	return errno ? -errno : -EIO;
495
496	log_debug("LZ4 compression finished (%zu -> %zu bytes, %.1f%%)",
497	total_in, total_out,
498	(double) total_out / total_in * 100);
499
500	return 0;
501	#else
502	return -EPROTONOSUPPORT;
503	#endif
504	}
505
506	int decompress_stream_xz(int fdf, int fdt, off_t max_bytes) {
507
508	#ifdef HAVE_XZ
355b59e2 ZJS	509	_cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
	510	lzma_ret ret;
	511
	512	uint8_t buf[BUFSIZ], out[BUFSIZ];
	513	lzma_action action = LZMA_RUN;
	514
	515	assert(fdf >= 0);
	516	assert(fdt >= 0);
	517
	518	ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
	519	if (ret != LZMA_OK) {
	520	log_error("Failed to initialize XZ decoder: code %d", ret);
d89c8fdf	521	return -ENOMEM;
355b59e2 ZJS	522	}
	523
	524	for (;;) {
	525	if (s.avail_in == 0 && action == LZMA_RUN) {
	526	ssize_t n;
	527
	528	n = read(fdf, buf, sizeof(buf));
	529	if (n < 0)
	530	return -errno;
	531	if (n == 0)
	532	action = LZMA_FINISH;
	533	else {
	534	s.next_in = buf;
	535	s.avail_in = n;
	536	}
	537	}
	538
	539	if (s.avail_out == 0) {
	540	s.next_out = out;
	541	s.avail_out = sizeof(out);
	542	}
	543
	544	ret = lzma_code(&s, action);
	545	if (ret != LZMA_OK && ret != LZMA_STREAM_END) {
	546	log_error("Decompression failed: code %d", ret);
	547	return -EBADMSG;
	548	}
	549
	550	if (s.avail_out == 0 \|\| ret == LZMA_STREAM_END) {
	551	ssize_t n, k;
	552
	553	n = sizeof(out) - s.avail_out;
	554
	555	if (max_bytes != -1) {
	556	if (max_bytes < n)
d89c8fdf	557	return -EFBIG;
355b59e2 ZJS	558
	559	max_bytes -= n;
	560	}
	561
	562	errno = 0;
	563	k = loop_write(fdt, out, n, false);
	564	if (k < 0)
	565	return k;
	566	if (k != n)
	567	return errno ? -errno : -EIO;
	568
	569	if (ret == LZMA_STREAM_END) {
fa1c4b51	570	log_debug("XZ decompression finished (%"PRIu64" -> %"PRIu64" bytes, %.1f%%)",
355b59e2 ZJS	571	s.total_in, s.total_out,
	572	(double) s.total_out / s.total_in * 100);
	573
	574	return 0;
	575	}
	576	}
	577	}
d89c8fdf ZJS	578	#else
	579	log_error("Cannot decompress file. Compiled without XZ support.");
	580	return -EPROTONOSUPPORT;
	581	#endif
	582	}
	583
	584	int decompress_stream_lz4(int fdf, int fdt, off_t max_bytes) {
	585
	586	#ifdef HAVE_LZ4
	587	_cleanup_free_ char buf = NULL, out = NULL;
	588	size_t buf_size = 0;
	589	LZ4_streamDecode_t lz4_data = {};
	590	le32_t header;
	591	size_t total_in = sizeof(header), total_out = 0;
	592
	593	assert(fdf >= 0);
	594	assert(fdt >= 0);
	595
	596	out = malloc(4*LZ4_BUFSIZE);
	597	if (!out)
	598	return log_oom();
	599
	600	for (;;) {
	601	ssize_t n, m;
	602	int r;
	603
	604	n = read(fdf, &header, sizeof(header));
	605	if (n < 0)
	606	return -errno;
	607	if (n != sizeof(header))
	608	return errno ? -errno : -EIO;
	609
	610	m = le32toh(header);
	611	if (m == 0)
	612	break;
	613
	614	/* We refuse to use a bigger decompression buffer than
	615	* the one used for compression by 4 times. This means
	616	* that compression buffer size can be enlarged 4
	617	* times. This can be changed, but old binaries might
	618	* not accept buffers compressed by newer binaries then.
	619	*/
	620	if (m > LZ4_COMPRESSBOUND(LZ4_BUFSIZE * 4)) {
	621	log_error("Compressed stream block too big: %zd bytes", m);
	622	return -EBADMSG;
	623	}
	624
	625	total_in += sizeof(header) + m;
	626
	627	if (!GREEDY_REALLOC(buf, buf_size, m))
	628	return log_oom();
	629
	630	errno = 0;
	631	n = loop_read(fdf, buf, m, false);
	632	if (n < 0)
	633	return n;
	634	if (n != m)
	635	return errno ? -errno : -EIO;
	636
	637	r = LZ4_decompress_safe_continue(&lz4_data, buf, out, m, 4*LZ4_BUFSIZE);
	638	if (r <= 0)
	639	log_error("LZ4 decompression failed.");
	640
	641	total_out += r;
642
643	if (max_bytes != -1 && total_out > (size_t) max_bytes) {
644	log_debug("Decompressed stream longer than %zd bytes", max_bytes);
645	return -EFBIG;
646	}
647
648	errno = 0;
649	n = loop_write(fdt, out, r, false);
650	if (n < 0)
651	return n;
652	if (n != r)
653	return errno ? -errno : -EIO;
654	}
655
656	log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)",
657	total_in, total_out,
658	(double) total_out / total_in * 100);
659
660	return 0;
661	#else
662	log_error("Cannot decompress file. Compiled without LZ4 support.");
663	return -EPROTONOSUPPORT;
664	#endif
665	}
666
667	int decompress_stream(const char *filename, int fdf, int fdt, off_t max_bytes) {
668
669	if (endswith(filename, ".lz4"))
670	return decompress_stream_lz4(fdf, fdt, max_bytes);
671	else if (endswith(filename, ".xz"))
672	return decompress_stream_xz(fdf, fdt, max_bytes);
673	else
674	return -EPROTONOSUPPORT;
355b59e2	675	}