libz = dependency('zlib',
required : get_option('zlib'))
conf.set10('HAVE_ZLIB', libz.found())
+libz_cflags = libz.partial_dependency(includes: true, compile_args: true)
feature = get_option('bzip2')
libbzip2 = dependency('bzip2',
libbzip2 = cc.find_library('bz2', required : feature)
endif
conf.set10('HAVE_BZIP2', libbzip2.found())
+libbzip2_cflags = libbzip2.partial_dependency(includes: true, compile_args: true)
libxz = dependency('liblzma',
required : get_option('xz'))
#include <fcntl.h>
#include <stdio.h>
-#include <sys/mman.h>
#include <sys/stat.h>
#include <unistd.h>
+#if HAVE_XZ
+#include <lzma.h>
+#endif
+
#if HAVE_LZ4
#include <lz4.h>
-#include <lz4hc.h>
#include <lz4frame.h>
-#endif
-
-#if HAVE_XZ
-#include <lzma.h>
+#include <lz4hc.h>
#endif
#if HAVE_ZSTD
#include <zstd_errors.h>
#endif
+#if HAVE_ZLIB
+#include <zlib.h>
+#endif
+
+#if HAVE_BZIP2
+#include <bzlib.h>
+#endif
+
#include "sd-dlopen.h"
#include "alloc-util.h"
#include "bitfield.h"
#include "compress.h"
#include "dlfcn-util.h"
-#include "fileio.h"
#include "io-util.h"
#include "log.h"
#include "string-table.h"
-#include "string-util.h"
#include "unaligned.h"
+#if HAVE_XZ
+static void *lzma_dl = NULL;
+
+static DLSYM_PROTOTYPE(lzma_code) = NULL;
+static DLSYM_PROTOTYPE(lzma_easy_encoder) = NULL;
+static DLSYM_PROTOTYPE(lzma_end) = NULL;
+static DLSYM_PROTOTYPE(lzma_stream_buffer_encode) = NULL;
+static DLSYM_PROTOTYPE(lzma_stream_decoder) = NULL;
+static DLSYM_PROTOTYPE(lzma_lzma_preset) = NULL;
+
+/* We can’t just do _cleanup_(sym_lzma_end) because a compiler bug makes
+ * this fail with:
+ * ../src/basic/compress.c: In function ‘decompress_blob_xz’:
+ * ../src/basic/compress.c:304:9: error: cleanup argument not a function
+ * 304 | _cleanup_(sym_lzma_end) lzma_stream s = LZMA_STREAM_INIT;
+ * | ^~~~~~~~~
+ */
+static inline void lzma_end_wrapper(lzma_stream *ls) {
+ sym_lzma_end(ls);
+}
+#endif
+
#if HAVE_LZ4
static void *lz4_dl = NULL;
static DLSYM_PROTOTYPE(LZ4F_freeDecompressionContext) = NULL;
static DLSYM_PROTOTYPE(LZ4F_isError) = NULL;
static DLSYM_PROTOTYPE(LZ4_compress_HC) = NULL;
-/* These are used in test-compress.c so we don't make them static. */
-// NOLINTBEGIN(misc-use-internal-linkage)
-DLSYM_PROTOTYPE(LZ4_compress_default) = NULL;
-DLSYM_PROTOTYPE(LZ4_decompress_safe) = NULL;
-DLSYM_PROTOTYPE(LZ4_decompress_safe_partial) = NULL;
-DLSYM_PROTOTYPE(LZ4_versionNumber) = NULL;
-// NOLINTEND(misc-use-internal-linkage)
+static DLSYM_PROTOTYPE(LZ4_compress_default) = NULL;
+static DLSYM_PROTOTYPE(LZ4_decompress_safe) = NULL;
+static DLSYM_PROTOTYPE(LZ4_decompress_safe_partial) = NULL;
+static DLSYM_PROTOTYPE(LZ4_versionNumber) = NULL;
-DEFINE_TRIVIAL_CLEANUP_FUNC_FULL_RENAME(LZ4F_compressionContext_t, sym_LZ4F_freeCompressionContext, LZ4F_freeCompressionContextp, NULL);
-DEFINE_TRIVIAL_CLEANUP_FUNC_FULL_RENAME(LZ4F_decompressionContext_t, sym_LZ4F_freeDecompressionContext, LZ4F_freeDecompressionContextp, NULL);
+static const LZ4F_preferences_t lz4_preferences = {
+ .frameInfo.blockSizeID = 5,
+};
#endif
#if HAVE_ZSTD
static DLSYM_PROTOTYPE(ZSTD_getFrameContentSize) = NULL;
static DLSYM_PROTOTYPE(ZSTD_isError) = NULL;
-DEFINE_TRIVIAL_CLEANUP_FUNC_FULL_RENAME(ZSTD_CCtx*, sym_ZSTD_freeCCtx, ZSTD_freeCCtxp, NULL);
DEFINE_TRIVIAL_CLEANUP_FUNC_FULL_RENAME(ZSTD_DCtx*, sym_ZSTD_freeDCtx, ZSTD_freeDCtxp, NULL);
static int zstd_ret_to_errno(size_t ret) {
}
#endif
-#if HAVE_XZ
-static void *lzma_dl = NULL;
+#if HAVE_ZLIB
+static void *zlib_dl = NULL;
-static DLSYM_PROTOTYPE(lzma_code) = NULL;
-static DLSYM_PROTOTYPE(lzma_easy_encoder) = NULL;
-static DLSYM_PROTOTYPE(lzma_end) = NULL;
-static DLSYM_PROTOTYPE(lzma_stream_buffer_encode) = NULL;
-static DLSYM_PROTOTYPE(lzma_stream_decoder) = NULL;
-static DLSYM_PROTOTYPE(lzma_lzma_preset) = NULL;
+static DLSYM_PROTOTYPE(deflateInit2_) = NULL;
+static DLSYM_PROTOTYPE(deflate) = NULL;
+static DLSYM_PROTOTYPE(deflateEnd) = NULL;
+static DLSYM_PROTOTYPE(inflateInit2_) = NULL;
+static DLSYM_PROTOTYPE(inflate) = NULL;
+static DLSYM_PROTOTYPE(inflateEnd) = NULL;
-/* We can't just do _cleanup_(sym_lzma_end) because a compiler bug makes
- * this fail with:
- * ../src/basic/compress.c: In function ‘decompress_blob_xz’:
- * ../src/basic/compress.c:304:9: error: cleanup argument not a function
- * 304 | _cleanup_(sym_lzma_end) lzma_stream s = LZMA_STREAM_INIT;
- * | ^~~~~~~~~
- */
-static inline void lzma_end_wrapper(lzma_stream *ls) {
- sym_lzma_end(ls);
+static inline void deflateEnd_wrapper(z_stream *s) {
+ sym_deflateEnd(s);
+}
+
+static inline void inflateEnd_wrapper(z_stream *s) {
+ sym_inflateEnd(s);
+}
+#endif
+
+#if HAVE_BZIP2
+static void *bzip2_dl = NULL;
+
+static DLSYM_PROTOTYPE(BZ2_bzCompressInit) = NULL;
+static DLSYM_PROTOTYPE(BZ2_bzCompress) = NULL;
+static DLSYM_PROTOTYPE(BZ2_bzCompressEnd) = NULL;
+static DLSYM_PROTOTYPE(BZ2_bzDecompressInit) = NULL;
+static DLSYM_PROTOTYPE(BZ2_bzDecompress) = NULL;
+static DLSYM_PROTOTYPE(BZ2_bzDecompressEnd) = NULL;
+
+static inline void BZ2_bzCompressEnd_wrapper(bz_stream *s) {
+ sym_BZ2_bzCompressEnd(s);
+}
+
+static inline void BZ2_bzDecompressEnd_wrapper(bz_stream *s) {
+ sym_BZ2_bzDecompressEnd(s);
}
#endif
+/* Opaque Compressor/Decompressor struct definition */
+struct Compressor {
+ Compression type;
+ bool encoding;
+ union {
+#if HAVE_XZ
+ lzma_stream xz;
+#endif
+#if HAVE_LZ4
+ struct {
+ LZ4F_compressionContext_t c_lz4;
+ void *lz4_header; /* stashed frame header from LZ4F_compressBegin */
+ size_t lz4_header_size;
+ };
+ LZ4F_decompressionContext_t d_lz4;
+#endif
+#if HAVE_ZSTD
+ ZSTD_CCtx *c_zstd;
+ ZSTD_DCtx *d_zstd;
+#endif
+#if HAVE_ZLIB
+ z_stream gzip;
+#endif
+#if HAVE_BZIP2
+ bz_stream bzip2;
+#endif
+ };
+};
+
#define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t))
+/* zlib windowBits value for gzip format: MAX_WBITS (15) + 16 to enable gzip header detection/generation */
+#define ZLIB_WBITS_GZIP (15 + 16)
+
static const char* const compression_table[_COMPRESSION_MAX] = {
- [COMPRESSION_NONE] = "NONE",
- [COMPRESSION_XZ] = "XZ",
- [COMPRESSION_LZ4] = "LZ4",
- [COMPRESSION_ZSTD] = "ZSTD",
+ [COMPRESSION_NONE] = "uncompressed", /* backwards compatibility with importd */
+ [COMPRESSION_XZ] = "xz",
+ [COMPRESSION_LZ4] = "lz4",
+ [COMPRESSION_ZSTD] = "zstd",
+ [COMPRESSION_GZIP] = "gzip",
+ [COMPRESSION_BZIP2] = "bzip2",
+};
+
+static const char* const compression_uppercase_table[_COMPRESSION_MAX] = {
+ [COMPRESSION_NONE] = "NONE", /* backwards compatibility with SYSTEMD_JOURNAL_COMPRESS=NONE */
+ [COMPRESSION_XZ] = "XZ",
+ [COMPRESSION_LZ4] = "LZ4",
+ [COMPRESSION_ZSTD] = "ZSTD",
+ [COMPRESSION_GZIP] = "GZIP",
+ [COMPRESSION_BZIP2] = "BZIP2",
};
-static const char* const compression_lowercase_table[_COMPRESSION_MAX] = {
- [COMPRESSION_NONE] = "none",
- [COMPRESSION_XZ] = "xz",
- [COMPRESSION_LZ4] = "lz4",
- [COMPRESSION_ZSTD] = "zstd",
+static const char* const compression_extension_table[_COMPRESSION_MAX] = {
+ [COMPRESSION_NONE] = "",
+ [COMPRESSION_XZ] = ".xz",
+ [COMPRESSION_LZ4] = ".lz4",
+ [COMPRESSION_ZSTD] = ".zst",
+ [COMPRESSION_GZIP] = ".gz",
+ [COMPRESSION_BZIP2] = ".bz2",
};
DEFINE_STRING_TABLE_LOOKUP(compression, Compression);
-DEFINE_STRING_TABLE_LOOKUP(compression_lowercase, Compression);
+DEFINE_STRING_TABLE_LOOKUP(compression_uppercase, Compression);
+DEFINE_STRING_TABLE_LOOKUP(compression_extension, Compression);
+
+Compression compression_from_string_harder(const char *s) {
+ Compression c;
+
+ assert(s);
+
+ c = compression_from_string(s);
+ if (c >= 0)
+ return c;
+
+ return compression_uppercase_from_string(s);
+}
+
+Compression compression_from_filename(const char *filename) {
+ Compression c;
+ const char *e;
+
+ assert(filename);
+
+ e = strrchr(filename, '.');
+ if (!e)
+ return COMPRESSION_NONE;
+
+ c = compression_extension_from_string(e);
+ if (c < 0)
+ return COMPRESSION_NONE;
+
+ return c;
+}
bool compression_supported(Compression c) {
static const unsigned supported =
(1U << COMPRESSION_NONE) |
(1U << COMPRESSION_XZ) * HAVE_XZ |
(1U << COMPRESSION_LZ4) * HAVE_LZ4 |
- (1U << COMPRESSION_ZSTD) * HAVE_ZSTD;
+ (1U << COMPRESSION_ZSTD) * HAVE_ZSTD |
+ (1U << COMPRESSION_GZIP) * HAVE_ZLIB |
+ (1U << COMPRESSION_BZIP2) * HAVE_BZIP2;
assert(c >= 0);
assert(c < _COMPRESSION_MAX);
return BIT_SET(supported, c);
}
-int dlopen_lzma(void) {
+int dlopen_xz(void) {
#if HAVE_XZ
SD_ELF_NOTE_DLOPEN(
"lzma",
#endif
}
-int compress_blob_xz(const void *src, uint64_t src_size,
- void *dst, size_t dst_alloc_size, size_t *dst_size, int level) {
+int dlopen_lz4(void) {
+#if HAVE_LZ4
+ SD_ELF_NOTE_DLOPEN(
+ "lz4",
+ "Support lz4 compression in journal and coredump files",
+ COMPRESSION_PRIORITY_LZ4,
+ "liblz4.so.1");
+
+ return dlopen_many_sym_or_warn(
+ &lz4_dl,
+ "liblz4.so.1", LOG_DEBUG,
+ DLSYM_ARG(LZ4F_compressBegin),
+ DLSYM_ARG(LZ4F_compressBound),
+ DLSYM_ARG(LZ4F_compressEnd),
+ DLSYM_ARG(LZ4F_compressUpdate),
+ DLSYM_ARG(LZ4F_createCompressionContext),
+ DLSYM_ARG(LZ4F_createDecompressionContext),
+ DLSYM_ARG(LZ4F_decompress),
+ DLSYM_ARG(LZ4F_freeCompressionContext),
+ DLSYM_ARG(LZ4F_freeDecompressionContext),
+ DLSYM_ARG(LZ4F_isError),
+ DLSYM_ARG(LZ4_compress_default),
+ DLSYM_ARG(LZ4_compress_HC),
+ DLSYM_ARG(LZ4_decompress_safe),
+ DLSYM_ARG(LZ4_decompress_safe_partial),
+ DLSYM_ARG(LZ4_versionNumber));
+#else
+ return -EOPNOTSUPP;
+#endif
+}
+
+int dlopen_zstd(void) {
+#if HAVE_ZSTD
+ SD_ELF_NOTE_DLOPEN(
+ "zstd",
+ "Support zstd compression in journal and coredump files",
+ COMPRESSION_PRIORITY_ZSTD,
+ "libzstd.so.1");
+
+ return dlopen_many_sym_or_warn(
+ &zstd_dl,
+ "libzstd.so.1", LOG_DEBUG,
+ DLSYM_ARG(ZSTD_getErrorCode),
+ DLSYM_ARG(ZSTD_compress),
+ DLSYM_ARG(ZSTD_getFrameContentSize),
+ DLSYM_ARG(ZSTD_decompressStream),
+ DLSYM_ARG(ZSTD_getErrorName),
+ DLSYM_ARG(ZSTD_DStreamOutSize),
+ DLSYM_ARG(ZSTD_CStreamInSize),
+ DLSYM_ARG(ZSTD_CStreamOutSize),
+ DLSYM_ARG(ZSTD_CCtx_setParameter),
+ DLSYM_ARG(ZSTD_compressStream2),
+ DLSYM_ARG(ZSTD_DStreamInSize),
+ DLSYM_ARG(ZSTD_freeCCtx),
+ DLSYM_ARG(ZSTD_freeDCtx),
+ DLSYM_ARG(ZSTD_isError),
+ DLSYM_ARG(ZSTD_createDCtx),
+ DLSYM_ARG(ZSTD_createCCtx));
+#else
+ return -EOPNOTSUPP;
+#endif
+}
+
+int dlopen_zlib(void) {
+#if HAVE_ZLIB
+ SD_ELF_NOTE_DLOPEN(
+ "zlib",
+ "Support gzip compression and decompression",
+ SD_ELF_NOTE_DLOPEN_PRIORITY_SUGGESTED,
+ "libz.so.1");
+
+ return dlopen_many_sym_or_warn(
+ &zlib_dl,
+ "libz.so.1", LOG_DEBUG,
+ DLSYM_ARG(deflateInit2_),
+ DLSYM_ARG(deflate),
+ DLSYM_ARG(deflateEnd),
+ DLSYM_ARG(inflateInit2_),
+ DLSYM_ARG(inflate),
+ DLSYM_ARG(inflateEnd));
+#else
+ return -EOPNOTSUPP;
+#endif
+}
+
+int dlopen_bzip2(void) {
+#if HAVE_BZIP2
+ SD_ELF_NOTE_DLOPEN(
+ "bzip2",
+ "Support bzip2 compression and decompression",
+ SD_ELF_NOTE_DLOPEN_PRIORITY_SUGGESTED,
+ "libbz2.so.1");
+
+ return dlopen_many_sym_or_warn(
+ &bzip2_dl,
+ "libbz2.so.1", LOG_DEBUG,
+ DLSYM_ARG(BZ2_bzCompressInit),
+ DLSYM_ARG(BZ2_bzCompress),
+ DLSYM_ARG(BZ2_bzCompressEnd),
+ DLSYM_ARG(BZ2_bzDecompressInit),
+ DLSYM_ARG(BZ2_bzDecompress),
+ DLSYM_ARG(BZ2_bzDecompressEnd));
+#else
+ return -EOPNOTSUPP;
+#endif
+}
+
+static int compress_blob_xz(
+ const void *src,
+ uint64_t src_size,
+ void *dst,
+ size_t dst_alloc_size,
+ size_t *dst_size,
+ int level) {
assert(src);
assert(src_size > 0);
size_t out_pos = 0;
int r;
- r = dlopen_lzma();
+ r = dlopen_xz();
if (r < 0)
return r;
#endif
}
-int dlopen_lz4(void) {
-#if HAVE_LZ4
- SD_ELF_NOTE_DLOPEN(
- "lz4",
- "Support lz4 compression in journal and coredump files",
- COMPRESSION_PRIORITY_LZ4,
- "liblz4.so.1");
-
- return dlopen_many_sym_or_warn(
- &lz4_dl,
- "liblz4.so.1", LOG_DEBUG,
- DLSYM_ARG(LZ4F_compressBegin),
- DLSYM_ARG(LZ4F_compressBound),
- DLSYM_ARG(LZ4F_compressEnd),
- DLSYM_ARG(LZ4F_compressUpdate),
- DLSYM_ARG(LZ4F_createCompressionContext),
- DLSYM_ARG(LZ4F_createDecompressionContext),
- DLSYM_ARG(LZ4F_decompress),
- DLSYM_ARG(LZ4F_freeCompressionContext),
- DLSYM_ARG(LZ4F_freeDecompressionContext),
- DLSYM_ARG(LZ4F_isError),
- DLSYM_ARG(LZ4_compress_default),
- DLSYM_ARG(LZ4_compress_HC),
- DLSYM_ARG(LZ4_decompress_safe),
- DLSYM_ARG(LZ4_decompress_safe_partial),
- DLSYM_ARG(LZ4_versionNumber));
-#else
- return -EOPNOTSUPP;
-#endif
-}
-
-int compress_blob_lz4(const void *src, uint64_t src_size,
- void *dst, size_t dst_alloc_size, size_t *dst_size, int level) {
+static int compress_blob_lz4(
+ const void *src,
+ uint64_t src_size,
+ void *dst,
+ size_t dst_alloc_size,
+ size_t *dst_size,
+ int level) {
assert(src);
assert(src_size > 0);
if (src_size < 9)
return -ENOBUFS;
+ if (src_size > INT_MAX)
+ return -EFBIG;
+ if (dst_alloc_size > INT_MAX)
+ dst_alloc_size = INT_MAX;
+
if (level <= 0)
r = sym_LZ4_compress_default(src, (char*)dst + 8, src_size, (int) dst_alloc_size - 8);
else
#endif
}
-int dlopen_zstd(void) {
-#if HAVE_ZSTD
- SD_ELF_NOTE_DLOPEN(
- "zstd",
- "Support zstd compression in journal and coredump files",
- COMPRESSION_PRIORITY_ZSTD,
- "libzstd.so.1");
-
- return dlopen_many_sym_or_warn(
- &zstd_dl,
- "libzstd.so.1", LOG_DEBUG,
- DLSYM_ARG(ZSTD_getErrorCode),
- DLSYM_ARG(ZSTD_compress),
- DLSYM_ARG(ZSTD_getFrameContentSize),
- DLSYM_ARG(ZSTD_decompressStream),
- DLSYM_ARG(ZSTD_getErrorName),
- DLSYM_ARG(ZSTD_DStreamOutSize),
- DLSYM_ARG(ZSTD_CStreamInSize),
- DLSYM_ARG(ZSTD_CStreamOutSize),
- DLSYM_ARG(ZSTD_CCtx_setParameter),
- DLSYM_ARG(ZSTD_compressStream2),
- DLSYM_ARG(ZSTD_DStreamInSize),
- DLSYM_ARG(ZSTD_freeCCtx),
- DLSYM_ARG(ZSTD_freeDCtx),
- DLSYM_ARG(ZSTD_isError),
- DLSYM_ARG(ZSTD_createDCtx),
- DLSYM_ARG(ZSTD_createCCtx));
-#else
- return -EOPNOTSUPP;
-#endif
-}
-
-int compress_blob_zstd(
- const void *src, uint64_t src_size,
- void *dst, size_t dst_alloc_size, size_t *dst_size, int level) {
+static int compress_blob_zstd(
+ const void *src,
+ uint64_t src_size,
+ void *dst,
+ size_t dst_alloc_size,
+ size_t *dst_size,
+ int level) {
assert(src);
assert(src_size > 0);
#endif
}
-int decompress_blob_xz(
- const void *src,
- uint64_t src_size,
- void **dst,
- size_t* dst_size,
- size_t dst_max) {
+static int compress_blob_gzip(const void *src, uint64_t src_size,
+ void *dst, size_t dst_alloc_size, size_t *dst_size, int level) {
assert(src);
assert(src_size > 0);
assert(dst);
+ assert(dst_alloc_size > 0);
assert(dst_size);
-#if HAVE_XZ
+#if HAVE_ZLIB
int r;
- r = dlopen_lzma();
+ r = dlopen_zlib();
if (r < 0)
return r;
- _cleanup_(lzma_end_wrapper) lzma_stream s = LZMA_STREAM_INIT;
- lzma_ret ret = sym_lzma_stream_decoder(&s, UINT64_MAX, 0);
- if (ret != LZMA_OK)
- return -ENOMEM;
-
- size_t space = MIN(src_size * 2, dst_max ?: SIZE_MAX);
- if (!greedy_realloc(dst, space, 1))
+ if (src_size > UINT_MAX)
+ return -EFBIG;
+ if (dst_alloc_size > UINT_MAX)
+ dst_alloc_size = UINT_MAX;
+
+ _cleanup_(deflateEnd_wrapper) z_stream s = {};
+
+ r = sym_deflateInit2_(&s, level < 0 ? Z_DEFAULT_COMPRESSION : level,
+ /* method= */ Z_DEFLATED,
+ /* windowBits= */ ZLIB_WBITS_GZIP,
+ /* memLevel= */ 8,
+ /* strategy= */ Z_DEFAULT_STRATEGY,
+ ZLIB_VERSION, (int) sizeof(s));
+ if (r != Z_OK)
return -ENOMEM;
- s.next_in = src;
+ s.next_in = (void*) src;
+ s.avail_in = src_size;
+ s.next_out = dst;
+ s.avail_out = dst_alloc_size;
+
+ r = sym_deflate(&s, Z_FINISH);
+ if (r != Z_STREAM_END)
+ return -ENOBUFS;
+
+ *dst_size = dst_alloc_size - s.avail_out;
+ return 0;
+#else
+ return -EPROTONOSUPPORT;
+#endif
+}
+
+static int compress_blob_bzip2(
+ const void *src, uint64_t src_size,
+ void *dst, size_t dst_alloc_size, size_t *dst_size, int level) {
+
+ assert(src);
+ assert(src_size > 0);
+ assert(dst);
+ assert(dst_alloc_size > 0);
+ assert(dst_size);
+
+#if HAVE_BZIP2
+ int r;
+
+ r = dlopen_bzip2();
+ if (r < 0)
+ return r;
+
+ if (src_size > UINT_MAX)
+ return -EFBIG;
+ if (dst_alloc_size > UINT_MAX)
+ dst_alloc_size = UINT_MAX;
+
+ _cleanup_(BZ2_bzCompressEnd_wrapper) bz_stream s = {};
+
+ r = sym_BZ2_bzCompressInit(&s, level < 0 ? 9 : level, /* verbosity= */ 0, /* workFactor= */ 0);
+ if (r != BZ_OK)
+ return -ENOMEM;
+
+ s.next_in = (char*) src;
+ s.avail_in = src_size;
+ s.next_out = (char*) dst;
+ s.avail_out = dst_alloc_size;
+
+ r = sym_BZ2_bzCompress(&s, BZ_FINISH);
+
+ if (r != BZ_STREAM_END)
+ return -ENOBUFS;
+
+ *dst_size = dst_alloc_size - s.avail_out;
+ return 0;
+#else
+ return -EPROTONOSUPPORT;
+#endif
+}
+
+int compress_blob(
+ Compression compression,
+ const void *src, uint64_t src_size,
+ void *dst, size_t dst_alloc_size, size_t *dst_size, int level) {
+
+ switch (compression) {
+ case COMPRESSION_XZ:
+ return compress_blob_xz(src, src_size, dst, dst_alloc_size, dst_size, level);
+ case COMPRESSION_LZ4:
+ return compress_blob_lz4(src, src_size, dst, dst_alloc_size, dst_size, level);
+ case COMPRESSION_ZSTD:
+ return compress_blob_zstd(src, src_size, dst, dst_alloc_size, dst_size, level);
+ case COMPRESSION_GZIP:
+ return compress_blob_gzip(src, src_size, dst, dst_alloc_size, dst_size, level);
+ case COMPRESSION_BZIP2:
+ return compress_blob_bzip2(src, src_size, dst, dst_alloc_size, dst_size, level);
+ default:
+ return -EOPNOTSUPP;
+ }
+}
+
+static int decompress_blob_xz(
+ const void *src,
+ uint64_t src_size,
+ void **dst,
+ size_t *dst_size,
+ size_t dst_max) {
+
+ assert(src);
+ assert(src_size > 0);
+ assert(dst);
+ assert(dst_size);
+
+#if HAVE_XZ
+ int r;
+
+ r = dlopen_xz();
+ if (r < 0)
+ return r;
+
+ _cleanup_(lzma_end_wrapper) lzma_stream s = LZMA_STREAM_INIT;
+ lzma_ret ret = sym_lzma_stream_decoder(&s, UINT64_MAX, /* flags= */ 0);
+ if (ret != LZMA_OK)
+ return -ENOMEM;
+
+ size_t space = MIN(src_size * 2, dst_max ?: SIZE_MAX);
+ if (!greedy_realloc(dst, space, 1))
+ return -ENOMEM;
+
+ s.next_in = src;
s.avail_in = src_size;
s.next_out = *dst;
#endif
}
-int decompress_blob_lz4(
+static int decompress_blob_lz4(
const void *src,
uint64_t src_size,
void **dst,
- size_t* dst_size,
+ size_t *dst_size,
size_t dst_max) {
assert(src);
if (src_size <= 8)
return -EBADMSG;
+ if (src_size - 8 > INT_MAX)
+ return -EFBIG;
+
size = unaligned_read_le64(src);
if (size < 0 || (unsigned) size != unaligned_read_le64(src))
return -EFBIG;
#endif
}
-int decompress_blob_zstd(
+static int decompress_blob_zstd(
const void *src,
uint64_t src_size,
void **dst,
#endif
}
+static int decompress_blob_gzip(
+ const void *src,
+ uint64_t src_size,
+ void **dst,
+ size_t *dst_size,
+ size_t dst_max) {
+
+ assert(src);
+ assert(src_size > 0);
+ assert(dst);
+ assert(dst_size);
+
+#if HAVE_ZLIB
+ int r;
+
+ r = dlopen_zlib();
+ if (r < 0)
+ return r;
+
+ if (src_size > UINT_MAX)
+ return -EFBIG;
+
+ _cleanup_(inflateEnd_wrapper) z_stream s = {};
+
+ r = sym_inflateInit2_(&s, /* windowBits= */ ZLIB_WBITS_GZIP, ZLIB_VERSION, (int) sizeof(s));
+ if (r != Z_OK)
+ return -ENOMEM;
+
+ size_t space = MIN3(src_size * 2, dst_max ?: SIZE_MAX, (size_t) UINT_MAX);
+ if (!greedy_realloc(dst, space, 1))
+ return -ENOMEM;
+
+ s.next_in = (void*) src;
+ s.avail_in = src_size;
+ s.next_out = *dst;
+ s.avail_out = space;
+
+ for (;;) {
+ size_t used;
+
+ r = sym_inflate(&s, Z_NO_FLUSH);
+ if (r == Z_STREAM_END)
+ break;
+ if (!IN_SET(r, Z_OK, Z_BUF_ERROR))
+ return -EBADMSG;
+
+ if (dst_max > 0 && (space - s.avail_out) >= dst_max)
+ break;
+ if (dst_max > 0 && space == dst_max)
+ return -ENOBUFS;
+
+ used = space - s.avail_out;
+ space = MIN3(2 * space, dst_max ?: SIZE_MAX, UINT_MAX);
+ if (!greedy_realloc(dst, space, 1))
+ return -ENOMEM;
+
+ s.avail_out = space - used;
+ s.next_out = *(uint8_t**)dst + used;
+ }
+
+ *dst_size = space - s.avail_out;
+ return 0;
+#else
+ return -EPROTONOSUPPORT;
+#endif
+}
+
+static int decompress_blob_bzip2(
+ const void *src,
+ uint64_t src_size,
+ void **dst,
+ size_t *dst_size,
+ size_t dst_max) {
+
+ assert(src);
+ assert(src_size > 0);
+ assert(dst);
+ assert(dst_size);
+
+#if HAVE_BZIP2
+ int r;
+
+ r = dlopen_bzip2();
+ if (r < 0)
+ return r;
+
+ if (src_size > UINT_MAX)
+ return -EFBIG;
+
+ _cleanup_(BZ2_bzDecompressEnd_wrapper) bz_stream s = {};
+
+ r = sym_BZ2_bzDecompressInit(&s, /* verbosity= */ 0, /* small= */ 0);
+ if (r != BZ_OK)
+ return -ENOMEM;
+
+ size_t space = MIN3(src_size * 2, dst_max ?: SIZE_MAX, (size_t) UINT_MAX);
+ if (!greedy_realloc(dst, space, 1))
+ return -ENOMEM;
+
+ s.next_in = (char*) src;
+ s.avail_in = src_size;
+ s.next_out = (char*) *dst;
+ s.avail_out = space;
+
+ for (;;) {
+ size_t used;
+
+ r = sym_BZ2_bzDecompress(&s);
+ if (r == BZ_STREAM_END)
+ break;
+ if (r != BZ_OK)
+ return -EBADMSG;
+
+ if (dst_max > 0 && (space - s.avail_out) >= dst_max)
+ break;
+ if (dst_max > 0 && space == dst_max)
+ return -ENOBUFS;
+
+ used = space - s.avail_out;
+ space = MIN3(2 * space, dst_max ?: SIZE_MAX, (size_t) UINT_MAX);
+ if (!greedy_realloc(dst, space, 1))
+ return -ENOMEM;
+
+ s.avail_out = space - used;
+ s.next_out = (char*) *dst + used;
+ }
+
+ *dst_size = space - s.avail_out;
+ return 0;
+#else
+ return -EPROTONOSUPPORT;
+#endif
+}
+
int decompress_blob(
Compression compression,
const void *src,
uint64_t src_size,
void **dst,
- size_t* dst_size,
+ size_t *dst_size,
size_t dst_max) {
switch (compression) {
return decompress_blob_zstd(
src, src_size,
dst, dst_size, dst_max);
+ case COMPRESSION_GZIP:
+ return decompress_blob_gzip(
+ src, src_size,
+ dst, dst_size, dst_max);
+ case COMPRESSION_BZIP2:
+ return decompress_blob_bzip2(
+ src, src_size,
+ dst, dst_size, dst_max);
default:
return -EPROTONOSUPPORT;
}
}
-int decompress_startswith_xz(
+int decompress_zlib_raw(
+ const void *src,
+ uint64_t src_size,
+ void *dst,
+ size_t dst_size,
+ int wbits) {
+
+#if HAVE_ZLIB
+ int r;
+
+ r = dlopen_zlib();
+ if (r < 0)
+ return r;
+
+ if (src_size > UINT_MAX)
+ return -EFBIG;
+ if (dst_size > UINT_MAX)
+ return -EFBIG;
+
+ _cleanup_(inflateEnd_wrapper) z_stream s = {
+ .next_in = (void*) src,
+ .avail_in = src_size,
+ .next_out = dst,
+ .avail_out = dst_size,
+ };
+
+ r = sym_inflateInit2_(&s, /* windowBits= */ wbits, ZLIB_VERSION, (int) sizeof(s));
+ if (r != Z_OK)
+ return -EIO;
+
+ r = sym_inflate(&s, Z_FINISH);
+ size_t produced = (uint8_t*) s.next_out - (uint8_t*) dst;
+
+ if (r != Z_STREAM_END || produced != dst_size)
+ return -EBADMSG;
+
+ return 0;
+#else
+ return -EPROTONOSUPPORT;
+#endif
+}
+
+static int decompress_startswith_xz(
const void *src,
uint64_t src_size,
void **buffer,
#if HAVE_XZ
int r;
- r = dlopen_lzma();
+ r = dlopen_xz();
if (r < 0)
return r;
_cleanup_(lzma_end_wrapper) lzma_stream s = LZMA_STREAM_INIT;
- lzma_ret ret = sym_lzma_stream_decoder(&s, UINT64_MAX, 0);
+ lzma_ret ret = sym_lzma_stream_decoder(&s, UINT64_MAX, /* flags= */ 0);
if (ret != LZMA_OK)
return -EBADMSG;
#endif
}
-int decompress_startswith_lz4(
+static int decompress_startswith_lz4(
const void *src,
uint64_t src_size,
void **buffer,
if (src_size <= 8)
return -EBADMSG;
+ if (src_size - 8 > INT_MAX)
+ return -EFBIG;
+
if (!(greedy_realloc(buffer, ALIGN_8(prefix_len + 1), 1)))
return -ENOMEM;
allocated = MALLOC_SIZEOF_SAFE(*buffer);
#endif
}
-int decompress_startswith_zstd(
+static int decompress_startswith_zstd(
const void *src,
uint64_t src_size,
void **buffer,
#endif
}
-int decompress_startswith(
- Compression compression,
+static int decompress_startswith_gzip(
const void *src,
uint64_t src_size,
void **buffer,
size_t prefix_len,
uint8_t extra) {
- switch (compression) {
-
- case COMPRESSION_XZ:
- return decompress_startswith_xz(
- src, src_size,
- buffer,
- prefix, prefix_len,
- extra);
-
- case COMPRESSION_LZ4:
- return decompress_startswith_lz4(
- src, src_size,
- buffer,
- prefix, prefix_len,
- extra);
- case COMPRESSION_ZSTD:
- return decompress_startswith_zstd(
- src, src_size,
- buffer,
- prefix, prefix_len,
- extra);
- default:
- return -EBADMSG;
- }
-}
-
-int compress_stream_xz(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_uncompressed_size) {
- assert(fdf >= 0);
- assert(fdt >= 0);
+ assert(src);
+ assert(src_size > 0);
+ assert(buffer);
+ assert(prefix);
-#if HAVE_XZ
+#if HAVE_ZLIB
int r;
- r = dlopen_lzma();
+ r = dlopen_zlib();
if (r < 0)
return r;
- _cleanup_(lzma_end_wrapper) lzma_stream s = LZMA_STREAM_INIT;
- lzma_ret ret = sym_lzma_easy_encoder(&s, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64);
- if (ret != LZMA_OK)
- return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
- "Failed to initialize XZ encoder: code %u",
- ret);
+ if (src_size > UINT_MAX)
+ return -EFBIG;
- uint8_t buf[BUFSIZ], out[BUFSIZ];
- lzma_action action = LZMA_RUN;
- for (;;) {
- if (s.avail_in == 0 && action == LZMA_RUN) {
- size_t m = sizeof(buf);
- ssize_t n;
-
- if (max_bytes != UINT64_MAX && (uint64_t) m > max_bytes)
- m = (size_t) max_bytes;
-
- n = read(fdf, buf, m);
- if (n < 0)
- return -errno;
- if (n == 0)
- action = LZMA_FINISH;
- else {
- s.next_in = buf;
- s.avail_in = n;
-
- if (max_bytes != UINT64_MAX) {
- assert(max_bytes >= (uint64_t) n);
- max_bytes -= n;
- }
- }
- }
+ _cleanup_(inflateEnd_wrapper) z_stream s = {};
- if (s.avail_out == 0) {
- s.next_out = out;
- s.avail_out = sizeof(out);
- }
+ r = sym_inflateInit2_(&s, /* windowBits= */ ZLIB_WBITS_GZIP, ZLIB_VERSION, (int) sizeof(s));
+ if (r != Z_OK)
+ return -EBADMSG;
- ret = sym_lzma_code(&s, action);
- if (!IN_SET(ret, LZMA_OK, LZMA_STREAM_END))
- return log_error_errno(SYNTHETIC_ERRNO(EBADMSG),
- "Compression failed: code %u",
- ret);
+ if (!(greedy_realloc(buffer, ALIGN_8(prefix_len + 1), 1)))
+ return -ENOMEM;
- if (s.avail_out == 0 || ret == LZMA_STREAM_END) {
- ssize_t n, k;
+ size_t allocated = MALLOC_SIZEOF_SAFE(*buffer);
- n = sizeof(out) - s.avail_out;
+ s.next_in = (void*) src;
+ s.avail_in = src_size;
- k = loop_write(fdt, out, n);
- if (k < 0)
- return k;
+ s.next_out = *buffer;
+ s.avail_out = MIN(allocated, (size_t) UINT_MAX);
- if (ret == LZMA_STREAM_END) {
- if (ret_uncompressed_size)
- *ret_uncompressed_size = s.total_in;
+ for (;;) {
+ r = sym_inflate(&s, Z_FINISH);
- if (s.total_in == 0)
- log_debug("XZ compression finished (no input data)");
- else
- log_debug("XZ compression finished (%"PRIu64" -> %"PRIu64" bytes, %.1f%%)",
- s.total_in, s.total_out,
- (double) s.total_out / s.total_in * 100);
+ if (!IN_SET(r, Z_OK, Z_STREAM_END, Z_BUF_ERROR))
+ return -EBADMSG;
- return 0;
- }
- }
+ if (allocated - s.avail_out >= prefix_len + 1)
+ return memcmp(*buffer, prefix, prefix_len) == 0 &&
+ ((const uint8_t*) *buffer)[prefix_len] == extra;
+
+ if (r == Z_STREAM_END)
+ return 0;
+
+ size_t used = allocated - s.avail_out;
+
+ if (!(greedy_realloc(buffer, allocated * 2, 1)))
+ return -ENOMEM;
+
+ allocated = MALLOC_SIZEOF_SAFE(*buffer);
+ s.avail_out = MIN(allocated - used, (size_t) UINT_MAX);
+ s.next_out = *(uint8_t**)buffer + used;
}
#else
return -EPROTONOSUPPORT;
#endif
}
-#define LZ4_BUFSIZE (512*1024u)
+static int decompress_startswith_bzip2(
+ const void *src,
+ uint64_t src_size,
+ void **buffer,
+ const void *prefix,
+ size_t prefix_len,
+ uint8_t extra) {
-int compress_stream_lz4(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_uncompressed_size) {
+ assert(src);
+ assert(src_size > 0);
+ assert(buffer);
+ assert(prefix);
-#if HAVE_LZ4
- LZ4F_errorCode_t c;
- _cleanup_(LZ4F_freeCompressionContextp) LZ4F_compressionContext_t ctx = NULL;
- _cleanup_free_ void *in_buff = NULL;
- _cleanup_free_ char *out_buff = NULL;
- size_t out_allocsize, n, offset = 0, frame_size;
- uint64_t total_in = 0, total_out;
+#if HAVE_BZIP2
int r;
- static const LZ4F_preferences_t preferences = {
- .frameInfo.blockSizeID = 5,
- };
- r = dlopen_lz4();
+ r = dlopen_bzip2();
if (r < 0)
return r;
- c = sym_LZ4F_createCompressionContext(&ctx, LZ4F_VERSION);
- if (sym_LZ4F_isError(c))
- return -ENOMEM;
+ if (src_size > UINT_MAX)
+ return -EFBIG;
- frame_size = sym_LZ4F_compressBound(LZ4_BUFSIZE, &preferences);
- out_allocsize = frame_size + 64*1024; /* add some space for header and trailer */
- out_buff = malloc(out_allocsize);
- if (!out_buff)
- return -ENOMEM;
+ _cleanup_(BZ2_bzDecompressEnd_wrapper) bz_stream s = {};
+
+ r = sym_BZ2_bzDecompressInit(&s, /* verbosity= */ 0, /* small= */ 0);
+ if (r != BZ_OK)
+ return -EBADMSG;
- in_buff = malloc(LZ4_BUFSIZE);
- if (!in_buff)
+ if (!(greedy_realloc(buffer, ALIGN_8(prefix_len + 1), 1)))
return -ENOMEM;
- n = offset = total_out = sym_LZ4F_compressBegin(ctx, out_buff, out_allocsize, &preferences);
- if (sym_LZ4F_isError(n))
- return -EINVAL;
+ size_t allocated = MALLOC_SIZEOF_SAFE(*buffer);
+
+ s.next_in = (char*) src;
+ s.avail_in = src_size;
- log_debug("Buffer size is %zu bytes, header size %zu bytes.", out_allocsize, n);
+ s.next_out = *buffer;
+ s.avail_out = MIN(allocated, (size_t) UINT_MAX);
for (;;) {
- ssize_t k;
+ r = sym_BZ2_bzDecompress(&s);
- k = loop_read(fdf, in_buff, LZ4_BUFSIZE, true);
- if (k < 0)
- return k;
- if (k == 0)
- break;
- n = sym_LZ4F_compressUpdate(ctx, out_buff + offset, out_allocsize - offset,
- in_buff, k, NULL);
- if (sym_LZ4F_isError(n))
- return -ENOTRECOVERABLE;
+ if (!IN_SET(r, BZ_OK, BZ_STREAM_END))
+ return -EBADMSG;
+
+ if (allocated - s.avail_out >= prefix_len + 1)
+ return memcmp(*buffer, prefix, prefix_len) == 0 &&
+ ((const uint8_t*) *buffer)[prefix_len] == extra;
- total_in += k;
- offset += n;
- total_out += n;
+ if (r == BZ_STREAM_END)
+ return 0;
- if (max_bytes != UINT64_MAX && total_out > (size_t) max_bytes)
- return log_debug_errno(SYNTHETIC_ERRNO(EFBIG),
- "Compressed stream longer than %" PRIu64 " bytes", max_bytes);
+ size_t used = allocated - s.avail_out;
- if (out_allocsize - offset < frame_size + 4) {
- k = loop_write(fdt, out_buff, offset);
- if (k < 0)
- return k;
- offset = 0;
- }
+ if (!(greedy_realloc(buffer, allocated * 2, 1)))
+ return -ENOMEM;
+
+ allocated = MALLOC_SIZEOF_SAFE(*buffer);
+ s.avail_out = MIN(allocated - used, (size_t) UINT_MAX);
+ s.next_out = (char*) *buffer + used;
+ }
+#else
+ return -EPROTONOSUPPORT;
+#endif
+}
+
+int decompress_startswith(
+ Compression compression,
+ const void *src, uint64_t src_size,
+ void **buffer,
+ const void *prefix, size_t prefix_len,
+ uint8_t extra) {
+
+ switch (compression) {
+ case COMPRESSION_XZ:
+ return decompress_startswith_xz(src, src_size, buffer, prefix, prefix_len, extra);
+ case COMPRESSION_LZ4:
+ return decompress_startswith_lz4(src, src_size, buffer, prefix, prefix_len, extra);
+ case COMPRESSION_ZSTD:
+ return decompress_startswith_zstd(src, src_size, buffer, prefix, prefix_len, extra);
+ case COMPRESSION_GZIP:
+ return decompress_startswith_gzip(src, src_size, buffer, prefix, prefix_len, extra);
+ case COMPRESSION_BZIP2:
+ return decompress_startswith_bzip2(src, src_size, buffer, prefix, prefix_len, extra);
+ default:
+ return -EOPNOTSUPP;
}
+}
+
+int compress_stream(
+ Compression type,
+ int fdf, int fdt,
+ uint64_t max_bytes,
+ uint64_t *ret_uncompressed_size) {
+
+ _cleanup_(compressor_freep) Compressor *c = NULL;
+ _cleanup_free_ void *buf = NULL;
+ _cleanup_free_ uint8_t *input = NULL;
+ size_t buf_size = 0, buf_alloc = 0;
+ uint64_t total_in = 0, total_out = 0;
+ int r;
- n = sym_LZ4F_compressEnd(ctx, out_buff + offset, out_allocsize - offset, NULL);
- if (sym_LZ4F_isError(n))
- return -ENOTRECOVERABLE;
+ assert(fdf >= 0);
+ assert(fdt >= 0);
- offset += n;
- total_out += n;
- r = loop_write(fdt, out_buff, offset);
+ r = compressor_new(&c, type);
if (r < 0)
return r;
+ input = new(uint8_t, COMPRESS_PIPE_BUFFER_SIZE);
+ if (!input)
+ return -ENOMEM;
+
+ for (;;) {
+ size_t m = COMPRESS_PIPE_BUFFER_SIZE;
+ ssize_t n;
+
+ if (max_bytes != UINT64_MAX && (uint64_t) m > max_bytes)
+ m = (size_t) max_bytes;
+
+ n = read(fdf, input, m);
+ if (n < 0)
+ return -errno;
+
+ if (n == 0) {
+ r = compressor_finish(c, &buf, &buf_size, &buf_alloc);
+ if (r < 0)
+ return r;
+
+ if (buf_size > 0) {
+ r = loop_write(fdt, buf, buf_size);
+ if (r < 0)
+ return r;
+ total_out += buf_size;
+ }
+ break;
+ }
+
+ total_in += n;
+ if (max_bytes != UINT64_MAX) {
+ assert(max_bytes >= (uint64_t) n);
+ max_bytes -= n;
+ }
+
+ r = compressor_start(c, input, n, &buf, &buf_size, &buf_alloc);
+ if (r < 0)
+ return r;
+
+ if (buf_size > 0) {
+ r = loop_write(fdt, buf, buf_size);
+ if (r < 0)
+ return r;
+ total_out += buf_size;
+ }
+ }
+
if (ret_uncompressed_size)
*ret_uncompressed_size = total_in;
if (total_in == 0)
- log_debug("LZ4 compression finished (no input data)");
+ log_debug("%s compression finished (no input data)", compression_to_string(type));
else
- log_debug("LZ4 compression finished (%" PRIu64 " -> %" PRIu64 " bytes, %.1f%%)",
- total_in, total_out,
- (double) total_out / total_in * 100);
+ log_debug("%s compression finished (%" PRIu64 " -> %" PRIu64 " bytes, %.1f%%)",
+ compression_to_string(type), total_in, total_out, (double) total_out / total_in * 100);
return 0;
-#else
- return -EPROTONOSUPPORT;
-#endif
}
-#if HAVE_COMPRESSION
/* Determine whether sparse writes should be used for this fd. Sparse writes are only safe on
* regular files without O_APPEND (O_APPEND ignores lseek position, which would collapse holes). */
static int should_sparse(int fd) {
return 0;
}
-static int maybe_sparse_write(int fd, const void *buf, size_t nbytes, bool sparse) {
- int r;
+/* Common helper for decompress_stream_*() wrappers */
+
+struct decompress_stream_userdata {
+ int fd;
+ uint64_t max_bytes;
+ uint64_t total_out;
+ bool sparse;
+};
+
+static int decompress_stream_write_callback(const void *data, size_t size, void *userdata) {
+ struct decompress_stream_userdata *u = ASSERT_PTR(userdata);
- if (sparse) {
- ssize_t k;
+ if (u->max_bytes != UINT64_MAX) {
+ if (u->max_bytes < size)
+ return -EFBIG;
+ u->max_bytes -= size;
+ }
+
+ u->total_out += size;
+ if (u->sparse) {
/* Note: sparse_write() does not retry on EINTR and converts short writes to -EIO.
* This is fine here since sparse mode is only used on regular files, where short
* writes and EINTR are not expected in practice. */
- k = sparse_write(fd, buf, nbytes, 64);
+ ssize_t k = sparse_write(u->fd, data, size, 64);
if (k < 0)
return (int) k;
- } else {
- r = loop_write_full(fd, buf, nbytes, USEC_INFINITY);
- if (r < 0)
- return r;
+ return 0;
}
- return 0;
+ return loop_write(u->fd, data, size);
}
+
+static int decompressor_new(Decompressor **ret, Compression type) {
+#if HAVE_XZ || HAVE_LZ4 || HAVE_ZSTD || HAVE_ZLIB || HAVE_BZIP2
+ int r;
#endif
-int decompress_stream_xz(int fdf, int fdt, uint64_t max_bytes) {
- assert(fdf >= 0);
- assert(fdt >= 0);
+ assert(ret);
-#if HAVE_XZ
- bool sparse = should_sparse(fdt) > 0;
- int r;
+ _cleanup_(compressor_freep) Decompressor *c = new0(Decompressor, 1);
+ if (!c)
+ return -ENOMEM;
- r = dlopen_lzma();
- if (r < 0)
- return r;
+ c->type = _COMPRESSION_INVALID;
- _cleanup_(lzma_end_wrapper) lzma_stream s = LZMA_STREAM_INIT;
- lzma_ret ret = sym_lzma_stream_decoder(&s, UINT64_MAX, 0);
- if (ret != LZMA_OK)
- return log_debug_errno(SYNTHETIC_ERRNO(ENOMEM),
- "Failed to initialize XZ decoder: code %u",
- ret);
+ switch (type) {
- uint8_t buf[BUFSIZ], out[BUFSIZ];
- lzma_action action = LZMA_RUN;
- for (;;) {
- if (s.avail_in == 0 && action == LZMA_RUN) {
- ssize_t n;
-
- n = read(fdf, buf, sizeof(buf));
- if (n < 0)
- return -errno;
- if (n == 0)
- action = LZMA_FINISH;
- else {
- s.next_in = buf;
- s.avail_in = n;
- }
- }
+#if HAVE_XZ
+ case COMPRESSION_XZ:
+ r = dlopen_xz();
+ if (r < 0)
+ return r;
- if (s.avail_out == 0) {
- s.next_out = out;
- s.avail_out = sizeof(out);
- }
+ if (sym_lzma_stream_decoder(&c->xz, UINT64_MAX, LZMA_TELL_UNSUPPORTED_CHECK | LZMA_CONCATENATED) != LZMA_OK)
+ return -EIO;
+ break;
+#endif
- ret = sym_lzma_code(&s, action);
- if (!IN_SET(ret, LZMA_OK, LZMA_STREAM_END))
- return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG),
- "Decompression failed: code %u",
- ret);
+#if HAVE_LZ4
+ case COMPRESSION_LZ4: {
+ r = dlopen_lz4();
+ if (r < 0)
+ return r;
+
+ size_t rc = sym_LZ4F_createDecompressionContext(&c->d_lz4, LZ4F_VERSION);
+ if (sym_LZ4F_isError(rc))
+ return -ENOMEM;
+
+ break;
+ }
+#endif
- if (s.avail_out == 0 || ret == LZMA_STREAM_END) {
- ssize_t n, k;
+#if HAVE_ZSTD
+ case COMPRESSION_ZSTD:
+ r = dlopen_zstd();
+ if (r < 0)
+ return r;
- n = sizeof(out) - s.avail_out;
+ c->d_zstd = sym_ZSTD_createDCtx();
+ if (!c->d_zstd)
+ return -ENOMEM;
+ break;
+#endif
- if (max_bytes != UINT64_MAX) {
- if (max_bytes < (uint64_t) n)
- return -EFBIG;
+#if HAVE_ZLIB
+ case COMPRESSION_GZIP:
+ r = dlopen_zlib();
+ if (r < 0)
+ return r;
- max_bytes -= n;
- }
+ r = sym_inflateInit2_(&c->gzip, /* windowBits= */ ZLIB_WBITS_GZIP, ZLIB_VERSION, (int) sizeof(c->gzip));
+ if (r != Z_OK)
+ return -EIO;
+ break;
+#endif
- k = maybe_sparse_write(fdt, out, n, sparse);
- if (k < 0)
- return k;
+#if HAVE_BZIP2
+ case COMPRESSION_BZIP2:
+ r = dlopen_bzip2();
+ if (r < 0)
+ return r;
- if (ret == LZMA_STREAM_END) {
- if (s.total_in == 0)
- log_debug("XZ decompression finished (no input data)");
- else
- log_debug("XZ decompression finished (%"PRIu64" -> %"PRIu64" bytes, %.1f%%)",
- s.total_in, s.total_out,
- (double) s.total_out / s.total_in * 100);
+ r = sym_BZ2_bzDecompressInit(&c->bzip2, /* verbosity= */ 0, /* small= */ 0);
+ if (r != BZ_OK)
+ return -EIO;
+ break;
+#endif
- return sparse ? finalize_sparse(fdt) : 0;
- }
- }
+ default:
+ return -EOPNOTSUPP;
}
-#else
- return log_debug_errno(SYNTHETIC_ERRNO(EPROTONOSUPPORT),
- "Cannot decompress file. Compiled without XZ support.");
-#endif
+
+ c->type = type;
+ c->encoding = false;
+ *ret = TAKE_PTR(c);
+ return 0;
}
-int decompress_stream_lz4(int fdf, int fdt, uint64_t max_bytes) {
-#if HAVE_LZ4
- size_t c;
- _cleanup_(LZ4F_freeDecompressionContextp) LZ4F_decompressionContext_t ctx = NULL;
- _cleanup_free_ char *buf = NULL;
- char *src;
- struct stat st;
- bool sparse = should_sparse(fdt) > 0;
+int decompress_stream(
+ Compression type,
+ int fdf, int fdt,
+ uint64_t max_bytes) {
+
+ _cleanup_(compressor_freep) Decompressor *c = NULL;
+ _cleanup_free_ uint8_t *buf = NULL;
+ uint64_t total_in = 0;
int r;
- size_t total_in = 0, total_out = 0;
- r = dlopen_lz4();
+ assert(fdf >= 0);
+ assert(fdt >= 0);
+
+ r = decompressor_new(&c, type);
if (r < 0)
return r;
- c = sym_LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION);
- if (sym_LZ4F_isError(c))
- return -ENOMEM;
-
- if (fstat(fdf, &st) < 0)
- return log_debug_errno(errno, "fstat() failed: %m");
-
- if (file_offset_beyond_memory_size(st.st_size))
- return -EFBIG;
+ struct decompress_stream_userdata userdata = {
+ .fd = fdt,
+ .max_bytes = max_bytes,
+ .sparse = should_sparse(fdt) > 0,
+ };
- buf = malloc(LZ4_BUFSIZE);
+ buf = new(uint8_t, COMPRESS_PIPE_BUFFER_SIZE);
if (!buf)
return -ENOMEM;
- src = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fdf, 0);
- if (src == MAP_FAILED)
- return -errno;
-
- while (total_in < (size_t) st.st_size) {
- size_t produced = LZ4_BUFSIZE;
- size_t used = st.st_size - total_in;
-
- c = sym_LZ4F_decompress(ctx, buf, &produced, src + total_in, &used, NULL);
- if (sym_LZ4F_isError(c)) {
- r = -EBADMSG;
- goto cleanup;
- }
+ for (;;) {
+ ssize_t n;
- total_in += used;
- total_out += produced;
+ n = read(fdf, buf, COMPRESS_PIPE_BUFFER_SIZE);
+ if (n < 0)
+ return -errno;
+ if (n == 0)
+ break;
- if (max_bytes != UINT64_MAX && total_out > (size_t) max_bytes) {
- log_debug("Decompressed stream longer than %"PRIu64" bytes", max_bytes);
- r = -EFBIG;
- goto cleanup;
- }
+ total_in += n;
- r = maybe_sparse_write(fdt, buf, produced, sparse);
+ r = decompressor_push(c, buf, n, decompress_stream_write_callback, &userdata);
if (r < 0)
- goto cleanup;
+ return r;
}
if (total_in == 0)
- log_debug("LZ4 decompression finished (no input data)");
- else
- log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)",
- total_in, total_out,
- (double) total_out / total_in * 100);
- r = sparse ? finalize_sparse(fdt) : 0;
- cleanup:
- munmap(src, st.st_size);
- return r;
-#else
- return log_debug_errno(SYNTHETIC_ERRNO(EPROTONOSUPPORT),
- "Cannot decompress file. Compiled without LZ4 support.");
-#endif
-}
+ return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG), "%s decompression failed: no data read",
+ compression_to_string(type));
-int compress_stream_zstd(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_uncompressed_size) {
- assert(fdf >= 0);
- assert(fdt >= 0);
+ if (userdata.sparse) {
+ r = finalize_sparse(fdt);
+ if (r < 0)
+ return r;
+ }
-#if HAVE_ZSTD
- _cleanup_(ZSTD_freeCCtxp) ZSTD_CCtx *cctx = NULL;
- _cleanup_free_ void *in_buff = NULL, *out_buff = NULL;
- size_t in_allocsize, out_allocsize;
- size_t z;
- uint64_t left = max_bytes, in_bytes = 0;
- int r;
+ log_debug("%s decompression finished (%" PRIu64 " -> %" PRIu64 " bytes, %.1f%%)",
+ compression_to_string(type), total_in, userdata.total_out,
+ (double) userdata.total_out / total_in * 100);
- r = dlopen_zstd();
- if (r < 0)
- return r;
+ return 0;
+}
- /* Create the context and buffers */
- in_allocsize = sym_ZSTD_CStreamInSize();
- out_allocsize = sym_ZSTD_CStreamOutSize();
- in_buff = malloc(in_allocsize);
- out_buff = malloc(out_allocsize);
- cctx = sym_ZSTD_createCCtx();
- if (!cctx || !out_buff || !in_buff)
- return -ENOMEM;
+int decompress_stream_by_filename(const char *filename, int fdf, int fdt, uint64_t max_bytes) {
+ Compression c = compression_from_filename(filename);
+ if (c == COMPRESSION_NONE)
+ return -EPROTONOSUPPORT;
- z = sym_ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1);
- if (sym_ZSTD_isError(z))
- log_debug("Failed to enable ZSTD checksum, ignoring: %s", sym_ZSTD_getErrorName(z));
+ return decompress_stream(c, fdf, fdt, max_bytes);
+}
- /* This loop read from the input file, compresses that entire chunk,
- * and writes all output produced to the output file.
- */
- for (;;) {
- bool is_last_chunk;
- ZSTD_inBuffer input = {
- .src = in_buff,
- .size = 0,
- .pos = 0
- };
- ssize_t red;
+/* Push-based streaming compression/decompression context API */
- red = loop_read(fdf, in_buff, in_allocsize, true);
- if (red < 0)
- return red;
- is_last_chunk = red == 0;
+Compressor* compressor_free(Compressor *c) {
+ if (!c)
+ return NULL;
- in_bytes += (size_t) red;
- input.size = (size_t) red;
+ switch (c->type) {
- for (bool finished = false; !finished;) {
- ZSTD_outBuffer output = {
- .dst = out_buff,
- .size = out_allocsize,
- .pos = 0
- };
- size_t remaining;
- ssize_t wrote;
-
- /* Compress into the output buffer and write all of the
- * output to the file so we can reuse the buffer next
- * iteration.
- */
- remaining = sym_ZSTD_compressStream2(
- cctx, &output, &input,
- is_last_chunk ? ZSTD_e_end : ZSTD_e_continue);
-
- if (sym_ZSTD_isError(remaining)) {
- log_debug("ZSTD encoder failed: %s", sym_ZSTD_getErrorName(remaining));
- return zstd_ret_to_errno(remaining);
- }
+#if HAVE_XZ
+ case COMPRESSION_XZ:
+ sym_lzma_end(&c->xz);
+ break;
+#endif
- if (left < output.pos)
- return -EFBIG;
+#if HAVE_LZ4
+ case COMPRESSION_LZ4:
+ if (c->encoding) {
+ sym_LZ4F_freeCompressionContext(c->c_lz4);
+ c->c_lz4 = NULL;
+ c->lz4_header = mfree(c->lz4_header);
+ } else {
+ sym_LZ4F_freeDecompressionContext(c->d_lz4);
+ c->d_lz4 = NULL;
+ }
+ break;
+#endif
- wrote = loop_write_full(fdt, output.dst, output.pos, USEC_INFINITY);
- if (wrote < 0)
- return wrote;
+#if HAVE_ZSTD
+ case COMPRESSION_ZSTD:
+ if (c->encoding) {
+ sym_ZSTD_freeCCtx(c->c_zstd);
+ c->c_zstd = NULL;
+ } else {
+ sym_ZSTD_freeDCtx(c->d_zstd);
+ c->d_zstd = NULL;
+ }
+ break;
+#endif
- left -= output.pos;
+#if HAVE_ZLIB
+ case COMPRESSION_GZIP:
+ if (c->encoding)
+ sym_deflateEnd(&c->gzip);
+ else
+ sym_inflateEnd(&c->gzip);
+ break;
+#endif
- /* If we're on the last chunk we're finished when zstd
- * returns 0, which means its consumed all the input AND
- * finished the frame. Otherwise, we're finished when
- * we've consumed all the input.
- */
- finished = is_last_chunk ? (remaining == 0) : (input.pos == input.size);
- }
+#if HAVE_BZIP2
+ case COMPRESSION_BZIP2:
+ if (c->encoding)
+ sym_BZ2_bzCompressEnd(&c->bzip2);
+ else
+ sym_BZ2_bzDecompressEnd(&c->bzip2);
+ break;
+#endif
- /* zstd only returns 0 when the input is completely consumed */
- assert(input.pos == input.size);
- if (is_last_chunk)
- break;
+ default:
+ break;
}
- if (ret_uncompressed_size)
- *ret_uncompressed_size = in_bytes;
-
- if (in_bytes == 0)
- log_debug("ZSTD compression finished (no input data)");
- else
- log_debug("ZSTD compression finished (%" PRIu64 " -> %" PRIu64 " bytes, %.1f%%)",
- in_bytes, max_bytes - left, (double) (max_bytes - left) / in_bytes * 100);
+ return mfree(c);
+}
- return 0;
-#else
- return -EPROTONOSUPPORT;
-#endif
+Compression compressor_type(const Compressor *c) {
+ return c ? c->type : _COMPRESSION_INVALID;
}
-int decompress_stream_zstd(int fdf, int fdt, uint64_t max_bytes) {
- assert(fdf >= 0);
- assert(fdt >= 0);
+int decompressor_detect(Decompressor **ret, const void *data, size_t size) {
+ static const uint8_t xz_signature[] = {
+ 0xfd, '7', 'z', 'X', 'Z', 0x00
+ };
+ static const uint8_t lz4_signature[] = {
+ 0x04, 0x22, 0x4d, 0x18
+ };
+ static const uint8_t zstd_signature[] = {
+ 0x28, 0xb5, 0x2f, 0xfd
+ };
+ static const uint8_t gzip_signature[] = {
+ 0x1f, 0x8b
+ };
+ static const uint8_t bzip2_signature[] = {
+ 'B', 'Z', 'h'
+ };
-#if HAVE_ZSTD
- _cleanup_(ZSTD_freeDCtxp) ZSTD_DCtx *dctx = NULL;
- _cleanup_free_ void *in_buff = NULL, *out_buff = NULL;
- bool sparse = should_sparse(fdt) > 0;
- size_t in_allocsize, out_allocsize;
- size_t last_result = 0;
- uint64_t left = max_bytes, in_bytes = 0;
+#if HAVE_XZ || HAVE_LZ4 || HAVE_ZSTD || HAVE_ZLIB || HAVE_BZIP2
int r;
+#endif
- r = dlopen_zstd();
- if (r < 0)
- return r;
- /* Create the context and buffers */
- in_allocsize = sym_ZSTD_DStreamInSize();
- out_allocsize = sym_ZSTD_DStreamOutSize();
- in_buff = malloc(in_allocsize);
- out_buff = malloc(out_allocsize);
- dctx = sym_ZSTD_createDCtx();
- if (!dctx || !out_buff || !in_buff)
- return -ENOMEM;
-
- /* This loop assumes that the input file is one or more concatenated
- * zstd streams. This example won't work if there is trailing non-zstd
- * data at the end, but streaming decompression in general handles this
- * case. ZSTD_decompressStream() returns 0 exactly when the frame is
- * completed, and doesn't consume input after the frame.
- */
- for (;;) {
- bool has_error = false;
- ZSTD_inBuffer input = {
- .src = in_buff,
- .size = 0,
- .pos = 0
- };
- ssize_t red;
+ assert(ret);
- red = loop_read(fdf, in_buff, in_allocsize, true);
- if (red < 0)
- return red;
- if (red == 0)
- break;
+ if (*ret)
+ return 1;
- in_bytes += (size_t) red;
- input.size = (size_t) red;
- input.pos = 0;
+ if (size < MAX5(sizeof(xz_signature),
+ sizeof(gzip_signature),
+ sizeof(zstd_signature),
+ sizeof(bzip2_signature),
+ sizeof(lz4_signature)))
+ return 0;
- /* Given a valid frame, zstd won't consume the last byte of the
- * frame until it has flushed all of the decompressed data of
- * the frame. So input.pos < input.size means frame is not done
- * or there is still output available.
- */
- while (input.pos < input.size) {
- ZSTD_outBuffer output = {
- .dst = out_buff,
- .size = out_allocsize,
- .pos = 0
- };
- ssize_t wrote;
- /* The return code is zero if the frame is complete, but
- * there may be multiple frames concatenated together.
- * Zstd will automatically reset the context when a
- * frame is complete. Still, calling ZSTD_DCtx_reset()
- * can be useful to reset the context to a clean state,
- * for instance if the last decompression call returned
- * an error.
- */
- last_result = sym_ZSTD_decompressStream(dctx, &output, &input);
- if (sym_ZSTD_isError(last_result)) {
- has_error = true;
- break;
- }
+ assert(data);
- if (left < output.pos)
- return -EFBIG;
+ _cleanup_(compressor_freep) Decompressor *c = new0(Decompressor, 1);
+ if (!c)
+ return -ENOMEM;
- wrote = maybe_sparse_write(fdt, output.dst, output.pos, sparse);
- if (wrote < 0)
- return wrote;
+ c->type = COMPRESSION_NONE;
- left -= output.pos;
- }
- if (has_error)
- break;
- }
+#if HAVE_XZ
+ if (c->type == COMPRESSION_NONE && memcmp(data, xz_signature, sizeof(xz_signature)) == 0) {
+ r = dlopen_xz();
+ if (r < 0)
+ return r;
- if (in_bytes == 0)
- return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG), "ZSTD decoder failed: no data read");
+ lzma_ret xzr = sym_lzma_stream_decoder(&c->xz, UINT64_MAX, LZMA_TELL_UNSUPPORTED_CHECK | LZMA_CONCATENATED);
+ if (xzr != LZMA_OK)
+ return -EIO;
- if (last_result != 0) {
- /* The last return value from ZSTD_decompressStream did not end
- * on a frame, but we reached the end of the file! We assume
- * this is an error, and the input was truncated.
- */
- log_debug("ZSTD decoder failed: %s", sym_ZSTD_getErrorName(last_result));
- return zstd_ret_to_errno(last_result);
+ c->type = COMPRESSION_XZ;
}
-
- if (in_bytes == 0)
- log_debug("ZSTD decompression finished (no input data)");
- else
- log_debug("ZSTD decompression finished (%" PRIu64 " -> %" PRIu64 " bytes, %.1f%%)",
- in_bytes,
- max_bytes - left,
- (double) (max_bytes - left) / in_bytes * 100);
- return sparse ? finalize_sparse(fdt) : 0;
-#else
- return log_debug_errno(SYNTHETIC_ERRNO(EPROTONOSUPPORT),
- "Cannot decompress file. Compiled without ZSTD support.");
#endif
-}
-
-int decompress_stream(const char *filename, int fdf, int fdt, uint64_t max_bytes) {
- if (endswith(filename, ".lz4"))
- return decompress_stream_lz4(fdf, fdt, max_bytes);
- if (endswith(filename, ".xz"))
- return decompress_stream_xz(fdf, fdt, max_bytes);
- if (endswith(filename, ".zst"))
- return decompress_stream_zstd(fdf, fdt, max_bytes);
+#if HAVE_LZ4
+ if (c->type == COMPRESSION_NONE && memcmp(data, lz4_signature, sizeof(lz4_signature)) == 0) {
+ r = dlopen_lz4();
+ if (r < 0)
+ return r;
- return -EPROTONOSUPPORT;
+ size_t rc = sym_LZ4F_createDecompressionContext(&c->d_lz4, LZ4F_VERSION);
+ if (sym_LZ4F_isError(rc))
+ return -ENOMEM;
+
+ c->type = COMPRESSION_LZ4;
+ }
+#endif
+
+#if HAVE_ZSTD
+ if (c->type == COMPRESSION_NONE && memcmp(data, zstd_signature, sizeof(zstd_signature)) == 0) {
+ r = dlopen_zstd();
+ if (r < 0)
+ return r;
+
+ c->d_zstd = sym_ZSTD_createDCtx();
+ if (!c->d_zstd)
+ return -ENOMEM;
+
+ c->type = COMPRESSION_ZSTD;
+ }
+#endif
+
+#if HAVE_ZLIB
+ if (c->type == COMPRESSION_NONE && memcmp(data, gzip_signature, sizeof(gzip_signature)) == 0) {
+ r = dlopen_zlib();
+ if (r < 0)
+ return r;
+
+ r = sym_inflateInit2_(&c->gzip, /* windowBits= */ ZLIB_WBITS_GZIP, ZLIB_VERSION, (int) sizeof(c->gzip));
+ if (r != Z_OK)
+ return -EIO;
+
+ c->type = COMPRESSION_GZIP;
+ }
+#endif
+
+#if HAVE_BZIP2
+ if (c->type == COMPRESSION_NONE && memcmp(data, bzip2_signature, sizeof(bzip2_signature)) == 0) {
+ r = dlopen_bzip2();
+ if (r < 0)
+ return r;
+
+ r = sym_BZ2_bzDecompressInit(&c->bzip2, /* verbosity= */ 0, /* small= */ 0);
+ if (r != BZ_OK)
+ return -EIO;
+
+ c->type = COMPRESSION_BZIP2;
+ }
+#endif
+
+ c->encoding = false;
+
+ log_debug("Detected compression type: %s", compression_to_string(c->type));
+ *ret = TAKE_PTR(c);
+ return 1;
+}
+
+int decompressor_force_off(Decompressor **ret) {
+ assert(ret);
+
+ *ret = compressor_free(*ret);
+
+ Decompressor *c = new0(Decompressor, 1);
+ if (!c)
+ return -ENOMEM;
+
+ c->type = COMPRESSION_NONE;
+ c->encoding = false;
+ *ret = c;
+ return 0;
+}
+
+int decompressor_push(Decompressor *c, const void *data, size_t size, DecompressorCallback callback, void *userdata) {
+#if HAVE_XZ || HAVE_LZ4 || HAVE_ZSTD || HAVE_ZLIB || HAVE_BZIP2
+ _cleanup_free_ uint8_t *buffer = NULL;
+#endif
+ int r;
+
+ assert(c);
+ assert(callback);
+
+ if (c->encoding)
+ return -EINVAL;
+
+ if (size == 0)
+ return 1;
+
+ assert(data);
+
+#if HAVE_XZ || HAVE_LZ4 || HAVE_ZSTD || HAVE_ZLIB || HAVE_BZIP2
+ if (c->type != COMPRESSION_NONE) {
+ buffer = new(uint8_t, COMPRESS_PIPE_BUFFER_SIZE);
+ if (!buffer)
+ return -ENOMEM;
+ }
+#endif
+
+ switch (c->type) {
+
+ case COMPRESSION_NONE:
+ r = callback(data, size, userdata);
+ if (r < 0)
+ return r;
+
+ break;
+
+#if HAVE_XZ
+ case COMPRESSION_XZ:
+ c->xz.next_in = data;
+ c->xz.avail_in = size;
+
+ while (c->xz.avail_in > 0) {
+ c->xz.next_out = buffer;
+ c->xz.avail_out = COMPRESS_PIPE_BUFFER_SIZE;
+
+ lzma_ret lzr = sym_lzma_code(&c->xz, LZMA_RUN);
+ if (!IN_SET(lzr, LZMA_OK, LZMA_STREAM_END))
+ return -EBADMSG;
+
+ if (c->xz.avail_out < COMPRESS_PIPE_BUFFER_SIZE) {
+ r = callback(buffer, COMPRESS_PIPE_BUFFER_SIZE - c->xz.avail_out, userdata);
+ if (r < 0)
+ return r;
+ }
+ }
+
+ break;
+#endif
+
+#if HAVE_LZ4
+ case COMPRESSION_LZ4: {
+ const uint8_t *src = data;
+ size_t src_remaining = size;
+
+ while (src_remaining > 0) {
+ size_t produced = COMPRESS_PIPE_BUFFER_SIZE;
+ size_t consumed = src_remaining;
+
+ size_t rc = sym_LZ4F_decompress(c->d_lz4, buffer, &produced, src, &consumed, NULL);
+ if (sym_LZ4F_isError(rc))
+ return -EBADMSG;
+
+ if (consumed == 0 && produced == 0)
+ break; /* No progress possible with current input */
+
+ src += consumed;
+ src_remaining -= consumed;
+
+ if (produced > 0) {
+ r = callback(buffer, produced, userdata);
+ if (r < 0)
+ return r;
+ }
+ }
+
+ break;
+ }
+#endif
+
+#if HAVE_ZSTD
+ case COMPRESSION_ZSTD: {
+ ZSTD_inBuffer input = {
+ .src = (void*) data,
+ .size = size,
+ };
+
+ while (input.pos < input.size) {
+ ZSTD_outBuffer output = {
+ .dst = buffer,
+ .size = COMPRESS_PIPE_BUFFER_SIZE,
+ };
+
+ size_t res = sym_ZSTD_decompressStream(c->d_zstd, &output, &input);
+ if (sym_ZSTD_isError(res))
+ return -EBADMSG;
+
+ if (output.pos > 0) {
+ r = callback(output.dst, output.pos, userdata);
+ if (r < 0)
+ return r;
+ }
+ }
+
+ break;
+ }
+#endif
+
+#if HAVE_ZLIB
+ case COMPRESSION_GZIP:
+ if (size > UINT_MAX)
+ return -EFBIG;
+
+ c->gzip.next_in = (void*) data;
+ c->gzip.avail_in = size;
+
+ while (c->gzip.avail_in > 0) {
+ c->gzip.next_out = buffer;
+ c->gzip.avail_out = COMPRESS_PIPE_BUFFER_SIZE;
+
+ int zr = sym_inflate(&c->gzip, Z_NO_FLUSH);
+ if (!IN_SET(zr, Z_OK, Z_STREAM_END))
+ return -EBADMSG;
+
+ if (c->gzip.avail_out < COMPRESS_PIPE_BUFFER_SIZE) {
+ r = callback(buffer, COMPRESS_PIPE_BUFFER_SIZE - c->gzip.avail_out, userdata);
+ if (r < 0)
+ return r;
+ }
+
+ if (zr == Z_STREAM_END)
+ break;
+ }
+
+ break;
+#endif
+
+#if HAVE_BZIP2
+ case COMPRESSION_BZIP2:
+ if (size > UINT_MAX)
+ return -EFBIG;
+
+ c->bzip2.next_in = (char*) data;
+ c->bzip2.avail_in = size;
+
+ while (c->bzip2.avail_in > 0) {
+ c->bzip2.next_out = (char*) buffer;
+ c->bzip2.avail_out = COMPRESS_PIPE_BUFFER_SIZE;
+
+ int bzr = sym_BZ2_bzDecompress(&c->bzip2);
+ if (!IN_SET(bzr, BZ_OK, BZ_STREAM_END))
+ return -EBADMSG;
+
+ if (c->bzip2.avail_out < COMPRESS_PIPE_BUFFER_SIZE) {
+ r = callback(buffer, COMPRESS_PIPE_BUFFER_SIZE - c->bzip2.avail_out, userdata);
+ if (r < 0)
+ return r;
+ }
+
+ if (bzr == BZ_STREAM_END)
+ break;
+ }
+
+ break;
+#endif
+
+ default:
+ assert_not_reached();
+ }
+
+ return 1;
+}
+
+int compressor_new(Compressor **ret, Compression type) {
+#if HAVE_XZ || HAVE_LZ4 || HAVE_ZSTD || HAVE_ZLIB || HAVE_BZIP2
+ int r;
+#endif
+
+ assert(ret);
+
+ _cleanup_(compressor_freep) Compressor *c = new0(Compressor, 1);
+ if (!c)
+ return -ENOMEM;
+
+ c->type = _COMPRESSION_INVALID;
+ /* Set encoding early so that compressor_freep calls the correct cleanup (compression vs
+ * decompression) if any operation in the switch fails after setting c->type. This is safe
+ * because _COMPRESSION_INVALID hits the default: break case regardless of the encoding flag. */
+ c->encoding = true;
+
+ switch (type) {
+
+#if HAVE_XZ
+ case COMPRESSION_XZ: {
+ r = dlopen_xz();
+ if (r < 0)
+ return r;
+
+ lzma_ret xzr = sym_lzma_easy_encoder(&c->xz, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64);
+ if (xzr != LZMA_OK)
+ return -EIO;
+
+ c->type = COMPRESSION_XZ;
+ break;
+ }
+#endif
+
+#if HAVE_LZ4
+ case COMPRESSION_LZ4: {
+ r = dlopen_lz4();
+ if (r < 0)
+ return r;
+
+ size_t rc = sym_LZ4F_createCompressionContext(&c->c_lz4, LZ4F_VERSION);
+ if (sym_LZ4F_isError(rc))
+ return -ENOMEM;
+
+ c->type = COMPRESSION_LZ4;
+
+ /* Generate the frame header and stash it for the first compressor_start call */
+ size_t header_bound = sym_LZ4F_compressBound(0, &lz4_preferences);
+ c->lz4_header = malloc(header_bound);
+ if (!c->lz4_header)
+ return -ENOMEM;
+
+ c->lz4_header_size = sym_LZ4F_compressBegin(c->c_lz4, c->lz4_header, header_bound, &lz4_preferences);
+ if (sym_LZ4F_isError(c->lz4_header_size))
+ return -EINVAL;
+
+ break;
+ }
+#endif
+
+#if HAVE_ZSTD
+ case COMPRESSION_ZSTD:
+ r = dlopen_zstd();
+ if (r < 0)
+ return r;
+
+ c->c_zstd = sym_ZSTD_createCCtx();
+ if (!c->c_zstd)
+ return -ENOMEM;
+
+ c->type = COMPRESSION_ZSTD;
+
+ size_t z = sym_ZSTD_CCtx_setParameter(c->c_zstd, ZSTD_c_compressionLevel, ZSTD_CLEVEL_DEFAULT);
+ if (sym_ZSTD_isError(z))
+ return -EIO;
+
+ z = sym_ZSTD_CCtx_setParameter(c->c_zstd, ZSTD_c_checksumFlag, /* enable= */ 1);
+ if (sym_ZSTD_isError(z))
+ log_debug("Failed to enable ZSTD checksum, ignoring: %s", sym_ZSTD_getErrorName(z));
+
+ break;
+#endif
+
+#if HAVE_ZLIB
+ case COMPRESSION_GZIP:
+ r = dlopen_zlib();
+ if (r < 0)
+ return r;
+
+ r = sym_deflateInit2_(&c->gzip,
+ Z_DEFAULT_COMPRESSION,
+ /* method= */ Z_DEFLATED,
+ /* windowBits= */ ZLIB_WBITS_GZIP,
+ /* memLevel= */ 8,
+ /* strategy= */ Z_DEFAULT_STRATEGY,
+ ZLIB_VERSION, (int) sizeof(c->gzip));
+ if (r != Z_OK)
+ return -EIO;
+
+ c->type = COMPRESSION_GZIP;
+ break;
+#endif
+
+#if HAVE_BZIP2
+ case COMPRESSION_BZIP2:
+ r = dlopen_bzip2();
+ if (r < 0)
+ return r;
+
+ r = sym_BZ2_bzCompressInit(&c->bzip2, /* blockSize100k= */ 9, /* verbosity= */ 0, /* workFactor= */ 0);
+ if (r != BZ_OK)
+ return -EIO;
+
+ c->type = COMPRESSION_BZIP2;
+ break;
+#endif
+
+ case COMPRESSION_NONE:
+ c->type = COMPRESSION_NONE;
+ break;
+
+ default:
+ return -EOPNOTSUPP;
+ }
+
+ *ret = TAKE_PTR(c);
+ return 0;
+}
+
+#if HAVE_XZ || HAVE_LZ4 || HAVE_ZSTD || HAVE_ZLIB || HAVE_BZIP2
+static int enlarge_buffer(void **buffer, size_t *buffer_size, size_t *buffer_allocated, size_t need) {
+ assert(buffer);
+ assert(buffer_size);
+ assert(buffer_allocated);
+
+ need = MAX3(need, *buffer_size + 1, (size_t) COMPRESS_PIPE_BUFFER_SIZE);
+ if (*buffer_allocated >= need)
+ return 0;
+
+ if (!greedy_realloc(buffer, need, 1))
+ return -ENOMEM;
+
+ *buffer_allocated = MALLOC_SIZEOF_SAFE(*buffer);
+ return 1;
+}
+#endif
+
+int compressor_start(
+ Compressor *c,
+ const void *data,
+ size_t size,
+ void **buffer,
+ size_t *buffer_size,
+ size_t *buffer_allocated) {
+
+#if HAVE_XZ || HAVE_LZ4 || HAVE_ZSTD || HAVE_ZLIB || HAVE_BZIP2
+ int r;
+#endif
+
+ assert(c);
+ assert(buffer);
+ assert(buffer_size);
+ assert(buffer_allocated);
+
+ if (!c->encoding)
+ return -EINVAL;
+
+ if (size == 0)
+ return 0;
+
+ assert(data);
+
+ *buffer_size = 0;
+
+ switch (c->type) {
+
+#if HAVE_XZ
+ case COMPRESSION_XZ:
+
+ c->xz.next_in = data;
+ c->xz.avail_in = size;
+
+ while (c->xz.avail_in > 0) {
+ lzma_ret lzr;
+
+ r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0);
+ if (r < 0)
+ return r;
+
+ c->xz.next_out = (uint8_t*) *buffer + *buffer_size;
+ c->xz.avail_out = *buffer_allocated - *buffer_size;
+
+ lzr = sym_lzma_code(&c->xz, LZMA_RUN);
+ if (lzr != LZMA_OK)
+ return -EIO;
+
+ *buffer_size += (*buffer_allocated - *buffer_size) - c->xz.avail_out;
+ }
+
+ break;
+#endif
+
+#if HAVE_LZ4
+ case COMPRESSION_LZ4: {
+ /* Prepend any stashed frame header from compressor_new */
+ if (c->lz4_header_size > 0) {
+ r = enlarge_buffer(buffer, buffer_size, buffer_allocated, c->lz4_header_size);
+ if (r < 0)
+ return r;
+
+ memcpy(*buffer, c->lz4_header, c->lz4_header_size);
+ *buffer_size = c->lz4_header_size;
+ c->lz4_header = mfree(c->lz4_header);
+ c->lz4_header_size = 0;
+ }
+
+ size_t bound = sym_LZ4F_compressBound(size, &lz4_preferences);
+ r = enlarge_buffer(buffer, buffer_size, buffer_allocated, *buffer_size + bound);
+ if (r < 0)
+ return r;
+
+ size_t n = sym_LZ4F_compressUpdate(c->c_lz4,
+ (uint8_t*) *buffer + *buffer_size,
+ *buffer_allocated - *buffer_size,
+ data, size, NULL);
+ if (sym_LZ4F_isError(n))
+ return -EIO;
+
+ *buffer_size += n;
+ break;
+ }
+#endif
+
+#if HAVE_ZSTD
+ case COMPRESSION_ZSTD: {
+ ZSTD_inBuffer input = {
+ .src = data,
+ .size = size,
+ };
+
+ while (input.pos < input.size) {
+ r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0);
+ if (r < 0)
+ return r;
+
+ ZSTD_outBuffer output = {
+ .dst = ((uint8_t *) *buffer + *buffer_size),
+ .size = *buffer_allocated - *buffer_size,
+ };
+
+ size_t res = sym_ZSTD_compressStream2(c->c_zstd, &output, &input, ZSTD_e_continue);
+ if (sym_ZSTD_isError(res))
+ return -EIO;
+
+ *buffer_size += output.pos;
+ }
+
+ break;
+ }
+#endif
+
+#if HAVE_ZLIB
+ case COMPRESSION_GZIP:
+ if (size > UINT_MAX)
+ return -EFBIG;
+
+ c->gzip.next_in = (void*) data;
+ c->gzip.avail_in = size;
+
+ while (c->gzip.avail_in > 0) {
+ r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0);
+ if (r < 0)
+ return r;
+
+ size_t avail = MIN(*buffer_allocated - *buffer_size, (size_t) UINT_MAX);
+ c->gzip.next_out = (uint8_t*) *buffer + *buffer_size;
+ c->gzip.avail_out = avail;
+
+ r = sym_deflate(&c->gzip, Z_NO_FLUSH);
+ if (r != Z_OK)
+ return -EIO;
+
+ *buffer_size += avail - c->gzip.avail_out;
+ }
+
+ break;
+#endif
+
+#if HAVE_BZIP2
+ case COMPRESSION_BZIP2:
+ if (size > UINT_MAX)
+ return -EFBIG;
+
+ c->bzip2.next_in = (void*) data;
+ c->bzip2.avail_in = size;
+
+ while (c->bzip2.avail_in > 0) {
+ r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0);
+ if (r < 0)
+ return r;
+
+ size_t avail = MIN(*buffer_allocated - *buffer_size, (size_t) UINT_MAX);
+ c->bzip2.next_out = (void*) ((uint8_t*) *buffer + *buffer_size);
+ c->bzip2.avail_out = avail;
+
+ r = sym_BZ2_bzCompress(&c->bzip2, BZ_RUN);
+ if (r != BZ_RUN_OK)
+ return -EIO;
+
+ *buffer_size += avail - c->bzip2.avail_out;
+ }
+
+ break;
+#endif
+
+ case COMPRESSION_NONE:
+
+ if (*buffer_allocated < size) {
+ void *p;
+
+ p = realloc(*buffer, size);
+ if (!p)
+ return -ENOMEM;
+
+ *buffer = p;
+ *buffer_allocated = size;
+ }
+
+ memcpy(*buffer, data, size);
+ *buffer_size = size;
+ break;
+
+ default:
+ return -EOPNOTSUPP;
+ }
+
+ return 0;
+}
+
+int compressor_finish(Compressor *c, void **buffer, size_t *buffer_size, size_t *buffer_allocated) {
+#if HAVE_XZ || HAVE_LZ4 || HAVE_ZSTD || HAVE_ZLIB || HAVE_BZIP2
+ int r;
+#endif
+
+ assert(c);
+ assert(buffer);
+ assert(buffer_size);
+ assert(buffer_allocated);
+
+ if (!c->encoding)
+ return -EINVAL;
+
+ *buffer_size = 0;
+
+ switch (c->type) {
+
+#if HAVE_XZ
+ case COMPRESSION_XZ: {
+ lzma_ret lzr;
+
+ c->xz.avail_in = 0;
+
+ do {
+ r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0);
+ if (r < 0)
+ return r;
+
+ c->xz.next_out = (uint8_t*) *buffer + *buffer_size;
+ c->xz.avail_out = *buffer_allocated - *buffer_size;
+
+ lzr = sym_lzma_code(&c->xz, LZMA_FINISH);
+ if (!IN_SET(lzr, LZMA_OK, LZMA_STREAM_END))
+ return -EIO;
+
+ *buffer_size += (*buffer_allocated - *buffer_size) - c->xz.avail_out;
+ } while (lzr != LZMA_STREAM_END);
+
+ break;
+ }
+#endif
+
+#if HAVE_LZ4
+ case COMPRESSION_LZ4: {
+ size_t bound = sym_LZ4F_compressBound(0, &lz4_preferences);
+ r = enlarge_buffer(buffer, buffer_size, buffer_allocated, bound);
+ if (r < 0)
+ return r;
+
+ size_t n = sym_LZ4F_compressEnd(c->c_lz4, *buffer, *buffer_allocated, NULL);
+ if (sym_LZ4F_isError(n))
+ return -EIO;
+
+ *buffer_size = n;
+ break;
+ }
+#endif
+
+#if HAVE_ZSTD
+ case COMPRESSION_ZSTD: {
+ ZSTD_inBuffer input = {};
+ size_t res;
+
+ do {
+ r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0);
+ if (r < 0)
+ return r;
+
+ ZSTD_outBuffer output = {
+ .dst = ((uint8_t *) *buffer + *buffer_size),
+ .size = *buffer_allocated - *buffer_size,
+ };
+
+ res = sym_ZSTD_compressStream2(c->c_zstd, &output, &input, ZSTD_e_end);
+ if (sym_ZSTD_isError(res))
+ return -EIO;
+
+ *buffer_size += output.pos;
+ } while (res != 0);
+
+ break;
+ }
+#endif
+
+#if HAVE_ZLIB
+ case COMPRESSION_GZIP:
+ c->gzip.avail_in = 0;
+
+ do {
+ r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0);
+ if (r < 0)
+ return r;
+
+ size_t avail = MIN(*buffer_allocated - *buffer_size, (size_t) UINT_MAX);
+ c->gzip.next_out = (uint8_t*) *buffer + *buffer_size;
+ c->gzip.avail_out = avail;
+
+ r = sym_deflate(&c->gzip, Z_FINISH);
+ if (!IN_SET(r, Z_OK, Z_STREAM_END))
+ return -EIO;
+
+ *buffer_size += avail - c->gzip.avail_out;
+ } while (r != Z_STREAM_END);
+
+ break;
+#endif
+
+#if HAVE_BZIP2
+ case COMPRESSION_BZIP2:
+ c->bzip2.avail_in = 0;
+
+ do {
+ r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0);
+ if (r < 0)
+ return r;
+
+ size_t avail = MIN(*buffer_allocated - *buffer_size, (size_t) UINT_MAX);
+ c->bzip2.next_out = (void*) ((uint8_t*) *buffer + *buffer_size);
+ c->bzip2.avail_out = avail;
+
+ r = sym_BZ2_bzCompress(&c->bzip2, BZ_FINISH);
+ if (!IN_SET(r, BZ_FINISH_OK, BZ_STREAM_END))
+ return -EIO;
+
+ *buffer_size += avail - c->bzip2.avail_out;
+ } while (r != BZ_STREAM_END);
+
+ break;
+#endif
+
+ case COMPRESSION_NONE:
+ break;
+
+ default:
+ return -EOPNOTSUPP;
+ }
+
+ return 0;
}
COMPRESSION_XZ,
COMPRESSION_LZ4,
COMPRESSION_ZSTD,
+ COMPRESSION_GZIP,
+ COMPRESSION_BZIP2,
_COMPRESSION_MAX,
_COMPRESSION_INVALID = -EINVAL,
} Compression;
DECLARE_STRING_TABLE_LOOKUP(compression, Compression);
-DECLARE_STRING_TABLE_LOOKUP(compression_lowercase, Compression);
+DECLARE_STRING_TABLE_LOOKUP(compression_uppercase, Compression);
+DECLARE_STRING_TABLE_LOOKUP(compression_extension, Compression);
+
+/* Try the lowercase string table first, fall back to the uppercase one. Useful for parsing user input
+ * where both forms (e.g. "xz" and "XZ") have historically been accepted. */
+Compression compression_from_string_harder(const char *s);
+
+/* Derives the compression type from a filename's extension, defaulting to COMPRESSION_NONE if the
+ * filename does not carry a recognized compression suffix. */
+Compression compression_from_filename(const char *filename);
bool compression_supported(Compression c);
-int compress_blob_xz(const void *src, uint64_t src_size,
- void *dst, size_t dst_alloc_size, size_t *dst_size, int level);
-int compress_blob_lz4(const void *src, uint64_t src_size,
- void *dst, size_t dst_alloc_size, size_t *dst_size, int level);
-int compress_blob_zstd(const void *src, uint64_t src_size,
- void *dst, size_t dst_alloc_size, size_t *dst_size, int level);
-
-int decompress_blob_xz(const void *src, uint64_t src_size,
- void **dst, size_t* dst_size, size_t dst_max);
-int decompress_blob_lz4(const void *src, uint64_t src_size,
- void **dst, size_t* dst_size, size_t dst_max);
-int decompress_blob_zstd(const void *src, uint64_t src_size,
- void **dst, size_t* dst_size, size_t dst_max);
+/* Buffer size used by streaming compression APIs and pipeline stages that feed into them. Sized to
+ * match the typical Linux pipe buffer so that pipeline stages don't lose throughput due to small
+ * intermediate buffers. */
+#define COMPRESS_PIPE_BUFFER_SIZE (128U*1024U)
+
+/* Compressor / Decompressor — opaque push-based streaming compression context */
+
+typedef struct Compressor Compressor;
+typedef Compressor Decompressor;
+
+typedef int (*DecompressorCallback)(const void *data, size_t size, void *userdata);
+
+Compressor* compressor_free(Compressor *c);
+DEFINE_TRIVIAL_CLEANUP_FUNC(Compressor*, compressor_free);
+
+int compressor_new(Compressor **ret, Compression type);
+int compressor_start(Compressor *c, const void *data, size_t size, void **buffer, size_t *buffer_size, size_t *buffer_allocated);
+int compressor_finish(Compressor *c, void **buffer, size_t *buffer_size, size_t *buffer_allocated);
+
+int decompressor_detect(Decompressor **ret, const void *data, size_t size);
+int decompressor_force_off(Decompressor **ret);
+int decompressor_push(Decompressor *c, const void *data, size_t size, DecompressorCallback callback, void *userdata);
+
+Compression compressor_type(const Compressor *c);
+
+/* Blob compression/decompression */
+
+int compress_blob(Compression compression,
+ const void *src, uint64_t src_size,
+ void *dst, size_t dst_alloc_size, size_t *dst_size, int level);
int decompress_blob(Compression compression,
const void *src, uint64_t src_size,
- void **dst, size_t* dst_size, size_t dst_max);
-
-int decompress_startswith_xz(const void *src, uint64_t src_size,
- void **buffer,
- const void *prefix, size_t prefix_len,
- uint8_t extra);
-int decompress_startswith_lz4(const void *src, uint64_t src_size,
- void **buffer,
- const void *prefix, size_t prefix_len,
- uint8_t extra);
-int decompress_startswith_zstd(const void *src, uint64_t src_size,
- void **buffer,
- const void *prefix, size_t prefix_len,
- uint8_t extra);
+ void **dst, size_t *dst_size, size_t dst_max);
+
+int decompress_zlib_raw(const void *src, uint64_t src_size,
+ void *dst, size_t dst_size, int wbits);
+
int decompress_startswith(Compression compression,
const void *src, uint64_t src_size,
void **buffer,
const void *prefix, size_t prefix_len,
uint8_t extra);
-int compress_stream_xz(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_uncompressed_size);
-int compress_stream_lz4(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_uncompressed_size);
-int compress_stream_zstd(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_uncompressed_size);
+/* Stream compression/decompression (fd-to-fd) */
-int decompress_stream_xz(int fdf, int fdt, uint64_t max_bytes);
-int decompress_stream_lz4(int fdf, int fdt, uint64_t max_bytes);
-int decompress_stream_zstd(int fdf, int fdt, uint64_t max_bytes);
+int compress_stream(Compression type, int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_uncompressed_size);
+int decompress_stream(Compression type, int fdf, int fdt, uint64_t max_bytes);
+int decompress_stream_by_filename(const char *filename, int fdf, int fdt, uint64_t max_bytes);
+int dlopen_xz(void);
int dlopen_lz4(void);
int dlopen_zstd(void);
-int dlopen_lzma(void);
-
-static inline int compress_blob(
- Compression compression,
- const void *src, uint64_t src_size,
- void *dst, size_t dst_alloc_size, size_t *dst_size, int level) {
-
- switch (compression) {
- case COMPRESSION_ZSTD:
- return compress_blob_zstd(src, src_size, dst, dst_alloc_size, dst_size, level);
- case COMPRESSION_LZ4:
- return compress_blob_lz4(src, src_size, dst, dst_alloc_size, dst_size, level);
- case COMPRESSION_XZ:
- return compress_blob_xz(src, src_size, dst, dst_alloc_size, dst_size, level);
- default:
- return -EOPNOTSUPP;
- }
-}
-
-static inline int compress_stream(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_uncompressed_size) {
- switch (DEFAULT_COMPRESSION) {
- case COMPRESSION_ZSTD:
- return compress_stream_zstd(fdf, fdt, max_bytes, ret_uncompressed_size);
- case COMPRESSION_LZ4:
- return compress_stream_lz4(fdf, fdt, max_bytes, ret_uncompressed_size);
- case COMPRESSION_XZ:
- return compress_stream_xz(fdf, fdt, max_bytes, ret_uncompressed_size);
- default:
- return -EOPNOTSUPP;
- }
-}
+int dlopen_zlib(void);
+int dlopen_bzip2(void);
static inline const char* default_compression_extension(void) {
- switch (DEFAULT_COMPRESSION) {
- case COMPRESSION_ZSTD:
- return ".zst";
- case COMPRESSION_LZ4:
- return ".lz4";
- case COMPRESSION_XZ:
- return ".xz";
- default:
- return "";
- }
+ return compression_extension_to_string(DEFAULT_COMPRESSION) ?: "";
}
-
-int decompress_stream(const char *filename, int fdf, int fdt, uint64_t max_bytes);
fundamental_sources,
include_directories : basic_includes,
implicit_include_directories : false,
- dependencies : [libdl,
+ dependencies : [libbzip2_cflags,
+ libdl,
libgcrypt_cflags,
liblz4_cflags,
libm,
librt,
libxz_cflags,
+ libz_cflags,
libzstd_cflags,
threads,
userspace],
assert_se(get_testdata_dir(path, &fn) >= 0);
assert_se(read_full_file_full(AT_FDCWD, fn, UINT64_MAX, SIZE_MAX, 0, NULL, &compressed, &len) >= 0);
- assert_se(decompress_blob_zstd(compressed, len, ret_bcd, ret_bcd_len, SIZE_MAX) >= 0);
+ assert_se(decompress_blob(COMPRESSION_ZSTD, compressed, len, ret_bcd, ret_bcd_len, SIZE_MAX) >= 0);
}
static void test_get_bcd_title_one(
if (fd_compressed < 0)
return log_error_errno(fd_compressed, "Failed to create temporary file for coredump %s: %m", fn_compressed);
- r = compress_stream(fd, fd_compressed, max_size, &uncompressed_size);
+ r = compress_stream(DEFAULT_COMPRESSION, fd, fd_compressed, max_size, &uncompressed_size);
if (r < 0)
return log_error_errno(r, "Failed to compress %s: %m", coredump_tmpfile_name(tmp_compressed));
tmp = unlink_and_free(tmp);
fd = safe_close(fd);
- r = compress_stream(context->input_fd, fd_compressed, max_size, &partial_uncompressed_size);
+ r = compress_stream(DEFAULT_COMPRESSION, context->input_fd, fd_compressed, max_size, &partial_uncompressed_size);
if (r < 0)
return log_error_errno(r, "Failed to compress %s: %m", coredump_tmpfile_name(tmp_compressed));
uncompressed_size += partial_uncompressed_size;
goto error;
}
- r = decompress_stream(filename, fdf, fd, -1);
+ r = decompress_stream_by_filename(filename, fdf, fd, -1);
if (r < 0) {
log_error_errno(r, "Failed to decompress %s: %m", filename);
goto error;
MAX(_d, a); \
})
+#define MAX5(x, y, z, a, b) \
+ ({ \
+ const typeof(x) _e = MAX4(x, y, z, a); \
+ MAX(_e, b); \
+ })
+
#undef MIN
#define MIN(a, b) __MIN(UNIQ, (a), UNIQ, (b))
#define __MIN(aq, a, bq, b) \
#include "fd-util.h"
#include "format-util.h"
#include "fs-util.h"
-#include "import-common.h"
#include "log.h"
#include "pretty-print.h"
#include "ratelimit.h"
int input_fd;
int output_fd;
- ImportCompress compress;
+ Compressor *compress;
sd_event_source *output_event_source;
sd_event_source_unref(e->output_event_source);
- import_compress_free(&e->compress);
+ e->compress = compressor_free(e->compress);
sd_event_unref(e->event);
assert(e);
- if (!e->tried_reflink && e->compress.type == IMPORT_COMPRESS_UNCOMPRESSED) {
+ if (!e->tried_reflink && compressor_type(e->compress) == COMPRESSION_NONE) {
/* If we shall take an uncompressed snapshot we can
* reflink source to destination directly. Let's see
e->tried_reflink = true;
}
- if (!e->tried_sendfile && e->compress.type == IMPORT_COMPRESS_UNCOMPRESSED) {
+ if (!e->tried_sendfile && compressor_type(e->compress) == COMPRESSION_NONE) {
- l = sendfile(e->output_fd, e->input_fd, NULL, IMPORT_BUFFER_SIZE);
+ l = sendfile(e->output_fd, e->input_fd, NULL, COMPRESS_PIPE_BUFFER_SIZE);
if (l < 0) {
if (errno == EAGAIN)
return 0;
}
while (e->buffer_size <= 0) {
- uint8_t input[IMPORT_BUFFER_SIZE];
+ uint8_t input[COMPRESS_PIPE_BUFFER_SIZE];
if (e->eof) {
r = 0;
if (l == 0) {
e->eof = true;
- r = import_compress_finish(&e->compress, &e->buffer, &e->buffer_size, &e->buffer_allocated);
+ r = compressor_finish(e->compress, &e->buffer, &e->buffer_size, &e->buffer_allocated);
} else {
e->written_uncompressed += l;
- r = import_compress(&e->compress, input, l, &e->buffer, &e->buffer_size, &e->buffer_allocated);
+ r = compressor_start(e->compress, input, l, &e->buffer, &e->buffer_size, &e->buffer_allocated);
}
if (r < 0) {
r = log_error_errno(r, "Failed to encode: %m");
return new_fd;
}
-int raw_export_start(RawExport *e, const char *path, int fd, ImportCompressType compress) {
+int raw_export_start(RawExport *e, const char *path, int fd, Compression compress) {
_cleanup_close_ int sfd = -EBADF, tfd = -EBADF;
int r;
assert(e);
assert(path);
assert(fd >= 0);
- assert(compress < _IMPORT_COMPRESS_TYPE_MAX);
- assert(compress != IMPORT_COMPRESS_UNKNOWN);
+ assert(compress >= 0);
+ assert(compress < _COMPRESSION_MAX);
if (e->output_fd >= 0)
return -EBUSY;
else
e->input_fd = TAKE_FD(sfd);
- r = import_compress_init(&e->compress, compress);
+ r = compressor_new(&e->compress, compress);
if (r < 0)
return r;
/* SPDX-License-Identifier: LGPL-2.1-or-later */
#pragma once
+#include "compress.h"
#include "shared-forward.h"
-#include "import-compress.h"
typedef struct RawExport RawExport;
DEFINE_TRIVIAL_CLEANUP_FUNC(RawExport*, raw_export_unref);
-int raw_export_start(RawExport *e, const char *path, int fd, ImportCompressType compress);
+int raw_export_start(RawExport *e, const char *path, int fd, Compression compress);
/* SPDX-License-Identifier: LGPL-2.1-or-later */
#include <sys/stat.h>
+#include <unistd.h>
#include "sd-daemon.h"
#include "sd-event.h"
int tree_fd; /* directory fd of the tree to set up */
int userns_fd;
- ImportCompress compress;
+ Compressor *compress;
sd_event_source *output_event_source;
free(e->temp_path);
}
- import_compress_free(&e->compress);
+ e->compress = compressor_free(e->compress);
sd_event_unref(e->event);
assert(e);
- if (!e->tried_splice && e->compress.type == IMPORT_COMPRESS_UNCOMPRESSED) {
+ if (!e->tried_splice && compressor_type(e->compress) == COMPRESSION_NONE) {
- l = splice(e->tar_fd, NULL, e->output_fd, NULL, IMPORT_BUFFER_SIZE, 0);
+ l = splice(e->tar_fd, NULL, e->output_fd, NULL, COMPRESS_PIPE_BUFFER_SIZE, 0);
if (l < 0) {
if (errno == EAGAIN)
return 0;
}
while (e->buffer_size <= 0) {
- uint8_t input[IMPORT_BUFFER_SIZE];
+ uint8_t input[COMPRESS_PIPE_BUFFER_SIZE];
if (e->eof) {
r = tar_export_finish(e);
if (l == 0) {
e->eof = true;
- r = import_compress_finish(&e->compress, &e->buffer, &e->buffer_size, &e->buffer_allocated);
+ r = compressor_finish(e->compress, &e->buffer, &e->buffer_size, &e->buffer_allocated);
} else {
e->written_uncompressed += l;
- r = import_compress(&e->compress, input, l, &e->buffer, &e->buffer_size, &e->buffer_allocated);
+ r = compressor_start(e->compress, input, l, &e->buffer, &e->buffer_size, &e->buffer_allocated);
}
if (r < 0) {
r = log_error_errno(r, "Failed to encode: %m");
TarExport *e,
const char *path,
int fd,
- ImportCompressType compress,
+ Compression compress,
ImportFlags flags) {
_cleanup_close_ int sfd = -EBADF;
assert(e);
assert(path);
assert(fd >= 0);
- assert(compress < _IMPORT_COMPRESS_TYPE_MAX);
- assert(compress != IMPORT_COMPRESS_UNKNOWN);
+ assert(compress >= 0);
+ assert(compress < _COMPRESSION_MAX);
if (e->output_fd >= 0)
return -EBUSY;
}
}
- r = import_compress_init(&e->compress, compress);
+ r = compressor_new(&e->compress, compress);
if (r < 0)
return r;
/* SPDX-License-Identifier: LGPL-2.1-or-later */
#pragma once
+#include "compress.h"
#include "import-common.h"
-#include "import-compress.h"
#include "shared-forward.h"
typedef struct TarExport TarExport;
DEFINE_TRIVIAL_CLEANUP_FUNC(TarExport*, tar_export_unref);
-int tar_export_start(TarExport *e, const char *path, int fd, ImportCompressType compress, ImportFlags flags);
+int tar_export_start(TarExport *e, const char *path, int fd, Compression compress, ImportFlags flags);
#include <getopt.h>
#include <locale.h>
+#include <unistd.h>
#include "sd-event.h"
#include "verbs.h"
static ImportFlags arg_import_flags = 0;
-static ImportCompressType arg_compress = IMPORT_COMPRESS_UNKNOWN;
+static Compression arg_compress = _COMPRESSION_INVALID;
static ImageClass arg_class = IMAGE_MACHINE;
static RuntimeScope arg_runtime_scope = _RUNTIME_SCOPE_INVALID;
static void determine_compression_from_filename(const char *p) {
-
- if (arg_compress != IMPORT_COMPRESS_UNKNOWN)
- return;
-
- if (!p) {
- arg_compress = IMPORT_COMPRESS_UNCOMPRESSED;
+ if (arg_compress >= 0)
return;
- }
- if (endswith(p, ".xz"))
- arg_compress = IMPORT_COMPRESS_XZ;
- else if (endswith(p, ".gz"))
- arg_compress = IMPORT_COMPRESS_GZIP;
- else if (endswith(p, ".bz2"))
- arg_compress = IMPORT_COMPRESS_BZIP2;
- else if (endswith(p, ".zst"))
- arg_compress = IMPORT_COMPRESS_ZSTD;
- else
- arg_compress = IMPORT_COMPRESS_UNCOMPRESSED;
+ arg_compress = p ? compression_from_filename(p) : COMPRESSION_NONE;
}
static void on_tar_finished(TarExport *export, int error, void *userdata) {
fd = open_fd;
- log_info("Exporting '%s', saving to '%s' with compression '%s'.", local, path, import_compress_type_to_string(arg_compress));
+ log_info("Exporting '%s', saving to '%s' with compression '%s'.", local, path, compression_to_string(arg_compress));
} else {
_cleanup_free_ char *pretty = NULL;
fd = STDOUT_FILENO;
(void) fd_get_path(fd, &pretty);
- log_info("Exporting '%s', saving to '%s' with compression '%s'.", local, strna(pretty), import_compress_type_to_string(arg_compress));
+ log_info("Exporting '%s', saving to '%s' with compression '%s'.", local, strna(pretty), compression_to_string(arg_compress));
}
r = import_allocate_event_with_signals(&event);
fd = open_fd;
- log_info("Exporting '%s', saving to '%s' with compression '%s'.", local, path, import_compress_type_to_string(arg_compress));
+ log_info("Exporting '%s', saving to '%s' with compression '%s'.", local, path, compression_to_string(arg_compress));
} else {
_cleanup_free_ char *pretty = NULL;
fd = STDOUT_FILENO;
(void) fd_get_path(fd, &pretty);
- log_info("Exporting '%s', saving to '%s' with compression '%s'.", local, strna(pretty), import_compress_type_to_string(arg_compress));
+ log_info("Exporting '%s', saving to '%s' with compression '%s'.", local, strna(pretty), compression_to_string(arg_compress));
}
r = import_allocate_event_with_signals(&event);
return version();
case ARG_FORMAT:
- arg_compress = import_compress_type_from_string(optarg);
- if (arg_compress < 0 || arg_compress == IMPORT_COMPRESS_UNKNOWN)
+ arg_compress = compression_from_string_harder(optarg);
+ if (arg_compress < 0)
return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
"Unknown format: %s", optarg);
break;
#include "sd-event.h"
#include "capability-util.h"
+#include "compress.h"
#include "dirent-util.h"
#include "dissect-image.h"
#include "fd-util.h"
if (pipe2(pipefd, O_CLOEXEC) < 0)
return log_error_errno(errno, "Failed to create pipe for tar: %m");
- (void) fcntl(pipefd[0], F_SETPIPE_SZ, IMPORT_BUFFER_SIZE);
+ (void) fcntl(pipefd[0], F_SETPIPE_SZ, COMPRESS_PIPE_BUFFER_SIZE);
r = pidref_safe_fork_full(
"tar-x",
if (pipe2(pipefd, O_CLOEXEC) < 0)
return log_error_errno(errno, "Failed to create pipe for tar: %m");
- (void) fcntl(pipefd[0], F_SETPIPE_SZ, IMPORT_BUFFER_SIZE);
+ (void) fcntl(pipefd[0], F_SETPIPE_SZ, COMPRESS_PIPE_BUFFER_SIZE);
r = pidref_safe_fork_full(
"tar-c",
int import_make_foreign_userns(int *userns_fd);
int import_remove_tree(const char *path, int *userns_fd, ImportFlags flags);
-
-#define IMPORT_BUFFER_SIZE (128U*1024U)
+++ /dev/null
-/* SPDX-License-Identifier: LGPL-2.1-or-later */
-
-#include <stdlib.h>
-#include <string.h>
-
-#include "import-common.h"
-#include "import-compress.h"
-#include "log.h"
-#include "string-table.h"
-
-void import_compress_free(ImportCompress *c) {
- assert(c);
-
- if (c->type == IMPORT_COMPRESS_XZ)
- lzma_end(&c->xz);
- else if (c->type == IMPORT_COMPRESS_GZIP) {
- if (c->encoding)
- deflateEnd(&c->gzip);
- else
- inflateEnd(&c->gzip);
-#if HAVE_BZIP2
- } else if (c->type == IMPORT_COMPRESS_BZIP2) {
- if (c->encoding)
- BZ2_bzCompressEnd(&c->bzip2);
- else
- BZ2_bzDecompressEnd(&c->bzip2);
-#endif
-#if HAVE_ZSTD
- } else if (c->type == IMPORT_COMPRESS_ZSTD) {
- if (c->encoding) {
- ZSTD_freeCCtx(c->c_zstd);
- c->c_zstd = NULL;
- } else {
- ZSTD_freeDCtx(c->d_zstd);
- c->d_zstd = NULL;
- }
-#endif
- }
-
- c->type = IMPORT_COMPRESS_UNKNOWN;
-}
-
-int import_uncompress_detect(ImportCompress *c, const void *data, size_t size) {
- static const uint8_t xz_signature[] = {
- 0xfd, '7', 'z', 'X', 'Z', 0x00
- };
- static const uint8_t gzip_signature[] = {
- 0x1f, 0x8b
- };
- static const uint8_t bzip2_signature[] = {
- 'B', 'Z', 'h'
- };
- static const uint8_t zstd_signature[] = {
- 0x28, 0xb5, 0x2f, 0xfd
- };
-
- int r;
-
- assert(c);
-
- if (c->type != IMPORT_COMPRESS_UNKNOWN)
- return 1;
-
- if (size < MAX4(sizeof(xz_signature),
- sizeof(gzip_signature),
- sizeof(zstd_signature),
- sizeof(bzip2_signature)))
- return 0;
-
- assert(data);
-
- if (memcmp(data, xz_signature, sizeof(xz_signature)) == 0) {
- lzma_ret xzr;
-
- xzr = lzma_stream_decoder(&c->xz, UINT64_MAX, LZMA_TELL_UNSUPPORTED_CHECK | LZMA_CONCATENATED);
- if (xzr != LZMA_OK)
- return -EIO;
-
- c->type = IMPORT_COMPRESS_XZ;
-
- } else if (memcmp(data, gzip_signature, sizeof(gzip_signature)) == 0) {
- r = inflateInit2(&c->gzip, 15+16);
- if (r != Z_OK)
- return -EIO;
-
- c->type = IMPORT_COMPRESS_GZIP;
-
-#if HAVE_BZIP2
- } else if (memcmp(data, bzip2_signature, sizeof(bzip2_signature)) == 0) {
- r = BZ2_bzDecompressInit(&c->bzip2, 0, 0);
- if (r != BZ_OK)
- return -EIO;
-
- c->type = IMPORT_COMPRESS_BZIP2;
-#endif
-#if HAVE_ZSTD
- } else if (memcmp(data, zstd_signature, sizeof(zstd_signature)) == 0) {
- c->d_zstd = ZSTD_createDCtx();
- if (!c->d_zstd)
- return -ENOMEM;
-
- c->type = IMPORT_COMPRESS_ZSTD;
-#endif
- } else
- c->type = IMPORT_COMPRESS_UNCOMPRESSED;
-
- c->encoding = false;
-
- log_debug("Detected compression type: %s", import_compress_type_to_string(c->type));
- return 1;
-}
-
-void import_uncompress_force_off(ImportCompress *c) {
- assert(c);
-
- c->type = IMPORT_COMPRESS_UNCOMPRESSED;
- c->encoding = false;
-}
-
-int import_uncompress(ImportCompress *c, const void *data, size_t size, ImportCompressCallback callback, void *userdata) {
- int r;
-
- assert(c);
- assert(callback);
-
- r = import_uncompress_detect(c, data, size);
- if (r <= 0)
- return r;
-
- if (c->encoding)
- return -EINVAL;
-
- if (size <= 0)
- return 1;
-
- assert(data);
-
- switch (c->type) {
-
- case IMPORT_COMPRESS_UNCOMPRESSED:
- r = callback(data, size, userdata);
- if (r < 0)
- return r;
-
- break;
-
- case IMPORT_COMPRESS_XZ:
- c->xz.next_in = data;
- c->xz.avail_in = size;
-
- while (c->xz.avail_in > 0) {
- uint8_t buffer[IMPORT_BUFFER_SIZE];
- lzma_ret lzr;
-
- c->xz.next_out = buffer;
- c->xz.avail_out = sizeof(buffer);
-
- lzr = lzma_code(&c->xz, LZMA_RUN);
- if (!IN_SET(lzr, LZMA_OK, LZMA_STREAM_END))
- return -EIO;
-
- if (c->xz.avail_out < sizeof(buffer)) {
- r = callback(buffer, sizeof(buffer) - c->xz.avail_out, userdata);
- if (r < 0)
- return r;
- }
- }
-
- break;
-
- case IMPORT_COMPRESS_GZIP:
- c->gzip.next_in = (void*) data;
- c->gzip.avail_in = size;
-
- while (c->gzip.avail_in > 0) {
- uint8_t buffer[IMPORT_BUFFER_SIZE];
-
- c->gzip.next_out = buffer;
- c->gzip.avail_out = sizeof(buffer);
-
- r = inflate(&c->gzip, Z_NO_FLUSH);
- if (!IN_SET(r, Z_OK, Z_STREAM_END))
- return -EIO;
-
- if (c->gzip.avail_out < sizeof(buffer)) {
- r = callback(buffer, sizeof(buffer) - c->gzip.avail_out, userdata);
- if (r < 0)
- return r;
- }
- }
-
- break;
-
-#if HAVE_BZIP2
- case IMPORT_COMPRESS_BZIP2:
- c->bzip2.next_in = (void*) data;
- c->bzip2.avail_in = size;
-
- while (c->bzip2.avail_in > 0) {
- uint8_t buffer[IMPORT_BUFFER_SIZE];
-
- c->bzip2.next_out = (char*) buffer;
- c->bzip2.avail_out = sizeof(buffer);
-
- r = BZ2_bzDecompress(&c->bzip2);
- if (!IN_SET(r, BZ_OK, BZ_STREAM_END))
- return -EIO;
-
- if (c->bzip2.avail_out < sizeof(buffer)) {
- r = callback(buffer, sizeof(buffer) - c->bzip2.avail_out, userdata);
- if (r < 0)
- return r;
- }
- }
-
- break;
-#endif
-#if HAVE_ZSTD
- case IMPORT_COMPRESS_ZSTD: {
- ZSTD_inBuffer input = {
- .src = (void*) data,
- .size = size,
- };
-
- while (input.pos < input.size) {
- uint8_t buffer[IMPORT_BUFFER_SIZE];
- ZSTD_outBuffer output = {
- .dst = buffer,
- .size = sizeof(buffer),
- };
- size_t res;
-
- res = ZSTD_decompressStream(c->d_zstd, &output, &input);
- if (ZSTD_isError(res))
- return -EIO;
-
- if (output.pos > 0) {
- r = callback(output.dst, output.pos, userdata);
- if (r < 0)
- return r;
- }
- }
-
- break;
- }
-#endif
-
- default:
- assert_not_reached();
- }
-
- return 1;
-}
-
-int import_compress_init(ImportCompress *c, ImportCompressType t) {
- int r;
-
- assert(c);
-
- switch (t) {
-
- case IMPORT_COMPRESS_XZ: {
- lzma_ret xzr;
-
- xzr = lzma_easy_encoder(&c->xz, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64);
- if (xzr != LZMA_OK)
- return -EIO;
-
- c->type = IMPORT_COMPRESS_XZ;
- break;
- }
-
- case IMPORT_COMPRESS_GZIP:
- r = deflateInit2(&c->gzip, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15 + 16, 8, Z_DEFAULT_STRATEGY);
- if (r != Z_OK)
- return -EIO;
-
- c->type = IMPORT_COMPRESS_GZIP;
- break;
-
-#if HAVE_BZIP2
- case IMPORT_COMPRESS_BZIP2:
- r = BZ2_bzCompressInit(&c->bzip2, 9, 0, 0);
- if (r != BZ_OK)
- return -EIO;
-
- c->type = IMPORT_COMPRESS_BZIP2;
- break;
-#endif
-
-#if HAVE_ZSTD
- case IMPORT_COMPRESS_ZSTD:
- c->c_zstd = ZSTD_createCCtx();
- if (!c->c_zstd)
- return -ENOMEM;
-
- r = ZSTD_CCtx_setParameter(c->c_zstd, ZSTD_c_compressionLevel, ZSTD_CLEVEL_DEFAULT);
- if (ZSTD_isError(r))
- return -EIO;
-
- c->type = IMPORT_COMPRESS_ZSTD;
- break;
-#endif
-
- case IMPORT_COMPRESS_UNCOMPRESSED:
- c->type = IMPORT_COMPRESS_UNCOMPRESSED;
- break;
-
- default:
- return -EOPNOTSUPP;
- }
-
- c->encoding = true;
- return 0;
-}
-
-static int enlarge_buffer(void **buffer, size_t *buffer_size, size_t *buffer_allocated) {
- size_t l;
- void *p;
-
- assert(buffer);
- assert(buffer_size);
- assert(buffer_allocated);
-
- if (*buffer_allocated > *buffer_size)
- return 0;
-
- l = MAX(IMPORT_BUFFER_SIZE, (*buffer_size * 2));
- p = realloc(*buffer, l);
- if (!p)
- return -ENOMEM;
-
- *buffer = p;
- *buffer_allocated = l;
-
- return 1;
-}
-
-int import_compress(ImportCompress *c, const void *data, size_t size, void **buffer, size_t *buffer_size, size_t *buffer_allocated) {
- int r;
-
- assert(c);
- assert(buffer);
- assert(buffer_size);
- assert(buffer_allocated);
-
- if (!c->encoding)
- return -EINVAL;
-
- if (size <= 0)
- return 0;
-
- assert(data);
-
- *buffer_size = 0;
-
- switch (c->type) {
-
- case IMPORT_COMPRESS_XZ:
-
- c->xz.next_in = data;
- c->xz.avail_in = size;
-
- while (c->xz.avail_in > 0) {
- lzma_ret lzr;
-
- r = enlarge_buffer(buffer, buffer_size, buffer_allocated);
- if (r < 0)
- return r;
-
- c->xz.next_out = (uint8_t*) *buffer + *buffer_size;
- c->xz.avail_out = *buffer_allocated - *buffer_size;
-
- lzr = lzma_code(&c->xz, LZMA_RUN);
- if (lzr != LZMA_OK)
- return -EIO;
-
- *buffer_size += (*buffer_allocated - *buffer_size) - c->xz.avail_out;
- }
-
- break;
-
- case IMPORT_COMPRESS_GZIP:
-
- c->gzip.next_in = (void*) data;
- c->gzip.avail_in = size;
-
- while (c->gzip.avail_in > 0) {
- r = enlarge_buffer(buffer, buffer_size, buffer_allocated);
- if (r < 0)
- return r;
-
- c->gzip.next_out = (uint8_t*) *buffer + *buffer_size;
- c->gzip.avail_out = *buffer_allocated - *buffer_size;
-
- r = deflate(&c->gzip, Z_NO_FLUSH);
- if (r != Z_OK)
- return -EIO;
-
- *buffer_size += (*buffer_allocated - *buffer_size) - c->gzip.avail_out;
- }
-
- break;
-
-#if HAVE_BZIP2
- case IMPORT_COMPRESS_BZIP2:
-
- c->bzip2.next_in = (void*) data;
- c->bzip2.avail_in = size;
-
- while (c->bzip2.avail_in > 0) {
- r = enlarge_buffer(buffer, buffer_size, buffer_allocated);
- if (r < 0)
- return r;
-
- c->bzip2.next_out = (void*) ((uint8_t*) *buffer + *buffer_size);
- c->bzip2.avail_out = *buffer_allocated - *buffer_size;
-
- r = BZ2_bzCompress(&c->bzip2, BZ_RUN);
- if (r != BZ_RUN_OK)
- return -EIO;
-
- *buffer_size += (*buffer_allocated - *buffer_size) - c->bzip2.avail_out;
- }
-
- break;
-#endif
-
-#if HAVE_ZSTD
- case IMPORT_COMPRESS_ZSTD: {
- ZSTD_inBuffer input = {
- .src = data,
- .size = size,
- };
-
- while (input.pos < input.size) {
- r = enlarge_buffer(buffer, buffer_size, buffer_allocated);
- if (r < 0)
- return r;
-
- ZSTD_outBuffer output = {
- .dst = ((uint8_t *) *buffer + *buffer_size),
- .size = *buffer_allocated - *buffer_size,
- };
- size_t res;
-
- res = ZSTD_compressStream2(c->c_zstd, &output, &input, ZSTD_e_continue);
- if (ZSTD_isError(res))
- return -EIO;
-
- *buffer_size += output.pos;
- }
-
- break;
- }
-#endif
-
- case IMPORT_COMPRESS_UNCOMPRESSED:
-
- if (*buffer_allocated < size) {
- void *p;
-
- p = realloc(*buffer, size);
- if (!p)
- return -ENOMEM;
-
- *buffer = p;
- *buffer_allocated = size;
- }
-
- memcpy(*buffer, data, size);
- *buffer_size = size;
- break;
-
- default:
- return -EOPNOTSUPP;
- }
-
- return 0;
-}
-
-int import_compress_finish(ImportCompress *c, void **buffer, size_t *buffer_size, size_t *buffer_allocated) {
- int r;
-
- assert(c);
- assert(buffer);
- assert(buffer_size);
- assert(buffer_allocated);
-
- if (!c->encoding)
- return -EINVAL;
-
- *buffer_size = 0;
-
- switch (c->type) {
-
- case IMPORT_COMPRESS_XZ: {
- lzma_ret lzr;
-
- c->xz.avail_in = 0;
-
- do {
- r = enlarge_buffer(buffer, buffer_size, buffer_allocated);
- if (r < 0)
- return r;
-
- c->xz.next_out = (uint8_t*) *buffer + *buffer_size;
- c->xz.avail_out = *buffer_allocated - *buffer_size;
-
- lzr = lzma_code(&c->xz, LZMA_FINISH);
- if (!IN_SET(lzr, LZMA_OK, LZMA_STREAM_END))
- return -EIO;
-
- *buffer_size += (*buffer_allocated - *buffer_size) - c->xz.avail_out;
- } while (lzr != LZMA_STREAM_END);
-
- break;
- }
-
- case IMPORT_COMPRESS_GZIP:
- c->gzip.avail_in = 0;
-
- do {
- r = enlarge_buffer(buffer, buffer_size, buffer_allocated);
- if (r < 0)
- return r;
-
- c->gzip.next_out = (uint8_t*) *buffer + *buffer_size;
- c->gzip.avail_out = *buffer_allocated - *buffer_size;
-
- r = deflate(&c->gzip, Z_FINISH);
- if (!IN_SET(r, Z_OK, Z_STREAM_END))
- return -EIO;
-
- *buffer_size += (*buffer_allocated - *buffer_size) - c->gzip.avail_out;
- } while (r != Z_STREAM_END);
-
- break;
-
-#if HAVE_BZIP2
- case IMPORT_COMPRESS_BZIP2:
- c->bzip2.avail_in = 0;
-
- do {
- r = enlarge_buffer(buffer, buffer_size, buffer_allocated);
- if (r < 0)
- return r;
-
- c->bzip2.next_out = (void*) ((uint8_t*) *buffer + *buffer_size);
- c->bzip2.avail_out = *buffer_allocated - *buffer_size;
-
- r = BZ2_bzCompress(&c->bzip2, BZ_FINISH);
- if (!IN_SET(r, BZ_FINISH_OK, BZ_STREAM_END))
- return -EIO;
-
- *buffer_size += (*buffer_allocated - *buffer_size) - c->bzip2.avail_out;
- } while (r != BZ_STREAM_END);
-
- break;
-#endif
-
-#if HAVE_ZSTD
- case IMPORT_COMPRESS_ZSTD: {
- ZSTD_inBuffer input = {};
- size_t res;
-
- do {
- r = enlarge_buffer(buffer, buffer_size, buffer_allocated);
- if (r < 0)
- return r;
-
- ZSTD_outBuffer output = {
- .dst = ((uint8_t *) *buffer + *buffer_size),
- .size = *buffer_allocated - *buffer_size,
- };
-
- res = ZSTD_compressStream2(c->c_zstd, &output, &input, ZSTD_e_end);
- if (ZSTD_isError(res))
- return -EIO;
-
- *buffer_size += output.pos;
- } while (res != 0);
-
- break;
- }
-#endif
-
- case IMPORT_COMPRESS_UNCOMPRESSED:
- break;
-
- default:
- return -EOPNOTSUPP;
- }
-
- return 0;
-}
-
-static const char* const import_compress_type_table[_IMPORT_COMPRESS_TYPE_MAX] = {
- [IMPORT_COMPRESS_UNKNOWN] = "unknown",
- [IMPORT_COMPRESS_UNCOMPRESSED] = "uncompressed",
- [IMPORT_COMPRESS_XZ] = "xz",
- [IMPORT_COMPRESS_GZIP] = "gzip",
-#if HAVE_BZIP2
- [IMPORT_COMPRESS_BZIP2] = "bzip2",
-#endif
-#if HAVE_ZSTD
- [IMPORT_COMPRESS_ZSTD] = "zstd",
-#endif
-};
-
-DEFINE_STRING_TABLE_LOOKUP(import_compress_type, ImportCompressType);
+++ /dev/null
-/* SPDX-License-Identifier: LGPL-2.1-or-later */
-#pragma once
-
-#if HAVE_BZIP2
-#include <bzlib.h>
-#endif
-#include <lzma.h>
-#include <zlib.h>
-#if HAVE_ZSTD
-#include <zstd.h>
-#endif
-
-#include "shared-forward.h"
-
-typedef enum ImportCompressType {
- IMPORT_COMPRESS_UNKNOWN,
- IMPORT_COMPRESS_UNCOMPRESSED,
- IMPORT_COMPRESS_XZ,
- IMPORT_COMPRESS_GZIP,
- IMPORT_COMPRESS_BZIP2,
- IMPORT_COMPRESS_ZSTD,
- _IMPORT_COMPRESS_TYPE_MAX,
- _IMPORT_COMPRESS_TYPE_INVALID = -EINVAL,
-} ImportCompressType;
-
-typedef struct ImportCompress {
- ImportCompressType type;
- bool encoding;
- union {
- lzma_stream xz;
- z_stream gzip;
-#if HAVE_BZIP2
- bz_stream bzip2;
-#endif
-#if HAVE_ZSTD
- ZSTD_CCtx *c_zstd;
- ZSTD_DCtx *d_zstd;
-#endif
- };
-} ImportCompress;
-
-typedef int (*ImportCompressCallback)(const void *data, size_t size, void *userdata);
-
-void import_compress_free(ImportCompress *c);
-
-int import_uncompress_detect(ImportCompress *c, const void *data, size_t size);
-void import_uncompress_force_off(ImportCompress *c);
-int import_uncompress(ImportCompress *c, const void *data, size_t size, ImportCompressCallback callback, void *userdata);
-
-int import_compress_init(ImportCompress *c, ImportCompressType t);
-int import_compress(ImportCompress *c, const void *data, size_t size, void **buffer, size_t *buffer_size, size_t *buffer_allocated);
-int import_compress_finish(ImportCompress *c, void **buffer, size_t *buffer_size, size_t *buffer_allocated);
-
-DECLARE_STRING_TABLE_LOOKUP(import_compress_type, ImportCompressType);
/* SPDX-License-Identifier: LGPL-2.1-or-later */
#include <sys/stat.h>
+#include <unistd.h>
#include "sd-daemon.h"
#include "sd-event.h"
#include "alloc-util.h"
+#include "compress.h"
#include "copy.h"
#include "fd-util.h"
#include "format-util.h"
#include "fs-util.h"
#include "import-common.h"
-#include "import-compress.h"
#include "import-raw.h"
#include "import-util.h"
#include "install-file.h"
int input_fd;
int output_fd;
- ImportCompress compress;
+ Compressor *compress;
sd_event_source *input_event_source;
- uint8_t buffer[IMPORT_BUFFER_SIZE];
+ uint8_t buffer[COMPRESS_PIPE_BUFFER_SIZE];
size_t buffer_size;
uint64_t written_compressed;
unlink_and_free(i->temp_path);
- import_compress_free(&i->compress);
+ i->compress = compressor_free(i->compress);
sd_event_unref(i->event);
assert(i->input_fd >= 0);
assert(i->output_fd >= 0);
- if (i->compress.type != IMPORT_COMPRESS_UNCOMPRESSED)
+ if (compressor_type(i->compress) != COMPRESSION_NONE)
return 0;
if (i->offset != UINT64_MAX || i->size_max != UINT64_MAX)
i->buffer_size += l;
- if (i->compress.type == IMPORT_COMPRESS_UNKNOWN) {
+ if (!i->compress) {
if (l == 0) { /* EOF */
log_debug("File too short to be compressed, as no compression signature fits in, thus assuming uncompressed.");
- import_uncompress_force_off(&i->compress);
+ decompressor_force_off(&i->compress);
} else {
- r = import_uncompress_detect(&i->compress, i->buffer, i->buffer_size);
+ r = decompressor_detect(&i->compress, i->buffer, i->buffer_size);
if (r < 0) {
log_error_errno(r, "Failed to detect file compression: %m");
goto finish;
goto complete;
}
- r = import_uncompress(&i->compress, i->buffer, i->buffer_size, raw_import_write, i);
+ r = decompressor_push(i->compress, i->buffer, i->buffer_size, raw_import_write, i);
if (r < 0) {
log_error_errno(r, "Failed to decode and write: %m");
goto finish;
/* SPDX-License-Identifier: LGPL-2.1-or-later */
#include <sys/stat.h>
+#include <unistd.h>
#include "sd-daemon.h"
#include "sd-event.h"
#include "alloc-util.h"
#include "btrfs-util.h"
+#include "compress.h"
#include "dissect-image.h"
#include "errno-util.h"
#include "fd-util.h"
#include "format-util.h"
#include "import-common.h"
-#include "import-compress.h"
#include "import-tar.h"
#include "import-util.h"
#include "install-file.h"
int tree_fd;
int userns_fd;
- ImportCompress compress;
+ Compressor *compress;
sd_event_source *input_event_source;
- uint8_t buffer[IMPORT_BUFFER_SIZE];
+ uint8_t buffer[COMPRESS_PIPE_BUFFER_SIZE];
size_t buffer_size;
uint64_t written_compressed;
free(i->temp_path);
}
- import_compress_free(&i->compress);
+ i->compress = compressor_free(i->compress);
sd_event_unref(i->event);
i->buffer_size += l;
- if (i->compress.type == IMPORT_COMPRESS_UNKNOWN) {
+ if (!i->compress) {
if (l == 0) { /* EOF */
log_debug("File too short to be compressed, as no compression signature fits in, thus assuming uncompressed.");
- import_uncompress_force_off(&i->compress);
+ decompressor_force_off(&i->compress);
} else {
- r = import_uncompress_detect(&i->compress, i->buffer, i->buffer_size);
+ r = decompressor_detect(&i->compress, i->buffer, i->buffer_size);
if (r < 0) {
log_error_errno(r, "Failed to detect file compression: %m");
goto finish;
goto finish;
}
- r = import_uncompress(&i->compress, i->buffer, i->buffer_size, tar_import_write, i);
+ r = decompressor_push(i->compress, i->buffer, i->buffer_size, tar_import_write, i);
if (r < 0) {
log_error_errno(r, "Failed to decode and write: %m");
goto finish;
endif
common_deps = [
- libbzip2,
libcurl,
- libxz,
- libz,
- libzstd,
]
executables += [
'extract' : files(
'oci-util.c',
'import-common.c',
- 'import-compress.c',
'qcow2-util.c',
),
'dependencies' : [common_deps, threads],
/* SPDX-License-Identifier: LGPL-2.1-or-later */
+#include <unistd.h>
+
#include "sd-id128.h"
#include "alloc-util.h"
#include <fcntl.h>
#include <sys/stat.h>
#include <sys/xattr.h>
+#include <unistd.h>
#include "alloc-util.h"
#include "curl-util.h"
curl_glue_remove_and_free(j->glue, j->curl);
curl_slist_free_all(j->request_header);
- import_compress_free(&j->compress);
+ j->compress = compressor_free(j->compress);
if (j->checksum_ctx)
EVP_MD_CTX_free(j->checksum_ctx);
curl_glue_remove_and_free(j->glue, j->curl);
j->curl = NULL;
- import_compress_free(&j->compress);
+ j->compress = compressor_free(j->compress);
if (j->checksum_ctx) {
EVP_MD_CTX_free(j->checksum_ctx);
"Could not hash chunk.");
}
- r = import_uncompress(&j->compress, data->iov_base, data->iov_len, pull_job_write_uncompressed, j);
+ r = decompressor_push(j->compress, data->iov_base, data->iov_len, pull_job_write_uncompressed, j);
if (r < 0)
return r;
assert(j);
- r = import_uncompress_detect(&j->compress, j->payload.iov_base, j->payload.iov_len);
+ r = decompressor_detect(&j->compress, j->payload.iov_base, j->payload.iov_len);
if (r < 0)
return log_error_errno(r, "Failed to initialize compressor: %m");
if (r == 0)
return 0;
- log_debug("Stream is compressed: %s", import_compress_type_to_string(j->compress.type));
+ log_debug("Stream is compressed: %s", compression_to_string(compressor_type(j->compress)));
r = pull_job_open_disk(j);
if (r < 0)
#include <curl/curl.h>
#include <sys/stat.h>
-#include "shared-forward.h"
-#include "import-compress.h"
+#include "compress.h"
#include "openssl-util.h"
+#include "shared-forward.h"
typedef struct CurlGlue CurlGlue;
typedef struct PullJob PullJob;
usec_t mtime;
char *content_type;
- ImportCompress compress;
+ Compressor *compress;
unsigned progress_percent;
usec_t start_usec;
/* SPDX-License-Identifier: LGPL-2.1-or-later */
+#include <unistd.h>
+
#include "sd-event.h"
#include "sd-json.h"
#include "sd-varlink.h"
/* SPDX-License-Identifier: LGPL-2.1-or-later */
+#include <unistd.h>
+
#include "sd-daemon.h"
#include "sd-event.h"
/* SPDX-License-Identifier: LGPL-2.1-or-later */
+#include <unistd.h>
+
#include "sd-daemon.h"
#include "sd-event.h"
#include "sd-varlink.h"
/* SPDX-License-Identifier: LGPL-2.1-or-later */
-#include <zlib.h>
+#include <unistd.h>
#include "alloc-util.h"
+#include "compress.h"
#include "copy.h"
#include "qcow2-util.h"
#include "sparse-endian.h"
void *buffer2) {
_cleanup_free_ void *large_buffer = NULL;
- z_stream s = {};
- uint64_t sz;
ssize_t l;
int r;
if ((uint64_t) l != compressed_size)
return -EIO;
- s.next_in = buffer1;
- s.avail_in = compressed_size;
- s.next_out = buffer2;
- s.avail_out = cluster_size;
-
- r = inflateInit2(&s, -12);
- if (r != Z_OK)
- return -EIO;
-
- r = inflate(&s, Z_FINISH);
- sz = (uint8_t*) s.next_out - (uint8_t*) buffer2;
- inflateEnd(&s);
- if (r != Z_STREAM_END || sz != cluster_size)
- return -EIO;
+ r = decompress_zlib_raw(buffer1, compressed_size, buffer2, cluster_size, /* wbits= */ -12);
+ if (r < 0)
+ return r;
l = pwrite(dfd, buffer2, cluster_size, doffset);
if (l < 0)
}
}
- Compression c = compression_lowercase_from_string(word);
+ Compression c = compression_from_string_harder(word);
if (c <= 0 || !compression_supported(c)) {
log_syntax(unit, LOG_WARNING, filename, line, c,
"Compression algorithm '%s' is not supported on the system, ignoring.", word);
const CompressionConfig *cc;
ORDERED_HASHMAP_FOREACH(cc, arg_compression) {
- const char *c = compression_lowercase_to_string(cc->algorithm);
+ const char *c = compression_to_string(cc->algorithm);
if (strextendf_with_separator(&buf, ",", "%s;q=%.1f", c, q) < 0)
return -ENOMEM;
q -= step;
RemoteSource *source = *connection_cls;
header = MHD_lookup_connection_value(connection, MHD_HEADER_KIND, "Content-Encoding");
if (header) {
- Compression c = compression_lowercase_from_string(header);
+ Compression c = compression_from_string_harder(header);
if (c <= 0 || !compression_supported(c))
return mhd_respondf(connection, 0, MHD_HTTP_UNSUPPORTED_MEDIA_TYPE,
"Unsupported Content-Encoding type: %s", header);
r = compress_blob(u->compression->algorithm, compression_buffer, filled, buf, size * nmemb, &compressed_size, u->compression->level);
if (r < 0) {
log_error_errno(r, "Failed to compress %zu bytes by %s with level %i: %m",
- filled, compression_lowercase_to_string(u->compression->algorithm), u->compression->level);
+ filled, compression_to_string(u->compression->algorithm), u->compression->level);
return CURL_READFUNC_ABORT;
}
h = l;
if (u->compression) {
- _cleanup_free_ char *header = strjoin("Content-Encoding: ", compression_lowercase_to_string(u->compression->algorithm));
+ _cleanup_free_ char *header = strjoin("Content-Encoding: ", compression_to_string(u->compression->algorithm));
if (!header)
return log_oom();
r = compress_blob(u->compression->algorithm, compression_buffer, n, buf, size * nmemb, &compressed_size, u->compression->level);
if (r < 0) {
log_error_errno(r, "Failed to compress %zd bytes by %s with level %i: %m",
- n, compression_lowercase_to_string(u->compression->algorithm), u->compression->level);
+ n, compression_to_string(u->compression->algorithm), u->compression->level);
return CURL_READFUNC_ABORT;
}
assert(compressed_size <= size * nmemb);
return 0; /* Already picked the algorithm. Let's shortcut. */
if (cc) {
- _cleanup_free_ char *header = strjoin("Content-Encoding: ", compression_lowercase_to_string(cc->algorithm));
+ _cleanup_free_ char *header = strjoin("Content-Encoding: ", compression_to_string(cc->algorithm));
if (!header)
return log_oom();
u->compression = cc;
if (cc)
- log_debug("Using compression algorithm %s with compression level %i.", compression_lowercase_to_string(cc->algorithm), cc->level);
+ log_debug("Using compression algorithm %s with compression level %i.", compression_to_string(cc->algorithm), cc->level);
else
log_debug("Disabled compression algorithm.");
return 0;
if (streq(word, "*"))
return update_content_encoding_header(u, ordered_hashmap_first(arg_compression));
- Compression c = compression_lowercase_from_string(word);
+ Compression c = compression_from_string_harder(word);
if (c <= 0 || !compression_supported(c))
continue; /* unsupported or invalid algorithm. */
if (r >= 0)
return r ? DEFAULT_COMPRESSION : COMPRESSION_NONE;
- c = compression_from_string(e);
+ c = compression_from_string_harder(e);
if (c < 0) {
log_debug_errno(c, "Failed to parse SYSTEMD_JOURNAL_COMPRESS value, ignoring: %s", e);
return DEFAULT_COMPRESSION;
/* SPDX-License-Identifier: LGPL-2.1-or-later */
#include "alloc-util.h"
+#include "argv-util.h"
#include "compress.h"
-#include "nulstr-util.h"
#include "parse-util.h"
#include "process-util.h"
#include "random-util.h"
+#include "string-table.h"
#include "tests.h"
#include "time-util.h"
-typedef int (compress_t)(const void *src, uint64_t src_size, void *dst,
- size_t dst_alloc_size, size_t *dst_size, int level);
-typedef int (decompress_t)(const void *src, uint64_t src_size,
- void **dst, size_t* dst_size, size_t dst_max);
-
-#if HAVE_COMPRESSION
-
static usec_t arg_duration;
static size_t arg_start;
#define MAX_SIZE (1024*1024LU)
#define PRIME 1048571 /* A prime close enough to one megabyte that mod 4 == 3 */
+typedef enum BenchmarkDataType {
+ BENCHMARK_DATA_ZEROS,
+ BENCHMARK_DATA_SIMPLE,
+ BENCHMARK_DATA_RANDOM,
+ _BENCHMARK_DATA_TYPE_MAX,
+} BenchmarkDataType;
+
+static const char* const benchmark_data_type_table[_BENCHMARK_DATA_TYPE_MAX] = {
+ [BENCHMARK_DATA_ZEROS] = "zeros",
+ [BENCHMARK_DATA_SIMPLE] = "simple",
+ [BENCHMARK_DATA_RANDOM] = "random",
+};
+
+DEFINE_PRIVATE_STRING_TABLE_LOOKUP_TO_STRING(benchmark_data_type, BenchmarkDataType);
+
static size_t _permute(size_t x) {
size_t residue;
return _permute((_permute(x) + arg_start) % MAX_SIZE ^ 0xFF345);
}
-static char* make_buf(size_t count, const char *type) {
+static char* make_buf(size_t count, BenchmarkDataType type) {
char *buf;
- size_t i;
buf = malloc(count);
- assert_se(buf);
+ ASSERT_NOT_NULL(buf);
- if (streq(type, "zeros"))
+ switch (type) {
+
+ case BENCHMARK_DATA_ZEROS:
memzero(buf, count);
- else if (streq(type, "simple"))
- for (i = 0; i < count; i++)
+ break;
+
+ case BENCHMARK_DATA_SIMPLE:
+ for (size_t i = 0; i < count; i++)
buf[i] = 'a' + i % ('z' - 'a' + 1);
- else if (streq(type, "random")) {
+ break;
+
+ case BENCHMARK_DATA_RANDOM: {
size_t step = count / 10;
random_bytes(buf, step);
memzero(buf + 7*step, step);
random_bytes(buf + 8*step, step);
memzero(buf + 9*step, step);
- } else
+ break;
+ }
+
+ default:
assert_not_reached();
+ }
return buf;
}
-static void test_compress_decompress(const char* label, const char* type,
- compress_t compress, decompress_t decompress) {
- usec_t n, n2 = 0;
- float dt;
+TEST(benchmark) {
+ for (BenchmarkDataType dt = 0; dt < _BENCHMARK_DATA_TYPE_MAX; dt++)
+ for (Compression c = 0; c < _COMPRESSION_MAX; c++) {
+ if (c == COMPRESSION_NONE || !compression_supported(c))
+ continue;
- _cleanup_free_ char *text = NULL, *buf = NULL;
- _cleanup_free_ void *buf2 = NULL;
- size_t skipped = 0, compressed = 0, total = 0;
+ const char *label = compression_to_string(c);
+ const char *type = benchmark_data_type_to_string(dt);
+ usec_t n, n2 = 0;
- text = make_buf(MAX_SIZE, type);
- buf = calloc(MAX_SIZE + 1, 1);
- assert_se(text && buf);
+ _cleanup_free_ char *text = NULL, *buf = NULL;
+ _cleanup_free_ void *buf2 = NULL;
+ size_t skipped = 0, compressed = 0, total = 0;
- n = now(CLOCK_MONOTONIC);
+ text = make_buf(MAX_SIZE, dt);
+ buf = calloc(MAX_SIZE + 1, 1);
+ ASSERT_NOT_NULL(text);
+ ASSERT_NOT_NULL(buf);
- for (size_t i = 0; i <= MAX_SIZE; i++) {
- size_t j = 0, k = 0, size;
- int r;
+ n = now(CLOCK_MONOTONIC);
- size = permute(i);
- if (size == 0)
- continue;
+ for (size_t i = 0; i <= MAX_SIZE; i++) {
+ size_t j = 0, k = 0, size;
+ int r;
- log_debug("%s %zu %zu", type, i, size);
+ size = permute(i);
+ if (size == 0)
+ continue;
- memzero(buf, MIN(size + 1000, MAX_SIZE));
+ log_debug("%s %zu %zu", type, i, size);
- r = compress(text, size, buf, size, &j, /* level= */ -1);
- /* assume compression must be successful except for small or random inputs */
- assert_se(r >= 0 || (size < 2048 && r == -ENOBUFS) || streq(type, "random"));
+ memzero(buf, MIN(size + 1000, MAX_SIZE));
- /* check for overwrites */
- assert_se(buf[size] == 0);
- if (r < 0) {
- skipped += size;
- continue;
- }
+ r = compress_blob(c, text, size, buf, size, &j, /* level= */ -1);
+ /* assume compression must be successful except for small or random inputs */
+ ASSERT_TRUE(r >= 0 || (size < 2048 && r == -ENOBUFS) || dt == BENCHMARK_DATA_RANDOM);
- assert_se(j > 0);
- if (j >= size)
- log_error("%s \"compressed\" %zu -> %zu", label, size, j);
+ /* check for overwrites */
+ ASSERT_EQ(buf[size], 0);
+ if (r < 0) {
+ skipped += size;
+ continue;
+ }
- r = decompress(buf, j, &buf2, &k, 0);
- assert_se(r == 0);
- assert_se(k == size);
+ ASSERT_TRUE(j > 0);
+ if (j >= size)
+ log_error("%s \"compressed\" %zu -> %zu", label, size, j);
- assert_se(memcmp(text, buf2, size) == 0);
+ ASSERT_OK_ZERO(decompress_blob(c, buf, j, &buf2, &k, 0));
+ ASSERT_EQ(k, size);
+ ASSERT_EQ(memcmp(text, buf2, size), 0);
- total += size;
- compressed += j;
+ total += size;
+ compressed += j;
- n2 = now(CLOCK_MONOTONIC);
- if (n2 - n > arg_duration)
- break;
- }
+ n2 = now(CLOCK_MONOTONIC);
+ if (n2 - n > arg_duration)
+ break;
+ }
- dt = (n2-n) / 1e6;
+ float elapsed = (n2-n) / 1e6;
- log_info("%s/%s: compressed & decompressed %zu bytes in %.2fs (%.2fMiB/s), "
- "mean compression %.2f%%, skipped %zu bytes",
- label, type, total, dt,
- total / 1024. / 1024 / dt,
- 100 - compressed * 100. / total,
- skipped);
+ log_info("%s/%s: compressed & decompressed %zu bytes in %.2fs (%.2fMiB/s), "
+ "mean compression %.2f%%, skipped %zu bytes",
+ label, type, total, elapsed,
+ total / 1024. / 1024 / elapsed,
+ 100 - compressed * 100. / total,
+ skipped);
+ }
}
-#endif
-
-int main(int argc, char *argv[]) {
-#if HAVE_COMPRESSION
- test_setup_logging(LOG_INFO);
- if (argc >= 2) {
+static int intro(void) {
+ if (saved_argc >= 2) {
unsigned x;
- assert_se(safe_atou(argv[1], &x) >= 0);
+ ASSERT_OK(safe_atou(saved_argv[1], &x));
arg_duration = x * USEC_PER_SEC;
} else
arg_duration = slow_tests_enabled() ?
2 * USEC_PER_SEC : USEC_PER_SEC / 50;
- if (argc == 3)
- (void) safe_atozu(argv[2], &arg_start);
+ if (saved_argc == 3)
+ (void) safe_atozu(saved_argv[2], &arg_start);
else
arg_start = getpid_cached();
- NULSTR_FOREACH(i, "zeros\0simple\0random\0") {
-#if HAVE_XZ
- test_compress_decompress("XZ", i, compress_blob_xz, decompress_blob_xz);
-#endif
-#if HAVE_LZ4
- test_compress_decompress("LZ4", i, compress_blob_lz4, decompress_blob_lz4);
-#endif
-#if HAVE_ZSTD
- test_compress_decompress("ZSTD", i, compress_blob_zstd, decompress_blob_zstd);
-#endif
- }
return 0;
-#else
- return log_tests_skipped("No compression feature is enabled");
-#endif
}
+
+DEFINE_TEST_MAIN_WITH_INTRO(LOG_INFO, intro);
#include <sys/stat.h>
#include <unistd.h>
-#if HAVE_LZ4
-#include <lz4.h>
-#endif
-
#include "alloc-util.h"
+#include "argv-util.h"
#include "compress.h"
-#include "dlfcn-util.h"
#include "fd-util.h"
#include "io-util.h"
#include "path-util.h"
#include "tests.h"
#include "tmpfile-util.h"
-#if HAVE_XZ
-# define XZ_OK 0
-#else
-# define XZ_OK -EPROTONOSUPPORT
-#endif
+#define HUGE_SIZE (4096*1024)
-#if HAVE_LZ4
-# define LZ4_OK 0
-#else
-# define LZ4_OK -EPROTONOSUPPORT
-#endif
+static const char text[] =
+ "text\0foofoofoofoo AAAA aaaaaaaaa ghost busters barbarbar FFF"
+ "foofoofoofoo AAAA aaaaaaaaa ghost busters barbarbar FFF";
+static char data[512] = "random\0";
+static char *huge = NULL;
+static const char *srcfile;
+
+static const char* cat_for_compression(Compression c) {
+ switch (c) {
+ case COMPRESSION_XZ: return "xzcat";
+ case COMPRESSION_LZ4: return "lz4cat";
+ case COMPRESSION_ZSTD: return "zstdcat";
+ case COMPRESSION_GZIP: return "zcat";
+ case COMPRESSION_BZIP2: return "bzcat";
+ default: return NULL;
+ }
+}
-#define HUGE_SIZE (4096*1024)
+TEST(compress_decompress_blob) {
+ for (Compression c = 0; c < _COMPRESSION_MAX; c++) {
+ if (c == COMPRESSION_NONE || !compression_supported(c))
+ continue;
+
+ const char *label = compression_to_string(c);
+
+ for (size_t t = 0; t < 2; t++) {
+ const char *input = t == 0 ? text : data;
+ size_t input_len = t == 0 ? sizeof(text) : sizeof(data);
+ bool may_fail = t == 1;
+
+ char compressed[512];
+ size_t csize;
+ _cleanup_free_ char *decompressed = NULL;
+ int r;
+
+ log_info("/* testing %s %s blob compression/decompression */", label, input);
+
+ r = compress_blob(c, input, input_len, compressed, sizeof(compressed), &csize, -1);
+ if (r == -ENOBUFS) {
+ log_info_errno(r, "compression failed: %m");
+ ASSERT_TRUE(may_fail);
+ } else {
+ ASSERT_OK(r);
+ ASSERT_OK_ZERO(decompress_blob(c, compressed, csize, (void **) &decompressed, &csize, 0));
+ ASSERT_NOT_NULL(decompressed);
+ ASSERT_EQ(memcmp(decompressed, input, input_len), 0);
+ }
+
+ ASSERT_FAIL(decompress_blob(c, "garbage", 7, (void **) &decompressed, &csize, 0));
+ }
+ }
+}
-typedef int (compress_blob_t)(const void *src, uint64_t src_size,
- void *dst, size_t dst_alloc_size, size_t *dst_size, int level);
-typedef int (decompress_blob_t)(const void *src, uint64_t src_size,
- void **dst,
- size_t* dst_size, size_t dst_max);
-typedef int (decompress_sw_t)(const void *src, uint64_t src_size,
- void **buffer,
- const void *prefix, size_t prefix_len,
- uint8_t extra);
-
-typedef int (compress_stream_t)(int fdf, int fdt, uint64_t max_bytes, uint64_t *uncompressed_size);
-typedef int (decompress_stream_t)(int fdf, int fdt, uint64_t max_size);
-
-#if HAVE_COMPRESSION
-_unused_ static void test_compress_decompress(
- const char *compression,
- compress_blob_t compress,
- decompress_blob_t decompress,
- const char *data,
- size_t data_len,
- bool may_fail) {
-
- char compressed[512];
- size_t csize;
- _cleanup_free_ char *decompressed = NULL;
- int r;
-
- log_info("/* testing %s %s blob compression/decompression */",
- compression, data);
-
- r = compress(data, data_len, compressed, sizeof(compressed), &csize, /* level= */ -1);
- if (r == -ENOBUFS) {
- log_info_errno(r, "compression failed: %m");
- assert_se(may_fail);
- } else {
- assert_se(r >= 0);
- r = decompress(compressed, csize,
- (void **) &decompressed, &csize, 0);
- assert_se(r == 0);
- assert_se(decompressed);
- assert_se(memcmp(decompressed, data, data_len) == 0);
+TEST(decompress_startswith) {
+ for (Compression c = 0; c < _COMPRESSION_MAX; c++) {
+ if (c == COMPRESSION_NONE || !compression_supported(c))
+ continue;
+
+ const char *label = compression_to_string(c);
+
+ struct { const char *buf; size_t len; bool may_fail; } inputs[] = {
+ { text, sizeof(text), false },
+ { data, sizeof(data), true },
+ { huge, HUGE_SIZE, true },
+ };
+
+ for (size_t t = 0; t < ELEMENTSOF(inputs); t++) {
+ char *compressed;
+ _cleanup_free_ char *compressed1 = NULL, *compressed2 = NULL, *decompressed = NULL;
+ size_t csize, len;
+ int r;
+
+ log_info("/* testing decompress_startswith with %s on %.20s */", label, inputs[t].buf);
+
+ compressed = compressed1 = malloc(512);
+ ASSERT_NOT_NULL(compressed1);
+ r = compress_blob(c, inputs[t].buf, inputs[t].len, compressed, 512, &csize, -1);
+ if (r == -ENOBUFS) {
+ log_info_errno(r, "compression failed: %m");
+ ASSERT_TRUE(inputs[t].may_fail);
+
+ compressed = compressed2 = malloc(20000);
+ ASSERT_NOT_NULL(compressed2);
+ r = compress_blob(c, inputs[t].buf, inputs[t].len, compressed, 20000, &csize, -1);
+ }
+ if (r == -ENOBUFS) {
+ log_info_errno(r, "compression failed again: %m");
+ ASSERT_TRUE(inputs[t].may_fail);
+ continue;
+ }
+ ASSERT_OK(r);
+
+ len = strlen(inputs[t].buf);
+
+ ASSERT_OK_POSITIVE(decompress_startswith(c, compressed, csize, (void **) &decompressed, inputs[t].buf, len, '\0'));
+ ASSERT_OK_ZERO(decompress_startswith(c, compressed, csize, (void **) &decompressed, inputs[t].buf, len, 'w'));
+ ASSERT_OK_POSITIVE(decompress_startswith(c, compressed, csize, (void **) &decompressed, inputs[t].buf, len - 1, inputs[t].buf[len-1]));
+ ASSERT_OK_ZERO(decompress_startswith(c, compressed, csize, (void **) &decompressed, inputs[t].buf, len - 1, 'w'));
+ }
}
+}
- r = decompress("garbage", 7,
- (void **) &decompressed, &csize, 0);
- assert_se(r < 0);
+TEST(decompress_startswith_large) {
+ /* Test decompress_startswith with large data to exercise the buffer growth path. */
- /* make sure to have the minimal lz4 compressed size */
- r = decompress("00000000\1g", 9,
- (void **) &decompressed, &csize, 0);
- assert_se(r < 0);
+ _cleanup_free_ char *large = NULL;
+ size_t large_size = 8 * 1024;
- r = decompress("\100000000g", 9,
- (void **) &decompressed, &csize, 0);
- assert_se(r < 0);
+ ASSERT_NOT_NULL(large = malloc(large_size));
+ for (size_t i = 0; i < large_size; i++)
+ large[i] = 'A' + (i % 26);
- explicit_bzero_safe(decompressed, MALLOC_SIZEOF_SAFE(decompressed));
-}
+ for (Compression c = 0; c < _COMPRESSION_MAX; c++) {
+ if (c == COMPRESSION_NONE || !compression_supported(c))
+ continue;
+
+ _cleanup_free_ char *compressed = NULL;
+ size_t csize;
-_unused_ static void test_decompress_startswith(const char *compression,
- compress_blob_t compress,
- decompress_sw_t decompress_sw,
- const char *data,
- size_t data_len,
- bool may_fail) {
-
- char *compressed;
- _cleanup_free_ char *compressed1 = NULL, *compressed2 = NULL, *decompressed = NULL;
- size_t csize, len;
- int r;
-
- log_info("/* testing decompress_startswith with %s on %.20s text */",
- compression, data);
-
-#define BUFSIZE_1 512
-#define BUFSIZE_2 20000
-
- compressed = compressed1 = malloc(BUFSIZE_1);
- assert_se(compressed1);
- r = compress(data, data_len, compressed, BUFSIZE_1, &csize, /* level= */ -1);
- if (r == -ENOBUFS) {
- log_info_errno(r, "compression failed: %m");
- assert_se(may_fail);
-
- compressed = compressed2 = malloc(BUFSIZE_2);
- assert_se(compressed2);
- r = compress(data, data_len, compressed, BUFSIZE_2, &csize, /* level= */ -1);
+ log_info("/* decompress_startswith_large with %s */", compression_to_string(c));
+
+ ASSERT_NOT_NULL(compressed = malloc(large_size));
+ int r = compress_blob(c, large, large_size, compressed, large_size, &csize, -1);
+ if (r == -ENOBUFS) {
+ log_info_errno(r, "compression failed: %m");
+ continue;
+ }
+ ASSERT_OK(r);
+
+ _cleanup_free_ void *buf = NULL;
+
+ ASSERT_OK_POSITIVE(decompress_startswith(c, compressed, csize, &buf, large, 1, large[1]));
+ ASSERT_OK_ZERO(decompress_startswith(c, compressed, csize, &buf, large, 1, 0xff));
+ ASSERT_OK_POSITIVE(decompress_startswith(c, compressed, csize, &buf, large, 512, large[512]));
+ ASSERT_OK_ZERO(decompress_startswith(c, compressed, csize, &buf, large, 512, 0xff));
+ ASSERT_OK_POSITIVE(decompress_startswith(c, compressed, csize, &buf, large, 4096, large[4096]));
+ ASSERT_OK_ZERO(decompress_startswith(c, compressed, csize, &buf, large, 4096, 0xff));
}
- assert_se(r >= 0);
-
- len = strlen(data);
-
- r = decompress_sw(compressed, csize, (void **) &decompressed, data, len, '\0');
- assert_se(r > 0);
- r = decompress_sw(compressed, csize, (void **) &decompressed, data, len, 'w');
- assert_se(r == 0);
- r = decompress_sw(compressed, csize, (void **) &decompressed, "barbarbar", 9, ' ');
- assert_se(r == 0);
- r = decompress_sw(compressed, csize, (void **) &decompressed, data, len - 1, data[len-1]);
- assert_se(r > 0);
- r = decompress_sw(compressed, csize, (void **) &decompressed, data, len - 1, 'w');
- assert_se(r == 0);
- r = decompress_sw(compressed, csize, (void **) &decompressed, data, len, '\0');
- assert_se(r > 0);
}
-_unused_ static void test_decompress_startswith_short(const char *compression,
- compress_blob_t compress,
- decompress_sw_t decompress_sw) {
-
+TEST(decompress_startswith_short) {
#define TEXT "HUGE=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
- char buf[1024];
- size_t csize;
- int r;
+ for (Compression c = 0; c < _COMPRESSION_MAX; c++) {
+ if (c == COMPRESSION_NONE || !compression_supported(c))
+ continue;
+
+ char buf[1024];
+ size_t csize;
- log_info("/* %s with %s */", __func__, compression);
+ log_info("/* decompress_startswith_short with %s */", compression_to_string(c));
- r = compress(TEXT, sizeof TEXT, buf, sizeof buf, &csize, /* level= */ -1);
- assert_se(r >= 0);
+ ASSERT_OK(compress_blob(c, TEXT, sizeof TEXT, buf, sizeof buf, &csize, -1));
- for (size_t i = 1; i < strlen(TEXT); i++) {
- _cleanup_free_ void *buf2 = NULL;
+ for (size_t i = 1; i < strlen(TEXT); i++) {
+ _cleanup_free_ void *buf2 = NULL;
- assert_se(buf2 = malloc(i));
+ ASSERT_NOT_NULL(buf2 = malloc(i));
- assert_se(decompress_sw(buf, csize, &buf2, TEXT, i, TEXT[i]) == 1);
- assert_se(decompress_sw(buf, csize, &buf2, TEXT, i, 'y') == 0);
+ ASSERT_OK_POSITIVE(decompress_startswith(c, buf, csize, &buf2, TEXT, i, TEXT[i]));
+ ASSERT_OK_ZERO(decompress_startswith(c, buf, csize, &buf2, TEXT, i, 'y'));
+ }
}
+#undef TEXT
}
-_unused_ static void test_compress_stream(const char *compression,
- const char *cat,
- compress_stream_t compress,
- decompress_stream_t decompress,
- const char *srcfile) {
-
- _cleanup_close_ int src = -EBADF, dst = -EBADF, dst2 = -EBADF;
- _cleanup_(unlink_tempfilep) char
- pattern[] = "/tmp/systemd-test.compressed.XXXXXX",
- pattern2[] = "/tmp/systemd-test.compressed.XXXXXX";
- int r;
- _cleanup_free_ char *cmd = NULL, *cmd2 = NULL;
- struct stat st = {};
- uint64_t uncompressed_size;
-
- r = find_executable(cat, NULL);
- if (r < 0) {
- log_error_errno(r, "Skipping %s, could not find %s binary: %m", __func__, cat);
- return;
- }
+TEST(compress_decompress_stream) {
+ for (Compression c = 0; c < _COMPRESSION_MAX; c++) {
+ if (c == COMPRESSION_NONE || !compression_supported(c))
+ continue;
- log_debug("/* testing %s compression */", compression);
+ const char *cat = cat_for_compression(c);
+ if (!cat)
+ continue;
- log_debug("/* create source from %s */", srcfile);
+ int r = find_executable(cat, NULL);
+ if (r < 0) {
+ log_error_errno(r, "Skipping %s, could not find %s binary: %m",
+ compression_to_string(c), cat);
+ continue;
+ }
- ASSERT_OK(src = open(srcfile, O_RDONLY|O_CLOEXEC));
+ _cleanup_close_ int src = -EBADF, dst = -EBADF, dst2 = -EBADF;
+ _cleanup_(unlink_tempfilep) char
+ pattern[] = "/tmp/systemd-test.compressed.XXXXXX",
+ pattern2[] = "/tmp/systemd-test.compressed.XXXXXX";
+ _cleanup_free_ char *cmd = NULL, *cmd2 = NULL;
+ struct stat st = {};
+ uint64_t uncompressed_size;
- log_debug("/* test compression */");
+ log_debug("/* testing %s stream compression */", compression_to_string(c));
- assert_se((dst = mkostemp_safe(pattern)) >= 0);
+ ASSERT_OK(src = open(srcfile, O_RDONLY|O_CLOEXEC));
+ ASSERT_OK(dst = mkostemp_safe(pattern));
- ASSERT_OK(compress(src, dst, -1, &uncompressed_size));
+ ASSERT_OK(compress_stream(c, src, dst, -1, &uncompressed_size));
- if (cat) {
- assert_se(asprintf(&cmd, "%s %s | diff '%s' -", cat, pattern, srcfile) > 0);
- assert_se(system(cmd) == 0);
- }
+ ASSERT_OK_POSITIVE(asprintf(&cmd, "%s %s | diff '%s' -", cat, pattern, srcfile));
+ ASSERT_OK_ZERO(system(cmd));
+
+ ASSERT_OK(dst2 = mkostemp_safe(pattern2));
+
+ ASSERT_OK_ZERO_ERRNO(stat(srcfile, &st));
+ ASSERT_EQ((uint64_t) st.st_size, uncompressed_size);
- log_debug("/* test decompression */");
+ ASSERT_OK_ERRNO(lseek(dst, 0, SEEK_SET));
+ ASSERT_OK_ZERO(decompress_stream(c, dst, dst2, st.st_size));
- assert_se((dst2 = mkostemp_safe(pattern2)) >= 0);
+ ASSERT_OK_POSITIVE(asprintf(&cmd2, "diff '%s' %s", srcfile, pattern2));
+ ASSERT_OK_ZERO(system(cmd2));
- assert_se(stat(srcfile, &st) == 0);
- assert_se((uint64_t)st.st_size == uncompressed_size);
+ log_debug("/* test faulty decompression */");
- assert_se(lseek(dst, 0, SEEK_SET) == 0);
- r = decompress(dst, dst2, st.st_size);
- assert_se(r == 0);
+ ASSERT_OK_ERRNO(lseek(dst, 1, SEEK_SET));
+ r = decompress_stream(c, dst, dst2, st.st_size);
+ ASSERT_TRUE(IN_SET(r, 0, -EBADMSG));
- assert_se(asprintf(&cmd2, "diff '%s' %s", srcfile, pattern2) > 0);
- assert_se(system(cmd2) == 0);
+ ASSERT_OK_ERRNO(lseek(dst, 0, SEEK_SET));
+ ASSERT_OK_ERRNO(lseek(dst2, 0, SEEK_SET));
+ ASSERT_ERROR(decompress_stream(c, dst, dst2, st.st_size - 1), EFBIG);
+ }
+}
+
+struct decompressor_test_data {
+ uint8_t *buf;
+ size_t size;
+};
- log_debug("/* test faulty decompression */");
+static int test_decompressor_callback(const void *p, size_t size, void *userdata) {
+ struct decompressor_test_data *d = ASSERT_PTR(userdata);
- assert_se(lseek(dst, 1, SEEK_SET) == 1);
- r = decompress(dst, dst2, st.st_size);
- assert_se(IN_SET(r, 0, -EBADMSG));
+ if (!GREEDY_REALLOC(d->buf, d->size + size))
+ return -ENOMEM;
- assert_se(lseek(dst, 0, SEEK_SET) == 0);
- assert_se(lseek(dst2, 0, SEEK_SET) == 0);
- r = decompress(dst, dst2, st.st_size - 1);
- assert_se(r == -EFBIG);
+ memcpy(d->buf + d->size, p, size);
+ d->size += size;
+ return 0;
}
-_unused_ static void test_decompress_stream_sparse(const char *compression,
- compress_stream_t compress,
- decompress_stream_t decompress) {
-
- _cleanup_close_ int src = -EBADF, compressed = -EBADF, decompressed = -EBADF;
- _cleanup_(unlink_tempfilep) char
- pattern_src[] = "/tmp/systemd-test.sparse-src.XXXXXX",
- pattern_compressed[] = "/tmp/systemd-test.sparse-compressed.XXXXXX",
- pattern_decompressed[] = "/tmp/systemd-test.sparse-decompressed.XXXXXX";
- /* Create a sparse-like input: 4K of data, 64K of zeros, 4K of data, 64K trailing zeros.
- * Total apparent size: 136K, but most of it is zeros. */
- uint8_t data_block[4096];
- struct stat st_src, st_decompressed;
- uint64_t uncompressed_size;
- int r;
-
- assert(compression);
-
- log_debug("/* testing %s sparse decompression */", compression);
-
- random_bytes(data_block, sizeof(data_block));
-
- assert_se((src = mkostemp_safe(pattern_src)) >= 0);
-
- /* Write: 4K data, 64K zeros, 4K data, 64K zeros */
- assert_se(loop_write(src, data_block, sizeof(data_block)) >= 0);
- assert_se(ftruncate(src, sizeof(data_block) + 65536) >= 0);
- assert_se(lseek(src, sizeof(data_block) + 65536, SEEK_SET) >= 0);
- assert_se(loop_write(src, data_block, sizeof(data_block)) >= 0);
- assert_se(ftruncate(src, 2 * sizeof(data_block) + 2 * 65536) >= 0);
- assert_se(lseek(src, 0, SEEK_SET) == 0);
-
- assert_se(fstat(src, &st_src) >= 0);
- assert_se(st_src.st_size == 2 * (off_t) sizeof(data_block) + 2 * 65536);
-
- /* Compress */
- assert_se((compressed = mkostemp_safe(pattern_compressed)) >= 0);
- ASSERT_OK(compress(src, compressed, -1, &uncompressed_size));
- assert_se((uint64_t) st_src.st_size == uncompressed_size);
-
- /* Decompress to a regular file (sparse writes auto-detected) */
- assert_se((decompressed = mkostemp_safe(pattern_decompressed)) >= 0);
- assert_se(lseek(compressed, 0, SEEK_SET) == 0);
- r = decompress(compressed, decompressed, st_src.st_size);
- assert_se(r == 0);
-
- /* Verify apparent size matches */
- assert_se(fstat(decompressed, &st_decompressed) >= 0);
- assert_se(st_decompressed.st_size == st_src.st_size);
-
- /* Verify content matches by comparing bytes */
- assert_se(lseek(src, 0, SEEK_SET) == 0);
- assert_se(lseek(decompressed, 0, SEEK_SET) == 0);
-
- for (off_t offset = 0; offset < st_src.st_size;) {
- uint8_t buf_src[4096], buf_dst[4096];
- size_t to_read = MIN((size_t) (st_src.st_size - offset), sizeof(buf_src));
- ssize_t n;
-
- n = loop_read(src, buf_src, to_read, true);
- assert_se(n == (ssize_t) to_read);
- n = loop_read(decompressed, buf_dst, to_read, true);
- assert_se(n == (ssize_t) to_read);
- assert_se(memcmp(buf_src, buf_dst, to_read) == 0);
- offset += to_read;
- }
+TEST(decompress_stream_sparse) {
+ for (Compression c = 0; c < _COMPRESSION_MAX; c++) {
+ if (c == COMPRESSION_NONE || !compression_supported(c))
+ continue;
- /* Verify the decompressed file is actually sparse (uses less disk than apparent size).
- * st_blocks is in 512-byte units. The file has 128K of zeros, so disk usage should be
- * noticeably less than the apparent size if sparse writes worked.
- * Only assert if the filesystem supports holes (SEEK_HOLE). */
- log_debug("%s sparse decompression: apparent=%jd disk=%jd",
- compression,
- (intmax_t) st_decompressed.st_size,
- (intmax_t) st_decompressed.st_blocks * 512);
- if (lseek(decompressed, 0, SEEK_HOLE) < st_decompressed.st_size)
- assert_se(st_decompressed.st_blocks * 512 < st_decompressed.st_size);
- else
- log_debug("Filesystem does not support holes, skipping sparsity check");
-
- /* Test all-zeros input: entire output should be a hole */
- log_debug("/* testing %s sparse decompression of all-zeros */", compression);
- {
- _cleanup_close_ int zsrc = -EBADF, zcompressed = -EBADF, zdecompressed = -EBADF;
+ _cleanup_close_ int src = -EBADF, compressed = -EBADF, decompressed = -EBADF;
_cleanup_(unlink_tempfilep) char
- zp_src[] = "/tmp/systemd-test.sparse-zero-src.XXXXXX",
- zp_compressed[] = "/tmp/systemd-test.sparse-zero-compressed.XXXXXX",
- zp_decompressed[] = "/tmp/systemd-test.sparse-zero-decompressed.XXXXXX";
- struct stat zst;
- uint64_t zsize;
- uint8_t zeros[65536] = {};
-
- assert_se((zsrc = mkostemp_safe(zp_src)) >= 0);
- assert_se(loop_write(zsrc, zeros, sizeof(zeros)) >= 0);
- assert_se(lseek(zsrc, 0, SEEK_SET) == 0);
-
- assert_se((zcompressed = mkostemp_safe(zp_compressed)) >= 0);
- ASSERT_OK(compress(zsrc, zcompressed, -1, &zsize));
- assert_se(zsize == sizeof(zeros));
-
- assert_se((zdecompressed = mkostemp_safe(zp_decompressed)) >= 0);
- assert_se(lseek(zcompressed, 0, SEEK_SET) == 0);
- assert_se(decompress(zcompressed, zdecompressed, sizeof(zeros)) == 0);
-
- assert_se(fstat(zdecompressed, &zst) >= 0);
- assert_se(zst.st_size == (off_t) sizeof(zeros));
- /* All zeros — disk usage should be minimal */
- log_debug("%s all-zeros sparse: apparent=%jd disk=%jd",
- compression, (intmax_t) zst.st_size, (intmax_t) zst.st_blocks * 512);
- if (lseek(zdecompressed, 0, SEEK_HOLE) < zst.st_size)
- assert_se(zst.st_blocks * 512 < zst.st_size);
+ pattern_src[] = "/tmp/systemd-test.sparse-src.XXXXXX",
+ pattern_compressed[] = "/tmp/systemd-test.sparse-compressed.XXXXXX",
+ pattern_decompressed[] = "/tmp/systemd-test.sparse-decompressed.XXXXXX";
+ /* Create a sparse-like input: 4K of data, 64K of zeros, 4K of data, 64K trailing zeros.
+ * Total apparent size: 136K, but most of it is zeros. */
+ uint8_t data_block[4096];
+ struct stat st_src, st_decompressed;
+ uint64_t uncompressed_size;
+
+ log_debug("/* testing %s sparse decompression */", compression_to_string(c));
+
+ random_bytes(data_block, sizeof(data_block));
+
+ ASSERT_OK(src = mkostemp_safe(pattern_src));
+
+ /* Write: 4K data, 64K zeros, 4K data, 64K zeros */
+ ASSERT_OK(loop_write(src, data_block, sizeof(data_block)));
+ ASSERT_OK_ERRNO(ftruncate(src, sizeof(data_block) + 65536));
+ ASSERT_OK_ERRNO(lseek(src, sizeof(data_block) + 65536, SEEK_SET));
+ ASSERT_OK(loop_write(src, data_block, sizeof(data_block)));
+ ASSERT_OK_ERRNO(ftruncate(src, 2 * sizeof(data_block) + 2 * 65536));
+ ASSERT_EQ(lseek(src, 0, SEEK_SET), (off_t) 0);
+
+ ASSERT_OK_ERRNO(fstat(src, &st_src));
+ ASSERT_EQ(st_src.st_size, 2 * (off_t) sizeof(data_block) + 2 * 65536);
+
+ /* Compress */
+ ASSERT_OK(compressed = mkostemp_safe(pattern_compressed));
+ ASSERT_OK(compress_stream(c, src, compressed, -1, &uncompressed_size));
+ ASSERT_EQ((uint64_t) st_src.st_size, uncompressed_size);
+
+ /* Decompress to a regular file (sparse writes auto-detected) */
+ ASSERT_OK(decompressed = mkostemp_safe(pattern_decompressed));
+ ASSERT_EQ(lseek(compressed, 0, SEEK_SET), (off_t) 0);
+ ASSERT_OK_ZERO(decompress_stream(c, compressed, decompressed, st_src.st_size));
+
+ /* Verify apparent size matches */
+ ASSERT_OK_ERRNO(fstat(decompressed, &st_decompressed));
+ ASSERT_EQ(st_decompressed.st_size, st_src.st_size);
+
+ /* Verify content matches by comparing bytes */
+ ASSERT_EQ(lseek(src, 0, SEEK_SET), (off_t) 0);
+ ASSERT_EQ(lseek(decompressed, 0, SEEK_SET), (off_t) 0);
+
+ for (off_t offset = 0; offset < st_src.st_size;) {
+ uint8_t buf_src[4096], buf_dst[4096];
+ size_t to_read = MIN((size_t) (st_src.st_size - offset), sizeof(buf_src));
+
+ ASSERT_EQ(loop_read(src, buf_src, to_read, true), (ssize_t) to_read);
+ ASSERT_EQ(loop_read(decompressed, buf_dst, to_read, true), (ssize_t) to_read);
+ ASSERT_EQ(memcmp(buf_src, buf_dst, to_read), 0);
+ offset += to_read;
+ }
+
+ /* Verify the decompressed file is actually sparse (uses less disk than apparent size).
+ * st_blocks is in 512-byte units. The file has 128K of zeros, so disk usage should be
+ * noticeably less than the apparent size if sparse writes worked.
+ * Only assert if the filesystem supports holes (SEEK_HOLE). */
+ log_debug("%s sparse decompression: apparent=%jd disk=%jd",
+ compression_to_string(c),
+ (intmax_t) st_decompressed.st_size,
+ (intmax_t) st_decompressed.st_blocks * 512);
+ if (lseek(decompressed, 0, SEEK_HOLE) < st_decompressed.st_size)
+ ASSERT_LT(st_decompressed.st_blocks * 512, st_decompressed.st_size);
else
log_debug("Filesystem does not support holes, skipping sparsity check");
- }
- /* Test data ending with non-zero bytes: ftruncate should be a no-op */
- log_debug("/* testing %s sparse decompression ending with data */", compression);
- {
- _cleanup_close_ int dsrc = -EBADF, dcompressed = -EBADF, ddecompressed = -EBADF;
- _cleanup_(unlink_tempfilep) char
- dp_src[] = "/tmp/systemd-test.sparse-end-src.XXXXXX",
- dp_compressed[] = "/tmp/systemd-test.sparse-end-compressed.XXXXXX",
- dp_decompressed[] = "/tmp/systemd-test.sparse-end-decompressed.XXXXXX";
- struct stat dst;
- uint64_t dsize;
- uint8_t zeros[65536] = {};
-
- /* 64K zeros followed by 4K random data */
- assert_se((dsrc = mkostemp_safe(dp_src)) >= 0);
- assert_se(loop_write(dsrc, zeros, sizeof(zeros)) >= 0);
- assert_se(loop_write(dsrc, data_block, sizeof(data_block)) >= 0);
- assert_se(lseek(dsrc, 0, SEEK_SET) == 0);
-
- assert_se((dcompressed = mkostemp_safe(dp_compressed)) >= 0);
- ASSERT_OK(compress(dsrc, dcompressed, -1, &dsize));
- assert_se(dsize == sizeof(zeros) + sizeof(data_block));
-
- assert_se((ddecompressed = mkostemp_safe(dp_decompressed)) >= 0);
- assert_se(lseek(dcompressed, 0, SEEK_SET) == 0);
- assert_se(decompress(dcompressed, ddecompressed, dsize) == 0);
-
- assert_se(fstat(ddecompressed, &dst) >= 0);
- assert_se(dst.st_size == (off_t)(sizeof(zeros) + sizeof(data_block)));
+ /* Test all-zeros input: entire output should be a hole */
+ log_debug("/* testing %s sparse decompression of all-zeros */", compression_to_string(c));
+ {
+ _cleanup_close_ int zsrc = -EBADF, zcompressed = -EBADF, zdecompressed = -EBADF;
+ _cleanup_(unlink_tempfilep) char
+ zp_src[] = "/tmp/systemd-test.sparse-zero-src.XXXXXX",
+ zp_compressed[] = "/tmp/systemd-test.sparse-zero-compressed.XXXXXX",
+ zp_decompressed[] = "/tmp/systemd-test.sparse-zero-decompressed.XXXXXX";
+ struct stat zst;
+ uint64_t zsize;
+ uint8_t zeros[65536] = {};
+
+ ASSERT_OK(zsrc = mkostemp_safe(zp_src));
+ ASSERT_OK(loop_write(zsrc, zeros, sizeof(zeros)));
+ ASSERT_EQ(lseek(zsrc, 0, SEEK_SET), (off_t) 0);
+
+ ASSERT_OK(zcompressed = mkostemp_safe(zp_compressed));
+ ASSERT_OK(compress_stream(c, zsrc, zcompressed, -1, &zsize));
+ ASSERT_EQ(zsize, (uint64_t) sizeof(zeros));
+
+ ASSERT_OK(zdecompressed = mkostemp_safe(zp_decompressed));
+ ASSERT_EQ(lseek(zcompressed, 0, SEEK_SET), (off_t) 0);
+ ASSERT_OK_ZERO(decompress_stream(c, zcompressed, zdecompressed, sizeof(zeros)));
+
+ ASSERT_OK_ERRNO(fstat(zdecompressed, &zst));
+ ASSERT_EQ(zst.st_size, (off_t) sizeof(zeros));
+ /* All zeros — disk usage should be minimal */
+ log_debug("%s all-zeros sparse: apparent=%jd disk=%jd",
+ compression_to_string(c), (intmax_t) zst.st_size, (intmax_t) zst.st_blocks * 512);
+ if (lseek(zdecompressed, 0, SEEK_HOLE) < zst.st_size)
+ ASSERT_LT(zst.st_blocks * 512, zst.st_size);
+ else
+ log_debug("Filesystem does not support holes, skipping sparsity check");
+ }
+
+ /* Test data ending with non-zero bytes: ftruncate should be a no-op */
+ log_debug("/* testing %s sparse decompression ending with data */", compression_to_string(c));
+ {
+ _cleanup_close_ int dsrc = -EBADF, dcompressed = -EBADF, ddecompressed = -EBADF;
+ _cleanup_(unlink_tempfilep) char
+ dp_src[] = "/tmp/systemd-test.sparse-end-src.XXXXXX",
+ dp_compressed[] = "/tmp/systemd-test.sparse-end-compressed.XXXXXX",
+ dp_decompressed[] = "/tmp/systemd-test.sparse-end-decompressed.XXXXXX";
+ struct stat dst;
+ uint64_t dsize;
+ uint8_t zeros[65536] = {};
+
+ /* 64K zeros followed by 4K random data */
+ ASSERT_OK(dsrc = mkostemp_safe(dp_src));
+ ASSERT_OK(loop_write(dsrc, zeros, sizeof(zeros)));
+ ASSERT_OK(loop_write(dsrc, data_block, sizeof(data_block)));
+ ASSERT_EQ(lseek(dsrc, 0, SEEK_SET), (off_t) 0);
+
+ ASSERT_OK(dcompressed = mkostemp_safe(dp_compressed));
+ ASSERT_OK(compress_stream(c, dsrc, dcompressed, -1, &dsize));
+ ASSERT_EQ(dsize, (uint64_t)(sizeof(zeros) + sizeof(data_block)));
+
+ ASSERT_OK(ddecompressed = mkostemp_safe(dp_decompressed));
+ ASSERT_EQ(lseek(dcompressed, 0, SEEK_SET), (off_t) 0);
+ ASSERT_OK_ZERO(decompress_stream(c, dcompressed, ddecompressed, dsize));
+
+ ASSERT_OK_ERRNO(fstat(ddecompressed, &dst));
+ ASSERT_EQ(dst.st_size, (off_t)(sizeof(zeros) + sizeof(data_block)));
+ }
}
}
-#endif
-#if HAVE_LZ4
-extern DLSYM_PROTOTYPE(LZ4_compress_default);
-extern DLSYM_PROTOTYPE(LZ4_decompress_safe);
-extern DLSYM_PROTOTYPE(LZ4_decompress_safe_partial);
-extern DLSYM_PROTOTYPE(LZ4_versionNumber);
+TEST(compressor_decompressor_push_api) {
+ for (Compression c = 0; c < _COMPRESSION_MAX; c++) {
+ if (c == COMPRESSION_NONE || !compression_supported(c))
+ continue;
-static void test_lz4_decompress_partial(void) {
- char buf[20000], buf2[100];
- size_t buf_size = sizeof(buf), compressed;
- int r;
- _cleanup_free_ char *huge = NULL;
+ log_info("/* testing %s Compressor/Decompressor push API */", compression_to_string(c));
- log_debug("/* %s */", __func__);
+ _cleanup_(compressor_freep) Compressor *compressor = NULL;
+ _cleanup_(compressor_freep) Decompressor *decompressor = NULL;
+ _cleanup_free_ void *compressed = NULL, *finish_buf = NULL;
+ size_t compressed_size = 0, compressed_alloc = 0;
+ size_t finish_size = 0, finish_alloc = 0;
- assert_se(huge = malloc(HUGE_SIZE));
- memcpy(huge, "HUGE=", STRLEN("HUGE="));
- memset(&huge[STRLEN("HUGE=")], 'x', HUGE_SIZE - STRLEN("HUGE=") - 1);
- huge[HUGE_SIZE - 1] = '\0';
+ /* Compress */
+ ASSERT_OK(compressor_new(&compressor, c));
+ ASSERT_EQ(compressor_type(compressor), c);
- r = sym_LZ4_compress_default(huge, buf, HUGE_SIZE, buf_size);
- assert_se(r >= 0);
- compressed = r;
- log_info("Compressed %i → %zu", HUGE_SIZE, compressed);
-
- r = sym_LZ4_decompress_safe(buf, huge, r, HUGE_SIZE);
- assert_se(r >= 0);
- log_info("Decompressed → %i", r);
-
- r = sym_LZ4_decompress_safe_partial(buf, huge,
- compressed,
- 12, HUGE_SIZE);
- assert_se(r >= 0);
- log_info("Decompressed partial %i/%i → %i", 12, HUGE_SIZE, r);
-
- for (size_t size = 1; size < sizeof(buf2); size++) {
- /* This failed in older lz4s but works in newer ones. */
- r = sym_LZ4_decompress_safe_partial(buf, buf2, compressed, size, size);
- log_info("Decompressed partial %zu/%zu → %i (%s)", size, size, r,
- r < 0 ? "bad" : "good");
- if (r >= 0 && sym_LZ4_versionNumber() >= 10803)
- /* lz4 <= 1.8.2 should fail that test, let's only check for newer ones */
- assert_se(memcmp(buf2, huge, r) == 0);
+ ASSERT_OK(compressor_start(compressor, text, sizeof(text), &compressed, &compressed_size, &compressed_alloc));
+ ASSERT_OK(compressor_finish(compressor, &finish_buf, &finish_size, &finish_alloc));
+
+ size_t total_compressed = compressed_size + finish_size;
+ _cleanup_free_ void *full_compressed = malloc(total_compressed);
+ ASSERT_NOT_NULL(full_compressed);
+ memcpy(full_compressed, compressed, compressed_size);
+ if (finish_size > 0)
+ memcpy((uint8_t*) full_compressed + compressed_size, finish_buf, finish_size);
+
+ compressor = compressor_free(compressor);
+
+ /* Decompress via detect + push and verify content */
+ ASSERT_OK_POSITIVE(decompressor_detect(&decompressor, full_compressed, total_compressed));
+ ASSERT_EQ(compressor_type(decompressor), c);
+
+ struct decompressor_test_data result = {};
+ ASSERT_OK(decompressor_push(decompressor, full_compressed, total_compressed, test_decompressor_callback, &result));
+ ASSERT_EQ(result.size, sizeof(text));
+ ASSERT_EQ(memcmp(result.buf, text, sizeof(text)), 0);
+ free(result.buf);
+
+ decompressor = compressor_free(decompressor);
}
-}
-#endif
-int main(int argc, char *argv[]) {
-#if HAVE_COMPRESSION
- _unused_ const char text[] =
- "text\0foofoofoofoo AAAA aaaaaaaaa ghost busters barbarbar FFF"
- "foofoofoofoo AAAA aaaaaaaaa ghost busters barbarbar FFF";
+ /* Test compressor_type on NULL */
+ ASSERT_EQ(compressor_type(NULL), _COMPRESSION_INVALID);
- /* The file to test compression on can be specified as the first argument */
- const char *srcfile = argc > 1 ? argv[1] : argv[0];
+ /* Test decompressor_force_off */
+ _cleanup_(compressor_freep) Decompressor *d = NULL;
+ ASSERT_OK(decompressor_force_off(&d));
+ ASSERT_EQ(compressor_type(d), COMPRESSION_NONE);
+ d = compressor_free(d);
- char data[512] = "random\0";
+ /* Test decompressor_detect returning 0 on too-small input */
+ ASSERT_OK_ZERO(decompressor_detect(&d, "x", 1));
+ ASSERT_NULL(d);
+}
- _cleanup_free_ char *huge = NULL;
+static int intro(void) {
+ srcfile = saved_argc > 1 ? saved_argv[1] : saved_argv[0];
- assert_se(huge = malloc(HUGE_SIZE));
+ ASSERT_NOT_NULL(huge = malloc(HUGE_SIZE));
memcpy(huge, "HUGE=", STRLEN("HUGE="));
memset(&huge[STRLEN("HUGE=")], 'x', HUGE_SIZE - STRLEN("HUGE=") - 1);
huge[HUGE_SIZE - 1] = '\0';
- test_setup_logging(LOG_DEBUG);
-
random_bytes(data + 7, sizeof(data) - 7);
-#if HAVE_XZ
- test_compress_decompress("XZ", compress_blob_xz, decompress_blob_xz,
- text, sizeof(text), false);
- test_compress_decompress("XZ", compress_blob_xz, decompress_blob_xz,
- data, sizeof(data), true);
-
- test_decompress_startswith("XZ",
- compress_blob_xz, decompress_startswith_xz,
- text, sizeof(text), false);
- test_decompress_startswith("XZ",
- compress_blob_xz, decompress_startswith_xz,
- data, sizeof(data), true);
- test_decompress_startswith("XZ",
- compress_blob_xz, decompress_startswith_xz,
- huge, HUGE_SIZE, true);
-
- test_compress_stream("XZ", "xzcat",
- compress_stream_xz, decompress_stream_xz, srcfile);
-
- test_decompress_stream_sparse("XZ", compress_stream_xz, decompress_stream_xz);
-
- test_decompress_startswith_short("XZ", compress_blob_xz, decompress_startswith_xz);
-
-#else
- log_info("/* XZ test skipped */");
-#endif
-
-#if HAVE_LZ4
- if (dlopen_lz4() >= 0) {
- test_compress_decompress("LZ4", compress_blob_lz4, decompress_blob_lz4,
- text, sizeof(text), false);
- test_compress_decompress("LZ4", compress_blob_lz4, decompress_blob_lz4,
- data, sizeof(data), true);
-
- test_decompress_startswith("LZ4",
- compress_blob_lz4, decompress_startswith_lz4,
- text, sizeof(text), false);
- test_decompress_startswith("LZ4",
- compress_blob_lz4, decompress_startswith_lz4,
- data, sizeof(data), true);
- test_decompress_startswith("LZ4",
- compress_blob_lz4, decompress_startswith_lz4,
- huge, HUGE_SIZE, true);
-
- test_compress_stream("LZ4", "lz4cat",
- compress_stream_lz4, decompress_stream_lz4, srcfile);
-
- test_decompress_stream_sparse("LZ4", compress_stream_lz4, decompress_stream_lz4);
-
- test_lz4_decompress_partial();
-
- test_decompress_startswith_short("LZ4", compress_blob_lz4, decompress_startswith_lz4);
- } else
- log_error("/* Can't load liblz4 */");
-#else
- log_info("/* LZ4 test skipped */");
-#endif
-
-#if HAVE_ZSTD
- test_compress_decompress("ZSTD", compress_blob_zstd, decompress_blob_zstd,
- text, sizeof(text), false);
- test_compress_decompress("ZSTD", compress_blob_zstd, decompress_blob_zstd,
- data, sizeof(data), true);
-
- test_decompress_startswith("ZSTD",
- compress_blob_zstd, decompress_startswith_zstd,
- text, sizeof(text), false);
- test_decompress_startswith("ZSTD",
- compress_blob_zstd, decompress_startswith_zstd,
- data, sizeof(data), true);
- test_decompress_startswith("ZSTD",
- compress_blob_zstd, decompress_startswith_zstd,
- huge, HUGE_SIZE, true);
-
- test_compress_stream("ZSTD", "zstdcat",
- compress_stream_zstd, decompress_stream_zstd, srcfile);
-
- test_decompress_stream_sparse("ZSTD", compress_stream_zstd, decompress_stream_zstd);
-
- test_decompress_startswith_short("ZSTD", compress_blob_zstd, decompress_startswith_zstd);
-#else
- log_info("/* ZSTD test skipped */");
-#endif
-
return 0;
-#else
- return log_tests_skipped("no compression algorithm supported");
-#endif
}
+
+DEFINE_TEST_MAIN_WITH_INTRO(LOG_DEBUG, intro);
* where .so versions change and distributions update, but systemd doesn't have the new so names
* around yet. */
+ ASSERT_DLOPEN(dlopen_bzip2, HAVE_BZIP2);
ASSERT_DLOPEN(dlopen_bpf, HAVE_LIBBPF);
ASSERT_DLOPEN(dlopen_cryptsetup, HAVE_LIBCRYPTSETUP);
ASSERT_DLOPEN(dlopen_dw, HAVE_ELFUTILS);
ASSERT_DLOPEN(dlopen_libpam, HAVE_PAM);
ASSERT_DLOPEN(dlopen_libseccomp, HAVE_SECCOMP);
ASSERT_DLOPEN(dlopen_libselinux, HAVE_SELINUX);
+ ASSERT_DLOPEN(dlopen_xz, HAVE_XZ);
ASSERT_DLOPEN(dlopen_lz4, HAVE_LZ4);
- ASSERT_DLOPEN(dlopen_lzma, HAVE_XZ);
ASSERT_DLOPEN(dlopen_p11kit, HAVE_P11KIT);
ASSERT_DLOPEN(dlopen_passwdqc, HAVE_PASSWDQC);
ASSERT_DLOPEN(dlopen_pcre2, HAVE_PCRE2);
ASSERT_DLOPEN(dlopen_pwquality, HAVE_PWQUALITY);
ASSERT_DLOPEN(dlopen_qrencode, HAVE_QRENCODE);
ASSERT_DLOPEN(dlopen_tpm2, HAVE_TPM2);
+ ASSERT_DLOPEN(dlopen_zlib, HAVE_ZLIB);
ASSERT_DLOPEN(dlopen_zstd, HAVE_ZSTD);
return 0;
systemctl reset-failed systemd-journald.service
for c in NONE XZ LZ4 ZSTD; do
+ # compression_to_string() returns "uncompressed" for COMPRESSION_NONE
+ if [[ "${c}" == NONE ]]; then
+ log_name="uncompressed"
+ else
+ log_name="${c,,}"
+ fi
+
cat >/run/systemd/system/systemd-journald.service.d/compress.conf <<EOF
[Service]
Environment=SYSTEMD_JOURNAL_COMPRESS=${c}
ID="$(systemd-id128 new)"
systemd-cat -t "$ID" bash -c "for ((i=0;i<100;i++)); do echo -n hoge with ${c}; done; echo"
journalctl --sync
- timeout 10 bash -c "until SYSTEMD_LOG_LEVEL=debug journalctl --verify --quiet --file /var/log/journal/$MACHINE_ID/system.journal 2>&1 | grep -F 'compress=${c}' >/dev/null; do sleep .5; done"
+ timeout 10 bash -c "until SYSTEMD_LOG_LEVEL=debug journalctl --verify --quiet --file /var/log/journal/$MACHINE_ID/system.journal 2>&1 | grep -F 'compress=${log_name}' >/dev/null; do sleep .5; done"
# $SYSTEMD_JOURNAL_COMPRESS= also works for journal-remote
if [[ -x /usr/lib/systemd/systemd-journal-remote ]]; then
for cc in NONE XZ LZ4 ZSTD; do
+ if [[ "${cc}" == NONE ]]; then
+ cc_log_name="uncompressed"
+ else
+ cc_log_name="${cc,,}"
+ fi
+
rm -f /tmp/foo.journal
SYSTEMD_JOURNAL_COMPRESS="${cc}" /usr/lib/systemd/systemd-journal-remote --split-mode=none -o /tmp/foo.journal --getter="journalctl -b -o export -t $ID"
- SYSTEMD_LOG_LEVEL=debug journalctl --verify --quiet --file /tmp/foo.journal 2>&1 | grep -F "compress=${cc}" >/dev/null
+ SYSTEMD_LOG_LEVEL=debug journalctl --verify --quiet --file /tmp/foo.journal 2>&1 | grep -F "compress=${cc_log_name}" >/dev/null
journalctl -t "$ID" -o cat --file /tmp/foo.journal | grep -F "hoge with ${c}" >/dev/null
done
fi
projects / thirdparty / systemd.git / commitdiff
