From: Matteo Croce Date: Tue, 27 Feb 2024 20:28:14 +0000 (+0100) Subject: dynamically load compression libraries X-Git-Tag: v256-rc1~635^2~1 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=3fc72d54132151c131301fc7954e0b44cdd3c860;p=thirdparty%2Fsystemd.git dynamically load compression libraries Dynamically load liblz4, libzstd and liblzma with dlopen(). This helps to reduce the size of the initrd image when these libraries are not really needed. --- diff --git a/meson.build b/meson.build index d4e89eae012..dc7e2b9a7b9 100644 --- a/meson.build +++ b/meson.build @@ -1371,16 +1371,19 @@ conf.set10('HAVE_BZIP2', libbzip2.found()) libxz = dependency('liblzma', required : get_option('xz')) conf.set10('HAVE_XZ', libxz.found()) +libxz_cflags = libxz.partial_dependency(includes: true, compile_args: true) liblz4 = dependency('liblz4', version : '>= 1.3.0', required : get_option('lz4')) conf.set10('HAVE_LZ4', liblz4.found()) +liblz4_cflags = liblz4.partial_dependency(includes: true, compile_args: true) libzstd = dependency('libzstd', version : '>= 1.4.0', required : get_option('zstd')) conf.set10('HAVE_ZSTD', libzstd.found()) +libzstd_cflags = libzstd.partial_dependency(includes: true, compile_args: true) conf.set10('HAVE_COMPRESSION', libxz.found() or liblz4.found() or libzstd.found()) @@ -1951,8 +1954,7 @@ libsystemd = shared_library( link_args : ['-shared', '-Wl,--version-script=' + libsystemd_sym_path], link_with : [libbasic, - libbasic_gcrypt, - libbasic_compress], + libbasic_gcrypt], link_whole : [libsystemd_static], dependencies : [librt, threads, @@ -1969,7 +1971,6 @@ install_libsystemd_static = static_library( libsystemd_sources, basic_sources, basic_gcrypt_sources, - basic_compress_sources, fundamental_sources, include_directories : libsystemd_includes, build_by_default : static_libsystemd != 'false', @@ -1981,12 +1982,12 @@ install_libsystemd_static = static_library( libcap, libdl, libgcrypt, - liblz4, + liblz4_cflags, libmount, libopenssl, librt, - libxz, - libzstd, + libxz_cflags, + libzstd_cflags, threads, userspace], c_args : libsystemd_c_args + (static_libsystemd_pic ? [] : ['-fno-PIC'])) diff --git a/src/basic/compress.c b/src/basic/compress.c index ac0bfdfcd5f..5a4293fa814 100644 --- a/src/basic/compress.c +++ b/src/basic/compress.c @@ -12,11 +12,6 @@ #include #endif -#if HAVE_LZ4 -#include -#include -#endif - #if HAVE_ZSTD #include #include @@ -34,16 +29,52 @@ #include "unaligned.h" #if HAVE_LZ4 -DEFINE_TRIVIAL_CLEANUP_FUNC_FULL(LZ4F_compressionContext_t, LZ4F_freeCompressionContext, NULL); -DEFINE_TRIVIAL_CLEANUP_FUNC_FULL(LZ4F_decompressionContext_t, LZ4F_freeDecompressionContext, NULL); +static void *lz4_dl = NULL; + +static DLSYM_FUNCTION(LZ4F_compressBegin); +static DLSYM_FUNCTION(LZ4F_compressBound); +static DLSYM_FUNCTION(LZ4F_compressEnd); +static DLSYM_FUNCTION(LZ4F_compressUpdate); +static DLSYM_FUNCTION(LZ4F_createCompressionContext); +static DLSYM_FUNCTION(LZ4F_createDecompressionContext); +static DLSYM_FUNCTION(LZ4F_decompress); +static DLSYM_FUNCTION(LZ4F_freeCompressionContext); +static DLSYM_FUNCTION(LZ4F_freeDecompressionContext); +static DLSYM_FUNCTION(LZ4F_isError); +DLSYM_FUNCTION(LZ4_compress_default); +DLSYM_FUNCTION(LZ4_decompress_safe); +DLSYM_FUNCTION(LZ4_decompress_safe_partial); +DLSYM_FUNCTION(LZ4_versionNumber); + +DEFINE_TRIVIAL_CLEANUP_FUNC_FULL(LZ4F_compressionContext_t, sym_LZ4F_freeCompressionContext, NULL); +DEFINE_TRIVIAL_CLEANUP_FUNC_FULL(LZ4F_decompressionContext_t, sym_LZ4F_freeDecompressionContext, NULL); #endif #if HAVE_ZSTD -DEFINE_TRIVIAL_CLEANUP_FUNC_FULL(ZSTD_CCtx*, ZSTD_freeCCtx, NULL); -DEFINE_TRIVIAL_CLEANUP_FUNC_FULL(ZSTD_DCtx*, ZSTD_freeDCtx, NULL); +static void *zstd_dl = NULL; + +static DLSYM_FUNCTION(ZSTD_CCtx_setParameter); +static DLSYM_FUNCTION(ZSTD_compress); +static DLSYM_FUNCTION(ZSTD_compressStream2); +static DLSYM_FUNCTION(ZSTD_createCCtx); +static DLSYM_FUNCTION(ZSTD_createDCtx); +static DLSYM_FUNCTION(ZSTD_CStreamInSize); +static DLSYM_FUNCTION(ZSTD_CStreamOutSize); +static DLSYM_FUNCTION(ZSTD_decompressStream); +static DLSYM_FUNCTION(ZSTD_DStreamInSize); +static DLSYM_FUNCTION(ZSTD_DStreamOutSize); +static DLSYM_FUNCTION(ZSTD_freeCCtx); +static DLSYM_FUNCTION(ZSTD_freeDCtx); +static DLSYM_FUNCTION(ZSTD_getErrorCode); +static DLSYM_FUNCTION(ZSTD_getErrorName); +static DLSYM_FUNCTION(ZSTD_getFrameContentSize); +static DLSYM_FUNCTION(ZSTD_isError); + +DEFINE_TRIVIAL_CLEANUP_FUNC_FULL(ZSTD_CCtx*, sym_ZSTD_freeCCtx, NULL); +DEFINE_TRIVIAL_CLEANUP_FUNC_FULL(ZSTD_DCtx*, sym_ZSTD_freeDCtx, NULL); static int zstd_ret_to_errno(size_t ret) { - switch (ZSTD_getErrorCode(ret)) { + switch (sym_ZSTD_getErrorCode(ret)) { case ZSTD_error_dstSize_tooSmall: return -ENOBUFS; case ZSTD_error_memory_allocation: @@ -54,6 +85,27 @@ static int zstd_ret_to_errno(size_t ret) { } #endif +#if HAVE_XZ +static void *lzma_dl = NULL; + +static DLSYM_FUNCTION(lzma_code); +static DLSYM_FUNCTION(lzma_easy_encoder); +static DLSYM_FUNCTION(lzma_end); +static DLSYM_FUNCTION(lzma_stream_buffer_encode); +static DLSYM_FUNCTION(lzma_stream_decoder); + +/* We can't just do _cleanup_(sym_lzma_end) because a compiler bug makes + * this fail with: + * ../src/basic/compress.c: In function ‘decompress_blob_xz’: + * ../src/basic/compress.c:304:9: error: cleanup argument not a function + * 304 | _cleanup_(sym_lzma_end) lzma_stream s = LZMA_STREAM_INIT; + * | ^~~~~~~~~ + */ +static inline void lzma_end_wrapper(lzma_stream *ls) { + sym_lzma_end(ls); +} +#endif + #define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t)) static const char* const compression_table[_COMPRESSION_MAX] = { @@ -75,8 +127,28 @@ bool compression_supported(Compression c) { return c >= 0 && c < _COMPRESSION_MAX && FLAGS_SET(supported, 1U << c); } +#if HAVE_XZ +int dlopen_lzma(void) { + return dlopen_many_sym_or_warn( + &lzma_dl, + "liblzma.so.5", LOG_DEBUG, + DLSYM_ARG(lzma_code), + DLSYM_ARG(lzma_easy_encoder), + DLSYM_ARG(lzma_end), + DLSYM_ARG(lzma_stream_buffer_encode), + DLSYM_ARG(lzma_stream_decoder)); +} +#endif + int compress_blob_xz(const void *src, uint64_t src_size, void *dst, size_t dst_alloc_size, size_t *dst_size) { + + assert(src); + assert(src_size > 0); + assert(dst); + assert(dst_alloc_size > 0); + assert(dst_size); + #if HAVE_XZ static const lzma_options_lzma opt = { 1u << 20u, NULL, 0, LZMA_LC_DEFAULT, LZMA_LP_DEFAULT, @@ -88,12 +160,11 @@ int compress_blob_xz(const void *src, uint64_t src_size, }; lzma_ret ret; size_t out_pos = 0; + int r; - assert(src); - assert(src_size > 0); - assert(dst); - assert(dst_alloc_size > 0); - assert(dst_size); + r = dlopen_lzma(); + if (r < 0) + return r; /* Returns < 0 if we couldn't compress the data or the * compressed result is longer than the original */ @@ -101,7 +172,7 @@ int compress_blob_xz(const void *src, uint64_t src_size, if (src_size < 80) return -ENOBUFS; - ret = lzma_stream_buffer_encode((lzma_filter*) filters, LZMA_CHECK_NONE, NULL, + ret = sym_lzma_stream_buffer_encode((lzma_filter*) filters, LZMA_CHECK_NONE, NULL, src, src_size, dst, &out_pos, dst_alloc_size); if (ret != LZMA_OK) return -ENOBUFS; @@ -113,10 +184,30 @@ int compress_blob_xz(const void *src, uint64_t src_size, #endif } +#if HAVE_LZ4 +int dlopen_lz4(void) { + return dlopen_many_sym_or_warn( + &lz4_dl, + "liblz4.so.1", LOG_DEBUG, + DLSYM_ARG(LZ4F_compressBegin), + DLSYM_ARG(LZ4F_compressBound), + DLSYM_ARG(LZ4F_compressEnd), + DLSYM_ARG(LZ4F_compressUpdate), + DLSYM_ARG(LZ4F_createCompressionContext), + DLSYM_ARG(LZ4F_createDecompressionContext), + DLSYM_ARG(LZ4F_decompress), + DLSYM_ARG(LZ4F_freeCompressionContext), + DLSYM_ARG(LZ4F_freeDecompressionContext), + DLSYM_ARG(LZ4F_isError), + DLSYM_ARG(LZ4_compress_default), + DLSYM_ARG(LZ4_decompress_safe), + DLSYM_ARG(LZ4_decompress_safe_partial), + DLSYM_ARG(LZ4_versionNumber)); +} +#endif + int compress_blob_lz4(const void *src, uint64_t src_size, void *dst, size_t dst_alloc_size, size_t *dst_size) { -#if HAVE_LZ4 - int r; assert(src); assert(src_size > 0); @@ -124,13 +215,19 @@ int compress_blob_lz4(const void *src, uint64_t src_size, assert(dst_alloc_size > 0); assert(dst_size); +#if HAVE_LZ4 + int r; + + r = dlopen_lz4(); + if (r < 0) + return r; /* Returns < 0 if we couldn't compress the data or the * compressed result is longer than the original */ if (src_size < 9) return -ENOBUFS; - r = LZ4_compress_default(src, (char*)dst + 8, src_size, (int) dst_alloc_size - 8); + r = sym_LZ4_compress_default(src, (char*)dst + 8, src_size, (int) dst_alloc_size - 8); if (r <= 0) return -ENOBUFS; @@ -143,11 +240,33 @@ int compress_blob_lz4(const void *src, uint64_t src_size, #endif } +#if HAVE_ZSTD +int dlopen_zstd(void) { + return dlopen_many_sym_or_warn( + &zstd_dl, + "libzstd.so.1", LOG_DEBUG, + DLSYM_ARG(ZSTD_getErrorCode), + DLSYM_ARG(ZSTD_compress), + DLSYM_ARG(ZSTD_getFrameContentSize), + DLSYM_ARG(ZSTD_decompressStream), + DLSYM_ARG(ZSTD_getErrorName), + DLSYM_ARG(ZSTD_DStreamOutSize), + DLSYM_ARG(ZSTD_CStreamInSize), + DLSYM_ARG(ZSTD_CStreamOutSize), + DLSYM_ARG(ZSTD_CCtx_setParameter), + DLSYM_ARG(ZSTD_compressStream2), + DLSYM_ARG(ZSTD_DStreamInSize), + DLSYM_ARG(ZSTD_freeCCtx), + DLSYM_ARG(ZSTD_freeDCtx), + DLSYM_ARG(ZSTD_isError), + DLSYM_ARG(ZSTD_createDCtx), + DLSYM_ARG(ZSTD_createCCtx)); +} +#endif + int compress_blob_zstd( const void *src, uint64_t src_size, void *dst, size_t dst_alloc_size, size_t *dst_size) { -#if HAVE_ZSTD - size_t k; assert(src); assert(src_size > 0); @@ -155,8 +274,16 @@ int compress_blob_zstd( assert(dst_alloc_size > 0); assert(dst_size); - k = ZSTD_compress(dst, dst_alloc_size, src, src_size, 0); - if (ZSTD_isError(k)) +#if HAVE_ZSTD + size_t k; + int r; + + r = dlopen_zstd(); + if (r < 0) + return r; + + k = sym_ZSTD_compress(dst, dst_alloc_size, src, src_size, 0); + if (sym_ZSTD_isError(k)) return zstd_ret_to_errno(k); *dst_size = k; @@ -173,17 +300,22 @@ int decompress_blob_xz( size_t* dst_size, size_t dst_max) { -#if HAVE_XZ - _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT; - lzma_ret ret; - size_t space; - assert(src); assert(src_size > 0); assert(dst); assert(dst_size); - ret = lzma_stream_decoder(&s, UINT64_MAX, 0); +#if HAVE_XZ + _cleanup_(lzma_end_wrapper) lzma_stream s = LZMA_STREAM_INIT; + lzma_ret ret; + size_t space; + int r; + + r = dlopen_lzma(); + if (r < 0) + return r; + + ret = sym_lzma_stream_decoder(&s, UINT64_MAX, 0); if (ret != LZMA_OK) return -ENOMEM; @@ -200,7 +332,7 @@ int decompress_blob_xz( for (;;) { size_t used; - ret = lzma_code(&s, LZMA_FINISH); + ret = sym_lzma_code(&s, LZMA_FINISH); if (ret == LZMA_STREAM_END) break; @@ -235,15 +367,19 @@ int decompress_blob_lz4( size_t* dst_size, size_t dst_max) { -#if HAVE_LZ4 - char* out; - int r, size; /* LZ4 uses int for size */ - assert(src); assert(src_size > 0); assert(dst); assert(dst_size); +#if HAVE_LZ4 + char* out; + int r, size; /* LZ4 uses int for size */ + + r = dlopen_lz4(); + if (r < 0) + return r; + if (src_size <= 8) return -EBADMSG; @@ -254,7 +390,7 @@ int decompress_blob_lz4( if (!out) return -ENOMEM; - r = LZ4_decompress_safe((char*)src + 8, out, src_size - 8, size); + r = sym_LZ4_decompress_safe((char*)src + 8, out, src_size - 8, size); if (r < 0 || r != size) return -EBADMSG; @@ -272,15 +408,20 @@ int decompress_blob_zstd( size_t *dst_size, size_t dst_max) { -#if HAVE_ZSTD - uint64_t size; - assert(src); assert(src_size > 0); assert(dst); assert(dst_size); - size = ZSTD_getFrameContentSize(src, src_size); +#if HAVE_ZSTD + uint64_t size; + int r; + + r = dlopen_zstd(); + if (r < 0) + return r; + + size = sym_ZSTD_getFrameContentSize(src, src_size); if (IN_SET(size, ZSTD_CONTENTSIZE_ERROR, ZSTD_CONTENTSIZE_UNKNOWN)) return -EBADMSG; @@ -289,10 +430,10 @@ int decompress_blob_zstd( if (size > SIZE_MAX) return -E2BIG; - if (!(greedy_realloc(dst, MAX(ZSTD_DStreamOutSize(), size), 1))) + if (!(greedy_realloc(dst, MAX(sym_ZSTD_DStreamOutSize(), size), 1))) return -ENOMEM; - _cleanup_(ZSTD_freeDCtxp) ZSTD_DCtx *dctx = ZSTD_createDCtx(); + _cleanup_(sym_ZSTD_freeDCtxp) ZSTD_DCtx *dctx = sym_ZSTD_createDCtx(); if (!dctx) return -ENOMEM; @@ -305,9 +446,9 @@ int decompress_blob_zstd( .size = MALLOC_SIZEOF_SAFE(*dst), }; - size_t k = ZSTD_decompressStream(dctx, &output, &input); - if (ZSTD_isError(k)) { - log_debug("ZSTD decoder failed: %s", ZSTD_getErrorName(k)); + size_t k = sym_ZSTD_decompressStream(dctx, &output, &input); + if (sym_ZSTD_isError(k)) { + log_debug("ZSTD decoder failed: %s", sym_ZSTD_getErrorName(k)); return zstd_ret_to_errno(k); } assert(output.pos >= size); @@ -351,11 +492,6 @@ int decompress_startswith_xz( size_t prefix_len, uint8_t extra) { -#if HAVE_XZ - _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT; - size_t allocated; - lzma_ret ret; - /* Checks whether the decompressed blob starts with the mentioned prefix. The byte extra needs to * follow the prefix */ @@ -364,7 +500,17 @@ int decompress_startswith_xz( assert(buffer); assert(prefix); - ret = lzma_stream_decoder(&s, UINT64_MAX, 0); +#if HAVE_XZ + _cleanup_(lzma_end_wrapper) lzma_stream s = LZMA_STREAM_INIT; + size_t allocated; + lzma_ret ret; + int r; + + r = dlopen_lzma(); + if (r < 0) + return r; + + ret = sym_lzma_stream_decoder(&s, UINT64_MAX, 0); if (ret != LZMA_OK) return -EBADMSG; @@ -380,7 +526,7 @@ int decompress_startswith_xz( s.avail_out = allocated; for (;;) { - ret = lzma_code(&s, LZMA_FINISH); + ret = sym_lzma_code(&s, LZMA_FINISH); if (!IN_SET(ret, LZMA_OK, LZMA_STREAM_END)) return -EBADMSG; @@ -414,18 +560,22 @@ int decompress_startswith_lz4( size_t prefix_len, uint8_t extra) { -#if HAVE_LZ4 /* Checks whether the decompressed blob starts with the mentioned prefix. The byte extra needs to * follow the prefix */ - size_t allocated; - int r; - assert(src); assert(src_size > 0); assert(buffer); assert(prefix); +#if HAVE_LZ4 + size_t allocated; + int r; + + r = dlopen_lz4(); + if (r < 0) + return r; + if (src_size <= 8) return -EBADMSG; @@ -433,7 +583,7 @@ int decompress_startswith_lz4( return -ENOMEM; allocated = MALLOC_SIZEOF_SAFE(*buffer); - r = LZ4_decompress_safe_partial( + r = sym_LZ4_decompress_safe_partial( (char*)src + 8, *buffer, src_size - 8, @@ -447,7 +597,7 @@ int decompress_startswith_lz4( if (r < 0 || (size_t) r < prefix_len + 1) { size_t size; - if (LZ4_versionNumber() >= 10803) + if (sym_LZ4_versionNumber() >= 10803) /* We trust that the newer lz4 decompresses the number of bytes we * requested if available in the compressed string. */ return 0; @@ -482,24 +632,31 @@ int decompress_startswith_zstd( const void *prefix, size_t prefix_len, uint8_t extra) { -#if HAVE_ZSTD + assert(src); assert(src_size > 0); assert(buffer); assert(prefix); - uint64_t size = ZSTD_getFrameContentSize(src, src_size); +#if HAVE_ZSTD + int r; + + r = dlopen_zstd(); + if (r < 0) + return r; + + uint64_t size = sym_ZSTD_getFrameContentSize(src, src_size); if (IN_SET(size, ZSTD_CONTENTSIZE_ERROR, ZSTD_CONTENTSIZE_UNKNOWN)) return -EBADMSG; if (size < prefix_len + 1) return 0; /* Decompressed text too short to match the prefix and extra */ - _cleanup_(ZSTD_freeDCtxp) ZSTD_DCtx *dctx = ZSTD_createDCtx(); + _cleanup_(sym_ZSTD_freeDCtxp) ZSTD_DCtx *dctx = sym_ZSTD_createDCtx(); if (!dctx) return -ENOMEM; - if (!(greedy_realloc(buffer, MAX(ZSTD_DStreamOutSize(), prefix_len + 1), 1))) + if (!(greedy_realloc(buffer, MAX(sym_ZSTD_DStreamOutSize(), prefix_len + 1), 1))) return -ENOMEM; ZSTD_inBuffer input = { @@ -512,9 +669,9 @@ int decompress_startswith_zstd( }; size_t k; - k = ZSTD_decompressStream(dctx, &output, &input); - if (ZSTD_isError(k)) { - log_debug("ZSTD decoder failed: %s", ZSTD_getErrorName(k)); + k = sym_ZSTD_decompressStream(dctx, &output, &input); + if (sym_ZSTD_isError(k)) { + log_debug("ZSTD decoder failed: %s", sym_ZSTD_getErrorName(k)); return zstd_ret_to_errno(k); } assert(output.pos >= prefix_len + 1); @@ -559,16 +716,21 @@ int decompress_startswith( } int compress_stream_xz(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_uncompressed_size) { + assert(fdf >= 0); + assert(fdt >= 0); + #if HAVE_XZ - _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT; + _cleanup_(lzma_end_wrapper) lzma_stream s = LZMA_STREAM_INIT; lzma_ret ret; uint8_t buf[BUFSIZ], out[BUFSIZ]; lzma_action action = LZMA_RUN; + int r; - assert(fdf >= 0); - assert(fdt >= 0); + r = dlopen_lzma(); + if (r < 0) + return r; - ret = lzma_easy_encoder(&s, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64); + ret = sym_lzma_easy_encoder(&s, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64); if (ret != LZMA_OK) return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Failed to initialize XZ encoder: code %u", @@ -603,7 +765,7 @@ int compress_stream_xz(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_uncom s.avail_out = sizeof(out); } - ret = lzma_code(&s, action); + ret = sym_lzma_code(&s, action); if (!IN_SET(ret, LZMA_OK, LZMA_STREAM_END)) return log_error_errno(SYNTHETIC_ERRNO(EBADMSG), "Compression failed: code %u", @@ -641,7 +803,7 @@ int compress_stream_lz4(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_unco #if HAVE_LZ4 LZ4F_errorCode_t c; - _cleanup_(LZ4F_freeCompressionContextp) LZ4F_compressionContext_t ctx = NULL; + _cleanup_(sym_LZ4F_freeCompressionContextp) LZ4F_compressionContext_t ctx = NULL; _cleanup_free_ void *in_buff = NULL; _cleanup_free_ char *out_buff = NULL; size_t out_allocsize, n, offset = 0, frame_size; @@ -651,11 +813,15 @@ int compress_stream_lz4(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_unco .frameInfo.blockSizeID = 5, }; - c = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); - if (LZ4F_isError(c)) + r = dlopen_lz4(); + if (r < 0) + return r; + + c = sym_LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); + if (sym_LZ4F_isError(c)) return -ENOMEM; - frame_size = LZ4F_compressBound(LZ4_BUFSIZE, &preferences); + frame_size = sym_LZ4F_compressBound(LZ4_BUFSIZE, &preferences); out_allocsize = frame_size + 64*1024; /* add some space for header and trailer */ out_buff = malloc(out_allocsize); if (!out_buff) @@ -665,8 +831,8 @@ int compress_stream_lz4(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_unco if (!in_buff) return -ENOMEM; - n = offset = total_out = LZ4F_compressBegin(ctx, out_buff, out_allocsize, &preferences); - if (LZ4F_isError(n)) + n = offset = total_out = sym_LZ4F_compressBegin(ctx, out_buff, out_allocsize, &preferences); + if (sym_LZ4F_isError(n)) return -EINVAL; log_debug("Buffer size is %zu bytes, header size %zu bytes.", out_allocsize, n); @@ -679,9 +845,9 @@ int compress_stream_lz4(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_unco return k; if (k == 0) break; - n = LZ4F_compressUpdate(ctx, out_buff + offset, out_allocsize - offset, + n = sym_LZ4F_compressUpdate(ctx, out_buff + offset, out_allocsize - offset, in_buff, k, NULL); - if (LZ4F_isError(n)) + if (sym_LZ4F_isError(n)) return -ENOTRECOVERABLE; total_in += k; @@ -700,8 +866,8 @@ int compress_stream_lz4(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_unco } } - n = LZ4F_compressEnd(ctx, out_buff + offset, out_allocsize - offset, NULL); - if (LZ4F_isError(n)) + n = sym_LZ4F_compressEnd(ctx, out_buff + offset, out_allocsize - offset, NULL); + if (sym_LZ4F_isError(n)) return -ENOTRECOVERABLE; offset += n; @@ -724,18 +890,22 @@ int compress_stream_lz4(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_unco } int decompress_stream_xz(int fdf, int fdt, uint64_t max_bytes) { + assert(fdf >= 0); + assert(fdt >= 0); #if HAVE_XZ - _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT; + _cleanup_(lzma_end_wrapper) lzma_stream s = LZMA_STREAM_INIT; lzma_ret ret; uint8_t buf[BUFSIZ], out[BUFSIZ]; lzma_action action = LZMA_RUN; + int r; - assert(fdf >= 0); - assert(fdt >= 0); + r = dlopen_lzma(); + if (r < 0) + return r; - ret = lzma_stream_decoder(&s, UINT64_MAX, 0); + ret = sym_lzma_stream_decoder(&s, UINT64_MAX, 0); if (ret != LZMA_OK) return log_debug_errno(SYNTHETIC_ERRNO(ENOMEM), "Failed to initialize XZ decoder: code %u", @@ -761,7 +931,7 @@ int decompress_stream_xz(int fdf, int fdt, uint64_t max_bytes) { s.avail_out = sizeof(out); } - ret = lzma_code(&s, action); + ret = sym_lzma_code(&s, action); if (!IN_SET(ret, LZMA_OK, LZMA_STREAM_END)) return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG), "Decompression failed: code %u", @@ -801,15 +971,19 @@ int decompress_stream_xz(int fdf, int fdt, uint64_t max_bytes) { int decompress_stream_lz4(int in, int out, uint64_t max_bytes) { #if HAVE_LZ4 size_t c; - _cleanup_(LZ4F_freeDecompressionContextp) LZ4F_decompressionContext_t ctx = NULL; + _cleanup_(sym_LZ4F_freeDecompressionContextp) LZ4F_decompressionContext_t ctx = NULL; _cleanup_free_ char *buf = NULL; char *src; struct stat st; - int r = 0; + int r; size_t total_in = 0, total_out = 0; - c = LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION); - if (LZ4F_isError(c)) + r = dlopen_lz4(); + if (r < 0) + return r; + + c = sym_LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION); + if (sym_LZ4F_isError(c)) return -ENOMEM; if (fstat(in, &st) < 0) @@ -830,8 +1004,8 @@ int decompress_stream_lz4(int in, int out, uint64_t max_bytes) { size_t produced = LZ4_BUFSIZE; size_t used = st.st_size - total_in; - c = LZ4F_decompress(ctx, buf, &produced, src + total_in, &used, NULL); - if (LZ4F_isError(c)) { + c = sym_LZ4F_decompress(ctx, buf, &produced, src + total_in, &used, NULL); + if (sym_LZ4F_isError(c)) { r = -EBADMSG; goto cleanup; } @@ -853,6 +1027,7 @@ int decompress_stream_lz4(int in, int out, uint64_t max_bytes) { log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)", total_in, total_out, total_in > 0 ? (double) total_out / total_in * 100 : 0.0); + r = 0; cleanup: munmap(src, st.st_size); return r; @@ -863,28 +1038,33 @@ int decompress_stream_lz4(int in, int out, uint64_t max_bytes) { } int compress_stream_zstd(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_uncompressed_size) { + assert(fdf >= 0); + assert(fdt >= 0); + #if HAVE_ZSTD - _cleanup_(ZSTD_freeCCtxp) ZSTD_CCtx *cctx = NULL; + _cleanup_(sym_ZSTD_freeCCtxp) ZSTD_CCtx *cctx = NULL; _cleanup_free_ void *in_buff = NULL, *out_buff = NULL; size_t in_allocsize, out_allocsize; size_t z; uint64_t left = max_bytes, in_bytes = 0; + int r; - assert(fdf >= 0); - assert(fdt >= 0); + r = dlopen_zstd(); + if (r < 0) + return r; /* Create the context and buffers */ - in_allocsize = ZSTD_CStreamInSize(); - out_allocsize = ZSTD_CStreamOutSize(); + in_allocsize = sym_ZSTD_CStreamInSize(); + out_allocsize = sym_ZSTD_CStreamOutSize(); in_buff = malloc(in_allocsize); out_buff = malloc(out_allocsize); - cctx = ZSTD_createCCtx(); + cctx = sym_ZSTD_createCCtx(); if (!cctx || !out_buff || !in_buff) return -ENOMEM; - z = ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1); - if (ZSTD_isError(z)) - log_debug("Failed to enable ZSTD checksum, ignoring: %s", ZSTD_getErrorName(z)); + z = sym_ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1); + if (sym_ZSTD_isError(z)) + log_debug("Failed to enable ZSTD checksum, ignoring: %s", sym_ZSTD_getErrorName(z)); /* This loop read from the input file, compresses that entire chunk, * and writes all output produced to the output file. @@ -919,12 +1099,12 @@ int compress_stream_zstd(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_unc * output to the file so we can reuse the buffer next * iteration. */ - remaining = ZSTD_compressStream2( + remaining = sym_ZSTD_compressStream2( cctx, &output, &input, is_last_chunk ? ZSTD_e_end : ZSTD_e_continue); - if (ZSTD_isError(remaining)) { - log_debug("ZSTD encoder failed: %s", ZSTD_getErrorName(remaining)); + if (sym_ZSTD_isError(remaining)) { + log_debug("ZSTD encoder failed: %s", sym_ZSTD_getErrorName(remaining)); return zstd_ret_to_errno(remaining); } @@ -968,22 +1148,26 @@ int compress_stream_zstd(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_unc } int decompress_stream_zstd(int fdf, int fdt, uint64_t max_bytes) { + assert(fdf >= 0); + assert(fdt >= 0); + #if HAVE_ZSTD - _cleanup_(ZSTD_freeDCtxp) ZSTD_DCtx *dctx = NULL; + _cleanup_(sym_ZSTD_freeDCtxp) ZSTD_DCtx *dctx = NULL; _cleanup_free_ void *in_buff = NULL, *out_buff = NULL; size_t in_allocsize, out_allocsize; size_t last_result = 0; uint64_t left = max_bytes, in_bytes = 0; + int r; - assert(fdf >= 0); - assert(fdt >= 0); - + r = dlopen_zstd(); + if (r < 0) + return r; /* Create the context and buffers */ - in_allocsize = ZSTD_DStreamInSize(); - out_allocsize = ZSTD_DStreamOutSize(); + in_allocsize = sym_ZSTD_DStreamInSize(); + out_allocsize = sym_ZSTD_DStreamOutSize(); in_buff = malloc(in_allocsize); out_buff = malloc(out_allocsize); - dctx = ZSTD_createDCtx(); + dctx = sym_ZSTD_createDCtx(); if (!dctx || !out_buff || !in_buff) return -ENOMEM; @@ -1032,8 +1216,8 @@ int decompress_stream_zstd(int fdf, int fdt, uint64_t max_bytes) { * for instance if the last decompression call returned * an error. */ - last_result = ZSTD_decompressStream(dctx, &output, &input); - if (ZSTD_isError(last_result)) { + last_result = sym_ZSTD_decompressStream(dctx, &output, &input); + if (sym_ZSTD_isError(last_result)) { has_error = true; break; } @@ -1059,7 +1243,7 @@ int decompress_stream_zstd(int fdf, int fdt, uint64_t max_bytes) { * on a frame, but we reached the end of the file! We assume * this is an error, and the input was truncated. */ - log_debug("ZSTD decoder failed: %s", ZSTD_getErrorName(last_result)); + log_debug("ZSTD decoder failed: %s", sym_ZSTD_getErrorName(last_result)); return zstd_ret_to_errno(last_result); } diff --git a/src/basic/compress.h b/src/basic/compress.h index 1b5c645e327..76ffe2ad428 100644 --- a/src/basic/compress.h +++ b/src/basic/compress.h @@ -6,6 +6,13 @@ #include #include +#if HAVE_LZ4 +#include +#include +#endif + +#include "dlfcn-util.h" + typedef enum Compression { COMPRESSION_NONE, COMPRESSION_XZ, @@ -63,6 +70,24 @@ int decompress_stream_xz(int fdf, int fdt, uint64_t max_size); int decompress_stream_lz4(int fdf, int fdt, uint64_t max_size); int decompress_stream_zstd(int fdf, int fdt, uint64_t max_size); +#if HAVE_LZ4 +DLSYM_PROTOTYPE(LZ4_compress_default); +DLSYM_PROTOTYPE(LZ4_decompress_safe); +DLSYM_PROTOTYPE(LZ4_decompress_safe_partial); +DLSYM_PROTOTYPE(LZ4_versionNumber); + +int dlopen_lz4(void); +#endif + +#if HAVE_ZSTD +int dlopen_zstd(void); +#endif + +#if HAVE_XZ +int dlopen_lzma(void); +#endif + + static inline int compress_blob( Compression compression, const void *src, uint64_t src_size, diff --git a/src/basic/meson.build b/src/basic/meson.build index b5c99383702..9cde549bc54 100644 --- a/src/basic/meson.build +++ b/src/basic/meson.build @@ -17,6 +17,7 @@ basic_sources = files( 'cgroup-util.c', 'chase.c', 'chattr-util.c', + 'compress.c', 'conf-files.c', 'confidential-virt.c', 'devnum-util.c', @@ -275,8 +276,11 @@ libbasic = static_library( fundamental_sources, include_directories : basic_includes, dependencies : [libcap, + liblz4_cflags, libm, librt, + libxz_cflags, + libzstd_cflags, threads, userspace], c_args : ['-fvisibility=default'], @@ -300,20 +304,3 @@ libbasic_gcrypt = static_library( build_by_default : false) ############################################################ - -basic_compress_sources = files( - 'compress.c', -) - -# A convenience library that is separate from libbasic to avoid unnecessary -# linking to the compression libraries. -libbasic_compress = static_library( - 'basic-compress', - basic_compress_sources, - include_directories : basic_includes, - dependencies : [liblz4, - libxz, - libzstd, - userspace], - c_args : ['-fvisibility=default'], - build_by_default : false) diff --git a/src/boot/efi/meson.build b/src/boot/efi/meson.build index b5d4e963032..6194099fe40 100644 --- a/src/boot/efi/meson.build +++ b/src/boot/efi/meson.build @@ -28,7 +28,7 @@ efi_fuzz_template = fuzz_template + efitest_base executables += [ efi_test_template + { 'sources' : files('test-bcd.c'), - 'dependencies' : libzstd, + 'dependencies' : libzstd_cflags, 'conditions' : ['ENABLE_BOOTLOADER', 'HAVE_ZSTD'], }, efi_test_template + { diff --git a/src/coredump/meson.build b/src/coredump/meson.build index a69974672cc..bb81149b60f 100644 --- a/src/coredump/meson.build +++ b/src/coredump/meson.build @@ -5,15 +5,10 @@ systemd_coredump_sources = files( 'coredump-vacuum.c', ) -common_link_with = [ - libshared, - libbasic_compress, -] - common_dependencies = [ - liblz4, - libxz, - libzstd, + liblz4_cflags, + libxz_cflags, + libzstd_cflags, threads, ] @@ -22,7 +17,7 @@ executables += [ 'name' : 'systemd-coredump', 'conditions' : ['ENABLE_COREDUMP'], 'sources' : systemd_coredump_sources, - 'link_with' : common_link_with, + 'link_with' : [libshared], 'dependencies' : common_dependencies + [libacl], }, executable_template + { @@ -30,7 +25,7 @@ executables += [ 'public' : true, 'conditions' : ['ENABLE_COREDUMP'], 'sources' : files('coredumpctl.c'), - 'link_with' : common_link_with, + 'link_with' : [libshared], 'dependencies' : common_dependencies, }, test_template + { diff --git a/src/journal-remote/meson.build b/src/journal-remote/meson.build index 964a2517744..10a82751d77 100644 --- a/src/journal-remote/meson.build +++ b/src/journal-remote/meson.build @@ -22,9 +22,9 @@ libsystemd_journal_remote = static_library( libsystemd_journal_remote_sources, include_directories : includes, dependencies : [libgnutls, - liblz4, + liblz4_cflags, libmicrohttpd, - libxz, + libxz_cflags, threads, userspace], build_by_default : false) @@ -38,9 +38,9 @@ systemd_journal_gatewayd_sources = files( common_deps = [ libgnutls, - liblz4, - libxz, - libzstd, + liblz4_cflags, + libxz_cflags, + libzstd_cflags, threads, ] diff --git a/src/journal/meson.build b/src/journal/meson.build index a3c57106101..1811feb02b8 100644 --- a/src/journal/meson.build +++ b/src/journal/meson.build @@ -63,10 +63,10 @@ executables += [ libshared, ], 'dependencies' : [ - liblz4, + liblz4_cflags, libselinux, - libxz, - libzstd, + libxz_cflags, + libzstd_cflags, threads, ], }, @@ -95,26 +95,26 @@ executables += [ 'link_with' : journalctl_link_with, 'dependencies' : [ libdl, - liblz4, - libxz, - libzstd, + liblz4_cflags, + libxz_cflags, + libzstd_cflags, threads, ], }, journal_test_template + { 'sources' : files('test-journald-config.c'), 'dependencies' : [ - liblz4, + liblz4_cflags, libselinux, - libxz, + libxz_cflags, ], }, journal_test_template + { 'sources' : files('test-journald-syslog.c'), 'dependencies' : [ - liblz4, + liblz4_cflags, libselinux, - libxz, + libxz_cflags, threads, ], }, diff --git a/src/libsystemd/meson.build b/src/libsystemd/meson.build index 5d18f974bae..817e3a5e090 100644 --- a/src/libsystemd/meson.build +++ b/src/libsystemd/meson.build @@ -118,8 +118,7 @@ libsystemd_static = static_library( libsystemd_sources, include_directories : libsystemd_includes, c_args : libsystemd_c_args, - link_with : [libbasic, - libbasic_compress], + link_with : [libbasic], dependencies : [threads, librt, userspace], diff --git a/src/login/meson.build b/src/login/meson.build index b5bb1502584..43db03184c5 100644 --- a/src/login/meson.build +++ b/src/login/meson.build @@ -65,9 +65,9 @@ executables += [ 'conditions' : ['ENABLE_LOGIND'], 'sources' : loginctl_sources, 'dependencies' : [ - liblz4, - libxz, - libzstd, + liblz4_cflags, + libxz_cflags, + libzstd_cflags, threads, ], }, diff --git a/src/machine/meson.build b/src/machine/meson.build index b3a1ffce8fc..c82a32589d0 100644 --- a/src/machine/meson.build +++ b/src/machine/meson.build @@ -35,9 +35,9 @@ executables += [ 'conditions' : ['ENABLE_MACHINED'], 'sources' : files('machinectl.c'), 'dependencies' : [ - liblz4, - libxz, - libzstd, + liblz4_cflags, + libxz_cflags, + libzstd_cflags, threads, ], }, diff --git a/src/pstore/meson.build b/src/pstore/meson.build index b6fda8723d5..27c2855fb8a 100644 --- a/src/pstore/meson.build +++ b/src/pstore/meson.build @@ -7,9 +7,9 @@ executables += [ 'sources' : files('pstore.c'), 'dependencies' : [ libacl, - liblz4, - libxz, - libzstd, + liblz4_cflags, + libxz_cflags, + libzstd_cflags, threads, ], }, diff --git a/src/shared/meson.build b/src/shared/meson.build index 7d1dee68ab2..d540631f211 100644 --- a/src/shared/meson.build +++ b/src/shared/meson.build @@ -324,7 +324,7 @@ libshared_deps = [threads, libgcrypt, libiptc_cflags, libkmod, - liblz4, + liblz4_cflags, libmount, libopenssl, libp11kit_cflags, @@ -333,8 +333,8 @@ libshared_deps = [threads, libseccomp, libselinux, libxenctrl_cflags, - libxz, - libzstd] + libxz_cflags, + libzstd_cflags] libshared_sym_path = meson.current_source_dir() / 'libshared.sym' libshared_build_dir = meson.current_build_dir() diff --git a/src/systemctl/meson.build b/src/systemctl/meson.build index 255c639b5f6..257c362cea4 100644 --- a/src/systemctl/meson.build +++ b/src/systemctl/meson.build @@ -56,10 +56,10 @@ executables += [ 'link_with' : systemctl_link_with, 'dependencies' : [ libcap, - liblz4, + liblz4_cflags, libselinux, - libxz, - libzstd, + libxz_cflags, + libzstd_cflags, threads, ], }, diff --git a/src/test/meson.build b/src/test/meson.build index a0f38824808..426a54fd29d 100644 --- a/src/test/meson.build +++ b/src/test/meson.build @@ -247,18 +247,12 @@ executables += [ }, test_template + { 'sources' : files('test-compress-benchmark.c'), - 'link_with' : [ - libbasic_compress, - libshared, - ], + 'link_with' : [libshared], 'timeout' : 90, }, test_template + { 'sources' : files('test-compress.c'), - 'link_with' : [ - libbasic_compress, - libshared, - ], + 'link_with' : [libshared], }, test_template + { 'sources' : files('test-cryptolib.c'), diff --git a/src/test/test-compress.c b/src/test/test-compress.c index 2f20d001167..57cd77d4e50 100644 --- a/src/test/test-compress.c +++ b/src/test/test-compress.c @@ -6,6 +6,7 @@ #include #endif +#include "dlfcn-util.h" #include "alloc-util.h" #include "compress.h" #include "fd-util.h" @@ -241,16 +242,16 @@ static void test_lz4_decompress_partial(void) { memset(&huge[STRLEN("HUGE=")], 'x', HUGE_SIZE - STRLEN("HUGE=") - 1); huge[HUGE_SIZE - 1] = '\0'; - r = LZ4_compress_default(huge, buf, HUGE_SIZE, buf_size); + r = sym_LZ4_compress_default(huge, buf, HUGE_SIZE, buf_size); assert_se(r >= 0); compressed = r; log_info("Compressed %i → %zu", HUGE_SIZE, compressed); - r = LZ4_decompress_safe(buf, huge, r, HUGE_SIZE); + r = sym_LZ4_decompress_safe(buf, huge, r, HUGE_SIZE); assert_se(r >= 0); log_info("Decompressed → %i", r); - r = LZ4_decompress_safe_partial(buf, huge, + r = sym_LZ4_decompress_safe_partial(buf, huge, compressed, 12, HUGE_SIZE); assert_se(r >= 0); @@ -258,10 +259,10 @@ static void test_lz4_decompress_partial(void) { for (size_t size = 1; size < sizeof(buf2); size++) { /* This failed in older lz4s but works in newer ones. */ - r = LZ4_decompress_safe_partial(buf, buf2, compressed, size, size); + r = sym_LZ4_decompress_safe_partial(buf, buf2, compressed, size, size); log_info("Decompressed partial %zu/%zu → %i (%s)", size, size, r, r < 0 ? "bad" : "good"); - if (r >= 0 && LZ4_versionNumber() >= 10803) + if (r >= 0 && sym_LZ4_versionNumber() >= 10803) /* lz4 <= 1.8.2 should fail that test, let's only check for newer ones */ assert_se(memcmp(buf2, huge, r) == 0); } @@ -316,28 +317,30 @@ int main(int argc, char *argv[]) { #endif #if HAVE_LZ4 - test_compress_decompress("LZ4", compress_blob_lz4, decompress_blob_lz4, - text, sizeof(text), false); - test_compress_decompress("LZ4", compress_blob_lz4, decompress_blob_lz4, - data, sizeof(data), true); - - test_decompress_startswith("LZ4", - compress_blob_lz4, decompress_startswith_lz4, - text, sizeof(text), false); - test_decompress_startswith("LZ4", - compress_blob_lz4, decompress_startswith_lz4, - data, sizeof(data), true); - test_decompress_startswith("LZ4", - compress_blob_lz4, decompress_startswith_lz4, - huge, HUGE_SIZE, true); - - test_compress_stream("LZ4", "lz4cat", - compress_stream_lz4, decompress_stream_lz4, srcfile); - - test_lz4_decompress_partial(); - - test_decompress_startswith_short("LZ4", compress_blob_lz4, decompress_startswith_lz4); - + if (dlopen_lz4() >= 0) { + test_compress_decompress("LZ4", compress_blob_lz4, decompress_blob_lz4, + text, sizeof(text), false); + test_compress_decompress("LZ4", compress_blob_lz4, decompress_blob_lz4, + data, sizeof(data), true); + + test_decompress_startswith("LZ4", + compress_blob_lz4, decompress_startswith_lz4, + text, sizeof(text), false); + test_decompress_startswith("LZ4", + compress_blob_lz4, decompress_startswith_lz4, + data, sizeof(data), true); + test_decompress_startswith("LZ4", + compress_blob_lz4, decompress_startswith_lz4, + huge, HUGE_SIZE, true); + + test_compress_stream("LZ4", "lz4cat", + compress_stream_lz4, decompress_stream_lz4, srcfile); + + test_lz4_decompress_partial(); + + test_decompress_startswith_short("LZ4", compress_blob_lz4, decompress_startswith_lz4); + } else + log_error("/* Can't load liblz4 */"); #else log_info("/* LZ4 test skipped */"); #endif diff --git a/src/test/test-dlopen-so.c b/src/test/test-dlopen-so.c index d2dbe729818..3232506d5a1 100644 --- a/src/test/test-dlopen-so.c +++ b/src/test/test-dlopen-so.c @@ -4,6 +4,7 @@ #include #include "bpf-dlopen.h" +#include "compress.h" #include "cryptsetup-util.h" #include "elf-util.h" #include "idn-util.h" @@ -75,6 +76,18 @@ static int run(int argc, char **argv) { assert_se(dlopen_libarchive() >= 0); #endif +#if HAVE_LZ4 + assert_se(dlopen_lz4() >= 0); +#endif + +#if HAVE_ZSTD + assert_se(dlopen_zstd() >= 0); +#endif + +#if HAVE_XZ + assert_se(dlopen_lzma() >= 0); +#endif + return 0; }