/* SPDX-License-Identifier: LGPL-2.1-or-later */
+#include <fcntl.h>
#include <stdio.h>
#include <sys/mman.h>
#include <sys/stat.h>
#endif
}
+#if HAVE_COMPRESSION
+/* Determine whether sparse writes should be used for this fd. Sparse writes are only safe on
+ * regular files without O_APPEND (O_APPEND ignores lseek position, which would collapse holes). */
+static int should_sparse(int fd) {
+ struct stat st;
+
+ assert(fd >= 0);
+
+ if (fstat(fd, &st) < 0)
+ return -errno;
+
+ int flags = fcntl(fd, F_GETFL);
+ if (flags < 0)
+ return -errno;
+
+ return S_ISREG(st.st_mode) && !FLAGS_SET(flags, O_APPEND);
+}
+
+/* After sparse decompression, set the file size to the current position to account for
+ * trailing holes that sparse_write() created via lseek but never extended the file size for. */
+static int finalize_sparse(int fd) {
+ off_t pos;
+
+ assert(fd >= 0);
+
+ pos = lseek(fd, 0, SEEK_CUR);
+ if (pos < 0)
+ return -errno;
+
+ if (ftruncate(fd, pos) < 0)
+ return -errno;
+
+ return 0;
+}
+
+static int maybe_sparse_write(int fd, const void *buf, size_t nbytes, bool sparse) {
+ int r;
+
+ if (sparse) {
+ ssize_t k;
+
+ /* Note: sparse_write() does not retry on EINTR and converts short writes to -EIO.
+ * This is fine here since sparse mode is only used on regular files, where short
+ * writes and EINTR are not expected in practice. */
+ k = sparse_write(fd, buf, nbytes, 64);
+ if (k < 0)
+ return (int) k;
+ } else {
+ r = loop_write_full(fd, buf, nbytes, USEC_INFINITY);
+ if (r < 0)
+ return r;
+ }
+
+ return 0;
+}
+#endif
+
int decompress_stream_xz(int fdf, int fdt, uint64_t max_bytes) {
assert(fdf >= 0);
assert(fdt >= 0);
#if HAVE_XZ
+ bool sparse = should_sparse(fdt) > 0;
int r;
r = dlopen_lzma();
max_bytes -= n;
}
- k = loop_write(fdt, out, n);
+ k = maybe_sparse_write(fdt, out, n, sparse);
if (k < 0)
return k;
s.total_in, s.total_out,
(double) s.total_out / s.total_in * 100);
- return 0;
+ return sparse ? finalize_sparse(fdt) : 0;
}
}
}
_cleanup_free_ char *buf = NULL;
char *src;
struct stat st;
+ bool sparse = should_sparse(fdt) > 0;
int r;
size_t total_in = 0, total_out = 0;
goto cleanup;
}
- r = loop_write(fdt, buf, produced);
+ r = maybe_sparse_write(fdt, buf, produced, sparse);
if (r < 0)
goto cleanup;
}
log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)",
total_in, total_out,
(double) total_out / total_in * 100);
- r = 0;
+ r = sparse ? finalize_sparse(fdt) : 0;
cleanup:
munmap(src, st.st_size);
return r;
#if HAVE_ZSTD
_cleanup_(ZSTD_freeDCtxp) ZSTD_DCtx *dctx = NULL;
_cleanup_free_ void *in_buff = NULL, *out_buff = NULL;
+ bool sparse = should_sparse(fdt) > 0;
size_t in_allocsize, out_allocsize;
size_t last_result = 0;
uint64_t left = max_bytes, in_bytes = 0;
if (left < output.pos)
return -EFBIG;
- wrote = loop_write_full(fdt, output.dst, output.pos, USEC_INFINITY);
+ wrote = maybe_sparse_write(fdt, output.dst, output.pos, sparse);
if (wrote < 0)
return wrote;
in_bytes,
max_bytes - left,
(double) (max_bytes - left) / in_bytes * 100);
- return 0;
+ return sparse ? finalize_sparse(fdt) : 0;
#else
return log_debug_errno(SYNTHETIC_ERRNO(EPROTONOSUPPORT),
"Cannot decompress file. Compiled without ZSTD support.");
#include "compress.h"
#include "dlfcn-util.h"
#include "fd-util.h"
+#include "io-util.h"
#include "path-util.h"
#include "random-util.h"
#include "tests.h"
r = decompress(dst, dst2, st.st_size - 1);
assert_se(r == -EFBIG);
}
+
+_unused_ static void test_decompress_stream_sparse(const char *compression,
+ compress_stream_t compress,
+ decompress_stream_t decompress) {
+
+ _cleanup_close_ int src = -EBADF, compressed = -EBADF, decompressed = -EBADF;
+ _cleanup_(unlink_tempfilep) char
+ pattern_src[] = "/tmp/systemd-test.sparse-src.XXXXXX",
+ pattern_compressed[] = "/tmp/systemd-test.sparse-compressed.XXXXXX",
+ pattern_decompressed[] = "/tmp/systemd-test.sparse-decompressed.XXXXXX";
+ /* Create a sparse-like input: 4K of data, 64K of zeros, 4K of data, 64K trailing zeros.
+ * Total apparent size: 136K, but most of it is zeros. */
+ uint8_t data_block[4096];
+ struct stat st_src, st_decompressed;
+ uint64_t uncompressed_size;
+ int r;
+
+ assert(compression);
+
+ log_debug("/* testing %s sparse decompression */", compression);
+
+ random_bytes(data_block, sizeof(data_block));
+
+ assert_se((src = mkostemp_safe(pattern_src)) >= 0);
+
+ /* Write: 4K data, 64K zeros, 4K data, 64K zeros */
+ assert_se(loop_write(src, data_block, sizeof(data_block)) >= 0);
+ assert_se(ftruncate(src, sizeof(data_block) + 65536) >= 0);
+ assert_se(lseek(src, sizeof(data_block) + 65536, SEEK_SET) >= 0);
+ assert_se(loop_write(src, data_block, sizeof(data_block)) >= 0);
+ assert_se(ftruncate(src, 2 * sizeof(data_block) + 2 * 65536) >= 0);
+ assert_se(lseek(src, 0, SEEK_SET) == 0);
+
+ assert_se(fstat(src, &st_src) >= 0);
+ assert_se(st_src.st_size == 2 * (off_t) sizeof(data_block) + 2 * 65536);
+
+ /* Compress */
+ assert_se((compressed = mkostemp_safe(pattern_compressed)) >= 0);
+ ASSERT_OK(compress(src, compressed, -1, &uncompressed_size));
+ assert_se((uint64_t) st_src.st_size == uncompressed_size);
+
+ /* Decompress to a regular file (sparse writes auto-detected) */
+ assert_se((decompressed = mkostemp_safe(pattern_decompressed)) >= 0);
+ assert_se(lseek(compressed, 0, SEEK_SET) == 0);
+ r = decompress(compressed, decompressed, st_src.st_size);
+ assert_se(r == 0);
+
+ /* Verify apparent size matches */
+ assert_se(fstat(decompressed, &st_decompressed) >= 0);
+ assert_se(st_decompressed.st_size == st_src.st_size);
+
+ /* Verify content matches by comparing bytes */
+ assert_se(lseek(src, 0, SEEK_SET) == 0);
+ assert_se(lseek(decompressed, 0, SEEK_SET) == 0);
+
+ for (off_t offset = 0; offset < st_src.st_size;) {
+ uint8_t buf_src[4096], buf_dst[4096];
+ size_t to_read = MIN((size_t) (st_src.st_size - offset), sizeof(buf_src));
+ ssize_t n;
+
+ n = loop_read(src, buf_src, to_read, true);
+ assert_se(n == (ssize_t) to_read);
+ n = loop_read(decompressed, buf_dst, to_read, true);
+ assert_se(n == (ssize_t) to_read);
+ assert_se(memcmp(buf_src, buf_dst, to_read) == 0);
+ offset += to_read;
+ }
+
+ /* Verify the decompressed file is actually sparse (uses less disk than apparent size).
+ * st_blocks is in 512-byte units. The file has 128K of zeros, so disk usage should be
+ * noticeably less than the apparent size if sparse writes worked.
+ * Only assert if the filesystem supports holes (SEEK_HOLE). */
+ log_debug("%s sparse decompression: apparent=%jd disk=%jd",
+ compression,
+ (intmax_t) st_decompressed.st_size,
+ (intmax_t) st_decompressed.st_blocks * 512);
+ if (lseek(decompressed, 0, SEEK_HOLE) < st_decompressed.st_size)
+ assert_se(st_decompressed.st_blocks * 512 < st_decompressed.st_size);
+ else
+ log_debug("Filesystem does not support holes, skipping sparsity check");
+
+ /* Test all-zeros input: entire output should be a hole */
+ log_debug("/* testing %s sparse decompression of all-zeros */", compression);
+ {
+ _cleanup_close_ int zsrc = -EBADF, zcompressed = -EBADF, zdecompressed = -EBADF;
+ _cleanup_(unlink_tempfilep) char
+ zp_src[] = "/tmp/systemd-test.sparse-zero-src.XXXXXX",
+ zp_compressed[] = "/tmp/systemd-test.sparse-zero-compressed.XXXXXX",
+ zp_decompressed[] = "/tmp/systemd-test.sparse-zero-decompressed.XXXXXX";
+ struct stat zst;
+ uint64_t zsize;
+ uint8_t zeros[65536] = {};
+
+ assert_se((zsrc = mkostemp_safe(zp_src)) >= 0);
+ assert_se(loop_write(zsrc, zeros, sizeof(zeros)) >= 0);
+ assert_se(lseek(zsrc, 0, SEEK_SET) == 0);
+
+ assert_se((zcompressed = mkostemp_safe(zp_compressed)) >= 0);
+ ASSERT_OK(compress(zsrc, zcompressed, -1, &zsize));
+ assert_se(zsize == sizeof(zeros));
+
+ assert_se((zdecompressed = mkostemp_safe(zp_decompressed)) >= 0);
+ assert_se(lseek(zcompressed, 0, SEEK_SET) == 0);
+ assert_se(decompress(zcompressed, zdecompressed, sizeof(zeros)) == 0);
+
+ assert_se(fstat(zdecompressed, &zst) >= 0);
+ assert_se(zst.st_size == (off_t) sizeof(zeros));
+ /* All zeros — disk usage should be minimal */
+ log_debug("%s all-zeros sparse: apparent=%jd disk=%jd",
+ compression, (intmax_t) zst.st_size, (intmax_t) zst.st_blocks * 512);
+ if (lseek(zdecompressed, 0, SEEK_HOLE) < zst.st_size)
+ assert_se(zst.st_blocks * 512 < zst.st_size);
+ else
+ log_debug("Filesystem does not support holes, skipping sparsity check");
+ }
+
+ /* Test data ending with non-zero bytes: ftruncate should be a no-op */
+ log_debug("/* testing %s sparse decompression ending with data */", compression);
+ {
+ _cleanup_close_ int dsrc = -EBADF, dcompressed = -EBADF, ddecompressed = -EBADF;
+ _cleanup_(unlink_tempfilep) char
+ dp_src[] = "/tmp/systemd-test.sparse-end-src.XXXXXX",
+ dp_compressed[] = "/tmp/systemd-test.sparse-end-compressed.XXXXXX",
+ dp_decompressed[] = "/tmp/systemd-test.sparse-end-decompressed.XXXXXX";
+ struct stat dst;
+ uint64_t dsize;
+ uint8_t zeros[65536] = {};
+
+ /* 64K zeros followed by 4K random data */
+ assert_se((dsrc = mkostemp_safe(dp_src)) >= 0);
+ assert_se(loop_write(dsrc, zeros, sizeof(zeros)) >= 0);
+ assert_se(loop_write(dsrc, data_block, sizeof(data_block)) >= 0);
+ assert_se(lseek(dsrc, 0, SEEK_SET) == 0);
+
+ assert_se((dcompressed = mkostemp_safe(dp_compressed)) >= 0);
+ ASSERT_OK(compress(dsrc, dcompressed, -1, &dsize));
+ assert_se(dsize == sizeof(zeros) + sizeof(data_block));
+
+ assert_se((ddecompressed = mkostemp_safe(dp_decompressed)) >= 0);
+ assert_se(lseek(dcompressed, 0, SEEK_SET) == 0);
+ assert_se(decompress(dcompressed, ddecompressed, dsize) == 0);
+
+ assert_se(fstat(ddecompressed, &dst) >= 0);
+ assert_se(dst.st_size == (off_t)(sizeof(zeros) + sizeof(data_block)));
+ }
+}
#endif
#if HAVE_LZ4
test_compress_stream("XZ", "xzcat",
compress_stream_xz, decompress_stream_xz, srcfile);
+ test_decompress_stream_sparse("XZ", compress_stream_xz, decompress_stream_xz);
+
test_decompress_startswith_short("XZ", compress_blob_xz, decompress_startswith_xz);
#else
test_compress_stream("LZ4", "lz4cat",
compress_stream_lz4, decompress_stream_lz4, srcfile);
+ test_decompress_stream_sparse("LZ4", compress_stream_lz4, decompress_stream_lz4);
+
test_lz4_decompress_partial();
test_decompress_startswith_short("LZ4", compress_blob_lz4, decompress_startswith_lz4);
test_compress_stream("ZSTD", "zstdcat",
compress_stream_zstd, decompress_stream_zstd, srcfile);
+ test_decompress_stream_sparse("ZSTD", compress_stream_zstd, decompress_stream_zstd);
+
test_decompress_startswith_short("ZSTD", compress_blob_zstd, decompress_startswith_zstd);
#else
log_info("/* ZSTD test skipped */");
projects / thirdparty / systemd.git / commitdiff
