1 /* SPDX-License-Identifier: LGPL-2.1-or-later */
21 #include <zstd_errors.h>
24 #include "alloc-util.h"
28 #include "journal-def.h"
30 #include "sparse-endian.h"
31 #include "string-table.h"
32 #include "string-util.h"
33 #include "unaligned.h"
37 DEFINE_TRIVIAL_CLEANUP_FUNC(LZ4F_compressionContext_t
, LZ4F_freeCompressionContext
);
38 DEFINE_TRIVIAL_CLEANUP_FUNC(LZ4F_decompressionContext_t
, LZ4F_freeDecompressionContext
);
42 DEFINE_TRIVIAL_CLEANUP_FUNC(ZSTD_CCtx
*, ZSTD_freeCCtx
);
43 DEFINE_TRIVIAL_CLEANUP_FUNC(ZSTD_DCtx
*, ZSTD_freeDCtx
);
45 static int zstd_ret_to_errno(size_t ret
) {
46 switch (ZSTD_getErrorCode(ret
)) {
47 case ZSTD_error_dstSize_tooSmall
:
49 case ZSTD_error_memory_allocation
:
57 #define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t))
59 static const char* const object_compressed_table
[_OBJECT_COMPRESSED_MAX
] = {
60 [OBJECT_COMPRESSED_XZ
] = "XZ",
61 [OBJECT_COMPRESSED_LZ4
] = "LZ4",
62 [OBJECT_COMPRESSED_ZSTD
] = "ZSTD",
63 /* If we add too many more entries here, it's going to grow quite large (and be mostly sparse), since
64 * the array key is actually a bitmask, not a plain enum */
67 DEFINE_STRING_TABLE_LOOKUP(object_compressed
, int);
69 int compress_blob_xz(const void *src
, uint64_t src_size
,
70 void *dst
, size_t dst_alloc_size
, size_t *dst_size
) {
72 static const lzma_options_lzma opt
= {
73 1u << 20u, NULL
, 0, LZMA_LC_DEFAULT
, LZMA_LP_DEFAULT
,
74 LZMA_PB_DEFAULT
, LZMA_MODE_FAST
, 128, LZMA_MF_HC3
, 4
76 static const lzma_filter filters
[] = {
77 { LZMA_FILTER_LZMA2
, (lzma_options_lzma
*) &opt
},
78 { LZMA_VLI_UNKNOWN
, NULL
}
86 assert(dst_alloc_size
> 0);
89 /* Returns < 0 if we couldn't compress the data or the
90 * compressed result is longer than the original */
95 ret
= lzma_stream_buffer_encode((lzma_filter
*) filters
, LZMA_CHECK_NONE
, NULL
,
96 src
, src_size
, dst
, &out_pos
, dst_alloc_size
);
103 return -EPROTONOSUPPORT
;
107 int compress_blob_lz4(const void *src
, uint64_t src_size
,
108 void *dst
, size_t dst_alloc_size
, size_t *dst_size
) {
113 assert(src_size
> 0);
115 assert(dst_alloc_size
> 0);
118 /* Returns < 0 if we couldn't compress the data or the
119 * compressed result is longer than the original */
124 r
= LZ4_compress_default(src
, (char*)dst
+ 8, src_size
, (int) dst_alloc_size
- 8);
128 unaligned_write_le64(dst
, src_size
);
133 return -EPROTONOSUPPORT
;
137 int compress_blob_zstd(
138 const void *src
, uint64_t src_size
,
139 void *dst
, size_t dst_alloc_size
, size_t *dst_size
) {
144 assert(src_size
> 0);
146 assert(dst_alloc_size
> 0);
149 k
= ZSTD_compress(dst
, dst_alloc_size
, src
, src_size
, 0);
151 return zstd_ret_to_errno(k
);
156 return -EPROTONOSUPPORT
;
160 int decompress_blob_xz(const void *src
, uint64_t src_size
,
161 void **dst
, size_t *dst_alloc_size
, size_t* dst_size
, size_t dst_max
) {
164 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
169 assert(src_size
> 0);
171 assert(dst_alloc_size
);
173 assert(*dst_alloc_size
== 0 || *dst
);
175 ret
= lzma_stream_decoder(&s
, UINT64_MAX
, 0);
179 space
= MIN(src_size
* 2, dst_max
?: (size_t) -1);
180 if (!greedy_realloc(dst
, dst_alloc_size
, space
, 1))
184 s
.avail_in
= src_size
;
192 ret
= lzma_code(&s
, LZMA_FINISH
);
194 if (ret
== LZMA_STREAM_END
)
196 else if (ret
!= LZMA_OK
)
199 if (dst_max
> 0 && (space
- s
.avail_out
) >= dst_max
)
201 else if (dst_max
> 0 && space
== dst_max
)
204 used
= space
- s
.avail_out
;
205 space
= MIN(2 * space
, dst_max
?: (size_t) -1);
206 if (!greedy_realloc(dst
, dst_alloc_size
, space
, 1))
209 s
.avail_out
= space
- used
;
210 s
.next_out
= *(uint8_t**)dst
+ used
;
213 *dst_size
= space
- s
.avail_out
;
216 return -EPROTONOSUPPORT
;
220 int decompress_blob_lz4(const void *src
, uint64_t src_size
,
221 void **dst
, size_t *dst_alloc_size
, size_t* dst_size
, size_t dst_max
) {
225 int r
, size
; /* LZ4 uses int for size */
228 assert(src_size
> 0);
230 assert(dst_alloc_size
);
232 assert(*dst_alloc_size
== 0 || *dst
);
237 size
= unaligned_read_le64(src
);
238 if (size
< 0 || (unsigned) size
!= unaligned_read_le64(src
))
240 if ((size_t) size
> *dst_alloc_size
) {
241 out
= realloc(*dst
, size
);
245 *dst_alloc_size
= size
;
249 r
= LZ4_decompress_safe((char*)src
+ 8, out
, src_size
- 8, size
);
250 if (r
< 0 || r
!= size
)
256 return -EPROTONOSUPPORT
;
260 int decompress_blob_zstd(
261 const void *src
, uint64_t src_size
,
262 void **dst
, size_t *dst_alloc_size
, size_t *dst_size
, size_t dst_max
) {
268 assert(src_size
> 0);
270 assert(dst_alloc_size
);
272 assert(*dst_alloc_size
== 0 || *dst
);
274 size
= ZSTD_getFrameContentSize(src
, src_size
);
275 if (IN_SET(size
, ZSTD_CONTENTSIZE_ERROR
, ZSTD_CONTENTSIZE_UNKNOWN
))
278 if (dst_max
> 0 && size
> dst_max
)
283 if (!(greedy_realloc(dst
, dst_alloc_size
, MAX(ZSTD_DStreamOutSize(), size
), 1)))
286 _cleanup_(ZSTD_freeDCtxp
) ZSTD_DCtx
*dctx
= ZSTD_createDCtx();
290 ZSTD_inBuffer input
= {
294 ZSTD_outBuffer output
= {
296 .size
= *dst_alloc_size
,
299 size_t k
= ZSTD_decompressStream(dctx
, &output
, &input
);
300 if (ZSTD_isError(k
)) {
301 log_debug("ZSTD decoder failed: %s", ZSTD_getErrorName(k
));
302 return zstd_ret_to_errno(k
);
304 assert(output
.pos
>= size
);
309 return -EPROTONOSUPPORT
;
315 const void *src
, uint64_t src_size
,
316 void **dst
, size_t *dst_alloc_size
, size_t* dst_size
, size_t dst_max
) {
318 if (compression
== OBJECT_COMPRESSED_XZ
)
319 return decompress_blob_xz(
321 dst
, dst_alloc_size
, dst_size
, dst_max
);
322 else if (compression
== OBJECT_COMPRESSED_LZ4
)
323 return decompress_blob_lz4(
325 dst
, dst_alloc_size
, dst_size
, dst_max
);
326 else if (compression
== OBJECT_COMPRESSED_ZSTD
)
327 return decompress_blob_zstd(
329 dst
, dst_alloc_size
, dst_size
, dst_max
);
331 return -EPROTONOSUPPORT
;
334 int decompress_startswith_xz(const void *src
, uint64_t src_size
,
335 void **buffer
, size_t *buffer_size
,
336 const void *prefix
, size_t prefix_len
,
340 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
343 /* Checks whether the decompressed blob starts with the
344 * mentioned prefix. The byte extra needs to follow the
348 assert(src_size
> 0);
352 assert(*buffer_size
== 0 || *buffer
);
354 ret
= lzma_stream_decoder(&s
, UINT64_MAX
, 0);
358 if (!(greedy_realloc(buffer
, buffer_size
, ALIGN_8(prefix_len
+ 1), 1)))
362 s
.avail_in
= src_size
;
364 s
.next_out
= *buffer
;
365 s
.avail_out
= *buffer_size
;
368 ret
= lzma_code(&s
, LZMA_FINISH
);
370 if (!IN_SET(ret
, LZMA_OK
, LZMA_STREAM_END
))
373 if (*buffer_size
- s
.avail_out
>= prefix_len
+ 1)
374 return memcmp(*buffer
, prefix
, prefix_len
) == 0 &&
375 ((const uint8_t*) *buffer
)[prefix_len
] == extra
;
377 if (ret
== LZMA_STREAM_END
)
380 s
.avail_out
+= *buffer_size
;
382 if (!(greedy_realloc(buffer
, buffer_size
, *buffer_size
* 2, 1)))
385 s
.next_out
= *(uint8_t**)buffer
+ *buffer_size
- s
.avail_out
;
389 return -EPROTONOSUPPORT
;
393 int decompress_startswith_lz4(const void *src
, uint64_t src_size
,
394 void **buffer
, size_t *buffer_size
,
395 const void *prefix
, size_t prefix_len
,
398 /* Checks whether the decompressed blob starts with the
399 * mentioned prefix. The byte extra needs to follow the
405 assert(src_size
> 0);
409 assert(*buffer_size
== 0 || *buffer
);
414 if (!(greedy_realloc(buffer
, buffer_size
, ALIGN_8(prefix_len
+ 1), 1)))
417 r
= LZ4_decompress_safe_partial((char*)src
+ 8, *buffer
, src_size
- 8,
418 prefix_len
+ 1, *buffer_size
);
419 /* One lz4 < 1.8.3, we might get "failure" (r < 0), or "success" where
420 * just a part of the buffer is decompressed. But if we get a smaller
421 * amount of bytes than requested, we don't know whether there isn't enough
422 * data to fill the requested size or whether we just got a partial answer.
424 if (r
< 0 || (size_t) r
< prefix_len
+ 1) {
427 if (LZ4_versionNumber() >= 10803)
428 /* We trust that the newer lz4 decompresses the number of bytes we
429 * requested if available in the compressed string. */
433 /* Compare what we have first, in case of mismatch we can
434 * shortcut the full comparison. */
435 if (memcmp(*buffer
, prefix
, r
) != 0)
438 /* Before version 1.8.3, lz4 always tries to decode full a "sequence",
439 * so in pathological cases might need to decompress the full field. */
440 r
= decompress_blob_lz4(src
, src_size
, buffer
, buffer_size
, &size
, 0);
444 if (size
< prefix_len
+ 1)
448 return memcmp(*buffer
, prefix
, prefix_len
) == 0 &&
449 ((const uint8_t*) *buffer
)[prefix_len
] == extra
;
451 return -EPROTONOSUPPORT
;
455 int decompress_startswith_zstd(
456 const void *src
, uint64_t src_size
,
457 void **buffer
, size_t *buffer_size
,
458 const void *prefix
, size_t prefix_len
,
462 assert(src_size
> 0);
466 assert(*buffer_size
== 0 || *buffer
);
468 uint64_t size
= ZSTD_getFrameContentSize(src
, src_size
);
469 if (IN_SET(size
, ZSTD_CONTENTSIZE_ERROR
, ZSTD_CONTENTSIZE_UNKNOWN
))
472 if (size
< prefix_len
+ 1)
473 return 0; /* Decompressed text too short to match the prefix and extra */
475 _cleanup_(ZSTD_freeDCtxp
) ZSTD_DCtx
*dctx
= ZSTD_createDCtx();
479 if (!(greedy_realloc(buffer
, buffer_size
, MAX(ZSTD_DStreamOutSize(), prefix_len
+ 1), 1)))
482 ZSTD_inBuffer input
= {
486 ZSTD_outBuffer output
= {
488 .size
= *buffer_size
,
492 k
= ZSTD_decompressStream(dctx
, &output
, &input
);
493 if (ZSTD_isError(k
)) {
494 log_debug("ZSTD decoder failed: %s", ZSTD_getErrorName(k
));
495 return zstd_ret_to_errno(k
);
497 assert(output
.pos
>= prefix_len
+ 1);
499 return memcmp(*buffer
, prefix
, prefix_len
) == 0 &&
500 ((const uint8_t*) *buffer
)[prefix_len
] == extra
;
502 return -EPROTONOSUPPORT
;
506 int decompress_startswith(
508 const void *src
, uint64_t src_size
,
509 void **buffer
, size_t *buffer_size
,
510 const void *prefix
, size_t prefix_len
,
513 if (compression
== OBJECT_COMPRESSED_XZ
)
514 return decompress_startswith_xz(
520 else if (compression
== OBJECT_COMPRESSED_LZ4
)
521 return decompress_startswith_lz4(
526 else if (compression
== OBJECT_COMPRESSED_ZSTD
)
527 return decompress_startswith_zstd(
536 int compress_stream_xz(int fdf
, int fdt
, uint64_t max_bytes
) {
538 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
540 uint8_t buf
[BUFSIZ
], out
[BUFSIZ
];
541 lzma_action action
= LZMA_RUN
;
546 ret
= lzma_easy_encoder(&s
, LZMA_PRESET_DEFAULT
, LZMA_CHECK_CRC64
);
548 return log_error_errno(SYNTHETIC_ERRNO(EINVAL
),
549 "Failed to initialize XZ encoder: code %u",
553 if (s
.avail_in
== 0 && action
== LZMA_RUN
) {
554 size_t m
= sizeof(buf
);
557 if (max_bytes
!= (uint64_t) -1 && (uint64_t) m
> max_bytes
)
558 m
= (size_t) max_bytes
;
560 n
= read(fdf
, buf
, m
);
564 action
= LZMA_FINISH
;
569 if (max_bytes
!= (uint64_t) -1) {
570 assert(max_bytes
>= (uint64_t) n
);
576 if (s
.avail_out
== 0) {
578 s
.avail_out
= sizeof(out
);
581 ret
= lzma_code(&s
, action
);
582 if (!IN_SET(ret
, LZMA_OK
, LZMA_STREAM_END
))
583 return log_error_errno(SYNTHETIC_ERRNO(EBADMSG
),
584 "Compression failed: code %u",
587 if (s
.avail_out
== 0 || ret
== LZMA_STREAM_END
) {
590 n
= sizeof(out
) - s
.avail_out
;
592 k
= loop_write(fdt
, out
, n
, false);
596 if (ret
== LZMA_STREAM_END
) {
597 log_debug("XZ compression finished (%"PRIu64
" -> %"PRIu64
" bytes, %.1f%%)",
598 s
.total_in
, s
.total_out
,
599 (double) s
.total_out
/ s
.total_in
* 100);
606 return -EPROTONOSUPPORT
;
610 #define LZ4_BUFSIZE (512*1024u)
612 int compress_stream_lz4(int fdf
, int fdt
, uint64_t max_bytes
) {
616 _cleanup_(LZ4F_freeCompressionContextp
) LZ4F_compressionContext_t ctx
= NULL
;
617 _cleanup_free_
char *buf
= NULL
;
619 size_t size
, n
, total_in
= 0, total_out
, offset
= 0, frame_size
;
622 static const LZ4F_compressOptions_t options
= {
625 static const LZ4F_preferences_t preferences
= {
626 .frameInfo
.blockSizeID
= 5,
629 c
= LZ4F_createCompressionContext(&ctx
, LZ4F_VERSION
);
633 if (fstat(fdf
, &st
) < 0)
634 return log_debug_errno(errno
, "fstat() failed: %m");
636 frame_size
= LZ4F_compressBound(LZ4_BUFSIZE
, &preferences
);
637 size
= frame_size
+ 64*1024; /* add some space for header and trailer */
642 n
= offset
= total_out
= LZ4F_compressBegin(ctx
, buf
, size
, &preferences
);
646 src
= mmap(NULL
, st
.st_size
, PROT_READ
, MAP_PRIVATE
, fdf
, 0);
647 if (src
== MAP_FAILED
)
650 log_debug("Buffer size is %zu bytes, header size %zu bytes.", size
, n
);
652 while (total_in
< (size_t) st
.st_size
) {
655 k
= MIN(LZ4_BUFSIZE
, st
.st_size
- total_in
);
656 n
= LZ4F_compressUpdate(ctx
, buf
+ offset
, size
- offset
,
657 src
+ total_in
, k
, &options
);
658 if (LZ4F_isError(n
)) {
659 r
= -ENOTRECOVERABLE
;
667 if (max_bytes
!= (uint64_t) -1 && total_out
> (size_t) max_bytes
)
668 return log_debug_errno(SYNTHETIC_ERRNO(EFBIG
),
669 "Compressed stream longer than %" PRIu64
" bytes",
672 if (size
- offset
< frame_size
+ 4) {
673 k
= loop_write(fdt
, buf
, offset
, false);
682 n
= LZ4F_compressEnd(ctx
, buf
+ offset
, size
- offset
, &options
);
683 if (LZ4F_isError(n
)) {
684 r
= -ENOTRECOVERABLE
;
690 r
= loop_write(fdt
, buf
, offset
, false);
694 log_debug("LZ4 compression finished (%zu -> %zu bytes, %.1f%%)",
696 (double) total_out
/ total_in
* 100);
698 munmap(src
, st
.st_size
);
701 return -EPROTONOSUPPORT
;
705 int decompress_stream_xz(int fdf
, int fdt
, uint64_t max_bytes
) {
708 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
711 uint8_t buf
[BUFSIZ
], out
[BUFSIZ
];
712 lzma_action action
= LZMA_RUN
;
717 ret
= lzma_stream_decoder(&s
, UINT64_MAX
, 0);
719 return log_debug_errno(SYNTHETIC_ERRNO(ENOMEM
),
720 "Failed to initialize XZ decoder: code %u",
724 if (s
.avail_in
== 0 && action
== LZMA_RUN
) {
727 n
= read(fdf
, buf
, sizeof(buf
));
731 action
= LZMA_FINISH
;
738 if (s
.avail_out
== 0) {
740 s
.avail_out
= sizeof(out
);
743 ret
= lzma_code(&s
, action
);
744 if (!IN_SET(ret
, LZMA_OK
, LZMA_STREAM_END
))
745 return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG
),
746 "Decompression failed: code %u",
749 if (s
.avail_out
== 0 || ret
== LZMA_STREAM_END
) {
752 n
= sizeof(out
) - s
.avail_out
;
754 if (max_bytes
!= (uint64_t) -1) {
755 if (max_bytes
< (uint64_t) n
)
761 k
= loop_write(fdt
, out
, n
, false);
765 if (ret
== LZMA_STREAM_END
) {
766 log_debug("XZ decompression finished (%"PRIu64
" -> %"PRIu64
" bytes, %.1f%%)",
767 s
.total_in
, s
.total_out
,
768 (double) s
.total_out
/ s
.total_in
* 100);
775 return log_debug_errno(SYNTHETIC_ERRNO(EPROTONOSUPPORT
),
776 "Cannot decompress file. Compiled without XZ support.");
780 int decompress_stream_lz4(int in
, int out
, uint64_t max_bytes
) {
783 _cleanup_(LZ4F_freeDecompressionContextp
) LZ4F_decompressionContext_t ctx
= NULL
;
784 _cleanup_free_
char *buf
= NULL
;
788 size_t total_in
= 0, total_out
= 0;
790 c
= LZ4F_createDecompressionContext(&ctx
, LZ4F_VERSION
);
794 if (fstat(in
, &st
) < 0)
795 return log_debug_errno(errno
, "fstat() failed: %m");
797 buf
= malloc(LZ4_BUFSIZE
);
801 src
= mmap(NULL
, st
.st_size
, PROT_READ
, MAP_PRIVATE
, in
, 0);
802 if (src
== MAP_FAILED
)
805 while (total_in
< (size_t) st
.st_size
) {
806 size_t produced
= LZ4_BUFSIZE
;
807 size_t used
= st
.st_size
- total_in
;
809 c
= LZ4F_decompress(ctx
, buf
, &produced
, src
+ total_in
, &used
, NULL
);
810 if (LZ4F_isError(c
)) {
816 total_out
+= produced
;
818 if (max_bytes
!= (uint64_t) -1 && total_out
> (size_t) max_bytes
) {
819 log_debug("Decompressed stream longer than %"PRIu64
" bytes", max_bytes
);
824 r
= loop_write(out
, buf
, produced
, false);
829 log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)",
831 total_in
> 0 ? (double) total_out
/ total_in
* 100 : 0.0);
833 munmap(src
, st
.st_size
);
836 return log_debug_errno(SYNTHETIC_ERRNO(EPROTONOSUPPORT
),
837 "Cannot decompress file. Compiled without LZ4 support.");
841 int compress_stream_zstd(int fdf
, int fdt
, uint64_t max_bytes
) {
843 _cleanup_(ZSTD_freeCCtxp
) ZSTD_CCtx
*cctx
= NULL
;
844 _cleanup_free_
void *in_buff
= NULL
, *out_buff
= NULL
;
845 size_t in_allocsize
, out_allocsize
;
847 uint64_t left
= max_bytes
, in_bytes
= 0;
852 /* Create the context and buffers */
853 in_allocsize
= ZSTD_CStreamInSize();
854 out_allocsize
= ZSTD_CStreamOutSize();
855 in_buff
= malloc(in_allocsize
);
856 out_buff
= malloc(out_allocsize
);
857 cctx
= ZSTD_createCCtx();
858 if (!cctx
|| !out_buff
|| !in_buff
)
861 z
= ZSTD_CCtx_setParameter(cctx
, ZSTD_c_checksumFlag
, 1);
863 log_debug("Failed to enable ZSTD checksum, ignoring: %s", ZSTD_getErrorName(z
));
865 /* This loop read from the input file, compresses that entire chunk,
866 * and writes all output produced to the output file.
870 ZSTD_inBuffer input
= {
877 red
= loop_read(fdf
, in_buff
, in_allocsize
, true);
880 is_last_chunk
= red
== 0;
882 in_bytes
+= (size_t) red
;
883 input
.size
= (size_t) red
;
885 for (bool finished
= false; !finished
;) {
886 ZSTD_outBuffer output
= {
888 .size
= out_allocsize
,
894 /* Compress into the output buffer and write all of the
895 * output to the file so we can reuse the buffer next
898 remaining
= ZSTD_compressStream2(
899 cctx
, &output
, &input
,
900 is_last_chunk
? ZSTD_e_end
: ZSTD_e_continue
);
902 if (ZSTD_isError(remaining
)) {
903 log_debug("ZSTD encoder failed: %s", ZSTD_getErrorName(remaining
));
904 return zstd_ret_to_errno(remaining
);
907 if (left
< output
.pos
)
910 wrote
= loop_write(fdt
, output
.dst
, output
.pos
, 1);
916 /* If we're on the last chunk we're finished when zstd
917 * returns 0, which means its consumed all the input AND
918 * finished the frame. Otherwise, we're finished when
919 * we've consumed all the input.
921 finished
= is_last_chunk
? (remaining
== 0) : (input
.pos
== input
.size
);
924 /* zstd only returns 0 when the input is completely consumed */
925 assert(input
.pos
== input
.size
);
931 log_debug("ZSTD compression finished (%" PRIu64
" -> %" PRIu64
" bytes, %.1f%%)",
932 in_bytes
, max_bytes
- left
, (double) (max_bytes
- left
) / in_bytes
* 100);
934 log_debug("ZSTD compression finished (%" PRIu64
" -> %" PRIu64
" bytes)",
935 in_bytes
, max_bytes
- left
);
939 return -EPROTONOSUPPORT
;
943 int decompress_stream_zstd(int fdf
, int fdt
, uint64_t max_bytes
) {
945 _cleanup_(ZSTD_freeDCtxp
) ZSTD_DCtx
*dctx
= NULL
;
946 _cleanup_free_
void *in_buff
= NULL
, *out_buff
= NULL
;
947 size_t in_allocsize
, out_allocsize
;
948 size_t last_result
= 0;
949 uint64_t left
= max_bytes
, in_bytes
= 0;
954 /* Create the context and buffers */
955 in_allocsize
= ZSTD_DStreamInSize();
956 out_allocsize
= ZSTD_DStreamOutSize();
957 in_buff
= malloc(in_allocsize
);
958 out_buff
= malloc(out_allocsize
);
959 dctx
= ZSTD_createDCtx();
960 if (!dctx
|| !out_buff
|| !in_buff
)
963 /* This loop assumes that the input file is one or more concatenated
964 * zstd streams. This example won't work if there is trailing non-zstd
965 * data at the end, but streaming decompression in general handles this
966 * case. ZSTD_decompressStream() returns 0 exactly when the frame is
967 * completed, and doesn't consume input after the frame.
970 bool has_error
= false;
971 ZSTD_inBuffer input
= {
978 red
= loop_read(fdf
, in_buff
, in_allocsize
, true);
984 in_bytes
+= (size_t) red
;
985 input
.size
= (size_t) red
;
988 /* Given a valid frame, zstd won't consume the last byte of the
989 * frame until it has flushed all of the decompressed data of
990 * the frame. So input.pos < input.size means frame is not done
991 * or there is still output available.
993 while (input
.pos
< input
.size
) {
994 ZSTD_outBuffer output
= {
996 .size
= out_allocsize
,
1000 /* The return code is zero if the frame is complete, but
1001 * there may be multiple frames concatenated together.
1002 * Zstd will automatically reset the context when a
1003 * frame is complete. Still, calling ZSTD_DCtx_reset()
1004 * can be useful to reset the context to a clean state,
1005 * for instance if the last decompression call returned
1008 last_result
= ZSTD_decompressStream(dctx
, &output
, &input
);
1009 if (ZSTD_isError(last_result
)) {
1014 if (left
< output
.pos
)
1017 wrote
= loop_write(fdt
, output
.dst
, output
.pos
, 1);
1028 return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG
), "ZSTD decoder failed: no data read");
1030 if (last_result
!= 0) {
1031 /* The last return value from ZSTD_decompressStream did not end
1032 * on a frame, but we reached the end of the file! We assume
1033 * this is an error, and the input was truncated.
1035 log_debug("ZSTD decoder failed: %s", ZSTD_getErrorName(last_result
));
1036 return zstd_ret_to_errno(last_result
);
1040 "ZSTD decompression finished (%" PRIu64
" -> %" PRIu64
" bytes, %.1f%%)",
1043 (double) (max_bytes
- left
) / in_bytes
* 100);
1046 return log_debug_errno(SYNTHETIC_ERRNO(EPROTONOSUPPORT
),
1047 "Cannot decompress file. Compiled without ZSTD support.");
1051 int decompress_stream(const char *filename
, int fdf
, int fdt
, uint64_t max_bytes
) {
1053 if (endswith(filename
, ".lz4"))
1054 return decompress_stream_lz4(fdf
, fdt
, max_bytes
);
1055 else if (endswith(filename
, ".xz"))
1056 return decompress_stream_xz(fdf
, fdt
, max_bytes
);
1057 else if (endswith(filename
, ".zst"))
1058 return decompress_stream_zstd(fdf
, fdt
, max_bytes
);
1060 return -EPROTONOSUPPORT
;