1 /* SPDX-License-Identifier: LGPL-2.1-or-later */
21 #include <zstd_errors.h>
24 #include "alloc-util.h"
28 #include "journal-def.h"
30 #include "sparse-endian.h"
31 #include "string-table.h"
32 #include "string-util.h"
33 #include "unaligned.h"
37 DEFINE_TRIVIAL_CLEANUP_FUNC(LZ4F_compressionContext_t
, LZ4F_freeCompressionContext
);
38 DEFINE_TRIVIAL_CLEANUP_FUNC(LZ4F_decompressionContext_t
, LZ4F_freeDecompressionContext
);
42 DEFINE_TRIVIAL_CLEANUP_FUNC(ZSTD_CCtx
*, ZSTD_freeCCtx
);
43 DEFINE_TRIVIAL_CLEANUP_FUNC(ZSTD_DCtx
*, ZSTD_freeDCtx
);
45 static int zstd_ret_to_errno(size_t ret
) {
46 switch (ZSTD_getErrorCode(ret
)) {
47 case ZSTD_error_dstSize_tooSmall
:
49 case ZSTD_error_memory_allocation
:
57 #define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t))
59 static const char* const object_compressed_table
[_OBJECT_COMPRESSED_MAX
] = {
60 [OBJECT_COMPRESSED_XZ
] = "XZ",
61 [OBJECT_COMPRESSED_LZ4
] = "LZ4",
62 [OBJECT_COMPRESSED_ZSTD
] = "ZSTD",
63 /* If we add too many more entries here, it's going to grow quite large (and be mostly sparse), since
64 * the array key is actually a bitmask, not a plain enum */
67 DEFINE_STRING_TABLE_LOOKUP(object_compressed
, int);
69 int compress_blob_xz(const void *src
, uint64_t src_size
,
70 void *dst
, size_t dst_alloc_size
, size_t *dst_size
) {
72 static const lzma_options_lzma opt
= {
73 1u << 20u, NULL
, 0, LZMA_LC_DEFAULT
, LZMA_LP_DEFAULT
,
74 LZMA_PB_DEFAULT
, LZMA_MODE_FAST
, 128, LZMA_MF_HC3
, 4
76 static const lzma_filter filters
[] = {
77 { LZMA_FILTER_LZMA2
, (lzma_options_lzma
*) &opt
},
78 { LZMA_VLI_UNKNOWN
, NULL
}
86 assert(dst_alloc_size
> 0);
89 /* Returns < 0 if we couldn't compress the data or the
90 * compressed result is longer than the original */
95 ret
= lzma_stream_buffer_encode((lzma_filter
*) filters
, LZMA_CHECK_NONE
, NULL
,
96 src
, src_size
, dst
, &out_pos
, dst_alloc_size
);
103 return -EPROTONOSUPPORT
;
107 int compress_blob_lz4(const void *src
, uint64_t src_size
,
108 void *dst
, size_t dst_alloc_size
, size_t *dst_size
) {
113 assert(src_size
> 0);
115 assert(dst_alloc_size
> 0);
118 /* Returns < 0 if we couldn't compress the data or the
119 * compressed result is longer than the original */
124 r
= LZ4_compress_default(src
, (char*)dst
+ 8, src_size
, (int) dst_alloc_size
- 8);
128 unaligned_write_le64(dst
, src_size
);
133 return -EPROTONOSUPPORT
;
137 int compress_blob_zstd(
138 const void *src
, uint64_t src_size
,
139 void *dst
, size_t dst_alloc_size
, size_t *dst_size
) {
144 assert(src_size
> 0);
146 assert(dst_alloc_size
> 0);
149 k
= ZSTD_compress(dst
, dst_alloc_size
, src
, src_size
, 0);
151 return zstd_ret_to_errno(k
);
156 return -EPROTONOSUPPORT
;
160 int decompress_blob_xz(const void *src
, uint64_t src_size
,
161 void **dst
, size_t *dst_alloc_size
, size_t* dst_size
, size_t dst_max
) {
164 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
169 assert(src_size
> 0);
171 assert(dst_alloc_size
);
173 assert(*dst_alloc_size
== 0 || *dst
);
175 ret
= lzma_stream_decoder(&s
, UINT64_MAX
, 0);
179 space
= MIN(src_size
* 2, dst_max
?: (size_t) -1);
180 if (!greedy_realloc(dst
, dst_alloc_size
, space
, 1))
184 s
.avail_in
= src_size
;
192 ret
= lzma_code(&s
, LZMA_FINISH
);
194 if (ret
== LZMA_STREAM_END
)
196 else if (ret
!= LZMA_OK
)
199 if (dst_max
> 0 && (space
- s
.avail_out
) >= dst_max
)
201 else if (dst_max
> 0 && space
== dst_max
)
204 used
= space
- s
.avail_out
;
205 space
= MIN(2 * space
, dst_max
?: (size_t) -1);
206 if (!greedy_realloc(dst
, dst_alloc_size
, space
, 1))
209 s
.avail_out
= space
- used
;
210 s
.next_out
= *(uint8_t**)dst
+ used
;
213 *dst_size
= space
- s
.avail_out
;
216 return -EPROTONOSUPPORT
;
220 int decompress_blob_lz4(const void *src
, uint64_t src_size
,
221 void **dst
, size_t *dst_alloc_size
, size_t* dst_size
, size_t dst_max
) {
225 int r
, size
; /* LZ4 uses int for size */
228 assert(src_size
> 0);
230 assert(dst_alloc_size
);
232 assert(*dst_alloc_size
== 0 || *dst
);
237 size
= unaligned_read_le64(src
);
238 if (size
< 0 || (unsigned) size
!= unaligned_read_le64(src
))
240 if ((size_t) size
> *dst_alloc_size
) {
241 out
= realloc(*dst
, size
);
245 *dst_alloc_size
= size
;
249 r
= LZ4_decompress_safe((char*)src
+ 8, out
, src_size
- 8, size
);
250 if (r
< 0 || r
!= size
)
256 return -EPROTONOSUPPORT
;
260 int decompress_blob_zstd(
261 const void *src
, uint64_t src_size
,
262 void **dst
, size_t *dst_alloc_size
, size_t *dst_size
, size_t dst_max
) {
268 assert(src_size
> 0);
270 assert(dst_alloc_size
);
272 assert(*dst_alloc_size
== 0 || *dst
);
274 size
= ZSTD_getFrameContentSize(src
, src_size
);
275 if (IN_SET(size
, ZSTD_CONTENTSIZE_ERROR
, ZSTD_CONTENTSIZE_UNKNOWN
))
278 if (dst_max
> 0 && size
> dst_max
)
283 if (!(greedy_realloc(dst
, dst_alloc_size
, MAX(ZSTD_DStreamOutSize(), size
), 1)))
286 _cleanup_(ZSTD_freeDCtxp
) ZSTD_DCtx
*dctx
= ZSTD_createDCtx();
290 ZSTD_inBuffer input
= {
294 ZSTD_outBuffer output
= {
296 .size
= *dst_alloc_size
,
299 size_t k
= ZSTD_decompressStream(dctx
, &output
, &input
);
300 if (ZSTD_isError(k
)) {
301 log_debug("ZSTD decoder failed: %s", ZSTD_getErrorName(k
));
302 return zstd_ret_to_errno(k
);
304 assert(output
.pos
>= size
);
309 return -EPROTONOSUPPORT
;
315 const void *src
, uint64_t src_size
,
316 void **dst
, size_t *dst_alloc_size
, size_t* dst_size
, size_t dst_max
) {
318 if (compression
== OBJECT_COMPRESSED_XZ
)
319 return decompress_blob_xz(
321 dst
, dst_alloc_size
, dst_size
, dst_max
);
322 else if (compression
== OBJECT_COMPRESSED_LZ4
)
323 return decompress_blob_lz4(
325 dst
, dst_alloc_size
, dst_size
, dst_max
);
326 else if (compression
== OBJECT_COMPRESSED_ZSTD
)
327 return decompress_blob_zstd(
329 dst
, dst_alloc_size
, dst_size
, dst_max
);
331 return -EPROTONOSUPPORT
;
334 int decompress_startswith_xz(const void *src
, uint64_t src_size
,
335 void **buffer
, size_t *buffer_size
,
336 const void *prefix
, size_t prefix_len
,
340 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
343 /* Checks whether the decompressed blob starts with the
344 * mentioned prefix. The byte extra needs to follow the
348 assert(src_size
> 0);
352 assert(*buffer_size
== 0 || *buffer
);
354 ret
= lzma_stream_decoder(&s
, UINT64_MAX
, 0);
358 if (!(greedy_realloc(buffer
, buffer_size
, ALIGN_8(prefix_len
+ 1), 1)))
362 s
.avail_in
= src_size
;
364 s
.next_out
= *buffer
;
365 s
.avail_out
= *buffer_size
;
368 ret
= lzma_code(&s
, LZMA_FINISH
);
370 if (!IN_SET(ret
, LZMA_OK
, LZMA_STREAM_END
))
373 if (*buffer_size
- s
.avail_out
>= prefix_len
+ 1)
374 return memcmp(*buffer
, prefix
, prefix_len
) == 0 &&
375 ((const uint8_t*) *buffer
)[prefix_len
] == extra
;
377 if (ret
== LZMA_STREAM_END
)
380 s
.avail_out
+= *buffer_size
;
382 if (!(greedy_realloc(buffer
, buffer_size
, *buffer_size
* 2, 1)))
385 s
.next_out
= *(uint8_t**)buffer
+ *buffer_size
- s
.avail_out
;
389 return -EPROTONOSUPPORT
;
393 int decompress_startswith_lz4(const void *src
, uint64_t src_size
,
394 void **buffer
, size_t *buffer_size
,
395 const void *prefix
, size_t prefix_len
,
398 /* Checks whether the decompressed blob starts with the
399 * mentioned prefix. The byte extra needs to follow the
405 assert(src_size
> 0);
409 assert(*buffer_size
== 0 || *buffer
);
414 if (!(greedy_realloc(buffer
, buffer_size
, ALIGN_8(prefix_len
+ 1), 1)))
417 r
= LZ4_decompress_safe_partial((char*)src
+ 8, *buffer
, src_size
- 8,
418 prefix_len
+ 1, *buffer_size
);
419 /* One lz4 < 1.8.3, we might get "failure" (r < 0), or "success" where
420 * just a part of the buffer is decompressed. But if we get a smaller
421 * amount of bytes than requested, we don't know whether there isn't enough
422 * data to fill the requested size or whether we just got a partial answer.
424 if (r
< 0 || (size_t) r
< prefix_len
+ 1) {
427 if (LZ4_versionNumber() >= 10803)
428 /* We trust that the newer lz4 decompresses the number of bytes we
429 * requested if available in the compressed string. */
433 /* Compare what we have first, in case of mismatch we can
434 * shortcut the full comparison. */
435 if (memcmp(*buffer
, prefix
, r
) != 0)
438 /* Before version 1.8.3, lz4 always tries to decode full a "sequence",
439 * so in pathological cases might need to decompress the full field. */
440 r
= decompress_blob_lz4(src
, src_size
, buffer
, buffer_size
, &size
, 0);
444 if (size
< prefix_len
+ 1)
448 return memcmp(*buffer
, prefix
, prefix_len
) == 0 &&
449 ((const uint8_t*) *buffer
)[prefix_len
] == extra
;
451 return -EPROTONOSUPPORT
;
455 int decompress_startswith_zstd(
456 const void *src
, uint64_t src_size
,
457 void **buffer
, size_t *buffer_size
,
458 const void *prefix
, size_t prefix_len
,
462 assert(src_size
> 0);
466 assert(*buffer_size
== 0 || *buffer
);
468 uint64_t size
= ZSTD_getFrameContentSize(src
, src_size
);
469 if (IN_SET(size
, ZSTD_CONTENTSIZE_ERROR
, ZSTD_CONTENTSIZE_UNKNOWN
))
472 if (size
< prefix_len
+ 1)
473 return 0; /* Decompressed text too short to match the prefix and extra */
475 _cleanup_(ZSTD_freeDCtxp
) ZSTD_DCtx
*dctx
= ZSTD_createDCtx();
479 if (!(greedy_realloc(buffer
, buffer_size
, MAX(ZSTD_DStreamOutSize(), prefix_len
+ 1), 1)))
482 ZSTD_inBuffer input
= {
486 ZSTD_outBuffer output
= {
488 .size
= *buffer_size
,
492 k
= ZSTD_decompressStream(dctx
, &output
, &input
);
493 if (ZSTD_isError(k
)) {
494 log_debug("ZSTD decoder failed: %s", ZSTD_getErrorName(k
));
495 return zstd_ret_to_errno(k
);
497 assert(output
.pos
>= prefix_len
+ 1);
499 return memcmp(*buffer
, prefix
, prefix_len
) == 0 &&
500 ((const uint8_t*) *buffer
)[prefix_len
] == extra
;
502 return -EPROTONOSUPPORT
;
506 int decompress_startswith(
508 const void *src
, uint64_t src_size
,
509 void **buffer
, size_t *buffer_size
,
510 const void *prefix
, size_t prefix_len
,
513 if (compression
== OBJECT_COMPRESSED_XZ
)
514 return decompress_startswith_xz(
520 else if (compression
== OBJECT_COMPRESSED_LZ4
)
521 return decompress_startswith_lz4(
526 else if (compression
== OBJECT_COMPRESSED_ZSTD
)
527 return decompress_startswith_zstd(
536 int compress_stream_xz(int fdf
, int fdt
, uint64_t max_bytes
) {
538 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
540 uint8_t buf
[BUFSIZ
], out
[BUFSIZ
];
541 lzma_action action
= LZMA_RUN
;
546 ret
= lzma_easy_encoder(&s
, LZMA_PRESET_DEFAULT
, LZMA_CHECK_CRC64
);
548 return log_error_errno(SYNTHETIC_ERRNO(EINVAL
),
549 "Failed to initialize XZ encoder: code %u",
553 if (s
.avail_in
== 0 && action
== LZMA_RUN
) {
554 size_t m
= sizeof(buf
);
557 if (max_bytes
!= (uint64_t) -1 && (uint64_t) m
> max_bytes
)
558 m
= (size_t) max_bytes
;
560 n
= read(fdf
, buf
, m
);
564 action
= LZMA_FINISH
;
569 if (max_bytes
!= (uint64_t) -1) {
570 assert(max_bytes
>= (uint64_t) n
);
576 if (s
.avail_out
== 0) {
578 s
.avail_out
= sizeof(out
);
581 ret
= lzma_code(&s
, action
);
582 if (!IN_SET(ret
, LZMA_OK
, LZMA_STREAM_END
))
583 return log_error_errno(SYNTHETIC_ERRNO(EBADMSG
),
584 "Compression failed: code %u",
587 if (s
.avail_out
== 0 || ret
== LZMA_STREAM_END
) {
590 n
= sizeof(out
) - s
.avail_out
;
592 k
= loop_write(fdt
, out
, n
, false);
596 if (ret
== LZMA_STREAM_END
) {
597 log_debug("XZ compression finished (%"PRIu64
" -> %"PRIu64
" bytes, %.1f%%)",
598 s
.total_in
, s
.total_out
,
599 (double) s
.total_out
/ s
.total_in
* 100);
606 return -EPROTONOSUPPORT
;
610 #define LZ4_BUFSIZE (512*1024u)
612 int compress_stream_lz4(int fdf
, int fdt
, uint64_t max_bytes
) {
616 _cleanup_(LZ4F_freeCompressionContextp
) LZ4F_compressionContext_t ctx
= NULL
;
617 _cleanup_free_
char *buf
= NULL
;
619 size_t size
, n
, total_in
= 0, total_out
, offset
= 0, frame_size
;
622 static const LZ4F_compressOptions_t options
= {
625 static const LZ4F_preferences_t preferences
= {
626 .frameInfo
.blockSizeID
= 5,
629 c
= LZ4F_createCompressionContext(&ctx
, LZ4F_VERSION
);
633 if (fstat(fdf
, &st
) < 0)
634 return log_debug_errno(errno
, "fstat() failed: %m");
636 frame_size
= LZ4F_compressBound(LZ4_BUFSIZE
, &preferences
);
637 size
= frame_size
+ 64*1024; /* add some space for header and trailer */
642 n
= offset
= total_out
= LZ4F_compressBegin(ctx
, buf
, size
, &preferences
);
646 src
= mmap(NULL
, st
.st_size
, PROT_READ
, MAP_PRIVATE
, fdf
, 0);
647 if (src
== MAP_FAILED
)
650 log_debug("Buffer size is %zu bytes, header size %zu bytes.", size
, n
);
652 while (total_in
< (size_t) st
.st_size
) {
655 k
= MIN(LZ4_BUFSIZE
, st
.st_size
- total_in
);
656 n
= LZ4F_compressUpdate(ctx
, buf
+ offset
, size
- offset
,
657 src
+ total_in
, k
, &options
);
658 if (LZ4F_isError(n
)) {
659 r
= -ENOTRECOVERABLE
;
667 if (max_bytes
!= (uint64_t) -1 && total_out
> (size_t) max_bytes
) {
668 r
= log_debug_errno(SYNTHETIC_ERRNO(EFBIG
),
669 "Compressed stream longer than %" PRIu64
" bytes", max_bytes
);
673 if (size
- offset
< frame_size
+ 4) {
674 k
= loop_write(fdt
, buf
, offset
, false);
683 n
= LZ4F_compressEnd(ctx
, buf
+ offset
, size
- offset
, &options
);
684 if (LZ4F_isError(n
)) {
685 r
= -ENOTRECOVERABLE
;
691 r
= loop_write(fdt
, buf
, offset
, false);
695 log_debug("LZ4 compression finished (%zu -> %zu bytes, %.1f%%)",
697 (double) total_out
/ total_in
* 100);
699 munmap(src
, st
.st_size
);
702 return -EPROTONOSUPPORT
;
706 int decompress_stream_xz(int fdf
, int fdt
, uint64_t max_bytes
) {
709 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
712 uint8_t buf
[BUFSIZ
], out
[BUFSIZ
];
713 lzma_action action
= LZMA_RUN
;
718 ret
= lzma_stream_decoder(&s
, UINT64_MAX
, 0);
720 return log_debug_errno(SYNTHETIC_ERRNO(ENOMEM
),
721 "Failed to initialize XZ decoder: code %u",
725 if (s
.avail_in
== 0 && action
== LZMA_RUN
) {
728 n
= read(fdf
, buf
, sizeof(buf
));
732 action
= LZMA_FINISH
;
739 if (s
.avail_out
== 0) {
741 s
.avail_out
= sizeof(out
);
744 ret
= lzma_code(&s
, action
);
745 if (!IN_SET(ret
, LZMA_OK
, LZMA_STREAM_END
))
746 return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG
),
747 "Decompression failed: code %u",
750 if (s
.avail_out
== 0 || ret
== LZMA_STREAM_END
) {
753 n
= sizeof(out
) - s
.avail_out
;
755 if (max_bytes
!= (uint64_t) -1) {
756 if (max_bytes
< (uint64_t) n
)
762 k
= loop_write(fdt
, out
, n
, false);
766 if (ret
== LZMA_STREAM_END
) {
767 log_debug("XZ decompression finished (%"PRIu64
" -> %"PRIu64
" bytes, %.1f%%)",
768 s
.total_in
, s
.total_out
,
769 (double) s
.total_out
/ s
.total_in
* 100);
776 return log_debug_errno(SYNTHETIC_ERRNO(EPROTONOSUPPORT
),
777 "Cannot decompress file. Compiled without XZ support.");
781 int decompress_stream_lz4(int in
, int out
, uint64_t max_bytes
) {
784 _cleanup_(LZ4F_freeDecompressionContextp
) LZ4F_decompressionContext_t ctx
= NULL
;
785 _cleanup_free_
char *buf
= NULL
;
789 size_t total_in
= 0, total_out
= 0;
791 c
= LZ4F_createDecompressionContext(&ctx
, LZ4F_VERSION
);
795 if (fstat(in
, &st
) < 0)
796 return log_debug_errno(errno
, "fstat() failed: %m");
798 buf
= malloc(LZ4_BUFSIZE
);
802 src
= mmap(NULL
, st
.st_size
, PROT_READ
, MAP_PRIVATE
, in
, 0);
803 if (src
== MAP_FAILED
)
806 while (total_in
< (size_t) st
.st_size
) {
807 size_t produced
= LZ4_BUFSIZE
;
808 size_t used
= st
.st_size
- total_in
;
810 c
= LZ4F_decompress(ctx
, buf
, &produced
, src
+ total_in
, &used
, NULL
);
811 if (LZ4F_isError(c
)) {
817 total_out
+= produced
;
819 if (max_bytes
!= (uint64_t) -1 && total_out
> (size_t) max_bytes
) {
820 log_debug("Decompressed stream longer than %"PRIu64
" bytes", max_bytes
);
825 r
= loop_write(out
, buf
, produced
, false);
830 log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)",
832 total_in
> 0 ? (double) total_out
/ total_in
* 100 : 0.0);
834 munmap(src
, st
.st_size
);
837 return log_debug_errno(SYNTHETIC_ERRNO(EPROTONOSUPPORT
),
838 "Cannot decompress file. Compiled without LZ4 support.");
842 int compress_stream_zstd(int fdf
, int fdt
, uint64_t max_bytes
) {
844 _cleanup_(ZSTD_freeCCtxp
) ZSTD_CCtx
*cctx
= NULL
;
845 _cleanup_free_
void *in_buff
= NULL
, *out_buff
= NULL
;
846 size_t in_allocsize
, out_allocsize
;
848 uint64_t left
= max_bytes
, in_bytes
= 0;
853 /* Create the context and buffers */
854 in_allocsize
= ZSTD_CStreamInSize();
855 out_allocsize
= ZSTD_CStreamOutSize();
856 in_buff
= malloc(in_allocsize
);
857 out_buff
= malloc(out_allocsize
);
858 cctx
= ZSTD_createCCtx();
859 if (!cctx
|| !out_buff
|| !in_buff
)
862 z
= ZSTD_CCtx_setParameter(cctx
, ZSTD_c_checksumFlag
, 1);
864 log_debug("Failed to enable ZSTD checksum, ignoring: %s", ZSTD_getErrorName(z
));
866 /* This loop read from the input file, compresses that entire chunk,
867 * and writes all output produced to the output file.
871 ZSTD_inBuffer input
= {
878 red
= loop_read(fdf
, in_buff
, in_allocsize
, true);
881 is_last_chunk
= red
== 0;
883 in_bytes
+= (size_t) red
;
884 input
.size
= (size_t) red
;
886 for (bool finished
= false; !finished
;) {
887 ZSTD_outBuffer output
= {
889 .size
= out_allocsize
,
895 /* Compress into the output buffer and write all of the
896 * output to the file so we can reuse the buffer next
899 remaining
= ZSTD_compressStream2(
900 cctx
, &output
, &input
,
901 is_last_chunk
? ZSTD_e_end
: ZSTD_e_continue
);
903 if (ZSTD_isError(remaining
)) {
904 log_debug("ZSTD encoder failed: %s", ZSTD_getErrorName(remaining
));
905 return zstd_ret_to_errno(remaining
);
908 if (left
< output
.pos
)
911 wrote
= loop_write(fdt
, output
.dst
, output
.pos
, 1);
917 /* If we're on the last chunk we're finished when zstd
918 * returns 0, which means its consumed all the input AND
919 * finished the frame. Otherwise, we're finished when
920 * we've consumed all the input.
922 finished
= is_last_chunk
? (remaining
== 0) : (input
.pos
== input
.size
);
925 /* zstd only returns 0 when the input is completely consumed */
926 assert(input
.pos
== input
.size
);
932 log_debug("ZSTD compression finished (%" PRIu64
" -> %" PRIu64
" bytes, %.1f%%)",
933 in_bytes
, max_bytes
- left
, (double) (max_bytes
- left
) / in_bytes
* 100);
935 log_debug("ZSTD compression finished (%" PRIu64
" -> %" PRIu64
" bytes)",
936 in_bytes
, max_bytes
- left
);
940 return -EPROTONOSUPPORT
;
944 int decompress_stream_zstd(int fdf
, int fdt
, uint64_t max_bytes
) {
946 _cleanup_(ZSTD_freeDCtxp
) ZSTD_DCtx
*dctx
= NULL
;
947 _cleanup_free_
void *in_buff
= NULL
, *out_buff
= NULL
;
948 size_t in_allocsize
, out_allocsize
;
949 size_t last_result
= 0;
950 uint64_t left
= max_bytes
, in_bytes
= 0;
955 /* Create the context and buffers */
956 in_allocsize
= ZSTD_DStreamInSize();
957 out_allocsize
= ZSTD_DStreamOutSize();
958 in_buff
= malloc(in_allocsize
);
959 out_buff
= malloc(out_allocsize
);
960 dctx
= ZSTD_createDCtx();
961 if (!dctx
|| !out_buff
|| !in_buff
)
964 /* This loop assumes that the input file is one or more concatenated
965 * zstd streams. This example won't work if there is trailing non-zstd
966 * data at the end, but streaming decompression in general handles this
967 * case. ZSTD_decompressStream() returns 0 exactly when the frame is
968 * completed, and doesn't consume input after the frame.
971 bool has_error
= false;
972 ZSTD_inBuffer input
= {
979 red
= loop_read(fdf
, in_buff
, in_allocsize
, true);
985 in_bytes
+= (size_t) red
;
986 input
.size
= (size_t) red
;
989 /* Given a valid frame, zstd won't consume the last byte of the
990 * frame until it has flushed all of the decompressed data of
991 * the frame. So input.pos < input.size means frame is not done
992 * or there is still output available.
994 while (input
.pos
< input
.size
) {
995 ZSTD_outBuffer output
= {
997 .size
= out_allocsize
,
1001 /* The return code is zero if the frame is complete, but
1002 * there may be multiple frames concatenated together.
1003 * Zstd will automatically reset the context when a
1004 * frame is complete. Still, calling ZSTD_DCtx_reset()
1005 * can be useful to reset the context to a clean state,
1006 * for instance if the last decompression call returned
1009 last_result
= ZSTD_decompressStream(dctx
, &output
, &input
);
1010 if (ZSTD_isError(last_result
)) {
1015 if (left
< output
.pos
)
1018 wrote
= loop_write(fdt
, output
.dst
, output
.pos
, 1);
1029 return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG
), "ZSTD decoder failed: no data read");
1031 if (last_result
!= 0) {
1032 /* The last return value from ZSTD_decompressStream did not end
1033 * on a frame, but we reached the end of the file! We assume
1034 * this is an error, and the input was truncated.
1036 log_debug("ZSTD decoder failed: %s", ZSTD_getErrorName(last_result
));
1037 return zstd_ret_to_errno(last_result
);
1041 "ZSTD decompression finished (%" PRIu64
" -> %" PRIu64
" bytes, %.1f%%)",
1044 (double) (max_bytes
- left
) / in_bytes
* 100);
1047 return log_debug_errno(SYNTHETIC_ERRNO(EPROTONOSUPPORT
),
1048 "Cannot decompress file. Compiled without ZSTD support.");
1052 int decompress_stream(const char *filename
, int fdf
, int fdt
, uint64_t max_bytes
) {
1054 if (endswith(filename
, ".lz4"))
1055 return decompress_stream_lz4(fdf
, fdt
, max_bytes
);
1056 else if (endswith(filename
, ".xz"))
1057 return decompress_stream_xz(fdf
, fdt
, max_bytes
);
1058 else if (endswith(filename
, ".zst"))
1059 return decompress_stream_zstd(fdf
, fdt
, max_bytes
);
1061 return -EPROTONOSUPPORT
;