1 /* SPDX-License-Identifier: LGPL-2.1+ */
3 This file is part of systemd.
5 Copyright 2011 Lennart Poettering
23 #include "alloc-util.h"
27 #include "journal-def.h"
29 #include "sparse-endian.h"
30 #include "string-table.h"
31 #include "string-util.h"
35 DEFINE_TRIVIAL_CLEANUP_FUNC(LZ4F_compressionContext_t
, LZ4F_freeCompressionContext
);
36 DEFINE_TRIVIAL_CLEANUP_FUNC(LZ4F_decompressionContext_t
, LZ4F_freeDecompressionContext
);
39 #define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t))
41 static const char* const object_compressed_table
[_OBJECT_COMPRESSED_MAX
] = {
42 [OBJECT_COMPRESSED_XZ
] = "XZ",
43 [OBJECT_COMPRESSED_LZ4
] = "LZ4",
46 DEFINE_STRING_TABLE_LOOKUP(object_compressed
, int);
48 int compress_blob_xz(const void *src
, uint64_t src_size
,
49 void *dst
, size_t dst_alloc_size
, size_t *dst_size
) {
51 static const lzma_options_lzma opt
= {
52 1u << 20u, NULL
, 0, LZMA_LC_DEFAULT
, LZMA_LP_DEFAULT
,
53 LZMA_PB_DEFAULT
, LZMA_MODE_FAST
, 128, LZMA_MF_HC3
, 4
55 static const lzma_filter filters
[] = {
56 { LZMA_FILTER_LZMA2
, (lzma_options_lzma
*) &opt
},
57 { LZMA_VLI_UNKNOWN
, NULL
}
65 assert(dst_alloc_size
> 0);
68 /* Returns < 0 if we couldn't compress the data or the
69 * compressed result is longer than the original */
74 ret
= lzma_stream_buffer_encode((lzma_filter
*) filters
, LZMA_CHECK_NONE
, NULL
,
75 src
, src_size
, dst
, &out_pos
, dst_alloc_size
);
82 return -EPROTONOSUPPORT
;
86 int compress_blob_lz4(const void *src
, uint64_t src_size
,
87 void *dst
, size_t dst_alloc_size
, size_t *dst_size
) {
94 assert(dst_alloc_size
> 0);
97 /* Returns < 0 if we couldn't compress the data or the
98 * compressed result is longer than the original */
103 #if LZ4_VERSION_NUMBER >= 10700
104 r
= LZ4_compress_default(src
, (char*)dst
+ 8, src_size
, (int) dst_alloc_size
- 8);
106 r
= LZ4_compress_limitedOutput(src
, (char*)dst
+ 8, src_size
, (int) dst_alloc_size
- 8);
111 *(le64_t
*) dst
= htole64(src_size
);
116 return -EPROTONOSUPPORT
;
121 int decompress_blob_xz(const void *src
, uint64_t src_size
,
122 void **dst
, size_t *dst_alloc_size
, size_t* dst_size
, size_t dst_max
) {
125 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
130 assert(src_size
> 0);
132 assert(dst_alloc_size
);
134 assert(*dst_alloc_size
== 0 || *dst
);
136 ret
= lzma_stream_decoder(&s
, UINT64_MAX
, 0);
140 space
= MIN(src_size
* 2, dst_max
?: (size_t) -1);
141 if (!greedy_realloc(dst
, dst_alloc_size
, space
, 1))
145 s
.avail_in
= src_size
;
153 ret
= lzma_code(&s
, LZMA_FINISH
);
155 if (ret
== LZMA_STREAM_END
)
157 else if (ret
!= LZMA_OK
)
160 if (dst_max
> 0 && (space
- s
.avail_out
) >= dst_max
)
162 else if (dst_max
> 0 && space
== dst_max
)
165 used
= space
- s
.avail_out
;
166 space
= MIN(2 * space
, dst_max
?: (size_t) -1);
167 if (!greedy_realloc(dst
, dst_alloc_size
, space
, 1))
170 s
.avail_out
= space
- used
;
171 s
.next_out
= *(uint8_t**)dst
+ used
;
174 *dst_size
= space
- s
.avail_out
;
177 return -EPROTONOSUPPORT
;
181 int decompress_blob_lz4(const void *src
, uint64_t src_size
,
182 void **dst
, size_t *dst_alloc_size
, size_t* dst_size
, size_t dst_max
) {
186 int r
, size
; /* LZ4 uses int for size */
189 assert(src_size
> 0);
191 assert(dst_alloc_size
);
193 assert(*dst_alloc_size
== 0 || *dst
);
198 size
= le64toh( *(le64_t
*)src
);
199 if (size
< 0 || (unsigned) size
!= le64toh(*(le64_t
*)src
))
201 if ((size_t) size
> *dst_alloc_size
) {
202 out
= realloc(*dst
, size
);
206 *dst_alloc_size
= size
;
210 r
= LZ4_decompress_safe((char*)src
+ 8, out
, src_size
- 8, size
);
211 if (r
< 0 || r
!= size
)
217 return -EPROTONOSUPPORT
;
221 int decompress_blob(int compression
,
222 const void *src
, uint64_t src_size
,
223 void **dst
, size_t *dst_alloc_size
, size_t* dst_size
, size_t dst_max
) {
224 if (compression
== OBJECT_COMPRESSED_XZ
)
225 return decompress_blob_xz(src
, src_size
,
226 dst
, dst_alloc_size
, dst_size
, dst_max
);
227 else if (compression
== OBJECT_COMPRESSED_LZ4
)
228 return decompress_blob_lz4(src
, src_size
,
229 dst
, dst_alloc_size
, dst_size
, dst_max
);
235 int decompress_startswith_xz(const void *src
, uint64_t src_size
,
236 void **buffer
, size_t *buffer_size
,
237 const void *prefix
, size_t prefix_len
,
241 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
244 /* Checks whether the decompressed blob starts with the
245 * mentioned prefix. The byte extra needs to follow the
249 assert(src_size
> 0);
253 assert(*buffer_size
== 0 || *buffer
);
255 ret
= lzma_stream_decoder(&s
, UINT64_MAX
, 0);
259 if (!(greedy_realloc(buffer
, buffer_size
, ALIGN_8(prefix_len
+ 1), 1)))
263 s
.avail_in
= src_size
;
265 s
.next_out
= *buffer
;
266 s
.avail_out
= *buffer_size
;
269 ret
= lzma_code(&s
, LZMA_FINISH
);
271 if (!IN_SET(ret
, LZMA_OK
, LZMA_STREAM_END
))
274 if (*buffer_size
- s
.avail_out
>= prefix_len
+ 1)
275 return memcmp(*buffer
, prefix
, prefix_len
) == 0 &&
276 ((const uint8_t*) *buffer
)[prefix_len
] == extra
;
278 if (ret
== LZMA_STREAM_END
)
281 s
.avail_out
+= *buffer_size
;
283 if (!(greedy_realloc(buffer
, buffer_size
, *buffer_size
* 2, 1)))
286 s
.next_out
= *(uint8_t**)buffer
+ *buffer_size
- s
.avail_out
;
290 return -EPROTONOSUPPORT
;
294 int decompress_startswith_lz4(const void *src
, uint64_t src_size
,
295 void **buffer
, size_t *buffer_size
,
296 const void *prefix
, size_t prefix_len
,
299 /* Checks whether the decompressed blob starts with the
300 * mentioned prefix. The byte extra needs to follow the
307 assert(src_size
> 0);
311 assert(*buffer_size
== 0 || *buffer
);
316 if (!(greedy_realloc(buffer
, buffer_size
, ALIGN_8(prefix_len
+ 1), 1)))
319 r
= LZ4_decompress_safe_partial((char*)src
+ 8, *buffer
, src_size
- 8,
320 prefix_len
+ 1, *buffer_size
);
324 /* lz4 always tries to decode full "sequence", so in
325 * pathological cases might need to decompress the
327 r
= decompress_blob_lz4(src
, src_size
, buffer
, buffer_size
, &size
, 0);
332 if (size
>= prefix_len
+ 1)
333 return memcmp(*buffer
, prefix
, prefix_len
) == 0 &&
334 ((const uint8_t*) *buffer
)[prefix_len
] == extra
;
339 return -EPROTONOSUPPORT
;
343 int decompress_startswith(int compression
,
344 const void *src
, uint64_t src_size
,
345 void **buffer
, size_t *buffer_size
,
346 const void *prefix
, size_t prefix_len
,
348 if (compression
== OBJECT_COMPRESSED_XZ
)
349 return decompress_startswith_xz(src
, src_size
,
353 else if (compression
== OBJECT_COMPRESSED_LZ4
)
354 return decompress_startswith_lz4(src
, src_size
,
362 int compress_stream_xz(int fdf
, int fdt
, uint64_t max_bytes
) {
364 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
366 uint8_t buf
[BUFSIZ
], out
[BUFSIZ
];
367 lzma_action action
= LZMA_RUN
;
372 ret
= lzma_easy_encoder(&s
, LZMA_PRESET_DEFAULT
, LZMA_CHECK_CRC64
);
373 if (ret
!= LZMA_OK
) {
374 log_error("Failed to initialize XZ encoder: code %u", ret
);
379 if (s
.avail_in
== 0 && action
== LZMA_RUN
) {
380 size_t m
= sizeof(buf
);
383 if (max_bytes
!= (uint64_t) -1 && (uint64_t) m
> max_bytes
)
384 m
= (size_t) max_bytes
;
386 n
= read(fdf
, buf
, m
);
390 action
= LZMA_FINISH
;
395 if (max_bytes
!= (uint64_t) -1) {
396 assert(max_bytes
>= (uint64_t) n
);
402 if (s
.avail_out
== 0) {
404 s
.avail_out
= sizeof(out
);
407 ret
= lzma_code(&s
, action
);
408 if (!IN_SET(ret
, LZMA_OK
, LZMA_STREAM_END
)) {
409 log_error("Compression failed: code %u", ret
);
413 if (s
.avail_out
== 0 || ret
== LZMA_STREAM_END
) {
416 n
= sizeof(out
) - s
.avail_out
;
418 k
= loop_write(fdt
, out
, n
, false);
422 if (ret
== LZMA_STREAM_END
) {
423 log_debug("XZ compression finished (%"PRIu64
" -> %"PRIu64
" bytes, %.1f%%)",
424 s
.total_in
, s
.total_out
,
425 (double) s
.total_out
/ s
.total_in
* 100);
432 return -EPROTONOSUPPORT
;
436 #define LZ4_BUFSIZE (512*1024u)
438 int compress_stream_lz4(int fdf
, int fdt
, uint64_t max_bytes
) {
442 _cleanup_(LZ4F_freeCompressionContextp
) LZ4F_compressionContext_t ctx
= NULL
;
443 _cleanup_free_
char *buf
= NULL
;
445 size_t size
, n
, total_in
= 0, total_out
, offset
= 0, frame_size
;
448 static const LZ4F_compressOptions_t options
= {
451 static const LZ4F_preferences_t preferences
= {
452 .frameInfo
.blockSizeID
= 5,
455 c
= LZ4F_createCompressionContext(&ctx
, LZ4F_VERSION
);
459 if (fstat(fdf
, &st
) < 0)
460 return log_debug_errno(errno
, "fstat() failed: %m");
462 frame_size
= LZ4F_compressBound(LZ4_BUFSIZE
, &preferences
);
463 size
= frame_size
+ 64*1024; /* add some space for header and trailer */
468 n
= offset
= total_out
= LZ4F_compressBegin(ctx
, buf
, size
, &preferences
);
472 src
= mmap(NULL
, st
.st_size
, PROT_READ
, MAP_PRIVATE
, fdf
, 0);
473 if (src
== MAP_FAILED
)
476 log_debug("Buffer size is %zu bytes, header size %zu bytes.", size
, n
);
478 while (total_in
< (size_t) st
.st_size
) {
481 k
= MIN(LZ4_BUFSIZE
, st
.st_size
- total_in
);
482 n
= LZ4F_compressUpdate(ctx
, buf
+ offset
, size
- offset
,
483 src
+ total_in
, k
, &options
);
484 if (LZ4F_isError(n
)) {
485 r
= -ENOTRECOVERABLE
;
493 if (max_bytes
!= (uint64_t) -1 && total_out
> (size_t) max_bytes
) {
494 log_debug("Compressed stream longer than %"PRIu64
" bytes", max_bytes
);
498 if (size
- offset
< frame_size
+ 4) {
499 k
= loop_write(fdt
, buf
, offset
, false);
508 n
= LZ4F_compressEnd(ctx
, buf
+ offset
, size
- offset
, &options
);
509 if (LZ4F_isError(n
)) {
510 r
= -ENOTRECOVERABLE
;
516 r
= loop_write(fdt
, buf
, offset
, false);
520 log_debug("LZ4 compression finished (%zu -> %zu bytes, %.1f%%)",
522 (double) total_out
/ total_in
* 100);
524 munmap(src
, st
.st_size
);
527 return -EPROTONOSUPPORT
;
531 int decompress_stream_xz(int fdf
, int fdt
, uint64_t max_bytes
) {
534 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
537 uint8_t buf
[BUFSIZ
], out
[BUFSIZ
];
538 lzma_action action
= LZMA_RUN
;
543 ret
= lzma_stream_decoder(&s
, UINT64_MAX
, 0);
544 if (ret
!= LZMA_OK
) {
545 log_debug("Failed to initialize XZ decoder: code %u", ret
);
550 if (s
.avail_in
== 0 && action
== LZMA_RUN
) {
553 n
= read(fdf
, buf
, sizeof(buf
));
557 action
= LZMA_FINISH
;
564 if (s
.avail_out
== 0) {
566 s
.avail_out
= sizeof(out
);
569 ret
= lzma_code(&s
, action
);
570 if (!IN_SET(ret
, LZMA_OK
, LZMA_STREAM_END
)) {
571 log_debug("Decompression failed: code %u", ret
);
575 if (s
.avail_out
== 0 || ret
== LZMA_STREAM_END
) {
578 n
= sizeof(out
) - s
.avail_out
;
580 if (max_bytes
!= (uint64_t) -1) {
581 if (max_bytes
< (uint64_t) n
)
587 k
= loop_write(fdt
, out
, n
, false);
591 if (ret
== LZMA_STREAM_END
) {
592 log_debug("XZ decompression finished (%"PRIu64
" -> %"PRIu64
" bytes, %.1f%%)",
593 s
.total_in
, s
.total_out
,
594 (double) s
.total_out
/ s
.total_in
* 100);
601 log_debug("Cannot decompress file. Compiled without XZ support.");
602 return -EPROTONOSUPPORT
;
606 int decompress_stream_lz4(int in
, int out
, uint64_t max_bytes
) {
609 _cleanup_(LZ4F_freeDecompressionContextp
) LZ4F_decompressionContext_t ctx
= NULL
;
610 _cleanup_free_
char *buf
= NULL
;
614 size_t total_in
= 0, total_out
= 0;
616 c
= LZ4F_createDecompressionContext(&ctx
, LZ4F_VERSION
);
620 if (fstat(in
, &st
) < 0)
621 return log_debug_errno(errno
, "fstat() failed: %m");
623 buf
= malloc(LZ4_BUFSIZE
);
627 src
= mmap(NULL
, st
.st_size
, PROT_READ
, MAP_PRIVATE
, in
, 0);
628 if (src
== MAP_FAILED
)
631 while (total_in
< (size_t) st
.st_size
) {
632 size_t produced
= LZ4_BUFSIZE
;
633 size_t used
= st
.st_size
- total_in
;
635 c
= LZ4F_decompress(ctx
, buf
, &produced
, src
+ total_in
, &used
, NULL
);
636 if (LZ4F_isError(c
)) {
642 total_out
+= produced
;
644 if (max_bytes
!= (uint64_t) -1 && total_out
> (size_t) max_bytes
) {
645 log_debug("Decompressed stream longer than %"PRIu64
" bytes", max_bytes
);
650 r
= loop_write(out
, buf
, produced
, false);
655 log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)",
657 total_in
> 0 ? (double) total_out
/ total_in
* 100 : 0.0);
659 munmap(src
, st
.st_size
);
662 log_debug("Cannot decompress file. Compiled without LZ4 support.");
663 return -EPROTONOSUPPORT
;
667 int decompress_stream(const char *filename
, int fdf
, int fdt
, uint64_t max_bytes
) {
669 if (endswith(filename
, ".lz4"))
670 return decompress_stream_lz4(fdf
, fdt
, max_bytes
);
671 else if (endswith(filename
, ".xz"))
672 return decompress_stream_xz(fdf
, fdt
, max_bytes
);
674 return -EPROTONOSUPPORT
;