1 /* SPDX-License-Identifier: LGPL-2.1+ */
18 #include "alloc-util.h"
22 #include "journal-def.h"
24 #include "sparse-endian.h"
25 #include "string-table.h"
26 #include "string-util.h"
30 DEFINE_TRIVIAL_CLEANUP_FUNC(LZ4F_compressionContext_t
, LZ4F_freeCompressionContext
);
31 DEFINE_TRIVIAL_CLEANUP_FUNC(LZ4F_decompressionContext_t
, LZ4F_freeDecompressionContext
);
34 #define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t))
36 static const char* const object_compressed_table
[_OBJECT_COMPRESSED_MAX
] = {
37 [OBJECT_COMPRESSED_XZ
] = "XZ",
38 [OBJECT_COMPRESSED_LZ4
] = "LZ4",
41 DEFINE_STRING_TABLE_LOOKUP(object_compressed
, int);
43 int compress_blob_xz(const void *src
, uint64_t src_size
,
44 void *dst
, size_t dst_alloc_size
, size_t *dst_size
) {
46 static const lzma_options_lzma opt
= {
47 1u << 20u, NULL
, 0, LZMA_LC_DEFAULT
, LZMA_LP_DEFAULT
,
48 LZMA_PB_DEFAULT
, LZMA_MODE_FAST
, 128, LZMA_MF_HC3
, 4
50 static const lzma_filter filters
[] = {
51 { LZMA_FILTER_LZMA2
, (lzma_options_lzma
*) &opt
},
52 { LZMA_VLI_UNKNOWN
, NULL
}
60 assert(dst_alloc_size
> 0);
63 /* Returns < 0 if we couldn't compress the data or the
64 * compressed result is longer than the original */
69 ret
= lzma_stream_buffer_encode((lzma_filter
*) filters
, LZMA_CHECK_NONE
, NULL
,
70 src
, src_size
, dst
, &out_pos
, dst_alloc_size
);
77 return -EPROTONOSUPPORT
;
81 int compress_blob_lz4(const void *src
, uint64_t src_size
,
82 void *dst
, size_t dst_alloc_size
, size_t *dst_size
) {
89 assert(dst_alloc_size
> 0);
92 /* Returns < 0 if we couldn't compress the data or the
93 * compressed result is longer than the original */
98 #if LZ4_VERSION_NUMBER >= 10700
99 r
= LZ4_compress_default(src
, (char*)dst
+ 8, src_size
, (int) dst_alloc_size
- 8);
101 r
= LZ4_compress_limitedOutput(src
, (char*)dst
+ 8, src_size
, (int) dst_alloc_size
- 8);
106 *(le64_t
*) dst
= htole64(src_size
);
111 return -EPROTONOSUPPORT
;
115 int decompress_blob_xz(const void *src
, uint64_t src_size
,
116 void **dst
, size_t *dst_alloc_size
, size_t* dst_size
, size_t dst_max
) {
119 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
124 assert(src_size
> 0);
126 assert(dst_alloc_size
);
128 assert(*dst_alloc_size
== 0 || *dst
);
130 ret
= lzma_stream_decoder(&s
, UINT64_MAX
, 0);
134 space
= MIN(src_size
* 2, dst_max
?: (size_t) -1);
135 if (!greedy_realloc(dst
, dst_alloc_size
, space
, 1))
139 s
.avail_in
= src_size
;
147 ret
= lzma_code(&s
, LZMA_FINISH
);
149 if (ret
== LZMA_STREAM_END
)
151 else if (ret
!= LZMA_OK
)
154 if (dst_max
> 0 && (space
- s
.avail_out
) >= dst_max
)
156 else if (dst_max
> 0 && space
== dst_max
)
159 used
= space
- s
.avail_out
;
160 space
= MIN(2 * space
, dst_max
?: (size_t) -1);
161 if (!greedy_realloc(dst
, dst_alloc_size
, space
, 1))
164 s
.avail_out
= space
- used
;
165 s
.next_out
= *(uint8_t**)dst
+ used
;
168 *dst_size
= space
- s
.avail_out
;
171 return -EPROTONOSUPPORT
;
175 int decompress_blob_lz4(const void *src
, uint64_t src_size
,
176 void **dst
, size_t *dst_alloc_size
, size_t* dst_size
, size_t dst_max
) {
180 int r
, size
; /* LZ4 uses int for size */
183 assert(src_size
> 0);
185 assert(dst_alloc_size
);
187 assert(*dst_alloc_size
== 0 || *dst
);
192 size
= le64toh( *(le64_t
*)src
);
193 if (size
< 0 || (unsigned) size
!= le64toh(*(le64_t
*)src
))
195 if ((size_t) size
> *dst_alloc_size
) {
196 out
= realloc(*dst
, size
);
200 *dst_alloc_size
= size
;
204 r
= LZ4_decompress_safe((char*)src
+ 8, out
, src_size
- 8, size
);
205 if (r
< 0 || r
!= size
)
211 return -EPROTONOSUPPORT
;
215 int decompress_blob(int compression
,
216 const void *src
, uint64_t src_size
,
217 void **dst
, size_t *dst_alloc_size
, size_t* dst_size
, size_t dst_max
) {
218 if (compression
== OBJECT_COMPRESSED_XZ
)
219 return decompress_blob_xz(src
, src_size
,
220 dst
, dst_alloc_size
, dst_size
, dst_max
);
221 else if (compression
== OBJECT_COMPRESSED_LZ4
)
222 return decompress_blob_lz4(src
, src_size
,
223 dst
, dst_alloc_size
, dst_size
, dst_max
);
228 int decompress_startswith_xz(const void *src
, uint64_t src_size
,
229 void **buffer
, size_t *buffer_size
,
230 const void *prefix
, size_t prefix_len
,
234 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
237 /* Checks whether the decompressed blob starts with the
238 * mentioned prefix. The byte extra needs to follow the
242 assert(src_size
> 0);
246 assert(*buffer_size
== 0 || *buffer
);
248 ret
= lzma_stream_decoder(&s
, UINT64_MAX
, 0);
252 if (!(greedy_realloc(buffer
, buffer_size
, ALIGN_8(prefix_len
+ 1), 1)))
256 s
.avail_in
= src_size
;
258 s
.next_out
= *buffer
;
259 s
.avail_out
= *buffer_size
;
262 ret
= lzma_code(&s
, LZMA_FINISH
);
264 if (!IN_SET(ret
, LZMA_OK
, LZMA_STREAM_END
))
267 if (*buffer_size
- s
.avail_out
>= prefix_len
+ 1)
268 return memcmp(*buffer
, prefix
, prefix_len
) == 0 &&
269 ((const uint8_t*) *buffer
)[prefix_len
] == extra
;
271 if (ret
== LZMA_STREAM_END
)
274 s
.avail_out
+= *buffer_size
;
276 if (!(greedy_realloc(buffer
, buffer_size
, *buffer_size
* 2, 1)))
279 s
.next_out
= *(uint8_t**)buffer
+ *buffer_size
- s
.avail_out
;
283 return -EPROTONOSUPPORT
;
287 int decompress_startswith_lz4(const void *src
, uint64_t src_size
,
288 void **buffer
, size_t *buffer_size
,
289 const void *prefix
, size_t prefix_len
,
292 /* Checks whether the decompressed blob starts with the
293 * mentioned prefix. The byte extra needs to follow the
300 assert(src_size
> 0);
304 assert(*buffer_size
== 0 || *buffer
);
309 if (!(greedy_realloc(buffer
, buffer_size
, ALIGN_8(prefix_len
+ 1), 1)))
312 r
= LZ4_decompress_safe_partial((char*)src
+ 8, *buffer
, src_size
- 8,
313 prefix_len
+ 1, *buffer_size
);
317 /* lz4 always tries to decode full "sequence", so in
318 * pathological cases might need to decompress the
320 r
= decompress_blob_lz4(src
, src_size
, buffer
, buffer_size
, &size
, 0);
325 if (size
>= prefix_len
+ 1)
326 return memcmp(*buffer
, prefix
, prefix_len
) == 0 &&
327 ((const uint8_t*) *buffer
)[prefix_len
] == extra
;
332 return -EPROTONOSUPPORT
;
336 int decompress_startswith(int compression
,
337 const void *src
, uint64_t src_size
,
338 void **buffer
, size_t *buffer_size
,
339 const void *prefix
, size_t prefix_len
,
341 if (compression
== OBJECT_COMPRESSED_XZ
)
342 return decompress_startswith_xz(src
, src_size
,
346 else if (compression
== OBJECT_COMPRESSED_LZ4
)
347 return decompress_startswith_lz4(src
, src_size
,
355 int compress_stream_xz(int fdf
, int fdt
, uint64_t max_bytes
) {
357 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
359 uint8_t buf
[BUFSIZ
], out
[BUFSIZ
];
360 lzma_action action
= LZMA_RUN
;
365 ret
= lzma_easy_encoder(&s
, LZMA_PRESET_DEFAULT
, LZMA_CHECK_CRC64
);
366 if (ret
!= LZMA_OK
) {
367 log_error("Failed to initialize XZ encoder: code %u", ret
);
372 if (s
.avail_in
== 0 && action
== LZMA_RUN
) {
373 size_t m
= sizeof(buf
);
376 if (max_bytes
!= (uint64_t) -1 && (uint64_t) m
> max_bytes
)
377 m
= (size_t) max_bytes
;
379 n
= read(fdf
, buf
, m
);
383 action
= LZMA_FINISH
;
388 if (max_bytes
!= (uint64_t) -1) {
389 assert(max_bytes
>= (uint64_t) n
);
395 if (s
.avail_out
== 0) {
397 s
.avail_out
= sizeof(out
);
400 ret
= lzma_code(&s
, action
);
401 if (!IN_SET(ret
, LZMA_OK
, LZMA_STREAM_END
)) {
402 log_error("Compression failed: code %u", ret
);
406 if (s
.avail_out
== 0 || ret
== LZMA_STREAM_END
) {
409 n
= sizeof(out
) - s
.avail_out
;
411 k
= loop_write(fdt
, out
, n
, false);
415 if (ret
== LZMA_STREAM_END
) {
416 log_debug("XZ compression finished (%"PRIu64
" -> %"PRIu64
" bytes, %.1f%%)",
417 s
.total_in
, s
.total_out
,
418 (double) s
.total_out
/ s
.total_in
* 100);
425 return -EPROTONOSUPPORT
;
429 #define LZ4_BUFSIZE (512*1024u)
431 int compress_stream_lz4(int fdf
, int fdt
, uint64_t max_bytes
) {
435 _cleanup_(LZ4F_freeCompressionContextp
) LZ4F_compressionContext_t ctx
= NULL
;
436 _cleanup_free_
char *buf
= NULL
;
438 size_t size
, n
, total_in
= 0, total_out
, offset
= 0, frame_size
;
441 static const LZ4F_compressOptions_t options
= {
444 static const LZ4F_preferences_t preferences
= {
445 .frameInfo
.blockSizeID
= 5,
448 c
= LZ4F_createCompressionContext(&ctx
, LZ4F_VERSION
);
452 if (fstat(fdf
, &st
) < 0)
453 return log_debug_errno(errno
, "fstat() failed: %m");
455 frame_size
= LZ4F_compressBound(LZ4_BUFSIZE
, &preferences
);
456 size
= frame_size
+ 64*1024; /* add some space for header and trailer */
461 n
= offset
= total_out
= LZ4F_compressBegin(ctx
, buf
, size
, &preferences
);
465 src
= mmap(NULL
, st
.st_size
, PROT_READ
, MAP_PRIVATE
, fdf
, 0);
466 if (src
== MAP_FAILED
)
469 log_debug("Buffer size is %zu bytes, header size %zu bytes.", size
, n
);
471 while (total_in
< (size_t) st
.st_size
) {
474 k
= MIN(LZ4_BUFSIZE
, st
.st_size
- total_in
);
475 n
= LZ4F_compressUpdate(ctx
, buf
+ offset
, size
- offset
,
476 src
+ total_in
, k
, &options
);
477 if (LZ4F_isError(n
)) {
478 r
= -ENOTRECOVERABLE
;
486 if (max_bytes
!= (uint64_t) -1 && total_out
> (size_t) max_bytes
) {
487 log_debug("Compressed stream longer than %"PRIu64
" bytes", max_bytes
);
491 if (size
- offset
< frame_size
+ 4) {
492 k
= loop_write(fdt
, buf
, offset
, false);
501 n
= LZ4F_compressEnd(ctx
, buf
+ offset
, size
- offset
, &options
);
502 if (LZ4F_isError(n
)) {
503 r
= -ENOTRECOVERABLE
;
509 r
= loop_write(fdt
, buf
, offset
, false);
513 log_debug("LZ4 compression finished (%zu -> %zu bytes, %.1f%%)",
515 (double) total_out
/ total_in
* 100);
517 munmap(src
, st
.st_size
);
520 return -EPROTONOSUPPORT
;
524 int decompress_stream_xz(int fdf
, int fdt
, uint64_t max_bytes
) {
527 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
530 uint8_t buf
[BUFSIZ
], out
[BUFSIZ
];
531 lzma_action action
= LZMA_RUN
;
536 ret
= lzma_stream_decoder(&s
, UINT64_MAX
, 0);
537 if (ret
!= LZMA_OK
) {
538 log_debug("Failed to initialize XZ decoder: code %u", ret
);
543 if (s
.avail_in
== 0 && action
== LZMA_RUN
) {
546 n
= read(fdf
, buf
, sizeof(buf
));
550 action
= LZMA_FINISH
;
557 if (s
.avail_out
== 0) {
559 s
.avail_out
= sizeof(out
);
562 ret
= lzma_code(&s
, action
);
563 if (!IN_SET(ret
, LZMA_OK
, LZMA_STREAM_END
)) {
564 log_debug("Decompression failed: code %u", ret
);
568 if (s
.avail_out
== 0 || ret
== LZMA_STREAM_END
) {
571 n
= sizeof(out
) - s
.avail_out
;
573 if (max_bytes
!= (uint64_t) -1) {
574 if (max_bytes
< (uint64_t) n
)
580 k
= loop_write(fdt
, out
, n
, false);
584 if (ret
== LZMA_STREAM_END
) {
585 log_debug("XZ decompression finished (%"PRIu64
" -> %"PRIu64
" bytes, %.1f%%)",
586 s
.total_in
, s
.total_out
,
587 (double) s
.total_out
/ s
.total_in
* 100);
594 log_debug("Cannot decompress file. Compiled without XZ support.");
595 return -EPROTONOSUPPORT
;
599 int decompress_stream_lz4(int in
, int out
, uint64_t max_bytes
) {
602 _cleanup_(LZ4F_freeDecompressionContextp
) LZ4F_decompressionContext_t ctx
= NULL
;
603 _cleanup_free_
char *buf
= NULL
;
607 size_t total_in
= 0, total_out
= 0;
609 c
= LZ4F_createDecompressionContext(&ctx
, LZ4F_VERSION
);
613 if (fstat(in
, &st
) < 0)
614 return log_debug_errno(errno
, "fstat() failed: %m");
616 buf
= malloc(LZ4_BUFSIZE
);
620 src
= mmap(NULL
, st
.st_size
, PROT_READ
, MAP_PRIVATE
, in
, 0);
621 if (src
== MAP_FAILED
)
624 while (total_in
< (size_t) st
.st_size
) {
625 size_t produced
= LZ4_BUFSIZE
;
626 size_t used
= st
.st_size
- total_in
;
628 c
= LZ4F_decompress(ctx
, buf
, &produced
, src
+ total_in
, &used
, NULL
);
629 if (LZ4F_isError(c
)) {
635 total_out
+= produced
;
637 if (max_bytes
!= (uint64_t) -1 && total_out
> (size_t) max_bytes
) {
638 log_debug("Decompressed stream longer than %"PRIu64
" bytes", max_bytes
);
643 r
= loop_write(out
, buf
, produced
, false);
648 log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)",
650 total_in
> 0 ? (double) total_out
/ total_in
* 100 : 0.0);
652 munmap(src
, st
.st_size
);
655 log_debug("Cannot decompress file. Compiled without LZ4 support.");
656 return -EPROTONOSUPPORT
;
660 int decompress_stream(const char *filename
, int fdf
, int fdt
, uint64_t max_bytes
) {
662 if (endswith(filename
, ".lz4"))
663 return decompress_stream_lz4(fdf
, fdt
, max_bytes
);
664 else if (endswith(filename
, ".xz"))
665 return decompress_stream_xz(fdf
, fdt
, max_bytes
);
667 return -EPROTONOSUPPORT
;