1 /* SPDX-License-Identifier: LGPL-2.1+ */
3 This file is part of systemd.
5 Copyright 2011 Lennart Poettering
7 systemd is free software; you can redistribute it and/or modify it
8 under the terms of the GNU Lesser General Public License as published by
9 the Free Software Foundation; either version 2.1 of the License, or
10 (at your option) any later version.
12 systemd is distributed in the hope that it will be useful, but
13 WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 Lesser General Public License for more details.
17 You should have received a copy of the GNU Lesser General Public License
18 along with systemd; If not, see <http://www.gnu.org/licenses/>.
36 #include "alloc-util.h"
40 #include "journal-def.h"
42 #include "sparse-endian.h"
43 #include "string-table.h"
44 #include "string-util.h"
48 DEFINE_TRIVIAL_CLEANUP_FUNC(LZ4F_compressionContext_t
, LZ4F_freeCompressionContext
);
49 DEFINE_TRIVIAL_CLEANUP_FUNC(LZ4F_decompressionContext_t
, LZ4F_freeDecompressionContext
);
52 #define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t))
54 static const char* const object_compressed_table
[_OBJECT_COMPRESSED_MAX
] = {
55 [OBJECT_COMPRESSED_XZ
] = "XZ",
56 [OBJECT_COMPRESSED_LZ4
] = "LZ4",
59 DEFINE_STRING_TABLE_LOOKUP(object_compressed
, int);
61 int compress_blob_xz(const void *src
, uint64_t src_size
,
62 void *dst
, size_t dst_alloc_size
, size_t *dst_size
) {
64 static const lzma_options_lzma opt
= {
65 1u << 20u, NULL
, 0, LZMA_LC_DEFAULT
, LZMA_LP_DEFAULT
,
66 LZMA_PB_DEFAULT
, LZMA_MODE_FAST
, 128, LZMA_MF_HC3
, 4
68 static const lzma_filter filters
[] = {
69 { LZMA_FILTER_LZMA2
, (lzma_options_lzma
*) &opt
},
70 { LZMA_VLI_UNKNOWN
, NULL
}
78 assert(dst_alloc_size
> 0);
81 /* Returns < 0 if we couldn't compress the data or the
82 * compressed result is longer than the original */
87 ret
= lzma_stream_buffer_encode((lzma_filter
*) filters
, LZMA_CHECK_NONE
, NULL
,
88 src
, src_size
, dst
, &out_pos
, dst_alloc_size
);
95 return -EPROTONOSUPPORT
;
99 int compress_blob_lz4(const void *src
, uint64_t src_size
,
100 void *dst
, size_t dst_alloc_size
, size_t *dst_size
) {
105 assert(src_size
> 0);
107 assert(dst_alloc_size
> 0);
110 /* Returns < 0 if we couldn't compress the data or the
111 * compressed result is longer than the original */
116 #if LZ4_VERSION_NUMBER >= 10700
117 r
= LZ4_compress_default(src
, (char*)dst
+ 8, src_size
, (int) dst_alloc_size
- 8);
119 r
= LZ4_compress_limitedOutput(src
, (char*)dst
+ 8, src_size
, (int) dst_alloc_size
- 8);
124 *(le64_t
*) dst
= htole64(src_size
);
129 return -EPROTONOSUPPORT
;
134 int decompress_blob_xz(const void *src
, uint64_t src_size
,
135 void **dst
, size_t *dst_alloc_size
, size_t* dst_size
, size_t dst_max
) {
138 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
143 assert(src_size
> 0);
145 assert(dst_alloc_size
);
147 assert(*dst_alloc_size
== 0 || *dst
);
149 ret
= lzma_stream_decoder(&s
, UINT64_MAX
, 0);
153 space
= MIN(src_size
* 2, dst_max
?: (size_t) -1);
154 if (!greedy_realloc(dst
, dst_alloc_size
, space
, 1))
158 s
.avail_in
= src_size
;
166 ret
= lzma_code(&s
, LZMA_FINISH
);
168 if (ret
== LZMA_STREAM_END
)
170 else if (ret
!= LZMA_OK
)
173 if (dst_max
> 0 && (space
- s
.avail_out
) >= dst_max
)
175 else if (dst_max
> 0 && space
== dst_max
)
178 used
= space
- s
.avail_out
;
179 space
= MIN(2 * space
, dst_max
?: (size_t) -1);
180 if (!greedy_realloc(dst
, dst_alloc_size
, space
, 1))
183 s
.avail_out
= space
- used
;
184 s
.next_out
= *(uint8_t**)dst
+ used
;
187 *dst_size
= space
- s
.avail_out
;
190 return -EPROTONOSUPPORT
;
194 int decompress_blob_lz4(const void *src
, uint64_t src_size
,
195 void **dst
, size_t *dst_alloc_size
, size_t* dst_size
, size_t dst_max
) {
199 int r
, size
; /* LZ4 uses int for size */
202 assert(src_size
> 0);
204 assert(dst_alloc_size
);
206 assert(*dst_alloc_size
== 0 || *dst
);
211 size
= le64toh( *(le64_t
*)src
);
212 if (size
< 0 || (unsigned) size
!= le64toh(*(le64_t
*)src
))
214 if ((size_t) size
> *dst_alloc_size
) {
215 out
= realloc(*dst
, size
);
219 *dst_alloc_size
= size
;
223 r
= LZ4_decompress_safe((char*)src
+ 8, out
, src_size
- 8, size
);
224 if (r
< 0 || r
!= size
)
230 return -EPROTONOSUPPORT
;
234 int decompress_blob(int compression
,
235 const void *src
, uint64_t src_size
,
236 void **dst
, size_t *dst_alloc_size
, size_t* dst_size
, size_t dst_max
) {
237 if (compression
== OBJECT_COMPRESSED_XZ
)
238 return decompress_blob_xz(src
, src_size
,
239 dst
, dst_alloc_size
, dst_size
, dst_max
);
240 else if (compression
== OBJECT_COMPRESSED_LZ4
)
241 return decompress_blob_lz4(src
, src_size
,
242 dst
, dst_alloc_size
, dst_size
, dst_max
);
248 int decompress_startswith_xz(const void *src
, uint64_t src_size
,
249 void **buffer
, size_t *buffer_size
,
250 const void *prefix
, size_t prefix_len
,
254 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
257 /* Checks whether the decompressed blob starts with the
258 * mentioned prefix. The byte extra needs to follow the
262 assert(src_size
> 0);
266 assert(*buffer_size
== 0 || *buffer
);
268 ret
= lzma_stream_decoder(&s
, UINT64_MAX
, 0);
272 if (!(greedy_realloc(buffer
, buffer_size
, ALIGN_8(prefix_len
+ 1), 1)))
276 s
.avail_in
= src_size
;
278 s
.next_out
= *buffer
;
279 s
.avail_out
= *buffer_size
;
282 ret
= lzma_code(&s
, LZMA_FINISH
);
284 if (!IN_SET(ret
, LZMA_OK
, LZMA_STREAM_END
))
287 if (*buffer_size
- s
.avail_out
>= prefix_len
+ 1)
288 return memcmp(*buffer
, prefix
, prefix_len
) == 0 &&
289 ((const uint8_t*) *buffer
)[prefix_len
] == extra
;
291 if (ret
== LZMA_STREAM_END
)
294 s
.avail_out
+= *buffer_size
;
296 if (!(greedy_realloc(buffer
, buffer_size
, *buffer_size
* 2, 1)))
299 s
.next_out
= *(uint8_t**)buffer
+ *buffer_size
- s
.avail_out
;
303 return -EPROTONOSUPPORT
;
307 int decompress_startswith_lz4(const void *src
, uint64_t src_size
,
308 void **buffer
, size_t *buffer_size
,
309 const void *prefix
, size_t prefix_len
,
312 /* Checks whether the decompressed blob starts with the
313 * mentioned prefix. The byte extra needs to follow the
320 assert(src_size
> 0);
324 assert(*buffer_size
== 0 || *buffer
);
329 if (!(greedy_realloc(buffer
, buffer_size
, ALIGN_8(prefix_len
+ 1), 1)))
332 r
= LZ4_decompress_safe_partial((char*)src
+ 8, *buffer
, src_size
- 8,
333 prefix_len
+ 1, *buffer_size
);
337 /* lz4 always tries to decode full "sequence", so in
338 * pathological cases might need to decompress the
340 r
= decompress_blob_lz4(src
, src_size
, buffer
, buffer_size
, &size
, 0);
345 if (size
>= prefix_len
+ 1)
346 return memcmp(*buffer
, prefix
, prefix_len
) == 0 &&
347 ((const uint8_t*) *buffer
)[prefix_len
] == extra
;
352 return -EPROTONOSUPPORT
;
356 int decompress_startswith(int compression
,
357 const void *src
, uint64_t src_size
,
358 void **buffer
, size_t *buffer_size
,
359 const void *prefix
, size_t prefix_len
,
361 if (compression
== OBJECT_COMPRESSED_XZ
)
362 return decompress_startswith_xz(src
, src_size
,
366 else if (compression
== OBJECT_COMPRESSED_LZ4
)
367 return decompress_startswith_lz4(src
, src_size
,
375 int compress_stream_xz(int fdf
, int fdt
, uint64_t max_bytes
) {
377 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
379 uint8_t buf
[BUFSIZ
], out
[BUFSIZ
];
380 lzma_action action
= LZMA_RUN
;
385 ret
= lzma_easy_encoder(&s
, LZMA_PRESET_DEFAULT
, LZMA_CHECK_CRC64
);
386 if (ret
!= LZMA_OK
) {
387 log_error("Failed to initialize XZ encoder: code %u", ret
);
392 if (s
.avail_in
== 0 && action
== LZMA_RUN
) {
393 size_t m
= sizeof(buf
);
396 if (max_bytes
!= (uint64_t) -1 && (uint64_t) m
> max_bytes
)
397 m
= (size_t) max_bytes
;
399 n
= read(fdf
, buf
, m
);
403 action
= LZMA_FINISH
;
408 if (max_bytes
!= (uint64_t) -1) {
409 assert(max_bytes
>= (uint64_t) n
);
415 if (s
.avail_out
== 0) {
417 s
.avail_out
= sizeof(out
);
420 ret
= lzma_code(&s
, action
);
421 if (!IN_SET(ret
, LZMA_OK
, LZMA_STREAM_END
)) {
422 log_error("Compression failed: code %u", ret
);
426 if (s
.avail_out
== 0 || ret
== LZMA_STREAM_END
) {
429 n
= sizeof(out
) - s
.avail_out
;
431 k
= loop_write(fdt
, out
, n
, false);
435 if (ret
== LZMA_STREAM_END
) {
436 log_debug("XZ compression finished (%"PRIu64
" -> %"PRIu64
" bytes, %.1f%%)",
437 s
.total_in
, s
.total_out
,
438 (double) s
.total_out
/ s
.total_in
* 100);
445 return -EPROTONOSUPPORT
;
449 #define LZ4_BUFSIZE (512*1024u)
451 int compress_stream_lz4(int fdf
, int fdt
, uint64_t max_bytes
) {
455 _cleanup_(LZ4F_freeCompressionContextp
) LZ4F_compressionContext_t ctx
= NULL
;
456 _cleanup_free_
char *buf
= NULL
;
458 size_t size
, n
, total_in
= 0, total_out
, offset
= 0, frame_size
;
461 static const LZ4F_compressOptions_t options
= {
464 static const LZ4F_preferences_t preferences
= {
465 .frameInfo
.blockSizeID
= 5,
468 c
= LZ4F_createCompressionContext(&ctx
, LZ4F_VERSION
);
472 if (fstat(fdf
, &st
) < 0)
473 return log_debug_errno(errno
, "fstat() failed: %m");
475 frame_size
= LZ4F_compressBound(LZ4_BUFSIZE
, &preferences
);
476 size
= frame_size
+ 64*1024; /* add some space for header and trailer */
481 n
= offset
= total_out
= LZ4F_compressBegin(ctx
, buf
, size
, &preferences
);
485 src
= mmap(NULL
, st
.st_size
, PROT_READ
, MAP_PRIVATE
, fdf
, 0);
486 if (src
== MAP_FAILED
)
489 log_debug("Buffer size is %zu bytes, header size %zu bytes.", size
, n
);
491 while (total_in
< (size_t) st
.st_size
) {
494 k
= MIN(LZ4_BUFSIZE
, st
.st_size
- total_in
);
495 n
= LZ4F_compressUpdate(ctx
, buf
+ offset
, size
- offset
,
496 src
+ total_in
, k
, &options
);
497 if (LZ4F_isError(n
)) {
498 r
= -ENOTRECOVERABLE
;
506 if (max_bytes
!= (uint64_t) -1 && total_out
> (size_t) max_bytes
) {
507 log_debug("Compressed stream longer than %"PRIu64
" bytes", max_bytes
);
511 if (size
- offset
< frame_size
+ 4) {
512 k
= loop_write(fdt
, buf
, offset
, false);
521 n
= LZ4F_compressEnd(ctx
, buf
+ offset
, size
- offset
, &options
);
522 if (LZ4F_isError(n
)) {
523 r
= -ENOTRECOVERABLE
;
529 r
= loop_write(fdt
, buf
, offset
, false);
533 log_debug("LZ4 compression finished (%zu -> %zu bytes, %.1f%%)",
535 (double) total_out
/ total_in
* 100);
537 munmap(src
, st
.st_size
);
540 return -EPROTONOSUPPORT
;
544 int decompress_stream_xz(int fdf
, int fdt
, uint64_t max_bytes
) {
547 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
550 uint8_t buf
[BUFSIZ
], out
[BUFSIZ
];
551 lzma_action action
= LZMA_RUN
;
556 ret
= lzma_stream_decoder(&s
, UINT64_MAX
, 0);
557 if (ret
!= LZMA_OK
) {
558 log_debug("Failed to initialize XZ decoder: code %u", ret
);
563 if (s
.avail_in
== 0 && action
== LZMA_RUN
) {
566 n
= read(fdf
, buf
, sizeof(buf
));
570 action
= LZMA_FINISH
;
577 if (s
.avail_out
== 0) {
579 s
.avail_out
= sizeof(out
);
582 ret
= lzma_code(&s
, action
);
583 if (!IN_SET(ret
, LZMA_OK
, LZMA_STREAM_END
)) {
584 log_debug("Decompression failed: code %u", ret
);
588 if (s
.avail_out
== 0 || ret
== LZMA_STREAM_END
) {
591 n
= sizeof(out
) - s
.avail_out
;
593 if (max_bytes
!= (uint64_t) -1) {
594 if (max_bytes
< (uint64_t) n
)
600 k
= loop_write(fdt
, out
, n
, false);
604 if (ret
== LZMA_STREAM_END
) {
605 log_debug("XZ decompression finished (%"PRIu64
" -> %"PRIu64
" bytes, %.1f%%)",
606 s
.total_in
, s
.total_out
,
607 (double) s
.total_out
/ s
.total_in
* 100);
614 log_debug("Cannot decompress file. Compiled without XZ support.");
615 return -EPROTONOSUPPORT
;
619 int decompress_stream_lz4(int in
, int out
, uint64_t max_bytes
) {
622 _cleanup_(LZ4F_freeDecompressionContextp
) LZ4F_decompressionContext_t ctx
= NULL
;
623 _cleanup_free_
char *buf
= NULL
;
627 size_t total_in
= 0, total_out
= 0;
629 c
= LZ4F_createDecompressionContext(&ctx
, LZ4F_VERSION
);
633 if (fstat(in
, &st
) < 0)
634 return log_debug_errno(errno
, "fstat() failed: %m");
636 buf
= malloc(LZ4_BUFSIZE
);
640 src
= mmap(NULL
, st
.st_size
, PROT_READ
, MAP_PRIVATE
, in
, 0);
641 if (src
== MAP_FAILED
)
644 while (total_in
< (size_t) st
.st_size
) {
645 size_t produced
= LZ4_BUFSIZE
;
646 size_t used
= st
.st_size
- total_in
;
648 c
= LZ4F_decompress(ctx
, buf
, &produced
, src
+ total_in
, &used
, NULL
);
649 if (LZ4F_isError(c
)) {
655 total_out
+= produced
;
657 if (max_bytes
!= (uint64_t) -1 && total_out
> (size_t) max_bytes
) {
658 log_debug("Decompressed stream longer than %"PRIu64
" bytes", max_bytes
);
663 r
= loop_write(out
, buf
, produced
, false);
668 log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)",
670 total_in
> 0 ? (double) total_out
/ total_in
* 100 : 0.0);
672 munmap(src
, st
.st_size
);
675 log_debug("Cannot decompress file. Compiled without LZ4 support.");
676 return -EPROTONOSUPPORT
;
680 int decompress_stream(const char *filename
, int fdf
, int fdt
, uint64_t max_bytes
) {
682 if (endswith(filename
, ".lz4"))
683 return decompress_stream_lz4(fdf
, fdt
, max_bytes
);
684 else if (endswith(filename
, ".xz"))
685 return decompress_stream_xz(fdf
, fdt
, max_bytes
);
687 return -EPROTONOSUPPORT
;