1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
4 This file is part of systemd.
6 Copyright 2011 Lennart Poettering
8 systemd is free software; you can redistribute it and/or modify it
9 under the terms of the GNU Lesser General Public License as published by
10 the Free Software Foundation; either version 2.1 of the License, or
11 (at your option) any later version.
13 systemd is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 Lesser General Public License for more details.
18 You should have received a copy of the GNU Lesser General Public License
19 along with systemd; If not, see <http://www.gnu.org/licenses/>.
36 #include "alloc-util.h"
40 #include "journal-def.h"
42 #include "sparse-endian.h"
43 #include "string-table.h"
44 #include "string-util.h"
48 DEFINE_TRIVIAL_CLEANUP_FUNC(LZ4F_compressionContext_t
, LZ4F_freeCompressionContext
);
49 DEFINE_TRIVIAL_CLEANUP_FUNC(LZ4F_decompressionContext_t
, LZ4F_freeDecompressionContext
);
52 #define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t))
54 static const char* const object_compressed_table
[_OBJECT_COMPRESSED_MAX
] = {
55 [OBJECT_COMPRESSED_XZ
] = "XZ",
56 [OBJECT_COMPRESSED_LZ4
] = "LZ4",
59 DEFINE_STRING_TABLE_LOOKUP(object_compressed
, int);
61 int compress_blob_xz(const void *src
, uint64_t src_size
, void *dst
, size_t *dst_size
) {
63 static const lzma_options_lzma opt
= {
64 1u << 20u, NULL
, 0, LZMA_LC_DEFAULT
, LZMA_LP_DEFAULT
,
65 LZMA_PB_DEFAULT
, LZMA_MODE_FAST
, 128, LZMA_MF_HC3
, 4
67 static const lzma_filter filters
[] = {
68 { LZMA_FILTER_LZMA2
, (lzma_options_lzma
*) &opt
},
69 { LZMA_VLI_UNKNOWN
, NULL
}
79 /* Returns < 0 if we couldn't compress the data or the
80 * compressed result is longer than the original */
85 ret
= lzma_stream_buffer_encode((lzma_filter
*) filters
, LZMA_CHECK_NONE
, NULL
,
86 src
, src_size
, dst
, &out_pos
, src_size
- 1);
93 return -EPROTONOSUPPORT
;
97 int compress_blob_lz4(const void *src
, uint64_t src_size
, void *dst
, size_t *dst_size
) {
102 assert(src_size
> 0);
106 /* Returns < 0 if we couldn't compress the data or the
107 * compressed result is longer than the original */
112 r
= LZ4_compress_limitedOutput(src
, dst
+ 8, src_size
, src_size
- 8 - 1);
116 *(le64_t
*) dst
= htole64(src_size
);
121 return -EPROTONOSUPPORT
;
126 int decompress_blob_xz(const void *src
, uint64_t src_size
,
127 void **dst
, size_t *dst_alloc_size
, size_t* dst_size
, size_t dst_max
) {
130 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
135 assert(src_size
> 0);
137 assert(dst_alloc_size
);
139 assert(*dst_alloc_size
== 0 || *dst
);
141 ret
= lzma_stream_decoder(&s
, UINT64_MAX
, 0);
145 space
= MIN(src_size
* 2, dst_max
?: (size_t) -1);
146 if (!greedy_realloc(dst
, dst_alloc_size
, space
, 1))
150 s
.avail_in
= src_size
;
158 ret
= lzma_code(&s
, LZMA_FINISH
);
160 if (ret
== LZMA_STREAM_END
)
162 else if (ret
!= LZMA_OK
)
165 if (dst_max
> 0 && (space
- s
.avail_out
) >= dst_max
)
167 else if (dst_max
> 0 && space
== dst_max
)
170 used
= space
- s
.avail_out
;
171 space
= MIN(2 * space
, dst_max
?: (size_t) -1);
172 if (!greedy_realloc(dst
, dst_alloc_size
, space
, 1))
175 s
.avail_out
= space
- used
;
176 s
.next_out
= *dst
+ used
;
179 *dst_size
= space
- s
.avail_out
;
182 return -EPROTONOSUPPORT
;
186 int decompress_blob_lz4(const void *src
, uint64_t src_size
,
187 void **dst
, size_t *dst_alloc_size
, size_t* dst_size
, size_t dst_max
) {
191 int r
, size
; /* LZ4 uses int for size */
194 assert(src_size
> 0);
196 assert(dst_alloc_size
);
198 assert(*dst_alloc_size
== 0 || *dst
);
203 size
= le64toh( *(le64_t
*)src
);
204 if (size
< 0 || (unsigned) size
!= le64toh(*(le64_t
*)src
))
206 if ((size_t) size
> *dst_alloc_size
) {
207 out
= realloc(*dst
, size
);
211 *dst_alloc_size
= size
;
215 r
= LZ4_decompress_safe(src
+ 8, out
, src_size
- 8, size
);
216 if (r
< 0 || r
!= size
)
222 return -EPROTONOSUPPORT
;
226 int decompress_blob(int compression
,
227 const void *src
, uint64_t src_size
,
228 void **dst
, size_t *dst_alloc_size
, size_t* dst_size
, size_t dst_max
) {
229 if (compression
== OBJECT_COMPRESSED_XZ
)
230 return decompress_blob_xz(src
, src_size
,
231 dst
, dst_alloc_size
, dst_size
, dst_max
);
232 else if (compression
== OBJECT_COMPRESSED_LZ4
)
233 return decompress_blob_lz4(src
, src_size
,
234 dst
, dst_alloc_size
, dst_size
, dst_max
);
240 int decompress_startswith_xz(const void *src
, uint64_t src_size
,
241 void **buffer
, size_t *buffer_size
,
242 const void *prefix
, size_t prefix_len
,
246 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
249 /* Checks whether the decompressed blob starts with the
250 * mentioned prefix. The byte extra needs to follow the
254 assert(src_size
> 0);
258 assert(*buffer_size
== 0 || *buffer
);
260 ret
= lzma_stream_decoder(&s
, UINT64_MAX
, 0);
264 if (!(greedy_realloc(buffer
, buffer_size
, ALIGN_8(prefix_len
+ 1), 1)))
268 s
.avail_in
= src_size
;
270 s
.next_out
= *buffer
;
271 s
.avail_out
= *buffer_size
;
274 ret
= lzma_code(&s
, LZMA_FINISH
);
276 if (ret
!= LZMA_STREAM_END
&& ret
!= LZMA_OK
)
279 if (*buffer_size
- s
.avail_out
>= prefix_len
+ 1)
280 return memcmp(*buffer
, prefix
, prefix_len
) == 0 &&
281 ((const uint8_t*) *buffer
)[prefix_len
] == extra
;
283 if (ret
== LZMA_STREAM_END
)
286 s
.avail_out
+= *buffer_size
;
288 if (!(greedy_realloc(buffer
, buffer_size
, *buffer_size
* 2, 1)))
291 s
.next_out
= *buffer
+ *buffer_size
- s
.avail_out
;
295 return -EPROTONOSUPPORT
;
299 int decompress_startswith_lz4(const void *src
, uint64_t src_size
,
300 void **buffer
, size_t *buffer_size
,
301 const void *prefix
, size_t prefix_len
,
304 /* Checks whether the decompressed blob starts with the
305 * mentioned prefix. The byte extra needs to follow the
311 assert(src_size
> 0);
315 assert(*buffer_size
== 0 || *buffer
);
320 if (!(greedy_realloc(buffer
, buffer_size
, ALIGN_8(prefix_len
+ 1), 1)))
323 r
= LZ4_decompress_safe_partial(src
+ 8, *buffer
, src_size
- 8,
324 prefix_len
+ 1, *buffer_size
);
328 if ((unsigned) r
>= prefix_len
+ 1)
329 return memcmp(*buffer
, prefix
, prefix_len
) == 0 &&
330 ((const uint8_t*) *buffer
)[prefix_len
] == extra
;
335 return -EPROTONOSUPPORT
;
339 int decompress_startswith(int compression
,
340 const void *src
, uint64_t src_size
,
341 void **buffer
, size_t *buffer_size
,
342 const void *prefix
, size_t prefix_len
,
344 if (compression
== OBJECT_COMPRESSED_XZ
)
345 return decompress_startswith_xz(src
, src_size
,
349 else if (compression
== OBJECT_COMPRESSED_LZ4
)
350 return decompress_startswith_lz4(src
, src_size
,
358 int compress_stream_xz(int fdf
, int fdt
, uint64_t max_bytes
) {
360 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
362 uint8_t buf
[BUFSIZ
], out
[BUFSIZ
];
363 lzma_action action
= LZMA_RUN
;
368 ret
= lzma_easy_encoder(&s
, LZMA_PRESET_DEFAULT
, LZMA_CHECK_CRC64
);
369 if (ret
!= LZMA_OK
) {
370 log_error("Failed to initialize XZ encoder: code %u", ret
);
375 if (s
.avail_in
== 0 && action
== LZMA_RUN
) {
376 size_t m
= sizeof(buf
);
379 if (max_bytes
!= (uint64_t) -1 && (uint64_t) m
> max_bytes
)
380 m
= (size_t) max_bytes
;
382 n
= read(fdf
, buf
, m
);
386 action
= LZMA_FINISH
;
391 if (max_bytes
!= (uint64_t) -1) {
392 assert(max_bytes
>= (uint64_t) n
);
398 if (s
.avail_out
== 0) {
400 s
.avail_out
= sizeof(out
);
403 ret
= lzma_code(&s
, action
);
404 if (ret
!= LZMA_OK
&& ret
!= LZMA_STREAM_END
) {
405 log_error("Compression failed: code %u", ret
);
409 if (s
.avail_out
== 0 || ret
== LZMA_STREAM_END
) {
412 n
= sizeof(out
) - s
.avail_out
;
414 k
= loop_write(fdt
, out
, n
, false);
418 if (ret
== LZMA_STREAM_END
) {
419 log_debug("XZ compression finished (%"PRIu64
" -> %"PRIu64
" bytes, %.1f%%)",
420 s
.total_in
, s
.total_out
,
421 (double) s
.total_out
/ s
.total_in
* 100);
428 return -EPROTONOSUPPORT
;
432 #define LZ4_BUFSIZE (512*1024u)
434 int compress_stream_lz4(int fdf
, int fdt
, uint64_t max_bytes
) {
438 _cleanup_(LZ4F_freeCompressionContextp
) LZ4F_compressionContext_t ctx
= NULL
;
439 _cleanup_free_
char *buf
= NULL
;
441 size_t size
, n
, total_in
= 0, total_out
= 0, offset
= 0, frame_size
;
444 static const LZ4F_compressOptions_t options
= {
447 static const LZ4F_preferences_t preferences
= {
448 .frameInfo
.blockSizeID
= 5,
451 c
= LZ4F_createCompressionContext(&ctx
, LZ4F_VERSION
);
455 if (fstat(fdf
, &st
) < 0)
456 return log_debug_errno(errno
, "fstat() failed: %m");
458 frame_size
= LZ4F_compressBound(LZ4_BUFSIZE
, &preferences
);
459 size
= frame_size
+ 64*1024; /* add some space for header and trailer */
464 n
= offset
= LZ4F_compressBegin(ctx
, buf
, size
, &preferences
);
468 src
= mmap(NULL
, st
.st_size
, PROT_READ
, MAP_PRIVATE
, fdf
, 0);
469 if (src
== MAP_FAILED
)
472 log_debug("Buffer size is %zu bytes, header size %zu bytes.", size
, n
);
474 while (total_in
< (size_t) st
.st_size
) {
477 k
= MIN(LZ4_BUFSIZE
, st
.st_size
- total_in
);
478 n
= LZ4F_compressUpdate(ctx
, buf
+ offset
, size
- offset
,
479 src
+ total_in
, k
, &options
);
480 if (LZ4F_isError(n
)) {
481 r
= -ENOTRECOVERABLE
;
489 if (max_bytes
!= (uint64_t) -1 && total_out
> (size_t) max_bytes
) {
490 log_debug("Compressed stream longer than %zd bytes", max_bytes
);
494 if (size
- offset
< frame_size
+ 4) {
495 k
= loop_write(fdt
, buf
, offset
, false);
504 n
= LZ4F_compressEnd(ctx
, buf
+ offset
, size
- offset
, &options
);
505 if (LZ4F_isError(n
)) {
506 r
= -ENOTRECOVERABLE
;
512 r
= loop_write(fdt
, buf
, offset
, false);
516 log_debug("LZ4 compression finished (%zu -> %zu bytes, %.1f%%)",
518 (double) total_out
/ total_in
* 100);
520 munmap(src
, st
.st_size
);
523 return -EPROTONOSUPPORT
;
527 int decompress_stream_xz(int fdf
, int fdt
, uint64_t max_bytes
) {
530 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
533 uint8_t buf
[BUFSIZ
], out
[BUFSIZ
];
534 lzma_action action
= LZMA_RUN
;
539 ret
= lzma_stream_decoder(&s
, UINT64_MAX
, 0);
540 if (ret
!= LZMA_OK
) {
541 log_debug("Failed to initialize XZ decoder: code %u", ret
);
546 if (s
.avail_in
== 0 && action
== LZMA_RUN
) {
549 n
= read(fdf
, buf
, sizeof(buf
));
553 action
= LZMA_FINISH
;
560 if (s
.avail_out
== 0) {
562 s
.avail_out
= sizeof(out
);
565 ret
= lzma_code(&s
, action
);
566 if (ret
!= LZMA_OK
&& ret
!= LZMA_STREAM_END
) {
567 log_debug("Decompression failed: code %u", ret
);
571 if (s
.avail_out
== 0 || ret
== LZMA_STREAM_END
) {
574 n
= sizeof(out
) - s
.avail_out
;
576 if (max_bytes
!= (uint64_t) -1) {
577 if (max_bytes
< (uint64_t) n
)
583 k
= loop_write(fdt
, out
, n
, false);
587 if (ret
== LZMA_STREAM_END
) {
588 log_debug("XZ decompression finished (%"PRIu64
" -> %"PRIu64
" bytes, %.1f%%)",
589 s
.total_in
, s
.total_out
,
590 (double) s
.total_out
/ s
.total_in
* 100);
597 log_debug("Cannot decompress file. Compiled without XZ support.");
598 return -EPROTONOSUPPORT
;
603 static int decompress_stream_lz4_v1(int fdf
, int fdt
, uint64_t max_bytes
) {
605 _cleanup_free_
char *buf
= NULL
, *out
= NULL
;
607 LZ4_streamDecode_t lz4_data
= {};
609 size_t total_in
= sizeof(header
), total_out
= 0;
614 out
= malloc(4*LZ4_BUFSIZE
);
622 r
= loop_read_exact(fdf
, &header
, sizeof(header
), false);
630 /* We refuse to use a bigger decompression buffer than
631 * the one used for compression by 4 times. This means
632 * that compression buffer size can be enlarged 4
633 * times. This can be changed, but old binaries might
634 * not accept buffers compressed by newer binaries then.
636 if (m
> LZ4_COMPRESSBOUND(LZ4_BUFSIZE
* 4)) {
637 log_debug("Compressed stream block too big: %zd bytes", m
);
641 total_in
+= sizeof(header
) + m
;
643 if (!GREEDY_REALLOC(buf
, buf_size
, m
))
646 r
= loop_read_exact(fdf
, buf
, m
, false);
650 r
= LZ4_decompress_safe_continue(&lz4_data
, buf
, out
, m
, 4*LZ4_BUFSIZE
);
652 log_debug("LZ4 decompression failed (legacy format).");
658 if (max_bytes
!= (uint64_t) -1 && (uint64_t) total_out
> max_bytes
) {
659 log_debug("Decompressed stream longer than %" PRIu64
" bytes", max_bytes
);
663 r
= loop_write(fdt
, out
, r
, false);
668 log_debug("LZ4 decompression finished (legacy format, %zu -> %zu bytes, %.1f%%)",
670 (double) total_out
/ total_in
* 100);
675 static int decompress_stream_lz4_v2(int in
, int out
, uint64_t max_bytes
) {
677 _cleanup_(LZ4F_freeDecompressionContextp
) LZ4F_decompressionContext_t ctx
= NULL
;
678 _cleanup_free_
char *buf
= NULL
;
682 size_t total_in
= 0, total_out
= 0;
684 c
= LZ4F_createDecompressionContext(&ctx
, LZ4F_VERSION
);
688 if (fstat(in
, &st
) < 0)
689 return log_debug_errno(errno
, "fstat() failed: %m");
691 buf
= malloc(LZ4_BUFSIZE
);
695 src
= mmap(NULL
, st
.st_size
, PROT_READ
, MAP_PRIVATE
, in
, 0);
696 if (src
== MAP_FAILED
)
699 while (total_in
< (size_t) st
.st_size
) {
700 size_t produced
= LZ4_BUFSIZE
;
701 size_t used
= st
.st_size
- total_in
;
703 c
= LZ4F_decompress(ctx
, buf
, &produced
, src
+ total_in
, &used
, NULL
);
704 if (LZ4F_isError(c
)) {
710 total_out
+= produced
;
712 if (max_bytes
!= (uint64_t) -1 && total_out
> (size_t) max_bytes
) {
713 log_debug("Decompressed stream longer than %zd bytes", max_bytes
);
718 r
= loop_write(out
, buf
, produced
, false);
723 log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)",
725 (double) total_out
/ total_in
* 100);
727 munmap(src
, st
.st_size
);
732 int decompress_stream_lz4(int fdf
, int fdt
, uint64_t max_bytes
) {
736 r
= decompress_stream_lz4_v2(fdf
, fdt
, max_bytes
);
738 r
= decompress_stream_lz4_v1(fdf
, fdt
, max_bytes
);
741 log_debug("Cannot decompress file. Compiled without LZ4 support.");
742 return -EPROTONOSUPPORT
;
746 int decompress_stream(const char *filename
, int fdf
, int fdt
, uint64_t max_bytes
) {
748 if (endswith(filename
, ".lz4"))
749 return decompress_stream_lz4(fdf
, fdt
, max_bytes
);
750 else if (endswith(filename
, ".xz"))
751 return decompress_stream_xz(fdf
, fdt
, max_bytes
);
753 return -EPROTONOSUPPORT
;