2 This file is part of systemd.
4 Copyright 2011 Lennart Poettering
6 systemd is free software; you can redistribute it and/or modify it
7 under the terms of the GNU Lesser General Public License as published by
8 the Free Software Foundation; either version 2.1 of the License, or
9 (at your option) any later version.
11 systemd is distributed in the hope that it will be useful, but
12 WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 Lesser General Public License for more details.
16 You should have received a copy of the GNU Lesser General Public License
17 along with systemd; If not, see <http://www.gnu.org/licenses/>.
34 #include "alloc-util.h"
38 #include "journal-def.h"
40 #include "sparse-endian.h"
41 #include "string-table.h"
42 #include "string-util.h"
46 DEFINE_TRIVIAL_CLEANUP_FUNC(LZ4F_compressionContext_t
, LZ4F_freeCompressionContext
);
47 DEFINE_TRIVIAL_CLEANUP_FUNC(LZ4F_decompressionContext_t
, LZ4F_freeDecompressionContext
);
50 #define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t))
52 static const char* const object_compressed_table
[_OBJECT_COMPRESSED_MAX
] = {
53 [OBJECT_COMPRESSED_XZ
] = "XZ",
54 [OBJECT_COMPRESSED_LZ4
] = "LZ4",
57 DEFINE_STRING_TABLE_LOOKUP(object_compressed
, int);
59 int compress_blob_xz(const void *src
, uint64_t src_size
,
60 void *dst
, size_t dst_alloc_size
, size_t *dst_size
) {
62 static const lzma_options_lzma opt
= {
63 1u << 20u, NULL
, 0, LZMA_LC_DEFAULT
, LZMA_LP_DEFAULT
,
64 LZMA_PB_DEFAULT
, LZMA_MODE_FAST
, 128, LZMA_MF_HC3
, 4
66 static const lzma_filter filters
[] = {
67 { LZMA_FILTER_LZMA2
, (lzma_options_lzma
*) &opt
},
68 { LZMA_VLI_UNKNOWN
, NULL
}
76 assert(dst_alloc_size
> 0);
79 /* Returns < 0 if we couldn't compress the data or the
80 * compressed result is longer than the original */
85 ret
= lzma_stream_buffer_encode((lzma_filter
*) filters
, LZMA_CHECK_NONE
, NULL
,
86 src
, src_size
, dst
, &out_pos
, dst_alloc_size
);
93 return -EPROTONOSUPPORT
;
97 int compress_blob_lz4(const void *src
, uint64_t src_size
,
98 void *dst
, size_t dst_alloc_size
, size_t *dst_size
) {
103 assert(src_size
> 0);
105 assert(dst_alloc_size
> 0);
108 /* Returns < 0 if we couldn't compress the data or the
109 * compressed result is longer than the original */
114 r
= LZ4_compress_limitedOutput(src
, dst
+ 8, src_size
, (int) dst_alloc_size
- 8);
118 *(le64_t
*) dst
= htole64(src_size
);
123 return -EPROTONOSUPPORT
;
128 int decompress_blob_xz(const void *src
, uint64_t src_size
,
129 void **dst
, size_t *dst_alloc_size
, size_t* dst_size
, size_t dst_max
) {
132 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
137 assert(src_size
> 0);
139 assert(dst_alloc_size
);
141 assert(*dst_alloc_size
== 0 || *dst
);
143 ret
= lzma_stream_decoder(&s
, UINT64_MAX
, 0);
147 space
= MIN(src_size
* 2, dst_max
?: (size_t) -1);
148 if (!greedy_realloc(dst
, dst_alloc_size
, space
, 1))
152 s
.avail_in
= src_size
;
160 ret
= lzma_code(&s
, LZMA_FINISH
);
162 if (ret
== LZMA_STREAM_END
)
164 else if (ret
!= LZMA_OK
)
167 if (dst_max
> 0 && (space
- s
.avail_out
) >= dst_max
)
169 else if (dst_max
> 0 && space
== dst_max
)
172 used
= space
- s
.avail_out
;
173 space
= MIN(2 * space
, dst_max
?: (size_t) -1);
174 if (!greedy_realloc(dst
, dst_alloc_size
, space
, 1))
177 s
.avail_out
= space
- used
;
178 s
.next_out
= *dst
+ used
;
181 *dst_size
= space
- s
.avail_out
;
184 return -EPROTONOSUPPORT
;
188 int decompress_blob_lz4(const void *src
, uint64_t src_size
,
189 void **dst
, size_t *dst_alloc_size
, size_t* dst_size
, size_t dst_max
) {
193 int r
, size
; /* LZ4 uses int for size */
196 assert(src_size
> 0);
198 assert(dst_alloc_size
);
200 assert(*dst_alloc_size
== 0 || *dst
);
205 size
= le64toh( *(le64_t
*)src
);
206 if (size
< 0 || (unsigned) size
!= le64toh(*(le64_t
*)src
))
208 if ((size_t) size
> *dst_alloc_size
) {
209 out
= realloc(*dst
, size
);
213 *dst_alloc_size
= size
;
217 r
= LZ4_decompress_safe(src
+ 8, out
, src_size
- 8, size
);
218 if (r
< 0 || r
!= size
)
224 return -EPROTONOSUPPORT
;
228 int decompress_blob(int compression
,
229 const void *src
, uint64_t src_size
,
230 void **dst
, size_t *dst_alloc_size
, size_t* dst_size
, size_t dst_max
) {
231 if (compression
== OBJECT_COMPRESSED_XZ
)
232 return decompress_blob_xz(src
, src_size
,
233 dst
, dst_alloc_size
, dst_size
, dst_max
);
234 else if (compression
== OBJECT_COMPRESSED_LZ4
)
235 return decompress_blob_lz4(src
, src_size
,
236 dst
, dst_alloc_size
, dst_size
, dst_max
);
242 int decompress_startswith_xz(const void *src
, uint64_t src_size
,
243 void **buffer
, size_t *buffer_size
,
244 const void *prefix
, size_t prefix_len
,
248 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
251 /* Checks whether the decompressed blob starts with the
252 * mentioned prefix. The byte extra needs to follow the
256 assert(src_size
> 0);
260 assert(*buffer_size
== 0 || *buffer
);
262 ret
= lzma_stream_decoder(&s
, UINT64_MAX
, 0);
266 if (!(greedy_realloc(buffer
, buffer_size
, ALIGN_8(prefix_len
+ 1), 1)))
270 s
.avail_in
= src_size
;
272 s
.next_out
= *buffer
;
273 s
.avail_out
= *buffer_size
;
276 ret
= lzma_code(&s
, LZMA_FINISH
);
278 if (ret
!= LZMA_STREAM_END
&& ret
!= LZMA_OK
)
281 if (*buffer_size
- s
.avail_out
>= prefix_len
+ 1)
282 return memcmp(*buffer
, prefix
, prefix_len
) == 0 &&
283 ((const uint8_t*) *buffer
)[prefix_len
] == extra
;
285 if (ret
== LZMA_STREAM_END
)
288 s
.avail_out
+= *buffer_size
;
290 if (!(greedy_realloc(buffer
, buffer_size
, *buffer_size
* 2, 1)))
293 s
.next_out
= *buffer
+ *buffer_size
- s
.avail_out
;
297 return -EPROTONOSUPPORT
;
301 int decompress_startswith_lz4(const void *src
, uint64_t src_size
,
302 void **buffer
, size_t *buffer_size
,
303 const void *prefix
, size_t prefix_len
,
306 /* Checks whether the decompressed blob starts with the
307 * mentioned prefix. The byte extra needs to follow the
314 assert(src_size
> 0);
318 assert(*buffer_size
== 0 || *buffer
);
323 if (!(greedy_realloc(buffer
, buffer_size
, ALIGN_8(prefix_len
+ 1), 1)))
326 r
= LZ4_decompress_safe_partial(src
+ 8, *buffer
, src_size
- 8,
327 prefix_len
+ 1, *buffer_size
);
331 /* lz4 always tries to decode full "sequence", so in
332 * pathological cases might need to decompress the
334 r
= decompress_blob_lz4(src
, src_size
, buffer
, buffer_size
, &size
, 0);
339 if (size
>= prefix_len
+ 1)
340 return memcmp(*buffer
, prefix
, prefix_len
) == 0 &&
341 ((const uint8_t*) *buffer
)[prefix_len
] == extra
;
346 return -EPROTONOSUPPORT
;
350 int decompress_startswith(int compression
,
351 const void *src
, uint64_t src_size
,
352 void **buffer
, size_t *buffer_size
,
353 const void *prefix
, size_t prefix_len
,
355 if (compression
== OBJECT_COMPRESSED_XZ
)
356 return decompress_startswith_xz(src
, src_size
,
360 else if (compression
== OBJECT_COMPRESSED_LZ4
)
361 return decompress_startswith_lz4(src
, src_size
,
369 int compress_stream_xz(int fdf
, int fdt
, uint64_t max_bytes
) {
371 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
373 uint8_t buf
[BUFSIZ
], out
[BUFSIZ
];
374 lzma_action action
= LZMA_RUN
;
379 ret
= lzma_easy_encoder(&s
, LZMA_PRESET_DEFAULT
, LZMA_CHECK_CRC64
);
380 if (ret
!= LZMA_OK
) {
381 log_error("Failed to initialize XZ encoder: code %u", ret
);
386 if (s
.avail_in
== 0 && action
== LZMA_RUN
) {
387 size_t m
= sizeof(buf
);
390 if (max_bytes
!= (uint64_t) -1 && (uint64_t) m
> max_bytes
)
391 m
= (size_t) max_bytes
;
393 n
= read(fdf
, buf
, m
);
397 action
= LZMA_FINISH
;
402 if (max_bytes
!= (uint64_t) -1) {
403 assert(max_bytes
>= (uint64_t) n
);
409 if (s
.avail_out
== 0) {
411 s
.avail_out
= sizeof(out
);
414 ret
= lzma_code(&s
, action
);
415 if (ret
!= LZMA_OK
&& ret
!= LZMA_STREAM_END
) {
416 log_error("Compression failed: code %u", ret
);
420 if (s
.avail_out
== 0 || ret
== LZMA_STREAM_END
) {
423 n
= sizeof(out
) - s
.avail_out
;
425 k
= loop_write(fdt
, out
, n
, false);
429 if (ret
== LZMA_STREAM_END
) {
430 log_debug("XZ compression finished (%"PRIu64
" -> %"PRIu64
" bytes, %.1f%%)",
431 s
.total_in
, s
.total_out
,
432 (double) s
.total_out
/ s
.total_in
* 100);
439 return -EPROTONOSUPPORT
;
443 #define LZ4_BUFSIZE (512*1024u)
445 int compress_stream_lz4(int fdf
, int fdt
, uint64_t max_bytes
) {
449 _cleanup_(LZ4F_freeCompressionContextp
) LZ4F_compressionContext_t ctx
= NULL
;
450 _cleanup_free_
char *buf
= NULL
;
452 size_t size
, n
, total_in
= 0, total_out
, offset
= 0, frame_size
;
455 static const LZ4F_compressOptions_t options
= {
458 static const LZ4F_preferences_t preferences
= {
459 .frameInfo
.blockSizeID
= 5,
462 c
= LZ4F_createCompressionContext(&ctx
, LZ4F_VERSION
);
466 if (fstat(fdf
, &st
) < 0)
467 return log_debug_errno(errno
, "fstat() failed: %m");
469 frame_size
= LZ4F_compressBound(LZ4_BUFSIZE
, &preferences
);
470 size
= frame_size
+ 64*1024; /* add some space for header and trailer */
475 n
= offset
= total_out
= LZ4F_compressBegin(ctx
, buf
, size
, &preferences
);
479 src
= mmap(NULL
, st
.st_size
, PROT_READ
, MAP_PRIVATE
, fdf
, 0);
480 if (src
== MAP_FAILED
)
483 log_debug("Buffer size is %zu bytes, header size %zu bytes.", size
, n
);
485 while (total_in
< (size_t) st
.st_size
) {
488 k
= MIN(LZ4_BUFSIZE
, st
.st_size
- total_in
);
489 n
= LZ4F_compressUpdate(ctx
, buf
+ offset
, size
- offset
,
490 src
+ total_in
, k
, &options
);
491 if (LZ4F_isError(n
)) {
492 r
= -ENOTRECOVERABLE
;
500 if (max_bytes
!= (uint64_t) -1 && total_out
> (size_t) max_bytes
) {
501 log_debug("Compressed stream longer than %zd bytes", max_bytes
);
505 if (size
- offset
< frame_size
+ 4) {
506 k
= loop_write(fdt
, buf
, offset
, false);
515 n
= LZ4F_compressEnd(ctx
, buf
+ offset
, size
- offset
, &options
);
516 if (LZ4F_isError(n
)) {
517 r
= -ENOTRECOVERABLE
;
523 r
= loop_write(fdt
, buf
, offset
, false);
527 log_debug("LZ4 compression finished (%zu -> %zu bytes, %.1f%%)",
529 (double) total_out
/ total_in
* 100);
531 munmap(src
, st
.st_size
);
534 return -EPROTONOSUPPORT
;
538 int decompress_stream_xz(int fdf
, int fdt
, uint64_t max_bytes
) {
541 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
544 uint8_t buf
[BUFSIZ
], out
[BUFSIZ
];
545 lzma_action action
= LZMA_RUN
;
550 ret
= lzma_stream_decoder(&s
, UINT64_MAX
, 0);
551 if (ret
!= LZMA_OK
) {
552 log_debug("Failed to initialize XZ decoder: code %u", ret
);
557 if (s
.avail_in
== 0 && action
== LZMA_RUN
) {
560 n
= read(fdf
, buf
, sizeof(buf
));
564 action
= LZMA_FINISH
;
571 if (s
.avail_out
== 0) {
573 s
.avail_out
= sizeof(out
);
576 ret
= lzma_code(&s
, action
);
577 if (ret
!= LZMA_OK
&& ret
!= LZMA_STREAM_END
) {
578 log_debug("Decompression failed: code %u", ret
);
582 if (s
.avail_out
== 0 || ret
== LZMA_STREAM_END
) {
585 n
= sizeof(out
) - s
.avail_out
;
587 if (max_bytes
!= (uint64_t) -1) {
588 if (max_bytes
< (uint64_t) n
)
594 k
= loop_write(fdt
, out
, n
, false);
598 if (ret
== LZMA_STREAM_END
) {
599 log_debug("XZ decompression finished (%"PRIu64
" -> %"PRIu64
" bytes, %.1f%%)",
600 s
.total_in
, s
.total_out
,
601 (double) s
.total_out
/ s
.total_in
* 100);
608 log_debug("Cannot decompress file. Compiled without XZ support.");
609 return -EPROTONOSUPPORT
;
613 int decompress_stream_lz4(int in
, int out
, uint64_t max_bytes
) {
616 _cleanup_(LZ4F_freeDecompressionContextp
) LZ4F_decompressionContext_t ctx
= NULL
;
617 _cleanup_free_
char *buf
= NULL
;
621 size_t total_in
= 0, total_out
= 0;
623 c
= LZ4F_createDecompressionContext(&ctx
, LZ4F_VERSION
);
627 if (fstat(in
, &st
) < 0)
628 return log_debug_errno(errno
, "fstat() failed: %m");
630 buf
= malloc(LZ4_BUFSIZE
);
634 src
= mmap(NULL
, st
.st_size
, PROT_READ
, MAP_PRIVATE
, in
, 0);
635 if (src
== MAP_FAILED
)
638 while (total_in
< (size_t) st
.st_size
) {
639 size_t produced
= LZ4_BUFSIZE
;
640 size_t used
= st
.st_size
- total_in
;
642 c
= LZ4F_decompress(ctx
, buf
, &produced
, src
+ total_in
, &used
, NULL
);
643 if (LZ4F_isError(c
)) {
649 total_out
+= produced
;
651 if (max_bytes
!= (uint64_t) -1 && total_out
> (size_t) max_bytes
) {
652 log_debug("Decompressed stream longer than %zd bytes", max_bytes
);
657 r
= loop_write(out
, buf
, produced
, false);
662 log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)",
664 (double) total_out
/ total_in
* 100);
666 munmap(src
, st
.st_size
);
669 log_debug("Cannot decompress file. Compiled without LZ4 support.");
670 return -EPROTONOSUPPORT
;
674 int decompress_stream(const char *filename
, int fdf
, int fdt
, uint64_t max_bytes
) {
676 if (endswith(filename
, ".lz4"))
677 return decompress_stream_lz4(fdf
, fdt
, max_bytes
);
678 else if (endswith(filename
, ".xz"))
679 return decompress_stream_xz(fdf
, fdt
, max_bytes
);
681 return -EPROTONOSUPPORT
;