2 This file is part of systemd.
4 Copyright 2011 Lennart Poettering
6 systemd is free software; you can redistribute it and/or modify it
7 under the terms of the GNU Lesser General Public License as published by
8 the Free Software Foundation; either version 2.1 of the License, or
9 (at your option) any later version.
11 systemd is distributed in the hope that it will be useful, but
12 WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 Lesser General Public License for more details.
16 You should have received a copy of the GNU Lesser General Public License
17 along with systemd; If not, see <http://www.gnu.org/licenses/>.
35 #include "alloc-util.h"
39 #include "journal-def.h"
41 #include "sparse-endian.h"
42 #include "string-table.h"
43 #include "string-util.h"
47 DEFINE_TRIVIAL_CLEANUP_FUNC(LZ4F_compressionContext_t
, LZ4F_freeCompressionContext
);
48 DEFINE_TRIVIAL_CLEANUP_FUNC(LZ4F_decompressionContext_t
, LZ4F_freeDecompressionContext
);
51 #define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t))
53 static const char* const object_compressed_table
[_OBJECT_COMPRESSED_MAX
] = {
54 [OBJECT_COMPRESSED_XZ
] = "XZ",
55 [OBJECT_COMPRESSED_LZ4
] = "LZ4",
58 DEFINE_STRING_TABLE_LOOKUP(object_compressed
, int);
60 int compress_blob_xz(const void *src
, uint64_t src_size
,
61 void *dst
, size_t dst_alloc_size
, size_t *dst_size
) {
63 static const lzma_options_lzma opt
= {
64 1u << 20u, NULL
, 0, LZMA_LC_DEFAULT
, LZMA_LP_DEFAULT
,
65 LZMA_PB_DEFAULT
, LZMA_MODE_FAST
, 128, LZMA_MF_HC3
, 4
67 static const lzma_filter filters
[] = {
68 { LZMA_FILTER_LZMA2
, (lzma_options_lzma
*) &opt
},
69 { LZMA_VLI_UNKNOWN
, NULL
}
77 assert(dst_alloc_size
> 0);
80 /* Returns < 0 if we couldn't compress the data or the
81 * compressed result is longer than the original */
86 ret
= lzma_stream_buffer_encode((lzma_filter
*) filters
, LZMA_CHECK_NONE
, NULL
,
87 src
, src_size
, dst
, &out_pos
, dst_alloc_size
);
94 return -EPROTONOSUPPORT
;
98 int compress_blob_lz4(const void *src
, uint64_t src_size
,
99 void *dst
, size_t dst_alloc_size
, size_t *dst_size
) {
104 assert(src_size
> 0);
106 assert(dst_alloc_size
> 0);
109 /* Returns < 0 if we couldn't compress the data or the
110 * compressed result is longer than the original */
115 #if LZ4_VERSION_NUMBER >= 10700
116 r
= LZ4_compress_default(src
, (char*)dst
+ 8, src_size
, (int) dst_alloc_size
- 8);
118 r
= LZ4_compress_limitedOutput(src
, (char*)dst
+ 8, src_size
, (int) dst_alloc_size
- 8);
123 *(le64_t
*) dst
= htole64(src_size
);
128 return -EPROTONOSUPPORT
;
133 int decompress_blob_xz(const void *src
, uint64_t src_size
,
134 void **dst
, size_t *dst_alloc_size
, size_t* dst_size
, size_t dst_max
) {
137 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
142 assert(src_size
> 0);
144 assert(dst_alloc_size
);
146 assert(*dst_alloc_size
== 0 || *dst
);
148 ret
= lzma_stream_decoder(&s
, UINT64_MAX
, 0);
152 space
= MIN(src_size
* 2, dst_max
?: (size_t) -1);
153 if (!greedy_realloc(dst
, dst_alloc_size
, space
, 1))
157 s
.avail_in
= src_size
;
165 ret
= lzma_code(&s
, LZMA_FINISH
);
167 if (ret
== LZMA_STREAM_END
)
169 else if (ret
!= LZMA_OK
)
172 if (dst_max
> 0 && (space
- s
.avail_out
) >= dst_max
)
174 else if (dst_max
> 0 && space
== dst_max
)
177 used
= space
- s
.avail_out
;
178 space
= MIN(2 * space
, dst_max
?: (size_t) -1);
179 if (!greedy_realloc(dst
, dst_alloc_size
, space
, 1))
182 s
.avail_out
= space
- used
;
183 s
.next_out
= *(uint8_t**)dst
+ used
;
186 *dst_size
= space
- s
.avail_out
;
189 return -EPROTONOSUPPORT
;
193 int decompress_blob_lz4(const void *src
, uint64_t src_size
,
194 void **dst
, size_t *dst_alloc_size
, size_t* dst_size
, size_t dst_max
) {
198 int r
, size
; /* LZ4 uses int for size */
201 assert(src_size
> 0);
203 assert(dst_alloc_size
);
205 assert(*dst_alloc_size
== 0 || *dst
);
210 size
= le64toh( *(le64_t
*)src
);
211 if (size
< 0 || (unsigned) size
!= le64toh(*(le64_t
*)src
))
213 if ((size_t) size
> *dst_alloc_size
) {
214 out
= realloc(*dst
, size
);
218 *dst_alloc_size
= size
;
222 r
= LZ4_decompress_safe((char*)src
+ 8, out
, src_size
- 8, size
);
223 if (r
< 0 || r
!= size
)
229 return -EPROTONOSUPPORT
;
233 int decompress_blob(int compression
,
234 const void *src
, uint64_t src_size
,
235 void **dst
, size_t *dst_alloc_size
, size_t* dst_size
, size_t dst_max
) {
236 if (compression
== OBJECT_COMPRESSED_XZ
)
237 return decompress_blob_xz(src
, src_size
,
238 dst
, dst_alloc_size
, dst_size
, dst_max
);
239 else if (compression
== OBJECT_COMPRESSED_LZ4
)
240 return decompress_blob_lz4(src
, src_size
,
241 dst
, dst_alloc_size
, dst_size
, dst_max
);
247 int decompress_startswith_xz(const void *src
, uint64_t src_size
,
248 void **buffer
, size_t *buffer_size
,
249 const void *prefix
, size_t prefix_len
,
253 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
256 /* Checks whether the decompressed blob starts with the
257 * mentioned prefix. The byte extra needs to follow the
261 assert(src_size
> 0);
265 assert(*buffer_size
== 0 || *buffer
);
267 ret
= lzma_stream_decoder(&s
, UINT64_MAX
, 0);
271 if (!(greedy_realloc(buffer
, buffer_size
, ALIGN_8(prefix_len
+ 1), 1)))
275 s
.avail_in
= src_size
;
277 s
.next_out
= *buffer
;
278 s
.avail_out
= *buffer_size
;
281 ret
= lzma_code(&s
, LZMA_FINISH
);
283 if (ret
!= LZMA_STREAM_END
&& ret
!= LZMA_OK
)
286 if (*buffer_size
- s
.avail_out
>= prefix_len
+ 1)
287 return memcmp(*buffer
, prefix
, prefix_len
) == 0 &&
288 ((const uint8_t*) *buffer
)[prefix_len
] == extra
;
290 if (ret
== LZMA_STREAM_END
)
293 s
.avail_out
+= *buffer_size
;
295 if (!(greedy_realloc(buffer
, buffer_size
, *buffer_size
* 2, 1)))
298 s
.next_out
= *(uint8_t**)buffer
+ *buffer_size
- s
.avail_out
;
302 return -EPROTONOSUPPORT
;
306 int decompress_startswith_lz4(const void *src
, uint64_t src_size
,
307 void **buffer
, size_t *buffer_size
,
308 const void *prefix
, size_t prefix_len
,
311 /* Checks whether the decompressed blob starts with the
312 * mentioned prefix. The byte extra needs to follow the
319 assert(src_size
> 0);
323 assert(*buffer_size
== 0 || *buffer
);
328 if (!(greedy_realloc(buffer
, buffer_size
, ALIGN_8(prefix_len
+ 1), 1)))
331 r
= LZ4_decompress_safe_partial((char*)src
+ 8, *buffer
, src_size
- 8,
332 prefix_len
+ 1, *buffer_size
);
336 /* lz4 always tries to decode full "sequence", so in
337 * pathological cases might need to decompress the
339 r
= decompress_blob_lz4(src
, src_size
, buffer
, buffer_size
, &size
, 0);
344 if (size
>= prefix_len
+ 1)
345 return memcmp(*buffer
, prefix
, prefix_len
) == 0 &&
346 ((const uint8_t*) *buffer
)[prefix_len
] == extra
;
351 return -EPROTONOSUPPORT
;
355 int decompress_startswith(int compression
,
356 const void *src
, uint64_t src_size
,
357 void **buffer
, size_t *buffer_size
,
358 const void *prefix
, size_t prefix_len
,
360 if (compression
== OBJECT_COMPRESSED_XZ
)
361 return decompress_startswith_xz(src
, src_size
,
365 else if (compression
== OBJECT_COMPRESSED_LZ4
)
366 return decompress_startswith_lz4(src
, src_size
,
374 int compress_stream_xz(int fdf
, int fdt
, uint64_t max_bytes
) {
376 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
378 uint8_t buf
[BUFSIZ
], out
[BUFSIZ
];
379 lzma_action action
= LZMA_RUN
;
384 ret
= lzma_easy_encoder(&s
, LZMA_PRESET_DEFAULT
, LZMA_CHECK_CRC64
);
385 if (ret
!= LZMA_OK
) {
386 log_error("Failed to initialize XZ encoder: code %u", ret
);
391 if (s
.avail_in
== 0 && action
== LZMA_RUN
) {
392 size_t m
= sizeof(buf
);
395 if (max_bytes
!= (uint64_t) -1 && (uint64_t) m
> max_bytes
)
396 m
= (size_t) max_bytes
;
398 n
= read(fdf
, buf
, m
);
402 action
= LZMA_FINISH
;
407 if (max_bytes
!= (uint64_t) -1) {
408 assert(max_bytes
>= (uint64_t) n
);
414 if (s
.avail_out
== 0) {
416 s
.avail_out
= sizeof(out
);
419 ret
= lzma_code(&s
, action
);
420 if (ret
!= LZMA_OK
&& ret
!= LZMA_STREAM_END
) {
421 log_error("Compression failed: code %u", ret
);
425 if (s
.avail_out
== 0 || ret
== LZMA_STREAM_END
) {
428 n
= sizeof(out
) - s
.avail_out
;
430 k
= loop_write(fdt
, out
, n
, false);
434 if (ret
== LZMA_STREAM_END
) {
435 log_debug("XZ compression finished (%"PRIu64
" -> %"PRIu64
" bytes, %.1f%%)",
436 s
.total_in
, s
.total_out
,
437 (double) s
.total_out
/ s
.total_in
* 100);
444 return -EPROTONOSUPPORT
;
448 #define LZ4_BUFSIZE (512*1024u)
450 int compress_stream_lz4(int fdf
, int fdt
, uint64_t max_bytes
) {
454 _cleanup_(LZ4F_freeCompressionContextp
) LZ4F_compressionContext_t ctx
= NULL
;
455 _cleanup_free_
char *buf
= NULL
;
457 size_t size
, n
, total_in
= 0, total_out
, offset
= 0, frame_size
;
460 static const LZ4F_compressOptions_t options
= {
463 static const LZ4F_preferences_t preferences
= {
464 .frameInfo
.blockSizeID
= 5,
467 c
= LZ4F_createCompressionContext(&ctx
, LZ4F_VERSION
);
471 if (fstat(fdf
, &st
) < 0)
472 return log_debug_errno(errno
, "fstat() failed: %m");
474 frame_size
= LZ4F_compressBound(LZ4_BUFSIZE
, &preferences
);
475 size
= frame_size
+ 64*1024; /* add some space for header and trailer */
480 n
= offset
= total_out
= LZ4F_compressBegin(ctx
, buf
, size
, &preferences
);
484 src
= mmap(NULL
, st
.st_size
, PROT_READ
, MAP_PRIVATE
, fdf
, 0);
485 if (src
== MAP_FAILED
)
488 log_debug("Buffer size is %zu bytes, header size %zu bytes.", size
, n
);
490 while (total_in
< (size_t) st
.st_size
) {
493 k
= MIN(LZ4_BUFSIZE
, st
.st_size
- total_in
);
494 n
= LZ4F_compressUpdate(ctx
, buf
+ offset
, size
- offset
,
495 src
+ total_in
, k
, &options
);
496 if (LZ4F_isError(n
)) {
497 r
= -ENOTRECOVERABLE
;
505 if (max_bytes
!= (uint64_t) -1 && total_out
> (size_t) max_bytes
) {
506 log_debug("Compressed stream longer than %"PRIu64
" bytes", max_bytes
);
510 if (size
- offset
< frame_size
+ 4) {
511 k
= loop_write(fdt
, buf
, offset
, false);
520 n
= LZ4F_compressEnd(ctx
, buf
+ offset
, size
- offset
, &options
);
521 if (LZ4F_isError(n
)) {
522 r
= -ENOTRECOVERABLE
;
528 r
= loop_write(fdt
, buf
, offset
, false);
532 log_debug("LZ4 compression finished (%zu -> %zu bytes, %.1f%%)",
534 (double) total_out
/ total_in
* 100);
536 munmap(src
, st
.st_size
);
539 return -EPROTONOSUPPORT
;
543 int decompress_stream_xz(int fdf
, int fdt
, uint64_t max_bytes
) {
546 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
549 uint8_t buf
[BUFSIZ
], out
[BUFSIZ
];
550 lzma_action action
= LZMA_RUN
;
555 ret
= lzma_stream_decoder(&s
, UINT64_MAX
, 0);
556 if (ret
!= LZMA_OK
) {
557 log_debug("Failed to initialize XZ decoder: code %u", ret
);
562 if (s
.avail_in
== 0 && action
== LZMA_RUN
) {
565 n
= read(fdf
, buf
, sizeof(buf
));
569 action
= LZMA_FINISH
;
576 if (s
.avail_out
== 0) {
578 s
.avail_out
= sizeof(out
);
581 ret
= lzma_code(&s
, action
);
582 if (ret
!= LZMA_OK
&& ret
!= LZMA_STREAM_END
) {
583 log_debug("Decompression failed: code %u", ret
);
587 if (s
.avail_out
== 0 || ret
== LZMA_STREAM_END
) {
590 n
= sizeof(out
) - s
.avail_out
;
592 if (max_bytes
!= (uint64_t) -1) {
593 if (max_bytes
< (uint64_t) n
)
599 k
= loop_write(fdt
, out
, n
, false);
603 if (ret
== LZMA_STREAM_END
) {
604 log_debug("XZ decompression finished (%"PRIu64
" -> %"PRIu64
" bytes, %.1f%%)",
605 s
.total_in
, s
.total_out
,
606 (double) s
.total_out
/ s
.total_in
* 100);
613 log_debug("Cannot decompress file. Compiled without XZ support.");
614 return -EPROTONOSUPPORT
;
618 int decompress_stream_lz4(int in
, int out
, uint64_t max_bytes
) {
621 _cleanup_(LZ4F_freeDecompressionContextp
) LZ4F_decompressionContext_t ctx
= NULL
;
622 _cleanup_free_
char *buf
= NULL
;
626 size_t total_in
= 0, total_out
= 0;
628 c
= LZ4F_createDecompressionContext(&ctx
, LZ4F_VERSION
);
632 if (fstat(in
, &st
) < 0)
633 return log_debug_errno(errno
, "fstat() failed: %m");
635 buf
= malloc(LZ4_BUFSIZE
);
639 src
= mmap(NULL
, st
.st_size
, PROT_READ
, MAP_PRIVATE
, in
, 0);
640 if (src
== MAP_FAILED
)
643 while (total_in
< (size_t) st
.st_size
) {
644 size_t produced
= LZ4_BUFSIZE
;
645 size_t used
= st
.st_size
- total_in
;
647 c
= LZ4F_decompress(ctx
, buf
, &produced
, src
+ total_in
, &used
, NULL
);
648 if (LZ4F_isError(c
)) {
654 total_out
+= produced
;
656 if (max_bytes
!= (uint64_t) -1 && total_out
> (size_t) max_bytes
) {
657 log_debug("Decompressed stream longer than %"PRIu64
" bytes", max_bytes
);
662 r
= loop_write(out
, buf
, produced
, false);
667 log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)",
669 (double) total_out
/ total_in
* 100);
671 munmap(src
, st
.st_size
);
674 log_debug("Cannot decompress file. Compiled without LZ4 support.");
675 return -EPROTONOSUPPORT
;
679 int decompress_stream(const char *filename
, int fdf
, int fdt
, uint64_t max_bytes
) {
681 if (endswith(filename
, ".lz4"))
682 return decompress_stream_lz4(fdf
, fdt
, max_bytes
);
683 else if (endswith(filename
, ".xz"))
684 return decompress_stream_xz(fdf
, fdt
, max_bytes
);
686 return -EPROTONOSUPPORT
;