2 This file is part of systemd.
4 Copyright 2011 Lennart Poettering
6 systemd is free software; you can redistribute it and/or modify it
7 under the terms of the GNU Lesser General Public License as published by
8 the Free Software Foundation; either version 2.1 of the License, or
9 (at your option) any later version.
11 systemd is distributed in the hope that it will be useful, but
12 WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 Lesser General Public License for more details.
16 You should have received a copy of the GNU Lesser General Public License
17 along with systemd; If not, see <http://www.gnu.org/licenses/>.
35 #include "alloc-util.h"
39 #include "journal-def.h"
41 #include "sparse-endian.h"
42 #include "string-table.h"
43 #include "string-util.h"
47 DEFINE_TRIVIAL_CLEANUP_FUNC(LZ4F_compressionContext_t
, LZ4F_freeCompressionContext
);
48 DEFINE_TRIVIAL_CLEANUP_FUNC(LZ4F_decompressionContext_t
, LZ4F_freeDecompressionContext
);
51 #define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t))
53 static const char* const object_compressed_table
[_OBJECT_COMPRESSED_MAX
] = {
54 [OBJECT_COMPRESSED_XZ
] = "XZ",
55 [OBJECT_COMPRESSED_LZ4
] = "LZ4",
58 DEFINE_STRING_TABLE_LOOKUP(object_compressed
, int);
60 int compress_blob_xz(const void *src
, uint64_t src_size
,
61 void *dst
, size_t dst_alloc_size
, size_t *dst_size
) {
63 static const lzma_options_lzma opt
= {
64 1u << 20u, NULL
, 0, LZMA_LC_DEFAULT
, LZMA_LP_DEFAULT
,
65 LZMA_PB_DEFAULT
, LZMA_MODE_FAST
, 128, LZMA_MF_HC3
, 4
67 static const lzma_filter filters
[] = {
68 { LZMA_FILTER_LZMA2
, (lzma_options_lzma
*) &opt
},
69 { LZMA_VLI_UNKNOWN
, NULL
}
77 assert(dst_alloc_size
> 0);
80 /* Returns < 0 if we couldn't compress the data or the
81 * compressed result is longer than the original */
86 ret
= lzma_stream_buffer_encode((lzma_filter
*) filters
, LZMA_CHECK_NONE
, NULL
,
87 src
, src_size
, dst
, &out_pos
, dst_alloc_size
);
94 return -EPROTONOSUPPORT
;
98 int compress_blob_lz4(const void *src
, uint64_t src_size
,
99 void *dst
, size_t dst_alloc_size
, size_t *dst_size
) {
104 assert(src_size
> 0);
106 assert(dst_alloc_size
> 0);
109 /* Returns < 0 if we couldn't compress the data or the
110 * compressed result is longer than the original */
115 r
= LZ4_compress_limitedOutput(src
, dst
+ 8, src_size
, (int) dst_alloc_size
- 8);
119 *(le64_t
*) dst
= htole64(src_size
);
124 return -EPROTONOSUPPORT
;
129 int decompress_blob_xz(const void *src
, uint64_t src_size
,
130 void **dst
, size_t *dst_alloc_size
, size_t* dst_size
, size_t dst_max
) {
133 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
138 assert(src_size
> 0);
140 assert(dst_alloc_size
);
142 assert(*dst_alloc_size
== 0 || *dst
);
144 ret
= lzma_stream_decoder(&s
, UINT64_MAX
, 0);
148 space
= MIN(src_size
* 2, dst_max
?: (size_t) -1);
149 if (!greedy_realloc(dst
, dst_alloc_size
, space
, 1))
153 s
.avail_in
= src_size
;
161 ret
= lzma_code(&s
, LZMA_FINISH
);
163 if (ret
== LZMA_STREAM_END
)
165 else if (ret
!= LZMA_OK
)
168 if (dst_max
> 0 && (space
- s
.avail_out
) >= dst_max
)
170 else if (dst_max
> 0 && space
== dst_max
)
173 used
= space
- s
.avail_out
;
174 space
= MIN(2 * space
, dst_max
?: (size_t) -1);
175 if (!greedy_realloc(dst
, dst_alloc_size
, space
, 1))
178 s
.avail_out
= space
- used
;
179 s
.next_out
= *dst
+ used
;
182 *dst_size
= space
- s
.avail_out
;
185 return -EPROTONOSUPPORT
;
189 int decompress_blob_lz4(const void *src
, uint64_t src_size
,
190 void **dst
, size_t *dst_alloc_size
, size_t* dst_size
, size_t dst_max
) {
194 int r
, size
; /* LZ4 uses int for size */
197 assert(src_size
> 0);
199 assert(dst_alloc_size
);
201 assert(*dst_alloc_size
== 0 || *dst
);
206 size
= le64toh( *(le64_t
*)src
);
207 if (size
< 0 || (unsigned) size
!= le64toh(*(le64_t
*)src
))
209 if ((size_t) size
> *dst_alloc_size
) {
210 out
= realloc(*dst
, size
);
214 *dst_alloc_size
= size
;
218 r
= LZ4_decompress_safe(src
+ 8, out
, src_size
- 8, size
);
219 if (r
< 0 || r
!= size
)
225 return -EPROTONOSUPPORT
;
229 int decompress_blob(int compression
,
230 const void *src
, uint64_t src_size
,
231 void **dst
, size_t *dst_alloc_size
, size_t* dst_size
, size_t dst_max
) {
232 if (compression
== OBJECT_COMPRESSED_XZ
)
233 return decompress_blob_xz(src
, src_size
,
234 dst
, dst_alloc_size
, dst_size
, dst_max
);
235 else if (compression
== OBJECT_COMPRESSED_LZ4
)
236 return decompress_blob_lz4(src
, src_size
,
237 dst
, dst_alloc_size
, dst_size
, dst_max
);
243 int decompress_startswith_xz(const void *src
, uint64_t src_size
,
244 void **buffer
, size_t *buffer_size
,
245 const void *prefix
, size_t prefix_len
,
249 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
252 /* Checks whether the decompressed blob starts with the
253 * mentioned prefix. The byte extra needs to follow the
257 assert(src_size
> 0);
261 assert(*buffer_size
== 0 || *buffer
);
263 ret
= lzma_stream_decoder(&s
, UINT64_MAX
, 0);
267 if (!(greedy_realloc(buffer
, buffer_size
, ALIGN_8(prefix_len
+ 1), 1)))
271 s
.avail_in
= src_size
;
273 s
.next_out
= *buffer
;
274 s
.avail_out
= *buffer_size
;
277 ret
= lzma_code(&s
, LZMA_FINISH
);
279 if (ret
!= LZMA_STREAM_END
&& ret
!= LZMA_OK
)
282 if (*buffer_size
- s
.avail_out
>= prefix_len
+ 1)
283 return memcmp(*buffer
, prefix
, prefix_len
) == 0 &&
284 ((const uint8_t*) *buffer
)[prefix_len
] == extra
;
286 if (ret
== LZMA_STREAM_END
)
289 s
.avail_out
+= *buffer_size
;
291 if (!(greedy_realloc(buffer
, buffer_size
, *buffer_size
* 2, 1)))
294 s
.next_out
= *buffer
+ *buffer_size
- s
.avail_out
;
298 return -EPROTONOSUPPORT
;
302 int decompress_startswith_lz4(const void *src
, uint64_t src_size
,
303 void **buffer
, size_t *buffer_size
,
304 const void *prefix
, size_t prefix_len
,
307 /* Checks whether the decompressed blob starts with the
308 * mentioned prefix. The byte extra needs to follow the
315 assert(src_size
> 0);
319 assert(*buffer_size
== 0 || *buffer
);
324 if (!(greedy_realloc(buffer
, buffer_size
, ALIGN_8(prefix_len
+ 1), 1)))
327 r
= LZ4_decompress_safe_partial(src
+ 8, *buffer
, src_size
- 8,
328 prefix_len
+ 1, *buffer_size
);
332 /* lz4 always tries to decode full "sequence", so in
333 * pathological cases might need to decompress the
335 r
= decompress_blob_lz4(src
, src_size
, buffer
, buffer_size
, &size
, 0);
340 if (size
>= prefix_len
+ 1)
341 return memcmp(*buffer
, prefix
, prefix_len
) == 0 &&
342 ((const uint8_t*) *buffer
)[prefix_len
] == extra
;
347 return -EPROTONOSUPPORT
;
351 int decompress_startswith(int compression
,
352 const void *src
, uint64_t src_size
,
353 void **buffer
, size_t *buffer_size
,
354 const void *prefix
, size_t prefix_len
,
356 if (compression
== OBJECT_COMPRESSED_XZ
)
357 return decompress_startswith_xz(src
, src_size
,
361 else if (compression
== OBJECT_COMPRESSED_LZ4
)
362 return decompress_startswith_lz4(src
, src_size
,
370 int compress_stream_xz(int fdf
, int fdt
, uint64_t max_bytes
) {
372 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
374 uint8_t buf
[BUFSIZ
], out
[BUFSIZ
];
375 lzma_action action
= LZMA_RUN
;
380 ret
= lzma_easy_encoder(&s
, LZMA_PRESET_DEFAULT
, LZMA_CHECK_CRC64
);
381 if (ret
!= LZMA_OK
) {
382 log_error("Failed to initialize XZ encoder: code %u", ret
);
387 if (s
.avail_in
== 0 && action
== LZMA_RUN
) {
388 size_t m
= sizeof(buf
);
391 if (max_bytes
!= (uint64_t) -1 && (uint64_t) m
> max_bytes
)
392 m
= (size_t) max_bytes
;
394 n
= read(fdf
, buf
, m
);
398 action
= LZMA_FINISH
;
403 if (max_bytes
!= (uint64_t) -1) {
404 assert(max_bytes
>= (uint64_t) n
);
410 if (s
.avail_out
== 0) {
412 s
.avail_out
= sizeof(out
);
415 ret
= lzma_code(&s
, action
);
416 if (ret
!= LZMA_OK
&& ret
!= LZMA_STREAM_END
) {
417 log_error("Compression failed: code %u", ret
);
421 if (s
.avail_out
== 0 || ret
== LZMA_STREAM_END
) {
424 n
= sizeof(out
) - s
.avail_out
;
426 k
= loop_write(fdt
, out
, n
, false);
430 if (ret
== LZMA_STREAM_END
) {
431 log_debug("XZ compression finished (%"PRIu64
" -> %"PRIu64
" bytes, %.1f%%)",
432 s
.total_in
, s
.total_out
,
433 (double) s
.total_out
/ s
.total_in
* 100);
440 return -EPROTONOSUPPORT
;
444 #define LZ4_BUFSIZE (512*1024u)
446 int compress_stream_lz4(int fdf
, int fdt
, uint64_t max_bytes
) {
450 _cleanup_(LZ4F_freeCompressionContextp
) LZ4F_compressionContext_t ctx
= NULL
;
451 _cleanup_free_
char *buf
= NULL
;
453 size_t size
, n
, total_in
= 0, total_out
, offset
= 0, frame_size
;
456 static const LZ4F_compressOptions_t options
= {
459 static const LZ4F_preferences_t preferences
= {
460 .frameInfo
.blockSizeID
= 5,
463 c
= LZ4F_createCompressionContext(&ctx
, LZ4F_VERSION
);
467 if (fstat(fdf
, &st
) < 0)
468 return log_debug_errno(errno
, "fstat() failed: %m");
470 frame_size
= LZ4F_compressBound(LZ4_BUFSIZE
, &preferences
);
471 size
= frame_size
+ 64*1024; /* add some space for header and trailer */
476 n
= offset
= total_out
= LZ4F_compressBegin(ctx
, buf
, size
, &preferences
);
480 src
= mmap(NULL
, st
.st_size
, PROT_READ
, MAP_PRIVATE
, fdf
, 0);
481 if (src
== MAP_FAILED
)
484 log_debug("Buffer size is %zu bytes, header size %zu bytes.", size
, n
);
486 while (total_in
< (size_t) st
.st_size
) {
489 k
= MIN(LZ4_BUFSIZE
, st
.st_size
- total_in
);
490 n
= LZ4F_compressUpdate(ctx
, buf
+ offset
, size
- offset
,
491 src
+ total_in
, k
, &options
);
492 if (LZ4F_isError(n
)) {
493 r
= -ENOTRECOVERABLE
;
501 if (max_bytes
!= (uint64_t) -1 && total_out
> (size_t) max_bytes
) {
502 log_debug("Compressed stream longer than %"PRIu64
" bytes", max_bytes
);
506 if (size
- offset
< frame_size
+ 4) {
507 k
= loop_write(fdt
, buf
, offset
, false);
516 n
= LZ4F_compressEnd(ctx
, buf
+ offset
, size
- offset
, &options
);
517 if (LZ4F_isError(n
)) {
518 r
= -ENOTRECOVERABLE
;
524 r
= loop_write(fdt
, buf
, offset
, false);
528 log_debug("LZ4 compression finished (%zu -> %zu bytes, %.1f%%)",
530 (double) total_out
/ total_in
* 100);
532 munmap(src
, st
.st_size
);
535 return -EPROTONOSUPPORT
;
539 int decompress_stream_xz(int fdf
, int fdt
, uint64_t max_bytes
) {
542 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
545 uint8_t buf
[BUFSIZ
], out
[BUFSIZ
];
546 lzma_action action
= LZMA_RUN
;
551 ret
= lzma_stream_decoder(&s
, UINT64_MAX
, 0);
552 if (ret
!= LZMA_OK
) {
553 log_debug("Failed to initialize XZ decoder: code %u", ret
);
558 if (s
.avail_in
== 0 && action
== LZMA_RUN
) {
561 n
= read(fdf
, buf
, sizeof(buf
));
565 action
= LZMA_FINISH
;
572 if (s
.avail_out
== 0) {
574 s
.avail_out
= sizeof(out
);
577 ret
= lzma_code(&s
, action
);
578 if (ret
!= LZMA_OK
&& ret
!= LZMA_STREAM_END
) {
579 log_debug("Decompression failed: code %u", ret
);
583 if (s
.avail_out
== 0 || ret
== LZMA_STREAM_END
) {
586 n
= sizeof(out
) - s
.avail_out
;
588 if (max_bytes
!= (uint64_t) -1) {
589 if (max_bytes
< (uint64_t) n
)
595 k
= loop_write(fdt
, out
, n
, false);
599 if (ret
== LZMA_STREAM_END
) {
600 log_debug("XZ decompression finished (%"PRIu64
" -> %"PRIu64
" bytes, %.1f%%)",
601 s
.total_in
, s
.total_out
,
602 (double) s
.total_out
/ s
.total_in
* 100);
609 log_debug("Cannot decompress file. Compiled without XZ support.");
610 return -EPROTONOSUPPORT
;
614 int decompress_stream_lz4(int in
, int out
, uint64_t max_bytes
) {
617 _cleanup_(LZ4F_freeDecompressionContextp
) LZ4F_decompressionContext_t ctx
= NULL
;
618 _cleanup_free_
char *buf
= NULL
;
622 size_t total_in
= 0, total_out
= 0;
624 c
= LZ4F_createDecompressionContext(&ctx
, LZ4F_VERSION
);
628 if (fstat(in
, &st
) < 0)
629 return log_debug_errno(errno
, "fstat() failed: %m");
631 buf
= malloc(LZ4_BUFSIZE
);
635 src
= mmap(NULL
, st
.st_size
, PROT_READ
, MAP_PRIVATE
, in
, 0);
636 if (src
== MAP_FAILED
)
639 while (total_in
< (size_t) st
.st_size
) {
640 size_t produced
= LZ4_BUFSIZE
;
641 size_t used
= st
.st_size
- total_in
;
643 c
= LZ4F_decompress(ctx
, buf
, &produced
, src
+ total_in
, &used
, NULL
);
644 if (LZ4F_isError(c
)) {
650 total_out
+= produced
;
652 if (max_bytes
!= (uint64_t) -1 && total_out
> (size_t) max_bytes
) {
653 log_debug("Decompressed stream longer than %"PRIu64
" bytes", max_bytes
);
658 r
= loop_write(out
, buf
, produced
, false);
663 log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)",
665 (double) total_out
/ total_in
* 100);
667 munmap(src
, st
.st_size
);
670 log_debug("Cannot decompress file. Compiled without LZ4 support.");
671 return -EPROTONOSUPPORT
;
675 int decompress_stream(const char *filename
, int fdf
, int fdt
, uint64_t max_bytes
) {
677 if (endswith(filename
, ".lz4"))
678 return decompress_stream_lz4(fdf
, fdt
, max_bytes
);
679 else if (endswith(filename
, ".xz"))
680 return decompress_stream_xz(fdf
, fdt
, max_bytes
);
682 return -EPROTONOSUPPORT
;