1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
4 This file is part of systemd.
6 Copyright 2011 Lennart Poettering
8 systemd is free software; you can redistribute it and/or modify it
9 under the terms of the GNU Lesser General Public License as published by
10 the Free Software Foundation; either version 2.1 of the License, or
11 (at your option) any later version.
13 systemd is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 Lesser General Public License for more details.
18 You should have received a copy of the GNU Lesser General Public License
19 along with systemd; If not, see <http://www.gnu.org/licenses/>.
38 #include "journal-def.h"
40 #include "sparse-endian.h"
41 #include "string-util.h"
45 DEFINE_TRIVIAL_CLEANUP_FUNC(LZ4F_compressionContext_t
, LZ4F_freeCompressionContext
);
46 DEFINE_TRIVIAL_CLEANUP_FUNC(LZ4F_decompressionContext_t
, LZ4F_freeDecompressionContext
);
49 #define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t))
51 static const char* const object_compressed_table
[_OBJECT_COMPRESSED_MAX
] = {
52 [OBJECT_COMPRESSED_XZ
] = "XZ",
53 [OBJECT_COMPRESSED_LZ4
] = "LZ4",
56 DEFINE_STRING_TABLE_LOOKUP(object_compressed
, int);
58 int compress_blob_xz(const void *src
, uint64_t src_size
, void *dst
, size_t *dst_size
) {
60 static const lzma_options_lzma opt
= {
61 1u << 20u, NULL
, 0, LZMA_LC_DEFAULT
, LZMA_LP_DEFAULT
,
62 LZMA_PB_DEFAULT
, LZMA_MODE_FAST
, 128, LZMA_MF_HC3
, 4
64 static const lzma_filter filters
[] = {
65 { LZMA_FILTER_LZMA2
, (lzma_options_lzma
*) &opt
},
66 { LZMA_VLI_UNKNOWN
, NULL
}
76 /* Returns < 0 if we couldn't compress the data or the
77 * compressed result is longer than the original */
82 ret
= lzma_stream_buffer_encode((lzma_filter
*) filters
, LZMA_CHECK_NONE
, NULL
,
83 src
, src_size
, dst
, &out_pos
, src_size
- 1);
90 return -EPROTONOSUPPORT
;
94 int compress_blob_lz4(const void *src
, uint64_t src_size
, void *dst
, size_t *dst_size
) {
103 /* Returns < 0 if we couldn't compress the data or the
104 * compressed result is longer than the original */
109 r
= LZ4_compress_limitedOutput(src
, dst
+ 8, src_size
, src_size
- 8 - 1);
113 *(le64_t
*) dst
= htole64(src_size
);
118 return -EPROTONOSUPPORT
;
123 int decompress_blob_xz(const void *src
, uint64_t src_size
,
124 void **dst
, size_t *dst_alloc_size
, size_t* dst_size
, size_t dst_max
) {
127 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
132 assert(src_size
> 0);
134 assert(dst_alloc_size
);
136 assert(*dst_alloc_size
== 0 || *dst
);
138 ret
= lzma_stream_decoder(&s
, UINT64_MAX
, 0);
142 space
= MIN(src_size
* 2, dst_max
?: (size_t) -1);
143 if (!greedy_realloc(dst
, dst_alloc_size
, space
, 1))
147 s
.avail_in
= src_size
;
155 ret
= lzma_code(&s
, LZMA_FINISH
);
157 if (ret
== LZMA_STREAM_END
)
159 else if (ret
!= LZMA_OK
)
162 if (dst_max
> 0 && (space
- s
.avail_out
) >= dst_max
)
164 else if (dst_max
> 0 && space
== dst_max
)
167 used
= space
- s
.avail_out
;
168 space
= MIN(2 * space
, dst_max
?: (size_t) -1);
169 if (!greedy_realloc(dst
, dst_alloc_size
, space
, 1))
172 s
.avail_out
= space
- used
;
173 s
.next_out
= *dst
+ used
;
176 *dst_size
= space
- s
.avail_out
;
179 return -EPROTONOSUPPORT
;
183 int decompress_blob_lz4(const void *src
, uint64_t src_size
,
184 void **dst
, size_t *dst_alloc_size
, size_t* dst_size
, size_t dst_max
) {
188 int r
, size
; /* LZ4 uses int for size */
191 assert(src_size
> 0);
193 assert(dst_alloc_size
);
195 assert(*dst_alloc_size
== 0 || *dst
);
200 size
= le64toh( *(le64_t
*)src
);
201 if (size
< 0 || (le64_t
) size
!= *(le64_t
*)src
)
203 if ((size_t) size
> *dst_alloc_size
) {
204 out
= realloc(*dst
, size
);
208 *dst_alloc_size
= size
;
212 r
= LZ4_decompress_safe(src
+ 8, out
, src_size
- 8, size
);
213 if (r
< 0 || r
!= size
)
219 return -EPROTONOSUPPORT
;
223 int decompress_blob(int compression
,
224 const void *src
, uint64_t src_size
,
225 void **dst
, size_t *dst_alloc_size
, size_t* dst_size
, size_t dst_max
) {
226 if (compression
== OBJECT_COMPRESSED_XZ
)
227 return decompress_blob_xz(src
, src_size
,
228 dst
, dst_alloc_size
, dst_size
, dst_max
);
229 else if (compression
== OBJECT_COMPRESSED_LZ4
)
230 return decompress_blob_lz4(src
, src_size
,
231 dst
, dst_alloc_size
, dst_size
, dst_max
);
237 int decompress_startswith_xz(const void *src
, uint64_t src_size
,
238 void **buffer
, size_t *buffer_size
,
239 const void *prefix
, size_t prefix_len
,
243 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
246 /* Checks whether the decompressed blob starts with the
247 * mentioned prefix. The byte extra needs to follow the
251 assert(src_size
> 0);
255 assert(*buffer_size
== 0 || *buffer
);
257 ret
= lzma_stream_decoder(&s
, UINT64_MAX
, 0);
261 if (!(greedy_realloc(buffer
, buffer_size
, ALIGN_8(prefix_len
+ 1), 1)))
265 s
.avail_in
= src_size
;
267 s
.next_out
= *buffer
;
268 s
.avail_out
= *buffer_size
;
271 ret
= lzma_code(&s
, LZMA_FINISH
);
273 if (ret
!= LZMA_STREAM_END
&& ret
!= LZMA_OK
)
276 if (*buffer_size
- s
.avail_out
>= prefix_len
+ 1)
277 return memcmp(*buffer
, prefix
, prefix_len
) == 0 &&
278 ((const uint8_t*) *buffer
)[prefix_len
] == extra
;
280 if (ret
== LZMA_STREAM_END
)
283 s
.avail_out
+= *buffer_size
;
285 if (!(greedy_realloc(buffer
, buffer_size
, *buffer_size
* 2, 1)))
288 s
.next_out
= *buffer
+ *buffer_size
- s
.avail_out
;
292 return -EPROTONOSUPPORT
;
296 int decompress_startswith_lz4(const void *src
, uint64_t src_size
,
297 void **buffer
, size_t *buffer_size
,
298 const void *prefix
, size_t prefix_len
,
301 /* Checks whether the decompressed blob starts with the
302 * mentioned prefix. The byte extra needs to follow the
308 assert(src_size
> 0);
312 assert(*buffer_size
== 0 || *buffer
);
317 if (!(greedy_realloc(buffer
, buffer_size
, ALIGN_8(prefix_len
+ 1), 1)))
320 r
= LZ4_decompress_safe_partial(src
+ 8, *buffer
, src_size
- 8,
321 prefix_len
+ 1, *buffer_size
);
325 if ((unsigned) r
>= prefix_len
+ 1)
326 return memcmp(*buffer
, prefix
, prefix_len
) == 0 &&
327 ((const uint8_t*) *buffer
)[prefix_len
] == extra
;
332 return -EPROTONOSUPPORT
;
336 int decompress_startswith(int compression
,
337 const void *src
, uint64_t src_size
,
338 void **buffer
, size_t *buffer_size
,
339 const void *prefix
, size_t prefix_len
,
341 if (compression
== OBJECT_COMPRESSED_XZ
)
342 return decompress_startswith_xz(src
, src_size
,
346 else if (compression
== OBJECT_COMPRESSED_LZ4
)
347 return decompress_startswith_lz4(src
, src_size
,
355 int compress_stream_xz(int fdf
, int fdt
, uint64_t max_bytes
) {
357 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
359 uint8_t buf
[BUFSIZ
], out
[BUFSIZ
];
360 lzma_action action
= LZMA_RUN
;
365 ret
= lzma_easy_encoder(&s
, LZMA_PRESET_DEFAULT
, LZMA_CHECK_CRC64
);
366 if (ret
!= LZMA_OK
) {
367 log_error("Failed to initialize XZ encoder: code %u", ret
);
372 if (s
.avail_in
== 0 && action
== LZMA_RUN
) {
373 size_t m
= sizeof(buf
);
376 if (max_bytes
!= (uint64_t) -1 && (uint64_t) m
> max_bytes
)
377 m
= (size_t) max_bytes
;
379 n
= read(fdf
, buf
, m
);
383 action
= LZMA_FINISH
;
388 if (max_bytes
!= (uint64_t) -1) {
389 assert(max_bytes
>= (uint64_t) n
);
395 if (s
.avail_out
== 0) {
397 s
.avail_out
= sizeof(out
);
400 ret
= lzma_code(&s
, action
);
401 if (ret
!= LZMA_OK
&& ret
!= LZMA_STREAM_END
) {
402 log_error("Compression failed: code %u", ret
);
406 if (s
.avail_out
== 0 || ret
== LZMA_STREAM_END
) {
409 n
= sizeof(out
) - s
.avail_out
;
411 k
= loop_write(fdt
, out
, n
, false);
415 if (ret
== LZMA_STREAM_END
) {
416 log_debug("XZ compression finished (%"PRIu64
" -> %"PRIu64
" bytes, %.1f%%)",
417 s
.total_in
, s
.total_out
,
418 (double) s
.total_out
/ s
.total_in
* 100);
425 return -EPROTONOSUPPORT
;
429 #define LZ4_BUFSIZE (512*1024u)
431 int compress_stream_lz4(int fdf
, int fdt
, uint64_t max_bytes
) {
435 _cleanup_(LZ4F_freeCompressionContextp
) LZ4F_compressionContext_t ctx
= NULL
;
436 _cleanup_free_
char *buf
= NULL
;
438 size_t size
, n
, total_in
= 0, total_out
= 0, offset
= 0, frame_size
;
441 static const LZ4F_compressOptions_t options
= {
444 static const LZ4F_preferences_t preferences
= {
445 .frameInfo
.blockSizeID
= 5,
448 c
= LZ4F_createCompressionContext(&ctx
, LZ4F_VERSION
);
452 if (fstat(fdf
, &st
) < 0)
453 return log_debug_errno(errno
, "fstat() failed: %m");
455 frame_size
= LZ4F_compressBound(LZ4_BUFSIZE
, &preferences
);
456 size
= frame_size
+ 64*1024; /* add some space for header and trailer */
461 n
= offset
= LZ4F_compressBegin(ctx
, buf
, size
, &preferences
);
465 src
= mmap(NULL
, st
.st_size
, PROT_READ
, MAP_PRIVATE
, fdf
, 0);
466 if (src
== MAP_FAILED
)
469 log_debug("Buffer size is %zu bytes, header size %zu bytes.", size
, n
);
471 while (total_in
< (size_t) st
.st_size
) {
474 k
= MIN(LZ4_BUFSIZE
, st
.st_size
- total_in
);
475 n
= LZ4F_compressUpdate(ctx
, buf
+ offset
, size
- offset
,
476 src
+ total_in
, k
, &options
);
477 if (LZ4F_isError(n
)) {
478 r
= -ENOTRECOVERABLE
;
486 if (max_bytes
!= (uint64_t) -1 && total_out
> (size_t) max_bytes
) {
487 log_debug("Compressed stream longer than %zd bytes", max_bytes
);
491 if (size
- offset
< frame_size
+ 4) {
492 k
= loop_write(fdt
, buf
, offset
, false);
501 n
= LZ4F_compressEnd(ctx
, buf
+ offset
, size
- offset
, &options
);
502 if (LZ4F_isError(n
)) {
503 r
= -ENOTRECOVERABLE
;
509 r
= loop_write(fdt
, buf
, offset
, false);
513 log_debug("LZ4 compression finished (%zu -> %zu bytes, %.1f%%)",
515 (double) total_out
/ total_in
* 100);
517 munmap(src
, st
.st_size
);
520 return -EPROTONOSUPPORT
;
524 int decompress_stream_xz(int fdf
, int fdt
, uint64_t max_bytes
) {
527 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
530 uint8_t buf
[BUFSIZ
], out
[BUFSIZ
];
531 lzma_action action
= LZMA_RUN
;
536 ret
= lzma_stream_decoder(&s
, UINT64_MAX
, 0);
537 if (ret
!= LZMA_OK
) {
538 log_debug("Failed to initialize XZ decoder: code %u", ret
);
543 if (s
.avail_in
== 0 && action
== LZMA_RUN
) {
546 n
= read(fdf
, buf
, sizeof(buf
));
550 action
= LZMA_FINISH
;
557 if (s
.avail_out
== 0) {
559 s
.avail_out
= sizeof(out
);
562 ret
= lzma_code(&s
, action
);
563 if (ret
!= LZMA_OK
&& ret
!= LZMA_STREAM_END
) {
564 log_debug("Decompression failed: code %u", ret
);
568 if (s
.avail_out
== 0 || ret
== LZMA_STREAM_END
) {
571 n
= sizeof(out
) - s
.avail_out
;
573 if (max_bytes
!= (uint64_t) -1) {
574 if (max_bytes
< (uint64_t) n
)
580 k
= loop_write(fdt
, out
, n
, false);
584 if (ret
== LZMA_STREAM_END
) {
585 log_debug("XZ decompression finished (%"PRIu64
" -> %"PRIu64
" bytes, %.1f%%)",
586 s
.total_in
, s
.total_out
,
587 (double) s
.total_out
/ s
.total_in
* 100);
594 log_debug("Cannot decompress file. Compiled without XZ support.");
595 return -EPROTONOSUPPORT
;
600 static int decompress_stream_lz4_v1(int fdf
, int fdt
, uint64_t max_bytes
) {
602 _cleanup_free_
char *buf
= NULL
, *out
= NULL
;
604 LZ4_streamDecode_t lz4_data
= {};
606 size_t total_in
= sizeof(header
), total_out
= 0;
611 out
= malloc(4*LZ4_BUFSIZE
);
619 r
= loop_read_exact(fdf
, &header
, sizeof(header
), false);
627 /* We refuse to use a bigger decompression buffer than
628 * the one used for compression by 4 times. This means
629 * that compression buffer size can be enlarged 4
630 * times. This can be changed, but old binaries might
631 * not accept buffers compressed by newer binaries then.
633 if (m
> LZ4_COMPRESSBOUND(LZ4_BUFSIZE
* 4)) {
634 log_debug("Compressed stream block too big: %zd bytes", m
);
638 total_in
+= sizeof(header
) + m
;
640 if (!GREEDY_REALLOC(buf
, buf_size
, m
))
643 r
= loop_read_exact(fdf
, buf
, m
, false);
647 r
= LZ4_decompress_safe_continue(&lz4_data
, buf
, out
, m
, 4*LZ4_BUFSIZE
);
649 log_debug("LZ4 decompression failed (legacy format).");
655 if (max_bytes
!= (uint64_t) -1 && (uint64_t) total_out
> max_bytes
) {
656 log_debug("Decompressed stream longer than %" PRIu64
" bytes", max_bytes
);
660 r
= loop_write(fdt
, out
, r
, false);
665 log_debug("LZ4 decompression finished (legacy format, %zu -> %zu bytes, %.1f%%)",
667 (double) total_out
/ total_in
* 100);
672 static int decompress_stream_lz4_v2(int in
, int out
, uint64_t max_bytes
) {
674 _cleanup_(LZ4F_freeDecompressionContextp
) LZ4F_decompressionContext_t ctx
= NULL
;
675 _cleanup_free_
char *buf
= NULL
;
679 size_t total_in
= 0, total_out
= 0;
681 c
= LZ4F_createDecompressionContext(&ctx
, LZ4F_VERSION
);
685 if (fstat(in
, &st
) < 0)
686 return log_debug_errno(errno
, "fstat() failed: %m");
688 buf
= malloc(LZ4_BUFSIZE
);
692 src
= mmap(NULL
, st
.st_size
, PROT_READ
, MAP_PRIVATE
, in
, 0);
693 if (src
== MAP_FAILED
)
696 while (total_in
< (size_t) st
.st_size
) {
697 size_t produced
= LZ4_BUFSIZE
;
698 size_t used
= st
.st_size
- total_in
;
700 c
= LZ4F_decompress(ctx
, buf
, &produced
, src
+ total_in
, &used
, NULL
);
701 if (LZ4F_isError(c
)) {
707 total_out
+= produced
;
709 if (max_bytes
!= (uint64_t) -1 && total_out
> (size_t) max_bytes
) {
710 log_debug("Decompressed stream longer than %zd bytes", max_bytes
);
715 r
= loop_write(out
, buf
, produced
, false);
720 log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)",
722 (double) total_out
/ total_in
* 100);
724 munmap(src
, st
.st_size
);
729 int decompress_stream_lz4(int fdf
, int fdt
, uint64_t max_bytes
) {
733 r
= decompress_stream_lz4_v2(fdf
, fdt
, max_bytes
);
735 r
= decompress_stream_lz4_v1(fdf
, fdt
, max_bytes
);
738 log_debug("Cannot decompress file. Compiled without LZ4 support.");
739 return -EPROTONOSUPPORT
;
743 int decompress_stream(const char *filename
, int fdf
, int fdt
, uint64_t max_bytes
) {
745 if (endswith(filename
, ".lz4"))
746 return decompress_stream_lz4(fdf
, fdt
, max_bytes
);
747 else if (endswith(filename
, ".xz"))
748 return decompress_stream_xz(fdf
, fdt
, max_bytes
);
750 return -EPROTONOSUPPORT
;