1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
4 This file is part of systemd.
6 Copyright 2011 Lennart Poettering
8 systemd is free software; you can redistribute it and/or modify it
9 under the terms of the GNU Lesser General Public License as published by
10 the Free Software Foundation; either version 2.1 of the License, or
11 (at your option) any later version.
13 systemd is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 Lesser General Public License for more details.
18 You should have received a copy of the GNU Lesser General Public License
19 along with systemd; If not, see <http://www.gnu.org/licenses/>.
37 #include "sparse-endian.h"
38 #include "journal-def.h"
40 #define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t))
42 static const char* const object_compressed_table
[_OBJECT_COMPRESSED_MAX
] = {
43 [OBJECT_COMPRESSED_XZ
] = "XZ",
44 [OBJECT_COMPRESSED_LZ4
] = "LZ4",
47 DEFINE_STRING_TABLE_LOOKUP(object_compressed
, int);
49 int compress_blob_xz(const void *src
, uint64_t src_size
, void *dst
, size_t *dst_size
) {
51 static const lzma_options_lzma opt
= {
52 1u << 20u, NULL
, 0, LZMA_LC_DEFAULT
, LZMA_LP_DEFAULT
,
53 LZMA_PB_DEFAULT
, LZMA_MODE_FAST
, 128, LZMA_MF_HC3
, 4};
54 static const lzma_filter filters
[2] = {
55 {LZMA_FILTER_LZMA2
, (lzma_options_lzma
*) &opt
},
56 {LZMA_VLI_UNKNOWN
, NULL
}
66 /* Returns < 0 if we couldn't compress the data or the
67 * compressed result is longer than the original */
72 ret
= lzma_stream_buffer_encode((lzma_filter
*) filters
, LZMA_CHECK_NONE
, NULL
,
73 src
, src_size
, dst
, &out_pos
, src_size
- 1);
80 return -EPROTONOSUPPORT
;
84 int compress_blob_lz4(const void *src
, uint64_t src_size
, void *dst
, size_t *dst_size
) {
93 /* Returns < 0 if we couldn't compress the data or the
94 * compressed result is longer than the original */
99 r
= LZ4_compress_limitedOutput(src
, dst
+ 8, src_size
, src_size
- 8 - 1);
103 *(le64_t
*) dst
= htole64(src_size
);
108 return -EPROTONOSUPPORT
;
113 int decompress_blob_xz(const void *src
, uint64_t src_size
,
114 void **dst
, size_t *dst_alloc_size
, size_t* dst_size
, size_t dst_max
) {
117 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
122 assert(src_size
> 0);
124 assert(dst_alloc_size
);
126 assert(*dst_alloc_size
== 0 || *dst
);
128 ret
= lzma_stream_decoder(&s
, UINT64_MAX
, 0);
132 space
= MIN(src_size
* 2, dst_max
?: (size_t) -1);
133 if (!greedy_realloc(dst
, dst_alloc_size
, space
, 1))
137 s
.avail_in
= src_size
;
145 ret
= lzma_code(&s
, LZMA_FINISH
);
147 if (ret
== LZMA_STREAM_END
)
149 else if (ret
!= LZMA_OK
)
152 if (dst_max
> 0 && (space
- s
.avail_out
) >= dst_max
)
154 else if (dst_max
> 0 && space
== dst_max
)
157 used
= space
- s
.avail_out
;
158 space
= MIN(2 * space
, dst_max
?: (size_t) -1);
159 if (!greedy_realloc(dst
, dst_alloc_size
, space
, 1))
162 s
.avail_out
= space
- used
;
163 s
.next_out
= *dst
+ used
;
166 *dst_size
= space
- s
.avail_out
;
169 return -EPROTONOSUPPORT
;
173 int decompress_blob_lz4(const void *src
, uint64_t src_size
,
174 void **dst
, size_t *dst_alloc_size
, size_t* dst_size
, size_t dst_max
) {
178 int r
, size
; /* LZ4 uses int for size */
181 assert(src_size
> 0);
183 assert(dst_alloc_size
);
185 assert(*dst_alloc_size
== 0 || *dst
);
190 size
= le64toh( *(le64_t
*)src
);
191 if (size
< 0 || (le64_t
) size
!= *(le64_t
*)src
)
193 if ((size_t) size
> *dst_alloc_size
) {
194 out
= realloc(*dst
, size
);
198 *dst_alloc_size
= size
;
202 r
= LZ4_decompress_safe(src
+ 8, out
, src_size
- 8, size
);
203 if (r
< 0 || r
!= size
)
209 return -EPROTONOSUPPORT
;
213 int decompress_blob(int compression
,
214 const void *src
, uint64_t src_size
,
215 void **dst
, size_t *dst_alloc_size
, size_t* dst_size
, size_t dst_max
) {
216 if (compression
== OBJECT_COMPRESSED_XZ
)
217 return decompress_blob_xz(src
, src_size
,
218 dst
, dst_alloc_size
, dst_size
, dst_max
);
219 else if (compression
== OBJECT_COMPRESSED_LZ4
)
220 return decompress_blob_lz4(src
, src_size
,
221 dst
, dst_alloc_size
, dst_size
, dst_max
);
227 int decompress_startswith_xz(const void *src
, uint64_t src_size
,
228 void **buffer
, size_t *buffer_size
,
229 const void *prefix
, size_t prefix_len
,
233 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
236 /* Checks whether the decompressed blob starts with the
237 * mentioned prefix. The byte extra needs to follow the
241 assert(src_size
> 0);
245 assert(*buffer_size
== 0 || *buffer
);
247 ret
= lzma_stream_decoder(&s
, UINT64_MAX
, 0);
251 if (!(greedy_realloc(buffer
, buffer_size
, ALIGN_8(prefix_len
+ 1), 1)))
255 s
.avail_in
= src_size
;
257 s
.next_out
= *buffer
;
258 s
.avail_out
= *buffer_size
;
261 ret
= lzma_code(&s
, LZMA_FINISH
);
263 if (ret
!= LZMA_STREAM_END
&& ret
!= LZMA_OK
)
266 if (*buffer_size
- s
.avail_out
>= prefix_len
+ 1)
267 return memcmp(*buffer
, prefix
, prefix_len
) == 0 &&
268 ((const uint8_t*) *buffer
)[prefix_len
] == extra
;
270 if (ret
== LZMA_STREAM_END
)
273 s
.avail_out
+= *buffer_size
;
275 if (!(greedy_realloc(buffer
, buffer_size
, *buffer_size
* 2, 1)))
278 s
.next_out
= *buffer
+ *buffer_size
- s
.avail_out
;
282 return -EPROTONOSUPPORT
;
286 int decompress_startswith_lz4(const void *src
, uint64_t src_size
,
287 void **buffer
, size_t *buffer_size
,
288 const void *prefix
, size_t prefix_len
,
291 /* Checks whether the decompressed blob starts with the
292 * mentioned prefix. The byte extra needs to follow the
298 assert(src_size
> 0);
302 assert(*buffer_size
== 0 || *buffer
);
307 if (!(greedy_realloc(buffer
, buffer_size
, ALIGN_8(prefix_len
+ 1), 1)))
310 r
= LZ4_decompress_safe_partial(src
+ 8, *buffer
, src_size
- 8,
311 prefix_len
+ 1, *buffer_size
);
315 if ((unsigned) r
>= prefix_len
+ 1)
316 return memcmp(*buffer
, prefix
, prefix_len
) == 0 &&
317 ((const uint8_t*) *buffer
)[prefix_len
] == extra
;
322 return -EPROTONOSUPPORT
;
326 int decompress_startswith(int compression
,
327 const void *src
, uint64_t src_size
,
328 void **buffer
, size_t *buffer_size
,
329 const void *prefix
, size_t prefix_len
,
331 if (compression
== OBJECT_COMPRESSED_XZ
)
332 return decompress_startswith_xz(src
, src_size
,
336 else if (compression
== OBJECT_COMPRESSED_LZ4
)
337 return decompress_startswith_lz4(src
, src_size
,
345 int compress_stream_xz(int fdf
, int fdt
, uint64_t max_bytes
) {
347 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
349 uint8_t buf
[BUFSIZ
], out
[BUFSIZ
];
350 lzma_action action
= LZMA_RUN
;
355 ret
= lzma_easy_encoder(&s
, LZMA_PRESET_DEFAULT
, LZMA_CHECK_CRC64
);
356 if (ret
!= LZMA_OK
) {
357 log_error("Failed to initialize XZ encoder: code %u", ret
);
362 if (s
.avail_in
== 0 && action
== LZMA_RUN
) {
363 size_t m
= sizeof(buf
);
366 if (max_bytes
!= (uint64_t) -1 && (uint64_t) m
> max_bytes
)
367 m
= (size_t) max_bytes
;
369 n
= read(fdf
, buf
, m
);
373 action
= LZMA_FINISH
;
378 if (max_bytes
!= (uint64_t) -1) {
379 assert(max_bytes
>= (uint64_t) n
);
385 if (s
.avail_out
== 0) {
387 s
.avail_out
= sizeof(out
);
390 ret
= lzma_code(&s
, action
);
391 if (ret
!= LZMA_OK
&& ret
!= LZMA_STREAM_END
) {
392 log_error("Compression failed: code %u", ret
);
396 if (s
.avail_out
== 0 || ret
== LZMA_STREAM_END
) {
399 n
= sizeof(out
) - s
.avail_out
;
401 k
= loop_write(fdt
, out
, n
, false);
405 if (ret
== LZMA_STREAM_END
) {
406 log_debug("XZ compression finished (%"PRIu64
" -> %"PRIu64
" bytes, %.1f%%)",
407 s
.total_in
, s
.total_out
,
408 (double) s
.total_out
/ s
.total_in
* 100);
415 return -EPROTONOSUPPORT
;
419 #define LZ4_BUFSIZE (512*1024)
421 int compress_stream_lz4(int fdf
, int fdt
, uint64_t max_bytes
) {
425 _cleanup_free_
char *buf1
= NULL
, *buf2
= NULL
, *out
= NULL
;
427 LZ4_stream_t lz4_data
= {};
429 size_t total_in
= 0, total_out
= sizeof(header
);
435 buf1
= malloc(LZ4_BUFSIZE
);
436 buf2
= malloc(LZ4_BUFSIZE
);
437 out
= malloc(LZ4_COMPRESSBOUND(LZ4_BUFSIZE
));
438 if (!buf1
|| !buf2
|| !out
)
447 if (max_bytes
!= (uint64_t) -1 && (uint64_t) m
> (max_bytes
- total_in
))
448 m
= (size_t) (max_bytes
- total_in
);
450 n
= read(fdf
, buf
, m
);
458 r
= LZ4_compress_continue(&lz4_data
, buf
, out
, n
);
460 log_error("LZ4 compression failed.");
467 n
= write(fdt
, &header
, sizeof(header
));
470 if (n
!= sizeof(header
))
471 return errno
? -errno
: -EIO
;
473 n
= loop_write(fdt
, out
, r
, false);
477 total_out
+= sizeof(header
) + r
;
479 buf
= buf
== buf1
? buf2
: buf1
;
483 n
= write(fdt
, &header
, sizeof(header
));
486 if (n
!= sizeof(header
))
487 return errno
? -errno
: -EIO
;
489 log_debug("LZ4 compression finished (%zu -> %zu bytes, %.1f%%)",
491 (double) total_out
/ total_in
* 100);
495 return -EPROTONOSUPPORT
;
499 int decompress_stream_xz(int fdf
, int fdt
, uint64_t max_bytes
) {
502 _cleanup_(lzma_end
) lzma_stream s
= LZMA_STREAM_INIT
;
505 uint8_t buf
[BUFSIZ
], out
[BUFSIZ
];
506 lzma_action action
= LZMA_RUN
;
511 ret
= lzma_stream_decoder(&s
, UINT64_MAX
, 0);
512 if (ret
!= LZMA_OK
) {
513 log_error("Failed to initialize XZ decoder: code %u", ret
);
518 if (s
.avail_in
== 0 && action
== LZMA_RUN
) {
521 n
= read(fdf
, buf
, sizeof(buf
));
525 action
= LZMA_FINISH
;
532 if (s
.avail_out
== 0) {
534 s
.avail_out
= sizeof(out
);
537 ret
= lzma_code(&s
, action
);
538 if (ret
!= LZMA_OK
&& ret
!= LZMA_STREAM_END
) {
539 log_error("Decompression failed: code %u", ret
);
543 if (s
.avail_out
== 0 || ret
== LZMA_STREAM_END
) {
546 n
= sizeof(out
) - s
.avail_out
;
548 if (max_bytes
!= (uint64_t) -1) {
549 if (max_bytes
< (uint64_t) n
)
555 k
= loop_write(fdt
, out
, n
, false);
559 if (ret
== LZMA_STREAM_END
) {
560 log_debug("XZ decompression finished (%"PRIu64
" -> %"PRIu64
" bytes, %.1f%%)",
561 s
.total_in
, s
.total_out
,
562 (double) s
.total_out
/ s
.total_in
* 100);
569 log_error("Cannot decompress file. Compiled without XZ support.");
570 return -EPROTONOSUPPORT
;
574 int decompress_stream_lz4(int fdf
, int fdt
, uint64_t max_bytes
) {
577 _cleanup_free_
char *buf
= NULL
, *out
= NULL
;
579 LZ4_streamDecode_t lz4_data
= {};
581 size_t total_in
= sizeof(header
), total_out
= 0;
586 out
= malloc(4*LZ4_BUFSIZE
);
594 r
= loop_read_exact(fdf
, &header
, sizeof(header
), false);
602 /* We refuse to use a bigger decompression buffer than
603 * the one used for compression by 4 times. This means
604 * that compression buffer size can be enlarged 4
605 * times. This can be changed, but old binaries might
606 * not accept buffers compressed by newer binaries then.
608 if (m
> LZ4_COMPRESSBOUND(LZ4_BUFSIZE
* 4)) {
609 log_error("Compressed stream block too big: %zd bytes", m
);
613 total_in
+= sizeof(header
) + m
;
615 if (!GREEDY_REALLOC(buf
, buf_size
, m
))
618 r
= loop_read_exact(fdf
, buf
, m
, false);
622 r
= LZ4_decompress_safe_continue(&lz4_data
, buf
, out
, m
, 4*LZ4_BUFSIZE
);
624 log_error("LZ4 decompression failed.");
628 if (max_bytes
!= (uint64_t) -1 && (uint64_t) total_out
> max_bytes
) {
629 log_debug("Decompressed stream longer than %" PRIu64
" bytes", max_bytes
);
633 r
= loop_write(fdt
, out
, r
, false);
638 log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)",
640 (double) total_out
/ total_in
* 100);
644 log_error("Cannot decompress file. Compiled without LZ4 support.");
645 return -EPROTONOSUPPORT
;
649 int decompress_stream(const char *filename
, int fdf
, int fdt
, uint64_t max_bytes
) {
651 if (endswith(filename
, ".lz4"))
652 return decompress_stream_lz4(fdf
, fdt
, max_bytes
);
653 else if (endswith(filename
, ".xz"))
654 return decompress_stream_xz(fdf
, fdt
, max_bytes
);
656 return -EPROTONOSUPPORT
;